def build_elu_cnn(input_shape, output_size): """Build a variation of the CNN implemented in the ELU paper. https://arxiv.org/abs/1511.07289 """ def layers(n, channels, kernel): return sum(([ Convolution2D(channels, kernel_size=kernel, padding="same"), ELU() ] for i in range(n)), []) model = Sequential( [ Convolution2D( 384, kernel_size=3, padding="same", input_shape=input_shape) ] + layers(1, 384, 3) + [MaxPooling2D(pool_size=(2, 2))] + layers(1, 384, 1) + layers(1, 384, 2) + layers(2, 640, 2) + [MaxPooling2D(pool_size=(2, 2))] + layers(1, 640, 1) + layers(3, 768, 2) + [MaxPooling2D(pool_size=(2, 2))] + layers(1, 768, 1) + layers(2, 896, 2) + [MaxPooling2D(pool_size=(2, 2))] + layers(1, 896, 3) + layers(2, 1024, 2) + [ MaxPooling2D(pool_size=(2, 2)), Convolution2D(output_size, kernel_size=1, padding="same"), GlobalAveragePooling2D(), Activation("softmax") ]) model.compile(optimizer=SGD(momentum=0.9), loss="categorical_crossentropy", metrics=["accuracy"]) return model
def get_seq_model(): """Define three channel input shape depending on image data format.""" if K.image_data_format() == 'channels_first': input_shape = (3, img_width, img_height) else: input_shape = (img_width, img_height, 3) # Initialize CNN by creating a sequential model. model = Sequential() model.add(Conv2D(32, (3, 3), input_shape=input_shape)) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(32, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Conv2D(64, (3, 3))) model.add(Activation('relu')) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) model.add(Dense(64)) model.add(Activation('relu')) model.add(Dropout(0.5)) model.add(Dense(2)) model.add(Activation('sigmoid')) model.compile( loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy']) return model
def build_small_cnn(input_shape, output_size): model = Sequential([ # conv1_* Convolution2D(32, kernel_size=3, padding="same", input_shape=input_shape), Activation("relu"), Convolution2D(32, kernel_size=3, padding="same"), Activation("relu"), MaxPooling2D(pool_size=(2, 2)), # conv2_* Convolution2D(64, kernel_size=3, padding="same"), Activation("relu"), Convolution2D(64, kernel_size=3, padding="same"), Activation("relu"), MaxPooling2D(pool_size=(2, 2)), # Fully connected Flatten(), Dense(512), Activation("relu"), Dense(512), Activation("relu"), Dense(output_size), Activation("softmax") ]) model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) return model
def inception_block_1c(X): X_3x3 = fr_utils.conv2d_bn(X, layer='inception_3c_3x3', cv1_out=128, cv1_filter=(1, 1), cv2_out=256, cv2_filter=(3, 3), cv2_strides=(2, 2), padding=(1, 1)) X_5x5 = fr_utils.conv2d_bn(X, layer='inception_3c_5x5', cv1_out=32, cv1_filter=(1, 1), cv2_out=64, cv2_filter=(5, 5), cv2_strides=(2, 2), padding=(2, 2)) X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X) X_pool = ZeroPadding2D(padding=((0, 1), (0, 1)), data_format='channels_first')(X_pool) inception = concatenate([X_3x3, X_5x5, X_pool], axis=1) return inception
def inception_block_1a(X): """ Implementation of an inception block """ X_3x3 = Conv2D(96, (1, 1), data_format='channels_first', name='inception_3a_3x3_conv1')(X) X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_3x3_bn1')(X_3x3) X_3x3 = Activation('relu')(X_3x3) X_3x3 = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_3x3) X_3x3 = Conv2D(128, (3, 3), data_format='channels_first', name='inception_3a_3x3_conv2')(X_3x3) X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_3x3_bn2')(X_3x3) X_3x3 = Activation('relu')(X_3x3) X_5x5 = Conv2D(16, (1, 1), data_format='channels_first', name='inception_3a_5x5_conv1')(X) X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_5x5_bn1')(X_5x5) X_5x5 = Activation('relu')(X_5x5) X_5x5 = ZeroPadding2D(padding=(2, 2), data_format='channels_first')(X_5x5) X_5x5 = Conv2D(32, (5, 5), data_format='channels_first', name='inception_3a_5x5_conv2')(X_5x5) X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_5x5_bn2')(X_5x5) X_5x5 = Activation('relu')(X_5x5) X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X) X_pool = Conv2D(32, (1, 1), data_format='channels_first', name='inception_3a_pool_conv')(X_pool) X_pool = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_pool_bn')(X_pool) X_pool = Activation('relu')(X_pool) X_pool = ZeroPadding2D(padding=((3, 4), (3, 4)), data_format='channels_first')(X_pool) X_1x1 = Conv2D(64, (1, 1), data_format='channels_first', name='inception_3a_1x1_conv')(X) X_1x1 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_1x1_bn')(X_1x1) X_1x1 = Activation('relu')(X_1x1) # CONCAT inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1) return inception
def build_cnn(input_shape, output_size): kwargs = {"kernel_size": 3, "activation": "relu", "padding": "same"} model = Sequential([ # conv1_* Convolution2D(64, input_shape=input_shape, **kwargs), BatchRenormalization(), Convolution2D(64, **kwargs), BatchRenormalization(), MaxPooling2D(pool_size=(2, 2)), Dropout(0.25), # conv2_* Convolution2D(128, **kwargs), BatchRenormalization(), Convolution2D(128, **kwargs), BatchRenormalization(), MaxPooling2D(pool_size=(2, 2)), Dropout(0.25), # conv3_* Convolution2D(256, **kwargs), BatchRenormalization(), Convolution2D(256, **kwargs), BatchRenormalization(), MaxPooling2D(pool_size=(2, 2)), Dropout(0.25), # Fully connected Flatten(), Dense(1024), Activation("relu"), Dropout(0.5), Dense(512), Activation("relu"), Dropout(0.5), Dense(output_size), Activation("softmax") ]) model.compile(loss="categorical_crossentropy", optimizer="adam", metrics=["accuracy"]) return model
def Train(self): # self.loadDataFeature() self.loadDataTxt() self.train_and_test_split(0.75) # model model = Sequential() # model.add(Dense(392, activation='relu')) # model.add(Dense(128, activation='relu')) # model.add(Dense(36, activation='softmax')) #cnn model model.add( Conv2D(64, (3, 3), activation='relu', input_shape=(28, 28, 1))) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D((2, 2))) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(Conv2D(64, (3, 3), activation='relu')) model.add(MaxPooling2D((2, 2))) model.add(Flatten()) model.add(Dense(128, activation='relu')) model.add(Dense(128, activation='relu')) model.add(Dense(36, activation='softmax')) # model.compile(loss='mse', optimizer='adam', metrics=['accuracy']) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) model.fit( self.train_data['data'], self.train_data['class_name'], batch_size=25, epochs=100, verbose=1, validation_data=(self.test_data['data'], self.test_data['class_name']), ) self.model = model model.save('digit_classification_model1.h5') # Y_pred = model.predict(self.test_data['data']) # self.metric(self.test_data['class_name'], Y_pred, data_type='binary') self.metric()
def neural_network(input_shape): inputs = keras.Input(shape=input_shape) #Layer 1 x = MaxPooling2D(pool_size=(2, 2), name='MaxPooling2D_1')(inputs) x = Conv2D(32, kernel_size=(5, 5), padding='same')(x) x = BatchNormalization()(x) x = LeakyReLU(alpha=0.1)(x) x = MaxPooling2D(pool_size=(4, 4))(x) #Layer 2 x = Conv2D(64, kernel_size=(5, 5), padding='same', name='Conv2D_2')(x) x = BatchNormalization()(x) x = LeakyReLU(alpha=0.1)(x) x = MaxPooling2D(pool_size=(2, 2), name='MaxPooling2D_3')(x) x = Flatten(name='Flatten')(x) #Layer 3 #model.add(Dense(256,name = 'Dense_1')) #model.add(BatchNormalization(name = 'BatchNormalization_2')) #model.add(LeakyReLU(alpha=0.1)) #model.add(Dropout(0.5,name = 'Dropout_1')) #Layer 4 x = Dense(128, name='Dense_2')(x) x = BatchNormalization(name='BatchNormalization_3')(x) x = LeakyReLU(alpha=0.1)(x) x = Dropout(0.5, name='Dropout_2')(x) #Layer 5 x = Dense(128, name='Dense_3')(x) x = BatchNormalization(name='BatchNormalization_4')(x) x = LeakyReLU(alpha=0.1)(x) #model.add(Dropout(0.5,name = 'Dropout_3')) outputs = Dense(1, activation='sigmoid', name='Dense_4')(x) model = Model(inputs, outputs) return model
def inception_block_3b(X): X_3x3 = fr_utils.conv2d_bn(X, layer='inception_5b_3x3', cv1_out=96, cv1_filter=(1, 1), cv2_out=384, cv2_filter=(3, 3), cv2_strides=(1, 1), padding=(1, 1)) X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X) X_pool = fr_utils.conv2d_bn(X_pool, layer='inception_5b_pool', cv1_out=96, cv1_filter=(1, 1)) X_pool = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_pool) X_1x1 = fr_utils.conv2d_bn(X, layer='inception_5b_1x1', cv1_out=256, cv1_filter=(1, 1)) inception = concatenate([X_3x3, X_pool, X_1x1], axis=1) return inception
def main(_): kernel_posterior_fn = tfp_layers_util.default_mean_field_normal_fn( untransformed_scale_initializer=tf.compat.v1.initializers. random_normal(mean=FLAGS.var, stddev=0.1)) encoder_w0 = tf.keras.Sequential([ tfp.layers.Convolution2DReparameterization( filters=32, kernel_size=3, strides=(2, 2), activation='relu', padding='SAME', kernel_posterior_fn=kernel_posterior_fn), tfp.layers.Convolution2DReparameterization( filters=48, kernel_size=3, strides=(2, 2), activation='relu', padding='SAME', kernel_posterior_fn=kernel_posterior_fn), MaxPooling2D(pool_size=(2, 2)), tfp.layers.Convolution2DReparameterization( filters=64, kernel_size=3, strides=(2, 2), activation='relu', padding='SAME', kernel_posterior_fn=kernel_posterior_fn), tf.keras.layers.Flatten(), tfp.layers.DenseReparameterization( FLAGS.dim_w, kernel_posterior_fn=kernel_posterior_fn), ]) decoder0 = tf.keras.Sequential([ tf.keras.layers.Dense(100, activation=tf.nn.relu), tf.keras.layers.Dense(100, activation=tf.nn.relu), tf.keras.layers.Dense(FLAGS.dim_y), ]) dim_output = FLAGS.dim_y dim_input = FLAGS.dim_im * FLAGS.dim_im * 1 exp_name = '%s.beta-%g.update_lr-%g.trial-%d' % ( 'np_bbb', FLAGS.beta, FLAGS.update_lr, FLAGS.trial) checkpoint_dir = os.path.join(FLAGS.logdir, exp_name) x_train, y_train = pickle.load( tf.io.gfile.GFile(os.path.join(get_data_dir(), FLAGS.data[0]), 'rb')) x_val, y_val = pickle.load( tf.io.gfile.GFile(os.path.join(get_data_dir(), FLAGS.data[1]), 'rb')) x_train, y_train = np.array(x_train), np.array(y_train) y_train = y_train[:, :, -1, None] x_val, y_val = np.array(x_val), np.array(y_val) y_val = y_val[:, :, -1, None] ds_train = tf.data.Dataset.from_generator( functools.partial(gen, x_train, y_train), (tf.float32, tf.float32, tf.float32, tf.float32), (tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]))) ds_val = tf.data.Dataset.from_generator( functools.partial(gen, x_val, y_val), (tf.float32, tf.float32, tf.float32, tf.float32), (tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]))) inputa, labela, inputb, labelb = ds_train.make_one_shot_iterator( ).get_next() input_tensors = {'inputa': inputa,\ 'inputb': inputb,\ 'labela': labela, 'labelb': labelb} inputa_val, labela_val, inputb_val, labelb_val = ds_val.make_one_shot_iterator( ).get_next() metaval_input_tensors = {'inputa': inputa_val,\ 'inputb': inputb_val,\ 'labela': labela_val, 'labelb': labelb_val} loss, train_op, facto = construct_model(input_tensors, encoder_w0, decoder0, prefix='metatrain_') loss_val = construct_model(metaval_input_tensors, encoder_w0, decoder0, prefix='metaval_') ########### summ_op = tf.summary.merge_all() sess = tf.InteractiveSession() summary_writer = tf.summary.FileWriter(checkpoint_dir, sess.graph) tf.global_variables_initializer().run() PRINT_INTERVAL = 50 # pylint: disable=invalid-name SUMMARY_INTERVAL = 5 # pylint: disable=invalid-name prelosses, prelosses_val = [], [] old_time = time.time() for itr in range(FLAGS.num_updates): feed_dict = {facto: FLAGS.facto} if itr % SUMMARY_INTERVAL == 0: summary, cost, cost_val = sess.run([summ_op, loss, loss_val], feed_dict) summary_writer.add_summary(summary, itr) prelosses.append(cost) # 0 step loss on training set prelosses_val.append( cost_val) # 0 step loss on meta_val training set sess.run(train_op, feed_dict) if (itr != 0) and itr % PRINT_INTERVAL == 0: print('Iteration ' + str(itr) + ': ' + str(np.mean(prelosses)), 'time =', time.time() - old_time) prelosses = [] old_time = time.time() print('Validation results: ' + str(np.mean(prelosses_val))) prelosses_val = []
def main(_): dim_output = FLAGS.dim_y dim_input = FLAGS.dim_im * FLAGS.dim_im * 1 exp_name = '%s.beta-%g.meta_lr-%g.update_lr-%g.trial-%d' % ( 'maml_bbb', FLAGS.beta, FLAGS.meta_lr, FLAGS.update_lr, FLAGS.trial) checkpoint_dir = os.path.join(FLAGS.logdir, exp_name) x_train, y_train = pickle.load( tf.io.gfile.GFile(os.path.join(get_data_dir(), FLAGS.data[0]), 'rb')) x_val, y_val = pickle.load( tf.io.gfile.GFile(os.path.join(get_data_dir(), FLAGS.data[1]), 'rb')) x_train, y_train = np.array(x_train), np.array(y_train) y_train = y_train[:, :, -1, None] x_val, y_val = np.array(x_val), np.array(y_val) y_val = y_val[:, :, -1, None] ds_train = tf.data.Dataset.from_generator( functools.partial(gen, x_train, y_train), (tf.float32, tf.float32, tf.float32, tf.float32), (tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]))) ds_val = tf.data.Dataset.from_generator( functools.partial(gen, x_val, y_val), (tf.float32, tf.float32, tf.float32, tf.float32), (tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]))) kernel_posterior_fn = tfp_layers_util.default_mean_field_normal_fn( untransformed_scale_initializer=tf.compat.v1.initializers. random_normal(mean=FLAGS.var, stddev=0.1)) encoder_w = tf.keras.Sequential([ tfp.layers.Convolution2DReparameterization( filters=32, kernel_size=3, strides=(2, 2), activation='relu', padding='SAME', kernel_posterior_fn=kernel_posterior_fn), tfp.layers.Convolution2DReparameterization( filters=48, kernel_size=3, strides=(2, 2), activation='relu', padding='SAME', kernel_posterior_fn=kernel_posterior_fn), MaxPooling2D(pool_size=(2, 2)), tfp.layers.Convolution2DReparameterization( filters=64, kernel_size=3, strides=(2, 2), activation='relu', padding='SAME', kernel_posterior_fn=kernel_posterior_fn), tf.keras.layers.Flatten(), tfp.layers.DenseReparameterization( FLAGS.dim_w, kernel_posterior_fn=kernel_posterior_fn), ]) xa, labela, xb, labelb = ds_train.make_one_shot_iterator().get_next() xa = tf.reshape(xa, [-1, 128, 128, 1]) xb = tf.reshape(xb, [-1, 128, 128, 1]) with tf.variable_scope('encoder'): inputa = encoder_w(xa) inputa = tf.reshape( inputa, [-1, FLAGS.update_batch_size * FLAGS.num_classes, FLAGS.dim_w]) inputb = encoder_w(xb) inputb = tf.reshape( inputb, [-1, FLAGS.update_batch_size * FLAGS.num_classes, FLAGS.dim_w]) input_tensors = {'inputa': inputa,\ 'inputb': inputb, \ 'labela': labela, 'labelb': labelb} # n_task * n_im_per_task * dim_w xa_val, labela_val, xb_val, labelb_val = ds_val.make_one_shot_iterator( ).get_next() xa_val = tf.reshape(xa_val, [-1, 128, 128, 1]) xb_val = tf.reshape(xb_val, [-1, 128, 128, 1]) inputa_val = encoder_w(xa_val) inputa_val = tf.reshape( inputa_val, [-1, FLAGS.update_batch_size * FLAGS.num_classes, FLAGS.dim_w]) inputb_val = encoder_w(xb_val) inputb_val = tf.reshape( inputb_val, [-1, FLAGS.update_batch_size * FLAGS.num_classes, FLAGS.dim_w]) metaval_input_tensors = {'inputa': inputa_val,\ 'inputb': inputb_val, \ 'labela': labela_val, 'labelb': labelb_val} # num_updates = max(self.test_num_updates, FLAGS.num_updates) model = MAML(encoder_w, FLAGS.dim_w, dim_output) model.construct_model(input_tensors=input_tensors, prefix='metatrain_') model.construct_model(input_tensors=metaval_input_tensors, prefix='metaval_', test_num_updates=FLAGS.test_num_updates) # model.construct_model(input_tensors=input_tensors, prefix='metaval_') model.summ_op = tf.summary.merge_all() sess = tf.InteractiveSession() tf.global_variables_initializer().run() if FLAGS.train: train(model, sess, checkpoint_dir, exp_name)
NN2 = Sequential() NN2.add(InputLayer(input_shape=(Nmaturities, Nstrikes, 1))) NN2.add( Conv2D(64, (3, 3), use_bias=True, padding='valid', strides=(1, 1), activation='tanh')) NN2.add( Conv2D(64, (2, 2), use_bias=True, padding='valid', strides=(1, 1), activation='tanh')) NN2.add(MaxPooling2D(pool_size=(2, 2))) NN2.add( Conv2D(64, (2, 2), use_bias=True, padding='valid', strides=(1, 1), activation='tanh')) NN2.add(ZeroPadding2D(padding=(1, 1))) NN2.add( Conv2D(64, (2, 2), use_bias=True, padding='valid', strides=(1, 1), activation='tanh')) NN2.add(ZeroPadding2D(padding=(1, 1))) NN2.add(
layer_sizes = [4, 8, 16] conv_layers = [1, 2] for dense_layer in dense_layers: for layer_size in layer_sizes: for conv_layer in conv_layers: NAME = f'Pneumonia-{IMG_SIZE}px-{NUM_SAMPLES}samples-{conv_layer}conv-{layer_size}nodes-{dense_layer}dense-{int(time.time())}' tensorboard = TensorBoard(log_dir=f'logs/{NAME}') print(NAME) model = Sequential() # format: Num of filters, window/step, dimensions model.add(Conv2D(layer_size, (3, 3), input_shape=x_train.shape[1:])) model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) print('Layer 0 generated') for i in range(conv_layer - 1): print(f'Layer {i + 1} generated.') model.add(Conv2D(layer_size, (3, 3))) model.add(Activation("relu")) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(Flatten()) for l in range(dense_layer): model.add(Dense(layer_size)) model.add(Activation("relu")) model.add(Dense(1)) # Final activation function
def stack_layers(inputs, layers, kernel_initializer='glorot_uniform'): ''' Builds the architecture of the network by applying each layer specified in layers to inputs. inputs: a dict containing input_types and input_placeholders for each key and value pair, respecively. for spectralnet, this means the input_types 'Unlabeled' and 'Orthonorm'* layers: a list of dicts containing all layers to be used in the network, where each dict describes one such layer. each dict requires the key 'type'. all other keys are dependent on the layer type kernel_initializer: initialization configuration passed to keras (see keras initializers) returns: outputs, a dict formatted in much the same way as inputs. it contains input_types and output_tensors for each key and value pair, respectively, where output_tensors are the outputs of the input_placeholders in inputs after each layer in layers is applied * this is necessary since spectralnet takes multiple inputs and performs special computations on the orthonorm layer ''' outputs = dict() for key in inputs: outputs[key] = inputs[key] for layer in layers: # check for l2_reg argument l2_reg = layer.get('l2_reg') if l2_reg: l2_reg = l2(layer['l2_reg']) # create the layer if layer['type'] == 'softplus_reg': l = Dense(layer['size'], activation='softplus', kernel_initializer=kernel_initializer, kernel_regularizer=l2(0.001), name=layer.get('name')) elif layer['type'] == 'softplus': l = Dense(layer['size'], activation='softplus', kernel_initializer=kernel_initializer, kernel_regularizer=l2_reg, name=layer.get('name')) elif layer['type'] == 'softmax': l = Dense(layer['size'], activation='softmax', kernel_initializer=kernel_initializer, kernel_regularizer=l2_reg, name=layer.get('name')) elif layer['type'] == 'tanh': l = Dense(layer['size'], activation='tanh', kernel_initializer=kernel_initializer, kernel_regularizer=l2_reg, name=layer.get('name')) elif layer['type'] == 'relu': l = Dense(layer['size'], activation='relu', kernel_initializer=kernel_initializer, kernel_regularizer=l2_reg, name=layer.get('name')) elif layer['type'] == 'selu': l = Dense(layer['size'], activation='selu', kernel_initializer=kernel_initializer, kernel_regularizer=l2_reg, name=layer.get('name')) elif layer['type'] == 'Conv2D': l = Conv2D(layer['channels'], kernel_size=layer['kernel'], activation='relu', data_format='channels_last', kernel_regularizer=l2_reg, name=layer.get('name')) elif layer['type'] == 'BatchNormalization': l = BatchNormalization(name=layer.get('name')) elif layer['type'] == 'MaxPooling2D': l = MaxPooling2D(pool_size=layer['pool_size'], data_format='channels_first', name=layer.get('name')) elif layer['type'] == 'Dropout': l = Dropout(layer['rate'], name=layer.get('name')) elif layer['type'] == 'Flatten': l = Flatten(name=layer.get('name')) elif layer['type'] == 'Orthonorm': l = Orthonorm(outputs['Orthonorm'], name=layer.get('name')) else: raise ValueError("Invalid layer type '{}'".format(layer['type'])) # apply the layer to each input in inputs for k in outputs: outputs[k] = l(outputs[k]) return outputs
def main(_): encoder_w0 = tf.keras.Sequential([ Conv2D(filters=32, kernel_size=3, strides=(2, 2), activation='relu', padding='same'), Conv2D(filters=48, kernel_size=3, strides=(2, 2), activation='relu', padding='same'), MaxPooling2D(pool_size=(2, 2)), Conv2D(filters=64, kernel_size=3, strides=(2, 2), activation='relu', padding='same'), tf.keras.layers.Flatten(), tf.keras.layers.Dense(FLAGS.dim_w), ]) decoder0 = tf.keras.Sequential([ tf.keras.layers.Dense(100, activation=tf.nn.relu), tf.keras.layers.Dense(100, activation=tf.nn.relu), tf.keras.layers.Dense(FLAGS.dim_y), ]) dim_output = FLAGS.dim_y dim_input = FLAGS.dim_im * FLAGS.dim_im * 1 exp_basename = 'np_vanilla' if FLAGS.noise_scale: exp_basename = 'np_vanilla_noised_scale' + str(FLAGS.noise_scale) elif FLAGS.num_noise > 0: exp_basename = 'np_vanilla_noise' + str(FLAGS.num_noise) if FLAGS.weight_decay: exp_name = '%s.update_lr-%g.beta-%g.trial-%d' % ( exp_basename, FLAGS.update_lr, FLAGS.beta, FLAGS.trial) else: exp_name = '%s.update_lr-%g.trial-%d' % (exp_basename, FLAGS.update_lr, FLAGS.trial) if FLAGS.testing: exp_name += "-test" checkpoint_dir = os.path.join(FLAGS.logdir, exp_name) if FLAGS.testing: data = [FLAGS.data[0], FLAGS.data[1]] else: data = [FLAGS.data[0], FLAGS.data[0]] x_train, y_train = pickle.load( tf.io.gfile.GFile(os.path.join(get_data_dir(), data[0]), 'rb')) x_val, y_val = pickle.load( tf.io.gfile.GFile(os.path.join(get_data_dir(), data[1]), 'rb')) if not FLAGS.testing: # Split the train dataset into val and train x_train, y_train = x_train[:-5], y_train[:-5] x_val, y_val = x_val[-5:], y_val[-5:] x_train, y_train = np.array(x_train), np.array(y_train) y_train = y_train[:, :, -1, None] x_val, y_val = np.array(x_val), np.array(y_val) y_val = y_val[:, :, -1, None] ds_train = tf.data.Dataset.from_generator( functools.partial(gen, x_train, y_train, True), (tf.float32, tf.float32, tf.float32, tf.float32), (tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]))) ds_val = tf.data.Dataset.from_generator( functools.partial(gen, x_val, y_val, False), (tf.float32, tf.float32, tf.float32, tf.float32), (tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]), tf.TensorShape( [None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]))) inputa, labela, inputb, labelb = ds_train.make_one_shot_iterator( ).get_next() input_tensors = {'inputa': inputa,\ 'inputb': inputb,\ 'labela': labela, 'labelb': labelb} inputa_val, labela_val, inputb_val, labelb_val = ds_val.make_one_shot_iterator( ).get_next() metaval_input_tensors = {'inputa': inputa_val,\ 'inputb': inputb_val,\ 'labela': labela_val, 'labelb': labelb_val} loss, train_op, facto = construct_model(input_tensors, encoder_w0, decoder0, prefix='metatrain_') loss_val = construct_model(metaval_input_tensors, encoder_w0, decoder0, prefix='metaval_') ########### summ_op = tf.summary.merge_all() sess = tf.InteractiveSession() summary_writer = tf.summary.FileWriter(checkpoint_dir, sess.graph) tf.global_variables_initializer().run() PRINT_INTERVAL = 50 # pylint: disable=invalid-name SUMMARY_INTERVAL = 5 # pylint: disable=invalid-name val_step = [] train_k, val_k = [], [] # scratch buffers prelosses, prelosses_val = [], [] old_time = time.time() for itr in range(FLAGS.num_updates): feed_dict = {facto: FLAGS.facto} if itr % SUMMARY_INTERVAL == 0: summary, cost, cost_val = sess.run([summ_op, loss, loss_val], feed_dict) summary_writer.add_summary(summary, itr) prelosses.append(cost) # 0 step loss on training set prelosses_val.append( cost_val) # 0 step loss on meta_val training set sess.run(train_op, feed_dict) if (itr != 0) and itr % PRINT_INTERVAL == 0: val_step.append(itr) print('Iteration ' + str(itr) + ': ' + str(np.mean(prelosses)), 'time =', time.time() - old_time) prelosses = [] old_time = time.time() print('Validation results: ' + str(np.mean(prelosses_val))) # Dump (theres no inner loss?) train_k.append(np.mean(prelosses)) val_k.append(np.mean(prelosses_val)) all_ = (val_step, train_k, val_k) pickle.dump(all_, open(os.path.join(checkpoint_dir, 'results.p'), 'wb')) prelosses_val = [] prelosses = []
def InceptionModel(input_shape): """ Implementation of the Inception model used for FaceNet Arguments: input_shape -- shape of the images of the dataset Returns: model -- a Model() instance in Keras """ # Define the input as a tensor with shape input_shape X_input = Input(input_shape) # Zero-Padding X = ZeroPadding2D((3, 3))(X_input) # First Block X = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(X) X = BatchNormalization(axis=1, name='bn1')(X) X = Activation('relu')(X) # Zero-Padding + MAXPOOL X = ZeroPadding2D((1, 1))(X) X = MaxPooling2D((3, 3), strides=2)(X) # Second Block X = Conv2D(64, (1, 1), strides=(1, 1), name='conv2')(X) X = BatchNormalization(axis=1, epsilon=0.00001, name='bn2')(X) X = Activation('relu')(X) # Zero-Padding + MAXPOOL X = ZeroPadding2D((1, 1))(X) # Second Block X = Conv2D(192, (3, 3), strides=(1, 1), name='conv3')(X) X = BatchNormalization(axis=1, epsilon=0.00001, name='bn3')(X) X = Activation('relu')(X) # Zero-Padding + MAXPOOL X = ZeroPadding2D((1, 1))(X) X = MaxPooling2D(pool_size=3, strides=2)(X) # Inception 1: a/b/c X = inception_block_1a(X) X = inception_block_1b(X) X = inception_block_1c(X) # Inception 2: a/b X = inception_block_2a(X) X = inception_block_2b(X) # Inception 3: a/b X = inception_block_3a(X) X = inception_block_3b(X) # Top layer X = AveragePooling2D(pool_size=(3, 3), strides=(1, 1), data_format='channels_first')(X) X = Flatten()(X) X = Dense(128, name='dense_layer')(X) # L2 normalization X = Lambda(lambda x: K.l2_normalize(x, axis=1))(X) # Create model instance model = Model(inputs=X_input, outputs=X, name='FaceRecoModel') return model