def fcn_model(inputs, num_classes): # TODO Add Encoder Blocks. filters = 16 keep_prob = 1 # Remember that with each encoder layer, the depth of your model (the number of filters) increases. l1 = encoder_block(inputs, filters) l1 = keras.layers.core.Dropout(keep_prob)(l1) l2 = encoder_block(l1, filters * 2) l2 = keras.layers.core.Dropout(keep_prob)(l2) l3 = encoder_block(l2, filters * 3) l3 = keras.layers.core.Dropout(keep_prob)(l3) l4 = encoder_block(l3, filters * 4) fcn = conv2d_batchnorm(l4, filters * 4, strides=1, kernel=1) # decoder x = decoder_block(fcn, filters * 4) x = decoder_block(x, filters * 3) x = keras.layers.core.Dropout(keep_prob)(x) x = decoder_block(x, filters * 2) x = layers.concatenate([x, l1]) x = keras.layers.core.Dropout(keep_prob)(x) x = decoder_block(x, filters) x = layers.concatenate([x, inputs]) # The function returns the output layer of your model. "x" is the final layer obtained from the last decoder_block() return layers.Conv2D(num_classes, 3, activation='softmax', padding='same')(x)
def generator(Z_dim, y_dim, image_size=32): gf_dim = 64 s16 = int(image_size / 16) c_dim = 3 z_in = Input(shape=(Z_dim, ), name='Z_input') y_in = Input(shape=(y_dim, ), name='y_input') inputs = concatenate([z_in, y_in]) G_h = Dense(gf_dim * 8 * s16 * s16)(inputs) G_h = Activation('relu')(BatchNormalization()( Reshape(target_shape=[s16, s16, gf_dim * 8])(G_h))) G_h = Activation('relu')(BatchNormalization()(Conv2DTranspose( gf_dim * 4, kernel_size=(5, 5), strides=(2, 2), padding='same')(G_h))) G_h = Activation('relu')(BatchNormalization()(Conv2DTranspose( gf_dim * 2, kernel_size=(5, 5), strides=(2, 2), padding='same')(G_h))) G_h = Activation('relu')(BatchNormalization()(Conv2DTranspose( gf_dim * 2, kernel_size=(5, 5), strides=(1, 1), padding='same')(G_h))) G_h = Activation('relu')(BatchNormalization()(Conv2DTranspose( gf_dim, kernel_size=(5, 5), strides=(2, 2), padding='same')(G_h))) G_prob = Conv2DTranspose(c_dim, kernel_size=(5, 5), strides=(2, 2), padding='same', activation='sigmoid')(G_h) G = Model(inputs=[z_in, y_in], outputs=G_prob, name='Generator') print('=== Generator ===') G.summary() print('\n\n') return G
def decoder_block(small_ip_layer, large_ip_layer, filters): # Upsample the small input layer using the bilinear_upsample() function. output_layer = bilinear_upsample(small_ip_layer) # Concatenate the upsampled and large input layers using layers.concatenate output_layer = layers.concatenate([output_layer, large_ip_layer]) # Add some number of separable convolution layers output_layer = separable_conv2d_batchnorm(output_layer, filters) # output_layer = separable_conv2d_batchnorm(output_layer, filters) return output_layer
def decoder_block(small_ip_layer, large_ip_layer, filters): # TODO Upsample the small input layer using the bilinear_upsample() function. output = bilinear_upsample(small_ip_layer) # TODO Concatenate the upsampled and large input layers using layers.concatenate print ('db_output_{} large_ip_{}'.format(output.get_shape().as_list(), large_ip_layer.get_shape().as_list())) output = layers.concatenate([output, large_ip_layer], axis=-1) # TODO Add some number of separable convolution layers output_layer = separable_conv2d_batchnorm(output, filters) return output_layer
def decoder_block(small_ip_layer, large_ip_layer, filters): # TODO Upsample the small input layer using the bilinear_upsample() function. upsampled_layer = bilinear_upsample(small_ip_layer) # TODO Concatenate the upsampled and large input layers using layers.concatenate concatenated_layers = layers.concatenate([upsampled_layer, large_ip_layer]) # TODO Add some number of separable convolution layers lay1 = separable_conv2d_batchnorm(concatenated_layers, filters, strides=1) output_layer = separable_conv2d_batchnorm(lay1, filters, strides=1) return output_layer
def dense_block(input_, num_nodes, update_filter, growth_rate): concat_feat = input_ for i in range(num_nodes): x = conv_block(concat_feat, growth_rate) # concat_feat=ZeroPadding2D((1,1))(concat_feat) concat_feat = concatenate([concat_feat, x], axis=3) update_filter += growth_rate return concat_feat, update_filter
def decoder_block(small_ip_layer, large_ip_layer, filters): # TODO Upsample the small input layer using the bilinear_upsample() function. small_ip_layer_up = bilinear_upsample(small_ip_layer) # TODO Concatenate the upsampled and large input layers using layers.concatenate output_layer = layers.concatenate([small_ip_layer_up, large_ip_layer]) # TODO Add some number of separable convolution layers # What? return output_layer
def decode_block(x, skip_layer, layer_fn, upsample_type, n_layers, n_out_filts=None): x = upsample(x, upsample_type, n_out_filts) if upsample_type == 'nearest_neighbor': x = layer_fn(x) x = l.concatenate([x, skip_layer]) for i in range(n_layers - 1): x = layer_fn(x) return x
def decoder_block(small_ip_layer, large_ip_layer, filters): # TODO Upsample the small input layer using the bilinear_upsample() function. upsampled_small_ip_layer = bilinear_upsample(small_ip_layer) # print("small", upsampled_small_ip_layer.get_shape()) # print("large", large_ip_layer.get_shape()) # TODO Concatenate the upsampled and large input layers using layers.concatenate concatenated = layers.concatenate( [upsampled_small_ip_layer, large_ip_layer]) # TODO Add some number of separable convolution layers conv_layer_1 = separable_conv2d_batchnorm(concatenated, filters) conv_layer_2 = separable_conv2d_batchnorm(conv_layer_1, filters) output_layer = separable_conv2d_batchnorm(conv_layer_2, filters) return output_layer
def decoder_block(small_ip_layer, large_ip_layer, filters): # TODO Upsample the small input layer using the bilinear_upsample() function. upsampled = bilinear_upsample(small_ip_layer) if large_ip_layer != None: # TODO Concatenate the upsampled and large input layers using layers.concatenate concatenated = layers.concatenate([upsampled, large_ip_layer]) # TODO Add some number of separable convolution layers output_layer = separable_conv2d_batchnorm(concatenated, filters) else: output_layer = separable_conv2d_batchnorm(upsampled, filters) return output_layer
def make_parallel(model, gpu_count): def get_slice(data, idx, parts): shape = tf.shape(data) size = tf.concat([shape[:1] // parts, shape[1:]], axis=0) stride = tf.concat([shape[:1] // parts, shape[1:] * 0], axis=0) start = stride * idx return tf.slice(data, start, size) outputs_all = [] for i in range(len(model.outputs)): outputs_all.append([]) #Place a copy of the model on each GPU, each getting a slice of the batch for i in range(gpu_count): with tf.device('/gpu:%d' % i): with tf.name_scope('tower_%d' % i) as scope: inputs = [] #Slice each input into a piece for processing on this GPU for x in model.inputs: input_shape = tuple(x.get_shape().as_list())[1:] slice_n = Lambda(get_slice, arguments={ 'idx': i, 'parts': gpu_count })(x) inputs.append(slice_n) outputs = model(inputs) if not isinstance(outputs, list): outputs = [outputs] #Save all the outputs for merging back together later for l in range(len(outputs)): outputs_all[l].append(outputs[l]) # merge outputs on CPU with tf.device('/cpu:0'): merged = [] for outputs in outputs_all: merged.append(concatenate(outputs, axis=0)) return Model(model.inputs[0], merged)
def decoder_block(small_ip_layer, large_ip_layer, filters): # TODO Upsample the small input layer using the bilinear_upsample() function. upsampled_layer = bilinear_upsample(small_ip_layer) # print('upsampled_layer in decoder_block:', upsampled_layer) # print('large_ip_layer in decoder_block:', large_ip_layer) # TODO Concatenate the upsampled and large input layers using layers.concatenate if large_ip_layer != None: concatenate_layer = layers.concatenate( [upsampled_layer, large_ip_layer]) else: concatenate_layer = upsampled_layer # print('concatenate_layer in decoder_block:', concatenate_layer) # TODO Add some number of separable convolution layers output_layer = separable_conv2d_batchnorm(concatenate_layer, filters) # output_layer = separable_conv2d_batchnorm(output_layer, 3) # print('output_layer last Shape in decoder_block: {}'.format(output_layer.shape)) return output_layer
def generator_model(): noise_input = Input(batch_shape=(BATCH_SIZE, Z_DIM), name='z_input') disc_input = Input(batch_shape=(BATCH_SIZE, DISC_DIM), name='disc_input') cont_input = Input(batch_shape=(BATCH_SIZE, CONT_DIM), name='cont_input') input_list = [noise_input, disc_input, cont_input] gen_input = concatenate(input_list, axis=1, name="generator_input") h = Dense(1024)(gen_input) h = Activation('tanh')(h) h = Dense(128 * 7 * 7)(h) h = BatchNormalization()(h) h = Activation('tanh')(h) h = Reshape((7, 7, 128), input_shape=(128 * 7 * 7, ))(h) h = UpSampling2D(size=(2, 2))(h) h = Conv2D(64, (5, 5), padding='same')(h) h = Activation('tanh')(h) h = UpSampling2D(size=(2, 2))(h) h = Conv2D(1, (5, 5), padding='same')(h) h = Activation('tanh')(h) return Model(inputs=input_list, outputs=[h], name="generator")
def discriminator(X_dim, y_dim): df_dim = 64 x_in = Input(shape=(X_dim), name='X_input') y_in = Input(shape=(y_dim, ), name='Y_input') D_h = LeakyReLU(0.2)(Convolution2D(df_dim, kernel_size=(5, 5), strides=(2, 2), padding='same')(x_in)) D_h = LeakyReLU(0.2)(BatchNormalization()(Convolution2D( df_dim * 2, kernel_size=(5, 5), strides=(2, 2), padding='same')(D_h))) D_h = LeakyReLU(0.2)(BatchNormalization()(Convolution2D( df_dim * 2, kernel_size=(5, 5), strides=(2, 2), padding='same')(D_h))) D_h = LeakyReLU(0.2)(BatchNormalization()(Convolution2D( df_dim * 4, kernel_size=(5, 5), strides=(2, 2), padding='same')(D_h))) D_h = LeakyReLU(0.2)(BatchNormalization()(Convolution2D( df_dim * 8, kernel_size=(5, 5), strides=(2, 2), padding='same')(D_h))) D_h = Flatten()(D_h) D_h = concatenate([D_h, y_in]) D_logit = Dense(1, name='Discriminator_Output')(D_h) D = Model(inputs=[x_in, y_in], outputs=D_logit, name='Discriminator') print('=== Discriminator ===') D.summary() print('\n\n') return D
def train(): NUM_EPOCHS = options['num_epochs'] LOAD_PATH = options['load_path'] SAVE_PATH = options['save_path'] PSIZE = options['psize'] HSIZE = options['hsize'] WSIZE = options['wsize'] CSIZE = options['csize'] model_name = options['model_name'] BATCH_SIZE = options['batch_size'] continue_training = options['continue_training'] files = [] num_labels = 5 with open('train.txt') as f: for line in f: files.append(line[:-1]) print '%d training samples' % len(files) flair_t2_node = tf.placeholder(dtype=tf.float32, shape=(None, HSIZE, WSIZE, CSIZE, 2)) t1_t1ce_node = tf.placeholder(dtype=tf.float32, shape=(None, HSIZE, WSIZE, CSIZE, 2)) flair_t2_gt_node = tf.placeholder(dtype=tf.int32, shape=(None, PSIZE, PSIZE, PSIZE, 2)) t1_t1ce_gt_node = tf.placeholder(dtype=tf.int32, shape=(None, PSIZE, PSIZE, PSIZE, 5)) if model_name == 'dense48': flair_t2_15, flair_t2_27 = tf_models.BraTS2ScaleDenseNetConcat_large( input=flair_t2_node, name='flair') t1_t1ce_15, t1_t1ce_27 = tf_models.BraTS2ScaleDenseNetConcat_large( input=t1_t1ce_node, name='t1') elif model_name == 'no_dense': flair_t2_15, flair_t2_27 = tf_models.PlainCounterpart( input=flair_t2_node, name='flair') t1_t1ce_15, t1_t1ce_27 = tf_models.PlainCounterpart(input=t1_t1ce_node, name='t1') elif model_name == 'dense24': flair_t2_15, flair_t2_27 = tf_models.BraTS2ScaleDenseNetConcat( input=flair_t2_node, name='flair') t1_t1ce_15, t1_t1ce_27 = tf_models.BraTS2ScaleDenseNetConcat( input=t1_t1ce_node, name='t1') else: print ' No such model name ' t1_t1ce_15 = concatenate([t1_t1ce_15, flair_t2_15]) t1_t1ce_27 = concatenate([t1_t1ce_27, flair_t2_27]) flair_t2_15 = Conv3D(2, kernel_size=1, strides=1, padding='same', name='flair_t2_15_cls')(flair_t2_15) flair_t2_27 = Conv3D(2, kernel_size=1, strides=1, padding='same', name='flair_t2_27_cls')(flair_t2_27) t1_t1ce_15 = Conv3D(num_labels, kernel_size=1, strides=1, padding='same', name='t1_t1ce_15_cls')(t1_t1ce_15) t1_t1ce_27 = Conv3D(num_labels, kernel_size=1, strides=1, padding='same', name='t1_t1ce_27_cls')(t1_t1ce_27) flair_t2_score = flair_t2_15[:, 13:25, 13:25, 13:25, :] + \ flair_t2_27[:, 13:25, 13:25, 13:25, :] t1_t1ce_score = t1_t1ce_15[:, 13:25, 13:25, 13:25, :] + \ t1_t1ce_27[:, 13:25, 13:25, 13:25, :] loss = segmentation_loss(flair_t2_gt_node, flair_t2_score, 2) + \ segmentation_loss(t1_t1ce_gt_node, t1_t1ce_score, 5) acc_flair_t2 = acc_tf(y_pred=flair_t2_score, y_true=flair_t2_gt_node) acc_t1_t1ce = acc_tf(y_pred=t1_t1ce_score, y_true=t1_t1ce_gt_node) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): optimizer = tf.train.AdamOptimizer(learning_rate=5e-4).minimize(loss) saver = tf.train.Saver(max_to_keep=15) data_gen_train = vox_generator(all_files=files, n_pos=200, n_neg=200, correction=options['correction']) with tf.Session() as sess: if continue_training: saver.restore(sess, LOAD_PATH) else: sess.run(tf.global_variables_initializer()) for ei in range(NUM_EPOCHS): for pi in range(len(files)): acc_pi, loss_pi = [], [] data, labels, centers = data_gen_train.next() n_batches = int(np.ceil(float(centers.shape[1]) / BATCH_SIZE)) for nb in range(n_batches): offset_batch = min(nb * BATCH_SIZE, centers.shape[1] - BATCH_SIZE) data_batch, label_batch = get_patches_3d( data, labels, centers[:, offset_batch:offset_batch + BATCH_SIZE], HSIZE, WSIZE, CSIZE, PSIZE, False) label_batch = label_transform(label_batch, 5) _, l, acc_ft, acc_t1c = sess.run( fetches=[optimizer, loss, acc_flair_t2, acc_t1_t1ce], feed_dict={ flair_t2_node: data_batch[:, :, :, :, :2], t1_t1ce_node: data_batch[:, :, :, :, 2:], flair_t2_gt_node: label_batch[0], t1_t1ce_gt_node: label_batch[1], learning_phase(): 1 }) acc_pi.append([acc_ft, acc_t1c]) loss_pi.append(l) n_pos_sum = np.sum(np.reshape(label_batch[0], (-1, 2)), axis=0) print 'epoch-patient: %d, %d, iter: %d-%d, p%%: %.4f, loss: %.4f, acc_flair_t2: %.2f%%, acc_t1_t1ce: %.2f%%' % \ (ei + 1, pi + 1, nb + 1, n_batches, n_pos_sum[1]/float(np.sum(n_pos_sum)), l, acc_ft, acc_t1c) print 'patient loss: %.4f, patient acc: %.4f' % ( np.mean(loss_pi), np.mean(acc_pi)) saver.save(sess, SAVE_PATH, global_step=ei) print 'model saved'
# Now let's get a tensor with the output of our vision model: image_input = Input(shape=(224, 224, 3)) encoded_image = vision_model(image_input) # Next, let's define a language model to encode the question into a vector. # Each question will be at most 100 word long, # and we will index words as integers from 1 to 9999. question_input = Input(shape=(100, ), dtype='int32') embedded_question = Embedding(input_dim=10000, output_dim=256, input_length=100)( question_input) # (?, 100, 256) encoded_question = LSTM(256)(embedded_question) # (?, 256) # Let's concatenate the question vector and the image vector: merged = concatenate([encoded_question, encoded_image]) # And let's train a logistic regression over 1000 words on top: output = Dense(1000, activation='softmax')(merged) # This is our final model: vqa_model = Model(inputs=[image_input, question_input], outputs=output) # The next stage would be training this model on actual data. """ ### Video question answering model Now that we have trained our image QA model, we can quickly turn it into a video QA model. With appropriate training, you will be able to show it a short video (e.g. 100-frame human action) and ask a natural language question about the video (e.g. "what sport is the boy playing?" -> "football"). """
def InceptionV3(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000): """Instantiates the Inception v3 architecture. Optionally loads weights pre-trained on ImageNet. Note that when using TensorFlow, for best performance you should set `image_data_format="channels_last"` in your Keras config at ~/.keras/keras.json. The model and the weights are compatible with both TensorFlow and Theano. The data format convention used by the model is the one specified in your Keras config file. Note that the default input image size for this model is 299x299. Arguments: include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet). input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(299, 299, 3)` (with `channels_last` data format) or `(3, 299, 299)` (with `channels_first` data format). It should have exactly 3 input channels, and width and height should be no smaller than 139. E.g. `(150, 150, 3)` would be one valid value. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. Returns: A Keras model instance. Raises: ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if weights not in {'imagenet', None}: raise ValueError('The `weights` argument should be either ' '`None` (random initialization) or `imagenet` ' '(pre-training on ImageNet).') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as imagenet with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape input_shape = _obtain_input_shape( input_shape, default_size=299, min_size=139, data_format=K.image_data_format(), require_flatten=False, weights=weights) if input_tensor is None: img_input = Input(shape=input_shape) else: img_input = Input(tensor=input_tensor, shape=input_shape) if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = 3 x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid') x = conv2d_bn(x, 32, 3, 3, padding='valid') x = conv2d_bn(x, 64, 3, 3) x = MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv2d_bn(x, 80, 1, 1, padding='valid') x = conv2d_bn(x, 192, 3, 3, padding='valid') x = MaxPooling2D((3, 3), strides=(2, 2))(x) # mixed 0, 1, 2: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 32, 1, 1) x = layers.concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed0') # mixed 1: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = layers.concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed1') # mixed 2: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = layers.concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed2') # mixed 3: 17 x 17 x 768 branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid') branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn( branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid') branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) x = layers.concatenate( [branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3') # mixed 4: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 128, 1, 1) branch7x7 = conv2d_bn(branch7x7, 128, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 128, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed4') # mixed 5, 6: 17 x 17 x 768 for i in range(2): branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 160, 1, 1) branch7x7 = conv2d_bn(branch7x7, 160, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 160, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed' + str(5 + i)) # mixed 7: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(branch7x7, 192, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 192, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed7') # mixed 8: 8 x 8 x 1280 branch3x3 = conv2d_bn(x, 192, 1, 1) branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, strides=(2, 2), padding='valid') branch7x7x3 = conv2d_bn(x, 192, 1, 1) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1) branch7x7x3 = conv2d_bn( branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid') branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) x = layers.concatenate( [branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8') # mixed 9: 8 x 8 x 2048 for i in range(2): branch1x1 = conv2d_bn(x, 320, 1, 1) branch3x3 = conv2d_bn(x, 384, 1, 1) branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) branch3x3 = layers.concatenate( [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) branch3x3dbl = conv2d_bn(x, 448, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) branch3x3dbl = layers.concatenate( [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed' + str(9 + i)) if include_top: # Classification block x = GlobalAveragePooling2D(name='avg_pool')(x) x = Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='inception_v3') # load weights if weights == 'imagenet': if include_top: weights_path = get_file( 'inception_v3_weights_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH, cache_subdir='models', md5_hash='9a0d58056eeedaa3f26cb7ebd46da564') else: weights_path = get_file( 'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5', WEIGHTS_PATH_NO_TOP, cache_subdir='models', md5_hash='bcbd6486424b2319ff4ef7d526e38f63') model.load_weights(weights_path) return model
# A LSTM will transform the vector sequence into a single vector, # containing information about the entire sequence lstm_out = LSTM(32)(x) # output (?, 32) # Here we insert the auxiliary loss, allowing the LSTM and Embedding layer to be trained smoothly even though the main loss will be much higher in the model.# output (?, 1) auxiliary_output = Dense(1, activation='sigmoid', name='aux_output')(lstm_out) # knowing its shape, we can create auxiliary_output_array auxiliary_output_array = np.random.random((1000, 1)) # At this point, we feed into the model our auxiliary input data by concatenating it with the LSTM output: auxiliary_input = Input(shape=(5, ), name='aux_input') # create auxiliary_input_array auxiliary_input_array = np.random.random((1000, 5)) x = concatenate([lstm_out, auxiliary_input]) # We stack a deep densely-connected network on top x = Dense(64, activation='relu')(x) x = Dense(64, activation='relu')(x) x = Dense(64, activation='relu')(x) # And finally we add the main logistic regression layer main_output = Dense(1, activation='sigmoid', name='main_output')(x) # create main_output_array for training later main_output_array = np.random.random((1000, 1)) # This defines a model with two inputs and two outputs: model = Model(inputs=[main_input, auxiliary_input], outputs=[main_output, auxiliary_output])
def decoder_block(small_ip_layer, large_ip_layer, filters): output_layer = bilinear_upsample(small_ip_layer) output_layer = layers.concatenate([output_layer, large_ip_layer]) output_layer = separable_conv2d_batchnorm(output_layer, filters, strides=1) return output_layer
""" from tensorflow.contrib.keras.python.keras.layers import Conv2D, MaxPooling2D, Input, Dense, Flatten, concatenate from tensorflow.contrib.keras.python.keras.models import Model # First, define the vision modules # digit_input = Input(shape=(1, 27, 27)) digit_input = Input(shape=(27, 27, 1)) x = Conv2D(64, (3, 3))(digit_input) # padding 'valid' or 'same' affect shape x = Conv2D(64, (3, 3))(x) # padding='same', can keep image size the same x = MaxPooling2D((2, 2))(x) # downscale image size out = Flatten()(x) vision_model = Model(digit_input, out) # Then define the tell-digits-apart model # digit_a = Input(shape=(1, 27, 27)) # digit_b = Input(shape=(1, 27, 27)) digit_a = Input(shape=(27, 27, 1)) digit_b = Input(shape=(27, 27, 1)) # The vision model will be shared, weights and all out_a = vision_model(digit_a) out_b = vision_model(digit_b) concatenated = concatenate([out_a, out_b]) # cbind tensors out = Dense(1, activation='sigmoid')(concatenated) classification_model = Model([digit_a, digit_b], out)
def create_model(embeddings, config=get_config(), sentence_length=100): config['sentence_length'] = sentence_length # sentence attention attention_input = Input(shape=(config['sentence_length'], config['embedding_size'],), dtype='float32') x = Permute((2, 1))(attention_input) x = Reshape((config['embedding_size'], config['sentence_length']))(x) x = Dense(config['sentence_length'], activation='softmax', bias=True)(x) x = Lambda(lambda x: K.mean(x, axis=1), name='attention_vector_sentence')(x) x = RepeatVector(config['embedding_size'])(x) # x = Lambda(lambda x: x, name='attention_vector_sentence')(x) attention_probabilities = Permute((2, 1))(x) x = merge.multiply([attention_input, attention_probabilities], name='attention_mul') x = Lambda(lambda x: K.sum(x, axis=1))(x) sentence_attention = Model(attention_input, x, name='sentence_attention') embedding_layer = Embedding( embeddings.shape[0], embeddings.shape[1], input_length=config['sentence_length'], trainable=False, weights=[embeddings], ) pos_mbedding_layer = Embedding( 15, config['embedding_size'], input_length=config['sentence_length'], trainable=True, # weights=[embeddings], ) input = Input(shape=(config['sentence_length'],), dtype='int32', name='input_1') x = embedding_layer(input) x = SpatialDropout1D(config['embedding_dropout'])(x) x1 = Attention()(x) input2 = Input(shape=(config['sentence_length'],), dtype='int32', name='input_2') x2 = pos_mbedding_layer(input2) x2 = sentence_attention(x2) # res x_sum = Lambda(lambda x: K.mean(x, axis=1))(x) x1 = add([x1, x_sum]) x = concatenate([x1, x2]) if config['dense_layer']: x = Dense(config['dense_layer'], activation='relu')(x) x = Dropout(config['dropout_prob'])(x) output = Dense(1, activation='sigmoid',)(x) model = Model(inputs=[input, input2], outputs=output) return model, config
x = MaxPooling1D(5)(x) x = Conv1D(128, 5, activation='relu')(x) x = MaxPooling1D(5)(x) x = Conv1D(128, 5, activation='relu')(x) x = MaxPooling1D(35)(x) # global max pooling x = Dropout (0.2)(x) x = Flatten()(x) x = Dense(512, activation='relu')(x) model_text = Model(sequence_input, x) model_gen = Model(sequence_input_gen, x_gen) print(model_text.output.shape) print(model_gen.output.shape) merged = concatenate([model_text.output,model_gen.output]) preds = Dense(len(labels_index), activation='softmax')(merged) model = Model(inputs=[sequence_input,sequence_input_gen],outputs=preds) model.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=['acc']) embedding_matrix=None embeddings_index=None for epoch in range(NUM_EPOCHS):
def decoder_block(small_ip_layer, large_ip_layer, filters): upsampled = bilinear_upsample(small_ip_layer) concatenated = layers.concatenate([upsampled, large_ip_layer]) output_layer = separable_conv2d_batchnorm(concatenated, filters) return output_layer
def InceptionV3(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000): """Instantiates the Inception v3 architecture. Optionally loads weights pre-trained on ImageNet. Note that when using TensorFlow, for best performance you should set `image_data_format="channels_last"` in your Keras config at ~/.keras/keras.json. The model and the weights are compatible with both TensorFlow and Theano. The data format convention used by the model is the one specified in your Keras config file. Note that the default input image size for this model is 299x299. Arguments: include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet). input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(299, 299, 3)` (with `channels_last` data format) or `(3, 299, 299)` (with `channels_first` data format). It should have exactly 3 inputs channels, and width and height should be no smaller than 139. E.g. `(150, 150, 3)` would be one valid value. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. Returns: A Keras model instance. Raises: ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if weights not in {'imagenet', None}: raise ValueError('The `weights` argument should be either ' '`None` (random initialization) or `imagenet` ' '(pre-training on ImageNet).') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as imagenet with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape input_shape = _obtain_input_shape( input_shape, default_size=299, min_size=139, data_format=K.image_data_format(), include_top=include_top) if input_tensor is None: img_input = Input(shape=input_shape) else: img_input = Input(tensor=input_tensor, shape=input_shape) if K.image_data_format() == 'channels_first': channel_axis = 1 else: channel_axis = 3 x = conv2d_bn(img_input, 32, 3, 3, strides=(2, 2), padding='valid') x = conv2d_bn(x, 32, 3, 3, padding='valid') x = conv2d_bn(x, 64, 3, 3) x = MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv2d_bn(x, 80, 1, 1, padding='valid') x = conv2d_bn(x, 192, 3, 3, padding='valid') x = MaxPooling2D((3, 3), strides=(2, 2))(x) # mixed 0, 1, 2: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 32, 1, 1) x = layers.concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed0') # mixed 1: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = layers.concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed1') # mixed 2: 35 x 35 x 256 branch1x1 = conv2d_bn(x, 64, 1, 1) branch5x5 = conv2d_bn(x, 48, 1, 1) branch5x5 = conv2d_bn(branch5x5, 64, 5, 5) branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 64, 1, 1) x = layers.concatenate( [branch1x1, branch5x5, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed2') # mixed 3: 17 x 17 x 768 branch3x3 = conv2d_bn(x, 384, 3, 3, strides=(2, 2), padding='valid') branch3x3dbl = conv2d_bn(x, 64, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 96, 3, 3) branch3x3dbl = conv2d_bn( branch3x3dbl, 96, 3, 3, strides=(2, 2), padding='valid') branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) x = layers.concatenate( [branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed3') # mixed 4: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 128, 1, 1) branch7x7 = conv2d_bn(branch7x7, 128, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 128, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 128, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed4') # mixed 5, 6: 17 x 17 x 768 for i in range(2): branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 160, 1, 1) branch7x7 = conv2d_bn(branch7x7, 160, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 160, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 160, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed' + str(5 + i)) # mixed 7: 17 x 17 x 768 branch1x1 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(x, 192, 1, 1) branch7x7 = conv2d_bn(branch7x7, 192, 1, 7) branch7x7 = conv2d_bn(branch7x7, 192, 7, 1) branch7x7dbl = conv2d_bn(x, 192, 1, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 7, 1) branch7x7dbl = conv2d_bn(branch7x7dbl, 192, 1, 7) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch7x7, branch7x7dbl, branch_pool], axis=channel_axis, name='mixed7') # mixed 8: 8 x 8 x 1280 branch3x3 = conv2d_bn(x, 192, 1, 1) branch3x3 = conv2d_bn(branch3x3, 320, 3, 3, strides=(2, 2), padding='valid') branch7x7x3 = conv2d_bn(x, 192, 1, 1) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 1, 7) branch7x7x3 = conv2d_bn(branch7x7x3, 192, 7, 1) branch7x7x3 = conv2d_bn( branch7x7x3, 192, 3, 3, strides=(2, 2), padding='valid') branch_pool = MaxPooling2D((3, 3), strides=(2, 2))(x) x = layers.concatenate( [branch3x3, branch7x7x3, branch_pool], axis=channel_axis, name='mixed8') # mixed 9: 8 x 8 x 2048 for i in range(2): branch1x1 = conv2d_bn(x, 320, 1, 1) branch3x3 = conv2d_bn(x, 384, 1, 1) branch3x3_1 = conv2d_bn(branch3x3, 384, 1, 3) branch3x3_2 = conv2d_bn(branch3x3, 384, 3, 1) branch3x3 = layers.concatenate( [branch3x3_1, branch3x3_2], axis=channel_axis, name='mixed9_' + str(i)) branch3x3dbl = conv2d_bn(x, 448, 1, 1) branch3x3dbl = conv2d_bn(branch3x3dbl, 384, 3, 3) branch3x3dbl_1 = conv2d_bn(branch3x3dbl, 384, 1, 3) branch3x3dbl_2 = conv2d_bn(branch3x3dbl, 384, 3, 1) branch3x3dbl = layers.concatenate( [branch3x3dbl_1, branch3x3dbl_2], axis=channel_axis) branch_pool = AveragePooling2D((3, 3), strides=(1, 1), padding='same')(x) branch_pool = conv2d_bn(branch_pool, 192, 1, 1) x = layers.concatenate( [branch1x1, branch3x3, branch3x3dbl, branch_pool], axis=channel_axis, name='mixed' + str(9 + i)) if include_top: # Classification block x = GlobalAveragePooling2D(name='avg_pool')(x) x = Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='inception_v3') # load weights if weights == 'imagenet': if K.image_data_format() == 'channels_first': if K.backend() == 'tensorflow': warnings.warn('You are using the TensorFlow backend, yet you ' 'are using the Theano ' 'image data format convention ' '(`image_data_format="channels_first"`). ' 'For best performance, set ' '`image_data_format="channels_last"` in ' 'your Keras config ' 'at ~/.keras/keras.json.') if include_top: weights_path = get_file( 'inception_v3_weights_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH, cache_subdir='models', md5_hash='9a0d58056eeedaa3f26cb7ebd46da564') else: weights_path = get_file( 'inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5', WEIGHTS_PATH_NO_TOP, cache_subdir='models', md5_hash='bcbd6486424b2319ff4ef7d526e38f63') model.load_weights(weights_path) if K.backend() == 'theano': convert_all_kernels_in_model(model) return model
1. build a simple inception model: multiple inputs, multiple towers 2. Residual connection on a convolution layer 3. a shared vision model 4. Visual question answering model 5. Video question answering model ### Inception module For more information about the Inception architecture, see [Going Deeper with Convolutions](http://arxiv.org/abs/1409.4842). """ from tensorflow.contrib.keras.python.keras.layers import Conv2D, MaxPooling2D, Input, concatenate from tensorflow.contrib.keras.python.keras.models import Model input_img = Input(shape=(256, 256, 3)) tower_1 = Conv2D(64, (1, 1), padding='same', activation='relu')(input_img) tower_1 = Conv2D(64, (3, 3), padding='same', activation='relu')(tower_1) tower_2 = Conv2D(64, (1, 1), padding='same', activation='relu')(input_img) tower_2 = Conv2D(64, (5, 5), padding='same', activation='relu')(tower_2) tower_3 = MaxPooling2D((3, 3), strides=(1, 1), padding='same')(input_img) tower_3 = Conv2D(64, (1, 1), padding='same', activation='relu')(tower_3) output = concatenate([tower_1, tower_2, tower_3], axis=1) # add up on second dim axi=1 model = Model(input_img, output) model.summary()