def get_unet_model(IMG_HEIGHT=300, IMG_WIDTH=300, IMG_CHANNELS=3): inputs = layers.Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS)) s = Lambda(lambda x: x / 1)(inputs) c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(s) c1 = Dropout(0.1)(c1) c1 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c1) p1 = MaxPooling2D((2, 2))(c1) c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p1) c2 = Dropout(0.1)(c2) c2 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c2) p2 = MaxPooling2D((2, 2))(c2) c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p2) c3 = Dropout(0.2)(c3) c3 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c3) p3 = MaxPooling2D((2, 2))(c3) c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p3) c4 = Dropout(0.2)(c4) c4 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c4) p4 = MaxPooling2D(pool_size=(2, 2))(c4) c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(p4) c5 = Dropout(0.3)(c5) c5 = Conv2D(256, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c5) u6 = Conv2DTranspose(128, (2, 2), strides=(2, 2), padding='same')(c5) u6 = concatenate([u6, c4]) c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u6) c6 = Dropout(0.2)(c6) c6 = Conv2D(128, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c6) u7 = Conv2DTranspose(64, (2, 2), strides=(2, 2), padding='same')(c6) u7 = concatenate([u7, c3]) c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u7) c7 = Dropout(0.2)(c7) c7 = Conv2D(64, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c7) u8 = Conv2DTranspose(32, (2, 2), strides=(2, 2), padding='same')(c7) u8 = concatenate([u8, c2]) c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u8) c8 = Dropout(0.1)(c8) c8 = Conv2D(32, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c8) u9 = Conv2DTranspose(16, (2, 2), strides=(2, 2), padding='same')(c8) u9 = concatenate([u9, c1], axis=3) c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(u9) c9 = Dropout(0.1)(c9) c9 = Conv2D(16, (3, 3), activation='elu', kernel_initializer='he_normal', padding='same')(c9) outputs = Conv2D(1, (1, 1), activation='sigmoid')(c9) model = Model(inputs=[inputs], outputs=[outputs]) model.compile(optimizer='adam', loss='binary_crossentropy', metrics=[mean_iou]) model.summary() return model
def model(reader): inshape = (None, ) known_in = L.Input(shape=inshape, name='known_input', dtype='int32') unknown_in = L.Input(shape=inshape, name='unknown_input', dtype='int32') embedding = L.Embedding( len(reader.channels[0].vocabulary_above_cutoff) + 2, 5) known_embed = embedding(known_in) unknown_embed = embedding(unknown_in) conv8 = L.Convolution1D(filters=500, kernel_size=8, strides=1, activation='relu', name='convolutional_8') conv4 = L.Convolution1D(filters=500, kernel_size=4, strides=1, activation='relu', name='convolutional_4') conv16 = L.Convolution1D(filters=200, kernel_size=8, strides=1, activation='relu', name='convolutional_8_2') repr_known1 = L.GlobalMaxPooling1D(name='known_repr_8')(conv8(known_embed)) repr_unknown1 = L.GlobalMaxPooling1D(name='unknown_repr_8')( conv8(unknown_embed)) repr_known2 = L.GlobalMaxPooling1D(name='known_repr_4')(conv4(known_embed)) repr_unknown2 = L.GlobalMaxPooling1D(name='unknown_repr_4')( conv4(unknown_embed)) repr_known3 = L.GlobalMaxPooling1D(name='known_repr_8_2')( conv16(known_embed)) repr_unknown3 = L.GlobalMaxPooling1D(name='unknown_repr_8_2')( conv16(unknown_embed)) repr_known = L.Concatenate()([repr_known1, repr_known2, repr_known3]) repr_unknown = L.Concatenate()( [repr_unknown1, repr_unknown2, repr_unknown3]) abs_diff = L.merge(inputs=[repr_known, repr_unknown], mode=lambda x: abs(x[0] - x[1]), output_shape=lambda x: x[0], name='absolute_difference') dense1 = L.Dense(500, activation='relu')(abs_diff) dense2 = L.Dense(500, activation='relu')(dense1) dense3 = L.Dense(500, activation='relu')(dense2) dense4 = L.Dense(500, activation='relu')(dense3) pruned = L.Dropout(0.3)(dense4) output = L.Dense(2, activation='softmax', name='output')(pruned) model = Model(inputs=[known_in, unknown_in], outputs=output) optimizer = O.Adam(lr=0.0005) model.compile(optimizer=optimizer, loss='categorical_crossentropy', metrics=['accuracy']) return model
def add_prior(input_model, prior_shape, name='prior_model', prefix=None, use_logp=True, final_pred_activation='softmax', add_prior_layer_reg=0): """ Append post-prior layer to a given model """ # naming model_name = name if prefix is None: prefix = model_name # prior input layer prior_input_name = '%s-input' % prefix prior_tensor = KL.Input(shape=prior_shape, name=prior_input_name) prior_tensor_input = prior_tensor like_tensor = input_model.output # operation varies depending on whether we log() prior or not. if use_logp: # name = '%s-log' % prefix # prior_tensor = KL.Lambda(_log_layer_wrap(add_prior_layer_reg), name=name)(prior_tensor) print("Breaking change: use_logp option now requires log input!", file=sys.stderr) merge_op = KL.add else: # using sigmoid to get the likelihood values between 0 and 1 # note: they won't add up to 1. name = '%s_likelihood_sigmoid' % prefix like_tensor = KL.Activation('sigmoid', name=name)(like_tensor) merge_op = KL.multiply # merge the likelihood and prior layers into posterior layer name = '%s_posterior' % prefix post_tensor = merge_op([prior_tensor, like_tensor], name=name) # output prediction layer # we use a softmax to compute P(L_x|I) where x is each location pred_name = '%s_prediction' % prefix if final_pred_activation == 'softmax': assert use_logp, 'cannot do softmax when adding prior via P()' print("using final_pred_activation %s for %s" % (final_pred_activation, model_name)) softmax_lambda_fcn = lambda x: keras.activations.softmax(x, axis=-1) pred_tensor = KL.Lambda(softmax_lambda_fcn, name=pred_name)(post_tensor) else: pred_tensor = KL.Activation('linear', name=pred_name)(post_tensor) # create the model model_inputs = [*input_model.inputs, prior_tensor_input] model = Model(inputs=model_inputs, outputs=[pred_tensor], name=model_name) # compile return model
def design_dnn(nb_features, input_shape, nb_levels, conv_size, nb_labels, feat_mult=1, pool_size=2, padding='same', activation='elu', final_layer='dense-sigmoid', conv_dropout=0, conv_maxnorm=0, nb_input_features=1, batch_norm=False, name=None, prefix=None, use_strided_convolution_maxpool=True, nb_conv_per_level=2): """ "deep" cnn with dense or global max pooling layer @ end... Could use sequential... """ model_name = name if model_name is None: model_name = 'model_1' if prefix is None: prefix = model_name ndims = len(input_shape) input_shape = tuple(input_shape) convL = getattr(KL, 'Conv%dD' % ndims) maxpool = KL.MaxPooling3D if len(input_shape) == 3 else KL.MaxPooling2D if isinstance(pool_size, int): pool_size = (pool_size, ) * ndims # kwargs for the convolution layer conv_kwargs = {'padding': padding, 'activation': activation} if conv_maxnorm > 0: conv_kwargs['kernel_constraint'] = maxnorm(conv_maxnorm) # initialize a dictionary enc_tensors = {} # first layer: input name = '%s_input' % prefix enc_tensors[name] = KL.Input(shape=input_shape + (nb_input_features, ), name=name) last_tensor = enc_tensors[name] # down arm: # add nb_levels of conv + ReLu + conv + ReLu. Pool after each of first nb_levels - 1 layers for level in range(nb_levels): for conv in range(nb_conv_per_level): if conv_dropout > 0: name = '%s_dropout_%d_%d' % (prefix, level, conv) enc_tensors[name] = KL.Dropout(conv_dropout)(last_tensor) last_tensor = enc_tensors[name] name = '%s_conv_%d_%d' % (prefix, level, conv) nb_lvl_feats = np.round(nb_features * feat_mult**level).astype(int) enc_tensors[name] = convL(nb_lvl_feats, conv_size, **conv_kwargs, name=name)(last_tensor) last_tensor = enc_tensors[name] # max pool if use_strided_convolution_maxpool: name = '%s_strided_conv_%d' % (prefix, level) enc_tensors[name] = convL(nb_lvl_feats, pool_size, **conv_kwargs, name=name)(last_tensor) last_tensor = enc_tensors[name] else: name = '%s_maxpool_%d' % (prefix, level) enc_tensors[name] = maxpool(pool_size=pool_size, name=name, padding=padding)(last_tensor) last_tensor = enc_tensors[name] # dense layer if final_layer == 'dense-sigmoid': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(1, name=name, activation="sigmoid")(last_tensor) elif final_layer == 'dense-tanh': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(1, name=name)(last_tensor) last_tensor = enc_tensors[name] # Omittting BatchNorm for now, it seems to have a cpu vs gpu problem # https://github.com/tensorflow/tensorflow/pull/8906 # https://github.com/fchollet/keras/issues/5802 # name = '%s_%s_bn' % prefix # enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # last_tensor = enc_tensors[name] name = '%s_%s_tanh' % prefix enc_tensors[name] = KL.Activation(activation="tanh", name=name)(last_tensor) elif final_layer == 'dense-softmax': name = "%s_flatten" % prefix enc_tensors[name] = KL.Flatten(name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_dense' % prefix enc_tensors[name] = KL.Dense(nb_labels, name=name, activation="softmax")(last_tensor) # global max pooling layer elif final_layer == 'myglobalmaxpooling': name = '%s_batch_norm' % prefix enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool' % prefix enc_tensors[name] = KL.Lambda(_global_max_nd, name=name)(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool_reshape' % prefix enc_tensors[name] = KL.Reshape((1, 1), name=name)(last_tensor) last_tensor = enc_tensors[name] # cannot do activation in lambda layer. Could code inside, but will do extra lyaer name = '%s_global_max_pool_sigmoid' % prefix enc_tensors[name] = KL.Conv1D(1, 1, name=name, activation="sigmoid", use_bias=True)(last_tensor) elif final_layer == 'globalmaxpooling': name = '%s_conv_to_featmaps' % prefix enc_tensors[name] = KL.Conv3D(2, 1, name=name, activation="relu")(last_tensor) last_tensor = enc_tensors[name] name = '%s_global_max_pool' % prefix enc_tensors[name] = KL.GlobalMaxPooling3D(name=name)(last_tensor) last_tensor = enc_tensors[name] # cannot do activation in lambda layer. Could code inside, but will do extra lyaer name = '%s_global_max_pool_softmax' % prefix enc_tensors[name] = KL.Activation('softmax', name=name)(last_tensor) last_tensor = enc_tensors[name] # create the model model = Model(inputs=[enc_tensors['%s_input' % prefix]], outputs=[last_tensor], name=model_name) return model
def VGG19_l2(input_shape=None, classes=5, use_soft=True): img_input = layers.Input(shape=input_shape) # Block 1 x = layers.Conv2D(2, (3, 3), activation='relu', padding='same', name='block1_conv1', kernel_initializer="he_normal")(img_input) x = layers.Conv2D(2, (3, 3), activation='relu', padding='same', name='block1_conv2', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x) # Block 2 x = layers.Conv2D(4, (3, 3), activation='relu', padding='same', name='block2_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(4, (3, 3), activation='relu', padding='same', name='block2_conv2', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x) # Block 3 x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x) # Block 4 x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x) # Block 5 x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x) # Classification block x = layers.Flatten(name='flatten')(x) x = layers.Dense(512, activation='relu', name='fc1', kernel_regularizer=regularizers.l2(1e-4), bias_regularizer=regularizers.l2(1e-4), activity_regularizer=regularizers.l2(1e-4))(x) x = layers.Dropout(0.5)(x) x = layers.Dense(128, activation='relu', name='fc2', kernel_regularizer=regularizers.l2(1e-4), bias_regularizer=regularizers.l2(1e-4), activity_regularizer=regularizers.l2(1e-4))(x) #x=layers.Dropout(0.8)(x) if use_soft: x = Dense(classes, activation="softmax", name='predictions', kernel_regularizer=regularizers.l2(1e-4), bias_regularizer=regularizers.l2(1e-4), activity_regularizer=regularizers.l2(1e-4))(x) else: x = Dense(classes, activation="linear", name="Z_4", kernel_regularizer=regularizers.l2(1e-4), bias_regularizer=regularizers.l2(1e-4), activity_regularizer=regularizers.l2(1e-4))(x) model = models.Model(img_input, x, name='vgg19') return model
# 验证 # input_shape越大,精度会上升,但速度会下降。 # input_shape = (320, 320) # input_shape = (416, 416) input_shape = (608, 608) # 验证时的分数阈值和nms_iou阈值 conf_thresh = 0.001 nms_thresh = 0.45 # 是否画出验证集图片 draw_image = False # 验证时的批大小 eval_batch_size = 4 # 多尺度训练 inputs = layers.Input(shape=(None, None, 3)) model_body = YOLOv4(inputs, num_classes, num_anchors) _decode = Decode(conf_thresh, nms_thresh, input_shape, model_body, class_names) # 模式。 0-从头训练,1-读取之前的模型继续训练(model_path可以是'yolov4.h5'、'./weights/step00001000.h5'这些。) pattern = 1 max_bbox_per_scale = 150 iou_loss_thresh = 0.7 if pattern == 1: lr = 0.0001 batch_size = 8 model_path = 'yolov4.h5' # model_path = './weights/step00001000.h5' model_body.load_weights(model_path, by_name=True) strs = model_path.split('step') if len(strs) == 2:
def conv_enc(nb_features, input_shape, nb_levels, conv_size, name=None, prefix=None, feat_mult=1, pool_size=2, dilation_rate_mult=1, padding='same', activation='elu', layer_nb_feats=None, use_residuals=False, nb_conv_per_level=2, conv_dropout=0, batch_norm=None): """ Fully Convolutional Encoder """ # naming model_name = name if prefix is None: prefix = model_name # volume size data ndims = len(input_shape) - 1 input_shape = tuple(input_shape) if isinstance(pool_size, int): pool_size = (pool_size, ) * ndims # prepare layers convL = getattr(KL, 'Conv%dD' % ndims) conv_kwargs = {'padding': padding, 'activation': activation} maxpool = getattr(KL, 'MaxPooling%dD' % ndims) # first layer: input name = '%s_input' % prefix last_tensor = KL.Input(shape=input_shape, name=name) input_tensor = last_tensor # down arm: # add nb_levels of conv + ReLu + conv + ReLu. Pool after each of first nb_levels - 1 layers lfidx = 0 for level in range(nb_levels): lvl_first_tensor = last_tensor nb_lvl_feats = np.round(nb_features * feat_mult**level).astype(int) conv_kwargs['dilation_rate'] = dilation_rate_mult**level for conv in range(nb_conv_per_level): if layer_nb_feats is not None: nb_lvl_feats = layer_nb_feats[lfidx] lfidx += 1 name = '%s_conv_downarm_%d_%d' % (prefix, level, conv) if conv < (nb_conv_per_level - 1) or (not use_residuals): last_tensor = convL(nb_lvl_feats, conv_size, **conv_kwargs, name=name)(last_tensor) else: # no activation last_tensor = convL(nb_lvl_feats, conv_size, padding=padding, name=name)(last_tensor) if conv_dropout > 0: # conv dropout along feature space only name = '%s_dropout_downarm_%d_%d' % (prefix, level, conv) noise_shape = [None, *[1] * ndims, nb_lvl_feats] last_tensor = KL.Dropout(conv_dropout, noise_shape=noise_shape)(last_tensor) if use_residuals: convarm_layer = last_tensor # the "add" layer is the original input # However, it may not have the right number of features to be added nb_feats_in = lvl_first_tensor.get_shape()[-1] nb_feats_out = convarm_layer.get_shape()[-1] add_layer = lvl_first_tensor if nb_feats_in > 1 and nb_feats_out > 1 and (nb_feats_in != nb_feats_out): name = '%s_expand_down_merge_%d' % (prefix, level) last_tensor = convL(nb_lvl_feats, conv_size, **conv_kwargs, name=name)(lvl_first_tensor) add_layer = last_tensor if conv_dropout > 0: name = '%s_dropout_down_merge_%d_%d' % (prefix, level, conv) noise_shape = [None, *[1] * ndims, nb_lvl_feats] last_tensor = KL.Dropout( conv_dropout, noise_shape=noise_shape)(last_tensor) name = '%s_res_down_merge_%d' % (prefix, level) last_tensor = KL.add([add_layer, convarm_layer], name=name) name = '%s_res_down_merge_act_%d' % (prefix, level) last_tensor = KL.Activation(activation, name=name)(last_tensor) if batch_norm is not None: name = '%s_bn_down_%d' % (prefix, level) last_tensor = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # max pool if we're not at the last level if level < (nb_levels - 1): name = '%s_maxpool_%d' % (prefix, level) last_tensor = maxpool(pool_size=pool_size, name=name, padding=padding)(last_tensor) # create the model and return model = Model(inputs=input_tensor, outputs=[last_tensor], name=model_name) return model
def model(self, model_type): input_image = KL.Input(shape=[None, None, 3], name='input_image') input_bbox = KL.Input(shape=[None, 4], name='input_bbox') input_image_meta = KL.Input(shape=[self.cfg.IMAGE_META_SIZE], name="input_image_meta") input_anchors = KL.Input(shape=[None, 4], name="input_anchors") detection_map, reid_map = self.reid(input_image) rpn = self.build_rpn_model(self.cfg.RPN_ANCHOR_STRIDE, len(self.cfg.RPN_ANCHOR_RATIOS), self.cfg.TOP_DOWN_PYRAMID_SIZE) outputs = list(zip(*[rpn(p) for p in detection_map])) rpn_class_logits, rpn_class, rpn_bbox = [ KL.Concatenate(axis=1, name=n)(list(o)) for o, n in zip( outputs, ["rpn_class_logits", "rpn_class", "rpn_bbox"]) ] rpn_rois = ProposalLayer( proposal_count=self.cfg.POST_NMS_ROIS_INFERENCE, nms_threshold=self.cfg.RPN_NMS_THRESHOLD, name="ROI", config=self.cfg)([rpn_class, rpn_bbox, input_anchors]) rois = KL.Lambda(lambda x: tf.concat(x, axis=1))( [input_bbox, rpn_rois]) x = PyramidROIAlign( [self.cfg.POOL_SIZE, self.cfg.POOL_SIZE], name="roi_align_classifier")([rois, input_image_meta] + detection_map[:-1]) x = self.mrcnn_feature(x) scores, bboxes = self.mrcnn_box_reg([rois, x]) detection_scores, regression_scores, detection, regression = KL.Lambda(lambda x : [x[0][:, tf.shape(input_bbox)[1]:, :], \ x[0][:, :tf.shape(input_bbox)[1], :], \ x[1][:, tf.shape(input_bbox)[1]:, :], \ x[1][:, :tf.shape(input_bbox)[1], :] ])([scores, bboxes]) detection_scores, detection = KL.Lambda( lambda x: self.nms_selection(x[0], x[1]))( [detection_scores, detection]) if model_type == 'detection': return KM.Model( [input_image, input_bbox, input_image_meta, input_anchors], [detection, detection_scores], name='mrcnn') bboxes = KL.Lambda(lambda x: tf.concat(x, axis=1))( [input_bbox, regression, detection]) reid_map = ATLnet(reid_map, layer=self.cfg.layer, SEnet=self.cfg.SEnet) pooled = feature_pooling(self.cfg, name='alignedROIPooling')([bboxes] + reid_map) pooled = KL.Lambda(lambda x: tf.squeeze(x, axis=0))(pooled) vectors = sMGN(pooled, _eval=True, return_all=self.cfg.mgn, return_mgn=True, l2_norm=self.cfg.l2_norm) vectors = KL.Lambda(lambda x: tf.expand_dims(x[:, 0, 0, :], axis=0))( vectors) prediction_vector, regression_vector, detection_vector = KL.Lambda(lambda x : [x[:, :tf.shape(input_bbox)[1], :], \ x[:, tf.shape(input_bbox)[1]:(2*tf.shape(input_bbox)[1]), :], \ x[:, (2*tf.shape(input_bbox)[1]):, :], \ ])(vectors) return KM.Model( [input_image, input_bbox, input_image_meta, input_anchors], [ prediction_vector, regression_vector, regression, regression_scores, detection_vector, detection, detection_scores ])
def build_rpn_model(self, anchor_stride, anchors_per_location, depth): input_feature_map = KL.Input(shape=[None, None, depth], name="input_rpn_feature_map") outputs = self.rpn_graph(input_feature_map, anchors_per_location, anchor_stride) return KM.Model([input_feature_map], outputs, name="rpn_model")
# import ipdb; ipdb.set_trace() # train dataset dataset_train = u.get_dataset(coco_path, 'train') gen_train = prepare(dataset_train, epochs, batch_size, input_shape, output_shape) callback = ModelSaveBestAvgAcc(filepath="model-{epoch:02d}-{avgacc:.2f}.hdf5", verbose=True, cond=filter_val('fmeasure')) losses = [] for i in range(0, 2): losses.append(binary_focal_loss(gamma=2.)) input_tensor = layers.Input(shape=input_shape) model = get_model(input_tensor=input_tensor) outputs = model.outputs x = layers.Multiply()([input_tensor, model.output]) x = model(x) # import ipdb; ipdb.set_trace() model = models.Model(inputs=input_tensor, outputs=outputs + [x]) model.compile(optimizer=opt.Adam(lr=1e-4), loss=losses, metrics=['accuracy', precision, recall, fmeasure]) model.summary()
def ResNet(stack_fn, preact, use_bias, model_name='resnet', include_top=True, weights=None, input_tensor=None, input_shape=None, pooling=None, nclass=1000, **kwargs): """Instantiates the ResNet, ResNetV2, and ResNeXt architecture. Optionally loads weights pre-trained on ImageNet. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. # Arguments stack_fn: a function that returns output tensor for the stacked residual blocks. preact: whether to use pre-activation or not (True for ResNetV2, False for ResNet and ResNeXt). use_bias: whether to use biases for convolutional layers or not (True for ResNet and ResNetV2, False for ResNeXt). model_name: string, model name. include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization), 'imagenet' (pre-training on ImageNet), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or `(3, 224, 224)` (with `channels_first` data format). It should have exactly 3 inputs channels. pooling: optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ # Determine proper input shape # input_shape = input_shape if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor bn_axis = 3 if backend.image_data_format() == 'channels_last' else 1 x = layers.ZeroPadding2D(padding=((3, 3), (3, 3)), name='conv1_pad')(img_input) x = layers.Conv2D(64, 7, strides=2, use_bias=use_bias, name='conv1_conv')(x) if preact is False: x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='conv1_bn')(x) x = layers.Activation('relu', name='conv1_relu')(x) x = layers.ZeroPadding2D(padding=((1, 1), (1, 1)), name='pool1_pad')(x) x = layers.MaxPooling2D(3, strides=2, name='pool1_pool')(x) x = stack_fn(x) if preact is True: x = layers.BatchNormalization(axis=bn_axis, epsilon=1.001e-5, name='post_bn')(x) x = layers.Activation('relu', name='post_relu')(x) if include_top: x = layers.GlobalAveragePooling2D(name='avg_pool')(x) x = layers.Dense(nclass, activation='softmax', name='probs')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling2D(name='avg_pool')(x) elif pooling == 'max': x = layers.GlobalMaxPooling2D(name='max_pool')(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = keras_utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = models.Model(inputs, x, name=model_name) # Load weights. if weights is not None: model.load_weights(weights) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def CapsNet(input_shape, n_class, num_routing): from keras import layers, models from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask from keras.preprocessing import sequence """ A Capsule Network on MNIST. :param input_shape: data shape, 4d, [None, width, height, channels] :param n_class: number of classes :param num_routing: number of routing iterations :return: A Keras Model with 2 inputs and 2 outputs """ x = layers.Input(shape=(maxlen, ), dtype='int32') embed = layers.Embedding(max_features, embed_dim, input_length=maxlen)(x) # conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(embed) conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(embed) pool = layers.MaxPooling1D(pool_size=231, strides=1)(conv1) # lstm = layers.LSTM(9, return_sequences=True, recurrent_dropout=0.6, dropout = 0.9, name='lstm1')(conv1) # lstm = layers.CuDNNLSTM(9, return_sequences=True, name='lstm1')(pool) # lstm2 = layers.CuDNNLSTM(9, return_sequences=True, name='lstm2')(lstm) # drop = layers.Dropout(0.5, name='drop1')(lstm) # td = layers.TimeDistributed(layers.Dense(81, activation='softmax'))(lstm) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector] # primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') primarycaps = PrimaryCap(pool, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. # digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps) digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='out_caps')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked = Mask()( [digitcaps, y]) # The true label is used to mask the output of capsule layer. # x_recon = layers.Dense(512, activation='relu')(masked) # x_recon = layers.Dense(1024, activation='relu')(x_recon) x_recon = layers.Dense(512, activation='relu')(masked) # x_recon = layers.Dropout(0.9)(x_recon) x_recon = layers.Dense(1024, activation='relu')(x_recon) # x_recon = layers.Dropout(0.9)(x_recon) x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon) # x_recon = layers.Reshape(target_shape=[1], name='out_recon')(x_recon) # two-input-two-output keras Model return models.Model([x, y], [out_caps, x_recon])
################## train_dataset = '../../200_dataset/norm_audio_video/tr_set.hdf5' val_dataset = '../../200_dataset/norm_audio_video/val_set.hdf5' test_dataset = '../../200_dataset/norm_audio_video/test_set.hdf5' batch_size = 10 epochs = 100 train_generator = data_generator(train_dataset, batch_size) val_generator = data_generator(val_dataset, batch_size) ##V:video1 VV:video2 V1 = layers.Input(shape=(75, 512), name='Video1_input') V2 = layers.Conv1D(256, kernel_size=7, dilation_rate=1, padding='same', activation='relu')(V1) V3 = layers.BatchNormalization(axis=-1)(V2) V4 = layers.Conv1D(256, kernel_size=5, dilation_rate=1, padding='same', activation='relu')(V3) V5 = layers.BatchNormalization(axis=-1)(V4) V6 = layers.Conv1D(256, kernel_size=5, dilation_rate=2,
def get_model(img_size=(1, 300, 300, 3), num_classes=4): inputs = layers.Input(img_size, name='RGB_Input') print('input shape ', inputs.shape) ### [First half of the network: downsampling inputs] ### # Entry block x = layers.Conv2D(32, 3, strides=2, padding="same")(inputs) x = layers.BatchNormalization()(x) x = layers.Activation("relu")(x) previous_block_activation = x # Set aside residual # Blocks 1, 2, 3 are identical apart from the feature depth. for filters in [64, 128, 256]: x = layers.Activation("relu")(x) x = layers.SeparableConv2D(filters, 3, padding="same")(x) x = layers.BatchNormalization()(x) x = layers.Activation("relu")(x) x = layers.SeparableConv2D(filters, 3, padding="same")(x) x = layers.BatchNormalization()(x) x = layers.MaxPooling2D(3, strides=2, padding="same")(x) # Project residual residual = layers.Conv2D(filters, 1, strides=2, padding="same")(previous_block_activation) x = layers.add([x, residual]) # Add back residual previous_block_activation = x # Set aside next residual ### [Second half of the network: upsampling inputs] ### for filters in [256, 128, 64, 32]: x = layers.Activation("relu")(x) x = layers.Conv2DTranspose(filters, 3, padding="same")(x) x = layers.BatchNormalization()(x) x = layers.Activation("relu")(x) x = layers.Conv2DTranspose(filters, 3, padding="same")(x) x = layers.BatchNormalization()(x) x = layers.UpSampling2D(2)(x) # Project residual residual = layers.UpSampling2D(2)(previous_block_activation) residual = layers.Conv2D(filters, 1, padding="same")(residual) x = layers.add([x, residual]) # Add back residual previous_block_activation = x # Set aside next residual # Add a per-pixel classification layer #outputs = layers.Conv2D(num_classes, 0, activation="softmax", padding="same")(x) d = layers.Convolution2D(1, (1, 1), activation='sigmoid', padding='same')(x) d = layers.Cropping2D((EDGE_CROP, EDGE_CROP))(d) d = layers.ZeroPadding2D((EDGE_CROP, EDGE_CROP))(d) # Define the model model0 = models.Model(inputs=[inputs], outputs=[d]) #model = models.Model(inputs, outputs) return model0
# In[6]: print('Pad sequences (samples x time)') x_train = sequence.pad_sequences(x_train, maxlen=maxlen) x_test = sequence.pad_sequences(x_test, maxlen=maxlen) print('x_train shape:', x_train.shape) print('x_test shape:', x_test.shape) # In[10]: import keras.layers as layers from sklearn import metrics metrics.classification layers.Input() print('Build model...') model = Sequential() model.add(Embedding(max_features, 128)) model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2)) model.add(Dense(1, activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) print('Train...') model.fit(x_train, y_train, batch_size=batch_size, epochs=15,
first_model = models.Model(inputs=model1.input, outputs=first_model) second_model = model2.layers[9].output second_model = models.Model(inputs=model2.input, outputs=second_model) first_model.summary() second_model.summary() pred1_train = first_model.predict(x_train) pred1_test = first_model.predict(x_test) pred2_train = second_model.predict(x_train) pred2_test = second_model.predict(x_test) x_train = np.column_stack((pred1_train, pred2_train)) x_test = np.column_stack((pred1_test, pred2_test)) print(x_train.shape) print(x_test.shape) input_audio = layers.Input(shape=(x_train.shape[1], )) model = layers.Dense(32, activation='relu')(input_audio) #model = layers.Dense(32, activation='relu')(model) model = layers.Dense(3, activation='softmax')(model) model = Model(input_audio, model) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adam(), metrics=['accuracy']) model.summary() path_save = os.path.join("/users/home/s18023/DCASE2020/taskb/final/model_d") os.chdir(path_save) checkpointer = ModelCheckpoint(filepath=weightpath_name, monitor='val_loss',
def Xception(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000, **kwargs): """Instantiates the Xception architecture. Optionally loads weights pre-trained on ImageNet. Note that the data format convention used by the model is the one specified in your Keras config at `~/.keras/keras.json`. Note that the default input image size for this model is 299x299. # Arguments include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization), 'imagenet' (pre-training on ImageNet), or the path to the weights file to be loaded. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(299, 299, 3)`. It should have exactly 3 inputs channels, and width and height should be no smaller than 71. E.g. `(150, 150, 3)` would be one valid value. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional block. - `avg` means that global average pooling will be applied to the output of the last convolutional block, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. RuntimeError: If attempting to run this model with a backend that does not support separable convolutions. """ # backend, layers, models, keras_utils = get_submodules_from_kwargs(kwargs) if not (weights in {'imagenet', None} or os.path.exists(weights)): raise ValueError('The `weights` argument should be either ' '`None` (random initialization), `imagenet` ' '(pre-training on ImageNet), ' 'or the path to the weights file to be loaded.') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as `"imagenet"` with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape input_shape = _obtain_input_shape(input_shape, default_size=299, min_size=71, data_format=backend.image_data_format(), require_flatten=include_top, weights=weights) if input_tensor is None: img_input = layers.Input(shape=input_shape) else: if not backend.is_keras_tensor(input_tensor): img_input = layers.Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1 x = layers.Conv2D(32, (3, 3), strides=(2, 2), use_bias=False, name='block1_conv1')(img_input) x = layers.BatchNormalization(axis=channel_axis, name='block1_conv1_bn')(x) x = layers.Activation('relu', name='block1_conv1_act')(x) x = layers.Conv2D(64, (3, 3), use_bias=False, name='block1_conv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block1_conv2_bn')(x) x = layers.Activation('relu', name='block1_conv2_act')(x) residual = layers.Conv2D(128, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x) residual = layers.BatchNormalization(axis=channel_axis)(residual) x = layers.SeparableConv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block2_sepconv1_bn')(x) x = layers.Activation('relu', name='block2_sepconv2_act')(x) x = layers.SeparableConv2D(128, (3, 3), padding='same', use_bias=False, name='block2_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block2_sepconv2_bn')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block2_pool')(x) x = layers.add([x, residual]) residual = layers.Conv2D(256, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x) residual = layers.BatchNormalization(axis=channel_axis)(residual) x = layers.Activation('relu', name='block3_sepconv1_act')(x) x = layers.SeparableConv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block3_sepconv1_bn')(x) x = layers.Activation('relu', name='block3_sepconv2_act')(x) x = layers.SeparableConv2D(256, (3, 3), padding='same', use_bias=False, name='block3_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block3_sepconv2_bn')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block3_pool')(x) x = layers.add([x, residual]) residual = layers.Conv2D(728, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x) residual = layers.BatchNormalization(axis=channel_axis)(residual) x = layers.Activation('relu', name='block4_sepconv1_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block4_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block4_sepconv1_bn')(x) x = layers.Activation('relu', name='block4_sepconv2_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block4_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block4_sepconv2_bn')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block4_pool')(x) x = layers.add([x, residual]) for i in range(8): residual = x prefix = 'block' + str(i + 5) x = layers.Activation('relu', name=prefix + '_sepconv1_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name=prefix + '_sepconv1_bn')(x) x = layers.Activation('relu', name=prefix + '_sepconv2_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name=prefix + '_sepconv2_bn')(x) x = layers.Activation('relu', name=prefix + '_sepconv3_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name=prefix + '_sepconv3')(x) x = layers.BatchNormalization(axis=channel_axis, name=prefix + '_sepconv3_bn')(x) x = layers.add([x, residual]) residual = layers.Conv2D(1024, (1, 1), strides=(2, 2), padding='same', use_bias=False)(x) residual = layers.BatchNormalization(axis=channel_axis)(residual) x = layers.Activation('relu', name='block13_sepconv1_act')(x) x = layers.SeparableConv2D(728, (3, 3), padding='same', use_bias=False, name='block13_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block13_sepconv1_bn')(x) x = layers.Activation('relu', name='block13_sepconv2_act')(x) x = layers.SeparableConv2D(1024, (3, 3), padding='same', use_bias=False, name='block13_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block13_sepconv2_bn')(x) x = layers.MaxPooling2D((3, 3), strides=(2, 2), padding='same', name='block13_pool')(x) x = layers.add([x, residual]) x = layers.SeparableConv2D(1536, (3, 3), padding='same', use_bias=False, name='block14_sepconv1')(x) x = layers.BatchNormalization(axis=channel_axis, name='block14_sepconv1_bn')(x) x = layers.Activation('relu', name='block14_sepconv1_act')(x) x = layers.SeparableConv2D(2048, (3, 3), padding='same', use_bias=False, name='block14_sepconv2')(x) x = layers.BatchNormalization(axis=channel_axis, name='block14_sepconv2_bn')(x) x = layers.Activation('relu', name='block14_sepconv2_act')(x) if include_top: x = layers.GlobalAveragePooling2D(name='avg_pool')(x) x = layers.Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = layers.GlobalAveragePooling2D()(x) elif pooling == 'max': x = layers.GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = keras_utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = models.Model(inputs, x, name='xception') # Load weights. if weights == 'imagenet': if include_top: weights_path = keras_utils.get_file( 'xception_weights_tf_dim_ordering_tf_kernels.h5', TF_WEIGHTS_PATH, cache_subdir='models', file_hash='0a58e3b7378bc2990ea3b43d5981f1f6') else: weights_path = keras_utils.get_file( 'xception_weights_tf_dim_ordering_tf_kernels_notop.h5', TF_WEIGHTS_PATH_NO_TOP, cache_subdir='models', file_hash='b0042744bf5b25fce3cb969f33bebb97') model.load_weights(weights_path) if backend.backend() == 'theano': keras_utils.convert_all_kernels_in_model(model) elif weights is not None: model.load_weights(weights) return model
x = layers.LeakyReLU()(x) x = layers.Reshape((6, 6, 128))(x) x = layers.Conv2D(256, 5, padding='same')(x) x = layers.LeakyReLU()(x) x = layers.Conv2DTranspose(256, 4, strides=1, padding='same')(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(256, 5, padding='same')(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 5, padding='same')(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(channels, 7, activation='tanh', padding='same')(x) generator = keras.models.Model(generator_input, x) generator.summary() discriminator_input = layers.Input(shape=(height, width, channels)) x = layers.Conv2D(128, 1)(discriminator_input) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 1, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 1, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Conv2D(128, 1, strides=2)(x) x = layers.LeakyReLU()(x) x = layers.Flatten()(x) x = layers.Dropout(0, 4)(x) x = layers.Dense(1, activation='sigmoid')(x) discriminator = keras.models.Model(discriminator_input, x) discriminator.summary()
def main(): # user options batch_size = 128 val_in_train = False # not sure how the validation part works during fit. use_model_checkpt = False # demo processing sess = tf.Session() KB.set_session(sess) gdev_list = get_available_gpus() ngpus = len(gdev_list) batch_size = batch_size * ngpus data = mnist.load_mnist() X_train = data.train.images # X_test = data.test.images train_samples = X_train.shape[0] # 60000 # test_samples = X_test.shape[0] # 10000 height_nrows = 28 width_ncols = 28 batch_shape = [batch_size, height_nrows, width_ncols, 1] epochs = 5 steps_per_epoch = train_samples // batch_size # validations_steps = test_samples / batch_size nclasses = 10 # The capacity variable controls the maximum queue size # allowed when prefetching data for training. capacity = 10000 # min_after_dequeue is the minimum number elements in the queue # after a dequeue, which ensures sufficient mixing of elements. min_after_dequeue = 3000 # If `enqueue_many` is `False`, `tensors` is assumed to represent a # single example. An input tensor with shape `[x, y, z]` will be output # as a tensor with shape `[batch_size, x, y, z]`. # # If `enqueue_many` is `True`, `tensors` is assumed to represent a # batch of examples, where the first dimension is indexed by example, # and all members of `tensors` should have the same size in the # first dimension. If an input tensor has shape `[*, x, y, z]`, the # output will have shape `[batch_size, x, y, z]`. enqueue_many = True x_train_batch, y_train_batch = tf.train.shuffle_batch( tensors=[data.train.images, data.train.labels.astype(np.int32)], batch_size=batch_size, capacity=capacity, min_after_dequeue=min_after_dequeue, enqueue_many=enqueue_many, num_threads=8) x_train_batch = tf.cast(x_train_batch, tf.float32) x_train_batch = tf.reshape(x_train_batch, shape=batch_shape) y_train_batch = tf.cast(y_train_batch, tf.int32) y_train_batch = tf.one_hot(y_train_batch, nclasses) x_train_input = Input(tensor=x_train_batch) # x_test_batch, y_test_batch = tf.train.batch( # tensors=[data.test.images, data.test.labels.astype(np.int32)], # batch_size=batch_size, # capacity=capacity, # enqueue_many=enqueue_many, # num_threads=8) # I like the non-functional definition of model more. # model_init = make_model(x_train_input, nclasses) # x_train_out = model_init.output # train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) x_train_out = cnn_layers(x_train_input, nclasses) train_model = Model(inputs=[x_train_input], outputs=[x_train_out]) if ngpus > 1: train_model = make_parallel(train_model, gdev_list) lr = 2e-3 * ngpus train_model.compile(optimizer=RMSprop(lr=lr, decay=1e-5), loss='categorical_crossentropy', metrics=['accuracy'], target_tensors=[y_train_batch]) if ngpus > 1: print_mgpu_modelsummary(train_model) else: train_model.summary() # Callbacks if use_model_checkpt: mon = 'val_acc' if val_in_train else 'acc' checkpoint = ModelCheckpoint( 'saved_wt.h5', monitor=mon, verbose=0, save_best_only=True, save_weights_only=True) checkpoint = [checkpoint] else: checkpoint = [] callbacks = checkpoint # Training slower with callback. Multigpu slower with callback during # training than 1 GPU. Again, mnist is too trivial of a model and dataset # to benchmark or stress GPU compute capabilities. I set up this example # to illustrate potential for speedup of multigpu case trying to use mnist # as a stressor. # It's like comparing a 5 ft race between a person and a truck. A truck is # obviously faster than a person but in a 5 ft race the person will likely # win due to slower startup for the truck. # I will re-implement this with Cifar that should be a better benchmark. # Start the queue runners. tf.train.start_queue_runners(sess=sess) # Fit the model using data from the TFRecord data tensors. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess, coord) start_time = time.time() train_model.fit( # validation_data=(x_test_batch, y_test_batch) # if val_in_train else None, # validation data is not used??? # validations_steps if val_in_train else None, # validation_steps=val_in_train, steps_per_epoch=steps_per_epoch, epochs=epochs, callbacks=callbacks) elapsed_time = time.time() - start_time print('[{}] finished in {} s'.format('TRAINING', round(elapsed_time, 3))) if not checkpoint: # empty list train_model.save_weights('./saved_wt.h5') # Clean up the TF session. coord.request_stop() coord.join(threads) KB.clear_session() # Second Session. Demonstrate that the model works and is independent of # the TFRecord pipeline, and to test loading trained model without tensors. x_test = np.reshape(data.validation.images, (data.validation.images.shape[0], 28, 28, 1)) y_test = data.validation.labels x_test_inp = KL.Input(shape=(x_test.shape[1:])) test_out = cnn_layers(x_test_inp, nclasses) test_model = Model(inputs=x_test_inp, outputs=test_out) test_model.load_weights('saved_wt.h5') test_model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy']) test_model.summary() loss, acc = test_model.evaluate(x_test, to_categorical(y_test)) print('\nTest loss: {0}'.format(loss)) print('\nTest accuracy: {0}'.format(acc))
time.strftime('%Y-%m-%d_%H.%M.%S', time.gmtime()) os.mkdir('bb/models/' + run_id) print('Loading data') train_X = np.load('bb/data/sequences_train-' + str(TIMESTEPS) + 'steps.npy')[:, :, [0, 2, 4]] val_X = np.load('bb/data/sequences_val-' + str(TIMESTEPS) + 'steps.npy')[:, :, [0, 2, 4]] if len(train_X) % BATCH_SIZE != 0 or len(val_X) % BATCH_SIZE != 0: raise ValueError( 'Data size incompatible with batch size (must be evenly divisible)') num_inputs = len(train_X[0][0]) print('Number of inputs in loaded data: ' + str(num_inputs)) # Create graph structure. input_placeholder = layers.Input(shape=[TIMESTEPS, num_inputs]) # Encoder. encoded = layers.LSTM(64, return_sequences=True)(input_placeholder) encoded = advanced_activations.ELU(alpha=.5)(encoded) encoded = layers.LSTM(40)(encoded) encoded = advanced_activations.ELU(alpha=.5)(encoded) encoded = layers.Dense(3)(encoded) encoded = advanced_activations.ELU(alpha=.5)(encoded) encoded = layers.BatchNormalization(name='embedding')(encoded) # Decoder. decoded = layers.Dense(8)(encoded) decoded = advanced_activations.ELU(alpha=.5)(decoded) decoded = layers.Dense(16)(decoded) decoded = advanced_activations.ELU(alpha=.5)(decoded) decoded = layers.Dense(TIMESTEPS * num_inputs, activation='sigmoid')(decoded) decoded = layers.Reshape([TIMESTEPS, num_inputs])(decoded)
def conv_dec(nb_features, input_shape, nb_levels, conv_size, nb_labels, name=None, prefix=None, feat_mult=1, pool_size=2, use_skip_connections=False, padding='same', dilation_rate_mult=1, activation='elu', use_residuals=False, final_pred_activation='softmax', nb_conv_per_level=2, layer_nb_feats=None, batch_norm=None, conv_dropout=0, input_model=None): """ Fully Convolutional Decoder Parameters: ... use_skip_connections (bool): if true, turns an Enc-Dec to a U-Net. If true, input_tensor and tensors are required. It assumes a particular naming of layers. conv_enc... """ # naming model_name = name if prefix is None: prefix = model_name # if using skip connections, make sure need to use them. if use_skip_connections: assert input_model is not None, "is using skip connections, tensors dictionary is required" # first layer: input input_name = '%s_input' % prefix if input_model is None: input_tensor = KL.Input(shape=input_shape, name=input_name) last_tensor = input_tensor else: input_tensor = input_model.input last_tensor = input_model.output input_shape = last_tensor.shape.as_list()[1:] # vol size info ndims = len(input_shape) - 1 input_shape = tuple(input_shape) if isinstance(pool_size, int): pool_size = (pool_size, ) * ndims # prepare layers convL = getattr(KL, 'Conv%dD' % ndims) conv_kwargs = {'padding': padding, 'activation': activation} upsample = getattr(KL, 'UpSampling%dD' % ndims) # up arm: # nb_levels - 1 layers of Deconvolution3D # (approx via up + conv + ReLu) + merge + conv + ReLu + conv + ReLu lfidx = 0 for level in range(nb_levels - 1): nb_lvl_feats = np.round(nb_features * feat_mult**(nb_levels - 2 - level)).astype(int) conv_kwargs['dilation_rate'] = dilation_rate_mult**(nb_levels - 2 - level) # upsample matching the max pooling layers size name = '%s_up_%d' % (prefix, nb_levels + level) last_tensor = upsample(size=pool_size, name=name)(last_tensor) up_tensor = last_tensor # merge layers combining previous layer # TODO: add Cropping3D or Cropping2D if 'valid' padding if use_skip_connections: conv_name = '%s_conv_downarm_%d_%d' % ( prefix, nb_levels - 2 - level, nb_conv_per_level - 1) cat_tensor = input_model.get_layer(conv_name).output name = '%s_merge_%d' % (prefix, nb_levels + level) last_tensor = KL.concatenate([cat_tensor, last_tensor], axis=ndims + 1, name=name) # convolution layers for conv in range(nb_conv_per_level): if layer_nb_feats is not None: nb_lvl_feats = layer_nb_feats[lfidx] lfidx += 1 name = '%s_conv_uparm_%d_%d' % (prefix, nb_levels + level, conv) if conv < (nb_conv_per_level - 1) or (not use_residuals): last_tensor = convL(nb_lvl_feats, conv_size, **conv_kwargs, name=name)(last_tensor) else: last_tensor = convL(nb_lvl_feats, conv_size, padding=padding, name=name)(last_tensor) if conv_dropout > 0: name = '%s_dropout_uparm_%d_%d' % (prefix, level, conv) noise_shape = [None, *[1] * ndims, nb_lvl_feats] last_tensor = KL.Dropout(conv_dropout, noise_shape=noise_shape)(last_tensor) # residual block if use_residuals: # the "add" layer is the original input # However, it may not have the right number of features to be added add_layer = up_tensor nb_feats_in = add_layer.get_shape()[-1] nb_feats_out = last_tensor.get_shape()[-1] if nb_feats_in > 1 and nb_feats_out > 1 and (nb_feats_in != nb_feats_out): name = '%s_expand_up_merge_%d' % (prefix, level) add_layer = convL(nb_lvl_feats, conv_size, **conv_kwargs, name=name)(add_layer) if conv_dropout > 0: name = '%s_dropout_up_merge_%d_%d' % (prefix, level, conv) noise_shape = [None, *[1] * ndims, nb_lvl_feats] last_tensor = KL.Dropout( conv_dropout, noise_shape=noise_shape)(last_tensor) name = '%s_res_up_merge_%d' % (prefix, level) last_tensor = KL.add([last_tensor, add_layer], name=name) name = '%s_res_up_merge_act_%d' % (prefix, level) last_tensor = KL.Activation(activation, name=name)(last_tensor) if batch_norm is not None: name = '%s_bn_up_%d' % (prefix, level) last_tensor = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # Compute likelyhood prediction (no activation yet) name = '%s_likelihood' % prefix last_tensor = convL(nb_labels, 1, activation=None, name=name)(last_tensor) like_tensor = last_tensor # output prediction layer # we use a softmax to compute P(L_x|I) where x is each location if final_pred_activation == 'softmax': print("using final_pred_activation %s for %s" % (final_pred_activation, model_name)) name = '%s_prediction' % prefix softmax_lambda_fcn = lambda x: keras.activations.softmax( x, axis=ndims + 1) pred_tensor = KL.Lambda(softmax_lambda_fcn, name=name)(last_tensor) # otherwise create a layer that does nothing. else: name = '%s_prediction' % prefix pred_tensor = KL.Activation('linear', name=name)(like_tensor) # create the model and retun model = Model(inputs=input_tensor, outputs=pred_tensor, name=model_name) return model
def CapsNet_WithDecoder(input_shape, n_class, kernel, primary_channel, primary_veclen, digit_veclen, dropout, routings, decoderParm): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ if kernel > 19: conv_padding = 'same' print('kernel size big, padding to same') else: conv_padding = 'valid' x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=kernel, strides=1, padding=conv_padding, activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=primary_veclen, n_channels=primary_channel, kernel_size=kernel, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=digit_veclen, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction NumDecoderLayer, DecoderLayerUnit = decoderParm decoder = models.Sequential(name='decoder') decoder.add( layers.Dense(DecoderLayerUnit[0], activation='relu', input_dim=digit_veclen * n_class)) #16*n_class)) for i in range(NumDecoderLayer - 2): decoder.add(layers.Dense(DecoderLayerUnit[i + 1], activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, digit_veclen)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def VGG19_dense(input_shape=None, classes=4): img_input = layers.Input(shape=input_shape) # Block 1 x = layers.Conv2D(2, (3, 3), activation='relu', padding='same', name='block1_conv1', kernel_initializer="he_normal")(img_input) x = layers.Conv2D(2, (3, 3), activation='relu', padding='same', name='block1_conv2', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x) # Block 2 x = layers.Conv2D(4, (3, 3), activation='relu', padding='same', name='block2_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(4, (3, 3), activation='relu', padding='same', name='block2_conv2', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x) # Block 3 x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x) # Block 4 x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x) # Block 5 x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x) x = layers.Conv2D(32, (3, 3), activation='relu', name="feature")(x) x = layers.Conv2D(32, (1, 1), activation='relu')(x) x = layers.Conv2D(classes, (1, 1), activation='softmax')(x) x = layers.AveragePooling2D((9, 9))(x) model = models.Model(img_input, x, name='vgg19') model.summary() return model
def build_model(self): """Build a critic (value) network that maps (state, action) pairs -> Q-values.""" # Define input layers states = layers.Input(shape=(self.state_size,), name='states') actions = layers.Input(shape=(self.action_size,), name='actions') # Parameters for critic network ki = initializers.glorot_uniform(); kr = None if self.model_params["use_l2"] > 0: l2 = self.model_params["use_l2"] kr = regularizers.l2(l2) dropout_rate = self.model_params["dropout_rate"] use_bias = True use_bn = self.model_params["use_bn"] if use_bn: use_bias=False act_fn = self.model_params["act_fn"] n1 = self.model_params["layer1_n"] n2 = self.model_params["layer2_n"] # The critic network s_fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='s_fc1')(states) if use_bn: s_fc1 = layers.BatchNormalization()(s_fc1) if (dropout_rate > 0): s_fc1 = layers.Dropout(dropout_rate)(s_fc1) s_fc1 = layers.Activation(act_fn)(s_fc1) a_fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='a_fc1')(actions) if use_bn: a_fc1 = layers.BatchNormalization()(a_fc1) if (dropout_rate > 0): a_fc1 = layers.Dropout(dropout_rate)(a_fc1) a_fc1 = layers.Activation(act_fn)(a_fc1) # Combine state and action pathways net = layers.Add()([s_fc1, a_fc1]) net = layers.Dense(units=n2, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc2')(net) if use_bn: net = layers.BatchNormalization()(net) if (dropout_rate > 0): net = layers.Dropout(dropout_rate)(net) net = layers.Activation(act_fn)(net) # Add final output layer to prduce action values (Q values) ki = initializers.RandomUniform(minval=-3e-3, maxval=3e-3) Q_values = layers.Dense(units=1, kernel_initializer=ki, kernel_regularizer=kr, activation='linear', name='q_values')(net) # Create Keras model self.model = models.Model(inputs=[states, actions], outputs=Q_values) # Define optimizer and compile model for training with built-in loss function optimizer = optimizers.Adam(lr=0.001) self.model.compile(optimizer=optimizer, loss='mse') # Compute action gradients (derivative of Q values w.r.t. to actions) action_gradients = K.gradients(Q_values, actions) # Define an additional function to fetch action gradients (to be used by actor model) self.get_action_gradients = K.function( inputs=[*self.model.input, K.learning_phase()], outputs=action_gradients)
def adversarial_training(data_dir, generator_model_path, discriminator_model_path): """ Adversarial training of the generator network Gθ and discriminator network Dφ. """ # # define model input and output tensors # generator_input_tensor = layers.Input(shape=(rand_dim, )) generated_image_tensor = generator_network(generator_input_tensor) generated_or_real_image_tensor = layers.Input(shape=(img_height, img_width, img_channels)) discriminator_output = discriminator_network( generated_or_real_image_tensor) combined_output = discriminator_network( generator_network(generator_input_tensor)) # # define models # generator_model = models.Model(input=generator_input_tensor, output=generated_image_tensor, name='generator') discriminator_model = models.Model(input=generated_or_real_image_tensor, output=discriminator_output, name='discriminator') combined_model = models.Model(input=generator_input_tensor, output=combined_output, name='combined') # # compile models # adam = optimizers.Adam( lr=0.0002, beta_1=0.5, beta_2=0.999) # as described in appendix A of DeepMind's AC-GAN paper generator_model.compile(optimizer=adam, loss='binary_crossentropy') discriminator_model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy']) discriminator_model.trainable = False combined_model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy']) print(generator_model.summary()) print(discriminator_model.summary()) # # data generators # data_generator = image.ImageDataGenerator( preprocessing_function=applications.xception.preprocess_input, dim_ordering='tf') flow_from_directory_params = { 'target_size': (img_height, img_width), 'color_mode': 'grayscale' if img_channels == 1 else 'rgb', 'class_mode': None, 'batch_size': batch_size } real_image_generator = data_generator.flow_from_directory( directory=data_dir, **flow_from_directory_params) def get_image_batch(): img_batch = real_image_generator.next() # keras generators may generate an incomplete batch for the last batch if len(img_batch) != batch_size: img_batch = real_image_generator.next() assert img_batch.shape == (batch_size, img_height, img_width, img_channels), img_batch.shape return img_batch # the target labels for the binary cross-entropy loss layer are 0 for every yj (real) and 1 for every xi (generated) y_real = np.array([0] * batch_size) y_generated = np.array([1] * batch_size) combined_loss = np.zeros(shape=len(combined_model.metrics_names)) disc_loss_real = np.zeros(shape=len(discriminator_model.metrics_names)) disc_loss_generated = np.zeros( shape=len(discriminator_model.metrics_names)) if generator_model_path: generator_model.load_weights(generator_model_path, by_name=True) if discriminator_model_path: discriminator_model.load_weights(discriminator_model_path, by_name=True) for i in range(nb_steps): print('Step: {} of {}.'.format(i, nb_steps)) # train the discriminator for _ in range(k_d): generator_input = np.random.normal(size=(batch_size, rand_dim)) # sample a mini-batch of real images real_image_batch = get_image_batch() # generate a batch of images with the current generator generated_image_batch = generator_model.predict(generator_input) # update φ by taking an SGD step on mini-batch loss LD(φ) disc_loss_real = np.add( discriminator_model.train_on_batch(real_image_batch, y_real), disc_loss_real) disc_loss_generated = np.add( discriminator_model.train_on_batch(generated_image_batch, y_generated), disc_loss_generated) # train the generator for _ in range(k_g * 2): generator_input = np.random.normal(size=(batch_size, rand_dim)) # update θ by taking an SGD step on mini-batch loss LR(θ) combined_loss = np.add( combined_model.train_on_batch(generator_input, y_real), combined_loss) if not i % log_interval and i != 0: # plot batch of generated images w/ current generator figure_name = 'generated_image_batch_step_{}.png'.format(i) print('Saving batch of generated images at adversarial step: {}.'. format(i)) plot_images.plot_batch( generator_model.predict( np.random.normal(size=(batch_size, rand_dim))), get_image_batch(), os.path.join(cache_dir, figure_name)) # log loss summary print('Generator model loss: {}.'.format(combined_loss / (log_interval * k_g * 2))) print('Discriminator model loss real: {}.'.format( disc_loss_real / (log_interval * k_d * 2))) print('Discriminator model loss generated: {}.'.format( disc_loss_generated / (log_interval * k_d * 2))) combined_loss = np.zeros(shape=len(combined_model.metrics_names)) disc_loss_real = np.zeros( shape=len(discriminator_model.metrics_names)) disc_loss_generated = np.zeros( shape=len(discriminator_model.metrics_names)) # save model checkpoints model_checkpoint_base_name = os.path.join( cache_dir, '{}_model_weights_step_{}.h5') generator_model.save_weights( model_checkpoint_base_name.format('generator', i)) discriminator_model.save_weights( model_checkpoint_base_name.format('discriminator', i))
def build_model(self): """Build an actor (policy) network that maps states -> actions.""" # Define input layer (states) states = layers.Input(shape=(self.state_size,), name='states') # Parameters for actor network ki = initializers.glorot_uniform(); kr = None if self.model_params["use_l2"] > 0: l2 = self.model_params["use_l2"] kr = regularizers.l2(l2) dropout_rate = self.model_params["dropout_rate"] use_bias = True use_bn = self.model_params["use_bn"] if use_bn: use_bias=False act_fn = self.model_params["act_fn"] n1 = self.model_params["layer1_n"] n2 = self.model_params["layer2_n"] # the actor network fc1 = layers.Dense(units=n1, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc1')(states) if use_bn: fc1 = layers.BatchNormalization()(fc1) if (dropout_rate > 0): fc1 = layers.Dropout(dropout_rate)(fc1) fc1 = layers.Activation(act_fn)(fc1) fc2 = layers.Dense(units=n2, kernel_initializer=ki, kernel_regularizer=kr, use_bias=use_bias, name='fc2')(fc1) if use_bn: fc2 = layers.BatchNormalization()(fc2) if (dropout_rate > 0): fc2 = layers.Dropout(dropout_rate)(fc2) fc2 = layers.Activation(act_fn)(fc2) # Add final output layer ki = initializers.RandomUniform(minval=-3e-3, maxval=3e-3) raw_actions = layers.Dense(units=self.action_size, activation='tanh', kernel_initializer=ki, kernel_regularizer=kr, use_bias=False, name='raw_actions')(fc2) # Scale [-1, 1] output for each action dimension to proper range actions = layers.Lambda(lambda x: (0.5*(x + 1.0)*self.action_range + self.action_low), name='actions')(raw_actions) # Create Keras model self.model = models.Model(inputs=states, outputs=actions) # Define loss function using action value (Q value) gradients action_gradients = layers.Input(shape=(self.action_size,)) loss = K.mean(-action_gradients * actions) if (kr is not None): loss = loss + l2*K.sum(K.square(self.model.get_layer('fc1').get_weights()[0])) \ + l2*K.sum(K.square(self.model.get_layer('fc2').get_weights()[0])) \ + l2*K.sum(K.square(self.model.get_layer('raw_actions').get_weights()[0])) # Define optimizer and training function optimizer = optimizers.Adam(lr=0.0001) updates_op = optimizer.get_updates(params=self.model.trainable_weights, loss=loss) self.train_fn = K.function( inputs=[self.model.input, action_gradients, K.learning_phase()], outputs=[], updates=updates_op)
def densenet(channels, init_block_channels, dropout_rate=0.0, in_channels=3, in_size=(224, 224), classes=1000): """ DenseNet model from 'Densely Connected Convolutional Networks,' https://arxiv.org/abs/1608.06993. Parameters: ---------- channels : list of list of int Number of output channels for each unit. init_block_channels : int Number of output channels for the initial unit. dropout_rate : float, default 0.0 Parameter of Dropout layer. Faction of the input units to drop. in_channels : int, default 3 Number of input channels. in_size : tuple of two ints, default (224, 224) Spatial size of the expected input image. classes : int, default 1000 Number of classification classes. """ input_shape = (in_channels, 224, 224) if is_channels_first() else (224, 224, in_channels) input = nn.Input(shape=input_shape) x = preres_init_block( x=input, in_channels=in_channels, out_channels=init_block_channels, name="features/init_block") in_channels = init_block_channels for i, channels_per_stage in enumerate(channels): if i != 0: x = transition_block( x=x, in_channels=in_channels, out_channels=(in_channels // 2), name="features/stage{}/trans{}".format(i + 1, i + 1)) in_channels = in_channels // 2 for j, out_channels in enumerate(channels_per_stage): x = dense_unit( x=x, in_channels=in_channels, out_channels=out_channels, dropout_rate=dropout_rate, name="features/stage{}/unit{}".format(i + 1, j + 1)) in_channels = out_channels x = preres_activation( x=x, name="features/post_activ") x = nn.AvgPool2D( pool_size=7, strides=1, name="features/final_pool")(x) # x = nn.Flatten()(x) x = flatten(x) x = nn.Dense( units=classes, input_dim=in_channels, name="output")(x) model = Model(inputs=input, outputs=x) model.in_size = in_size model.classes = classes return model
def VGG19(input_shape=None, classes=[3, 5], use_soft=True): img_input = layers.Input(shape=input_shape) # Block 1 x = layers.Conv2D(2, (3, 3), activation='relu', padding='same', name='block1_conv1', kernel_initializer="he_normal")(img_input) x = layers.Conv2D(2, (3, 3), activation='relu', padding='same', name='block1_conv2', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x) # Block 2 x = layers.Conv2D(4, (3, 3), activation='relu', padding='same', name='block2_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(4, (3, 3), activation='relu', padding='same', name='block2_conv2', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x) # Block 3 x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(8, (3, 3), activation='relu', padding='same', name='block3_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x) # Block 4 x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block4_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x) # Block 5 x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv1', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv2', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv3', kernel_initializer="he_normal")(x) x = layers.Conv2D(16, (3, 3), activation='relu', padding='same', name='block5_conv4', kernel_initializer="he_normal")(x) x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x) # Classification block x = layers.Flatten(name='flatten')(x) x1 = layers.Dense(512, activation='relu', name='fc1')(x) x1 = layers.Dropout(0.5, name="dropout_1")(x1) x1 = layers.Dense(128, activation='relu', name='fc2')(x1) #x=layers.Dropout(0.8)(x) x2 = layers.Dense(512, activation='relu', name='fc1_branch')(x) x2 = layers.Dropout(0.5, name="dropout_1_branch")(x2) x2 = layers.Dense(128, activation='relu', name='fc2_branch')(x2) if use_soft: x1 = Dense(classes[0], activation="softmax", name='predictions')(x1) x2 = Dense(classes[1], activation="softmax", name='predictions_branch')(x2) else: x1 = Dense(classes[0], activation="linear", name="Z_4")(x) model = models.Model(img_input, [x1, x2], name='vgg19') return model
def single_ae( enc_size, input_shape, name='single_ae', prefix=None, ae_type='dense', # 'dense', or 'conv' conv_size=None, input_model=None, enc_lambda_layers=None, batch_norm=True, padding='same', activation=None, include_mu_shift_layer=False, do_vae=False): """ single-layer Autoencoder (i.e. input - encoding - output) """ # naming model_name = name if prefix is None: prefix = model_name if enc_lambda_layers is None: enc_lambda_layers = [] # prepare input input_name = '%s_input' % prefix if input_model is None: assert input_shape is not None, 'input_shape of input_model is necessary' input_tensor = KL.Input(shape=input_shape, name=input_name) last_tensor = input_tensor else: input_tensor = input_model.input last_tensor = input_model.output input_shape = last_tensor.shape.as_list()[1:] input_nb_feats = last_tensor.shape.as_list()[-1] # prepare conv type based on input if ae_type == 'conv': ndims = len(input_shape) - 1 convL = getattr(KL, 'Conv%dD' % ndims) assert conv_size is not None, 'with conv ae, need conv_size' conv_kwargs = {'padding': padding, 'activation': activation} # if want to go through a dense layer in the middle of the U, need to: # - flatten last layer if not flat # - do dense encoding and decoding # - unflatten (rehsape spatially) at end if ae_type == 'dense' and len(input_shape) > 1: name = '%s_ae_%s_down_flat' % (prefix, ae_type) last_tensor = KL.Flatten(name=name)(last_tensor) # recall this layer pre_enc_layer = last_tensor # encoding layer if ae_type == 'dense': assert len( enc_size) == 1, "enc_size should be of length 1 for dense layer" enc_size_str = ''.join(['%d_' % d for d in enc_size])[:-1] name = '%s_ae_mu_enc_dense_%s' % (prefix, enc_size_str) last_tensor = KL.Dense(enc_size[0], name=name)(pre_enc_layer) else: # convolution # convolve then resize. enc_size should be [nb_dim1, nb_dim2, ..., nb_feats] assert len(enc_size) == len(input_shape), \ "encoding size does not match input shape %d %d" % (len(enc_size), len(input_shape)) if list(enc_size)[:-1] != list(input_shape)[:-1] and \ all([f is not None for f in input_shape[:-1]]) and \ all([f is not None for f in enc_size[:-1]]): # assert len(enc_size) - 1 == 2, "Sorry, I have not yet implemented non-2D resizing -- need to check out interpn!" name = '%s_ae_mu_enc_conv' % (prefix) last_tensor = convL(enc_size[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) name = '%s_ae_mu_enc' % (prefix) zf = [ enc_size[:-1][f] / last_tensor.shape.as_list()[1:-1][f] for f in range(len(enc_size) - 1) ] last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor) # resize_fn = lambda x: tf.image.resize_bilinear(x, enc_size[:-1]) # last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor) elif enc_size[ -1] is None: # convolutional, but won't tell us bottleneck name = '%s_ae_mu_enc' % (prefix) last_tensor = KL.Lambda(lambda x: x, name=name)(pre_enc_layer) else: name = '%s_ae_mu_enc' % (prefix) last_tensor = convL(enc_size[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) if include_mu_shift_layer: # shift name = '%s_ae_mu_shift' % (prefix) last_tensor = layers.LocalBias(name=name)(last_tensor) # encoding clean-up layers for layer_fcn in enc_lambda_layers: lambda_name = layer_fcn.__name__ name = '%s_ae_mu_%s' % (prefix, lambda_name) last_tensor = KL.Lambda(layer_fcn, name=name)(last_tensor) if batch_norm is not None: name = '%s_ae_mu_bn' % (prefix) last_tensor = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # have a simple layer that does nothing to have a clear name before sampling name = '%s_ae_mu' % (prefix) last_tensor = KL.Lambda(lambda x: x, name=name)(last_tensor) # if doing variational AE, will need the sigma layer as well. if do_vae: mu_tensor = last_tensor # encoding layer if ae_type == 'dense': name = '%s_ae_sigma_enc_dense_%s' % (prefix, enc_size_str) last_tensor = KL.Dense( enc_size[0], name=name, # kernel_initializer=keras.initializers.RandomNormal(mean=0.0, stddev=1e-5), # bias_initializer=keras.initializers.RandomNormal(mean=-5.0, stddev=1e-5) )(pre_enc_layer) else: if list(enc_size)[:-1] != list(input_shape)[:-1] and \ all([f is not None for f in input_shape[:-1]]) and \ all([f is not None for f in enc_size[:-1]]): # assert len(enc_size) - 1 == 2, "Sorry, I have not yet implemented non-2D resizing..." name = '%s_ae_sigma_enc_conv' % (prefix) last_tensor = convL(enc_size[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) name = '%s_ae_sigma_enc' % (prefix) zf = [ enc_size[:-1][f] / last_tensor.shape.as_list()[1:-1][f] for f in range(len(enc_size) - 1) ] last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor) # resize_fn = lambda x: tf.image.resize_bilinear(x, enc_size[:-1]) # last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor) elif enc_size[ -1] is None: # convolutional, but won't tell us bottleneck name = '%s_ae_sigma_enc' % (prefix) last_tensor = convL(pre_enc_layer.shape.as_list()[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) # cannot use lambda, then mu and sigma will be same layer. # last_tensor = KL.Lambda(lambda x: x, name=name)(pre_enc_layer) else: name = '%s_ae_sigma_enc' % (prefix) last_tensor = convL(enc_size[-1], conv_size, name=name, **conv_kwargs)(pre_enc_layer) # encoding clean-up layers for layer_fcn in enc_lambda_layers: lambda_name = layer_fcn.__name__ name = '%s_ae_sigma_%s' % (prefix, lambda_name) last_tensor = KL.Lambda(layer_fcn, name=name)(last_tensor) if batch_norm is not None: name = '%s_ae_sigma_bn' % (prefix) last_tensor = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # have a simple layer that does nothing to have a clear name before sampling name = '%s_ae_sigma' % (prefix) last_tensor = KL.Lambda(lambda x: x, name=name)(last_tensor) logvar_tensor = last_tensor # VAE sampling name = '%s_ae_sample' % (prefix) last_tensor = layers.SampleNormalLogVar(name=name)( [mu_tensor, logvar_tensor]) if include_mu_shift_layer: # shift name = '%s_ae_sample_shift' % (prefix) last_tensor = layers.LocalBias(name=name)(last_tensor) # decoding layer if ae_type == 'dense': name = '%s_ae_%s_dec_flat_%s' % (prefix, ae_type, enc_size_str) last_tensor = KL.Dense(np.prod(input_shape), name=name)(last_tensor) # unflatten if dense method if len(input_shape) > 1: name = '%s_ae_%s_dec' % (prefix, ae_type) last_tensor = KL.Reshape(input_shape, name=name)(last_tensor) else: if list(enc_size)[:-1] != list(input_shape)[:-1] and \ all([f is not None for f in input_shape[:-1]]) and \ all([f is not None for f in enc_size[:-1]]): name = '%s_ae_mu_dec' % (prefix) zf = [ input_shape[:-1][f] / enc_size[:-1][f] for f in range(len(enc_size) - 1) ] last_tensor = layers.Resize(zoom_factor=zf, name=name)(last_tensor) # resize_fn = lambda x: tf.image.resize_bilinear(x, input_shape[:-1]) # last_tensor = KL.Lambda(resize_fn, name=name)(last_tensor) name = '%s_ae_%s_dec' % (prefix, ae_type) last_tensor = convL(input_nb_feats, conv_size, name=name, **conv_kwargs)(last_tensor) if batch_norm is not None: name = '%s_bn_ae_%s_dec' % (prefix, ae_type) last_tensor = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor) # create the model and retun model = Model(inputs=input_tensor, outputs=[last_tensor], name=model_name) return model
def build_model(): from keras import models, layers # Build U-Net model def upsample_conv(filters, kernel_size, strides, padding): return layers.Conv2DTranspose(filters, kernel_size, strides=strides, padding=padding) def upsample_simple(filters, kernel_size, strides, padding): return layers.UpSampling2D(strides) if UPSAMPLE_MODE=='DECONV': upsample=upsample_conv else: upsample=upsample_simple input_img = layers.Input(t_x.shape[1:], name = 'RGB_Input') pp_in_layer = input_img if NET_SCALING is not None: pp_in_layer = layers.AvgPool2D(NET_SCALING)(pp_in_layer) pp_in_layer = layers.GaussianNoise(GAUSSIAN_NOISE)(pp_in_layer) pp_in_layer = layers.BatchNormalization()(pp_in_layer) c1 = layers.Conv2D(8, (3, 3), activation='relu', padding='same') (pp_in_layer) c1 = layers.Conv2D(8, (3, 3), activation='relu', padding='same') (c1) p1 = layers.MaxPooling2D((2, 2)) (c1) c2 = layers.Conv2D(16, (3, 3), activation='relu', padding='same') (p1) c2 = layers.Conv2D(16, (3, 3), activation='relu', padding='same') (c2) p2 = layers.MaxPooling2D((2, 2)) (c2) c3 = layers.Conv2D(32, (3, 3), activation='relu', padding='same') (p2) c3 = layers.Conv2D(32, (3, 3), activation='relu', padding='same') (c3) p3 = layers.MaxPooling2D((2, 2)) (c3) c4 = layers.Conv2D(64, (3, 3), activation='relu', padding='same') (p3) c4 = layers.Conv2D(64, (3, 3), activation='relu', padding='same') (c4) p4 = layers.MaxPooling2D(pool_size=(2, 2)) (c4) c5 = layers.Conv2D(128, (3, 3), activation='relu', padding='same') (p4) c5 = layers.Conv2D(128, (3, 3), activation='relu', padding='same') (c5) u6 = upsample(64, (2, 2), strides=(2, 2), padding='same') (c5) u6 = layers.concatenate([u6, c4]) c6 = layers.Conv2D(64, (3, 3), activation='relu', padding='same') (u6) c6 = layers.Conv2D(64, (3, 3), activation='relu', padding='same') (c6) u7 = upsample(32, (2, 2), strides=(2, 2), padding='same') (c6) u7 = layers.concatenate([u7, c3]) c7 = layers.Conv2D(32, (3, 3), activation='relu', padding='same') (u7) c7 = layers.Conv2D(32, (3, 3), activation='relu', padding='same') (c7) u8 = upsample(16, (2, 2), strides=(2, 2), padding='same') (c7) u8 = layers.concatenate([u8, c2]) c8 = layers.Conv2D(16, (3, 3), activation='relu', padding='same') (u8) c8 = layers.Conv2D(16, (3, 3), activation='relu', padding='same') (c8) u9 = upsample(8, (2, 2), strides=(2, 2), padding='same') (c8) u9 = layers.concatenate([u9, c1], axis=3) c9 = layers.Conv2D(8, (3, 3), activation='relu', padding='same') (u9) c9 = layers.Conv2D(8, (3, 3), activation='relu', padding='same') (c9) d = layers.Conv2D(1, (1, 1), activation='sigmoid') (c9) # d = layers.Cropping2D((EDGE_CROP, EDGE_CROP))(d) # d = layers.ZeroPadding2D((EDGE_CROP, EDGE_CROP))(d) if NET_SCALING is not None: d = layers.UpSampling2D(NET_SCALING)(d) seg_model = models.Model(inputs=[input_img], outputs=[d]) seg_model.summary()