def new_model(image_size=299, video_length=40, cnn_trainable=False): inputs = Input(shape=(video_length, image_size, image_size, 3)) cnn = inception_v3.InceptionV3(include_top=False, weights='imagenet') model = TimeDistributed(cnn)(inputs) model.trainable = cnn_trainable model = LSTM(512)(model) model = Dropout(0.5)(model) model = Dense(1, activation='softmax')(model) model = Model(inputs=inputs, outputs=model) adam = keras.optimizers.Adam(learning_rate=1e-5) model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy']) model.summary() return model
def create_posenet(num_beacon, weights_path=None, trainable=True): beacon_input = Input(shape=(None, num_beacon, 1, 1)) beacon_icp1_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp1_out1'))(beacon_input) beacon_icp1_out1.trainable = False ''' beacon_cls1_fc1_flat = TimeDistributed(Flatten())(beacon_icp1_out1) beacon_cls1_fc1_pose = TimeDistributed(Dense(1024,activation='relu',name='beacon_cls1_fc1_pose'))(beacon_cls1_fc1_flat) beacon_cls1_fc_pose_xyz = TimeDistributed(Dense(3,name='beacon_cls1_fc_pose_xyz'))(beacon_cls1_fc1_pose) beacon_cls1_fc_pose_wpqr = TimeDistributed(Dense(4,name='beacon_cls1_fc_pose_wpqr'))(beacon_cls1_fc1_pose) ''' beacon_icp4_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp4_out1'))(beacon_icp1_out1) beacon_icp4_out1.trainable = False ''' beacon_cls2_fc1_flat = TimeDistributed(Flatten())(beacon_icp4_out1) beacon_cls2_fc1 = TimeDistributed(Dense(1024,activation='relu',name='beacon_cls2_fc1'))(beacon_cls2_fc1_flat) beacon_cls2_fc_pose_xyz = TimeDistributed(Dense(3,name='beacon_cls2_fc_pose_xyz'))(beacon_cls2_fc1) beacon_cls2_fc_pose_wpqr = TimeDistributed(Dense(4,name='beacon_cls2_fc_pose_wpqr'))(beacon_cls2_fc1) ''' beacon_icp7_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp7_out1'))(beacon_icp4_out1) beacon_icp7_out1.trainable = False beacon_cls3_fc1_flat = TimeDistributed(Flatten())(beacon_icp7_out1) beacon_cls3_fc1_flat.trainable = False beacon_cls3_fc1_pose = TimeDistributed( Dense(2048, activation='relu', name='beacon_cls3_fc1_pose'))(beacon_cls3_fc1_flat) beacon_cls3_fc1_pose.trainable = False beacon_lstm = LSTM(256, return_sequences=True, name='beacon_lstm')(beacon_cls3_fc1_pose) beacon_lstm_dense_xyz = TimeDistributed( Dense(128, activation='relu'), name='beacon_lstm_dense_xyz')(beacon_lstm) beacon_lstm_pose_xyz = TimeDistributed( Dense(3), name='beacon_lstm_pose_xyz')(beacon_lstm_dense_xyz) beacon_lstm_dense_wpqr = TimeDistributed( Dense(128, activation='relu'), name='beacon_lstm_dense_wpqr')(beacon_lstm) beacon_lstm_pose_wpqr = TimeDistributed( Dense(4), name='beacon_lstm_pose_wpqr')(beacon_lstm_dense_wpqr) beacon_posenet = Model( inputs=beacon_input, outputs=[beacon_lstm_pose_xyz, beacon_lstm_pose_wpqr]) if weights_path: print("start load image network weights") beacon_posenet.load_weights(weights_path, by_name=True) print("finish load image network weights") if not trainable: for layer in beacon_posenet.layers: layer.trainable = False return beacon_posenet
def build_model(p): """ build a Keras model using the parameters in p """ max_posts = p['max_posts'] max_length = p['max_length'] filters = p['filters'] filtlen = p['filtlen'] poollen = p['poollen'] densed = p['densed'] embed_size = p['embed_size'] batch = p['batch'] random.seed(p['seed']) np.random.seed(p['seed']) # https://github.com/fchollet/keras/issues/2280 tf.reset_default_graph() if len(tf.get_default_graph()._nodes_by_id.keys()) > 0: raise RuntimeError( "Seeding is not supported after building part of the graph. " "Please move set_seed to the beginning of your code.") tf.set_random_seed(p['seed']) sess = tf.Session() K.set_session(sess) nb_words, genf, tok = datagen(max_posts, max_length, stype='training', batch_size=batch, randposts=p['randposts'], mintf=p['mintf'], mindf=p['mindf'], noempty=p['noempty'], prep=p['prep'], returntok=True) n_classes = 2 inp = Input(shape=(max_posts, max_length), dtype='int32') nextin = inp if p['cosine']: from keras.constraints import unitnorm wconstrain = unitnorm() else: wconstrain = None if p['w2v']: embeddings_index = {} fn = 'data/w2v_50_sg_export.txt' with open(fn) as f: for line in f: values = line.strip().split() word = values[0] if word in tok.word_index: coefs = np.asarray(values[1:], dtype='float32') embeddings_index[word] = coefs print('Found %s word vectors.' % len(embeddings_index)) embedding_matrix = np.zeros((nb_words, embed_size)) for word, i in tok.word_index.items(): embedding_vector = embeddings_index.get(word) if embedding_vector is not None: embedding_matrix[i] = embedding_vector else: embedding_matrix[i] = np.random.uniform(-0.2, 0.2, size=embed_size) emb = Embedding(nb_words, embed_size, mask_zero=True, input_length=max_length, W_constraint=wconstrain, weights=[embedding_matrix], trainable=p['etrain']) if not p['etrain']: print("making not trainable") emb.trainable = False else: assert p['etrain'], "must have etrain=True with w2v=False" emb = Embedding(nb_words, embed_size, mask_zero=True, W_constraint=wconstrain) embedded = TimeDistributed(emb)(nextin) if not p['etrain']: emb.trainable = False embedded.trainable = False conv = Sequential() conv.add( Convolution1D(nb_filter=filters, filter_length=filtlen, border_mode='valid', W_constraint=wconstrain, activation='linear', subsample_length=1, input_shape=(max_length, embed_size))) conv.add(Activation(p['af'])) conv.add(GlobalAveragePooling1D()) posts = TimeDistributed(conv)(embedded) combined = Convolution1D(nb_filter=filters, filter_length=p['acl'], border_mode='valid', activation=p['af'], subsample_length=p['acl'])(posts) combined = Flatten()(combined) if densed != 0: combined = Dense(densed, activation=p['af'])(combined) outlayer = Dense(2, activation='softmax')(combined) model = Model(inp, outlayer) return model, genf
def create_posenet_inception_v3(num_beacon, image_beacon_weights_path=None, trainable=True): # create the base pre-trained model image_input = Input(shape=(None, 299, 299, 3)) base_model = InceptionV3(weights='imagenet', include_top=False, pooling='avg') base_model.trainable = False # add top layer model_output = TimeDistributed(base_model)(image_input) # beacon subnet 1 beacon_input = Input(shape=(None, num_beacon, 1, 1)) beacon_icp1_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp1_out1'))(beacon_input) beacon_icp1_out1.trainable = False # beacon subnet 2 beacon_icp4_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp4_out1'))(beacon_icp1_out1) beacon_icp4_out1.trainable = False # beacon subnet 3 beacon_icp7_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp7_out1'))(beacon_icp4_out1) beacon_icp7_out1.trainable = False beacon_cls3_fc1_flat = TimeDistributed(Flatten())(beacon_icp7_out1) beacon_cls3_fc1_flat.trainable = False beacon_cls3_fc1_pose = TimeDistributed( Dense(2048, activation='relu', name='beacon_cls3_fc1_pose'))(beacon_cls3_fc1_flat) beacon_cls3_fc1_pose.trainable = False # image, beacon classify 3 image_beacon_cls3_fc1_pose = concatenate( [model_output, beacon_cls3_fc1_pose], name='image_beacon_cls3_fc1_pose') image_beacon_lstm = LSTM( 256, return_sequences=True, name='image_beacon_lstm')(image_beacon_cls3_fc1_pose) image_beacon_lstm_dense_xyz = TimeDistributed( Dense(128, activation='relu'), name='image_beacon_lstm_dense_xyz')(image_beacon_lstm) image_beacon_lstm_pose_xyz = TimeDistributed( Dense(3), name='image_beacon_lstm_pose_xyz')(image_beacon_lstm_dense_xyz) image_beacon_lstm_dense_wpqr = TimeDistributed( Dense(128, activation='relu'), name='image_beacon_lstm_dense_wpqr')(image_beacon_lstm) image_beacon_lstm_pose_wpqr = TimeDistributed( Dense(4), name='image_beacon_lstm_pose_wpqr')(image_beacon_lstm_dense_wpqr) image_beacon_posenet = Model( inputs=[image_input, beacon_input], outputs=[image_beacon_lstm_pose_xyz, image_beacon_lstm_pose_wpqr]) if image_beacon_weights_path: print("start load image beacon network weights") image_beacon_weights_path_ext = os.path.splitext( image_beacon_weights_path)[-1] if image_beacon_weights_path_ext == ".npy": weights_data = np.load(image_beacon_weights_path).item() for layer in image_beacon_posenet.layers: if layer.name in weights_data.keys(): layer_weights = weights_data[layer.name] layer.set_weights( (layer_weights['weights'], layer_weights['biases'])) print("finish load imaege beacon network weights") elif image_beacon_weights_path_ext == ".h5": image_beacon_posenet.load_weights(image_beacon_weights_path, by_name=True) print("finish load image beacon network weights") else: print("invalid weight file : " + image_weights_path) sys.exit() if not trainable: for layer in image_beacon_posenet.layers: layer.trainable = False return image_beacon_posenet
def create_posenet_mobilenet_v1(num_beacon, image_beacon_weights_path=None, trainable=True): # create the base pre-trained model if K.image_data_format() == 'channels_first': image_input = Input(shape=(None, 3, 224, 224), name='input_1') else: image_input = Input(shape=(None, 224, 224, 3), name='input_1') alpha = 1.0 dropout = 1e-3 if K.image_data_format() == 'channels_first': input_shape = (3, 224, 224) else: input_shape = (224, 224, 3) base_model = MobileNet(input_shape=input_shape, alpha=alpha, dropout=dropout, weights='imagenet', include_top=False) base_model.trainable = False # add top layer model_output = TimeDistributed(base_model)(image_input) if K.image_data_format() == 'channels_first': shape = (int(1024 * alpha), 1, 1) else: shape = (1, 1, int(1024 * alpha)) model_output = TimeDistributed(GlobalAveragePooling2D())(model_output) model_output = TimeDistributed(Reshape(shape, name='reshape_1'))(model_output) model_output = TimeDistributed(Dropout(dropout, name='dropout'))(model_output) image_conv_pose_xyz = TimeDistributed( Conv2D(1024, (1, 1), padding='same', name='conv_pose_xyz'))(model_output) image_conv_pose_xyz_flat = TimeDistributed(Flatten())(image_conv_pose_xyz) image_conv_pose_wpqr = TimeDistributed( Conv2D(1024, (1, 1), padding='same', name='conv_pose_wpqr'))(model_output) image_conv_pose_wpqr_flat = TimeDistributed( Flatten())(image_conv_pose_wpqr) # beacon subnet 1 beacon_input = Input(shape=(None, num_beacon, 1, 1), name='input_2') beacon_icp1_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp1_out1'))(beacon_input) beacon_icp1_out1.trainable = False # beacon subnet 2 beacon_icp4_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp4_out1'))(beacon_icp1_out1) beacon_icp4_out1.trainable = False # beacon subnet 3 beacon_icp7_out1 = TimeDistributed( Conv2D(16, (1, 1), padding='same', activation='relu', name='beacon_icp7_out1'))(beacon_icp4_out1) beacon_icp7_out1.trainable = False beacon_cls3_fc1_flat = TimeDistributed(Flatten())(beacon_icp7_out1) beacon_cls3_fc1_flat.trainable = False beacon_cls3_fc1_pose = TimeDistributed( Dense(2048, activation='relu', name='beacon_cls3_fc1_pose'))(beacon_cls3_fc1_flat) beacon_cls3_fc1_pose.trainable = False # image, beacon classify 3 image_beacon_cls3_fc1_pose_xyz = concatenate( [image_conv_pose_xyz_flat, beacon_cls3_fc1_pose], name='image_beacon_cls3_fc1_pose_xyz') image_beacon_lstm_xyz = LSTM( 256, return_sequences=True, name='image_beacon_lstm_xyz')(image_beacon_cls3_fc1_pose_xyz) image_beacon_lstm_dense_xyz = TimeDistributed( Dense(128, activation='relu'), name='image_beacon_lstm_dense_xyz')(image_beacon_lstm_xyz) image_beacon_lstm_pose_xyz = TimeDistributed( Dense(3), name='image_beacon_lstm_pose_xyz')(image_beacon_lstm_dense_xyz) image_beacon_cls3_fc1_pose_wpqr = concatenate( [image_conv_pose_wpqr_flat, beacon_cls3_fc1_pose], name='image_beacon_cls3_fc1_pose_wpqr') image_beacon_lstm_wpqr = LSTM( 256, return_sequences=True, name='image_beacon_lstm_wpqr')(image_beacon_cls3_fc1_pose_wpqr) image_beacon_lstm_dense_wpqr = TimeDistributed( Dense(128, activation='relu'), name='image_beacon_lstm_dense_wpqr')(image_beacon_lstm_wpqr) image_beacon_lstm_pose_wpqr = TimeDistributed( Dense(4), name='image_beacon_lstm_pose_wpqr')(image_beacon_lstm_dense_wpqr) image_beacon_posenet = Model( inputs=[image_input, beacon_input], outputs=[image_beacon_lstm_pose_xyz, image_beacon_lstm_pose_wpqr]) if image_beacon_weights_path: print("start load image beacon network weights") image_beacon_weights_path_ext = os.path.splitext( image_beacon_weights_path)[-1] if image_beacon_weights_path_ext == ".npy": weights_data = np.load(image_beacon_weights_path).item() for layer in image_beacon_posenet.layers: if layer.name in weights_data.keys(): layer_weights = weights_data[layer.name] layer.set_weights( (layer_weights['weights'], layer_weights['biases'])) print("finish load imaege beacon network weights") elif image_beacon_weights_path_ext == ".h5": image_beacon_posenet.load_weights(image_beacon_weights_path, by_name=True) print("finish load image beacon network weights") else: print("invalid weight file : " + image_weights_path) sys.exit() if not trainable: for layer in image_beacon_posenet.layers: layer.trainable = False return image_beacon_posenet