def new_model(image_size=299, video_length=40, cnn_trainable=False):
inputs = Input(shape=(video_length, image_size, image_size, 3))
cnn = inception_v3.InceptionV3(include_top=False, weights='imagenet')
model = TimeDistributed(cnn)(inputs)
model.trainable = cnn_trainable
model = LSTM(512)(model)
model = Dropout(0.5)(model)
model = Dense(1, activation='softmax')(model)
model = Model(inputs=inputs, outputs=model)
adam = keras.optimizers.Adam(learning_rate=1e-5)
model.compile(loss='binary_crossentropy',
optimizer=adam,
metrics=['accuracy'])
model.summary()
return model
def create_posenet(num_beacon, weights_path=None, trainable=True):
beacon_input = Input(shape=(None, num_beacon, 1, 1))
beacon_icp1_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp1_out1'))(beacon_input)
beacon_icp1_out1.trainable = False
'''
beacon_cls1_fc1_flat = TimeDistributed(Flatten())(beacon_icp1_out1)
beacon_cls1_fc1_pose = TimeDistributed(Dense(1024,activation='relu',name='beacon_cls1_fc1_pose'))(beacon_cls1_fc1_flat)
beacon_cls1_fc_pose_xyz = TimeDistributed(Dense(3,name='beacon_cls1_fc_pose_xyz'))(beacon_cls1_fc1_pose)
beacon_cls1_fc_pose_wpqr = TimeDistributed(Dense(4,name='beacon_cls1_fc_pose_wpqr'))(beacon_cls1_fc1_pose)
'''
beacon_icp4_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp4_out1'))(beacon_icp1_out1)
beacon_icp4_out1.trainable = False
'''
beacon_cls2_fc1_flat = TimeDistributed(Flatten())(beacon_icp4_out1)
beacon_cls2_fc1 = TimeDistributed(Dense(1024,activation='relu',name='beacon_cls2_fc1'))(beacon_cls2_fc1_flat)
beacon_cls2_fc_pose_xyz = TimeDistributed(Dense(3,name='beacon_cls2_fc_pose_xyz'))(beacon_cls2_fc1)
beacon_cls2_fc_pose_wpqr = TimeDistributed(Dense(4,name='beacon_cls2_fc_pose_wpqr'))(beacon_cls2_fc1)
'''
beacon_icp7_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp7_out1'))(beacon_icp4_out1)
beacon_icp7_out1.trainable = False
beacon_cls3_fc1_flat = TimeDistributed(Flatten())(beacon_icp7_out1)
beacon_cls3_fc1_flat.trainable = False
beacon_cls3_fc1_pose = TimeDistributed(
Dense(2048, activation='relu',
name='beacon_cls3_fc1_pose'))(beacon_cls3_fc1_flat)
beacon_cls3_fc1_pose.trainable = False
beacon_lstm = LSTM(256, return_sequences=True,
name='beacon_lstm')(beacon_cls3_fc1_pose)
beacon_lstm_dense_xyz = TimeDistributed(
Dense(128,
activation='relu'), name='beacon_lstm_dense_xyz')(beacon_lstm)
beacon_lstm_pose_xyz = TimeDistributed(
Dense(3), name='beacon_lstm_pose_xyz')(beacon_lstm_dense_xyz)
beacon_lstm_dense_wpqr = TimeDistributed(
Dense(128,
activation='relu'), name='beacon_lstm_dense_wpqr')(beacon_lstm)
beacon_lstm_pose_wpqr = TimeDistributed(
Dense(4), name='beacon_lstm_pose_wpqr')(beacon_lstm_dense_wpqr)
beacon_posenet = Model(
inputs=beacon_input,
outputs=[beacon_lstm_pose_xyz, beacon_lstm_pose_wpqr])
if weights_path:
print("start load image network weights")
beacon_posenet.load_weights(weights_path, by_name=True)
print("finish load image network weights")
if not trainable:
for layer in beacon_posenet.layers:
layer.trainable = False
return beacon_posenet
def build_model(p):
""" build a Keras model using the parameters in p """
max_posts = p['max_posts']
max_length = p['max_length']
filters = p['filters']
filtlen = p['filtlen']
poollen = p['poollen']
densed = p['densed']
embed_size = p['embed_size']
batch = p['batch']
random.seed(p['seed'])
np.random.seed(p['seed'])
# https://github.com/fchollet/keras/issues/2280
tf.reset_default_graph()
if len(tf.get_default_graph()._nodes_by_id.keys()) > 0:
raise RuntimeError(
"Seeding is not supported after building part of the graph. "
"Please move set_seed to the beginning of your code.")
tf.set_random_seed(p['seed'])
sess = tf.Session()
K.set_session(sess)
nb_words, genf, tok = datagen(max_posts,
max_length,
stype='training',
batch_size=batch,
randposts=p['randposts'],
mintf=p['mintf'],
mindf=p['mindf'],
noempty=p['noempty'],
prep=p['prep'],
returntok=True)
n_classes = 2
inp = Input(shape=(max_posts, max_length), dtype='int32')
nextin = inp
if p['cosine']:
from keras.constraints import unitnorm
wconstrain = unitnorm()
else:
wconstrain = None
if p['w2v']:
embeddings_index = {}
fn = 'data/w2v_50_sg_export.txt'
with open(fn) as f:
for line in f:
values = line.strip().split()
word = values[0]
if word in tok.word_index:
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
print('Found %s word vectors.' % len(embeddings_index))
embedding_matrix = np.zeros((nb_words, embed_size))
for word, i in tok.word_index.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
else:
embedding_matrix[i] = np.random.uniform(-0.2,
0.2,
size=embed_size)
emb = Embedding(nb_words,
embed_size,
mask_zero=True,
input_length=max_length,
W_constraint=wconstrain,
weights=[embedding_matrix],
trainable=p['etrain'])
if not p['etrain']:
print("making not trainable")
emb.trainable = False
else:
assert p['etrain'], "must have etrain=True with w2v=False"
emb = Embedding(nb_words,
embed_size,
mask_zero=True,
W_constraint=wconstrain)
embedded = TimeDistributed(emb)(nextin)
if not p['etrain']:
emb.trainable = False
embedded.trainable = False
conv = Sequential()
conv.add(
Convolution1D(nb_filter=filters,
filter_length=filtlen,
border_mode='valid',
W_constraint=wconstrain,
activation='linear',
subsample_length=1,
input_shape=(max_length, embed_size)))
conv.add(Activation(p['af']))
conv.add(GlobalAveragePooling1D())
posts = TimeDistributed(conv)(embedded)
combined = Convolution1D(nb_filter=filters,
filter_length=p['acl'],
border_mode='valid',
activation=p['af'],
subsample_length=p['acl'])(posts)
combined = Flatten()(combined)
if densed != 0:
combined = Dense(densed, activation=p['af'])(combined)
outlayer = Dense(2, activation='softmax')(combined)
model = Model(inp, outlayer)
return model, genf
def create_posenet_inception_v3(num_beacon,
image_beacon_weights_path=None,
trainable=True):
# create the base pre-trained model
image_input = Input(shape=(None, 299, 299, 3))
base_model = InceptionV3(weights='imagenet',
include_top=False,
pooling='avg')
base_model.trainable = False
# add top layer
model_output = TimeDistributed(base_model)(image_input)
# beacon subnet 1
beacon_input = Input(shape=(None, num_beacon, 1, 1))
beacon_icp1_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp1_out1'))(beacon_input)
beacon_icp1_out1.trainable = False
# beacon subnet 2
beacon_icp4_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp4_out1'))(beacon_icp1_out1)
beacon_icp4_out1.trainable = False
# beacon subnet 3
beacon_icp7_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp7_out1'))(beacon_icp4_out1)
beacon_icp7_out1.trainable = False
beacon_cls3_fc1_flat = TimeDistributed(Flatten())(beacon_icp7_out1)
beacon_cls3_fc1_flat.trainable = False
beacon_cls3_fc1_pose = TimeDistributed(
Dense(2048, activation='relu',
name='beacon_cls3_fc1_pose'))(beacon_cls3_fc1_flat)
beacon_cls3_fc1_pose.trainable = False
# image, beacon classify 3
image_beacon_cls3_fc1_pose = concatenate(
[model_output, beacon_cls3_fc1_pose],
name='image_beacon_cls3_fc1_pose')
image_beacon_lstm = LSTM(
256, return_sequences=True,
name='image_beacon_lstm')(image_beacon_cls3_fc1_pose)
image_beacon_lstm_dense_xyz = TimeDistributed(
Dense(128, activation='relu'),
name='image_beacon_lstm_dense_xyz')(image_beacon_lstm)
image_beacon_lstm_pose_xyz = TimeDistributed(
Dense(3),
name='image_beacon_lstm_pose_xyz')(image_beacon_lstm_dense_xyz)
image_beacon_lstm_dense_wpqr = TimeDistributed(
Dense(128, activation='relu'),
name='image_beacon_lstm_dense_wpqr')(image_beacon_lstm)
image_beacon_lstm_pose_wpqr = TimeDistributed(
Dense(4),
name='image_beacon_lstm_pose_wpqr')(image_beacon_lstm_dense_wpqr)
image_beacon_posenet = Model(
inputs=[image_input, beacon_input],
outputs=[image_beacon_lstm_pose_xyz, image_beacon_lstm_pose_wpqr])
if image_beacon_weights_path:
print("start load image beacon network weights")
image_beacon_weights_path_ext = os.path.splitext(
image_beacon_weights_path)[-1]
if image_beacon_weights_path_ext == ".npy":
weights_data = np.load(image_beacon_weights_path).item()
for layer in image_beacon_posenet.layers:
if layer.name in weights_data.keys():
layer_weights = weights_data[layer.name]
layer.set_weights(
(layer_weights['weights'], layer_weights['biases']))
print("finish load imaege beacon network weights")
elif image_beacon_weights_path_ext == ".h5":
image_beacon_posenet.load_weights(image_beacon_weights_path,
by_name=True)
print("finish load image beacon network weights")
else:
print("invalid weight file : " + image_weights_path)
sys.exit()
if not trainable:
for layer in image_beacon_posenet.layers:
layer.trainable = False
return image_beacon_posenet
def create_posenet_mobilenet_v1(num_beacon,
image_beacon_weights_path=None,
trainable=True):
# create the base pre-trained model
if K.image_data_format() == 'channels_first':
image_input = Input(shape=(None, 3, 224, 224), name='input_1')
else:
image_input = Input(shape=(None, 224, 224, 3), name='input_1')
alpha = 1.0
dropout = 1e-3
if K.image_data_format() == 'channels_first':
input_shape = (3, 224, 224)
else:
input_shape = (224, 224, 3)
base_model = MobileNet(input_shape=input_shape,
alpha=alpha,
dropout=dropout,
weights='imagenet',
include_top=False)
base_model.trainable = False
# add top layer
model_output = TimeDistributed(base_model)(image_input)
if K.image_data_format() == 'channels_first':
shape = (int(1024 * alpha), 1, 1)
else:
shape = (1, 1, int(1024 * alpha))
model_output = TimeDistributed(GlobalAveragePooling2D())(model_output)
model_output = TimeDistributed(Reshape(shape,
name='reshape_1'))(model_output)
model_output = TimeDistributed(Dropout(dropout,
name='dropout'))(model_output)
image_conv_pose_xyz = TimeDistributed(
Conv2D(1024, (1, 1), padding='same',
name='conv_pose_xyz'))(model_output)
image_conv_pose_xyz_flat = TimeDistributed(Flatten())(image_conv_pose_xyz)
image_conv_pose_wpqr = TimeDistributed(
Conv2D(1024, (1, 1), padding='same',
name='conv_pose_wpqr'))(model_output)
image_conv_pose_wpqr_flat = TimeDistributed(
Flatten())(image_conv_pose_wpqr)
# beacon subnet 1
beacon_input = Input(shape=(None, num_beacon, 1, 1), name='input_2')
beacon_icp1_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp1_out1'))(beacon_input)
beacon_icp1_out1.trainable = False
# beacon subnet 2
beacon_icp4_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp4_out1'))(beacon_icp1_out1)
beacon_icp4_out1.trainable = False
# beacon subnet 3
beacon_icp7_out1 = TimeDistributed(
Conv2D(16, (1, 1),
padding='same',
activation='relu',
name='beacon_icp7_out1'))(beacon_icp4_out1)
beacon_icp7_out1.trainable = False
beacon_cls3_fc1_flat = TimeDistributed(Flatten())(beacon_icp7_out1)
beacon_cls3_fc1_flat.trainable = False
beacon_cls3_fc1_pose = TimeDistributed(
Dense(2048, activation='relu',
name='beacon_cls3_fc1_pose'))(beacon_cls3_fc1_flat)
beacon_cls3_fc1_pose.trainable = False
# image, beacon classify 3
image_beacon_cls3_fc1_pose_xyz = concatenate(
[image_conv_pose_xyz_flat, beacon_cls3_fc1_pose],
name='image_beacon_cls3_fc1_pose_xyz')
image_beacon_lstm_xyz = LSTM(
256, return_sequences=True,
name='image_beacon_lstm_xyz')(image_beacon_cls3_fc1_pose_xyz)
image_beacon_lstm_dense_xyz = TimeDistributed(
Dense(128, activation='relu'),
name='image_beacon_lstm_dense_xyz')(image_beacon_lstm_xyz)
image_beacon_lstm_pose_xyz = TimeDistributed(
Dense(3),
name='image_beacon_lstm_pose_xyz')(image_beacon_lstm_dense_xyz)
image_beacon_cls3_fc1_pose_wpqr = concatenate(
[image_conv_pose_wpqr_flat, beacon_cls3_fc1_pose],
name='image_beacon_cls3_fc1_pose_wpqr')
image_beacon_lstm_wpqr = LSTM(
256, return_sequences=True,
name='image_beacon_lstm_wpqr')(image_beacon_cls3_fc1_pose_wpqr)
image_beacon_lstm_dense_wpqr = TimeDistributed(
Dense(128, activation='relu'),
name='image_beacon_lstm_dense_wpqr')(image_beacon_lstm_wpqr)
image_beacon_lstm_pose_wpqr = TimeDistributed(
Dense(4),
name='image_beacon_lstm_pose_wpqr')(image_beacon_lstm_dense_wpqr)
image_beacon_posenet = Model(
inputs=[image_input, beacon_input],
outputs=[image_beacon_lstm_pose_xyz, image_beacon_lstm_pose_wpqr])
if image_beacon_weights_path:
print("start load image beacon network weights")
image_beacon_weights_path_ext = os.path.splitext(
image_beacon_weights_path)[-1]
if image_beacon_weights_path_ext == ".npy":
weights_data = np.load(image_beacon_weights_path).item()
for layer in image_beacon_posenet.layers:
if layer.name in weights_data.keys():
layer_weights = weights_data[layer.name]
layer.set_weights(
(layer_weights['weights'], layer_weights['biases']))
print("finish load imaege beacon network weights")
elif image_beacon_weights_path_ext == ".h5":
image_beacon_posenet.load_weights(image_beacon_weights_path,
by_name=True)
print("finish load image beacon network weights")
else:
print("invalid weight file : " + image_weights_path)
sys.exit()
if not trainable:
for layer in image_beacon_posenet.layers:
layer.trainable = False
return image_beacon_posenet
