def boosting_rank_net(input_shape, hns=[8, 6, 4, 4], classes=2):
"""
"""
res_model = boosting_res_net((input_shape[1], ),
hns,
out_layer_name='proba')
res_model = Model(res_model.input,
res_model.get_layer('pre_sigmoid').output)
inputs = Input(input_shape)
minor_inputs = Lambda(lambda x: x[:, 0], name='minor_input')(inputs)
pred_minor = res_model(minor_inputs)
minor_out_proba = Lambda(lambda x: x, name='minor_out_proba')(pred_minor)
major_inputs = Lambda(lambda x: x[:, 1], name='major_input')(inputs)
pred_major = res_model(major_inputs)
major_out_proba = Lambda(lambda x: x, name='major_out_proba')(pred_major)
sub = Subtract()([major_out_proba, minor_out_proba])
sub = Lambda(lambda x: x * RANK_SCALE, name='rank_scale_layer')(sub)
proba = Activation('sigmoid')(sub)
model = Model(inputs, proba)
model.compile(optimizer=Nadam(lr=0.001), loss=min_pred)
return model
def layer_test_helper_merge_2d(layer, channel_index, data_format):
# This should test that the output is the correct shape so it should pass
# into a Dense layer rather than a Conv layer.
# The weighted layer is the previous layer,
# Create model
input_shape = list(random.randint(10, 20, size=3))
input_1 = Input(shape=input_shape)
input_2 = Input(shape=input_shape)
x = Conv2D(3, [3, 3], data_format=data_format, name='conv_1')(input_1)
y = Conv2D(3, [3, 3], data_format=data_format, name='conv_2')(input_2)
x = layer([x, y])
x = Flatten()(x)
main_output = Dense(5, name='dense_1')(x)
model = Model(inputs=[input_1, input_2], outputs=main_output)
# Delete channels
del_layer = model.get_layer('conv_1')
del_layer_2 = model.get_layer('conv_2')
surgeon = Surgeon(model)
surgeon.add_job('delete_channels', del_layer, channels=channel_index)
surgeon.add_job('delete_channels', del_layer_2, channels=channel_index)
new_model = surgeon.operate()
new_w = new_model.get_layer('dense_1').get_weights()
# Calculate next layer's correct weights
flat_sz = np.prod(layer.get_output_shape_at(0)[1:])
channel_count = getattr(del_layer, utils.get_channels_attr(del_layer))
channel_index = [i % channel_count for i in channel_index]
if data_format == 'channels_first':
delete_indices = [
x * flat_sz // channel_count + i for x in channel_index
for i in range(
0,
flat_sz // channel_count,
)
]
elif data_format == 'channels_last':
delete_indices = [
x + i for i in range(0, flat_sz, channel_count)
for x in channel_index
]
else:
raise ValueError
correct_w = model.get_layer('dense_1').get_weights()
correct_w[0] = np.delete(correct_w[0], delete_indices, axis=0)
assert weights_equal(correct_w, new_w)
def create_hooknet_encoder(hooknet: Model):
"""Creates a new Keras model with bottleneck of the target branch as ouput after combining the context features
Args:
hooknet (Model): HookNet Model
Returns:
Model: return new model with encoding as output
"""
encoding_layer = hooknet.get_layer("target-branchbottle").output
return Model(hooknet.inputs, encoding_layer)
def _load_model(self, model='VGG'):
if model=='VGG':
print("Loading VGG19 pre-trained model...")
base_model = VGG19(weights='imagenet')
base_model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output)
x = base_model.output
x = GlobalAveragePooling2D()(x)
self.VGG_model = Model(inputs=base_model.input, outputs=x)
if model=='Inception_Resnet':
#load Inception_Resnet model
print("Loading Inception_Resnet_V2 pre-trained model...")
base_model = InceptionResNetV2(weights='imagenet', include_top=False)
x = base_model.output
x = GlobalAveragePooling2D()(x)
self.IR_model = Model(inputs=base_model.input, outputs=x)
def create_model(sequence_length):
# Build model
input_shape = (sequence_length, )
model_input = Input(shape=input_shape)
z = Embedding(len(vocabulary_inv),
embedding_dim,
input_length=sequence_length,
name="embedding")(model_input)
z = Dropout(dropout_prob[0])(z)
# Convolutional block
conv_blocks = []
for sz in filter_sizes:
conv = Convolution1D(filters=num_filters,
kernel_size=sz,
padding="valid",
activation="relu",
strides=1)(z)
conv = MaxPooling1D(pool_size=2)(conv)
conv = Flatten()(conv)
conv_blocks.append(conv)
z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0]
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
# Initialize weights with word2vec
weights = np.array([v for v in embedding_weights.values()])
print("Initializing embedding layer with word2vec weights, shape",
weights.shape)
embedding_layer = model.get_layer("embedding")
embedding_layer.set_weights([weights])
print(model.summary())
return model
def get(self, request):
id = request.query_params['id']
image_path = 'media/' + id
print(image_path)
# load VGG19 model
print("Loading VGG19 pre-trained model...")
base_model = VGG19(weights='imagenet')
base_model = Model(inputs=base_model.input,
outputs=base_model.get_layer('block4_pool').output)
x = base_model.output
x = GlobalAveragePooling2D()(x)
VGG_model = Model(inputs=base_model.input, outputs=x)
img_VGG = image.load_img(image_path, target_size=(224, 224))
img = image.img_to_array(img_VGG) # convert to array
img = np.expand_dims(img, axis=0)
img = ppVGG19(img)
features = VGG_model.predict(img).flatten()
point_a = features.reshape(1, 512)
filename = []
score = []
for i in os.listdir('data/features'):
point_b = np.load('data/features/' + i).reshape(1, 512)
filename.append(i)
distance = np.linalg.norm(point_a - point_b)
cos_lib = cosine_similarity(point_a, point_b)
score.append(cos_lib[0][0])
ind = score.index((max(score)))
print(ind)
print(filename[ind])
product = products.objects.get(obj_id=filename[ind])
serialier = productsSerializer(product)
return Response(serialier.data)
def autoencoder_with_node_features(dataset, adj, feats, weights=None):
aug_adj = sp.hstack([adj, feats])
h, w = aug_adj.shape
kwargs = dict(
use_bias=True,
kernel_initializer='glorot_normal',
kernel_regularizer=None,
bias_initializer='zeros',
bias_regularizer=None,
trainable=True,
)
data = Input(shape=(w, ), dtype=np.float32, name='data')
if dataset in ['protein', 'cora', 'citeseer', 'pubmed']:
# dropout 0.5 is needed for protein and citation (large nets)
noisy_data = Dropout(rate=0.5, name='drop1')(data)
else:
# dropout 0.2 is needed for conflict and metabolic (small nets)
noisy_data = Dropout(rate=0.2, name='drop1')(data)
### First set of encoding transformation ###
encoded = Dense(256, activation='relu', name='encoded1',
**kwargs)(noisy_data)
encoded = Lambda(mvn, name='mvn1')(encoded)
### Second set of encoding transformation ###
encoded = Dense(128, activation='relu', name='encoded2', **kwargs)(encoded)
encoded = Lambda(mvn, name='mvn2')(encoded)
encoded = Dropout(rate=0.5, name='drop2')(encoded)
# the encoder model maps an input to its encoded representation
encoder = Model([data], encoded)
encoded1 = encoder.get_layer('encoded1')
encoded2 = encoder.get_layer('encoded2')
### First set of decoding transformation ###
decoded = DenseTied(256,
tie_to=encoded2,
transpose=True,
activation='relu',
name='decoded2')(encoded)
decoded = Lambda(mvn, name='mvn3')(decoded)
### Second set of decoding transformation - reconstruction ###
decoded = DenseTied(w,
tie_to=encoded1,
transpose=True,
activation='linear',
name='decoded1')(decoded)
# compile the autoencoder
adam = optimizers.Adam(lr=0.001, decay=0.0)
if dataset in ['metabolic', 'conflict']:
# datasets with mixture of binary and real features
decoded_feats = Lambda(lambda x: x[:, adj.shape[1]:],
name='decoded_feats')(decoded)
decoded = Lambda(lambda x: x[:, :adj.shape[1]],
name='decoded')(decoded)
autoencoder = Model(inputs=[data], outputs=[decoded, decoded_feats])
autoencoder.compile(optimizer=adam,
loss={
'decoded': mbce,
'decoded_feats': ce
},
loss_weights={
'decoded': 1.0,
'decoded_feats': 1.0
})
else:
# datasets with only binary graph and node features
autoencoder = Model(inputs=[data], outputs=decoded)
autoencoder.compile(optimizer=adam, loss=mbce)
if weights is not None:
autoencoder.load_weights(weights)
return encoder, autoencoder
class CI_Model(gx_model):
def __init__(self, old_new):
super(CI_Model, self).__init__()
self.pairwise_model = None
self.predict_model = None
self.old_new = old_new # 新模型还是旧模型
self.CI_handle_slt_apis_mode = new_Para.param.CI_handle_slt_apis_mode
self.lr = new_Para.param.CI_learning_rate # 内容部分学习率
self.optimizer = Adam(lr=self.lr)
self.set_simple_name()
self.set_paths()
def set_simple_name(self):
self.simple_name = 'new_func_' if self.old_new == 'new' else 'old_func_' # 新情景，只用功能
self.simple_name += new_Para.param.text_extracter_mode
if new_Para.param.text_extracter_mode == 'inception':
self.simple_name += '_'
self.simple_name += new_Para.param.inception_pooling
if not new_Para.param.if_inception_MLP:
self.simple_name += '_NO_extract_MLP'
if self.old_new == 'new':
if not self.CI_handle_slt_apis_mode:
self.simple_name += '_noSlt'
else:
self.simple_name += ('_' + self.CI_handle_slt_apis_mode
) # 处理方式是全连接还是attention
self.simple_name += '_{}'.format(self.lr) # 学习率
if self.old_new == 'new':
self.simple_name += '_' + new_Para.param.final_activation # + '_New' # 新模型可选sigmoid和softmax
# self.simple_name += '_2'
# self.simple_name += '_newPartTarget' # !!!临时路径！！！效果不好，不用
# self.simple_name += '_reductData' # !!!
def set_paths(self):
# 路径设置
self.model_dir = dataset.crt_ds.model_path.format(
self.get_simple_name()) # 模型路径
if not os.path.exists(self.model_dir):
os.makedirs(self.model_dir)
self.model_name_path = os.path.join(self.model_dir, 'model_name.dat')
self.CI_features_path = os.path.join(self.model_dir, 'CI_features.fea')
self.train_slt_apis_mid_features_path = os.path.join(
self.model_dir, 'train_slt_apis_mid_features.csv')
self.test_slt_apis_mid_features_path = os.path.join(
self.model_dir, 'test_slt_apis_mid_features.csv')
self.ma_text_tag_feas_path = os.path.join(
self.model_dir,
'mashup_api_text_tag_feas.dat') # mashup和api的提取的文本特征
def get_simple_name(self):
return self.simple_name
def get_name(self):
"""
self.model_name区分模型框架，返回的name用于记录在evalute中，区分不同的模型，架构
:return:
"""
return self.model_name
def set_attribute(self):
self.all_mashup_num = meta_data.mashup_num
self.all_api_num = meta_data.api_num
self.his_m_ids = dataset.crt_ds.his_mashup_ids
def set_embedding_matrixs(self):
# 把每个mashup和api的文本和tag编码，得到四个矩阵，输入id即可映射到这些embedding上,节省get_instaces部分的内存
if self.encoded_texts is None:
self.process_texts() # 首先对数据padding，编码处理
if self.encoded_tags is None:
self.process_tags()
# text
self.mid2text_wordindex = np.array(
self.encoded_texts.get_texts_in_index(range(self.all_mashup_num),
'keras_setting', 0))
self.aid2text_wordindex = np.array(
self.encoded_texts.get_texts_in_index(range(self.all_api_num),
'keras_setting',
self.all_mashup_num))
# tag
self.mid2tag_wordindex = np.array(
self.encoded_tags.get_texts_in_index(range(self.all_mashup_num),
'keras_setting', 0))
self.aid2tag_wordindex = np.array(
self.encoded_tags.get_texts_in_index(range(self.all_api_num),
'keras_setting',
self.all_mashup_num))
# instances中slt apis不满足数目时，需要用value(api_num)填充，最终全部映射为0
self.aid2text_wordindex = np.vstack(
(self.aid2text_wordindex,
np.zeros((1, new_Para.param.MAX_SEQUENCE_LENGTH),
dtype=int))).astype(int)
self.aid2tag_wordindex = np.vstack(
(self.aid2tag_wordindex,
np.zeros((1, new_Para.param.MAX_SEQUENCE_LENGTH),
dtype=int))).astype(int)
# print('mid2text_wordindex[0]:',self.mid2text_wordindex[0])
# print('mid2tag_wordindex[0:5]:', self.mid2tag_wordindex[0:5])
# print('aid2text_wordindex[0]:',self.aid2text_wordindex[0])
# print('aid2tag_wordindex[0:5]:', self.aid2tag_wordindex[0:5])
def set_embedding_layers(self):
# embedding
# using the embedding layer instead of packing in the instance, to save memory
self.mid2text_embedding_layer = Embedding(
self.all_mashup_num,
new_Para.param.MAX_SEQUENCE_LENGTH,
embeddings_initializer=Constant(self.mid2text_wordindex),
mask_zero=False,
input_length=1,
trainable=False,
name='mashup_text_encoding_embedding_layer')
self.aid2text_embedding_layer = Embedding(
self.all_api_num + 1,
new_Para.param.MAX_SEQUENCE_LENGTH,
embeddings_initializer=Constant(self.aid2text_wordindex),
mask_zero=False,
trainable=False,
name='api_text_encoding_embedding_layer')
self.mid2tag_embedding_layer = Embedding(
self.all_mashup_num,
new_Para.param.MAX_SEQUENCE_LENGTH,
embeddings_initializer=Constant(self.mid2tag_wordindex),
mask_zero=False,
input_length=1,
trainable=False,
name='mashup_tag_encoding_embedding_layer')
self.aid2tag_embedding_layer = Embedding(
self.all_api_num + 1,
new_Para.param.MAX_SEQUENCE_LENGTH,
embeddings_initializer=Constant(self.aid2tag_wordindex),
mask_zero=False,
trainable=False,
name='api_tag_encoding_embedding_layer')
self.user_text_feature_extracter = None
self.item_text_feature_extracter = None
self.user_tag_feature_extracter = None
self.item_tag_feature_extracter = None
def prepare(self):
self.set_attribute()
self.set_embedding_matrixs()
self.set_embedding_layers()
def user_text_feature_extractor(self):
if not self.user_text_feature_extracter:
user_id_input = Input(shape=(1, ),
dtype='int32') # , name='mashup_id_input'
user_text_input = Lambda(
lambda x: tf.cast(tf.squeeze(x, axis=1), 'int32'))(
self.mid2text_embedding_layer(user_id_input))
user_text_vec = self.feature_extracter_from_texts()(
user_text_input) # (?,50)
self.user_text_feature_extracter = Model(
user_id_input,
user_text_vec,
name='user_text_feature_extracter')
return self.user_text_feature_extracter
# mashup和api的文本特征提取器，区别在于ID到文本编码的embedding矩阵不同，但又要公用相同的word_embedding层和inception特征提取器
def item_text_feature_extractor(self):
if not self.item_text_feature_extracter:
item_id_input = Input(shape=(1, ),
dtype='int32') # , name='api_id_input'
item_text_input = Lambda(
lambda x: tf.cast(tf.squeeze(x, axis=1), 'int32'))(
self.aid2text_embedding_layer(item_id_input))
item_text_vec = self.feature_extracter_from_texts()(
item_text_input) # (?,50)
self.item_text_feature_extracter = Model(
item_id_input,
item_text_vec,
name='item_text_feature_extracter')
return self.item_text_feature_extracter
def user_tag_feature_extractor(self):
if not self.user_tag_feature_extracter:
user_id_input = Input(shape=(1, ),
dtype='int32') # , name='user_id_input'
user_tag_input = Lambda(
lambda x: tf.cast(tf.squeeze(x, axis=1), 'int32'))(
self.mid2tag_embedding_layer(user_id_input))
user_tag_embedding = self.get_text_embedding_layer()(
user_tag_input)
user_tag_vec = Lambda(lambda x: tf.squeeze(x, axis=1))(
SequencePoolingLayer(
'mean', supports_masking=True)(user_tag_embedding))
self.user_tag_feature_extracter = Model(
user_id_input, user_tag_vec, name='user_tag_feature_extracter')
return self.user_tag_feature_extracter
def item_tag_feature_extractor(self):
if not self.item_tag_feature_extracter:
item_id_input = Input(shape=(1, ),
dtype='int32') # , name='api_id_input'
item_tag_input = Lambda(
lambda x: tf.cast(tf.squeeze(x, axis=1), 'int32'))(
self.aid2tag_embedding_layer(item_id_input))
item_tag_embedding = self.get_tag_embedding_layer()(item_tag_input)
item_tag_vec = Lambda(lambda x: tf.squeeze(x, axis=1))(
SequencePoolingLayer(
'mean', supports_masking=True)(item_tag_embedding))
self.item_tag_feature_extracter = Model(
item_id_input, item_tag_vec, name='item_tag_feature_extracter')
return self.item_tag_feature_extracter
def get_model(self):
if not self.model:
mashup_id_input = Input(shape=(1, ),
dtype='int32',
name='mashup_id_input')
api_id_input = Input(shape=(1, ),
dtype='int32',
name='api_id_input')
inputs = [mashup_id_input, api_id_input]
user_text_vec = self.user_text_feature_extractor()(mashup_id_input)
item_text_vec = self.item_text_feature_extractor()(api_id_input)
user_tag_vec = self.user_tag_feature_extractor()(mashup_id_input)
item_tag_vec = self.item_tag_feature_extractor()(api_id_input)
feature_list = [
user_text_vec, item_text_vec, user_tag_vec, item_tag_vec
]
if self.old_new == 'LR_PNCF': # 旧场景，使用GMF形式的双塔模型
x = Concatenate(name='user_concatenate')(
[user_text_vec, user_tag_vec])
y = Concatenate(name='item_concatenate')(
[item_text_vec, item_tag_vec])
output = Multiply()([x, y])
predict_result = Dense(1,
activation='sigmoid',
use_bias=False,
kernel_initializer='lecun_uniform',
name="prediction")(output) # 参数学习权重，非线性
self.model = Model(inputs=inputs,
outputs=[predict_result],
name='predict_model')
return self.model
elif self.old_new == 'new' and self.CI_handle_slt_apis_mode:
# 已选择的服务
mashup_slt_apis_input = Input(
shape=(new_Para.param.slt_item_num, ),
dtype='int32',
name='slt_api_ids_input')
mashup_slt_apis_input_3D = Reshape(
(new_Para.param.slt_item_num, 1))(mashup_slt_apis_input)
# mashup_slt_apis_num_input = Input(shape=(1,), dtype='int32', name='mashup_slt_apis_num_input')
inputs.append(mashup_slt_apis_input)
mask = Lambda(lambda x: K.not_equal(x, self.all_api_num))(
mashup_slt_apis_input) # (?, 3) !!!
# 已选择的服务直接复用item_feature_extractor
slt_text_vec_list, slt_tag_vec_list = [], []
for i in range(new_Para.param.slt_item_num):
x = Lambda(slice, arguments={'index': i})(
mashup_slt_apis_input_3D) # (?,1,1)
x = Reshape((1, ))(x)
temp_item_text_vec = self.item_text_feature_extractor()(x)
temp_item_tag_vec = self.item_tag_feature_extractor()(x)
slt_text_vec_list.append(temp_item_text_vec)
slt_tag_vec_list.append(temp_item_tag_vec)
if self.CI_handle_slt_apis_mode in ('attention', 'average'):
# text和tag使用各自的attention block
slt_text_vec_list = [
Reshape((1, new_Para.param.embedding_dim))(key_2D)
for key_2D in slt_text_vec_list
]
slt_tag_vec_list = [
Reshape((1, new_Para.param.embedding_dim))(key_2D)
for key_2D in slt_tag_vec_list
] # 增加了一维 eg:[None,50]->[None,1,50]
text_keys_embs = Concatenate(axis=1)(
slt_text_vec_list) # [?,3,50]
tag_keys_embs = Concatenate(axis=1)(
slt_tag_vec_list) # [?,3,50]
if self.CI_handle_slt_apis_mode == 'attention':
query_item_text_vec = Lambda(
lambda x: tf.expand_dims(x, axis=1))(
item_text_vec) # (?, 50)->(?, 1, 50)
query_item_tag_vec = Lambda(
lambda x: tf.expand_dims(x, axis=1))(item_tag_vec)
# 压缩历史，得到向量
text_hist = AttentionSequencePoolingLayer(
supports_masking=True)(
[query_item_text_vec, text_keys_embs],
mask=mask)
tag_hist = AttentionSequencePoolingLayer(
supports_masking=True)(
[query_item_tag_vec, tag_keys_embs], mask=mask)
else: # 'average'
text_hist = SequencePoolingLayer(
'mean', supports_masking=True)(text_keys_embs,
mask=mask)
tag_hist = SequencePoolingLayer('mean',
supports_masking=True)(
tag_keys_embs,
mask=mask)
text_hist = Lambda(lambda x: tf.squeeze(x, axis=1))(
text_hist) # (?, 1, 50)->(?, 50)
tag_hist = Lambda(lambda x: tf.squeeze(x, axis=1))(
tag_hist)
elif self.CI_handle_slt_apis_mode == 'full_concate':
text_hist = Concatenate(axis=1)(
slt_text_vec_list) # [?,150]
tag_hist = Concatenate(axis=1)(slt_tag_vec_list) # [?,150]
else:
raise ValueError('wrong CI_handle_slt_apis_mode!')
feature_list.extend([text_hist, tag_hist])
else: # 包括新模型不处理已选择服务和旧模型
pass
feature_list = list(map(NoMask(),
feature_list)) # DNN不支持mak，所以不能再传递mask
all_features = Concatenate(
name='all_content_concatenate')(feature_list)
output = DNN(self.content_fc_unit_nums[:-1])(all_features)
output = Dense(self.content_fc_unit_nums[-1],
activation='relu',
kernel_regularizer=l2(new_Para.param.l2_reg),
name='text_tag_feature_extracter')(output)
# 输出层
if new_Para.param.final_activation == 'softmax':
predict_result = Dense(2,
activation='softmax',
name="prediction")(output)
elif new_Para.param.final_activation == 'sigmoid':
predict_result = Dense(1,
activation='sigmoid',
kernel_initializer='lecun_uniform',
name="prediction")(output)
# Model
# if self.IfUniteNI:
# inputs.append(user_NI_input)
self.model = Model(inputs=inputs,
outputs=[predict_result],
name='predict_model')
for layer in self.model.layers:
print(layer.name)
print('built CI model, done!')
return self.model
def show_text_tag_features(self, train_data, show_num=10):
"""
检查生成的mashup和api的text和tag的特征是否正常
"""
if self.old_new == 'old':
m_ids, a_ids = train_data[:-1]
instances_tuple = self.get_instances(m_ids[:show_num],
a_ids[:show_num])
elif self.old_new == 'new':
m_ids, a_ids, slt_a_ids = train_data[:-1]
instances_tuple = self.get_instances(m_ids[:show_num],
a_ids[:show_num],
slt_a_ids[:show_num])
text_tag_middle_model = Model(
inputs=[*self.model.inputs],
outputs=[
*self.model.get_layer('all_content_concatenate').input[:4]
])
mashup_text_features, apis_text_features, mashup_tag_features, apis_tag_features = text_tag_middle_model.predict(
[*instances_tuple], verbose=0)
mashup_text_features_path = os.path.join(self.model_dir,
'mashup_text_features.dat')
apis_text_features_path = os.path.join(self.model_dir,
'apis_text_features.dat')
mashup_tag_features_path = os.path.join(self.model_dir,
'mashup_tag_features.dat')
apis_tag_features_path = os.path.join(self.model_dir,
'apis_tag_features.dat')
save_2D_list(mashup_text_features_path, mashup_text_features, 'a+')
save_2D_list(apis_text_features_path, apis_text_features, 'a+')
save_2D_list(mashup_tag_features_path, mashup_tag_features, 'a+')
save_2D_list(apis_tag_features_path, apis_tag_features, 'a+')
def get_pairwise_model(self):
if self.pairwise_model is None:
if new_Para.param.pairwise and self.model is not None: # 如果使用pairwise型的目标函数
mashup_id_input = Input(shape=(1, ),
dtype='int32',
name='mashup_id_input')
api_id_input = Input(shape=(1, ),
dtype='int32',
name='api_id_input')
neg_api_id_input = Input(shape=(1, ),
dtype='int32',
name='neg_api_id_input')
mashup_slt_apis_input = Input(
shape=(new_Para.param.slt_item_num, ),
dtype='int32',
name='slt_api_ids_input')
if self.old_new == 'new':
pos_ratings = self.model(
[mashup_id_input, api_id_input, mashup_slt_apis_input])
neg_ratings = self.model([
mashup_id_input, neg_api_id_input,
mashup_slt_apis_input
]) # 再加一个负例api id
elif self.old_new == 'old':
pos_ratings = self.model([mashup_id_input, api_id_input])
neg_ratings = self.model(
[mashup_id_input, neg_api_id_input])
loss = Lambda(
lambda x: K.relu(new_Para.param.margin + x[0] - x[1]),
name='sub_result')([neg_ratings, pos_ratings])
# 注意输入格式！
if self.old_new == 'new':
self.pairwise_model = Model(inputs=[
mashup_id_input, api_id_input, mashup_slt_apis_input,
neg_api_id_input
],
outputs=loss)
elif self.old_new == 'old':
self.pairwise_model = Model(inputs=[
mashup_id_input, api_id_input, neg_api_id_input
],
outputs=loss)
for layer in self.pairwise_model.layers:
print(layer.name)
# # 复用的是同一个对象！
# print(self.pairwise_model.get_layer('predict_model'),id(self.pairwise_model.get_layer('predict_model')))
# print(self.model,id(self.model))
return self.pairwise_model
def get_instances(self,
mashup_id_instances,
api_id_instances,
slt_api_ids_instances=None,
neg_api_id_instances=None,
if_Train=False,
test_phase_flag=True):
"""
生成该模型需要的样本
slt_api_ids_instances是每个样本中，已经选择的api的id序列 变长二维序列
train和test样例都可用 但是针对一维列表形式，所以测试时先需拆分（text的数据是二维列表）！！！
:param args:
:return:
"""
examples = [np.array(mashup_id_instances), np.array(api_id_instances)]
# if new_Para.param.need_slt_apis and slt_api_ids_instances: # 是否加入slt_api_ids_instances
if self.old_new == 'new' and self.CI_handle_slt_apis_mode: # 根据模型变化调整决定，同时输入的数据本身也是对应的
# 节省内存版, 不够slt_item_num的要padding
instance_num = len(slt_api_ids_instances)
padded_slt_api_instances = np.ones(
(instance_num, new_Para.param.slt_item_num)) * self.all_api_num
for index1 in range(instance_num):
a_slt_api_ids = slt_api_ids_instances[index1]
for index2 in range(len(a_slt_api_ids)):
padded_slt_api_instances[index1][index2] = a_slt_api_ids[
index2]
examples.append(padded_slt_api_instances)
if new_Para.param.pairwise and if_Train: # pair test时，不需要neg_api_id_instances
examples.append(np.array(neg_api_id_instances))
examples_tuples = tuple(examples)
return examples_tuples
def get_mashup_api_features(self, mashup_num, api_num):
"""
得到每个mashup和api经过特征提取器或者平均池化得到的特征，可以直接用id索引，供构造instance的文本部分使用
:param text_tag_recommend_model:
:param mashup_num:
:param api_num:
:return:
"""
if os.path.exists(self.ma_text_tag_feas_path):
with open(self.ma_text_tag_feas_path, 'rb') as f1:
mashup_texts_features, mashup_tag_features, api_texts_features, api_tag_features = pickle.load(
f1)
else:
# 前四个分别是 user_text_vec, item_text_vec, user_tag_vec, item_tag_vec
text_tag_middle_model = Model(
inputs=[*self.model.inputs[:2]],
outputs=[
self.model.get_layer('all_content_concatenate').input[0],
self.model.get_layer('all_content_concatenate').input[1],
self.model.get_layer('all_content_concatenate').input[2],
self.model.get_layer('all_content_concatenate').input[3]
])
feature_mashup_ids = list(
np.unique([m_id for m_id in range(mashup_num)]))
feature_instances_tuple = self.get_instances(
feature_mashup_ids, [0] * len(feature_mashup_ids))
mashup_texts_features, _1, mashup_tag_features, _2 = text_tag_middle_model.predict(
[*feature_instances_tuple], verbose=0)
feature_api_ids = list(np.unique([a_id
for a_id in range(api_num)]))
feature_instances_tuple = self.get_instances(
[0] * len(feature_api_ids), feature_api_ids)
_1, api_texts_features, _2, api_tag_features = text_tag_middle_model.predict(
[*feature_instances_tuple], verbose=0)
with open(self.ma_text_tag_feas_path, 'wb') as f2:
pickle.dump((mashup_texts_features, mashup_tag_features,
api_texts_features, api_tag_features), f2)
return mashup_texts_features, mashup_tag_features, api_texts_features, api_tag_features
class MakeMovie(BaseCommand):
def __init__(self):
self.deg_to_rad = math.pi / 180.0
def parse_args(self, args):
parser = argparse.ArgumentParser(prog='makemovie')
parser.add_argument('--tub', help='The tub to make movie from')
parser.add_argument(
'--out',
default='tub_movie.mp4',
help='The movie filename to create. default: tub_movie.mp4')
parser.add_argument(
'--config',
default='./config.py',
help='location of config file to use. default: ./config.py')
parser.add_argument('--model',
help='the model to use to show control outputs')
parser.add_argument('--type', help='the model type to load')
parser.add_argument(
'--salient',
action="store_true",
help='should we overlay salient map showing avtivations')
parser.add_argument('--start',
type=int,
default=1,
help='first frame to process')
parser.add_argument('--end',
type=int,
default=-1,
help='last frame to process')
parser.add_argument('--scale',
type=int,
default=2,
help='make image frame output larger by X mult')
parsed_args = parser.parse_args(args)
return parsed_args, parser
def run(self, args):
'''
Load the images from a tub and create a movie from them.
Movie
'''
import moviepy.editor as mpy
args, parser = self.parse_args(args)
if args.tub is None:
print("ERR>> --tub argument missing.")
parser.print_help()
return
if args.model is not None and args.type is None:
print("ERR>> --type argument missing.")
parser.print_help()
return
if args.salient:
if args.model is None in args.model:
print(
"ERR>> salient visualization requires a model. Pass with the --model arg."
)
parser.print_help()
return
# imported like this, we make TF conditional on use of --salient
# and we keep the context maintained throughout our callbacks to
# compute the salient mask
from tensorflow.python.keras import backend as K
import tensorflow as tf
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
conf = os.path.expanduser(args.config)
if not os.path.exists(conf):
print("No config file at location: %s. Add --config to specify\
location or run from dir containing config.py." % conf)
return
self.cfg = dk.load_config(conf)
self.tub = Tub(args.tub)
self.index = self.tub.get_index(shuffled=False)
start = args.start
self.end = args.end if args.end != -1 else len(self.index)
if self.end >= len(self.index):
self.end = len(self.index) - 1
num_frames = self.end - start
self.iRec = start
self.scale = args.scale
self.keras_part = None
self.convolution_part = None
if not args.model is None:
self.keras_part = get_model_by_type(args.type, cfg=self.cfg)
self.keras_part.load(args.model)
self.keras_part.compile()
if args.salient:
self.init_salient(self.keras_part.model)
#This method nested in this way to take the conditional import of TF
#in a manner that extends to this callback. Done this way, we avoid
#importing in the below method, which triggers a new cuda device allocation
#each call.
def compute_visualisation_mask(img):
#from https://github.com/ermolenkodev/keras-salient-object-visualisation
activations = self.functor([np.array([img])])
activations = [
np.reshape(
img, (1, img.shape[0], img.shape[1], img.shape[2]))
] + activations
upscaled_activation = np.ones((3, 6))
for layer in [5, 4, 3, 2, 1]:
averaged_activation = np.mean(
activations[layer],
axis=3).squeeze(axis=0) * upscaled_activation
output_shape = (activations[layer - 1].shape[1],
activations[layer - 1].shape[2])
x = tf.constant(
np.reshape(averaged_activation,
(1, averaged_activation.shape[0],
averaged_activation.shape[1], 1)),
tf.float32)
conv = tf.nn.conv2d_transpose(
x,
self.layers_kernels[layer],
output_shape=(1, output_shape[0], output_shape[1],
1),
strides=self.layers_strides[layer],
padding='VALID')
with tf.Session() as session:
result = session.run(conv)
upscaled_activation = np.reshape(result, output_shape)
final_visualisation_mask = upscaled_activation
return (final_visualisation_mask -
np.min(final_visualisation_mask)) / (
np.max(final_visualisation_mask) -
np.min(final_visualisation_mask))
self.compute_visualisation_mask = compute_visualisation_mask
print('making movie', args.out, 'from', num_frames, 'images')
clip = mpy.VideoClip(self.make_frame,
duration=((num_frames - 1) /
self.cfg.DRIVE_LOOP_HZ))
clip.write_videofile(args.out, fps=self.cfg.DRIVE_LOOP_HZ)
def draw_model_prediction(self, record, img):
'''
query the model for it's prediction, draw the user input and the predictions
as green and blue lines on the image
'''
if self.keras_part is None:
return
import cv2
user_angle = float(record["user/angle"])
user_throttle = float(record["user/throttle"])
expected = self.keras_part.model.inputs[0].shape[1:]
actual = img.shape
pred_img = img
# check input depth
if expected[2] == 1 and actual[2] == 3:
pred_img = rgb2gray(pred_img)
pred_img = pred_img.reshape(pred_img.shape + (1, ))
actual = pred_img.shape
if expected != actual:
print("expected input dim", expected, "didn't match actual dim",
actual)
return
pilot_angle, pilot_throttle = self.keras_part.run(pred_img)
length = self.cfg.IMAGE_H
a1 = user_angle * 45.0
l1 = user_throttle * length
a2 = pilot_angle * 45.0
l2 = pilot_throttle * length
mid = self.cfg.IMAGE_W // 2 - 1
p1 = tuple((mid - 2, self.cfg.IMAGE_H - 1))
p2 = tuple((mid + 2, self.cfg.IMAGE_H - 1))
p11 = tuple(
(int(p1[0] + l1 * math.cos((a1 + 270.0) * self.deg_to_rad)),
int(p1[1] + l1 * math.sin((a1 + 270.0) * self.deg_to_rad))))
p22 = tuple(
(int(p2[0] + l2 * math.cos((a2 + 270.0) * self.deg_to_rad)),
int(p2[1] + l2 * math.sin((a2 + 270.0) * self.deg_to_rad))))
# user is green, pilot is blue
cv2.line(img, p1, p11, (0, 255, 0), 2)
cv2.line(img, p2, p22, (0, 0, 255), 2)
def draw_steering_distribution(self, record, img):
'''
query the model for it's prediction, draw the distribution of steering choices
'''
from donkeycar.parts.keras import KerasCategorical
if self.keras_part is None or type(
self.keras_part) is not KerasCategorical:
return
import cv2
pred_img = img.reshape((1, ) + img.shape)
angle_binned, throttle = self.keras_part.model.predict(pred_img)
x = 4
dx = 4
y = 120 - 4
iArgMax = np.argmax(angle_binned)
for i in range(15):
p1 = (x, y)
p2 = (x, y - int(angle_binned[0][i] * 100.0))
if i == iArgMax:
cv2.line(img, p1, p2, (255, 0, 0), 2)
else:
cv2.line(img, p1, p2, (200, 200, 200), 2)
x += dx
def init_salient(self, model):
#from https://github.com/ermolenkodev/keras-salient-object-visualisation
from tensorflow.python.keras.layers import Input, Dense, merge
from tensorflow.python.keras.models import Model
from tensorflow.python.keras.layers import Convolution2D, MaxPooling2D, Reshape, BatchNormalization
from tensorflow.python.keras.layers import Activation, Dropout, Flatten, Dense
input_shape = model.inputs[0].shape
img_in = Input(shape=(input_shape[1], input_shape[2], input_shape[3]),
name='img_in')
x = img_in
x = Convolution2D(24, (5, 5),
strides=(2, 2),
activation='relu',
name='conv1')(x)
x = Convolution2D(32, (5, 5),
strides=(2, 2),
activation='relu',
name='conv2')(x)
x = Convolution2D(64, (5, 5),
strides=(2, 2),
activation='relu',
name='conv3')(x)
x = Convolution2D(64, (3, 3),
strides=(2, 2),
activation='relu',
name='conv4')(x)
conv_5 = Convolution2D(64, (3, 3),
strides=(1, 1),
activation='relu',
name='conv5')(x)
self.convolution_part = Model(inputs=[img_in], outputs=[conv_5])
for layer_num in ('1', '2', '3', '4', '5'):
try:
self.convolution_part.get_layer(
'conv' + layer_num).set_weights(
model.get_layer('conv2d_' + layer_num).get_weights())
except Exception as e:
print(e)
print("Failed to load layer weights for layer", layer_num)
raise Exception("Failed to load weights")
from tensorflow.python.keras import backend as K
import tensorflow as tf
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
self.inp = self.convolution_part.input # input placeholder
self.outputs = [
layer.output for layer in self.convolution_part.layers[1:]
] # all layer outputs
self.functor = K.function([self.inp], self.outputs)
kernel_3x3 = tf.constant(
np.array([[[[1]], [[1]], [[1]]], [[[1]], [[1]], [[1]]],
[[[1]], [[1]], [[1]]]]), tf.float32)
kernel_5x5 = tf.constant(
np.array([[[[1]], [[1]], [[1]], [[1]], [[1]]],
[[[1]], [[1]], [[1]], [[1]], [[1]]],
[[[1]], [[1]], [[1]], [[1]], [[1]]],
[[[1]], [[1]], [[1]], [[1]], [[1]]],
[[[1]], [[1]], [[1]], [[1]], [[1]]]]), tf.float32)
self.layers_kernels = {
5: kernel_3x3,
4: kernel_3x3,
3: kernel_5x5,
2: kernel_5x5,
1: kernel_5x5
}
self.layers_strides = {
5: [1, 1, 1, 1],
4: [1, 2, 2, 1],
3: [1, 2, 2, 1],
2: [1, 2, 2, 1],
1: [1, 2, 2, 1]
}
def draw_salient(self, img):
# from https://github.com/ermolenkodev/keras-salient-object-visualisation
import cv2
alpha = 0.004
beta = 1.0 - alpha
expected = self.keras_part.model.inputs[0].shape[1:]
actual = img.shape
pred_img = img
# check input depth
if expected[2] == 1 and actual[2] == 3:
pred_img = rgb2gray(pred_img)
pred_img = pred_img.reshape(pred_img.shape + (1, ))
actual = pred_img.shape
salient_mask = self.compute_visualisation_mask(pred_img)
salient_mask_stacked = np.dstack((salient_mask, salient_mask))
salient_mask_stacked = np.dstack((salient_mask_stacked, salient_mask))
blend = cv2.addWeighted(img.astype('float32'), alpha,
salient_mask_stacked, beta, 0.0)
return blend
def make_frame(self, t):
'''
Callback to return an image from from our tub records.
This is called from the VideoClip as it references a time.
We don't use t to reference the frame, but instead increment
a frame counter. This assumes sequential access.
'''
if self.iRec >= self.end or self.iRec >= len(self.index):
return None
rec_ix = self.index[self.iRec]
rec = self.tub.get_record(rec_ix)
image = rec['cam/image_array']
if self.convolution_part:
image = self.draw_salient(image)
image = image * 255
image = image.astype('uint8')
self.draw_model_prediction(rec, image)
self.draw_steering_distribution(rec, image)
if self.scale != 1:
import cv2
h, w, d = image.shape
dsize = (w * self.scale, h * self.scale)
image = cv2.resize(image,
dsize=dsize,
interpolation=cv2.INTER_CUBIC)
self.iRec += 1
# returns a 8-bit RGB array
return image
def main():
disp_needs_resizing = False
target_disp_shape = (501, 501)
target_disp_length = 501 * 501
datafile = sys.argv[1]
clustering = load_single(CLUSTERS_FILE)
model = None
if MODEL_FILE is not None:
##load model
##stacked lstm
# from tensorflow.python.keras.models import load_model
#model = load_model("model.hdf5")
# print("model loaded")
#sys.path.insert(1,'../weather2')
#os.chdir("../weather2")
import demo
#model=demo.stacked_lstm_ae(8,4096,'relu',32,'sgd',0.2) #simples
model = demo.cnn_bilstm()
#print(model.summary())
#sys.path.insert(1,'../final_eval')
#os.chdir("../final_eval")
model.load_weights(MODEL_FILE)
from tensorflow.python.keras.models import Model
#model = Model(inputs=model.inputs, outputs=model.get_layer("encoder").output)
model = Model(inputs=model.inputs,
outputs=model.get_layer("bidirectional").output)
else:
#log('no model')
pass
# Iterate through the samples
mm = np.memmap(datafile, dtype='float32', mode='r', shape=MM_SHAPE)
# TGIORGOS: CHANGE
# For for each sixhour slot (change to 8 slots)
for s_i, sample in enumerate(mm[0:]):
origin_i = int(sample[SAMPLE_SPEC['origin']]) # real origin
disp = np.array(sample[SAMPLE_SPEC['disp']])
disp = preprocessing.maxabs_scale(disp) * 1000 # scale disp [-1..1)
disp += PAD
if len(disp) < OR_DISP_SIZE:
disp_needs_resizing = True
x = int(np.sqrt(len(disp)))
target_disp_shape = (x, x)
target_disp_length = target_disp_shape[0] * target_disp_shape[1]
# log('Target dispersion shape: ' + str(target_disp_shape))
assert np.sqrt(len(disp)).is_integer() # sanity check...
lis = list()
if s_i + 8 > len(mm):
# log(str("returning...."))
# log(s_i)
# log(s_i+8)
return
for i in range(s_i, s_i + 8, 1): ##1-8 , 2-9 .....
# print("adding",i)
# print 'shape:' , mm[i][[SAMPLE_SPEC['weath']]].shape
lis.append(mm[i][[SAMPLE_SPEC['weath']]])
weather = np.array(lis)
weather = weather.reshape(1, 64, 64, 8, 1) #conv
#weather1 = weather.reshape(4096, 8,1) #add for lstm
ds = Dataset_transformations(weather, 1, weather.shape)
#ds1 = Dataset_transformations(weather1, 1, weather1.shape) #lstm
#print ds._items
ds.normalize()
#ds1.normalize()
# TGIORGOS CHANGE:
# 6 hour slot
ds._items = ds._items.T
#ds1._items = ds1._items.T
ds_hidden = None
#print(ds._items.shape)
if MODEL_FILE is not None:
#h = model.get_hidden(ds._items)
# print(model.summary())
# sys.path.insert(1,'../final_eval')
# os.chdir("../final_eval")
#log(str(ds._items.shape))
h = model.predict(ds._items)
# h = h.reshape(h.shape[0],h.shape[1]*h.shape[2])
#log(str(h))
#log(str(h.shape))
ds_hidden = Dataset_transformations(h, 1, h.shape)
else:
ds_hidden = ds ## unfortunate naming but...
#log('EEEEEEEEEEE')
assert ds_hidden._items.shape == (1, 4096)
# display_array(ds._items.reshape(61, 64))
# display_array(h[:,:2496].reshape(50, 50))
# get the closest cluster to the current weather
cl_order = clustering.centroids_distance(ds_hidden)
# log(cl_order)
cl_cluster = cl_order[0][0]
cl_score = cl_order[0][1]
cluster_date = clustering._desc_date[cl_cluster]
cl_id = reconstruct_date(cluster_date)
scores = []
scores_euc = []
scores_cos = []
for d in glob(DISPERSIONS_DIR + '/' + cl_id + '/*' + SPECIES + '.npy'):
origin = d[d.rfind('/') + 1:]
origin = origin[:origin.find('-')]
cl_dispersion = np.load(d)
if disp_needs_resizing:
# resize the 501x501 into whatever is needed
cl_dispersion = imresize(cl_dispersion,
target_disp_shape,
mode='F')
# p real, q model
# display_array(disp.reshape(167,167))
# display_array(cl_dispersion)
cl_dispersion = preprocessing.maxabs_scale(cl_dispersion) * 1000
cl_dispersion += PAD
scor = euclidean(cl_dispersion.reshape(target_disp_length), disp)
# scor = entropy(disp, cl_dispersion.reshape(target_disp_length))
scores.append((STATIONS.index(origin), origin, scor))
# Calculate cosine distance:
scor_euc = cosine(cl_dispersion.reshape(target_disp_length), disp)
scores_euc.append((STATIONS.index(origin), origin, scor_euc))
scor_cos = correlation(cl_dispersion.reshape(target_disp_length),
disp)
scores_cos.append((STATIONS.index(origin), origin, scor_cos))
assert scor != float('Inf')
assert scor_euc != float('Inf')
assert scor_cos != float('Inf')
scores.sort(key=operator.itemgetter(2))
scores_euc.sort(key=operator.itemgetter(2))
scores_cos.sort(key=operator.itemgetter(2))
#log(str(scores))
#log(str(scores_euc))
#log(str(scores_cos))
pos = 0
pos_euc = 0
pos_cos = 0
# try:
for i in range(0, len(STATIONS)):
#log('BIKA STIN FOR')
#log(str(STATIONS.index(origin)))
if origin_i == scores[i][0]:
pos = i + 1
if origin_i == scores_euc[i][0]:
pos_euc = i + 1
if origin_i == scores_cos[i][0]:
pos_cos = i + 1
if pos > 0 and pos_euc > 0 and pos_cos > 0:
break
# log(str(origin_i) + '> ' + str(s_i) + ' ' + str(pos) )
log(
str(origin_i) + '\t' + str(pos) + '\t' + str(pos_euc) + '\t' +
str(pos_cos))
class textgenrnn:
META_TOKEN = '<s>'
config = {
'rnn_layers': 2,
'rnn_size': 128,
'rnn_bidirectional': False,
'max_length': 40,
'max_words': 10000,
'dim_embeddings': 100,
'word_level': False,
'single_text': False
}
default_config = config.copy()
def __init__(self,
weights_path=None,
vocab_path=None,
config_path=None,
name="textgenrnn_tf"):
if weights_path is None:
weights_path = resource_filename(__name__,
'textgenrnn_weights.hdf5')
if vocab_path is None:
vocab_path = resource_filename(__name__, 'textgenrnn_vocab.json')
if config_path is not None:
with open(config_path, 'r', encoding='utf8',
errors='ignore') as json_file:
self.config = json.load(json_file)
self.config.update({'name': name})
self.default_config.update({'name': name})
with open(vocab_path, 'r', encoding='utf8',
errors='ignore') as json_file:
self.vocab = json.load(json_file)
self.tokenizer = Tokenizer(filters='', lower=False, char_level=True)
self.tokenizer.word_index = self.vocab
self.num_classes = len(self.vocab) + 1
self.model = textgenrnn_model(self.num_classes,
cfg=self.config,
weights_path=weights_path)
self.indices_char = dict((self.vocab[c], c) for c in self.vocab)
def generate(self,
n=1,
return_as_list=False,
prefix=None,
temperature=[1.0, 0.5, 0.2, 0.2],
max_gen_length=300,
interactive=False,
top_n=3,
progress=True):
gen_texts = []
iterable = trange(n) if progress and n > 1 else range(n)
for _ in iterable:
gen_text, _ = textgenrnn_generate(
self.model, self.vocab, self.indices_char, temperature,
self.config['max_length'],
self.META_TOKEN, self.config['word_level'],
self.config.get('single_text', False), max_gen_length,
interactive, top_n, prefix)
if not return_as_list:
print("{}\n".format(gen_text))
gen_texts.append(gen_text)
if return_as_list:
return gen_texts
def generate_samples(self, n=3, temperatures=[0.2, 0.5, 1.0], **kwargs):
for temperature in temperatures:
print('#' * 20 + '\nTemperature: {}\n'.format(temperature) +
'#' * 20)
self.generate(n, temperature=temperature, progress=False, **kwargs)
def train_on_texts(self,
texts,
context_labels=None,
batch_size=128,
num_epochs=50,
verbose=1,
new_model=False,
gen_epochs=1,
train_size=1.0,
max_gen_length=300,
validation=True,
dropout=0.0,
via_new_model=False,
save_epochs=0,
multi_gpu=False,
**kwargs):
if new_model and not via_new_model:
self.train_new_model(texts,
context_labels=context_labels,
num_epochs=num_epochs,
gen_epochs=gen_epochs,
train_size=train_size,
batch_size=batch_size,
dropout=dropout,
validation=validation,
save_epochs=save_epochs,
multi_gpu=multi_gpu,
**kwargs)
return
if context_labels:
context_labels = LabelBinarizer().fit_transform(context_labels)
if 'prop_keep' in kwargs:
train_size = prop_keep
if self.config['word_level']:
texts = [text_to_word_sequence(text, filters='') for text in texts]
# calculate all combinations of text indices + token indices
indices_list = [
np.meshgrid(np.array(i), np.arange(len(text) + 1))
for i, text in enumerate(texts)
]
indices_list = np.block(indices_list)
# If a single text, there will be 2 extra indices, so remove them
# Also remove first sequences which use padding
if self.config['single_text']:
indices_list = indices_list[self.config['max_length']:-2, :]
indices_mask = np.random.rand(indices_list.shape[0]) < train_size
if multi_gpu:
num_gpus = len(K.tensorflow_backend._get_available_gpus())
batch_size = batch_size * num_gpus
gen_val = None
val_steps = None
if train_size < 1.0 and validation:
indices_list_val = indices_list[~indices_mask, :]
gen_val = generate_sequences_from_texts(texts, indices_list_val,
self, context_labels,
batch_size)
val_steps = max(
int(np.floor(indices_list_val.shape[0] / batch_size)), 1)
indices_list = indices_list[indices_mask, :]
num_tokens = indices_list.shape[0]
assert num_tokens >= batch_size, "Fewer tokens than batch_size."
level = 'word' if self.config['word_level'] else 'character'
print("Training on {:,} {} sequences.".format(num_tokens, level))
steps_per_epoch = max(int(np.floor(num_tokens / batch_size)), 1)
gen = generate_sequences_from_texts(texts, indices_list, self,
context_labels, batch_size)
base_lr = 4e-3
# scheduler function must be defined inline.
def lr_linear_decay(epoch):
return (base_lr * (1 - (epoch / num_epochs)))
if context_labels is not None:
if new_model:
weights_path = None
else:
weights_path = "{}_weights.hdf5".format(self.config['name'])
self.save(weights_path)
self.model = textgenrnn_model(self.num_classes,
dropout=dropout,
cfg=self.config,
context_size=context_labels.shape[1],
weights_path=weights_path)
model_t = self.model
if multi_gpu:
# Do not locate model/merge on CPU since sample sizes are small.
parallel_model = multi_gpu_model(self.model,
gpus=num_gpus,
cpu_merge=False)
parallel_model.compile(loss='categorical_crossentropy',
optimizer=RMSprop(lr=4e-3, rho=0.99))
model_t = parallel_model
print("Training on {} GPUs.".format(num_gpus))
model_t.fit_generator(gen,
steps_per_epoch=steps_per_epoch,
epochs=num_epochs,
callbacks=[
LearningRateScheduler(lr_linear_decay),
generate_after_epoch(self, gen_epochs,
max_gen_length),
save_model_weights(self, num_epochs,
save_epochs)
],
verbose=verbose,
max_queue_size=10,
validation_data=gen_val,
validation_steps=val_steps)
# Keep the text-only version of the model if using context labels
if context_labels is not None:
self.model = Model(inputs=self.model.input[0],
outputs=self.model.output[1])
def train_new_model(self,
texts,
context_labels=None,
num_epochs=50,
gen_epochs=1,
batch_size=128,
dropout=0.0,
train_size=1.0,
validation=True,
save_epochs=0,
multi_gpu=False,
**kwargs):
self.config = self.default_config.copy()
self.config.update(**kwargs)
print("Training new model w/ {}-layer, {}-cell {}LSTMs".format(
self.config['rnn_layers'], self.config['rnn_size'],
'Bidirectional ' if self.config['rnn_bidirectional'] else ''))
# If training word level, must add spaces around each punctuation.
# https://stackoverflow.com/a/3645946/9314418
if self.config['word_level']:
punct = '!"#$%&()*+,-./:;<=>?@[\]^_`{|}~\\n\\t\'‘’“”’–—'
for i in range(len(texts)):
texts[i] = re.sub('([{}])'.format(punct), r' \1 ', texts[i])
texts[i] = re.sub(' {2,}', ' ', texts[i])
# Create text vocabulary for new texts
# if word-level, lowercase; if char-level, uppercase
self.tokenizer = Tokenizer(filters='',
lower=self.config['word_level'],
char_level=(not self.config['word_level']))
self.tokenizer.fit_on_texts(texts)
# Limit vocab to max_words
max_words = self.config['max_words']
self.tokenizer.word_index = {
k: v
for (k, v) in self.tokenizer.word_index.items() if v <= max_words
}
if not self.config.get('single_text', False):
self.tokenizer.word_index[self.META_TOKEN] = len(
self.tokenizer.word_index) + 1
self.vocab = self.tokenizer.word_index
self.num_classes = len(self.vocab) + 1
self.indices_char = dict((self.vocab[c], c) for c in self.vocab)
# Create a new, blank model w/ given params
self.model = textgenrnn_model(self.num_classes,
dropout=dropout,
cfg=self.config)
# Save the files needed to recreate the model
with open('{}_vocab.json'.format(self.config['name']),
'w',
encoding='utf8') as outfile:
json.dump(self.tokenizer.word_index, outfile, ensure_ascii=False)
with open('{}_config.json'.format(self.config['name']),
'w',
encoding='utf8') as outfile:
json.dump(self.config, outfile, ensure_ascii=False)
self.train_on_texts(texts,
new_model=True,
via_new_model=True,
context_labels=context_labels,
num_epochs=num_epochs,
gen_epochs=gen_epochs,
train_size=train_size,
batch_size=batch_size,
dropout=dropout,
validation=validation,
save_epochs=save_epochs,
multi_gpu=multi_gpu,
**kwargs)
def save(self, weights_path="textgenrnn_weights_saved.hdf5"):
self.model.save_weights(weights_path)
def load(self, weights_path):
self.model = textgenrnn_model(self.num_classes,
cfg=self.config,
weights_path=weights_path)
def reset(self):
self.config = self.default_config.copy()
self.__init__(name=self.config['name'])
def train_from_file(self,
file_path,
header=True,
delim="\n",
new_model=False,
context=None,
is_csv=False,
**kwargs):
context_labels = None
if context:
texts, context_labels = textgenrnn_texts_from_file_context(
file_path)
else:
texts = textgenrnn_texts_from_file(file_path, header, delim,
is_csv)
print("{:,} texts collected.".format(len(texts)))
if new_model:
self.train_new_model(texts,
context_labels=context_labels,
**kwargs)
else:
self.train_on_texts(texts, context_labels=context_labels, **kwargs)
def train_from_largetext_file(self, file_path, new_model=True, **kwargs):
with open(file_path, 'r', encoding='utf8', errors='ignore') as f:
texts = [f.read()]
if new_model:
self.train_new_model(texts, single_text=True, **kwargs)
else:
self.train_on_texts(texts, single_text=True, **kwargs)
def generate_to_file(self, destination_path, **kwargs):
texts = self.generate(return_as_list=True, **kwargs)
with open(destination_path, 'w') as f:
for text in texts:
f.write("{}\n".format(text))
def encode_text_vectors(self,
texts,
pca_dims=50,
tsne_dims=None,
tsne_seed=None,
return_pca=False,
return_tsne=False):
# if a single text, force it into a list:
if isinstance(texts, str):
texts = [texts]
vector_output = Model(inputs=self.model.input,
outputs=self.model.get_layer('attention').output)
encoded_vectors = []
maxlen = self.config['max_length']
for text in texts:
if self.config['word_level']:
text = text_to_word_sequence(text, filters='')
text_aug = [self.META_TOKEN] + list(text[0:maxlen])
encoded_text = textgenrnn_encode_sequence(text_aug, self.vocab,
maxlen)
encoded_vector = vector_output.predict(encoded_text)
encoded_vectors.append(encoded_vector)
encoded_vectors = np.squeeze(np.array(encoded_vectors), axis=1)
if pca_dims is not None:
assert len(texts) > 1, "Must use more than 1 text for PCA"
pca = PCA(pca_dims)
encoded_vectors = pca.fit_transform(encoded_vectors)
if tsne_dims is not None:
tsne = TSNE(tsne_dims, random_state=tsne_seed)
encoded_vectors = tsne.fit_transform(encoded_vectors)
return_objects = encoded_vectors
if return_pca or return_tsne:
return_objects = [return_objects]
if return_pca:
return_objects.append(pca)
if return_tsne:
return_objects.append(tsne)
return return_objects
def similarity(self, text, texts, use_pca=True):
text_encoded = self.encode_text_vectors(text, pca_dims=None)
if use_pca:
texts_encoded, pca = self.encode_text_vectors(texts,
return_pca=True)
text_encoded = pca.transform(text_encoded)
else:
texts_encoded = self.encode_text_vectors(texts, pca_dims=None)
cos_similairity = cosine_similarity(text_encoded, texts_encoded)[0]
text_sim_pairs = list(zip(texts, cos_similairity))
text_sim_pairs = sorted(text_sim_pairs, key=lambda x: -x[1])
return text_sim_pairs
data = np.load(GHT_FILE)[_LEVEL]
# Make 2 days (overlap 1 day)
data, _, _ = utils.overlap(data, win = 4, t = 8)
if MODEL=='':
print('insert model..')
exit()
print('Data Shape...',data.shape)
##predict data...
import demo
model=demo.stacked_lstm_ae(8,4096,'relu',32,'sgd',0.2,0.1)
model.load_weights(MODEL_FILE)
from tensorflow.python.keras.models import Model
model = Model(inputs=model.inputs, outputs=model.get_layer("encoder").output)
data = model.predict(data)
# Reshape
data = data.reshape(data.shape[1], data.shape[0])
ds = Dataset_transformations(data.T, 1000, data.shape)
if os.path.exists(PREFIX+CONFIG_NAME+'.zip'):
clust_obj = dataset_utils.load_single(PREFIX+CONFIG_NAME+'.zip')
else:
print 'Doing kmeans.....'
clust_obj = Clustering(ds,n_clusters=15,n_init=100,features_first=False)
clust_obj.batch_kmeans(10)
print 'Saving .....'
clust_obj.save(PREFIX+CONFIG_NAME+'.zip')
# Descriptor num_min: 1
def run_tests_file_output(model_name,
delex_slots=(),
alpha=-1.0,
beta=-1.0,
rev_pen=-1.0,
gamma=-1.0,
p=-1.0,
num_samp=-1,
file_suff="",
maximise_bleu=False):
"""
Runs tests and outputs to file
"""
data = load_data_chars("devset.csv",
delex_slots=delex_slots,
delex_both=False)
data = vectorise_chars(data)
data = create_reverse_indices(data)
model, encoder, decoder = load_models(model_name)
decoder = Model(
inputs=decoder.input,
outputs=[decoder.output,
decoder.get_layer('attention').output])
rev_model, rev_encoder, rev_decoder = load_models("basicreverse")
rev_decoder = Model(inputs=rev_decoder.input,
outputs=[
rev_decoder.output,
rev_decoder.get_layer('attention').output
])
output_file = open(model_name + file_suff + "_output.txt", "w")
prev_mr = ""
for seq_index in range(len(data["mr_input"])):
print(seq_index)
input = data["encoder_input"][seq_index:seq_index + 1]
correct = data['ref_sentences'][seq_index].strip("\t\n")
mr = data["mr_input"][seq_index]
if mr == prev_mr:
continue
prev_mr = mr
paths = decode_beam_search(input,
data,
encoder,
decoder,
mr=data['mr_input'][seq_index],
reverse_models=[rev_encoder, rev_decoder],
beam_width=10,
alpha=alpha,
beta=beta,
gamma=gamma,
rev_pen=rev_pen,
p=p,
num_samp=num_samp)
if len(delex_slots) > 0:
for i in range(len(paths)):
paths[i] = relexicalise_sentence(paths[i],
data['mr_input'][seq_index],
delex_slots)
if not maximise_bleu:
decoded_sentence = paths[0]
else:
max_bleu = 0
max_sent = ""
for p in paths:
bleu = sentence_bleu(references=[correct], hypothesis=p)
if bleu > max_bleu:
max_bleu = bleu
max_sent = p
decoded_sentence = max_sent
output_file.write(decoded_sentence)
print(decoded_sentence)
return
class JointEmbeddingModel:
def __init__(self, config):
self.data_dir = config.data_dir
self.model_name = config.model_name
self.meth_name_len = config.methname_len # the max length of method name
self.apiseq_len = config.apiseq_len
self.tokens_len = config.tokens_len
self.desc_len = config.desc_len
self.vocab_size = config.n_words # the size of vocab
self.embed_dims = config.embed_dims
self.lstm_dims = config.lstm_dims
self.hidden_dims = config.hidden_dims
self.margin = 0.05
self.init_embed_weights_meth_name = config.init_embed_weights_methodname
self.init_embed_weights_tokens = config.init_embed_weights_tokens
self.init_embed_weights_desc = config.init_embed_weights_desc
self.meth_name = Input(shape=(self.meth_name_len,), dtype='int32', name='meth_name')
self.apiseq = Input(shape=(self.apiseq_len,), dtype='int32', name='apiseq')
self.tokens = Input(shape=(self.tokens_len,), dtype='int32', name='tokens2')
self.desc_good = Input(shape=(self.desc_len,), dtype='int32', name='desc_good')
self.desc_bad = Input(shape=(self.desc_len,), dtype='int32', name='desc_bad')
if not os.path.exists(self.data_dir + 'model/' + self.model_name):
os.makedirs(self.data_dir + 'model/' + self.model_name)
def build(self):
self.transformer_meth = transformer.EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims,
embed_dim=self.embed_dims, ffn_dim=self.lstm_dims,
droput_rate=0.2, n_heads=2, max_len=self.meth_name_len,
name='methT')
self.transformer_apiseq = transformer.EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims,
embed_dim=self.embed_dims, ffn_dim=self.lstm_dims,
droput_rate=0.2, n_heads=4, max_len=self.apiseq_len,
name='apiseqT')
self.transformer_desc = transformer.EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims,
embed_dim=self.embed_dims, ffn_dim=self.lstm_dims,
droput_rate=0.2, n_heads=4, max_len=self.desc_len, name='descT')
# self.transformer_ast = EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims, embed_dim=self.embed_dims, ffn_dim=self.lstm_dims, droput_rate=0.2, n_heads=4, max_len=128)
self.transformer_tokens = transformer.EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims,
embed_dim=self.embed_dims, ffn_dim=self.lstm_dims,
droput_rate=0.2, n_heads=8, max_len=self.tokens_len,
name='tokensT')
# create path to store model Info
# 1 -- CodeNN
meth_name = Input(shape=(self.meth_name_len,), dtype='int32', name='meth_name')
apiseq = Input(shape=(self.apiseq_len,), dtype='int32', name='apiseq')
tokens3 = Input(shape=(self.tokens_len,), dtype='int32', name='tokens3')
# method name
# embedding layer
meth_name_out = self.transformer_meth(meth_name)
# max pooling
maxpool = Lambda(lambda x: k.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]),
name='maxpooling_methodname')
method_name_pool = maxpool(meth_name_out)
activation = Activation('tanh', name='active_method_name')
method_name_repr = activation(method_name_pool)
# apiseq
# embedding layer
apiseq_out = self.transformer_apiseq(apiseq)
# max pooling
maxpool = Lambda(lambda x: k.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]),
name='maxpooling_apiseq')
apiseq_pool = maxpool(apiseq_out)
activation = Activation('tanh', name='active_apiseq')
apiseq_repr = activation(apiseq_pool)
# tokens
# embedding layer
tokens_out = self.transformer_tokens(tokens3)
# max pooling
maxpool = Lambda(lambda x: k.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]),
name='maxpooling_tokens')
tokens_pool = maxpool(tokens_out)
activation = Activation('tanh', name='active_tokens')
tokens_repr = activation(tokens_pool)
# fusion method_name, apiseq, tokens
merge_method_name_api = Concatenate(name='merge_methname_api')([method_name_repr, apiseq_repr])
merge_code_repr = Concatenate(name='merge_code_repr')([merge_method_name_api, tokens_repr])
code_repr = Dense(self.hidden_dims, activation='tanh', name='dense_coderepr')(merge_code_repr)
self.code_repr_model = Model(inputs=[meth_name, apiseq, tokens3], outputs=[code_repr], name='code_repr_model')
self.code_repr_model.summary()
self.output = Model(inputs=self.code_repr_model.input, outputs=self.code_repr_model.get_layer('tokensT').output)
self.output.summary()
# 2 -- description
desc = Input(shape=(self.desc_len,), dtype='int32', name='desc')
# desc
# embedding layer
desc_out = self.transformer_desc(desc)
# max pooling
maxpool = Lambda(lambda x: k.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]),
name='maxpooling_desc')
desc_pool = maxpool(desc_out)
activation = Activation('tanh', name='active_desc')
desc_repr = activation(desc_pool)
self.desc_repr_model = Model(inputs=[desc], outputs=[desc_repr], name='desc_repr_model')
self.desc_repr_model.summary()
# 3 -- cosine similarity
code_repr = self.code_repr_model([meth_name, apiseq, tokens3])
desc_repr = self.desc_repr_model([desc])
cos_sim = Dot(axes=1, normalize=True, name='cos_sim')([code_repr, desc_repr])
sim_model = Model(inputs=[meth_name, apiseq, tokens3, desc], outputs=[cos_sim], name='sim_model')
self.sim_model = sim_model
self.sim_model.summary()
# 4 -- build training model
good_sim = sim_model([self.meth_name, self.apiseq, self.tokens, self.desc_good])
bad_sim = sim_model([self.meth_name, self.apiseq, self.tokens, self.desc_bad])
loss = Lambda(lambda x: k.maximum(1e-6, self.margin - (x[0] - x[1])), output_shape=lambda x: x[0], name='loss')(
[good_sim, bad_sim])
self.training_model = Model(inputs=[self.meth_name, self.apiseq, self.tokens, self.desc_good, self.desc_bad],
outputs=[loss], name='training_model')
self.training_model.summary()
def compile(self, optimizer, **kwargs):
optimizer = keras.optimizers.SGD(lr=0.0001, momentum=0.9, nesterov=True)
# optimizer = keras.optimizers.Adam(lr=0.0001)
# print(self.code_repr_model.layers, self.desc_repr_model.layers, self.training_model.layers, self.sim_model.layers)
self.code_repr_model.compile(loss='cosine_proximity', optimizer=optimizer, **kwargs)
self.desc_repr_model.compile(loss='cosine_proximity', optimizer=optimizer, **kwargs)
self.training_model.compile(loss=lambda y_true, y_pred: y_pred + y_true - y_true, optimizer=optimizer, **kwargs)
self.sim_model.compile(loss='binary_crossentropy', optimizer=optimizer, **kwargs)
def fit(self, x, **kwargs):
y = np.zeros(shape=x[0].shape[:1], dtype=np.float32)
return self.training_model.fit(x, y, **kwargs)
def getOutput(self, x):
# functor = k.function([self.code_repr_model.layers[0].input, k.learning_phase()], [self.code_repr_model.layers[0].output])
# print(functor(x)[0])
print(self.output.predict(x))
def repr_code(self, x, **kwargs):
return self.code_repr_model.predict(x, **kwargs)
def repr_desc(self, x, **kwargs):
return self.desc_repr_model.predict(x, **kwargs)
def predict(self, x, **kwargs):
return self.sim_model.predict(x, **kwargs)
def save(self, code_model_file, desc_model_file, **kwargs):
file = h5py.File(code_model_file, 'w')
weight_code = self.code_repr_model.get_weights()
for i in range(len(weight_code)):
file.create_dataset('weight_code'+str(i), data=weight_code[i])
file.close()
file = h5py.File(desc_model_file, 'w')
weight_desc = self.desc_repr_model.get_weights()
for i in range(len(weight_desc)):
file.create_dataset('weight_desc'+str(i), data=weight_desc[i])
file.close()
# self.code_repr_model.save_weights(code_model_file, **kwargs)
# self.desc_repr_model.save_weights(desc_model_file, **kwargs)
def load(self, code_model_file, desc_model_file, **kwargs):
# self.code_repr_model.load_weights(code_model_file, **kwargs)
# self.desc_repr_model.load_weights(desc_model_file, **kwargs)
file = h5py.File(code_model_file, 'r')
weight_code = []
for i in range(len(file.keys())):
weight_code.append(file['weight_code'+str(i)][:])
self.code_repr_model.set_weights(weight_code)
file.close()
file = h5py.File(desc_model_file, 'r')
weight_desc = []
for i in range(len(file.keys())):
weight_desc.append(file['weight_desc'+str(i)][:])
self.desc_repr_model.set_weights(weight_desc)
file.close()
seq_in, _, _ = demo.overlap(ght_700, 4, t=8)
seq_in = seq_in.reshape((seq_in.shape[0], seq_in.shape[1], 64, 64, 1))
if MODEL == '':
print('insert model..')
exit()
##predict data...
model = demo.model_conv_lstm(TIMESTEPS)
model.load_weights(MODEL)
print(model.summary())
from tensorflow.python.keras.models import Model
model = Model(inputs=model.input, outputs=model.get_layer('encoder').output)
data = model.predict(seq_in)
print("Prediction shape:", data.shape)
exit()
data = data.reshape((data.shape[0] * data.shape[1], 8 * 8 * 2))
print('reshape encoding..:', data.shape)
# Reshape
data = data.reshape(data.shape[1], data.shape[0])
ds = Dataset_transformations(data.T, 1000, data.shape)
if os.path.exists(PREFIX + CONFIG_NAME + '.zip'):
clust_obj = dataset_utils.load_single(PREFIX + CONFIG_NAME + '.zip')
else:
print 'Doing kmeans.....'
def main():
# Counting Dataset
counting_dataset_path = 'counting_data_UCF'
counting_dataset = list()
train_labels = {}
val_labels = {}
for im_path in glob.glob(os.path.join(counting_dataset_path, '*.jpg')):
counting_dataset.append(im_path)
img = image.load_img(im_path)
gt_file = im_path.replace('.jpg', '_ann.mat')
h, w = img.size
dmap, crowd_number = load_gt_from_mat(gt_file, (w, h))
train_labels[im_path] = dmap
val_labels[im_path] = crowd_number
counting_dataset_pyramid, train_labels_pyramid = multiscale_pyramid(
counting_dataset, train_labels)
# Ranking Dataset
ranking_dataset_path = 'ranking_data'
ranking_dataset = list()
for im_path in glob.glob(os.path.join(ranking_dataset_path, '*.jpg')):
ranking_dataset.append(im_path)
# randomize the order of images before splitting
np.random.shuffle(counting_dataset)
split_size = int(round(len(counting_dataset) / 5))
splits_list = list()
for t in range(5):
splits_list.append(counting_dataset[t * split_size:t * split_size +
split_size])
split_val_labels = {}
mae_sum = 0.0
mse_sum = 0.0
# create folder to save results
date = str(datetime.datetime.now())
d = date.split()
d1 = d[0]
d2 = d[1].split(':')
results_folder = 'Results-' + d1 + '-' + d2[0] + '.' + d2[1]
if not os.path.exists(results_folder):
os.makedirs(results_folder)
# 5-fold cross validation
epochs = int(round(iterations / iterations_per_epoch))
n_fold = 5
for f in range(0, n_fold):
print('\nFold ' + str(f))
# Model
model = VGG16(include_top=False, weights='imagenet')
transfer_layer = model.get_layer('block5_conv3')
conv_model = Model(inputs=[model.input],
outputs=[transfer_layer.output],
name='vgg_partial')
counting_input = Input(shape=(224, 224, 3),
dtype='float32',
name='counting_input')
ranking_input = Input(shape=(224, 224, 3),
dtype='float32',
name='ranking_input')
x = conv_model([counting_input, ranking_input])
counting_output = Conv2D(1, (3, 3),
strides=(1, 1),
padding='same',
data_format=None,
dilation_rate=(1, 1),
activation='relu',
use_bias=True,
kernel_initializer='glorot_uniform',
bias_initializer='zeros',
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
bias_constraint=None,
name='counting_output')(x)
# The ranking output is computed using SUM pool. Here I use
# GlobalAveragePooling2D followed by a multiplication by 14^2 to do
# this.
ranking_output = Lambda(
lambda i: 14.0 * 14.0 * i,
name='ranking_output')(GlobalAveragePooling2D(
name='global_average_pooling2d')(counting_output))
train_model = Model(inputs=[counting_input, ranking_input],
outputs=[counting_output, ranking_output])
train_model.summary()
# l2 weight decay
for layer in train_model.layers:
if hasattr(layer, 'kernel_regularizer'):
layer.kernel_regularizer = regularizers.l2(5e-4)
elif layer.name == 'vgg_partial':
for l in layer.layers:
if hasattr(l, 'kernel_regularizer'):
l.kernel_regularizer = regularizers.l2(5e-4)
optimizer = SGD(lr=0.0, decay=0.0, momentum=0.9, nesterov=False)
loss = {
'counting_output': euclideanDistanceCountingLoss,
'ranking_output': pairwiseRankingHingeLoss
}
loss_weights = [1.0, 0.0]
train_model.compile(optimizer=optimizer,
loss=loss,
loss_weights=loss_weights)
splits_list_tmp = splits_list.copy()
# counting validation split
split_val = splits_list_tmp[f]
del splits_list_tmp[f]
flat = itertools.chain.from_iterable(splits_list_tmp)
# counting train split
split_train = list(flat)
# counting validation split labels
split_val_labels = {k: val_labels[k] for k in split_val}
counting_dataset_pyramid_split = []
train_labels_pyramid_split = []
for key in split_train:
counting_dataset_pyramid_split.append(
counting_dataset_pyramid[key][0])
counting_dataset_pyramid_split.append(
counting_dataset_pyramid[key][1])
counting_dataset_pyramid_split.append(
counting_dataset_pyramid[key][2])
counting_dataset_pyramid_split.append(
counting_dataset_pyramid[key][3])
counting_dataset_pyramid_split.append(
counting_dataset_pyramid[key][4])
train_labels_pyramid_split.append(train_labels_pyramid[key][0])
train_labels_pyramid_split.append(train_labels_pyramid[key][1])
train_labels_pyramid_split.append(train_labels_pyramid[key][2])
train_labels_pyramid_split.append(train_labels_pyramid[key][3])
train_labels_pyramid_split.append(train_labels_pyramid[key][4])
index_shuf = np.arange(len(counting_dataset_pyramid_split))
np.random.shuffle(index_shuf)
counting_dataset_pyramid_split_shuf = []
train_labels_pyramid_split_shuf = []
for i in index_shuf:
counting_dataset_pyramid_split_shuf.append(
counting_dataset_pyramid_split[i])
train_labels_pyramid_split_shuf.append(
train_labels_pyramid_split[i])
train_generator = DataGenerator(counting_dataset_pyramid_split_shuf,
train_labels_pyramid_split_shuf,
ranking_dataset, **params)
lrate = LearningRateScheduler(step_decay)
callbacks_list = [lrate]
train_model.fit_generator(generator=train_generator,
epochs=epochs,
callbacks=callbacks_list)
#test images
tmp_model = train_model.get_layer('vgg_partial')
test_input = Input(shape=(None, None, 3),
dtype='float32',
name='test_input')
new_input = tmp_model(test_input)
co = train_model.get_layer('counting_output')(new_input)
test_output = Lambda(lambda i: K.sum(i, axis=(1, 2)),
name='test_output')(co)
test_model = Model(inputs=[test_input], outputs=[test_output])
predictions = np.empty((len(split_val), 1))
y_validation = np.empty((len(split_val), 1))
for i in range(len(split_val)):
img = image.load_img(split_val[i], target_size=(224, 224))
img_to_array = image.img_to_array(img)
img_to_array = preprocess_input(img_to_array)
img_to_array = np.expand_dims(img_to_array, axis=0)
pred_test = test_model.predict(img_to_array)
predictions[i] = pred_test
y_validation[i] = split_val_labels[split_val[i]]
mean_abs_err = mae(predictions, y_validation)
mean_sqr_err = mse(predictions, y_validation)
# serialize model to JSON
model_json = test_model.to_json()
model_json_name = "test_model_" + str(f) + ".json"
with open(model_json_name, "w") as json_file:
json_file.write(model_json)
# serialize weights to HDF5
model_h5_name = "test_model_" + str(f) + ".h5"
test_model.save_weights(model_h5_name)
print("Saved model to disk")
print('\n######################')
print('Results on TEST SPLIT:')
print(' MAE: {}'.format(mean_abs_err))
print(' MSE: {}'.format(mean_sqr_err))
print("Took %f seconds" % (time.time() - s))
path1 = results_folder + '/test_split_results_fold-' + str(f) + '.txt'
with open(path1, 'w') as f:
f.write('mae: %f,\nmse: %f, \nTook %f seconds' %
(mean_abs_err, mean_sqr_err, time.time() - s))
mae_sum = mae_sum + mean_abs_err
mse_sum = mse_sum + mean_sqr_err
print('\n################################')
print('Average Results on TEST SPLIT:')
print(' AVE MAE: {}'.format(mae_sum / n_fold))
print(' AVE MSE: {}'.format(mse_sum / n_fold))
print("Took %f seconds" % (time.time() - s))
path2 = results_folder + '/test_split_results_avg.txt'
with open(path2, 'w') as f:
f.write('avg_mae: %f, \navg_mse: %f, \nTook %f seconds' %
(mae_sum / n_fold, mse_sum / n_fold, time.time() - s))
def seq2seq_architecture(latent_size, vocabulary_size, article_max_len,
embedding_matrix, batch_size, epochs, train_article,
train_summary, train_target):
encoder_inputs = Input(shape=(article_max_len, ), name='Encoder-Input')
encoder_embeddings = Embedding(
vocabulary_size + 1,
300,
weights=[embedding_matrix],
trainable=False,
mask_zero=False,
name='Encoder-Word-Embedding')(encoder_inputs)
encoder_embeddings = BatchNormalization(
name='Encoder-Batch-Normalization')(encoder_embeddings)
encoder_conv = Conv1D(filters=4,
kernel_size=8,
padding='same',
activation='relu')(encoder_embeddings)
encoder_drop = Dropout(0.25)(encoder_conv)
encoder_pool = MaxPooling1D(pool_size=1)(encoder_drop)
encoder_flatten = Flatten()(encoder_pool)
encoder_model = Model(inputs=encoder_inputs,
outputs=encoder_flatten,
name='Encoder-Model')
encoder_outputs = encoder_model(encoder_inputs)
decoder_inputs = Input(shape=(None, ), name='Decoder-Input')
decoder_embeddings = Embedding(
vocabulary_size + 1,
300,
weights=[embedding_matrix],
trainable=False,
mask_zero=False,
name='Decoder-Word-Embedding')(decoder_inputs)
decoder_embeddings = BatchNormalization(
name='Decoder-Batch-Normalization-1')(decoder_embeddings)
decoder_conv = Conv1D(filters=32, kernel_size=4, padding='same', activation='relu', name='Decoder-Conv1D') \
(decoder_embeddings)
decoder_drop = Dropout(0.25, name='Decoder-Conv1D-Dropout')(decoder_conv)
decoder_pool = MaxPooling1D(pool_size=1, name='Decoder-MaxPool1D')(
decoder_drop) # GlobalMaxPool1D()
decoder_gru = GRU(latent_size,
return_state=True,
return_sequences=True,
name='Decoder-GRU')
decoder_gru_outputs, _ = decoder_gru(decoder_pool,
initial_state=encoder_outputs)
decoder_outputs = BatchNormalization(
name='Decoder-Batch-Normalization-2')(decoder_gru_outputs)
decoder_outputs = Dense(vocabulary_size + 1,
activation='softmax',
name='Final-Output-Dense')(decoder_outputs)
seq2seq_model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
seq2seq_model.compile(optimizer="adam",
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
seq2seq_model.summary()
classes = [item for sublist in train_summary.tolist() for item in sublist]
class_weights = class_weight.compute_class_weight('balanced',
np.unique(classes),
classes)
e_stopping = EarlyStopping(monitor='val_loss',
patience=4,
verbose=1,
mode='min',
restore_best_weights=True)
history = seq2seq_model.fit(x=[train_article, train_summary],
y=np.expand_dims(train_target, -1),
batch_size=batch_size,
epochs=epochs,
validation_split=0.1,
callbacks=[e_stopping],
class_weight=class_weights)
f = open("data/models/convgru_results.txt", "w", encoding="utf-8")
f.write("ConvGRU \n layers: 1 \n latent size: " + str(latent_size) +
"\n vocab size: " + str(vocabulary_size) + "\n")
f.close()
history_dict = history.history
plot_loss(history_dict)
# inference
encoder_model = seq2seq_model.get_layer('Encoder-Model')
decoder_inputs = seq2seq_model.get_layer('Decoder-Input').input
decoder_embeddings = seq2seq_model.get_layer('Decoder-Word-Embedding')(
decoder_inputs)
decoder_embeddings = seq2seq_model.get_layer(
'Decoder-Batch-Normalization-1')(decoder_embeddings)
decoder_conv = seq2seq_model.get_layer('Decoder-Conv1D')(
decoder_embeddings)
decoder_drop = seq2seq_model.get_layer('Decoder-Conv1D-Dropout')(
decoder_conv)
decoder_pool = seq2seq_model.get_layer('Decoder-MaxPool1D')(decoder_drop)
gru_inference_state_input = Input(shape=(latent_size, ),
name='Hidden-State-Input')
gru_out, gru_state_out = seq2seq_model.get_layer('Decoder-GRU')(
[decoder_pool, gru_inference_state_input])
decoder_outputs = seq2seq_model.get_layer('Decoder-Batch-Normalization-2')(
gru_out)
dense_out = seq2seq_model.get_layer('Final-Output-Dense')(decoder_outputs)
decoder_model = Model([decoder_inputs, gru_inference_state_input],
[dense_out, gru_state_out])
return encoder_model, decoder_model
def MobileNetV2(classes=1000,
input_tensor=None,
input_shape=(512, 512, 3),
weights_info=None,
OS=16,
alpha=1.,
include_top=True):
""" Instantiates the Deeplabv3+ architecture
Optionally loads weights pre-trained
on PASCAL VOC or Cityscapes. This model is available for TensorFlow only.
# Arguments
classes: Integer, optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: shape of input image. format HxWxC
PASCAL VOC model was trained on (512,512,3) images. None is allowed as shape/width
weights_info: this dict is consisted of `classes` and `weghts`.
`classes` is number of `weights` output units.
`weights` is one of 'imagenet' (pre-training on ImageNet), 'pascal_voc', 'cityscapes',
original weights path (pre-training on original data) or None (random initialization)
OS: determines input_shape/feature_extractor_output ratio. One of {8,16}.
Used only for xception backbone.
alpha: controls the width of the MobileNetV2 network. This is known as the
width multiplier in the MobileNetV2 paper.
- If `alpha` < 1.0, proportionally decreases the number
of filters in each layer.
- If `alpha` > 1.0, proportionally increases the number
of filters in each layer.
- If `alpha` = 1, default number of filters from the paper
are used at each layer.
Used only for mobilenetv2 backbone. Pretrained is only available for alpha=1.
include_top: Boolean, whether to include the fully-connected
layer at the top of the network. Defaults to `True`.
# Returns
A Keras model instance.
"""
if weights_info is not None:
if weights_info.get("weights") is None:
weights = None
elif weights_info["weights"] in {
'imagenet', 'pascal_voc', 'cityscapes', None
}:
weights = weights_info["weights"]
elif os.path.exists(weights_info["weights"]) and weights_info.get(
"classes") is not None:
classes = int(weights_info["classes"])
weights = weights_info["weights"]
else:
raise ValueError(
'The `weights` should be either '
'`None` (random initialization), `imagenet`, `pascal_voc`, `cityscapes`, '
'original weights path (pre-training on original data), '
'or the path to the weights file to be loaded and'
'`classes` should be number of original weights output units')
else:
weights = None
if classes is None:
raise ValueError('`classes` should be any number')
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
img_input = input_tensor
# If input_shape is None, infer shape from input_tensor
if backend.image_data_format() == 'channels_first':
rows = input_shape[1]
cols = input_shape[2]
else:
rows = input_shape[0]
cols = input_shape[1]
if rows == cols and rows in [96, 128, 160, 192, 224]:
default_size = rows
else:
default_size = 224
if weights == 'imagenet':
if alpha not in [0.35, 0.50, 0.75, 1.0, 1.3, 1.4]:
raise ValueError('If imagenet weights are being loaded, '
'alpha can be one of `0.35`, `0.50`, `0.75`, '
'`1.0`, `1.3` or `1.4` only.')
if rows != cols or rows not in [96, 128, 160, 192, 224]:
rows = 224
OS = 8
first_block_filters = _make_divisible(32 * alpha, 8)
x = Conv2D(first_block_filters,
kernel_size=3,
strides=(2, 2),
padding='same',
use_bias=False,
name='Conv')(img_input)
x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_BN')(x)
x = Activation(relu6, name='Conv_Relu6')(x)
x = _inverted_res_block(x,
filters=16,
alpha=alpha,
stride=1,
expansion=1,
block_id=0,
skip_connection=False)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=2,
expansion=6,
block_id=1,
skip_connection=False)
x = _inverted_res_block(x,
filters=24,
alpha=alpha,
stride=1,
expansion=6,
block_id=2,
skip_connection=True)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=2,
expansion=6,
block_id=3,
skip_connection=False)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=4,
skip_connection=True)
x = _inverted_res_block(x,
filters=32,
alpha=alpha,
stride=1,
expansion=6,
block_id=5,
skip_connection=True)
# stride in block 6 changed from 2 -> 1, so we need to use rate = 2
x = _inverted_res_block(
x,
filters=64,
alpha=alpha,
stride=1, # 1!
expansion=6,
block_id=6,
skip_connection=False)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=7,
skip_connection=True)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=8,
skip_connection=True)
x = _inverted_res_block(x,
filters=64,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=9,
skip_connection=True)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=10,
skip_connection=False)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=11,
skip_connection=True)
x = _inverted_res_block(x,
filters=96,
alpha=alpha,
stride=1,
rate=2,
expansion=6,
block_id=12,
skip_connection=True)
x = _inverted_res_block(
x,
filters=160,
alpha=alpha,
stride=1,
rate=2, # 1!
expansion=6,
block_id=13,
skip_connection=False)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=14,
skip_connection=True)
x = _inverted_res_block(x,
filters=160,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=15,
skip_connection=True)
x = _inverted_res_block(x,
filters=320,
alpha=alpha,
stride=1,
rate=4,
expansion=6,
block_id=16,
skip_connection=False)
if alpha > 1.0:
last_block_filters = _make_divisible(1280 * alpha, 8)
else:
last_block_filters = 1280
x = Conv2D(last_block_filters,
kernel_size=1,
use_bias=False,
name='Conv_1')(x)
x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_1_bn')(x)
x = Activation(relu6, name='out_relu')(x)
x = GlobalAveragePooling2D()(x)
x = Dense(classes, activation='softmax')(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = layer_utils.get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='mobilenetv2')
# Load weights.
if weights == 'imagenet':
print("movilenetv2 load model imagenet")
model_name = ('mobilenet_v2_weights_tf_dim_ordering_tf_kernels_' +
str(alpha) + '_' + str(rows) + '.h5')
weight_path = BASE_WEIGHT_PATH + model_name
weights_path = data_utils.get_file(model_name,
weight_path,
cache_subdir='models')
model.load_weights(weights_path)
elif not (weights in {'pascal_voc', 'cityscapes', None}):
model.load_weights(weights)
if include_top:
return model
else:
# get last _inverted_res_block layer
no_top_model = Model(inputs=model.input,
outputs=model.get_layer(index=-6).output)
return no_top_model
def SENET50(include_top=True, weights='vggface',
input_tensor=None, input_shape=None,
pooling=None,
classes=8631):
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=197,
data_format=K.image_data_format(),
require_flatten=include_top,
weights=weights)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = Conv2D(
64, (7, 7), use_bias=False, strides=(2, 2), padding='same',
name='conv1/7x7_s2')(img_input)
x = BatchNormalization(axis=bn_axis, name='conv1/7x7_s2/bn')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = senet_conv_block(x, 3, [64, 64, 256], stage=2, block=1, strides=(1, 1))
x = senet_identity_block(x, 3, [64, 64, 256], stage=2, block=2)
x = senet_identity_block(x, 3, [64, 64, 256], stage=2, block=3)
x = senet_conv_block(x, 3, [128, 128, 512], stage=3, block=1)
x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=2)
x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=3)
x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=4)
x = senet_conv_block(x, 3, [256, 256, 1024], stage=4, block=1)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=2)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=3)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=4)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=5)
x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=6)
x = senet_conv_block(x, 3, [512, 512, 2048], stage=5, block=1)
x = senet_identity_block(x, 3, [512, 512, 2048], stage=5, block=2)
x = senet_identity_block(x, 3, [512, 512, 2048], stage=5, block=3)
x = AveragePooling2D((7, 7), name='avg_pool')(x)
if include_top:
x = Flatten()(x)
x = Dense(classes, activation='softmax', name='classifier')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='vggface_senet50')
# load weights
if weights == 'vggface':
if include_top:
weights_path = get_file('rcmalli_vggface_tf_senet50.h5',
utils.SENET50_WEIGHTS_PATH,
cache_subdir=utils.VGGFACE_DIR)
else:
weights_path = get_file('rcmalli_vggface_tf_notop_senet50.h5',
utils.SENET50_WEIGHTS_PATH_NO_TOP,
cache_subdir=utils.VGGFACE_DIR)
model.load_weights(weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if include_top:
maxpool = model.get_layer(name='avg_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='classifier')
layer_utils.convert_dense_weights_data_format(dense, shape,
'channels_first')
if K.image_data_format() == 'channels_first' and K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
elif weights is not None:
model.load_weights(weights)
return model
def VGG16(include_top=True, weights='vggface',
input_tensor=None, input_shape=None,
pooling=None,
classes=2622):
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=48,
data_format=K.image_data_format(),
require_flatten=include_top)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
# Block 1
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_1')(
img_input)
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x)
# Block 2
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_1')(
x)
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_2')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2')(x)
# Block 3
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_1')(
x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_2')(
x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_3')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3')(x)
# Block 4
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_1')(
x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_2')(
x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_3')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4')(x)
# Block 5
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_1')(
x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_2')(
x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_3')(
x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5')(x)
if include_top:
# Classification block
x = Flatten(name='flatten')(x)
x = Dense(4096, name='fc6')(x)
x = Activation('relu', name='fc6/relu')(x)
x = Dense(4096, name='fc7')(x)
x = Activation('relu', name='fc7/relu')(x)
x = Dense(classes, name='fc8')(x)
x = Activation('softmax', name='fc8/softmax')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='vggface_vgg16') # load weights
if weights == 'vggface':
if include_top:
weights_path = get_file('rcmalli_vggface_tf_vgg16.h5',
utils.
VGG16_WEIGHTS_PATH,
cache_subdir=utils.VGGFACE_DIR)
else:
weights_path = get_file('rcmalli_vggface_tf_notop_vgg16.h5',
utils.VGG16_WEIGHTS_PATH_NO_TOP,
cache_subdir=utils.VGGFACE_DIR)
model.load_weights(weights_path, by_name=True)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first':
if include_top:
maxpool = model.get_layer(name='pool5')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='fc6')
layer_utils.convert_dense_weights_data_format(dense, shape,
'channels_first')
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
return model
def autoencoder_multitask(dataset, adj, feats, labels, weights=None):
adj = sp.hstack([adj, feats])
h, w = adj.shape
kwargs = dict(
use_bias=True,
kernel_initializer='glorot_normal',
kernel_regularizer=None,
bias_initializer='zeros',
bias_regularizer=None,
trainable=True,
)
data = Input(shape=(w, ), dtype=np.float32, name='data')
### First set of encoding transformation ###
encoded = Dense(256, activation='relu', name='encoded1', **kwargs)(data)
### Second set of encoding transformation ###
encoded = Dense(128, activation='relu', name='encoded2', **kwargs)(encoded)
if dataset == 'pubmed':
encoded = Dropout(rate=0.5, name='drop')(encoded)
else:
encoded = Dropout(rate=0.8, name='drop')(encoded)
# the encoder model maps an input to its encoded representation
encoder = Model([data], encoded)
encoded1 = encoder.get_layer('encoded1')
encoded2 = encoder.get_layer('encoded2')
### First set of decoding transformation ###
decoded = DenseTied(256,
tie_to=encoded2,
transpose=True,
activation='relu',
name='decoded2')(encoded)
### Node classification ###
feat_data = Input(shape=(feats.shape[1], ))
pred1 = Dense(labels.shape[1], activation='linear')(feat_data)
pred2 = Dense(labels.shape[1], activation='linear')(decoded)
prediction = add([pred1, pred2], name='prediction')
### Second set of decoding transformation - reconstruction ###
decoded = DenseTied(w,
tie_to=encoded1,
transpose=True,
activation='linear',
name='decoded1')(decoded)
# compile the autoencoder
adam = optimizers.Adam(lr=0.001, decay=0.0)
autoencoder = Model(inputs=[data, feat_data],
outputs=[decoded, prediction])
autoencoder.compile(optimizer=adam,
loss={
'decoded1': mbce,
'prediction': masked_categorical_crossentropy
},
loss_weights={
'decoded1': 1.0,
'prediction': 1.0
})
if weights is not None:
autoencoder.load_weights(weights)
return encoder, autoencoder
z = Dropout(dropout_prob[1])(z)
z = Dense(hidden_dims, activation="relu")(z)
model_output = Dense(1, activation="sigmoid")(z)
model = Model(model_input, model_output)
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
#print(model.summary())
# Initialize weights with word2vec
weights = np.array([v for v in embedding_weights.values()])
print("Initializing embedding layer with word2vec weights, shape",
weights.shape)
embedding_layer = model.get_layer("embedding")
embedding_layer.set_weights([weights])
#-------------------------------------------------------------------------------------
history = model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=num_epochs,
validation_data=(x_test, y_test),
verbose=2)
epoch_nr = 0
val_loss = 1.00
for epoch in range(0, num_epochs, 1):
def autoencoder(sparse_net, adj, weights=None):
h, w = adj.shape
kwargs = dict(
use_bias=True,
kernel_initializer='glorot_normal',
kernel_regularizer=None,
bias_initializer='zeros',
bias_regularizer=None,
trainable=True,
)
data = Input(shape=(w, ), dtype=np.float32, name='data')
if sparse_net:
# for conflict, metabolic, protein networks
noisy_data = Dropout(rate=0.2, name='drop0')(data)
else:
# for citation, blogcatalog, arxiv-grqc, and powergrid networks
noisy_data = Dropout(rate=0.5, name='drop0')(data)
### First set of encoding transformation ###
encoded = Dense(256, activation='relu', name='encoded1',
**kwargs)(noisy_data)
if sparse_net:
encoded = Lambda(mvn, name='mvn1')(encoded)
encoded = Dropout(rate=0.5, name='drop1')(encoded)
### Second set of encoding transformation ###
encoded = Dense(128, activation='relu', name='encoded2', **kwargs)(encoded)
if sparse_net:
encoded = Lambda(mvn, name='mvn2')(encoded)
encoded = Dropout(rate=0.5, name='drop2')(encoded)
# the encoder model maps an input to its encoded representation
encoder = Model([data], encoded)
encoded1 = encoder.get_layer('encoded1')
encoded2 = encoder.get_layer('encoded2')
### First set of decoding transformation ###
decoded = DenseTied(256,
tie_to=encoded2,
transpose=True,
activation='relu',
name='decoded2')(encoded)
if sparse_net:
decoded = Lambda(mvn, name='mvn3')(decoded)
decoded = Dropout(rate=0.5, name='drop3')(decoded)
### Second set of decoding transformation - reconstruction ###
decoded = DenseTied(w,
tie_to=encoded1,
transpose=True,
activation='linear',
name='decoded1')(decoded)
# compile the autoencoder
adam = optimizers.Adam(lr=0.001, decay=0.0)
autoencoder = Model(inputs=[data], outputs=[decoded])
autoencoder.compile(optimizer=adam, loss=mbce)
if weights is not None:
autoencoder.load_weights(weights)
return encoder, autoencoder
new_lay.set_weights(lay.get_weights())
model = Model(inp, out)
model.summary()
# Define the loss
# Content loss
content_layers = ['block4_conv2']
content_loss_op = tf.Variable(0.0)
coef = 1
for i in range(len(content_layers)):
layer_name = content_layers[i]
content_features = model.get_layer(layer_name).output
content_features = sess.run(content_features, feed_dict = {creation: np.expand_dims(content_img, 0)})
content_features = tf.constant(content_features)
creation_features = model.get_layer(layer_name).output
weight = coef ** i
content_loss_op += weight * utils.mse(content_features, creation_features)
content_loss_op /= float(len(content_layers))
# Style loss
style_layers = [
'block1_conv1', #'block1_conv2',
'block2_conv1', #'block2_conv2',
'block3_conv1', #'block3_conv2', 'block3_conv3',
'block4_conv1', #'block4_conv2', 'block4_conv3',
def ResNet50(require_flatten=False,
weights='imagenet',
input_tensor=None,
input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the ResNet50 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format="channels_last"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
# Arguments
include_top: whether to include the fully-connected
layer at the top of the network.
weights: one of `None` (random initialization)
or "imagenet" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 244)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 197.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and require_flatten and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
input_shape = _obtain_input_shape(input_shape,
default_size=224,
min_size=197,
data_format=K.image_data_format(),
require_flatten=require_flatten)
if input_tensor is None:
img_input = Input(shape=input_shape)
else:
if not K.is_keras_tensor(input_tensor):
img_input = Input(tensor=input_tensor, shape=input_shape)
else:
img_input = input_tensor
if K.image_data_format() == 'channels_last':
bn_axis = 3
else:
bn_axis = 1
x = ZeroPadding2D((3, 3))(img_input)
x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x)
x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = AveragePooling2D((7, 7), name='avg_pool')(x)
if require_flatten:
x = Flatten(name='flatten1')(x)
x = Dense(classes, activation='softmax', name='fc1000')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
# Ensure that the model takes into account
# any potential predecessors of `input_tensor`.
if input_tensor is not None:
inputs = get_source_inputs(input_tensor)
else:
inputs = img_input
# Create model.
model = Model(inputs, x, name='resnet50')
# load weights
if weights == 'imagenet':
if require_flatten:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models',
md5_hash='a7b3fe01876f51b976af0dea6bc144eb')
else:
weights_path = get_file(
'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models',
md5_hash='a268eb855778b3df3c7506639542a6af')
model.load_weights(weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first':
if require_flatten:
maxpool = model.get_layer(name='avg_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='fc1000')
layer_utils.convert_dense_weights_data_format(
dense, shape, 'channels_first')
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
return model
def seq2seq_architecture(latent_size, vocabulary_size, embedding_matrix,
batch_size, epochs, train_article, train_summary,
train_target):
# encoder
encoder_inputs = Input(shape=(None, ), name='Encoder-Input')
encoder_embeddings = Embedding(
vocabulary_size + 1,
300,
weights=[embedding_matrix],
trainable=False,
mask_zero=True,
name='Encoder-Word-Embedding')(encoder_inputs)
encoder_embeddings = BatchNormalization(
name='Encoder-Batch-Normalization')(encoder_embeddings)
_, state_h, state_c = LSTM(latent_size,
return_state=True,
dropout=0.2,
recurrent_dropout=0.2,
name='Encoder-LSTM')(encoder_embeddings)
encoder_states = [state_h, state_c]
encoder_model = Model(inputs=encoder_inputs,
outputs=encoder_states,
name='Encoder-Model')
encoder_outputs = encoder_model(encoder_inputs)
# decoder
decoder_inputs = Input(shape=(None, ), name='Decoder-Input')
decoder_embeddings = Embedding(
vocabulary_size + 1,
300,
weights=[embedding_matrix],
trainable=False,
mask_zero=True,
name='Decoder-Word-Embedding')(decoder_inputs)
decoder_embeddings = BatchNormalization(
name='Decoder-Batch-Normalization-1')(decoder_embeddings)
decoder_lstm = LSTM(latent_size,
return_state=True,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2,
name='Decoder-LSTM')
decoder_lstm_outputs, _, _ = decoder_lstm(decoder_embeddings,
initial_state=encoder_outputs)
decoder_batchnorm = BatchNormalization(
name='Decoder-Batch-Normalization-2')(decoder_lstm_outputs)
decoder_outputs = Dense(vocabulary_size + 1,
activation='softmax',
name='Final-Output-Dense')(decoder_batchnorm)
seq2seq_model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
seq2seq_model.compile(optimizer="adam",
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
seq2seq_model.summary()
classes = [item for sublist in train_summary.tolist() for item in sublist]
class_weights = class_weight.compute_class_weight('balanced',
np.unique(classes),
classes)
e_stopping = EarlyStopping(monitor='val_loss',
patience=4,
verbose=1,
mode='min',
restore_best_weights=True)
history = seq2seq_model.fit(x=[train_article, train_summary],
y=np.expand_dims(train_target, -1),
batch_size=batch_size,
epochs=epochs,
validation_split=0.1,
callbacks=[e_stopping],
class_weight=class_weights)
f = open("data/models/lstm_results.txt", "w", encoding="utf-8")
f.write("LSTM \n layers: 1 \n latent size: " + str(latent_size) +
"\n vocab size: " + str(vocabulary_size) + "\n")
f.close()
history_dict = history.history
plot_loss(history_dict)
# inference
encoder_model = seq2seq_model.get_layer('Encoder-Model')
decoder_inputs = seq2seq_model.get_layer('Decoder-Input').input
decoder_embeddings = seq2seq_model.get_layer('Decoder-Word-Embedding')(
decoder_inputs)
decoder_embeddings = seq2seq_model.get_layer(
'Decoder-Batch-Normalization-1')(decoder_embeddings)
inference_state_h_input = Input(shape=(latent_size, ),
name='Hidden-State-Input')
inference_state_c_input = Input(shape=(latent_size, ),
name='Cell-State-Input')
lstm_out, lstm_state_h_out, lstm_state_c_out = seq2seq_model.get_layer(
'Decoder-LSTM')([
decoder_embeddings, inference_state_h_input,
inference_state_c_input
])
decoder_outputs = seq2seq_model.get_layer('Decoder-Batch-Normalization-2')(
lstm_out)
dense_out = seq2seq_model.get_layer('Final-Output-Dense')(decoder_outputs)
decoder_model = Model(
[decoder_inputs, inference_state_h_input, inference_state_c_input],
[dense_out, lstm_state_h_out, lstm_state_c_out])
return encoder_model, decoder_model
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Project : tql-Python. # @File : mid_layer # @Time : 2019-07-12 16:33 # @Author : yuanjie # @Email : [email protected] # @Software : PyCharm # @Description : """ https://www.tensorflow.org/beta/tutorials/keras/feature_columns """ from tensorflow.python.keras.layers import Input, Embedding, Reshape, Activation from tensorflow.python.keras.models import Model input_model = Input(shape=(1, )) output_store = Embedding(1115, 10, name='store_embedding')(input_model) output_store = Reshape(target_shape=(10, ))(output_store) output_model = Activation('sigmoid')(output_store) model = Model(inputs=input_model, outputs=output_model) model.summary() embed = Model(inputs=model.input, outputs=model.get_layer(index=1).output) # 以这个model的预测值作为输出 embed.predict([[1]])
from fashion.data.data_generatorAGR import DataGeneratorAgro
from fashion.settings import settings
# set gpu
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
os.environ["KERAS_BACKEND"] = "tensorflow"
# gpu growth
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
model = ResNet50(weights='imagenet')
# input = (224,224,3)
LAYER = "avg_pool"
model = Model(inputs=model.input, outputs=model.get_layer(LAYER).output)
pic_paths = glob.glob(settings["agro_path"] + "**/*", recursive=True)
# WARNING WE SLICE SOME VALUES
pic_df = pd.DataFrame({"path":
pic_paths}) #[:len(pic_paths) - (len(pic_paths) % 16)]
data_generatorAgro = DataGeneratorAgro(pic_df, batch_size=128)
predict = model.predict_generator(data_generatorAgro, verbose=1)
np.save("AGRO.npy", predict)
class DEC(object):
def __init__(self, dims, n_clusters=10, alpha=1.0, init='glorot_uniform'):
super(DEC, self).__init__()
self.dims = dims
self.input_dim = dims[0]
self.n_stacks = len(self.dims) - 1
self.n_clusters = n_clusters
self.alpha = alpha
self.encoder = autoencoder(self.dims, init=init)
# prepare DEC model
clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')(
self.encoder.output)
self.model = Model(inputs=self.encoder.input, outputs=clustering_layer)
def load_weights(self, weights): # load weights of DEC model
self.model.load_weights(weights)
def extract_features(self, x):
return self.encoder.predict(x)
def predict(
self,
x): # predict cluster labels using the output of clustering layer
q = self.model.predict(x, verbose=0)
return q.argmax(1)
@staticmethod
def target_distribution(q):
weight = q**2 / q.sum(0)
return (weight.T / weight.sum(1)).T
def compile(self, optimizer='sgd', loss='kld'):
self.model.compile(optimizer=optimizer, loss=loss)
def fit(self,
x,
y=None,
maxiter=2e4,
batch_size=256,
tol=1e-3,
update_interval=140,
save_dir='./results/temp'):
print('Update interval', update_interval)
save_interval = int(x.shape[0] / batch_size) * 5 # 5 epochs
print('Save interval', save_interval)
# Step 1: initialize cluster centers using k-means
t1 = time()
print('Initializing cluster centers with k-means.')
kmeans = KMeans(n_clusters=self.n_clusters, n_init=20)
y_pred = kmeans.fit_predict(self.encoder.predict(x))
y_pred_last = np.copy(y_pred)
self.model.get_layer(name='clustering').set_weights(
[kmeans.cluster_centers_])
# Step 2: deep clustering
# logging file
import csv
logfile = open(save_dir + '/dec_log.csv', 'w')
logwriter = csv.DictWriter(
logfile, fieldnames=['iter', 'acc', 'nmi', 'ari', 'loss'])
logwriter.writeheader()
loss = 0
index = 0
index_array = np.arange(x.shape[0])
for ite in range(int(maxiter)):
if ite % update_interval == 0:
q = self.model.predict(x, verbose=0)
p = self.target_distribution(
q) # update the auxiliary target distribution p
# evaluate the clustering performance
y_pred = q.argmax(1)
if y is not None:
acc = np.round(metrics.acc(y, y_pred), 5)
nmi = np.round(metrics.nmi(y, y_pred), 5)
ari = np.round(metrics.ari(y, y_pred), 5)
loss = np.round(loss, 5)
logdict = dict(iter=ite,
acc=acc,
nmi=nmi,
ari=ari,
loss=loss)
logwriter.writerow(logdict)
print(
'Iter %d: acc = %.5f, nmi = %.5f, ari = %.5f' %
(ite, acc, nmi, ari), ' ; loss=', loss)
# check stop criterion
delta_label = np.sum(y_pred != y_pred_last).astype(
np.float32) / y_pred.shape[0]
y_pred_last = np.copy(y_pred)
if ite > 0 and delta_label < tol:
print('delta_label ', delta_label, '< tol ', tol)
print('Reached tolerance threshold. Stopping training.')
logfile.close()
break
# train on batch
# if index == 0:
# np.random.shuffle(index_array)
idx = index_array[index * batch_size:min((index + 1) *
batch_size, x.shape[0])]
loss = self.model.train_on_batch(x=x[idx], y=p[idx])
index = index + 1 if (index + 1) * batch_size <= x.shape[0] else 0
# save intermediate model
if ite % save_interval == 0:
print('saving model to:',
save_dir + '/DEC_model_' + str(ite) + '.h5')
self.model.save_weights(save_dir + '/DEC_model_' + str(ite) +
'.h5')
ite += 1
# save the trained model
logfile.close()
print('saving model to:', save_dir + '/DEC_model_final.h5')
self.model.save_weights(save_dir + '/DEC_model_final.h5')
return y_pred
def VGG16(include_top=True, weights='imagenet',
input_tensor=None, input_shape=None,
pooling=None,
classes=1000):
"""Instantiates the VGG16 architecture.
Optionally loads weights pre-trained
on ImageNet. Note that when using TensorFlow,
for best performance you should set
`image_data_format="channels_last"` in your Keras config
at ~/.keras/keras.json.
The model and the weights are compatible with both
TensorFlow and Theano. The data format
convention used by the model is the one
specified in your Keras config file.
# Arguments
include_top: whether to include the 3 fully-connected
layers at the top of the network.
weights: one of `None` (random initialization)
or "imagenet" (pre-training on ImageNet).
input_tensor: optional Keras tensor (i.e. output of `layers.Input()`)
to use as image input for the model.
input_shape: optional shape tuple, only to be specified
if `include_top` is False (otherwise the input shape
has to be `(224, 224, 3)` (with `channels_last` data format)
or `(3, 224, 244)` (with `channels_first` data format).
It should have exactly 3 inputs channels,
and width and height should be no smaller than 48.
E.g. `(200, 200, 3)` would be one valid value.
pooling: Optional pooling mode for feature extraction
when `include_top` is `False`.
- `None` means that the output of the model will be
the 4D tensor output of the
last convolutional layer.
- `avg` means that global average pooling
will be applied to the output of the
last convolutional layer, and thus
the output of the model will be a 2D tensor.
- `max` means that global max pooling will
be applied.
classes: optional number of classes to classify images
into, only to be specified if `include_top` is True, and
if no `weights` argument is specified.
# Returns
A Keras model instance.
# Raises
ValueError: in case of invalid argument for `weights`,
or invalid input shape.
"""
if weights not in {'imagenet', None}:
raise ValueError('The `weights` argument should be either '
'`None` (random initialization) or `imagenet` '
'(pre-training on ImageNet).')
if weights == 'imagenet' and include_top and classes != 1000:
raise ValueError('If using `weights` as imagenet with `include_top`'
' as true, `classes` should be 1000')
# Determine proper input shape
img_input = Input(tensor=input_tensor, shape=input_shape)
# Block 1
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input)
x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x)
x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x)
x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Block 5
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x)
x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x)
if include_top:
# Classification block
x = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dense(classes, activation='softmax', name='predictions')(x)
else:
if pooling == 'avg':
x = GlobalAveragePooling2D()(x)
elif pooling == 'max':
x = GlobalMaxPooling2D()(x)
inputs = img_input
# Create model.
model = Model(inputs, x, name='vgg16')
# load weights
if weights == 'imagenet':
if include_top:
weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels.h5',
WEIGHTS_PATH,
cache_subdir='models')
else:
weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5',
WEIGHTS_PATH_NO_TOP,
cache_subdir='models')
model.load_weights(weights_path)
if K.backend() == 'theano':
layer_utils.convert_all_kernels_in_model(model)
if K.image_data_format() == 'channels_first':
if include_top:
maxpool = model.get_layer(name='block5_pool')
shape = maxpool.output_shape[1:]
dense = model.get_layer(name='fc1')
layer_utils.convert_dense_weights_data_format(dense, shape, 'channels_first')
if K.backend() == 'tensorflow':
warnings.warn('You are using the TensorFlow backend, yet you '
'are using the Theano '
'image data format convention '
'(`image_data_format="channels_first"`). '
'For best performance, set '
'`image_data_format="channels_last"` in '
'your Keras config '
'at ~/.keras/keras.json.')
return model
