def boosting_rank_net(input_shape, hns=[8, 6, 4, 4], classes=2): """ """ res_model = boosting_res_net((input_shape[1], ), hns, out_layer_name='proba') res_model = Model(res_model.input, res_model.get_layer('pre_sigmoid').output) inputs = Input(input_shape) minor_inputs = Lambda(lambda x: x[:, 0], name='minor_input')(inputs) pred_minor = res_model(minor_inputs) minor_out_proba = Lambda(lambda x: x, name='minor_out_proba')(pred_minor) major_inputs = Lambda(lambda x: x[:, 1], name='major_input')(inputs) pred_major = res_model(major_inputs) major_out_proba = Lambda(lambda x: x, name='major_out_proba')(pred_major) sub = Subtract()([major_out_proba, minor_out_proba]) sub = Lambda(lambda x: x * RANK_SCALE, name='rank_scale_layer')(sub) proba = Activation('sigmoid')(sub) model = Model(inputs, proba) model.compile(optimizer=Nadam(lr=0.001), loss=min_pred) return model
def layer_test_helper_merge_2d(layer, channel_index, data_format): # This should test that the output is the correct shape so it should pass # into a Dense layer rather than a Conv layer. # The weighted layer is the previous layer, # Create model input_shape = list(random.randint(10, 20, size=3)) input_1 = Input(shape=input_shape) input_2 = Input(shape=input_shape) x = Conv2D(3, [3, 3], data_format=data_format, name='conv_1')(input_1) y = Conv2D(3, [3, 3], data_format=data_format, name='conv_2')(input_2) x = layer([x, y]) x = Flatten()(x) main_output = Dense(5, name='dense_1')(x) model = Model(inputs=[input_1, input_2], outputs=main_output) # Delete channels del_layer = model.get_layer('conv_1') del_layer_2 = model.get_layer('conv_2') surgeon = Surgeon(model) surgeon.add_job('delete_channels', del_layer, channels=channel_index) surgeon.add_job('delete_channels', del_layer_2, channels=channel_index) new_model = surgeon.operate() new_w = new_model.get_layer('dense_1').get_weights() # Calculate next layer's correct weights flat_sz = np.prod(layer.get_output_shape_at(0)[1:]) channel_count = getattr(del_layer, utils.get_channels_attr(del_layer)) channel_index = [i % channel_count for i in channel_index] if data_format == 'channels_first': delete_indices = [ x * flat_sz // channel_count + i for x in channel_index for i in range( 0, flat_sz // channel_count, ) ] elif data_format == 'channels_last': delete_indices = [ x + i for i in range(0, flat_sz, channel_count) for x in channel_index ] else: raise ValueError correct_w = model.get_layer('dense_1').get_weights() correct_w[0] = np.delete(correct_w[0], delete_indices, axis=0) assert weights_equal(correct_w, new_w)
def create_hooknet_encoder(hooknet: Model): """Creates a new Keras model with bottleneck of the target branch as ouput after combining the context features Args: hooknet (Model): HookNet Model Returns: Model: return new model with encoding as output """ encoding_layer = hooknet.get_layer("target-branchbottle").output return Model(hooknet.inputs, encoding_layer)
def _load_model(self, model='VGG'): if model=='VGG': print("Loading VGG19 pre-trained model...") base_model = VGG19(weights='imagenet') base_model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output) x = base_model.output x = GlobalAveragePooling2D()(x) self.VGG_model = Model(inputs=base_model.input, outputs=x) if model=='Inception_Resnet': #load Inception_Resnet model print("Loading Inception_Resnet_V2 pre-trained model...") base_model = InceptionResNetV2(weights='imagenet', include_top=False) x = base_model.output x = GlobalAveragePooling2D()(x) self.IR_model = Model(inputs=base_model.input, outputs=x)
def create_model(sequence_length): # Build model input_shape = (sequence_length, ) model_input = Input(shape=input_shape) z = Embedding(len(vocabulary_inv), embedding_dim, input_length=sequence_length, name="embedding")(model_input) z = Dropout(dropout_prob[0])(z) # Convolutional block conv_blocks = [] for sz in filter_sizes: conv = Convolution1D(filters=num_filters, kernel_size=sz, padding="valid", activation="relu", strides=1)(z) conv = MaxPooling1D(pool_size=2)(conv) conv = Flatten()(conv) conv_blocks.append(conv) z = Concatenate()(conv_blocks) if len(conv_blocks) > 1 else conv_blocks[0] z = Dropout(dropout_prob[1])(z) z = Dense(hidden_dims, activation="relu")(z) model_output = Dense(1, activation="sigmoid")(z) model = Model(model_input, model_output) model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) # Initialize weights with word2vec weights = np.array([v for v in embedding_weights.values()]) print("Initializing embedding layer with word2vec weights, shape", weights.shape) embedding_layer = model.get_layer("embedding") embedding_layer.set_weights([weights]) print(model.summary()) return model
def get(self, request): id = request.query_params['id'] image_path = 'media/' + id print(image_path) # load VGG19 model print("Loading VGG19 pre-trained model...") base_model = VGG19(weights='imagenet') base_model = Model(inputs=base_model.input, outputs=base_model.get_layer('block4_pool').output) x = base_model.output x = GlobalAveragePooling2D()(x) VGG_model = Model(inputs=base_model.input, outputs=x) img_VGG = image.load_img(image_path, target_size=(224, 224)) img = image.img_to_array(img_VGG) # convert to array img = np.expand_dims(img, axis=0) img = ppVGG19(img) features = VGG_model.predict(img).flatten() point_a = features.reshape(1, 512) filename = [] score = [] for i in os.listdir('data/features'): point_b = np.load('data/features/' + i).reshape(1, 512) filename.append(i) distance = np.linalg.norm(point_a - point_b) cos_lib = cosine_similarity(point_a, point_b) score.append(cos_lib[0][0]) ind = score.index((max(score))) print(ind) print(filename[ind]) product = products.objects.get(obj_id=filename[ind]) serialier = productsSerializer(product) return Response(serialier.data)
def autoencoder_with_node_features(dataset, adj, feats, weights=None): aug_adj = sp.hstack([adj, feats]) h, w = aug_adj.shape kwargs = dict( use_bias=True, kernel_initializer='glorot_normal', kernel_regularizer=None, bias_initializer='zeros', bias_regularizer=None, trainable=True, ) data = Input(shape=(w, ), dtype=np.float32, name='data') if dataset in ['protein', 'cora', 'citeseer', 'pubmed']: # dropout 0.5 is needed for protein and citation (large nets) noisy_data = Dropout(rate=0.5, name='drop1')(data) else: # dropout 0.2 is needed for conflict and metabolic (small nets) noisy_data = Dropout(rate=0.2, name='drop1')(data) ### First set of encoding transformation ### encoded = Dense(256, activation='relu', name='encoded1', **kwargs)(noisy_data) encoded = Lambda(mvn, name='mvn1')(encoded) ### Second set of encoding transformation ### encoded = Dense(128, activation='relu', name='encoded2', **kwargs)(encoded) encoded = Lambda(mvn, name='mvn2')(encoded) encoded = Dropout(rate=0.5, name='drop2')(encoded) # the encoder model maps an input to its encoded representation encoder = Model([data], encoded) encoded1 = encoder.get_layer('encoded1') encoded2 = encoder.get_layer('encoded2') ### First set of decoding transformation ### decoded = DenseTied(256, tie_to=encoded2, transpose=True, activation='relu', name='decoded2')(encoded) decoded = Lambda(mvn, name='mvn3')(decoded) ### Second set of decoding transformation - reconstruction ### decoded = DenseTied(w, tie_to=encoded1, transpose=True, activation='linear', name='decoded1')(decoded) # compile the autoencoder adam = optimizers.Adam(lr=0.001, decay=0.0) if dataset in ['metabolic', 'conflict']: # datasets with mixture of binary and real features decoded_feats = Lambda(lambda x: x[:, adj.shape[1]:], name='decoded_feats')(decoded) decoded = Lambda(lambda x: x[:, :adj.shape[1]], name='decoded')(decoded) autoencoder = Model(inputs=[data], outputs=[decoded, decoded_feats]) autoencoder.compile(optimizer=adam, loss={ 'decoded': mbce, 'decoded_feats': ce }, loss_weights={ 'decoded': 1.0, 'decoded_feats': 1.0 }) else: # datasets with only binary graph and node features autoencoder = Model(inputs=[data], outputs=decoded) autoencoder.compile(optimizer=adam, loss=mbce) if weights is not None: autoencoder.load_weights(weights) return encoder, autoencoder
class CI_Model(gx_model): def __init__(self, old_new): super(CI_Model, self).__init__() self.pairwise_model = None self.predict_model = None self.old_new = old_new # 新模型还是旧模型 self.CI_handle_slt_apis_mode = new_Para.param.CI_handle_slt_apis_mode self.lr = new_Para.param.CI_learning_rate # 内容部分学习率 self.optimizer = Adam(lr=self.lr) self.set_simple_name() self.set_paths() def set_simple_name(self): self.simple_name = 'new_func_' if self.old_new == 'new' else 'old_func_' # 新情景,只用功能 self.simple_name += new_Para.param.text_extracter_mode if new_Para.param.text_extracter_mode == 'inception': self.simple_name += '_' self.simple_name += new_Para.param.inception_pooling if not new_Para.param.if_inception_MLP: self.simple_name += '_NO_extract_MLP' if self.old_new == 'new': if not self.CI_handle_slt_apis_mode: self.simple_name += '_noSlt' else: self.simple_name += ('_' + self.CI_handle_slt_apis_mode ) # 处理方式是全连接还是attention self.simple_name += '_{}'.format(self.lr) # 学习率 if self.old_new == 'new': self.simple_name += '_' + new_Para.param.final_activation # + '_New' # 新模型可选sigmoid和softmax # self.simple_name += '_2' # self.simple_name += '_newPartTarget' # !!!临时路径!!!效果不好,不用 # self.simple_name += '_reductData' # !!! def set_paths(self): # 路径设置 self.model_dir = dataset.crt_ds.model_path.format( self.get_simple_name()) # 模型路径 if not os.path.exists(self.model_dir): os.makedirs(self.model_dir) self.model_name_path = os.path.join(self.model_dir, 'model_name.dat') self.CI_features_path = os.path.join(self.model_dir, 'CI_features.fea') self.train_slt_apis_mid_features_path = os.path.join( self.model_dir, 'train_slt_apis_mid_features.csv') self.test_slt_apis_mid_features_path = os.path.join( self.model_dir, 'test_slt_apis_mid_features.csv') self.ma_text_tag_feas_path = os.path.join( self.model_dir, 'mashup_api_text_tag_feas.dat') # mashup和api的提取的文本特征 def get_simple_name(self): return self.simple_name def get_name(self): """ self.model_name区分模型框架,返回的name用于记录在evalute中,区分不同的模型,架构 :return: """ return self.model_name def set_attribute(self): self.all_mashup_num = meta_data.mashup_num self.all_api_num = meta_data.api_num self.his_m_ids = dataset.crt_ds.his_mashup_ids def set_embedding_matrixs(self): # 把每个mashup和api的文本和tag编码,得到四个矩阵,输入id即可映射到这些embedding上,节省get_instaces部分的内存 if self.encoded_texts is None: self.process_texts() # 首先对数据padding,编码处理 if self.encoded_tags is None: self.process_tags() # text self.mid2text_wordindex = np.array( self.encoded_texts.get_texts_in_index(range(self.all_mashup_num), 'keras_setting', 0)) self.aid2text_wordindex = np.array( self.encoded_texts.get_texts_in_index(range(self.all_api_num), 'keras_setting', self.all_mashup_num)) # tag self.mid2tag_wordindex = np.array( self.encoded_tags.get_texts_in_index(range(self.all_mashup_num), 'keras_setting', 0)) self.aid2tag_wordindex = np.array( self.encoded_tags.get_texts_in_index(range(self.all_api_num), 'keras_setting', self.all_mashup_num)) # instances中slt apis不满足数目时,需要用value(api_num)填充,最终全部映射为0 self.aid2text_wordindex = np.vstack( (self.aid2text_wordindex, np.zeros((1, new_Para.param.MAX_SEQUENCE_LENGTH), dtype=int))).astype(int) self.aid2tag_wordindex = np.vstack( (self.aid2tag_wordindex, np.zeros((1, new_Para.param.MAX_SEQUENCE_LENGTH), dtype=int))).astype(int) # print('mid2text_wordindex[0]:',self.mid2text_wordindex[0]) # print('mid2tag_wordindex[0:5]:', self.mid2tag_wordindex[0:5]) # print('aid2text_wordindex[0]:',self.aid2text_wordindex[0]) # print('aid2tag_wordindex[0:5]:', self.aid2tag_wordindex[0:5]) def set_embedding_layers(self): # embedding # using the embedding layer instead of packing in the instance, to save memory self.mid2text_embedding_layer = Embedding( self.all_mashup_num, new_Para.param.MAX_SEQUENCE_LENGTH, embeddings_initializer=Constant(self.mid2text_wordindex), mask_zero=False, input_length=1, trainable=False, name='mashup_text_encoding_embedding_layer') self.aid2text_embedding_layer = Embedding( self.all_api_num + 1, new_Para.param.MAX_SEQUENCE_LENGTH, embeddings_initializer=Constant(self.aid2text_wordindex), mask_zero=False, trainable=False, name='api_text_encoding_embedding_layer') self.mid2tag_embedding_layer = Embedding( self.all_mashup_num, new_Para.param.MAX_SEQUENCE_LENGTH, embeddings_initializer=Constant(self.mid2tag_wordindex), mask_zero=False, input_length=1, trainable=False, name='mashup_tag_encoding_embedding_layer') self.aid2tag_embedding_layer = Embedding( self.all_api_num + 1, new_Para.param.MAX_SEQUENCE_LENGTH, embeddings_initializer=Constant(self.aid2tag_wordindex), mask_zero=False, trainable=False, name='api_tag_encoding_embedding_layer') self.user_text_feature_extracter = None self.item_text_feature_extracter = None self.user_tag_feature_extracter = None self.item_tag_feature_extracter = None def prepare(self): self.set_attribute() self.set_embedding_matrixs() self.set_embedding_layers() def user_text_feature_extractor(self): if not self.user_text_feature_extracter: user_id_input = Input(shape=(1, ), dtype='int32') # , name='mashup_id_input' user_text_input = Lambda( lambda x: tf.cast(tf.squeeze(x, axis=1), 'int32'))( self.mid2text_embedding_layer(user_id_input)) user_text_vec = self.feature_extracter_from_texts()( user_text_input) # (?,50) self.user_text_feature_extracter = Model( user_id_input, user_text_vec, name='user_text_feature_extracter') return self.user_text_feature_extracter # mashup和api的文本特征提取器,区别在于ID到文本编码的embedding矩阵不同,但又要公用相同的word_embedding层和inception特征提取器 def item_text_feature_extractor(self): if not self.item_text_feature_extracter: item_id_input = Input(shape=(1, ), dtype='int32') # , name='api_id_input' item_text_input = Lambda( lambda x: tf.cast(tf.squeeze(x, axis=1), 'int32'))( self.aid2text_embedding_layer(item_id_input)) item_text_vec = self.feature_extracter_from_texts()( item_text_input) # (?,50) self.item_text_feature_extracter = Model( item_id_input, item_text_vec, name='item_text_feature_extracter') return self.item_text_feature_extracter def user_tag_feature_extractor(self): if not self.user_tag_feature_extracter: user_id_input = Input(shape=(1, ), dtype='int32') # , name='user_id_input' user_tag_input = Lambda( lambda x: tf.cast(tf.squeeze(x, axis=1), 'int32'))( self.mid2tag_embedding_layer(user_id_input)) user_tag_embedding = self.get_text_embedding_layer()( user_tag_input) user_tag_vec = Lambda(lambda x: tf.squeeze(x, axis=1))( SequencePoolingLayer( 'mean', supports_masking=True)(user_tag_embedding)) self.user_tag_feature_extracter = Model( user_id_input, user_tag_vec, name='user_tag_feature_extracter') return self.user_tag_feature_extracter def item_tag_feature_extractor(self): if not self.item_tag_feature_extracter: item_id_input = Input(shape=(1, ), dtype='int32') # , name='api_id_input' item_tag_input = Lambda( lambda x: tf.cast(tf.squeeze(x, axis=1), 'int32'))( self.aid2tag_embedding_layer(item_id_input)) item_tag_embedding = self.get_tag_embedding_layer()(item_tag_input) item_tag_vec = Lambda(lambda x: tf.squeeze(x, axis=1))( SequencePoolingLayer( 'mean', supports_masking=True)(item_tag_embedding)) self.item_tag_feature_extracter = Model( item_id_input, item_tag_vec, name='item_tag_feature_extracter') return self.item_tag_feature_extracter def get_model(self): if not self.model: mashup_id_input = Input(shape=(1, ), dtype='int32', name='mashup_id_input') api_id_input = Input(shape=(1, ), dtype='int32', name='api_id_input') inputs = [mashup_id_input, api_id_input] user_text_vec = self.user_text_feature_extractor()(mashup_id_input) item_text_vec = self.item_text_feature_extractor()(api_id_input) user_tag_vec = self.user_tag_feature_extractor()(mashup_id_input) item_tag_vec = self.item_tag_feature_extractor()(api_id_input) feature_list = [ user_text_vec, item_text_vec, user_tag_vec, item_tag_vec ] if self.old_new == 'LR_PNCF': # 旧场景,使用GMF形式的双塔模型 x = Concatenate(name='user_concatenate')( [user_text_vec, user_tag_vec]) y = Concatenate(name='item_concatenate')( [item_text_vec, item_tag_vec]) output = Multiply()([x, y]) predict_result = Dense(1, activation='sigmoid', use_bias=False, kernel_initializer='lecun_uniform', name="prediction")(output) # 参数学习权重,非线性 self.model = Model(inputs=inputs, outputs=[predict_result], name='predict_model') return self.model elif self.old_new == 'new' and self.CI_handle_slt_apis_mode: # 已选择的服务 mashup_slt_apis_input = Input( shape=(new_Para.param.slt_item_num, ), dtype='int32', name='slt_api_ids_input') mashup_slt_apis_input_3D = Reshape( (new_Para.param.slt_item_num, 1))(mashup_slt_apis_input) # mashup_slt_apis_num_input = Input(shape=(1,), dtype='int32', name='mashup_slt_apis_num_input') inputs.append(mashup_slt_apis_input) mask = Lambda(lambda x: K.not_equal(x, self.all_api_num))( mashup_slt_apis_input) # (?, 3) !!! # 已选择的服务直接复用item_feature_extractor slt_text_vec_list, slt_tag_vec_list = [], [] for i in range(new_Para.param.slt_item_num): x = Lambda(slice, arguments={'index': i})( mashup_slt_apis_input_3D) # (?,1,1) x = Reshape((1, ))(x) temp_item_text_vec = self.item_text_feature_extractor()(x) temp_item_tag_vec = self.item_tag_feature_extractor()(x) slt_text_vec_list.append(temp_item_text_vec) slt_tag_vec_list.append(temp_item_tag_vec) if self.CI_handle_slt_apis_mode in ('attention', 'average'): # text和tag使用各自的attention block slt_text_vec_list = [ Reshape((1, new_Para.param.embedding_dim))(key_2D) for key_2D in slt_text_vec_list ] slt_tag_vec_list = [ Reshape((1, new_Para.param.embedding_dim))(key_2D) for key_2D in slt_tag_vec_list ] # 增加了一维 eg:[None,50]->[None,1,50] text_keys_embs = Concatenate(axis=1)( slt_text_vec_list) # [?,3,50] tag_keys_embs = Concatenate(axis=1)( slt_tag_vec_list) # [?,3,50] if self.CI_handle_slt_apis_mode == 'attention': query_item_text_vec = Lambda( lambda x: tf.expand_dims(x, axis=1))( item_text_vec) # (?, 50)->(?, 1, 50) query_item_tag_vec = Lambda( lambda x: tf.expand_dims(x, axis=1))(item_tag_vec) # 压缩历史,得到向量 text_hist = AttentionSequencePoolingLayer( supports_masking=True)( [query_item_text_vec, text_keys_embs], mask=mask) tag_hist = AttentionSequencePoolingLayer( supports_masking=True)( [query_item_tag_vec, tag_keys_embs], mask=mask) else: # 'average' text_hist = SequencePoolingLayer( 'mean', supports_masking=True)(text_keys_embs, mask=mask) tag_hist = SequencePoolingLayer('mean', supports_masking=True)( tag_keys_embs, mask=mask) text_hist = Lambda(lambda x: tf.squeeze(x, axis=1))( text_hist) # (?, 1, 50)->(?, 50) tag_hist = Lambda(lambda x: tf.squeeze(x, axis=1))( tag_hist) elif self.CI_handle_slt_apis_mode == 'full_concate': text_hist = Concatenate(axis=1)( slt_text_vec_list) # [?,150] tag_hist = Concatenate(axis=1)(slt_tag_vec_list) # [?,150] else: raise ValueError('wrong CI_handle_slt_apis_mode!') feature_list.extend([text_hist, tag_hist]) else: # 包括新模型不处理已选择服务和旧模型 pass feature_list = list(map(NoMask(), feature_list)) # DNN不支持mak,所以不能再传递mask all_features = Concatenate( name='all_content_concatenate')(feature_list) output = DNN(self.content_fc_unit_nums[:-1])(all_features) output = Dense(self.content_fc_unit_nums[-1], activation='relu', kernel_regularizer=l2(new_Para.param.l2_reg), name='text_tag_feature_extracter')(output) # 输出层 if new_Para.param.final_activation == 'softmax': predict_result = Dense(2, activation='softmax', name="prediction")(output) elif new_Para.param.final_activation == 'sigmoid': predict_result = Dense(1, activation='sigmoid', kernel_initializer='lecun_uniform', name="prediction")(output) # Model # if self.IfUniteNI: # inputs.append(user_NI_input) self.model = Model(inputs=inputs, outputs=[predict_result], name='predict_model') for layer in self.model.layers: print(layer.name) print('built CI model, done!') return self.model def show_text_tag_features(self, train_data, show_num=10): """ 检查生成的mashup和api的text和tag的特征是否正常 """ if self.old_new == 'old': m_ids, a_ids = train_data[:-1] instances_tuple = self.get_instances(m_ids[:show_num], a_ids[:show_num]) elif self.old_new == 'new': m_ids, a_ids, slt_a_ids = train_data[:-1] instances_tuple = self.get_instances(m_ids[:show_num], a_ids[:show_num], slt_a_ids[:show_num]) text_tag_middle_model = Model( inputs=[*self.model.inputs], outputs=[ *self.model.get_layer('all_content_concatenate').input[:4] ]) mashup_text_features, apis_text_features, mashup_tag_features, apis_tag_features = text_tag_middle_model.predict( [*instances_tuple], verbose=0) mashup_text_features_path = os.path.join(self.model_dir, 'mashup_text_features.dat') apis_text_features_path = os.path.join(self.model_dir, 'apis_text_features.dat') mashup_tag_features_path = os.path.join(self.model_dir, 'mashup_tag_features.dat') apis_tag_features_path = os.path.join(self.model_dir, 'apis_tag_features.dat') save_2D_list(mashup_text_features_path, mashup_text_features, 'a+') save_2D_list(apis_text_features_path, apis_text_features, 'a+') save_2D_list(mashup_tag_features_path, mashup_tag_features, 'a+') save_2D_list(apis_tag_features_path, apis_tag_features, 'a+') def get_pairwise_model(self): if self.pairwise_model is None: if new_Para.param.pairwise and self.model is not None: # 如果使用pairwise型的目标函数 mashup_id_input = Input(shape=(1, ), dtype='int32', name='mashup_id_input') api_id_input = Input(shape=(1, ), dtype='int32', name='api_id_input') neg_api_id_input = Input(shape=(1, ), dtype='int32', name='neg_api_id_input') mashup_slt_apis_input = Input( shape=(new_Para.param.slt_item_num, ), dtype='int32', name='slt_api_ids_input') if self.old_new == 'new': pos_ratings = self.model( [mashup_id_input, api_id_input, mashup_slt_apis_input]) neg_ratings = self.model([ mashup_id_input, neg_api_id_input, mashup_slt_apis_input ]) # 再加一个负例api id elif self.old_new == 'old': pos_ratings = self.model([mashup_id_input, api_id_input]) neg_ratings = self.model( [mashup_id_input, neg_api_id_input]) loss = Lambda( lambda x: K.relu(new_Para.param.margin + x[0] - x[1]), name='sub_result')([neg_ratings, pos_ratings]) # 注意输入格式! if self.old_new == 'new': self.pairwise_model = Model(inputs=[ mashup_id_input, api_id_input, mashup_slt_apis_input, neg_api_id_input ], outputs=loss) elif self.old_new == 'old': self.pairwise_model = Model(inputs=[ mashup_id_input, api_id_input, neg_api_id_input ], outputs=loss) for layer in self.pairwise_model.layers: print(layer.name) # # 复用的是同一个对象! # print(self.pairwise_model.get_layer('predict_model'),id(self.pairwise_model.get_layer('predict_model'))) # print(self.model,id(self.model)) return self.pairwise_model def get_instances(self, mashup_id_instances, api_id_instances, slt_api_ids_instances=None, neg_api_id_instances=None, if_Train=False, test_phase_flag=True): """ 生成该模型需要的样本 slt_api_ids_instances是每个样本中,已经选择的api的id序列 变长二维序列 train和test样例都可用 但是针对一维列表形式,所以测试时先需拆分(text的数据是二维列表)!!! :param args: :return: """ examples = [np.array(mashup_id_instances), np.array(api_id_instances)] # if new_Para.param.need_slt_apis and slt_api_ids_instances: # 是否加入slt_api_ids_instances if self.old_new == 'new' and self.CI_handle_slt_apis_mode: # 根据模型变化调整决定,同时输入的数据本身也是对应的 # 节省内存版, 不够slt_item_num的要padding instance_num = len(slt_api_ids_instances) padded_slt_api_instances = np.ones( (instance_num, new_Para.param.slt_item_num)) * self.all_api_num for index1 in range(instance_num): a_slt_api_ids = slt_api_ids_instances[index1] for index2 in range(len(a_slt_api_ids)): padded_slt_api_instances[index1][index2] = a_slt_api_ids[ index2] examples.append(padded_slt_api_instances) if new_Para.param.pairwise and if_Train: # pair test时,不需要neg_api_id_instances examples.append(np.array(neg_api_id_instances)) examples_tuples = tuple(examples) return examples_tuples def get_mashup_api_features(self, mashup_num, api_num): """ 得到每个mashup和api经过特征提取器或者平均池化得到的特征,可以直接用id索引,供构造instance的文本部分使用 :param text_tag_recommend_model: :param mashup_num: :param api_num: :return: """ if os.path.exists(self.ma_text_tag_feas_path): with open(self.ma_text_tag_feas_path, 'rb') as f1: mashup_texts_features, mashup_tag_features, api_texts_features, api_tag_features = pickle.load( f1) else: # 前四个分别是 user_text_vec, item_text_vec, user_tag_vec, item_tag_vec text_tag_middle_model = Model( inputs=[*self.model.inputs[:2]], outputs=[ self.model.get_layer('all_content_concatenate').input[0], self.model.get_layer('all_content_concatenate').input[1], self.model.get_layer('all_content_concatenate').input[2], self.model.get_layer('all_content_concatenate').input[3] ]) feature_mashup_ids = list( np.unique([m_id for m_id in range(mashup_num)])) feature_instances_tuple = self.get_instances( feature_mashup_ids, [0] * len(feature_mashup_ids)) mashup_texts_features, _1, mashup_tag_features, _2 = text_tag_middle_model.predict( [*feature_instances_tuple], verbose=0) feature_api_ids = list(np.unique([a_id for a_id in range(api_num)])) feature_instances_tuple = self.get_instances( [0] * len(feature_api_ids), feature_api_ids) _1, api_texts_features, _2, api_tag_features = text_tag_middle_model.predict( [*feature_instances_tuple], verbose=0) with open(self.ma_text_tag_feas_path, 'wb') as f2: pickle.dump((mashup_texts_features, mashup_tag_features, api_texts_features, api_tag_features), f2) return mashup_texts_features, mashup_tag_features, api_texts_features, api_tag_features
class MakeMovie(BaseCommand): def __init__(self): self.deg_to_rad = math.pi / 180.0 def parse_args(self, args): parser = argparse.ArgumentParser(prog='makemovie') parser.add_argument('--tub', help='The tub to make movie from') parser.add_argument( '--out', default='tub_movie.mp4', help='The movie filename to create. default: tub_movie.mp4') parser.add_argument( '--config', default='./config.py', help='location of config file to use. default: ./config.py') parser.add_argument('--model', help='the model to use to show control outputs') parser.add_argument('--type', help='the model type to load') parser.add_argument( '--salient', action="store_true", help='should we overlay salient map showing avtivations') parser.add_argument('--start', type=int, default=1, help='first frame to process') parser.add_argument('--end', type=int, default=-1, help='last frame to process') parser.add_argument('--scale', type=int, default=2, help='make image frame output larger by X mult') parsed_args = parser.parse_args(args) return parsed_args, parser def run(self, args): ''' Load the images from a tub and create a movie from them. Movie ''' import moviepy.editor as mpy args, parser = self.parse_args(args) if args.tub is None: print("ERR>> --tub argument missing.") parser.print_help() return if args.model is not None and args.type is None: print("ERR>> --type argument missing.") parser.print_help() return if args.salient: if args.model is None in args.model: print( "ERR>> salient visualization requires a model. Pass with the --model arg." ) parser.print_help() return # imported like this, we make TF conditional on use of --salient # and we keep the context maintained throughout our callbacks to # compute the salient mask from tensorflow.python.keras import backend as K import tensorflow as tf os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' conf = os.path.expanduser(args.config) if not os.path.exists(conf): print("No config file at location: %s. Add --config to specify\ location or run from dir containing config.py." % conf) return self.cfg = dk.load_config(conf) self.tub = Tub(args.tub) self.index = self.tub.get_index(shuffled=False) start = args.start self.end = args.end if args.end != -1 else len(self.index) if self.end >= len(self.index): self.end = len(self.index) - 1 num_frames = self.end - start self.iRec = start self.scale = args.scale self.keras_part = None self.convolution_part = None if not args.model is None: self.keras_part = get_model_by_type(args.type, cfg=self.cfg) self.keras_part.load(args.model) self.keras_part.compile() if args.salient: self.init_salient(self.keras_part.model) #This method nested in this way to take the conditional import of TF #in a manner that extends to this callback. Done this way, we avoid #importing in the below method, which triggers a new cuda device allocation #each call. def compute_visualisation_mask(img): #from https://github.com/ermolenkodev/keras-salient-object-visualisation activations = self.functor([np.array([img])]) activations = [ np.reshape( img, (1, img.shape[0], img.shape[1], img.shape[2])) ] + activations upscaled_activation = np.ones((3, 6)) for layer in [5, 4, 3, 2, 1]: averaged_activation = np.mean( activations[layer], axis=3).squeeze(axis=0) * upscaled_activation output_shape = (activations[layer - 1].shape[1], activations[layer - 1].shape[2]) x = tf.constant( np.reshape(averaged_activation, (1, averaged_activation.shape[0], averaged_activation.shape[1], 1)), tf.float32) conv = tf.nn.conv2d_transpose( x, self.layers_kernels[layer], output_shape=(1, output_shape[0], output_shape[1], 1), strides=self.layers_strides[layer], padding='VALID') with tf.Session() as session: result = session.run(conv) upscaled_activation = np.reshape(result, output_shape) final_visualisation_mask = upscaled_activation return (final_visualisation_mask - np.min(final_visualisation_mask)) / ( np.max(final_visualisation_mask) - np.min(final_visualisation_mask)) self.compute_visualisation_mask = compute_visualisation_mask print('making movie', args.out, 'from', num_frames, 'images') clip = mpy.VideoClip(self.make_frame, duration=((num_frames - 1) / self.cfg.DRIVE_LOOP_HZ)) clip.write_videofile(args.out, fps=self.cfg.DRIVE_LOOP_HZ) def draw_model_prediction(self, record, img): ''' query the model for it's prediction, draw the user input and the predictions as green and blue lines on the image ''' if self.keras_part is None: return import cv2 user_angle = float(record["user/angle"]) user_throttle = float(record["user/throttle"]) expected = self.keras_part.model.inputs[0].shape[1:] actual = img.shape pred_img = img # check input depth if expected[2] == 1 and actual[2] == 3: pred_img = rgb2gray(pred_img) pred_img = pred_img.reshape(pred_img.shape + (1, )) actual = pred_img.shape if expected != actual: print("expected input dim", expected, "didn't match actual dim", actual) return pilot_angle, pilot_throttle = self.keras_part.run(pred_img) length = self.cfg.IMAGE_H a1 = user_angle * 45.0 l1 = user_throttle * length a2 = pilot_angle * 45.0 l2 = pilot_throttle * length mid = self.cfg.IMAGE_W // 2 - 1 p1 = tuple((mid - 2, self.cfg.IMAGE_H - 1)) p2 = tuple((mid + 2, self.cfg.IMAGE_H - 1)) p11 = tuple( (int(p1[0] + l1 * math.cos((a1 + 270.0) * self.deg_to_rad)), int(p1[1] + l1 * math.sin((a1 + 270.0) * self.deg_to_rad)))) p22 = tuple( (int(p2[0] + l2 * math.cos((a2 + 270.0) * self.deg_to_rad)), int(p2[1] + l2 * math.sin((a2 + 270.0) * self.deg_to_rad)))) # user is green, pilot is blue cv2.line(img, p1, p11, (0, 255, 0), 2) cv2.line(img, p2, p22, (0, 0, 255), 2) def draw_steering_distribution(self, record, img): ''' query the model for it's prediction, draw the distribution of steering choices ''' from donkeycar.parts.keras import KerasCategorical if self.keras_part is None or type( self.keras_part) is not KerasCategorical: return import cv2 pred_img = img.reshape((1, ) + img.shape) angle_binned, throttle = self.keras_part.model.predict(pred_img) x = 4 dx = 4 y = 120 - 4 iArgMax = np.argmax(angle_binned) for i in range(15): p1 = (x, y) p2 = (x, y - int(angle_binned[0][i] * 100.0)) if i == iArgMax: cv2.line(img, p1, p2, (255, 0, 0), 2) else: cv2.line(img, p1, p2, (200, 200, 200), 2) x += dx def init_salient(self, model): #from https://github.com/ermolenkodev/keras-salient-object-visualisation from tensorflow.python.keras.layers import Input, Dense, merge from tensorflow.python.keras.models import Model from tensorflow.python.keras.layers import Convolution2D, MaxPooling2D, Reshape, BatchNormalization from tensorflow.python.keras.layers import Activation, Dropout, Flatten, Dense input_shape = model.inputs[0].shape img_in = Input(shape=(input_shape[1], input_shape[2], input_shape[3]), name='img_in') x = img_in x = Convolution2D(24, (5, 5), strides=(2, 2), activation='relu', name='conv1')(x) x = Convolution2D(32, (5, 5), strides=(2, 2), activation='relu', name='conv2')(x) x = Convolution2D(64, (5, 5), strides=(2, 2), activation='relu', name='conv3')(x) x = Convolution2D(64, (3, 3), strides=(2, 2), activation='relu', name='conv4')(x) conv_5 = Convolution2D(64, (3, 3), strides=(1, 1), activation='relu', name='conv5')(x) self.convolution_part = Model(inputs=[img_in], outputs=[conv_5]) for layer_num in ('1', '2', '3', '4', '5'): try: self.convolution_part.get_layer( 'conv' + layer_num).set_weights( model.get_layer('conv2d_' + layer_num).get_weights()) except Exception as e: print(e) print("Failed to load layer weights for layer", layer_num) raise Exception("Failed to load weights") from tensorflow.python.keras import backend as K import tensorflow as tf os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' self.inp = self.convolution_part.input # input placeholder self.outputs = [ layer.output for layer in self.convolution_part.layers[1:] ] # all layer outputs self.functor = K.function([self.inp], self.outputs) kernel_3x3 = tf.constant( np.array([[[[1]], [[1]], [[1]]], [[[1]], [[1]], [[1]]], [[[1]], [[1]], [[1]]]]), tf.float32) kernel_5x5 = tf.constant( np.array([[[[1]], [[1]], [[1]], [[1]], [[1]]], [[[1]], [[1]], [[1]], [[1]], [[1]]], [[[1]], [[1]], [[1]], [[1]], [[1]]], [[[1]], [[1]], [[1]], [[1]], [[1]]], [[[1]], [[1]], [[1]], [[1]], [[1]]]]), tf.float32) self.layers_kernels = { 5: kernel_3x3, 4: kernel_3x3, 3: kernel_5x5, 2: kernel_5x5, 1: kernel_5x5 } self.layers_strides = { 5: [1, 1, 1, 1], 4: [1, 2, 2, 1], 3: [1, 2, 2, 1], 2: [1, 2, 2, 1], 1: [1, 2, 2, 1] } def draw_salient(self, img): # from https://github.com/ermolenkodev/keras-salient-object-visualisation import cv2 alpha = 0.004 beta = 1.0 - alpha expected = self.keras_part.model.inputs[0].shape[1:] actual = img.shape pred_img = img # check input depth if expected[2] == 1 and actual[2] == 3: pred_img = rgb2gray(pred_img) pred_img = pred_img.reshape(pred_img.shape + (1, )) actual = pred_img.shape salient_mask = self.compute_visualisation_mask(pred_img) salient_mask_stacked = np.dstack((salient_mask, salient_mask)) salient_mask_stacked = np.dstack((salient_mask_stacked, salient_mask)) blend = cv2.addWeighted(img.astype('float32'), alpha, salient_mask_stacked, beta, 0.0) return blend def make_frame(self, t): ''' Callback to return an image from from our tub records. This is called from the VideoClip as it references a time. We don't use t to reference the frame, but instead increment a frame counter. This assumes sequential access. ''' if self.iRec >= self.end or self.iRec >= len(self.index): return None rec_ix = self.index[self.iRec] rec = self.tub.get_record(rec_ix) image = rec['cam/image_array'] if self.convolution_part: image = self.draw_salient(image) image = image * 255 image = image.astype('uint8') self.draw_model_prediction(rec, image) self.draw_steering_distribution(rec, image) if self.scale != 1: import cv2 h, w, d = image.shape dsize = (w * self.scale, h * self.scale) image = cv2.resize(image, dsize=dsize, interpolation=cv2.INTER_CUBIC) self.iRec += 1 # returns a 8-bit RGB array return image
def main(): disp_needs_resizing = False target_disp_shape = (501, 501) target_disp_length = 501 * 501 datafile = sys.argv[1] clustering = load_single(CLUSTERS_FILE) model = None if MODEL_FILE is not None: ##load model ##stacked lstm # from tensorflow.python.keras.models import load_model #model = load_model("model.hdf5") # print("model loaded") #sys.path.insert(1,'../weather2') #os.chdir("../weather2") import demo #model=demo.stacked_lstm_ae(8,4096,'relu',32,'sgd',0.2) #simples model = demo.cnn_bilstm() #print(model.summary()) #sys.path.insert(1,'../final_eval') #os.chdir("../final_eval") model.load_weights(MODEL_FILE) from tensorflow.python.keras.models import Model #model = Model(inputs=model.inputs, outputs=model.get_layer("encoder").output) model = Model(inputs=model.inputs, outputs=model.get_layer("bidirectional").output) else: #log('no model') pass # Iterate through the samples mm = np.memmap(datafile, dtype='float32', mode='r', shape=MM_SHAPE) # TGIORGOS: CHANGE # For for each sixhour slot (change to 8 slots) for s_i, sample in enumerate(mm[0:]): origin_i = int(sample[SAMPLE_SPEC['origin']]) # real origin disp = np.array(sample[SAMPLE_SPEC['disp']]) disp = preprocessing.maxabs_scale(disp) * 1000 # scale disp [-1..1) disp += PAD if len(disp) < OR_DISP_SIZE: disp_needs_resizing = True x = int(np.sqrt(len(disp))) target_disp_shape = (x, x) target_disp_length = target_disp_shape[0] * target_disp_shape[1] # log('Target dispersion shape: ' + str(target_disp_shape)) assert np.sqrt(len(disp)).is_integer() # sanity check... lis = list() if s_i + 8 > len(mm): # log(str("returning....")) # log(s_i) # log(s_i+8) return for i in range(s_i, s_i + 8, 1): ##1-8 , 2-9 ..... # print("adding",i) # print 'shape:' , mm[i][[SAMPLE_SPEC['weath']]].shape lis.append(mm[i][[SAMPLE_SPEC['weath']]]) weather = np.array(lis) weather = weather.reshape(1, 64, 64, 8, 1) #conv #weather1 = weather.reshape(4096, 8,1) #add for lstm ds = Dataset_transformations(weather, 1, weather.shape) #ds1 = Dataset_transformations(weather1, 1, weather1.shape) #lstm #print ds._items ds.normalize() #ds1.normalize() # TGIORGOS CHANGE: # 6 hour slot ds._items = ds._items.T #ds1._items = ds1._items.T ds_hidden = None #print(ds._items.shape) if MODEL_FILE is not None: #h = model.get_hidden(ds._items) # print(model.summary()) # sys.path.insert(1,'../final_eval') # os.chdir("../final_eval") #log(str(ds._items.shape)) h = model.predict(ds._items) # h = h.reshape(h.shape[0],h.shape[1]*h.shape[2]) #log(str(h)) #log(str(h.shape)) ds_hidden = Dataset_transformations(h, 1, h.shape) else: ds_hidden = ds ## unfortunate naming but... #log('EEEEEEEEEEE') assert ds_hidden._items.shape == (1, 4096) # display_array(ds._items.reshape(61, 64)) # display_array(h[:,:2496].reshape(50, 50)) # get the closest cluster to the current weather cl_order = clustering.centroids_distance(ds_hidden) # log(cl_order) cl_cluster = cl_order[0][0] cl_score = cl_order[0][1] cluster_date = clustering._desc_date[cl_cluster] cl_id = reconstruct_date(cluster_date) scores = [] scores_euc = [] scores_cos = [] for d in glob(DISPERSIONS_DIR + '/' + cl_id + '/*' + SPECIES + '.npy'): origin = d[d.rfind('/') + 1:] origin = origin[:origin.find('-')] cl_dispersion = np.load(d) if disp_needs_resizing: # resize the 501x501 into whatever is needed cl_dispersion = imresize(cl_dispersion, target_disp_shape, mode='F') # p real, q model # display_array(disp.reshape(167,167)) # display_array(cl_dispersion) cl_dispersion = preprocessing.maxabs_scale(cl_dispersion) * 1000 cl_dispersion += PAD scor = euclidean(cl_dispersion.reshape(target_disp_length), disp) # scor = entropy(disp, cl_dispersion.reshape(target_disp_length)) scores.append((STATIONS.index(origin), origin, scor)) # Calculate cosine distance: scor_euc = cosine(cl_dispersion.reshape(target_disp_length), disp) scores_euc.append((STATIONS.index(origin), origin, scor_euc)) scor_cos = correlation(cl_dispersion.reshape(target_disp_length), disp) scores_cos.append((STATIONS.index(origin), origin, scor_cos)) assert scor != float('Inf') assert scor_euc != float('Inf') assert scor_cos != float('Inf') scores.sort(key=operator.itemgetter(2)) scores_euc.sort(key=operator.itemgetter(2)) scores_cos.sort(key=operator.itemgetter(2)) #log(str(scores)) #log(str(scores_euc)) #log(str(scores_cos)) pos = 0 pos_euc = 0 pos_cos = 0 # try: for i in range(0, len(STATIONS)): #log('BIKA STIN FOR') #log(str(STATIONS.index(origin))) if origin_i == scores[i][0]: pos = i + 1 if origin_i == scores_euc[i][0]: pos_euc = i + 1 if origin_i == scores_cos[i][0]: pos_cos = i + 1 if pos > 0 and pos_euc > 0 and pos_cos > 0: break # log(str(origin_i) + '> ' + str(s_i) + ' ' + str(pos) ) log( str(origin_i) + '\t' + str(pos) + '\t' + str(pos_euc) + '\t' + str(pos_cos))
class textgenrnn: META_TOKEN = '<s>' config = { 'rnn_layers': 2, 'rnn_size': 128, 'rnn_bidirectional': False, 'max_length': 40, 'max_words': 10000, 'dim_embeddings': 100, 'word_level': False, 'single_text': False } default_config = config.copy() def __init__(self, weights_path=None, vocab_path=None, config_path=None, name="textgenrnn_tf"): if weights_path is None: weights_path = resource_filename(__name__, 'textgenrnn_weights.hdf5') if vocab_path is None: vocab_path = resource_filename(__name__, 'textgenrnn_vocab.json') if config_path is not None: with open(config_path, 'r', encoding='utf8', errors='ignore') as json_file: self.config = json.load(json_file) self.config.update({'name': name}) self.default_config.update({'name': name}) with open(vocab_path, 'r', encoding='utf8', errors='ignore') as json_file: self.vocab = json.load(json_file) self.tokenizer = Tokenizer(filters='', lower=False, char_level=True) self.tokenizer.word_index = self.vocab self.num_classes = len(self.vocab) + 1 self.model = textgenrnn_model(self.num_classes, cfg=self.config, weights_path=weights_path) self.indices_char = dict((self.vocab[c], c) for c in self.vocab) def generate(self, n=1, return_as_list=False, prefix=None, temperature=[1.0, 0.5, 0.2, 0.2], max_gen_length=300, interactive=False, top_n=3, progress=True): gen_texts = [] iterable = trange(n) if progress and n > 1 else range(n) for _ in iterable: gen_text, _ = textgenrnn_generate( self.model, self.vocab, self.indices_char, temperature, self.config['max_length'], self.META_TOKEN, self.config['word_level'], self.config.get('single_text', False), max_gen_length, interactive, top_n, prefix) if not return_as_list: print("{}\n".format(gen_text)) gen_texts.append(gen_text) if return_as_list: return gen_texts def generate_samples(self, n=3, temperatures=[0.2, 0.5, 1.0], **kwargs): for temperature in temperatures: print('#' * 20 + '\nTemperature: {}\n'.format(temperature) + '#' * 20) self.generate(n, temperature=temperature, progress=False, **kwargs) def train_on_texts(self, texts, context_labels=None, batch_size=128, num_epochs=50, verbose=1, new_model=False, gen_epochs=1, train_size=1.0, max_gen_length=300, validation=True, dropout=0.0, via_new_model=False, save_epochs=0, multi_gpu=False, **kwargs): if new_model and not via_new_model: self.train_new_model(texts, context_labels=context_labels, num_epochs=num_epochs, gen_epochs=gen_epochs, train_size=train_size, batch_size=batch_size, dropout=dropout, validation=validation, save_epochs=save_epochs, multi_gpu=multi_gpu, **kwargs) return if context_labels: context_labels = LabelBinarizer().fit_transform(context_labels) if 'prop_keep' in kwargs: train_size = prop_keep if self.config['word_level']: texts = [text_to_word_sequence(text, filters='') for text in texts] # calculate all combinations of text indices + token indices indices_list = [ np.meshgrid(np.array(i), np.arange(len(text) + 1)) for i, text in enumerate(texts) ] indices_list = np.block(indices_list) # If a single text, there will be 2 extra indices, so remove them # Also remove first sequences which use padding if self.config['single_text']: indices_list = indices_list[self.config['max_length']:-2, :] indices_mask = np.random.rand(indices_list.shape[0]) < train_size if multi_gpu: num_gpus = len(K.tensorflow_backend._get_available_gpus()) batch_size = batch_size * num_gpus gen_val = None val_steps = None if train_size < 1.0 and validation: indices_list_val = indices_list[~indices_mask, :] gen_val = generate_sequences_from_texts(texts, indices_list_val, self, context_labels, batch_size) val_steps = max( int(np.floor(indices_list_val.shape[0] / batch_size)), 1) indices_list = indices_list[indices_mask, :] num_tokens = indices_list.shape[0] assert num_tokens >= batch_size, "Fewer tokens than batch_size." level = 'word' if self.config['word_level'] else 'character' print("Training on {:,} {} sequences.".format(num_tokens, level)) steps_per_epoch = max(int(np.floor(num_tokens / batch_size)), 1) gen = generate_sequences_from_texts(texts, indices_list, self, context_labels, batch_size) base_lr = 4e-3 # scheduler function must be defined inline. def lr_linear_decay(epoch): return (base_lr * (1 - (epoch / num_epochs))) if context_labels is not None: if new_model: weights_path = None else: weights_path = "{}_weights.hdf5".format(self.config['name']) self.save(weights_path) self.model = textgenrnn_model(self.num_classes, dropout=dropout, cfg=self.config, context_size=context_labels.shape[1], weights_path=weights_path) model_t = self.model if multi_gpu: # Do not locate model/merge on CPU since sample sizes are small. parallel_model = multi_gpu_model(self.model, gpus=num_gpus, cpu_merge=False) parallel_model.compile(loss='categorical_crossentropy', optimizer=RMSprop(lr=4e-3, rho=0.99)) model_t = parallel_model print("Training on {} GPUs.".format(num_gpus)) model_t.fit_generator(gen, steps_per_epoch=steps_per_epoch, epochs=num_epochs, callbacks=[ LearningRateScheduler(lr_linear_decay), generate_after_epoch(self, gen_epochs, max_gen_length), save_model_weights(self, num_epochs, save_epochs) ], verbose=verbose, max_queue_size=10, validation_data=gen_val, validation_steps=val_steps) # Keep the text-only version of the model if using context labels if context_labels is not None: self.model = Model(inputs=self.model.input[0], outputs=self.model.output[1]) def train_new_model(self, texts, context_labels=None, num_epochs=50, gen_epochs=1, batch_size=128, dropout=0.0, train_size=1.0, validation=True, save_epochs=0, multi_gpu=False, **kwargs): self.config = self.default_config.copy() self.config.update(**kwargs) print("Training new model w/ {}-layer, {}-cell {}LSTMs".format( self.config['rnn_layers'], self.config['rnn_size'], 'Bidirectional ' if self.config['rnn_bidirectional'] else '')) # If training word level, must add spaces around each punctuation. # https://stackoverflow.com/a/3645946/9314418 if self.config['word_level']: punct = '!"#$%&()*+,-./:;<=>?@[\]^_`{|}~\\n\\t\'‘’“”’–—' for i in range(len(texts)): texts[i] = re.sub('([{}])'.format(punct), r' \1 ', texts[i]) texts[i] = re.sub(' {2,}', ' ', texts[i]) # Create text vocabulary for new texts # if word-level, lowercase; if char-level, uppercase self.tokenizer = Tokenizer(filters='', lower=self.config['word_level'], char_level=(not self.config['word_level'])) self.tokenizer.fit_on_texts(texts) # Limit vocab to max_words max_words = self.config['max_words'] self.tokenizer.word_index = { k: v for (k, v) in self.tokenizer.word_index.items() if v <= max_words } if not self.config.get('single_text', False): self.tokenizer.word_index[self.META_TOKEN] = len( self.tokenizer.word_index) + 1 self.vocab = self.tokenizer.word_index self.num_classes = len(self.vocab) + 1 self.indices_char = dict((self.vocab[c], c) for c in self.vocab) # Create a new, blank model w/ given params self.model = textgenrnn_model(self.num_classes, dropout=dropout, cfg=self.config) # Save the files needed to recreate the model with open('{}_vocab.json'.format(self.config['name']), 'w', encoding='utf8') as outfile: json.dump(self.tokenizer.word_index, outfile, ensure_ascii=False) with open('{}_config.json'.format(self.config['name']), 'w', encoding='utf8') as outfile: json.dump(self.config, outfile, ensure_ascii=False) self.train_on_texts(texts, new_model=True, via_new_model=True, context_labels=context_labels, num_epochs=num_epochs, gen_epochs=gen_epochs, train_size=train_size, batch_size=batch_size, dropout=dropout, validation=validation, save_epochs=save_epochs, multi_gpu=multi_gpu, **kwargs) def save(self, weights_path="textgenrnn_weights_saved.hdf5"): self.model.save_weights(weights_path) def load(self, weights_path): self.model = textgenrnn_model(self.num_classes, cfg=self.config, weights_path=weights_path) def reset(self): self.config = self.default_config.copy() self.__init__(name=self.config['name']) def train_from_file(self, file_path, header=True, delim="\n", new_model=False, context=None, is_csv=False, **kwargs): context_labels = None if context: texts, context_labels = textgenrnn_texts_from_file_context( file_path) else: texts = textgenrnn_texts_from_file(file_path, header, delim, is_csv) print("{:,} texts collected.".format(len(texts))) if new_model: self.train_new_model(texts, context_labels=context_labels, **kwargs) else: self.train_on_texts(texts, context_labels=context_labels, **kwargs) def train_from_largetext_file(self, file_path, new_model=True, **kwargs): with open(file_path, 'r', encoding='utf8', errors='ignore') as f: texts = [f.read()] if new_model: self.train_new_model(texts, single_text=True, **kwargs) else: self.train_on_texts(texts, single_text=True, **kwargs) def generate_to_file(self, destination_path, **kwargs): texts = self.generate(return_as_list=True, **kwargs) with open(destination_path, 'w') as f: for text in texts: f.write("{}\n".format(text)) def encode_text_vectors(self, texts, pca_dims=50, tsne_dims=None, tsne_seed=None, return_pca=False, return_tsne=False): # if a single text, force it into a list: if isinstance(texts, str): texts = [texts] vector_output = Model(inputs=self.model.input, outputs=self.model.get_layer('attention').output) encoded_vectors = [] maxlen = self.config['max_length'] for text in texts: if self.config['word_level']: text = text_to_word_sequence(text, filters='') text_aug = [self.META_TOKEN] + list(text[0:maxlen]) encoded_text = textgenrnn_encode_sequence(text_aug, self.vocab, maxlen) encoded_vector = vector_output.predict(encoded_text) encoded_vectors.append(encoded_vector) encoded_vectors = np.squeeze(np.array(encoded_vectors), axis=1) if pca_dims is not None: assert len(texts) > 1, "Must use more than 1 text for PCA" pca = PCA(pca_dims) encoded_vectors = pca.fit_transform(encoded_vectors) if tsne_dims is not None: tsne = TSNE(tsne_dims, random_state=tsne_seed) encoded_vectors = tsne.fit_transform(encoded_vectors) return_objects = encoded_vectors if return_pca or return_tsne: return_objects = [return_objects] if return_pca: return_objects.append(pca) if return_tsne: return_objects.append(tsne) return return_objects def similarity(self, text, texts, use_pca=True): text_encoded = self.encode_text_vectors(text, pca_dims=None) if use_pca: texts_encoded, pca = self.encode_text_vectors(texts, return_pca=True) text_encoded = pca.transform(text_encoded) else: texts_encoded = self.encode_text_vectors(texts, pca_dims=None) cos_similairity = cosine_similarity(text_encoded, texts_encoded)[0] text_sim_pairs = list(zip(texts, cos_similairity)) text_sim_pairs = sorted(text_sim_pairs, key=lambda x: -x[1]) return text_sim_pairs
data = np.load(GHT_FILE)[_LEVEL] # Make 2 days (overlap 1 day) data, _, _ = utils.overlap(data, win = 4, t = 8) if MODEL=='': print('insert model..') exit() print('Data Shape...',data.shape) ##predict data... import demo model=demo.stacked_lstm_ae(8,4096,'relu',32,'sgd',0.2,0.1) model.load_weights(MODEL_FILE) from tensorflow.python.keras.models import Model model = Model(inputs=model.inputs, outputs=model.get_layer("encoder").output) data = model.predict(data) # Reshape data = data.reshape(data.shape[1], data.shape[0]) ds = Dataset_transformations(data.T, 1000, data.shape) if os.path.exists(PREFIX+CONFIG_NAME+'.zip'): clust_obj = dataset_utils.load_single(PREFIX+CONFIG_NAME+'.zip') else: print 'Doing kmeans.....' clust_obj = Clustering(ds,n_clusters=15,n_init=100,features_first=False) clust_obj.batch_kmeans(10) print 'Saving .....' clust_obj.save(PREFIX+CONFIG_NAME+'.zip') # Descriptor num_min: 1
def run_tests_file_output(model_name, delex_slots=(), alpha=-1.0, beta=-1.0, rev_pen=-1.0, gamma=-1.0, p=-1.0, num_samp=-1, file_suff="", maximise_bleu=False): """ Runs tests and outputs to file """ data = load_data_chars("devset.csv", delex_slots=delex_slots, delex_both=False) data = vectorise_chars(data) data = create_reverse_indices(data) model, encoder, decoder = load_models(model_name) decoder = Model( inputs=decoder.input, outputs=[decoder.output, decoder.get_layer('attention').output]) rev_model, rev_encoder, rev_decoder = load_models("basicreverse") rev_decoder = Model(inputs=rev_decoder.input, outputs=[ rev_decoder.output, rev_decoder.get_layer('attention').output ]) output_file = open(model_name + file_suff + "_output.txt", "w") prev_mr = "" for seq_index in range(len(data["mr_input"])): print(seq_index) input = data["encoder_input"][seq_index:seq_index + 1] correct = data['ref_sentences'][seq_index].strip("\t\n") mr = data["mr_input"][seq_index] if mr == prev_mr: continue prev_mr = mr paths = decode_beam_search(input, data, encoder, decoder, mr=data['mr_input'][seq_index], reverse_models=[rev_encoder, rev_decoder], beam_width=10, alpha=alpha, beta=beta, gamma=gamma, rev_pen=rev_pen, p=p, num_samp=num_samp) if len(delex_slots) > 0: for i in range(len(paths)): paths[i] = relexicalise_sentence(paths[i], data['mr_input'][seq_index], delex_slots) if not maximise_bleu: decoded_sentence = paths[0] else: max_bleu = 0 max_sent = "" for p in paths: bleu = sentence_bleu(references=[correct], hypothesis=p) if bleu > max_bleu: max_bleu = bleu max_sent = p decoded_sentence = max_sent output_file.write(decoded_sentence) print(decoded_sentence) return
class JointEmbeddingModel: def __init__(self, config): self.data_dir = config.data_dir self.model_name = config.model_name self.meth_name_len = config.methname_len # the max length of method name self.apiseq_len = config.apiseq_len self.tokens_len = config.tokens_len self.desc_len = config.desc_len self.vocab_size = config.n_words # the size of vocab self.embed_dims = config.embed_dims self.lstm_dims = config.lstm_dims self.hidden_dims = config.hidden_dims self.margin = 0.05 self.init_embed_weights_meth_name = config.init_embed_weights_methodname self.init_embed_weights_tokens = config.init_embed_weights_tokens self.init_embed_weights_desc = config.init_embed_weights_desc self.meth_name = Input(shape=(self.meth_name_len,), dtype='int32', name='meth_name') self.apiseq = Input(shape=(self.apiseq_len,), dtype='int32', name='apiseq') self.tokens = Input(shape=(self.tokens_len,), dtype='int32', name='tokens2') self.desc_good = Input(shape=(self.desc_len,), dtype='int32', name='desc_good') self.desc_bad = Input(shape=(self.desc_len,), dtype='int32', name='desc_bad') if not os.path.exists(self.data_dir + 'model/' + self.model_name): os.makedirs(self.data_dir + 'model/' + self.model_name) def build(self): self.transformer_meth = transformer.EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims, embed_dim=self.embed_dims, ffn_dim=self.lstm_dims, droput_rate=0.2, n_heads=2, max_len=self.meth_name_len, name='methT') self.transformer_apiseq = transformer.EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims, embed_dim=self.embed_dims, ffn_dim=self.lstm_dims, droput_rate=0.2, n_heads=4, max_len=self.apiseq_len, name='apiseqT') self.transformer_desc = transformer.EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims, embed_dim=self.embed_dims, ffn_dim=self.lstm_dims, droput_rate=0.2, n_heads=4, max_len=self.desc_len, name='descT') # self.transformer_ast = EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims, embed_dim=self.embed_dims, ffn_dim=self.lstm_dims, droput_rate=0.2, n_heads=4, max_len=128) self.transformer_tokens = transformer.EncoderModel(vocab_size=self.vocab_size, model_dim=self.hidden_dims, embed_dim=self.embed_dims, ffn_dim=self.lstm_dims, droput_rate=0.2, n_heads=8, max_len=self.tokens_len, name='tokensT') # create path to store model Info # 1 -- CodeNN meth_name = Input(shape=(self.meth_name_len,), dtype='int32', name='meth_name') apiseq = Input(shape=(self.apiseq_len,), dtype='int32', name='apiseq') tokens3 = Input(shape=(self.tokens_len,), dtype='int32', name='tokens3') # method name # embedding layer meth_name_out = self.transformer_meth(meth_name) # max pooling maxpool = Lambda(lambda x: k.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]), name='maxpooling_methodname') method_name_pool = maxpool(meth_name_out) activation = Activation('tanh', name='active_method_name') method_name_repr = activation(method_name_pool) # apiseq # embedding layer apiseq_out = self.transformer_apiseq(apiseq) # max pooling maxpool = Lambda(lambda x: k.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]), name='maxpooling_apiseq') apiseq_pool = maxpool(apiseq_out) activation = Activation('tanh', name='active_apiseq') apiseq_repr = activation(apiseq_pool) # tokens # embedding layer tokens_out = self.transformer_tokens(tokens3) # max pooling maxpool = Lambda(lambda x: k.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]), name='maxpooling_tokens') tokens_pool = maxpool(tokens_out) activation = Activation('tanh', name='active_tokens') tokens_repr = activation(tokens_pool) # fusion method_name, apiseq, tokens merge_method_name_api = Concatenate(name='merge_methname_api')([method_name_repr, apiseq_repr]) merge_code_repr = Concatenate(name='merge_code_repr')([merge_method_name_api, tokens_repr]) code_repr = Dense(self.hidden_dims, activation='tanh', name='dense_coderepr')(merge_code_repr) self.code_repr_model = Model(inputs=[meth_name, apiseq, tokens3], outputs=[code_repr], name='code_repr_model') self.code_repr_model.summary() self.output = Model(inputs=self.code_repr_model.input, outputs=self.code_repr_model.get_layer('tokensT').output) self.output.summary() # 2 -- description desc = Input(shape=(self.desc_len,), dtype='int32', name='desc') # desc # embedding layer desc_out = self.transformer_desc(desc) # max pooling maxpool = Lambda(lambda x: k.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]), name='maxpooling_desc') desc_pool = maxpool(desc_out) activation = Activation('tanh', name='active_desc') desc_repr = activation(desc_pool) self.desc_repr_model = Model(inputs=[desc], outputs=[desc_repr], name='desc_repr_model') self.desc_repr_model.summary() # 3 -- cosine similarity code_repr = self.code_repr_model([meth_name, apiseq, tokens3]) desc_repr = self.desc_repr_model([desc]) cos_sim = Dot(axes=1, normalize=True, name='cos_sim')([code_repr, desc_repr]) sim_model = Model(inputs=[meth_name, apiseq, tokens3, desc], outputs=[cos_sim], name='sim_model') self.sim_model = sim_model self.sim_model.summary() # 4 -- build training model good_sim = sim_model([self.meth_name, self.apiseq, self.tokens, self.desc_good]) bad_sim = sim_model([self.meth_name, self.apiseq, self.tokens, self.desc_bad]) loss = Lambda(lambda x: k.maximum(1e-6, self.margin - (x[0] - x[1])), output_shape=lambda x: x[0], name='loss')( [good_sim, bad_sim]) self.training_model = Model(inputs=[self.meth_name, self.apiseq, self.tokens, self.desc_good, self.desc_bad], outputs=[loss], name='training_model') self.training_model.summary() def compile(self, optimizer, **kwargs): optimizer = keras.optimizers.SGD(lr=0.0001, momentum=0.9, nesterov=True) # optimizer = keras.optimizers.Adam(lr=0.0001) # print(self.code_repr_model.layers, self.desc_repr_model.layers, self.training_model.layers, self.sim_model.layers) self.code_repr_model.compile(loss='cosine_proximity', optimizer=optimizer, **kwargs) self.desc_repr_model.compile(loss='cosine_proximity', optimizer=optimizer, **kwargs) self.training_model.compile(loss=lambda y_true, y_pred: y_pred + y_true - y_true, optimizer=optimizer, **kwargs) self.sim_model.compile(loss='binary_crossentropy', optimizer=optimizer, **kwargs) def fit(self, x, **kwargs): y = np.zeros(shape=x[0].shape[:1], dtype=np.float32) return self.training_model.fit(x, y, **kwargs) def getOutput(self, x): # functor = k.function([self.code_repr_model.layers[0].input, k.learning_phase()], [self.code_repr_model.layers[0].output]) # print(functor(x)[0]) print(self.output.predict(x)) def repr_code(self, x, **kwargs): return self.code_repr_model.predict(x, **kwargs) def repr_desc(self, x, **kwargs): return self.desc_repr_model.predict(x, **kwargs) def predict(self, x, **kwargs): return self.sim_model.predict(x, **kwargs) def save(self, code_model_file, desc_model_file, **kwargs): file = h5py.File(code_model_file, 'w') weight_code = self.code_repr_model.get_weights() for i in range(len(weight_code)): file.create_dataset('weight_code'+str(i), data=weight_code[i]) file.close() file = h5py.File(desc_model_file, 'w') weight_desc = self.desc_repr_model.get_weights() for i in range(len(weight_desc)): file.create_dataset('weight_desc'+str(i), data=weight_desc[i]) file.close() # self.code_repr_model.save_weights(code_model_file, **kwargs) # self.desc_repr_model.save_weights(desc_model_file, **kwargs) def load(self, code_model_file, desc_model_file, **kwargs): # self.code_repr_model.load_weights(code_model_file, **kwargs) # self.desc_repr_model.load_weights(desc_model_file, **kwargs) file = h5py.File(code_model_file, 'r') weight_code = [] for i in range(len(file.keys())): weight_code.append(file['weight_code'+str(i)][:]) self.code_repr_model.set_weights(weight_code) file.close() file = h5py.File(desc_model_file, 'r') weight_desc = [] for i in range(len(file.keys())): weight_desc.append(file['weight_desc'+str(i)][:]) self.desc_repr_model.set_weights(weight_desc) file.close()
seq_in, _, _ = demo.overlap(ght_700, 4, t=8) seq_in = seq_in.reshape((seq_in.shape[0], seq_in.shape[1], 64, 64, 1)) if MODEL == '': print('insert model..') exit() ##predict data... model = demo.model_conv_lstm(TIMESTEPS) model.load_weights(MODEL) print(model.summary()) from tensorflow.python.keras.models import Model model = Model(inputs=model.input, outputs=model.get_layer('encoder').output) data = model.predict(seq_in) print("Prediction shape:", data.shape) exit() data = data.reshape((data.shape[0] * data.shape[1], 8 * 8 * 2)) print('reshape encoding..:', data.shape) # Reshape data = data.reshape(data.shape[1], data.shape[0]) ds = Dataset_transformations(data.T, 1000, data.shape) if os.path.exists(PREFIX + CONFIG_NAME + '.zip'): clust_obj = dataset_utils.load_single(PREFIX + CONFIG_NAME + '.zip') else: print 'Doing kmeans.....'
def main(): # Counting Dataset counting_dataset_path = 'counting_data_UCF' counting_dataset = list() train_labels = {} val_labels = {} for im_path in glob.glob(os.path.join(counting_dataset_path, '*.jpg')): counting_dataset.append(im_path) img = image.load_img(im_path) gt_file = im_path.replace('.jpg', '_ann.mat') h, w = img.size dmap, crowd_number = load_gt_from_mat(gt_file, (w, h)) train_labels[im_path] = dmap val_labels[im_path] = crowd_number counting_dataset_pyramid, train_labels_pyramid = multiscale_pyramid( counting_dataset, train_labels) # Ranking Dataset ranking_dataset_path = 'ranking_data' ranking_dataset = list() for im_path in glob.glob(os.path.join(ranking_dataset_path, '*.jpg')): ranking_dataset.append(im_path) # randomize the order of images before splitting np.random.shuffle(counting_dataset) split_size = int(round(len(counting_dataset) / 5)) splits_list = list() for t in range(5): splits_list.append(counting_dataset[t * split_size:t * split_size + split_size]) split_val_labels = {} mae_sum = 0.0 mse_sum = 0.0 # create folder to save results date = str(datetime.datetime.now()) d = date.split() d1 = d[0] d2 = d[1].split(':') results_folder = 'Results-' + d1 + '-' + d2[0] + '.' + d2[1] if not os.path.exists(results_folder): os.makedirs(results_folder) # 5-fold cross validation epochs = int(round(iterations / iterations_per_epoch)) n_fold = 5 for f in range(0, n_fold): print('\nFold ' + str(f)) # Model model = VGG16(include_top=False, weights='imagenet') transfer_layer = model.get_layer('block5_conv3') conv_model = Model(inputs=[model.input], outputs=[transfer_layer.output], name='vgg_partial') counting_input = Input(shape=(224, 224, 3), dtype='float32', name='counting_input') ranking_input = Input(shape=(224, 224, 3), dtype='float32', name='ranking_input') x = conv_model([counting_input, ranking_input]) counting_output = Conv2D(1, (3, 3), strides=(1, 1), padding='same', data_format=None, dilation_rate=(1, 1), activation='relu', use_bias=True, kernel_initializer='glorot_uniform', bias_initializer='zeros', kernel_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, bias_constraint=None, name='counting_output')(x) # The ranking output is computed using SUM pool. Here I use # GlobalAveragePooling2D followed by a multiplication by 14^2 to do # this. ranking_output = Lambda( lambda i: 14.0 * 14.0 * i, name='ranking_output')(GlobalAveragePooling2D( name='global_average_pooling2d')(counting_output)) train_model = Model(inputs=[counting_input, ranking_input], outputs=[counting_output, ranking_output]) train_model.summary() # l2 weight decay for layer in train_model.layers: if hasattr(layer, 'kernel_regularizer'): layer.kernel_regularizer = regularizers.l2(5e-4) elif layer.name == 'vgg_partial': for l in layer.layers: if hasattr(l, 'kernel_regularizer'): l.kernel_regularizer = regularizers.l2(5e-4) optimizer = SGD(lr=0.0, decay=0.0, momentum=0.9, nesterov=False) loss = { 'counting_output': euclideanDistanceCountingLoss, 'ranking_output': pairwiseRankingHingeLoss } loss_weights = [1.0, 0.0] train_model.compile(optimizer=optimizer, loss=loss, loss_weights=loss_weights) splits_list_tmp = splits_list.copy() # counting validation split split_val = splits_list_tmp[f] del splits_list_tmp[f] flat = itertools.chain.from_iterable(splits_list_tmp) # counting train split split_train = list(flat) # counting validation split labels split_val_labels = {k: val_labels[k] for k in split_val} counting_dataset_pyramid_split = [] train_labels_pyramid_split = [] for key in split_train: counting_dataset_pyramid_split.append( counting_dataset_pyramid[key][0]) counting_dataset_pyramid_split.append( counting_dataset_pyramid[key][1]) counting_dataset_pyramid_split.append( counting_dataset_pyramid[key][2]) counting_dataset_pyramid_split.append( counting_dataset_pyramid[key][3]) counting_dataset_pyramid_split.append( counting_dataset_pyramid[key][4]) train_labels_pyramid_split.append(train_labels_pyramid[key][0]) train_labels_pyramid_split.append(train_labels_pyramid[key][1]) train_labels_pyramid_split.append(train_labels_pyramid[key][2]) train_labels_pyramid_split.append(train_labels_pyramid[key][3]) train_labels_pyramid_split.append(train_labels_pyramid[key][4]) index_shuf = np.arange(len(counting_dataset_pyramid_split)) np.random.shuffle(index_shuf) counting_dataset_pyramid_split_shuf = [] train_labels_pyramid_split_shuf = [] for i in index_shuf: counting_dataset_pyramid_split_shuf.append( counting_dataset_pyramid_split[i]) train_labels_pyramid_split_shuf.append( train_labels_pyramid_split[i]) train_generator = DataGenerator(counting_dataset_pyramid_split_shuf, train_labels_pyramid_split_shuf, ranking_dataset, **params) lrate = LearningRateScheduler(step_decay) callbacks_list = [lrate] train_model.fit_generator(generator=train_generator, epochs=epochs, callbacks=callbacks_list) #test images tmp_model = train_model.get_layer('vgg_partial') test_input = Input(shape=(None, None, 3), dtype='float32', name='test_input') new_input = tmp_model(test_input) co = train_model.get_layer('counting_output')(new_input) test_output = Lambda(lambda i: K.sum(i, axis=(1, 2)), name='test_output')(co) test_model = Model(inputs=[test_input], outputs=[test_output]) predictions = np.empty((len(split_val), 1)) y_validation = np.empty((len(split_val), 1)) for i in range(len(split_val)): img = image.load_img(split_val[i], target_size=(224, 224)) img_to_array = image.img_to_array(img) img_to_array = preprocess_input(img_to_array) img_to_array = np.expand_dims(img_to_array, axis=0) pred_test = test_model.predict(img_to_array) predictions[i] = pred_test y_validation[i] = split_val_labels[split_val[i]] mean_abs_err = mae(predictions, y_validation) mean_sqr_err = mse(predictions, y_validation) # serialize model to JSON model_json = test_model.to_json() model_json_name = "test_model_" + str(f) + ".json" with open(model_json_name, "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 model_h5_name = "test_model_" + str(f) + ".h5" test_model.save_weights(model_h5_name) print("Saved model to disk") print('\n######################') print('Results on TEST SPLIT:') print(' MAE: {}'.format(mean_abs_err)) print(' MSE: {}'.format(mean_sqr_err)) print("Took %f seconds" % (time.time() - s)) path1 = results_folder + '/test_split_results_fold-' + str(f) + '.txt' with open(path1, 'w') as f: f.write('mae: %f,\nmse: %f, \nTook %f seconds' % (mean_abs_err, mean_sqr_err, time.time() - s)) mae_sum = mae_sum + mean_abs_err mse_sum = mse_sum + mean_sqr_err print('\n################################') print('Average Results on TEST SPLIT:') print(' AVE MAE: {}'.format(mae_sum / n_fold)) print(' AVE MSE: {}'.format(mse_sum / n_fold)) print("Took %f seconds" % (time.time() - s)) path2 = results_folder + '/test_split_results_avg.txt' with open(path2, 'w') as f: f.write('avg_mae: %f, \navg_mse: %f, \nTook %f seconds' % (mae_sum / n_fold, mse_sum / n_fold, time.time() - s))
def seq2seq_architecture(latent_size, vocabulary_size, article_max_len, embedding_matrix, batch_size, epochs, train_article, train_summary, train_target): encoder_inputs = Input(shape=(article_max_len, ), name='Encoder-Input') encoder_embeddings = Embedding( vocabulary_size + 1, 300, weights=[embedding_matrix], trainable=False, mask_zero=False, name='Encoder-Word-Embedding')(encoder_inputs) encoder_embeddings = BatchNormalization( name='Encoder-Batch-Normalization')(encoder_embeddings) encoder_conv = Conv1D(filters=4, kernel_size=8, padding='same', activation='relu')(encoder_embeddings) encoder_drop = Dropout(0.25)(encoder_conv) encoder_pool = MaxPooling1D(pool_size=1)(encoder_drop) encoder_flatten = Flatten()(encoder_pool) encoder_model = Model(inputs=encoder_inputs, outputs=encoder_flatten, name='Encoder-Model') encoder_outputs = encoder_model(encoder_inputs) decoder_inputs = Input(shape=(None, ), name='Decoder-Input') decoder_embeddings = Embedding( vocabulary_size + 1, 300, weights=[embedding_matrix], trainable=False, mask_zero=False, name='Decoder-Word-Embedding')(decoder_inputs) decoder_embeddings = BatchNormalization( name='Decoder-Batch-Normalization-1')(decoder_embeddings) decoder_conv = Conv1D(filters=32, kernel_size=4, padding='same', activation='relu', name='Decoder-Conv1D') \ (decoder_embeddings) decoder_drop = Dropout(0.25, name='Decoder-Conv1D-Dropout')(decoder_conv) decoder_pool = MaxPooling1D(pool_size=1, name='Decoder-MaxPool1D')( decoder_drop) # GlobalMaxPool1D() decoder_gru = GRU(latent_size, return_state=True, return_sequences=True, name='Decoder-GRU') decoder_gru_outputs, _ = decoder_gru(decoder_pool, initial_state=encoder_outputs) decoder_outputs = BatchNormalization( name='Decoder-Batch-Normalization-2')(decoder_gru_outputs) decoder_outputs = Dense(vocabulary_size + 1, activation='softmax', name='Final-Output-Dense')(decoder_outputs) seq2seq_model = Model([encoder_inputs, decoder_inputs], decoder_outputs) seq2seq_model.compile(optimizer="adam", loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy']) seq2seq_model.summary() classes = [item for sublist in train_summary.tolist() for item in sublist] class_weights = class_weight.compute_class_weight('balanced', np.unique(classes), classes) e_stopping = EarlyStopping(monitor='val_loss', patience=4, verbose=1, mode='min', restore_best_weights=True) history = seq2seq_model.fit(x=[train_article, train_summary], y=np.expand_dims(train_target, -1), batch_size=batch_size, epochs=epochs, validation_split=0.1, callbacks=[e_stopping], class_weight=class_weights) f = open("data/models/convgru_results.txt", "w", encoding="utf-8") f.write("ConvGRU \n layers: 1 \n latent size: " + str(latent_size) + "\n vocab size: " + str(vocabulary_size) + "\n") f.close() history_dict = history.history plot_loss(history_dict) # inference encoder_model = seq2seq_model.get_layer('Encoder-Model') decoder_inputs = seq2seq_model.get_layer('Decoder-Input').input decoder_embeddings = seq2seq_model.get_layer('Decoder-Word-Embedding')( decoder_inputs) decoder_embeddings = seq2seq_model.get_layer( 'Decoder-Batch-Normalization-1')(decoder_embeddings) decoder_conv = seq2seq_model.get_layer('Decoder-Conv1D')( decoder_embeddings) decoder_drop = seq2seq_model.get_layer('Decoder-Conv1D-Dropout')( decoder_conv) decoder_pool = seq2seq_model.get_layer('Decoder-MaxPool1D')(decoder_drop) gru_inference_state_input = Input(shape=(latent_size, ), name='Hidden-State-Input') gru_out, gru_state_out = seq2seq_model.get_layer('Decoder-GRU')( [decoder_pool, gru_inference_state_input]) decoder_outputs = seq2seq_model.get_layer('Decoder-Batch-Normalization-2')( gru_out) dense_out = seq2seq_model.get_layer('Final-Output-Dense')(decoder_outputs) decoder_model = Model([decoder_inputs, gru_inference_state_input], [dense_out, gru_state_out]) return encoder_model, decoder_model
def MobileNetV2(classes=1000, input_tensor=None, input_shape=(512, 512, 3), weights_info=None, OS=16, alpha=1., include_top=True): """ Instantiates the Deeplabv3+ architecture Optionally loads weights pre-trained on PASCAL VOC or Cityscapes. This model is available for TensorFlow only. # Arguments classes: Integer, optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: shape of input image. format HxWxC PASCAL VOC model was trained on (512,512,3) images. None is allowed as shape/width weights_info: this dict is consisted of `classes` and `weghts`. `classes` is number of `weights` output units. `weights` is one of 'imagenet' (pre-training on ImageNet), 'pascal_voc', 'cityscapes', original weights path (pre-training on original data) or None (random initialization) OS: determines input_shape/feature_extractor_output ratio. One of {8,16}. Used only for xception backbone. alpha: controls the width of the MobileNetV2 network. This is known as the width multiplier in the MobileNetV2 paper. - If `alpha` < 1.0, proportionally decreases the number of filters in each layer. - If `alpha` > 1.0, proportionally increases the number of filters in each layer. - If `alpha` = 1, default number of filters from the paper are used at each layer. Used only for mobilenetv2 backbone. Pretrained is only available for alpha=1. include_top: Boolean, whether to include the fully-connected layer at the top of the network. Defaults to `True`. # Returns A Keras model instance. """ if weights_info is not None: if weights_info.get("weights") is None: weights = None elif weights_info["weights"] in { 'imagenet', 'pascal_voc', 'cityscapes', None }: weights = weights_info["weights"] elif os.path.exists(weights_info["weights"]) and weights_info.get( "classes") is not None: classes = int(weights_info["classes"]) weights = weights_info["weights"] else: raise ValueError( 'The `weights` should be either ' '`None` (random initialization), `imagenet`, `pascal_voc`, `cityscapes`, ' 'original weights path (pre-training on original data), ' 'or the path to the weights file to be loaded and' '`classes` should be number of original weights output units') else: weights = None if classes is None: raise ValueError('`classes` should be any number') if input_tensor is None: img_input = Input(shape=input_shape) else: img_input = input_tensor # If input_shape is None, infer shape from input_tensor if backend.image_data_format() == 'channels_first': rows = input_shape[1] cols = input_shape[2] else: rows = input_shape[0] cols = input_shape[1] if rows == cols and rows in [96, 128, 160, 192, 224]: default_size = rows else: default_size = 224 if weights == 'imagenet': if alpha not in [0.35, 0.50, 0.75, 1.0, 1.3, 1.4]: raise ValueError('If imagenet weights are being loaded, ' 'alpha can be one of `0.35`, `0.50`, `0.75`, ' '`1.0`, `1.3` or `1.4` only.') if rows != cols or rows not in [96, 128, 160, 192, 224]: rows = 224 OS = 8 first_block_filters = _make_divisible(32 * alpha, 8) x = Conv2D(first_block_filters, kernel_size=3, strides=(2, 2), padding='same', use_bias=False, name='Conv')(img_input) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_BN')(x) x = Activation(relu6, name='Conv_Relu6')(x) x = _inverted_res_block(x, filters=16, alpha=alpha, stride=1, expansion=1, block_id=0, skip_connection=False) x = _inverted_res_block(x, filters=24, alpha=alpha, stride=2, expansion=6, block_id=1, skip_connection=False) x = _inverted_res_block(x, filters=24, alpha=alpha, stride=1, expansion=6, block_id=2, skip_connection=True) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=2, expansion=6, block_id=3, skip_connection=False) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=1, expansion=6, block_id=4, skip_connection=True) x = _inverted_res_block(x, filters=32, alpha=alpha, stride=1, expansion=6, block_id=5, skip_connection=True) # stride in block 6 changed from 2 -> 1, so we need to use rate = 2 x = _inverted_res_block( x, filters=64, alpha=alpha, stride=1, # 1! expansion=6, block_id=6, skip_connection=False) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=7, skip_connection=True) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=8, skip_connection=True) x = _inverted_res_block(x, filters=64, alpha=alpha, stride=1, rate=2, expansion=6, block_id=9, skip_connection=True) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=10, skip_connection=False) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=11, skip_connection=True) x = _inverted_res_block(x, filters=96, alpha=alpha, stride=1, rate=2, expansion=6, block_id=12, skip_connection=True) x = _inverted_res_block( x, filters=160, alpha=alpha, stride=1, rate=2, # 1! expansion=6, block_id=13, skip_connection=False) x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=4, expansion=6, block_id=14, skip_connection=True) x = _inverted_res_block(x, filters=160, alpha=alpha, stride=1, rate=4, expansion=6, block_id=15, skip_connection=True) x = _inverted_res_block(x, filters=320, alpha=alpha, stride=1, rate=4, expansion=6, block_id=16, skip_connection=False) if alpha > 1.0: last_block_filters = _make_divisible(1280 * alpha, 8) else: last_block_filters = 1280 x = Conv2D(last_block_filters, kernel_size=1, use_bias=False, name='Conv_1')(x) x = BatchNormalization(epsilon=1e-3, momentum=0.999, name='Conv_1_bn')(x) x = Activation(relu6, name='out_relu')(x) x = GlobalAveragePooling2D()(x) x = Dense(classes, activation='softmax')(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = layer_utils.get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='mobilenetv2') # Load weights. if weights == 'imagenet': print("movilenetv2 load model imagenet") model_name = ('mobilenet_v2_weights_tf_dim_ordering_tf_kernels_' + str(alpha) + '_' + str(rows) + '.h5') weight_path = BASE_WEIGHT_PATH + model_name weights_path = data_utils.get_file(model_name, weight_path, cache_subdir='models') model.load_weights(weights_path) elif not (weights in {'pascal_voc', 'cityscapes', None}): model.load_weights(weights) if include_top: return model else: # get last _inverted_res_block layer no_top_model = Model(inputs=model.input, outputs=model.get_layer(index=-6).output) return no_top_model
def SENET50(include_top=True, weights='vggface', input_tensor=None, input_shape=None, pooling=None, classes=8631): input_shape = _obtain_input_shape(input_shape, default_size=224, min_size=197, data_format=K.image_data_format(), require_flatten=include_top, weights=weights) if input_tensor is None: img_input = Input(shape=input_shape) else: if not K.is_keras_tensor(input_tensor): img_input = Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor if K.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 x = Conv2D( 64, (7, 7), use_bias=False, strides=(2, 2), padding='same', name='conv1/7x7_s2')(img_input) x = BatchNormalization(axis=bn_axis, name='conv1/7x7_s2/bn')(x) x = Activation('relu')(x) x = MaxPooling2D((3, 3), strides=(2, 2))(x) x = senet_conv_block(x, 3, [64, 64, 256], stage=2, block=1, strides=(1, 1)) x = senet_identity_block(x, 3, [64, 64, 256], stage=2, block=2) x = senet_identity_block(x, 3, [64, 64, 256], stage=2, block=3) x = senet_conv_block(x, 3, [128, 128, 512], stage=3, block=1) x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=2) x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=3) x = senet_identity_block(x, 3, [128, 128, 512], stage=3, block=4) x = senet_conv_block(x, 3, [256, 256, 1024], stage=4, block=1) x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=2) x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=3) x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=4) x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=5) x = senet_identity_block(x, 3, [256, 256, 1024], stage=4, block=6) x = senet_conv_block(x, 3, [512, 512, 2048], stage=5, block=1) x = senet_identity_block(x, 3, [512, 512, 2048], stage=5, block=2) x = senet_identity_block(x, 3, [512, 512, 2048], stage=5, block=3) x = AveragePooling2D((7, 7), name='avg_pool')(x) if include_top: x = Flatten()(x) x = Dense(classes, activation='softmax', name='classifier')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='vggface_senet50') # load weights if weights == 'vggface': if include_top: weights_path = get_file('rcmalli_vggface_tf_senet50.h5', utils.SENET50_WEIGHTS_PATH, cache_subdir=utils.VGGFACE_DIR) else: weights_path = get_file('rcmalli_vggface_tf_notop_senet50.h5', utils.SENET50_WEIGHTS_PATH_NO_TOP, cache_subdir=utils.VGGFACE_DIR) model.load_weights(weights_path) if K.backend() == 'theano': layer_utils.convert_all_kernels_in_model(model) if include_top: maxpool = model.get_layer(name='avg_pool') shape = maxpool.output_shape[1:] dense = model.get_layer(name='classifier') layer_utils.convert_dense_weights_data_format(dense, shape, 'channels_first') if K.image_data_format() == 'channels_first' and K.backend() == 'tensorflow': warnings.warn('You are using the TensorFlow backend, yet you ' 'are using the Theano ' 'image data format convention ' '(`image_data_format="channels_first"`). ' 'For best performance, set ' '`image_data_format="channels_last"` in ' 'your Keras config ' 'at ~/.keras/keras.json.') elif weights is not None: model.load_weights(weights) return model
def VGG16(include_top=True, weights='vggface', input_tensor=None, input_shape=None, pooling=None, classes=2622): input_shape = _obtain_input_shape(input_shape, default_size=224, min_size=48, data_format=K.image_data_format(), require_flatten=include_top) if input_tensor is None: img_input = Input(shape=input_shape) else: if not K.is_keras_tensor(input_tensor): img_input = Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor # Block 1 x = Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_1')( img_input) x = Conv2D(64, (3, 3), activation='relu', padding='same', name='conv1_2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool1')(x) # Block 2 x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_1')( x) x = Conv2D(128, (3, 3), activation='relu', padding='same', name='conv2_2')( x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool2')(x) # Block 3 x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_1')( x) x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_2')( x) x = Conv2D(256, (3, 3), activation='relu', padding='same', name='conv3_3')( x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool3')(x) # Block 4 x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_1')( x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_2')( x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv4_3')( x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool4')(x) # Block 5 x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_1')( x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_2')( x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='conv5_3')( x) x = MaxPooling2D((2, 2), strides=(2, 2), name='pool5')(x) if include_top: # Classification block x = Flatten(name='flatten')(x) x = Dense(4096, name='fc6')(x) x = Activation('relu', name='fc6/relu')(x) x = Dense(4096, name='fc7')(x) x = Activation('relu', name='fc7/relu')(x) x = Dense(classes, name='fc8')(x) x = Activation('softmax', name='fc8/softmax')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='vggface_vgg16') # load weights if weights == 'vggface': if include_top: weights_path = get_file('rcmalli_vggface_tf_vgg16.h5', utils. VGG16_WEIGHTS_PATH, cache_subdir=utils.VGGFACE_DIR) else: weights_path = get_file('rcmalli_vggface_tf_notop_vgg16.h5', utils.VGG16_WEIGHTS_PATH_NO_TOP, cache_subdir=utils.VGGFACE_DIR) model.load_weights(weights_path, by_name=True) if K.backend() == 'theano': layer_utils.convert_all_kernels_in_model(model) if K.image_data_format() == 'channels_first': if include_top: maxpool = model.get_layer(name='pool5') shape = maxpool.output_shape[1:] dense = model.get_layer(name='fc6') layer_utils.convert_dense_weights_data_format(dense, shape, 'channels_first') if K.backend() == 'tensorflow': warnings.warn('You are using the TensorFlow backend, yet you ' 'are using the Theano ' 'image data format convention ' '(`image_data_format="channels_first"`). ' 'For best performance, set ' '`image_data_format="channels_last"` in ' 'your Keras config ' 'at ~/.keras/keras.json.') return model
def autoencoder_multitask(dataset, adj, feats, labels, weights=None): adj = sp.hstack([adj, feats]) h, w = adj.shape kwargs = dict( use_bias=True, kernel_initializer='glorot_normal', kernel_regularizer=None, bias_initializer='zeros', bias_regularizer=None, trainable=True, ) data = Input(shape=(w, ), dtype=np.float32, name='data') ### First set of encoding transformation ### encoded = Dense(256, activation='relu', name='encoded1', **kwargs)(data) ### Second set of encoding transformation ### encoded = Dense(128, activation='relu', name='encoded2', **kwargs)(encoded) if dataset == 'pubmed': encoded = Dropout(rate=0.5, name='drop')(encoded) else: encoded = Dropout(rate=0.8, name='drop')(encoded) # the encoder model maps an input to its encoded representation encoder = Model([data], encoded) encoded1 = encoder.get_layer('encoded1') encoded2 = encoder.get_layer('encoded2') ### First set of decoding transformation ### decoded = DenseTied(256, tie_to=encoded2, transpose=True, activation='relu', name='decoded2')(encoded) ### Node classification ### feat_data = Input(shape=(feats.shape[1], )) pred1 = Dense(labels.shape[1], activation='linear')(feat_data) pred2 = Dense(labels.shape[1], activation='linear')(decoded) prediction = add([pred1, pred2], name='prediction') ### Second set of decoding transformation - reconstruction ### decoded = DenseTied(w, tie_to=encoded1, transpose=True, activation='linear', name='decoded1')(decoded) # compile the autoencoder adam = optimizers.Adam(lr=0.001, decay=0.0) autoencoder = Model(inputs=[data, feat_data], outputs=[decoded, prediction]) autoencoder.compile(optimizer=adam, loss={ 'decoded1': mbce, 'prediction': masked_categorical_crossentropy }, loss_weights={ 'decoded1': 1.0, 'prediction': 1.0 }) if weights is not None: autoencoder.load_weights(weights) return encoder, autoencoder
z = Dropout(dropout_prob[1])(z) z = Dense(hidden_dims, activation="relu")(z) model_output = Dense(1, activation="sigmoid")(z) model = Model(model_input, model_output) model.compile(loss="binary_crossentropy", optimizer="adam", metrics=["accuracy"]) #print(model.summary()) # Initialize weights with word2vec weights = np.array([v for v in embedding_weights.values()]) print("Initializing embedding layer with word2vec weights, shape", weights.shape) embedding_layer = model.get_layer("embedding") embedding_layer.set_weights([weights]) #------------------------------------------------------------------------------------- history = model.fit(x_train, y_train, batch_size=batch_size, epochs=num_epochs, validation_data=(x_test, y_test), verbose=2) epoch_nr = 0 val_loss = 1.00 for epoch in range(0, num_epochs, 1):
def autoencoder(sparse_net, adj, weights=None): h, w = adj.shape kwargs = dict( use_bias=True, kernel_initializer='glorot_normal', kernel_regularizer=None, bias_initializer='zeros', bias_regularizer=None, trainable=True, ) data = Input(shape=(w, ), dtype=np.float32, name='data') if sparse_net: # for conflict, metabolic, protein networks noisy_data = Dropout(rate=0.2, name='drop0')(data) else: # for citation, blogcatalog, arxiv-grqc, and powergrid networks noisy_data = Dropout(rate=0.5, name='drop0')(data) ### First set of encoding transformation ### encoded = Dense(256, activation='relu', name='encoded1', **kwargs)(noisy_data) if sparse_net: encoded = Lambda(mvn, name='mvn1')(encoded) encoded = Dropout(rate=0.5, name='drop1')(encoded) ### Second set of encoding transformation ### encoded = Dense(128, activation='relu', name='encoded2', **kwargs)(encoded) if sparse_net: encoded = Lambda(mvn, name='mvn2')(encoded) encoded = Dropout(rate=0.5, name='drop2')(encoded) # the encoder model maps an input to its encoded representation encoder = Model([data], encoded) encoded1 = encoder.get_layer('encoded1') encoded2 = encoder.get_layer('encoded2') ### First set of decoding transformation ### decoded = DenseTied(256, tie_to=encoded2, transpose=True, activation='relu', name='decoded2')(encoded) if sparse_net: decoded = Lambda(mvn, name='mvn3')(decoded) decoded = Dropout(rate=0.5, name='drop3')(decoded) ### Second set of decoding transformation - reconstruction ### decoded = DenseTied(w, tie_to=encoded1, transpose=True, activation='linear', name='decoded1')(decoded) # compile the autoencoder adam = optimizers.Adam(lr=0.001, decay=0.0) autoencoder = Model(inputs=[data], outputs=[decoded]) autoencoder.compile(optimizer=adam, loss=mbce) if weights is not None: autoencoder.load_weights(weights) return encoder, autoencoder
new_lay.set_weights(lay.get_weights()) model = Model(inp, out) model.summary() # Define the loss # Content loss content_layers = ['block4_conv2'] content_loss_op = tf.Variable(0.0) coef = 1 for i in range(len(content_layers)): layer_name = content_layers[i] content_features = model.get_layer(layer_name).output content_features = sess.run(content_features, feed_dict = {creation: np.expand_dims(content_img, 0)}) content_features = tf.constant(content_features) creation_features = model.get_layer(layer_name).output weight = coef ** i content_loss_op += weight * utils.mse(content_features, creation_features) content_loss_op /= float(len(content_layers)) # Style loss style_layers = [ 'block1_conv1', #'block1_conv2', 'block2_conv1', #'block2_conv2', 'block3_conv1', #'block3_conv2', 'block3_conv3', 'block4_conv1', #'block4_conv2', 'block4_conv3',
def ResNet50(require_flatten=False, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000): """Instantiates the ResNet50 architecture. Optionally loads weights pre-trained on ImageNet. Note that when using TensorFlow, for best performance you should set `image_data_format="channels_last"` in your Keras config at ~/.keras/keras.json. The model and the weights are compatible with both TensorFlow and Theano. The data format convention used by the model is the one specified in your Keras config file. # Arguments include_top: whether to include the fully-connected layer at the top of the network. weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet). input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or `(3, 224, 244)` (with `channels_first` data format). It should have exactly 3 inputs channels, and width and height should be no smaller than 197. E.g. `(200, 200, 3)` would be one valid value. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if weights not in {'imagenet', None}: raise ValueError('The `weights` argument should be either ' '`None` (random initialization) or `imagenet` ' '(pre-training on ImageNet).') if weights == 'imagenet' and require_flatten and classes != 1000: raise ValueError('If using `weights` as imagenet with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape input_shape = _obtain_input_shape(input_shape, default_size=224, min_size=197, data_format=K.image_data_format(), require_flatten=require_flatten) if input_tensor is None: img_input = Input(shape=input_shape) else: if not K.is_keras_tensor(input_tensor): img_input = Input(tensor=input_tensor, shape=input_shape) else: img_input = input_tensor if K.image_data_format() == 'channels_last': bn_axis = 3 else: bn_axis = 1 x = ZeroPadding2D((3, 3))(img_input) x = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(x) x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x) x = Activation('relu')(x) x = MaxPooling2D((3, 3), strides=(2, 2))(x) x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1)) x = identity_block(x, 3, [64, 64, 256], stage=2, block='b') x = identity_block(x, 3, [64, 64, 256], stage=2, block='c') x = conv_block(x, 3, [128, 128, 512], stage=3, block='a') x = identity_block(x, 3, [128, 128, 512], stage=3, block='b') x = identity_block(x, 3, [128, 128, 512], stage=3, block='c') x = identity_block(x, 3, [128, 128, 512], stage=3, block='d') x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e') x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f') x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b') x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c') x = AveragePooling2D((7, 7), name='avg_pool')(x) if require_flatten: x = Flatten(name='flatten1')(x) x = Dense(classes, activation='softmax', name='fc1000')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) # Ensure that the model takes into account # any potential predecessors of `input_tensor`. if input_tensor is not None: inputs = get_source_inputs(input_tensor) else: inputs = img_input # Create model. model = Model(inputs, x, name='resnet50') # load weights if weights == 'imagenet': if require_flatten: weights_path = get_file( 'resnet50_weights_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH, cache_subdir='models', md5_hash='a7b3fe01876f51b976af0dea6bc144eb') else: weights_path = get_file( 'resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5', WEIGHTS_PATH_NO_TOP, cache_subdir='models', md5_hash='a268eb855778b3df3c7506639542a6af') model.load_weights(weights_path) if K.backend() == 'theano': layer_utils.convert_all_kernels_in_model(model) if K.image_data_format() == 'channels_first': if require_flatten: maxpool = model.get_layer(name='avg_pool') shape = maxpool.output_shape[1:] dense = model.get_layer(name='fc1000') layer_utils.convert_dense_weights_data_format( dense, shape, 'channels_first') if K.backend() == 'tensorflow': warnings.warn('You are using the TensorFlow backend, yet you ' 'are using the Theano ' 'image data format convention ' '(`image_data_format="channels_first"`). ' 'For best performance, set ' '`image_data_format="channels_last"` in ' 'your Keras config ' 'at ~/.keras/keras.json.') return model
def seq2seq_architecture(latent_size, vocabulary_size, embedding_matrix, batch_size, epochs, train_article, train_summary, train_target): # encoder encoder_inputs = Input(shape=(None, ), name='Encoder-Input') encoder_embeddings = Embedding( vocabulary_size + 1, 300, weights=[embedding_matrix], trainable=False, mask_zero=True, name='Encoder-Word-Embedding')(encoder_inputs) encoder_embeddings = BatchNormalization( name='Encoder-Batch-Normalization')(encoder_embeddings) _, state_h, state_c = LSTM(latent_size, return_state=True, dropout=0.2, recurrent_dropout=0.2, name='Encoder-LSTM')(encoder_embeddings) encoder_states = [state_h, state_c] encoder_model = Model(inputs=encoder_inputs, outputs=encoder_states, name='Encoder-Model') encoder_outputs = encoder_model(encoder_inputs) # decoder decoder_inputs = Input(shape=(None, ), name='Decoder-Input') decoder_embeddings = Embedding( vocabulary_size + 1, 300, weights=[embedding_matrix], trainable=False, mask_zero=True, name='Decoder-Word-Embedding')(decoder_inputs) decoder_embeddings = BatchNormalization( name='Decoder-Batch-Normalization-1')(decoder_embeddings) decoder_lstm = LSTM(latent_size, return_state=True, return_sequences=True, dropout=0.2, recurrent_dropout=0.2, name='Decoder-LSTM') decoder_lstm_outputs, _, _ = decoder_lstm(decoder_embeddings, initial_state=encoder_outputs) decoder_batchnorm = BatchNormalization( name='Decoder-Batch-Normalization-2')(decoder_lstm_outputs) decoder_outputs = Dense(vocabulary_size + 1, activation='softmax', name='Final-Output-Dense')(decoder_batchnorm) seq2seq_model = Model([encoder_inputs, decoder_inputs], decoder_outputs) seq2seq_model.compile(optimizer="adam", loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy']) seq2seq_model.summary() classes = [item for sublist in train_summary.tolist() for item in sublist] class_weights = class_weight.compute_class_weight('balanced', np.unique(classes), classes) e_stopping = EarlyStopping(monitor='val_loss', patience=4, verbose=1, mode='min', restore_best_weights=True) history = seq2seq_model.fit(x=[train_article, train_summary], y=np.expand_dims(train_target, -1), batch_size=batch_size, epochs=epochs, validation_split=0.1, callbacks=[e_stopping], class_weight=class_weights) f = open("data/models/lstm_results.txt", "w", encoding="utf-8") f.write("LSTM \n layers: 1 \n latent size: " + str(latent_size) + "\n vocab size: " + str(vocabulary_size) + "\n") f.close() history_dict = history.history plot_loss(history_dict) # inference encoder_model = seq2seq_model.get_layer('Encoder-Model') decoder_inputs = seq2seq_model.get_layer('Decoder-Input').input decoder_embeddings = seq2seq_model.get_layer('Decoder-Word-Embedding')( decoder_inputs) decoder_embeddings = seq2seq_model.get_layer( 'Decoder-Batch-Normalization-1')(decoder_embeddings) inference_state_h_input = Input(shape=(latent_size, ), name='Hidden-State-Input') inference_state_c_input = Input(shape=(latent_size, ), name='Cell-State-Input') lstm_out, lstm_state_h_out, lstm_state_c_out = seq2seq_model.get_layer( 'Decoder-LSTM')([ decoder_embeddings, inference_state_h_input, inference_state_c_input ]) decoder_outputs = seq2seq_model.get_layer('Decoder-Batch-Normalization-2')( lstm_out) dense_out = seq2seq_model.get_layer('Final-Output-Dense')(decoder_outputs) decoder_model = Model( [decoder_inputs, inference_state_h_input, inference_state_c_input], [dense_out, lstm_state_h_out, lstm_state_c_out]) return encoder_model, decoder_model
#!/usr/bin/env python # -*- coding: utf-8 -*- # @Project : tql-Python. # @File : mid_layer # @Time : 2019-07-12 16:33 # @Author : yuanjie # @Email : [email protected] # @Software : PyCharm # @Description : """ https://www.tensorflow.org/beta/tutorials/keras/feature_columns """ from tensorflow.python.keras.layers import Input, Embedding, Reshape, Activation from tensorflow.python.keras.models import Model input_model = Input(shape=(1, )) output_store = Embedding(1115, 10, name='store_embedding')(input_model) output_store = Reshape(target_shape=(10, ))(output_store) output_model = Activation('sigmoid')(output_store) model = Model(inputs=input_model, outputs=output_model) model.summary() embed = Model(inputs=model.input, outputs=model.get_layer(index=1).output) # 以这个model的预测值作为输出 embed.predict([[1]])
from fashion.data.data_generatorAGR import DataGeneratorAgro from fashion.settings import settings # set gpu os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = "0" os.environ["KERAS_BACKEND"] = "tensorflow" # gpu growth config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) K.set_session(sess) model = ResNet50(weights='imagenet') # input = (224,224,3) LAYER = "avg_pool" model = Model(inputs=model.input, outputs=model.get_layer(LAYER).output) pic_paths = glob.glob(settings["agro_path"] + "**/*", recursive=True) # WARNING WE SLICE SOME VALUES pic_df = pd.DataFrame({"path": pic_paths}) #[:len(pic_paths) - (len(pic_paths) % 16)] data_generatorAgro = DataGeneratorAgro(pic_df, batch_size=128) predict = model.predict_generator(data_generatorAgro, verbose=1) np.save("AGRO.npy", predict)
class DEC(object): def __init__(self, dims, n_clusters=10, alpha=1.0, init='glorot_uniform'): super(DEC, self).__init__() self.dims = dims self.input_dim = dims[0] self.n_stacks = len(self.dims) - 1 self.n_clusters = n_clusters self.alpha = alpha self.encoder = autoencoder(self.dims, init=init) # prepare DEC model clustering_layer = ClusteringLayer(self.n_clusters, name='clustering')( self.encoder.output) self.model = Model(inputs=self.encoder.input, outputs=clustering_layer) def load_weights(self, weights): # load weights of DEC model self.model.load_weights(weights) def extract_features(self, x): return self.encoder.predict(x) def predict( self, x): # predict cluster labels using the output of clustering layer q = self.model.predict(x, verbose=0) return q.argmax(1) @staticmethod def target_distribution(q): weight = q**2 / q.sum(0) return (weight.T / weight.sum(1)).T def compile(self, optimizer='sgd', loss='kld'): self.model.compile(optimizer=optimizer, loss=loss) def fit(self, x, y=None, maxiter=2e4, batch_size=256, tol=1e-3, update_interval=140, save_dir='./results/temp'): print('Update interval', update_interval) save_interval = int(x.shape[0] / batch_size) * 5 # 5 epochs print('Save interval', save_interval) # Step 1: initialize cluster centers using k-means t1 = time() print('Initializing cluster centers with k-means.') kmeans = KMeans(n_clusters=self.n_clusters, n_init=20) y_pred = kmeans.fit_predict(self.encoder.predict(x)) y_pred_last = np.copy(y_pred) self.model.get_layer(name='clustering').set_weights( [kmeans.cluster_centers_]) # Step 2: deep clustering # logging file import csv logfile = open(save_dir + '/dec_log.csv', 'w') logwriter = csv.DictWriter( logfile, fieldnames=['iter', 'acc', 'nmi', 'ari', 'loss']) logwriter.writeheader() loss = 0 index = 0 index_array = np.arange(x.shape[0]) for ite in range(int(maxiter)): if ite % update_interval == 0: q = self.model.predict(x, verbose=0) p = self.target_distribution( q) # update the auxiliary target distribution p # evaluate the clustering performance y_pred = q.argmax(1) if y is not None: acc = np.round(metrics.acc(y, y_pred), 5) nmi = np.round(metrics.nmi(y, y_pred), 5) ari = np.round(metrics.ari(y, y_pred), 5) loss = np.round(loss, 5) logdict = dict(iter=ite, acc=acc, nmi=nmi, ari=ari, loss=loss) logwriter.writerow(logdict) print( 'Iter %d: acc = %.5f, nmi = %.5f, ari = %.5f' % (ite, acc, nmi, ari), ' ; loss=', loss) # check stop criterion delta_label = np.sum(y_pred != y_pred_last).astype( np.float32) / y_pred.shape[0] y_pred_last = np.copy(y_pred) if ite > 0 and delta_label < tol: print('delta_label ', delta_label, '< tol ', tol) print('Reached tolerance threshold. Stopping training.') logfile.close() break # train on batch # if index == 0: # np.random.shuffle(index_array) idx = index_array[index * batch_size:min((index + 1) * batch_size, x.shape[0])] loss = self.model.train_on_batch(x=x[idx], y=p[idx]) index = index + 1 if (index + 1) * batch_size <= x.shape[0] else 0 # save intermediate model if ite % save_interval == 0: print('saving model to:', save_dir + '/DEC_model_' + str(ite) + '.h5') self.model.save_weights(save_dir + '/DEC_model_' + str(ite) + '.h5') ite += 1 # save the trained model logfile.close() print('saving model to:', save_dir + '/DEC_model_final.h5') self.model.save_weights(save_dir + '/DEC_model_final.h5') return y_pred
def VGG16(include_top=True, weights='imagenet', input_tensor=None, input_shape=None, pooling=None, classes=1000): """Instantiates the VGG16 architecture. Optionally loads weights pre-trained on ImageNet. Note that when using TensorFlow, for best performance you should set `image_data_format="channels_last"` in your Keras config at ~/.keras/keras.json. The model and the weights are compatible with both TensorFlow and Theano. The data format convention used by the model is the one specified in your Keras config file. # Arguments include_top: whether to include the 3 fully-connected layers at the top of the network. weights: one of `None` (random initialization) or "imagenet" (pre-training on ImageNet). input_tensor: optional Keras tensor (i.e. output of `layers.Input()`) to use as image input for the model. input_shape: optional shape tuple, only to be specified if `include_top` is False (otherwise the input shape has to be `(224, 224, 3)` (with `channels_last` data format) or `(3, 224, 244)` (with `channels_first` data format). It should have exactly 3 inputs channels, and width and height should be no smaller than 48. E.g. `(200, 200, 3)` would be one valid value. pooling: Optional pooling mode for feature extraction when `include_top` is `False`. - `None` means that the output of the model will be the 4D tensor output of the last convolutional layer. - `avg` means that global average pooling will be applied to the output of the last convolutional layer, and thus the output of the model will be a 2D tensor. - `max` means that global max pooling will be applied. classes: optional number of classes to classify images into, only to be specified if `include_top` is True, and if no `weights` argument is specified. # Returns A Keras model instance. # Raises ValueError: in case of invalid argument for `weights`, or invalid input shape. """ if weights not in {'imagenet', None}: raise ValueError('The `weights` argument should be either ' '`None` (random initialization) or `imagenet` ' '(pre-training on ImageNet).') if weights == 'imagenet' and include_top and classes != 1000: raise ValueError('If using `weights` as imagenet with `include_top`' ' as true, `classes` should be 1000') # Determine proper input shape img_input = Input(tensor=input_tensor, shape=input_shape) # Block 1 x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv1')(img_input) x = Conv2D(64, (3, 3), activation='relu', padding='same', name='block1_conv2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x) # Block 2 x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv1')(x) x = Conv2D(128, (3, 3), activation='relu', padding='same', name='block2_conv2')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x) # Block 3 x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv1')(x) x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv2')(x) x = Conv2D(256, (3, 3), activation='relu', padding='same', name='block3_conv3')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x) # Block 4 x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv1')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv2')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block4_conv3')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x) # Block 5 x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv1')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv2')(x) x = Conv2D(512, (3, 3), activation='relu', padding='same', name='block5_conv3')(x) x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x) if include_top: # Classification block x = Flatten(name='flatten')(x) x = Dense(4096, activation='relu', name='fc1')(x) x = Dense(4096, activation='relu', name='fc2')(x) x = Dense(classes, activation='softmax', name='predictions')(x) else: if pooling == 'avg': x = GlobalAveragePooling2D()(x) elif pooling == 'max': x = GlobalMaxPooling2D()(x) inputs = img_input # Create model. model = Model(inputs, x, name='vgg16') # load weights if weights == 'imagenet': if include_top: weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels.h5', WEIGHTS_PATH, cache_subdir='models') else: weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', WEIGHTS_PATH_NO_TOP, cache_subdir='models') model.load_weights(weights_path) if K.backend() == 'theano': layer_utils.convert_all_kernels_in_model(model) if K.image_data_format() == 'channels_first': if include_top: maxpool = model.get_layer(name='block5_pool') shape = maxpool.output_shape[1:] dense = model.get_layer(name='fc1') layer_utils.convert_dense_weights_data_format(dense, shape, 'channels_first') if K.backend() == 'tensorflow': warnings.warn('You are using the TensorFlow backend, yet you ' 'are using the Theano ' 'image data format convention ' '(`image_data_format="channels_first"`). ' 'For best performance, set ' '`image_data_format="channels_last"` in ' 'your Keras config ' 'at ~/.keras/keras.json.') return model