def simple_dense(): """Creates a simple sequential model, with 5 dense layers""" model = Sequential() model.add( Dense(units=32, input_shape=(32, ), use_bias=True, bias_constraint=MinMaxNorm(min_value=-1, max_value=1, rate=1.0, axis=0), bias_initializer=glorot_normal(seed=32), kernel_constraint=MaxNorm(max_value=1.5), kernel_initializer=glorot_uniform(seed=45))) model.add(Activation('relu')) model.add( Dense(units=32, activation='tanh', use_bias=False, activity_regularizer=l1_l2(l1=0.05, l2=0.05), kernel_constraint=MaxNorm(max_value=1.5), kernel_initializer=glorot_uniform(seed=45))) model.add( Dense(units=10, activation='softmax', use_bias=False, activity_regularizer=l1_l2(l1=0.05, l2=0.05), kernel_constraint=MaxNorm(max_value=1.5), kernel_initializer=glorot_uniform(seed=45))) return model
def buildMultiEmbModel(numClasses, seqLen, contextDim, initWeights, eventMap): """ Does a RNN over a CNN model with context information """ cnnHidden = 150 rnnHidden = 64 cl2 = .000 drop = .5 convSize = [2, 3, 4, 5] entDim = 20 distDim = 2 #NOTE hard coded for dataset #shape = (seqLen, 300 + entDim + distDim + contextDim) shape = (seqLen, 300 + entDim + distDim) model = Sequential() init = {wordEmbName: [initWeights]} #NOTE hardcoded for dataset emb = tripleEmbedding(init, seqLen, entDim, distDim, 12, 13, 0) #TODO remove print("embedding input {}".format(emb.input_shape)) print("embedding output {}".format(emb.output_shape)) print("next level shape {}".format(shape)) cnn = multiCNN(shape, cnnHidden, convSize, cl2) model.add(emb) model.add(cnn) #model.add(Conv1D(512, 2)) #model.add(MaxPooling1D(2)) #model.add(Bidirectional(GRU(128))) simple = Input(shape=(contextDim, )) level2 = Model(input=simple, output=simple) level3 = Sequential() level3.add(Merge([model, level2], mode="concat")) #level3.add(Dense(256)) #level3.add(Activation("relu")) level3.add(Dropout(drop)) level3.add(Dense(numClasses, W_constraint=MaxNorm(3.0))) level3.add(Activation("softmax")) level3.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy', microF1(eventMap)]) print(level3.summary()) return level3
def residual_block(residual_tensor): # res1 = BatchNormalization()(residual_tensor) res1 = Dropout(0.25)(residual_tensor) res1 = Conv3D(int(64 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(1, 1, 1), activation='relu', data_format='channels_first')(res1) # res2 = BatchNormalization()(res1) res2 = Dropout(0.25)(res1) res2 = Conv3D(int(64 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(1, 1, 1), activation='relu', data_format='channels_first')(res2) res3 = layers.add([res2, residual_tensor]) return res3
def load_model(self, num_layers=10): self.add( Dense(units=32, input_shape=(32, ), use_bias=True, bias_constraint=MinMaxNorm(min_value=-1, max_value=1, rate=1.0, axis=0), bias_initializer=glorot_normal(seed=32), kernel_constraint=MaxNorm(max_value=1.5), kernel_initializer=glorot_uniform(seed=45))) self.add( Dense(units=32, use_bias=True, activation='tanh', bias_constraint=MinMaxNorm(min_value=-1, max_value=1, rate=1.0, axis=0), bias_initializer=glorot_normal(seed=32), kernel_constraint=MaxNorm(max_value=1.5), kernel_initializer=glorot_uniform(seed=45))) self.add(Dropout(rate=0.5)) self.add( Dense(units=10, use_bias=True, activation='softmax', bias_constraint=MinMaxNorm(min_value=-1, max_value=1, rate=1.0, axis=0), bias_initializer=glorot_normal(seed=32), kernel_constraint=MaxNorm(max_value=1.5), kernel_initializer=glorot_uniform(seed=45)))
def buildCNNEmbModel(numClasses, seqLen, contextDim, initWeights, eventMap): """ Reproduces an CNN with embedding """ drop = .5 model = buildBaseCNN(seqLen, initWeights, 0) model.add(Dropout(drop)) model.add(Dense(numClasses, W_constraint=MaxNorm(3.0))) model.add(Activation("softmax")) model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy', microF1(eventMap)]) model.summary() return model
def return_norm(name, lstm_config, minimum, maximum, logger): """ Return Norm object to norm weight of neural network. """ log_name = name name = lstm_config[name.lower()] if name == 'maxnorm': logger.info("In {} use {} constraint with max={}".format( log_name, name, maximum)) return MaxNorm(maximum) if name == 'nonnegnorm': logger.info("In {} use {} constraint ".format(log_name, name)) return NonNeg() if name == 'minmaxnorm': logger.info("In {} use {} constraint with min={} and max={}".format( log_name, name, minimum, maximum)) return MinMaxNorm(minimum, maximum) else: logger.info("None constraint in {}.".format(log_name)) return None
def buildCNNEmbModel(numClasses, seqLen, contextDim, initWeights, eventMap): """ Reproduces an CNN with embedding """ cnnHidden = 75 cl2 = .000 drop = .5 convSize = [2, 3, 4, 5] entDim = 20 distDim = 2 fullEmbeddingDim = 300 + entDim + distDim #NOTE hard coded for dataset shape = (seqLen, fullEmbeddingDim) model = Sequential() init = {wordEmbName: [initWeights]} #NOTE hardcoded for dataset emb = tripleEmbedding(init, seqLen, entDim, distDim, 12, 13) cnn = multiCNN(shape, cnnHidden, convSize, cl2) model.add(emb) model.add(cnn) model.add(Dropout(drop)) model.add(Dense(numClasses, W_constraint=MaxNorm(3.0))) model.add(Activation("softmax")) model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']) print(model.summary()) return model
def __init__(self, input_dim, embedding_dim=[128, 128], use_bn=False, use_dp=False): super(SimpleMLP, self).__init__(name='mlp') self.use_bn = use_bn self.use_dp = use_dp if not isinstance(embedding_dim, list): embedding_dim = [embedding_dim] * 2 constraint = MaxNorm(1, axis=1) self.e1 = Embedding(input_dim[0], embedding_dim[0], embeddings_constraint=constraint) self.e2 = Embedding(input_dim[1], embedding_dim[1], embeddings_constraint=constraint) self.m = Concatenate() self.rate = 0.2 self.ls = [ Concatenate(), Dense(32), Dropout(self.rate), BatchNormalization(axis=-1), Activation('relu'), Dense(1, activation='sigmoid'), Reshape((-1, )) ] if self.use_dp: self.dp = Dropout(self.rate) if self.use_bn: self.bn = BatchNormalization(axis=-1)
def buildMultiEmbModel(numClasses, seqLen, contextDim, initWeights, eventMap): """ Does a RNN over a CNN model with context information """ drop = .5 model = buildBaseCNN(seqLen, initWeights, 0) simple = Input(shape=(contextDim,)) level2 = Model(input=simple, output=simple) level3 = Sequential() level3.add(Merge([model, level2], mode="concat")) level3.add(Dropout(drop)) level3.add(Dense(numClasses, W_constraint=MaxNorm(3.0))) level3.add(Activation("softmax")) level3.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy', microF1(eventMap)]) level3.summary() return level3
def add_full( self, model, hidden_dim, drop_out_prob, nb_class, ): model.add(Flatten()) model.add(Dense(hidden_dim, W_constraint=MaxNorm(m=9, axis=1))) # model.add(ReNorm()) # model.add(Activation('relu')) model.add(Dropout(drop_out_prob)) # model.add(LSTM(70)) # model.add(Dense(hidden_dim, W_constraint=MaxNorm(9))) # model.add(Dropout(0.5)) # model.add(Dense(2)) # model.add(Activation('softmax')) assert (nb_class > 1) if nb_class == 2: model.add(Dense(1)) model.add(Activation('sigmoid')) print 'begin compile..' model.compile(optimizer='adadelta', loss='binary_crossentropy', class_mode='binary') print 'end compile' else: model.add(Dense(nb_class)) model.add(Activation('softmax')) print 'begin compile..' model.compile(optimizer='rmsprop', loss='categorical_crossentropy', class_mode='categorical') print 'end compile'
def create_model(args, maxlen, vocab): def ortho_reg(weight_matrix): ### orthogonal regularization for aspect embedding matrix ### w_n = K.l2_normalize(weight_matrix, axis=-1) # K表示调用该函数的当前layer reg = K.sum( K.square( K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value))) # 自身矩阵的内积的平方根-自身特征值 = 越小, 越说明分量为0 return args.ortho_reg * reg # 这东西越小越好, 因为能保证各个特征分的越开 vocab_size = len(vocab) if args.emb_name: # 获取已经保存的embedding??? from w2vEmbReader import W2VEmbReader as EmbReader emb_reader = EmbReader( os.path.join( "/content/drive/My Drive/Attention-Based-Aspect-Extraction-master", "preprocessed_data", args.domain), args.emb_name) aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size) args.aspect_size = emb_reader.aspect_size args.emb_dim = emb_reader.emb_dim ##### Inputs ##### sentence_input = Input(shape=(maxlen, ), dtype='int32', name='sentence_input') neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input') ##### Construct word embedding layer ##### word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb', embeddings_constraint=MaxNorm(10)) ##### Compute sentence representation ##### pre-processing 根据attention组合句子 e_w = word_emb(sentence_input) # 将input转换为embedding y_s = Average()(e_w) # 默认求平均 layer att_weights = Attention(name='att_weights', W_constraint=MaxNorm(10), b_constraint=MaxNorm(10))([e_w, y_s]) # attention layer z_s = WeightedSum()([e_w, att_weights]) # encoding layer ##### Compute representations of negative instances ##### 增加准确性的tricks e_neg = word_emb(neg_input) z_n = Average()(e_neg) ##### Reconstruction ##### 构建dense层, 希望能够decoding attention sentences的特征 p_t = Dense(args.aspect_size)(z_s) p_t = Activation('softmax', name='p_t')(p_t) # softmax一下, nodes数量不改变, 数值被soft了一下 r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb', W_constraint=MaxNorm(10), W_regularizer=ortho_reg)( p_t) # 标准化0-10区间, 且正则项为自定义的ortho_reg ##### Loss ##### loss = MaxMargin(name='max_margin')([z_s, z_n, r_s]) # 自定义loss function??? 这是在做啥??? model = Model(inputs=[sentence_input, neg_input], outputs=[loss]) # negative input是需要自己分开数据集的吗?? ### Word embedding and aspect embedding initialization ###### if args.emb_name: from w2vEmbReader import W2VEmbReader as EmbReader logger.info('Initializing word embedding matrix') embs = model.get_layer('word_emb').embeddings K.set_value( embs, emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs))) logger.info( 'Initializing aspect embedding matrix as centroid of kmean clusters' ) # 为何初始化要用到kmeans K.set_value(model.get_layer('aspect_emb').W, aspect_matrix) # r-s return model
def __init__(self, input_dim, embedding_dim=128, use_bn=False, use_dp=False, embedding_method='DistMult'): super(LinkPredict, self).__init__(name='lp') self.use_bn = use_bn self.use_dp = use_dp if embedding_method == 'HolE': constraint = MaxNorm(1, axis=1) elif embedding_method == 'TransE': constraint = None elif embedding_method == 'DistMult': constraint = MaxNorm(1, axis=1) self.e1 = Embedding(input_dim[0], embedding_dim, embeddings_constraint=constraint) self.e2 = Embedding(input_dim[1], embedding_dim, embeddings_constraint=constraint) self.r1 = Embedding(input_dim[2], embedding_dim) self.r2 = Embedding(input_dim[3], embedding_dim) if embedding_method == 'DistMult': self.embedding = [ Multiply(), Lambda(lambda x: K.sum(x, axis=-1)), Activation('sigmoid'), Reshape((-1, )) ] elif embedding_method == 'HolE': self.embedding = [ Lambda(lambda x: HolE(x[0], x[1], x[2])), Activation('sigmoid'), Reshape((-1, )) ] elif embedding_method == 'TransE': self.embedding = [ Lambda(lambda x: TransE(x[0], x[1], x[2])), Activation('tanh'), Reshape((-1, )) ] else: raise NotImplementedError(embedding_method + ' not implemented') self.rate = 0.2 self.ls = [ Concatenate(), Dense(32), Dropout(self.rate), BatchNormalization(axis=-1), Activation('relu'), Dense(1, activation='sigmoid'), Reshape((-1, )) ] if self.use_dp: self.dp = Dropout(self.rate) if self.use_bn: self.bn1 = BatchNormalization(axis=-1) self.bn2 = BatchNormalization(axis=-1)
def vox_net(input_shape=None, input_tensor=None, downsize_filters_factor=1, pool_size=(2, 2, 2), initial_learning_rate=0.00001, convolutions=4, dropout=.25, filter_shape=(3, 3, 3), num_outputs=1, deconvolution=True, regression=True, output_shape=None): def residual_block(residual_tensor): # res1 = BatchNormalization()(residual_tensor) res1 = Dropout(0.25)(residual_tensor) res1 = Conv3D(int(64 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(1, 1, 1), activation='relu', data_format='channels_first')(res1) # res2 = BatchNormalization()(res1) res2 = Dropout(0.25)(res1) res2 = Conv3D(int(64 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(1, 1, 1), activation='relu', data_format='channels_first')(res2) res3 = layers.add([res2, residual_tensor]) return res3 inputs = Input(input_shape) conv1 = Conv3D(int(32 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(1, 1, 1), data_format='channels_first')(inputs) # conv2 = BatchNormalization()(conv1) conv2 = Dropout(0.25)(conv1) conv2 = Conv3D(int(64 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(1, 1, 1), activation='relu', data_format='channels_first')(conv2) # conv3 = BatchNormalization()(conv2) conv3 = Dropout(0.25)(conv2) conv3 = Conv3D(int(64 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(2, 2, 2), activation='relu', data_format='channels_first')(conv3) conv3 = residual_block(conv3) conv3 = residual_block(conv3) # conv4 = BatchNormalization()(conv4) conv4 = Dropout(0.25)(conv3) conv4 = Conv3D(int(64 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(2, 2, 2), activation='relu', data_format='channels_first')(conv4) conv4 = residual_block(conv4) conv4 = residual_block(conv4) # conv5 = BatchNormalization()(conv4) conv5 = Dropout(0.25)(conv4) conv5 = Conv3D(int(64 / downsize_filters_factor), 3, padding='same', kernel_constraint=MaxNorm(2.), strides=(2, 2, 2), activation='relu', data_format='channels_first')(conv5) conv5 = residual_block(conv5) conv5 = residual_block(conv5) upx3 = UpSampling3D(size=(2, 2, 2), data_format='channels_first')(conv3) upx4 = UpSampling3D(size=(4, 4, 4), data_format='channels_first')(conv4) upx5 = UpSampling3D(size=(8, 8, 8), data_format='channels_first')(conv5) upx = layers.add([conv2, upx3, upx4, upx5]) conv_last = Conv3D(int(num_outputs), (1, 1, 1), data_format='channels_first')(upx) if regression: act = Activation('relu')(conv_last) model = Model(inputs=inputs, outputs=act) model.compile(optimizer=Adam(lr=initial_learning_rate), loss=msq_loss, metrics=[msq]) else: act = Activation('sigmoid')(conv_last) model = Model(inputs=inputs, outputs=act) model.compile(optimizer=Adam(lr=initial_learning_rate), loss=dice_coef_loss, metrics=[dice_coef]) return model
save_to_json("xval_man_tokens.json", str(word_index)) reg = l2(0.01) reg_2 = l1_l2() # embed_init = RandomUniform(seed=313) embed_init = RandomUniform(seed=15) # Activity reg deals with outputs # Embeddings regs deals with hidden values my_embedding = Embedding(num_words, embedding_vector_length, embeddings_initializer=embed_init, activity_regularizer=reg, embeddings_constraint=MaxNorm(max_value=2, axis=0), embeddings_regularizer=None, input_length=max_review_length, trainable=True) print('Shape of data tensor:', embed_xvals.shape) print("xval sample", embed_xvals[156]) print('Shape of test tensor:', test_x.shape) print("test xval sample", test_x[0]) # MODEL CONSTRUCTION main_input = Input(shape=(max_review_length, ), dtype='int32') embed = my_embedding(main_input) embed = Dropout(0.2)(embed)
def create_model(args, maxlen, vocab): def ortho_reg(weight_matrix): ### orthogonal regularization for aspect embedding matrix ### w_n = K.l2_normalize(weight_matrix, axis=-1) reg = K.sum( K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value))) return args.ortho_reg * reg vocab_size = len(vocab) if args.emb_name: from w2vEmbReader import W2VEmbReader as EmbReader emb_reader = EmbReader( os.path.join("..", "preprocessed_data", args.domain), args.emb_name) aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size) args.aspect_size = emb_reader.aspect_size args.emb_dim = emb_reader.emb_dim ##### Inputs ##### sentence_input = Input(shape=(maxlen, ), dtype='int32', name='sentence_input') neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input') ##### Construct word embedding layer ##### word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb', embeddings_constraint=MaxNorm(10)) ##### Compute sentence representation ##### e_w = word_emb(sentence_input) y_s = Average()(e_w) att_weights = Attention(name='att_weights', W_constraint=MaxNorm(10), b_constraint=MaxNorm(10))([e_w, y_s]) z_s = WeightedSum()([e_w, att_weights]) ##### Compute representations of negative instances ##### e_neg = word_emb(neg_input) z_n = Average()(e_neg) ##### Reconstruction ##### p_t = Dense(args.aspect_size)(z_s) p_t = Activation('softmax', name='p_t')(p_t) r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb', W_constraint=MaxNorm(10), W_regularizer=ortho_reg)(p_t) ##### Loss ##### loss = MaxMargin(name='max_margin')([z_s, z_n, r_s]) model = Model(inputs=[sentence_input, neg_input], outputs=[loss]) ### Word embedding and aspect embedding initialization ###### if args.emb_name: from w2vEmbReader import W2VEmbReader as EmbReader logger.info('Initializing word embedding matrix') embs = model.get_layer('word_emb').embeddings K.set_value( embs, emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs))) logger.info( 'Initializing aspect embedding matrix as centroid of kmean clusters' ) K.set_value(model.get_layer('aspect_emb').W, aspect_matrix) return model
def main(model, params): datafolder = params['d'] training_passes = params['t'] eval_passes = params['e'] predict_passes = params['p'] batch_size = params['bs'] drop = params['drop'] dim = params['ed'] constr_dict = { 'maxnorm': MaxNorm(1, axis=1), 'unitnorm': UnitNorm(axis=1), 'nonneg': NonNeg() } reg_dict = {'l1': l1(0.01), 'l2': l2(0.01), 'l1_l2': l1_l2(0.01, 0.01)} train_file = datafolder + "train.txt" valid_file = datafolder + "valid.txt" test_file = datafolder + "test.txt" false_train_file = datafolder + "false_train.txt" E_mapping, R_mapping = mapping( [train_file, valid_file, test_file, false_train_file]) VOC_SIZE = len(list(E_mapping.keys())) PRED_SIZE = len(list(R_mapping.keys())) true_train = np.squeeze( np.asarray( list( data_iterator(train_file, E_mapping, R_mapping, batch_size=-1, mode=params['training_mode'])))) if params['reverse_labels']: #TransE true_train_labels = np.zeros(len(true_train.T)) else: true_train_labels = np.ones(len(true_train.T)) if params['false_mode'] == 'fromfile': false_train = np.squeeze( np.asarray( list( data_iterator(false_train_file, E_mapping, R_mapping, batch_size=-1, mode=params['training_mode'])))) else: s, p, o = true_train false_train = np.asarray( corrupt_triples(s, p, o, params['check'], params['false_mode'])) if params['reverse_labels']: false_train_labels = np.ones(len(false_train.T)) else: false_train_labels = np.zeros(len(false_train.T)) if params['constraint']: const = constr_dict[params['constraint']] else: const = None if params['regularizer']: reg = reg_dict[params['regularizer']] else: reg = None m = model(VOC_SIZE, PRED_SIZE, dim, embeddings_regularizer=const, embeddings_constraint=reg, dropout=params['drop']) m.compile(loss=params['loss'], optimizer='adagrad', metrics=['mae']) for i in range(training_passes): if params['false_mode'] != 'fromfile': s, p, o = true_train false_train = np.asarray( corrupt_triples(s, p, o, params['check'], params['false_mode'])) tmpX = np.concatenate([false_train.T, true_train.T], axis=0) tmpY = np.concatenate([false_train_labels.T, true_train_labels.T], axis=0) tmpY = tmpY * (1 - params['ls']) + params['ls'] / 2 m.fit(tmpX, tmpY, epochs=1, shuffle=True, batch_size=batch_size) try: if (i % eval_passes == 0 and i != 0) or (i == training_passes - 1 and eval_passes > 0): if params['filtered']: tmp = true_train.T else: tmp = [] res = evaluate(m, valid_file, E_mapping, R_mapping, params['reverse_labels'], tmp) print(res) except ZeroDivisionError: pass if params['store']: store_embedding(m, E_mapping, R_mapping, datafolder) if predict_passes > 0: print(predict_passes) test = np.squeeze( np.asarray( list( data_iterator(test_file, E_mapping, R_mapping, batch_size=-1, mode=params['training_mode'])))).T pred = m.predict(test) pred = [p[0] for p in pred] mapping_e = reverse_dict(E_mapping) mapping_r = reverse_dict(R_mapping) with open(params['output_file'], 'w') as f: for t, p in zip(test, pred): s, r, o = t s, r, o = mapping_e[s], mapping_r[r], mapping_e[o] string = '\t'.join(map(str, [s, r, o, p])) + '\n' f.write(string)
def create_model(args, maxlen, vocab): def ortho_reg(weight_matrix): ### orthogonal regularization for aspect embedding matrix ### w_n = K.l2_normalize(weight_matrix, axis=-1) reg = K.sum( K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value))) return args.ortho_reg * reg vocab_size = len(vocab) if args.emb_name: if args.emb_technique == "w2v": logger.info("Load {} glove embedding for {}".format( args.lang, config.word_emb_training_type)) if args.lang == 'en': emb_reader = EmbReader( config.emb_dir_en["w2v"].format( config.word_emb_training_type), args.emb_name) elif args.lang == 'de': #emb_reader = EmbReader(config.emb_dir_de["w2v"].format(config.word_emb_training_type), #args.emb_name) emb_reader = FineTuneEmbed_cca( '../preprocessed_data/german/w2v/fine_tuned', 'w2v_emb', '../preprocessed_data/german/w2v/full_trained', 'w2v_embedding_300') elif args.emb_technique == 'fasttext': if args.lang == 'en': #emb_reader = FastTextEmbReader(config.emb_dir_en["fasttext"].format(config.word_emb_training_type), #args.emb_name, config.fine_tuned_enabled) emb_reader = FineTuneEmbed_ortho_procrustes( '../preprocessed_data/fasttext/fine_tuned', 'fasttext_pre_trained', '../preprocessed_data/fasttext/full_trained', 'w2v_embedding_skipgram_300') elif args.lang == 'de': emb_reader = FastTextEmbReader( config.emb_dir_de["fasttext"].format( config.word_emb_training_type), args.emb_name, config.fine_tuned_enabled) #emb_reader = FineTuneEmbed_ortho_procrustes('../preprocessed_data/fasttext/fine_tuned','fasttext_pre_trained','../preprocessed_data/fasttext/full_trained', 'w2v_embedding_skipgram_300') elif args.emb_technique == "glove": if args.lang == 'de': logger.info('Load german glove embedding') emb_reader = GloveEmbedding(config.emb_dir_de["glove"], args.emb_name) else: logger.info("Load en glove embedding for {}".format( config.word_emb_training_type)) emb_reader = GloveEmbedding( config.emb_dir_en["glove"].format( config.word_emb_training_type), args.emb_name) elif args.emb_technique == "MUSE_supervised": emb_reader = MUSEEmbedding(config.emb_dir_biling['supervised'], args.emb_name) elif args.emb_technique == "MUSE_unsupervised": emb_reader = MUSEEmbedding(config.emb_dir_biling['unsupervised'], args.emb_name) aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size) args.aspect_size = emb_reader.aspect_size args.emb_dim = emb_reader.emb_dim ##### Inputs ##### sentence_input = Input(shape=(maxlen, ), dtype='int32', name='sentence_input') neg_input = Input(shape=(args.neg_size, maxlen), dtype='int32', name='neg_input') ##### Construct word embedding layer ##### word_emb = Embedding(vocab_size, args.emb_dim, mask_zero=True, name='word_emb', embeddings_constraint=MaxNorm(10)) ##### Compute sentence representation ##### e_w = word_emb(sentence_input) y_s = Average()(e_w) att_weights = Attention(name='att_weights', W_constraint=MaxNorm(10), b_constraint=MaxNorm(10))([e_w, y_s]) z_s = WeightedSum()([e_w, att_weights]) ##### Compute representations of negative instances ##### e_neg = word_emb(neg_input) z_n = Average()(e_neg) ##### Reconstruction ##### p_t = Dense(args.aspect_size)(z_s) p_t = Activation('softmax', name='p_t')(p_t) r_s = WeightedAspectEmb(args.aspect_size, args.emb_dim, name='aspect_emb', W_constraint=MaxNorm(10), W_regularizer=ortho_reg)(p_t) ##### Loss ##### loss = MaxMargin(name='max_margin')([z_s, z_n, r_s]) model = Model(inputs=[sentence_input, neg_input], outputs=[loss]) ### Word embedding and aspect embedding initialization ###### if args.emb_name: logger.info('Initializing word embedding matrix') embs = model.get_layer('word_emb').embeddings K.set_value( embs, emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs))) logger.info( 'Initializing aspect embedding matrix as centroid of kmean clusters' ) K.set_value(model.get_layer('aspect_emb').W, aspect_matrix) return model
def main(): # DATA INITIALIZATION split = 10 x = np.load('cardiac_records.npy') y = np.load('cardiac_targets.npy') x_split, y_split = kfold_split(x, y, split) print(x.shape) print(sum(y)) # CNN INITIALIZATION model = Sequential() model.add(BatchNormalization()) model.add(Dropout(rate=0)) # Convolution 1: model.add( Conv1D(filters=9, kernel_size=5, input_shape=(x_split.shape[2], 1), kernel_constraint=MaxNorm(3), kernel_initializer='uniform', activation='relu', padding='valid')) model.add(Dropout(rate=0)) # Convolution 2: model.add( Conv1D(filters=3, kernel_size=11, input_shape=(x_split.shape[2], 1), kernel_constraint=MaxNorm(3), kernel_initializer='uniform', activation='relu', padding='valid')) model.add(Dropout(rate=0)) model.add(Flatten()) # Fully connected dense layer model.add( Dense(units=x_split.shape[2], kernel_initializer='uniform', kernel_constraint=MaxNorm(3), activation='relu')) model.add(Dropout(rate=0)) # Add a single output logit layer: model.add( Dense(units=1, kernel_initializer='uniform', kernel_constraint=MaxNorm(3), activation='sigmoid')) adm = adam( lr=0.01, beta_1=0.8, beta_2=0.98, ) model.compile(optimizer=adm, loss='mean_squared_error', metrics=['accuracy']) print('TRAINING') epochs = 3 for ep in range(epochs): print("epoch: " + str(ep)) for k in range(split): x_split, y_split = kfold_split(x, y, split) train = np.ones((split, ), bool) train[k] = False x_train = x_split[train, :, :] y_train = y_split[train, :] x_train = np.reshape( x_train, [x_train.shape[0] * x_train.shape[1], x_train.shape[2], 1]) y_train = np.reshape(y_train, [y_train.shape[0] * y_train.shape[1]]) x_val = x_split[k, :, :] y_val = y_split[k, :] x_val = np.reshape(x_val, [x_val.shape[0], x_val.shape[1], 1]) model.fit(x_train, y_train, validation_data=(x_val, y_val), epochs=1, shuffle=True, verbose=2) model.save('desi_cnn.h5') print("Done")