def simple_dense():
"""Creates a simple sequential model, with 5 dense layers"""
model = Sequential()
model.add(
Dense(units=32,
input_shape=(32, ),
use_bias=True,
bias_constraint=MinMaxNorm(min_value=-1,
max_value=1,
rate=1.0,
axis=0),
bias_initializer=glorot_normal(seed=32),
kernel_constraint=MaxNorm(max_value=1.5),
kernel_initializer=glorot_uniform(seed=45)))
model.add(Activation('relu'))
model.add(
Dense(units=32,
activation='tanh',
use_bias=False,
activity_regularizer=l1_l2(l1=0.05, l2=0.05),
kernel_constraint=MaxNorm(max_value=1.5),
kernel_initializer=glorot_uniform(seed=45)))
model.add(
Dense(units=10,
activation='softmax',
use_bias=False,
activity_regularizer=l1_l2(l1=0.05, l2=0.05),
kernel_constraint=MaxNorm(max_value=1.5),
kernel_initializer=glorot_uniform(seed=45)))
return model
def buildMultiEmbModel(numClasses, seqLen, contextDim, initWeights, eventMap):
"""
Does a RNN over a CNN model with context information
"""
cnnHidden = 150
rnnHidden = 64
cl2 = .000
drop = .5
convSize = [2, 3, 4, 5]
entDim = 20
distDim = 2
#NOTE hard coded for dataset
#shape = (seqLen, 300 + entDim + distDim + contextDim)
shape = (seqLen, 300 + entDim + distDim)
model = Sequential()
init = {wordEmbName: [initWeights]}
#NOTE hardcoded for dataset
emb = tripleEmbedding(init, seqLen, entDim, distDim, 12, 13, 0)
#TODO remove
print("embedding input {}".format(emb.input_shape))
print("embedding output {}".format(emb.output_shape))
print("next level shape {}".format(shape))
cnn = multiCNN(shape, cnnHidden, convSize, cl2)
model.add(emb)
model.add(cnn)
#model.add(Conv1D(512, 2))
#model.add(MaxPooling1D(2))
#model.add(Bidirectional(GRU(128)))
simple = Input(shape=(contextDim, ))
level2 = Model(input=simple, output=simple)
level3 = Sequential()
level3.add(Merge([model, level2], mode="concat"))
#level3.add(Dense(256))
#level3.add(Activation("relu"))
level3.add(Dropout(drop))
level3.add(Dense(numClasses, W_constraint=MaxNorm(3.0)))
level3.add(Activation("softmax"))
level3.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy', microF1(eventMap)])
print(level3.summary())
return level3
def residual_block(residual_tensor):
# res1 = BatchNormalization()(residual_tensor)
res1 = Dropout(0.25)(residual_tensor)
res1 = Conv3D(int(64 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(1, 1, 1),
activation='relu',
data_format='channels_first')(res1)
# res2 = BatchNormalization()(res1)
res2 = Dropout(0.25)(res1)
res2 = Conv3D(int(64 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(1, 1, 1),
activation='relu',
data_format='channels_first')(res2)
res3 = layers.add([res2, residual_tensor])
return res3
def load_model(self, num_layers=10):
self.add(
Dense(units=32,
input_shape=(32, ),
use_bias=True,
bias_constraint=MinMaxNorm(min_value=-1,
max_value=1,
rate=1.0,
axis=0),
bias_initializer=glorot_normal(seed=32),
kernel_constraint=MaxNorm(max_value=1.5),
kernel_initializer=glorot_uniform(seed=45)))
self.add(
Dense(units=32,
use_bias=True,
activation='tanh',
bias_constraint=MinMaxNorm(min_value=-1,
max_value=1,
rate=1.0,
axis=0),
bias_initializer=glorot_normal(seed=32),
kernel_constraint=MaxNorm(max_value=1.5),
kernel_initializer=glorot_uniform(seed=45)))
self.add(Dropout(rate=0.5))
self.add(
Dense(units=10,
use_bias=True,
activation='softmax',
bias_constraint=MinMaxNorm(min_value=-1,
max_value=1,
rate=1.0,
axis=0),
bias_initializer=glorot_normal(seed=32),
kernel_constraint=MaxNorm(max_value=1.5),
kernel_initializer=glorot_uniform(seed=45)))
def buildCNNEmbModel(numClasses, seqLen, contextDim, initWeights, eventMap):
"""
Reproduces an CNN with embedding
"""
drop = .5
model = buildBaseCNN(seqLen, initWeights, 0)
model.add(Dropout(drop))
model.add(Dense(numClasses, W_constraint=MaxNorm(3.0)))
model.add(Activation("softmax"))
model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy', microF1(eventMap)])
model.summary()
return model
def return_norm(name, lstm_config, minimum, maximum, logger):
"""
Return Norm object to norm weight of neural network.
"""
log_name = name
name = lstm_config[name.lower()]
if name == 'maxnorm':
logger.info("In {} use {} constraint with max={}".format(
log_name, name, maximum))
return MaxNorm(maximum)
if name == 'nonnegnorm':
logger.info("In {} use {} constraint ".format(log_name, name))
return NonNeg()
if name == 'minmaxnorm':
logger.info("In {} use {} constraint with min={} and max={}".format(
log_name, name, minimum, maximum))
return MinMaxNorm(minimum, maximum)
else:
logger.info("None constraint in {}.".format(log_name))
return None
def buildCNNEmbModel(numClasses, seqLen, contextDim, initWeights, eventMap):
"""
Reproduces an CNN with embedding
"""
cnnHidden = 75
cl2 = .000
drop = .5
convSize = [2, 3, 4, 5]
entDim = 20
distDim = 2
fullEmbeddingDim = 300 + entDim + distDim
#NOTE hard coded for dataset
shape = (seqLen, fullEmbeddingDim)
model = Sequential()
init = {wordEmbName: [initWeights]}
#NOTE hardcoded for dataset
emb = tripleEmbedding(init, seqLen, entDim, distDim, 12, 13)
cnn = multiCNN(shape, cnnHidden, convSize, cl2)
model.add(emb)
model.add(cnn)
model.add(Dropout(drop))
model.add(Dense(numClasses, W_constraint=MaxNorm(3.0)))
model.add(Activation("softmax"))
model.compile(loss='categorical_crossentropy',
optimizer='adadelta',
metrics=['accuracy'])
print(model.summary())
return model
def __init__(self,
input_dim,
embedding_dim=[128, 128],
use_bn=False,
use_dp=False):
super(SimpleMLP, self).__init__(name='mlp')
self.use_bn = use_bn
self.use_dp = use_dp
if not isinstance(embedding_dim, list):
embedding_dim = [embedding_dim] * 2
constraint = MaxNorm(1, axis=1)
self.e1 = Embedding(input_dim[0],
embedding_dim[0],
embeddings_constraint=constraint)
self.e2 = Embedding(input_dim[1],
embedding_dim[1],
embeddings_constraint=constraint)
self.m = Concatenate()
self.rate = 0.2
self.ls = [
Concatenate(),
Dense(32),
Dropout(self.rate),
BatchNormalization(axis=-1),
Activation('relu'),
Dense(1, activation='sigmoid'),
Reshape((-1, ))
]
if self.use_dp:
self.dp = Dropout(self.rate)
if self.use_bn:
self.bn = BatchNormalization(axis=-1)
def buildMultiEmbModel(numClasses, seqLen, contextDim, initWeights, eventMap):
"""
Does a RNN over a CNN model with context information
"""
drop = .5
model = buildBaseCNN(seqLen, initWeights, 0)
simple = Input(shape=(contextDim,))
level2 = Model(input=simple, output=simple)
level3 = Sequential()
level3.add(Merge([model, level2], mode="concat"))
level3.add(Dropout(drop))
level3.add(Dense(numClasses, W_constraint=MaxNorm(3.0)))
level3.add(Activation("softmax"))
level3.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy', microF1(eventMap)])
level3.summary()
return level3
def add_full(
self,
model,
hidden_dim,
drop_out_prob,
nb_class,
):
model.add(Flatten())
model.add(Dense(hidden_dim, W_constraint=MaxNorm(m=9, axis=1)))
# model.add(ReNorm())
# model.add(Activation('relu'))
model.add(Dropout(drop_out_prob))
# model.add(LSTM(70))
# model.add(Dense(hidden_dim, W_constraint=MaxNorm(9)))
# model.add(Dropout(0.5))
# model.add(Dense(2))
# model.add(Activation('softmax'))
assert (nb_class > 1)
if nb_class == 2:
model.add(Dense(1))
model.add(Activation('sigmoid'))
print 'begin compile..'
model.compile(optimizer='adadelta',
loss='binary_crossentropy',
class_mode='binary')
print 'end compile'
else:
model.add(Dense(nb_class))
model.add(Activation('softmax'))
print 'begin compile..'
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
class_mode='categorical')
print 'end compile'
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1) # K表示调用该函数的当前layer
reg = K.sum(
K.square(
K.dot(w_n, K.transpose(w_n)) -
K.eye(w_n.shape[0].value))) # 自身矩阵的内积的平方根-自身特征值 = 越小, 越说明分量为0
return args.ortho_reg * reg # 这东西越小越好, 因为能保证各个特征分的越开
vocab_size = len(vocab)
if args.emb_name: # 获取已经保存的embedding???
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(
os.path.join(
"/content/drive/My Drive/Attention-Based-Aspect-Extraction-master",
"preprocessed_data", args.domain), args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb',
embeddings_constraint=MaxNorm(10))
##### Compute sentence representation ##### pre-processing 根据attention组合句子
e_w = word_emb(sentence_input) # 将input转换为embedding
y_s = Average()(e_w) # 默认求平均 layer
att_weights = Attention(name='att_weights',
W_constraint=MaxNorm(10),
b_constraint=MaxNorm(10))([e_w,
y_s]) # attention layer
z_s = WeightedSum()([e_w, att_weights]) # encoding layer
##### Compute representations of negative instances ##### 增加准确性的tricks
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction ##### 构建dense层, 希望能够decoding attention sentences的特征
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax',
name='p_t')(p_t) # softmax一下, nodes数量不改变, 数值被soft了一下
r_s = WeightedAspectEmb(args.aspect_size,
args.emb_dim,
name='aspect_emb',
W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(
p_t) # 标准化0-10区间, 且正则项为自定义的ortho_reg
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n,
r_s]) # 自定义loss function??? 这是在做啥???
model = Model(inputs=[sentence_input, neg_input],
outputs=[loss]) # negative input是需要自己分开数据集的吗??
### Word embedding and aspect embedding initialization ######
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
) # 为何初始化要用到kmeans
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix) # r-s
return model
def __init__(self,
input_dim,
embedding_dim=128,
use_bn=False,
use_dp=False,
embedding_method='DistMult'):
super(LinkPredict, self).__init__(name='lp')
self.use_bn = use_bn
self.use_dp = use_dp
if embedding_method == 'HolE':
constraint = MaxNorm(1, axis=1)
elif embedding_method == 'TransE':
constraint = None
elif embedding_method == 'DistMult':
constraint = MaxNorm(1, axis=1)
self.e1 = Embedding(input_dim[0],
embedding_dim,
embeddings_constraint=constraint)
self.e2 = Embedding(input_dim[1],
embedding_dim,
embeddings_constraint=constraint)
self.r1 = Embedding(input_dim[2], embedding_dim)
self.r2 = Embedding(input_dim[3], embedding_dim)
if embedding_method == 'DistMult':
self.embedding = [
Multiply(),
Lambda(lambda x: K.sum(x, axis=-1)),
Activation('sigmoid'),
Reshape((-1, ))
]
elif embedding_method == 'HolE':
self.embedding = [
Lambda(lambda x: HolE(x[0], x[1], x[2])),
Activation('sigmoid'),
Reshape((-1, ))
]
elif embedding_method == 'TransE':
self.embedding = [
Lambda(lambda x: TransE(x[0], x[1], x[2])),
Activation('tanh'),
Reshape((-1, ))
]
else:
raise NotImplementedError(embedding_method + ' not implemented')
self.rate = 0.2
self.ls = [
Concatenate(),
Dense(32),
Dropout(self.rate),
BatchNormalization(axis=-1),
Activation('relu'),
Dense(1, activation='sigmoid'),
Reshape((-1, ))
]
if self.use_dp:
self.dp = Dropout(self.rate)
if self.use_bn:
self.bn1 = BatchNormalization(axis=-1)
self.bn2 = BatchNormalization(axis=-1)
def vox_net(input_shape=None,
input_tensor=None,
downsize_filters_factor=1,
pool_size=(2, 2, 2),
initial_learning_rate=0.00001,
convolutions=4,
dropout=.25,
filter_shape=(3, 3, 3),
num_outputs=1,
deconvolution=True,
regression=True,
output_shape=None):
def residual_block(residual_tensor):
# res1 = BatchNormalization()(residual_tensor)
res1 = Dropout(0.25)(residual_tensor)
res1 = Conv3D(int(64 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(1, 1, 1),
activation='relu',
data_format='channels_first')(res1)
# res2 = BatchNormalization()(res1)
res2 = Dropout(0.25)(res1)
res2 = Conv3D(int(64 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(1, 1, 1),
activation='relu',
data_format='channels_first')(res2)
res3 = layers.add([res2, residual_tensor])
return res3
inputs = Input(input_shape)
conv1 = Conv3D(int(32 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(1, 1, 1),
data_format='channels_first')(inputs)
# conv2 = BatchNormalization()(conv1)
conv2 = Dropout(0.25)(conv1)
conv2 = Conv3D(int(64 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(1, 1, 1),
activation='relu',
data_format='channels_first')(conv2)
# conv3 = BatchNormalization()(conv2)
conv3 = Dropout(0.25)(conv2)
conv3 = Conv3D(int(64 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(2, 2, 2),
activation='relu',
data_format='channels_first')(conv3)
conv3 = residual_block(conv3)
conv3 = residual_block(conv3)
# conv4 = BatchNormalization()(conv4)
conv4 = Dropout(0.25)(conv3)
conv4 = Conv3D(int(64 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(2, 2, 2),
activation='relu',
data_format='channels_first')(conv4)
conv4 = residual_block(conv4)
conv4 = residual_block(conv4)
# conv5 = BatchNormalization()(conv4)
conv5 = Dropout(0.25)(conv4)
conv5 = Conv3D(int(64 / downsize_filters_factor),
3,
padding='same',
kernel_constraint=MaxNorm(2.),
strides=(2, 2, 2),
activation='relu',
data_format='channels_first')(conv5)
conv5 = residual_block(conv5)
conv5 = residual_block(conv5)
upx3 = UpSampling3D(size=(2, 2, 2), data_format='channels_first')(conv3)
upx4 = UpSampling3D(size=(4, 4, 4), data_format='channels_first')(conv4)
upx5 = UpSampling3D(size=(8, 8, 8), data_format='channels_first')(conv5)
upx = layers.add([conv2, upx3, upx4, upx5])
conv_last = Conv3D(int(num_outputs), (1, 1, 1),
data_format='channels_first')(upx)
if regression:
act = Activation('relu')(conv_last)
model = Model(inputs=inputs, outputs=act)
model.compile(optimizer=Adam(lr=initial_learning_rate),
loss=msq_loss,
metrics=[msq])
else:
act = Activation('sigmoid')(conv_last)
model = Model(inputs=inputs, outputs=act)
model.compile(optimizer=Adam(lr=initial_learning_rate),
loss=dice_coef_loss,
metrics=[dice_coef])
return model
save_to_json("xval_man_tokens.json", str(word_index))
reg = l2(0.01)
reg_2 = l1_l2()
# embed_init = RandomUniform(seed=313)
embed_init = RandomUniform(seed=15)
# Activity reg deals with outputs
# Embeddings regs deals with hidden values
my_embedding = Embedding(num_words,
embedding_vector_length,
embeddings_initializer=embed_init,
activity_regularizer=reg,
embeddings_constraint=MaxNorm(max_value=2, axis=0),
embeddings_regularizer=None,
input_length=max_review_length,
trainable=True)
print('Shape of data tensor:', embed_xvals.shape)
print("xval sample", embed_xvals[156])
print('Shape of test tensor:', test_x.shape)
print("test xval sample", test_x[0])
# MODEL CONSTRUCTION
main_input = Input(shape=(max_review_length, ), dtype='int32')
embed = my_embedding(main_input)
embed = Dropout(0.2)(embed)
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1)
reg = K.sum(
K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value)))
return args.ortho_reg * reg
vocab_size = len(vocab)
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
emb_reader = EmbReader(
os.path.join("..", "preprocessed_data", args.domain),
args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb',
embeddings_constraint=MaxNorm(10))
##### Compute sentence representation #####
e_w = word_emb(sentence_input)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights',
W_constraint=MaxNorm(10),
b_constraint=MaxNorm(10))([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size,
args.emb_dim,
name='aspect_emb',
W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(p_t)
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(inputs=[sentence_input, neg_input], outputs=[loss])
### Word embedding and aspect embedding initialization ######
if args.emb_name:
from w2vEmbReader import W2VEmbReader as EmbReader
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
)
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix)
return model
def main(model, params):
datafolder = params['d']
training_passes = params['t']
eval_passes = params['e']
predict_passes = params['p']
batch_size = params['bs']
drop = params['drop']
dim = params['ed']
constr_dict = {
'maxnorm': MaxNorm(1, axis=1),
'unitnorm': UnitNorm(axis=1),
'nonneg': NonNeg()
}
reg_dict = {'l1': l1(0.01), 'l2': l2(0.01), 'l1_l2': l1_l2(0.01, 0.01)}
train_file = datafolder + "train.txt"
valid_file = datafolder + "valid.txt"
test_file = datafolder + "test.txt"
false_train_file = datafolder + "false_train.txt"
E_mapping, R_mapping = mapping(
[train_file, valid_file, test_file, false_train_file])
VOC_SIZE = len(list(E_mapping.keys()))
PRED_SIZE = len(list(R_mapping.keys()))
true_train = np.squeeze(
np.asarray(
list(
data_iterator(train_file,
E_mapping,
R_mapping,
batch_size=-1,
mode=params['training_mode']))))
if params['reverse_labels']: #TransE
true_train_labels = np.zeros(len(true_train.T))
else:
true_train_labels = np.ones(len(true_train.T))
if params['false_mode'] == 'fromfile':
false_train = np.squeeze(
np.asarray(
list(
data_iterator(false_train_file,
E_mapping,
R_mapping,
batch_size=-1,
mode=params['training_mode']))))
else:
s, p, o = true_train
false_train = np.asarray(
corrupt_triples(s, p, o, params['check'], params['false_mode']))
if params['reverse_labels']:
false_train_labels = np.ones(len(false_train.T))
else:
false_train_labels = np.zeros(len(false_train.T))
if params['constraint']:
const = constr_dict[params['constraint']]
else:
const = None
if params['regularizer']:
reg = reg_dict[params['regularizer']]
else:
reg = None
m = model(VOC_SIZE,
PRED_SIZE,
dim,
embeddings_regularizer=const,
embeddings_constraint=reg,
dropout=params['drop'])
m.compile(loss=params['loss'], optimizer='adagrad', metrics=['mae'])
for i in range(training_passes):
if params['false_mode'] != 'fromfile':
s, p, o = true_train
false_train = np.asarray(
corrupt_triples(s, p, o, params['check'],
params['false_mode']))
tmpX = np.concatenate([false_train.T, true_train.T], axis=0)
tmpY = np.concatenate([false_train_labels.T, true_train_labels.T],
axis=0)
tmpY = tmpY * (1 - params['ls']) + params['ls'] / 2
m.fit(tmpX, tmpY, epochs=1, shuffle=True, batch_size=batch_size)
try:
if (i % eval_passes == 0 and i != 0) or (i == training_passes - 1
and eval_passes > 0):
if params['filtered']:
tmp = true_train.T
else:
tmp = []
res = evaluate(m, valid_file, E_mapping, R_mapping,
params['reverse_labels'], tmp)
print(res)
except ZeroDivisionError:
pass
if params['store']:
store_embedding(m, E_mapping, R_mapping, datafolder)
if predict_passes > 0:
print(predict_passes)
test = np.squeeze(
np.asarray(
list(
data_iterator(test_file,
E_mapping,
R_mapping,
batch_size=-1,
mode=params['training_mode'])))).T
pred = m.predict(test)
pred = [p[0] for p in pred]
mapping_e = reverse_dict(E_mapping)
mapping_r = reverse_dict(R_mapping)
with open(params['output_file'], 'w') as f:
for t, p in zip(test, pred):
s, r, o = t
s, r, o = mapping_e[s], mapping_r[r], mapping_e[o]
string = '\t'.join(map(str, [s, r, o, p])) + '\n'
f.write(string)
def create_model(args, maxlen, vocab):
def ortho_reg(weight_matrix):
### orthogonal regularization for aspect embedding matrix ###
w_n = K.l2_normalize(weight_matrix, axis=-1)
reg = K.sum(
K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value)))
return args.ortho_reg * reg
vocab_size = len(vocab)
if args.emb_name:
if args.emb_technique == "w2v":
logger.info("Load {} glove embedding for {}".format(
args.lang, config.word_emb_training_type))
if args.lang == 'en':
emb_reader = EmbReader(
config.emb_dir_en["w2v"].format(
config.word_emb_training_type), args.emb_name)
elif args.lang == 'de':
#emb_reader = EmbReader(config.emb_dir_de["w2v"].format(config.word_emb_training_type),
#args.emb_name)
emb_reader = FineTuneEmbed_cca(
'../preprocessed_data/german/w2v/fine_tuned', 'w2v_emb',
'../preprocessed_data/german/w2v/full_trained',
'w2v_embedding_300')
elif args.emb_technique == 'fasttext':
if args.lang == 'en':
#emb_reader = FastTextEmbReader(config.emb_dir_en["fasttext"].format(config.word_emb_training_type),
#args.emb_name, config.fine_tuned_enabled)
emb_reader = FineTuneEmbed_ortho_procrustes(
'../preprocessed_data/fasttext/fine_tuned',
'fasttext_pre_trained',
'../preprocessed_data/fasttext/full_trained',
'w2v_embedding_skipgram_300')
elif args.lang == 'de':
emb_reader = FastTextEmbReader(
config.emb_dir_de["fasttext"].format(
config.word_emb_training_type), args.emb_name,
config.fine_tuned_enabled)
#emb_reader = FineTuneEmbed_ortho_procrustes('../preprocessed_data/fasttext/fine_tuned','fasttext_pre_trained','../preprocessed_data/fasttext/full_trained', 'w2v_embedding_skipgram_300')
elif args.emb_technique == "glove":
if args.lang == 'de':
logger.info('Load german glove embedding')
emb_reader = GloveEmbedding(config.emb_dir_de["glove"],
args.emb_name)
else:
logger.info("Load en glove embedding for {}".format(
config.word_emb_training_type))
emb_reader = GloveEmbedding(
config.emb_dir_en["glove"].format(
config.word_emb_training_type), args.emb_name)
elif args.emb_technique == "MUSE_supervised":
emb_reader = MUSEEmbedding(config.emb_dir_biling['supervised'],
args.emb_name)
elif args.emb_technique == "MUSE_unsupervised":
emb_reader = MUSEEmbedding(config.emb_dir_biling['unsupervised'],
args.emb_name)
aspect_matrix = emb_reader.get_aspect_matrix(args.aspect_size)
args.aspect_size = emb_reader.aspect_size
args.emb_dim = emb_reader.emb_dim
##### Inputs #####
sentence_input = Input(shape=(maxlen, ),
dtype='int32',
name='sentence_input')
neg_input = Input(shape=(args.neg_size, maxlen),
dtype='int32',
name='neg_input')
##### Construct word embedding layer #####
word_emb = Embedding(vocab_size,
args.emb_dim,
mask_zero=True,
name='word_emb',
embeddings_constraint=MaxNorm(10))
##### Compute sentence representation #####
e_w = word_emb(sentence_input)
y_s = Average()(e_w)
att_weights = Attention(name='att_weights',
W_constraint=MaxNorm(10),
b_constraint=MaxNorm(10))([e_w, y_s])
z_s = WeightedSum()([e_w, att_weights])
##### Compute representations of negative instances #####
e_neg = word_emb(neg_input)
z_n = Average()(e_neg)
##### Reconstruction #####
p_t = Dense(args.aspect_size)(z_s)
p_t = Activation('softmax', name='p_t')(p_t)
r_s = WeightedAspectEmb(args.aspect_size,
args.emb_dim,
name='aspect_emb',
W_constraint=MaxNorm(10),
W_regularizer=ortho_reg)(p_t)
##### Loss #####
loss = MaxMargin(name='max_margin')([z_s, z_n, r_s])
model = Model(inputs=[sentence_input, neg_input], outputs=[loss])
### Word embedding and aspect embedding initialization ######
if args.emb_name:
logger.info('Initializing word embedding matrix')
embs = model.get_layer('word_emb').embeddings
K.set_value(
embs,
emb_reader.get_emb_matrix_given_vocab(vocab, K.get_value(embs)))
logger.info(
'Initializing aspect embedding matrix as centroid of kmean clusters'
)
K.set_value(model.get_layer('aspect_emb').W, aspect_matrix)
return model
def main():
# DATA INITIALIZATION
split = 10
x = np.load('cardiac_records.npy')
y = np.load('cardiac_targets.npy')
x_split, y_split = kfold_split(x, y, split)
print(x.shape)
print(sum(y))
# CNN INITIALIZATION
model = Sequential()
model.add(BatchNormalization())
model.add(Dropout(rate=0))
# Convolution 1:
model.add(
Conv1D(filters=9,
kernel_size=5,
input_shape=(x_split.shape[2], 1),
kernel_constraint=MaxNorm(3),
kernel_initializer='uniform',
activation='relu',
padding='valid'))
model.add(Dropout(rate=0))
# Convolution 2:
model.add(
Conv1D(filters=3,
kernel_size=11,
input_shape=(x_split.shape[2], 1),
kernel_constraint=MaxNorm(3),
kernel_initializer='uniform',
activation='relu',
padding='valid'))
model.add(Dropout(rate=0))
model.add(Flatten())
# Fully connected dense layer
model.add(
Dense(units=x_split.shape[2],
kernel_initializer='uniform',
kernel_constraint=MaxNorm(3),
activation='relu'))
model.add(Dropout(rate=0))
# Add a single output logit layer:
model.add(
Dense(units=1,
kernel_initializer='uniform',
kernel_constraint=MaxNorm(3),
activation='sigmoid'))
adm = adam(
lr=0.01,
beta_1=0.8,
beta_2=0.98,
)
model.compile(optimizer=adm,
loss='mean_squared_error',
metrics=['accuracy'])
print('TRAINING')
epochs = 3
for ep in range(epochs):
print("epoch: " + str(ep))
for k in range(split):
x_split, y_split = kfold_split(x, y, split)
train = np.ones((split, ), bool)
train[k] = False
x_train = x_split[train, :, :]
y_train = y_split[train, :]
x_train = np.reshape(
x_train,
[x_train.shape[0] * x_train.shape[1], x_train.shape[2], 1])
y_train = np.reshape(y_train,
[y_train.shape[0] * y_train.shape[1]])
x_val = x_split[k, :, :]
y_val = y_split[k, :]
x_val = np.reshape(x_val, [x_val.shape[0], x_val.shape[1], 1])
model.fit(x_train,
y_train,
validation_data=(x_val, y_val),
epochs=1,
shuffle=True,
verbose=2)
model.save('desi_cnn.h5')
print("Done")
