def __init__(self,
output_dimesion,
vocab_size,
dropout_rate,
emb_dim,
max_len,
nb_filters,
init_W=None,
cae_N_hidden=50,
nb_features=17):
''' CNN Module'''
model_summary = open('cnn_cae_transfer_model_summary', 'w')
self.max_len = max_len
max_features = vocab_size
vanila_dimension = 200
projection_dimension = output_dimesion
filter_lengths = [3, 4, 5]
# self.model = Graph()
# input
doc_input = Input(shape=(max_len, ), dtype='int32', name='doc_input')
'''Embedding Layer'''
if init_W is None:
# self.model.add_node(Embedding(
# max_features, emb_dim, input_length=max_len), name='sentence_embeddings', input='input')
sentence_embeddings = Embedding(
output_dim=emb_dim,
input_dim=max_features,
input_length=max_len,
name='sentence_embeddings')(doc_input)
else:
# self.model.add_node(Embedding(max_features, emb_dim, input_length=max_len, weights=[
# init_W / 20]), name='sentence_embeddings', input='input')
sentence_embeddings = Embedding(
output_dim=emb_dim,
input_dim=max_features,
input_length=max_len,
weights=[init_W / 20],
name='sentence_embeddings')(doc_input)
'''Reshape Layer'''
reshape = Reshape(target_shape=(max_len, emb_dim, 1),
name='reshape')(sentence_embeddings) # chanels last
'''Convolution Layer & Max Pooling Layer'''
flatten_ = []
for i in filter_lengths:
model_internal = Sequential()
# model_internal.add(Conv2D(
# nb_filters, i, emb_dim, activation="relu"))
model_internal.add(
Conv2D(nb_filters, (i, emb_dim),
activation="relu",
name='conv2d_' + str(i),
input_shape=(self.max_len, emb_dim, 1)))
# model_internal.add(MaxPooling2D(
# pool_size=(self.max_len - i + 1, 1)))
model_internal.add(
MaxPooling2D(pool_size=(self.max_len - i + 1, 1),
name='maxpool2d_' + str(i)))
model_internal.add(Flatten())
# plot_model(model_internal, 'model_cnn_cae_transfer_conv2d_%d.png'%i, show_shapes=True)
model_internal.summary(
print_fn=lambda x: model_summary.write(x + '\n'))
flatten = model_internal(reshape)
flatten_.append(flatten)
''' Attributes module '''
lam = 1e-3
N = nb_features
# cae_N_hidden = 50
att_input = Input(shape=(N, ), name='cae_input')
encoded = Dense(cae_N_hidden, activation='tanh',
name='encoded')(att_input)
att_output = Dense(N, activation='linear', name='cae_output')(encoded)
# model = Model(input=att_input, output=att_output)
def contractive_loss(y_pred, y_true):
mse = K.mean(K.square(y_true - y_pred), axis=1)
W = K.variable(value=model.get_layer('encoded').get_weights()
[0]) # N x cae_N_hidden
W = K.transpose(W) # cae_N_hidden x N
h = model.get_layer('encoded').output
dh = h * (1 - h) # N_batch x cae_N_hidden
# N_batch x cae_N_hidden * cae_N_hidden x 1 = N_batch x 1
contractive = lam * K.sum(dh**2 * K.sum(W**2, axis=1), axis=1)
return mse + contractive
'''Transfer layer '''
# if cae_N_hidden != nb_filters:
# sys.exit("For the transfer layer to work ''for now'' the attributes latent vector dimension (--att_dim)"
# " must equal the number of filters (kernal) of the conv. layer (--num_kernel_per_ws)")
# combine the outputs of boths modules
model_internal = Sequential(name='Transfer_ResBlock')
model_internal.add(
Conv1D(cae_N_hidden / 2,
1,
activation="relu",
name='Res_conv2d_1',
input_shape=(cae_N_hidden, 1)))
model_internal.add(
Conv1D(cae_N_hidden, 1, activation="relu", name='Res_conv2d_2'))
model_internal.add(
MaxPooling1D(pool_size=cae_N_hidden, name='Res_maxpool1d'))
model_internal.add(Flatten())
reshape = Reshape(target_shape=(cae_N_hidden, 1),
name='reshape_encoded')(encoded) # chanels last
residual = model_internal(reshape)
transfered = residual
# plot_model(model_internal,'model_cnn_cae_transfer_transferblock.png',show_shapes=True)
model_internal.summary(
print_fn=lambda x: model_summary.write(x + '\n'))
# shortcut = encoded
# transfered = add([residual, shortcut])
''' Adding CAE output to CNN output '''
flatten_.append(transfered)
'''Fully Connect Layer & Dropout Layer'''
# self.model.add_node(Dense(vanila_dimension, activation='tanh'),
# name='fully_connect', inputs=['unit_' + str(i) for i in filter_lengths])
fully_connect = Dense(vanila_dimension,
activation='tanh',
name='fully_connect')(concatenate(flatten_,
axis=-1))
# self.model.add_node(Dropout(dropout_rate),
# name='dropout', input='fully_connect')
dropout = Dropout(dropout_rate, name='dropout')(fully_connect)
# joint_output = add([dropout, transfered], name='joint_output')
'''Projection Layer & Output Layer'''
# self.model.add_node(Dense(projection_dimension, activation='tanh'),
# name='projection', input='dropout')
pj = Dense(projection_dimension,
activation='tanh',
name='joint_output') # output layer
projection = pj(dropout)
# Output Layergit
model = Model(inputs=[doc_input, att_input],
outputs=[projection, att_output])
# todo: check the optimizer
model.compile(
optimizer='rmsprop',
# optimizer={'joint_output': 'rmsprop', 'cae_output': 'adam'},
loss={
'joint_output': 'mse',
'cae_output': contractive_loss
},
loss_weights={
'joint_output': 1.,
'cae_output': 1.
})
# plot_model(model, to_file='model.png')
self.model = model
# plot_model(model, to_file='model_cnn_cae_transfer.png',show_shapes=True)
model.summary(print_fn=lambda x: model_summary.write(x + '\n'))
def build_model():
'''
Build a Keras model with the functional API
'''
bn_mode = 1
if args.chars:
char_input = Input(shape=(args.max_word_len, ),
dtype='int32',
name='char_input')
x = Embedding(char_vocab_size,
args.char_embedding_dim,
input_length=args.max_word_len)(char_input)
x = Reshape((args.max_word_len, args.char_embedding_dim))(x)
prev_x = x
for rnet_idx in range(args.resnet):
if args.bn:
x = BatchNormalization()(x)
x = Activation('relu')(x)
if args.dropout:
x = Dropout(args.dropout)(x)
x = Convolution1D(args.char_embedding_dim,
8,
activation='relu',
padding='same')(x)
if args.bn:
x = BatchNormalization()(x)
x = Activation('relu')(x)
if args.dropout:
x = Dropout(args.dropout)(x)
x = Convolution1D(args.char_embedding_dim,
4,
activation='relu',
padding='same')(x)
x = add([prev_x, x])
x = MaxPooling1D(pool_size=2, padding='same')(x)
prev_x = x
if args.bn:
x = BatchNormalization()(x)
x = Activation('linear')(x)
feature_size = args.char_embedding_dim
char_embedding = Reshape(
(int(args.max_word_len / (2**args.resnet)), int(feature_size)))(x)
if args.words:
word_input = Input(shape=(args.max_sent_len, ),
dtype='int32',
name='word_input')
if not args.ignore_embeddings:
word_embedding = Embedding(vocab_size,
word_embedding_dim,
input_length=args.max_sent_len,
weights=[embedding_weights],
trainable=(not args.freeze),
name='word_embedding')(word_input)
else:
word_embedding = Embedding(vocab_size,
word_embedding_dim,
input_length=args.max_sent_len,
name='word_embedding')(word_input)
l = GRU(units=int(args.rnn_dim),
return_sequences=False,
dropout=args.dropout,
input_shape=(args.max_sent_len, word_embedding_dim),
activation='relu')(word_embedding)
#l = GRU(units=int(args.rnn_dim)/2, return_sequences=False, dropout=args.dropout, input_shape=(args.max_sent_len, word_embedding_dim), activation='relu')(l)
r = GRU(units=int(args.rnn_dim),
return_sequences=False,
go_backwards=True,
dropout=args.dropout,
input_shape=(args.max_sent_len, word_embedding_dim),
activation='relu')(word_embedding)
#r = GRU(units=int(args.rnn_dim)/2, return_sequences=False, go_backwards=True, dropout=args.dropout, input_shape=(args.max_sent_len, word_embedding_dim), activation='relu')(r)
word_x = concatenate([l, r])
if args.bn:
word_x = BatchNormalization()(word_x)
if args.dropout:
word_x = Dropout(args.dropout)(word_x)
if args.chars:
embedding = char_embedding
if args.bn:
embedding = BatchNormalization()(embedding)
if args.rnn:
# Bidirectional GRU
l = GRU(units=int(args.rnn_dim),
return_sequences=False,
dropout=args.dropout,
input_shape=(args.max_sent_len, args.char_embedding_dim),
activation='relu')(embedding)
#l = GRU(units=int(args.rnn_dim)/2, return_sequences=False, dropout=args.dropout, input_shape=(args.max_sent_len, args.char_embedding_dim), activation='relu')(l)
r = GRU(units=int(args.rnn_dim),
return_sequences=False,
go_backwards=True,
dropout=args.dropout,
input_shape=(args.max_sent_len, args.char_embedding_dim),
activation='relu')(embedding)
#r = GRU(units=int(args.rnn_dim)/2, return_sequences=False, go_backwards=True, dropout=args.dropout, input_shape=(args.max_sent_len, args.char_embedding_dim), activation='relu')(r)
x = concatenate([l, r])
if args.bn:
x = BatchNormalization()(x)
else:
x = Convolution1D(args.char_embedding_dim,
8,
activation='relu',
border_mode='same')(embedding)
#
x = MaxPooling1D(pool_length=8, border_mode='same')(x)
x = Convolution1D(args.char_embedding_dim,
4,
activation='relu',
border_mode='same')(x)
#
x = MaxPooling1D(pool_length=8, border_mode='same')(x)
x = Reshape((args.char_embedding_dim * 2, ))(x)
x = Dense(args.char_embedding_dim, activation='relu')(x)
if args.chars and args.words:
x = concatenate([x, word_x])
x = Dense(word_embedding_dim * 2, activation='relu')(x)
elif args.words:
x = word_x
anger_output = Dense(1, activation='sigmoid', name='anger_output')(x)
anticipation_output = Dense(1,
activation='sigmoid',
name='anticipation_output')(x)
disgust_output = Dense(1, activation='sigmoid', name='disgust_output')(x)
fear_output = Dense(1, activation='sigmoid', name='fear_output')(x)
joy_output = Dense(1, activation='sigmoid', name='joy_output')(x)
love_output = Dense(1, activation='sigmoid', name='love_output')(x)
optimism_output = Dense(1, activation='sigmoid', name='optimism_output')(x)
pessimism_output = Dense(1, activation='sigmoid',
name='pessimism_output')(x)
sadness_output = Dense(1, activation='sigmoid', name='sadness_output')(x)
surprise_output = Dense(1, activation='sigmoid', name='surprise_output')(x)
trust_output = Dense(1, activation='sigmoid', name='trust_output')(x)
main_output = Dense(1, activation='linear', name='main_output')(x)
if args.chars and args.words:
model_input = [word_input, char_input]
elif args.chars:
model_input = [
char_input,
]
elif args.words:
model_input = [
word_input,
]
model_output = [
main_output, anger_output, anticipation_output, disgust_output,
fear_output, joy_output, love_output, optimism_output,
pessimism_output, sadness_output, surprise_output, trust_output
]
model = Model(inputs=model_input, outputs=model_output)
return model
y_test=np.delete(y_test,k)
X_test_add=np.delete(X_test_add,k,0)
print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train_one_hot, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test_one_hot, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
#embedding model
emb_model = Sequential()
emb_model.add(Embedding(max_features, 128, input_length=maxlen,dropout=0.2))
emb_model.add(Convolution1D(nb_filter=32, filter_length=3, border_mode='same', activation='relu'))
emb_model.add(MaxPooling1D(pool_length=2))
# ? may merge a layer here which include features from POST (grammar info)
emb_model.add(LSTM(128, dropout_W=0.2, dropout_U=0.2)) # try using a GRU instead, for fun
#additional model
add_model = Sequential()
add_model.add(Dense(12, input_dim=2, init='normal', activation='relu',W_constraint = maxnorm(2)))
#merge model
model = Sequential()
model.add(Merge([add_model, emb_model], mode='concat', concat_axis=1))
model.add(Dense(32))
model.add(Dense(1))
model.add(Activation('sigmoid'))
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Dropout(0.25))
model.add(
Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(LSTM(lstm_output_size))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print('Train...')
model.fit(X_train,
y_train,
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data=(X_test, y_test))
score, acc = model.evaluate(X_test, y_test, batch_size=batch_size)
# CNN
# ===============================================================================================
print ("Method = CNN for Arabic Sentiment Analysis'")
model_variation = 'CNN-non-static'
np.random.seed(0)
nb_filter = embeddings_dim
main_input = Input(shape=(max_sent_len,))
embedding = Embedding(max_features, embeddings_dim, input_length=max_sent_len, mask_zero=False, weights=[embedding_weights] )(main_input)
Drop1 = Dropout(dropout_prob[0])(embedding)
convs = []
for n_gram in filter_sizes:
conv = Convolution1D(nb_filter=nb_filter, filter_length=n_gram, border_mode='valid', activation='relu', subsample_length=1, input_dim=embeddings_dim, input_length=max_sent_len)(Drop1)
pool = MaxPooling1D(pool_length=max_sent_len - n_gram + 1)(conv)
# flat = Flatten()(pool)
convs.append(pool)
if len(filter_sizes) > 1:
merged = merge(convs, mode='concat')
else:
merged = convs[0]
conv_model = Model(inputs=main_input, outputs=merged)
# add created model grapg in sequential model
model = Sequential()
model.add(conv_model) # add model just like layer
model.add(Convolution1D(nb_filter=nb_filter, filter_length=filter_sizes[1], border_mode='same', activation='relu'))
model.add(MaxPooling1D(pool_length=3, border_mode='same'))
model.add(Dropout(0.25))
for filter_len in cfg.get('cnn', 'filtlen').split(','):
branch = Sequential()
branch.add(
Embedding(len(dataset.word2int),
cfg.getint('cnn', 'embdims'),
input_length=maxlen,
weights=init_vectors))
branch.add(
Convolution1D(nb_filter=cfg.getint('cnn', 'filters'),
filter_length=int(filter_len),
border_mode='valid',
activation='relu',
subsample_length=1))
branch.add(MaxPooling1D(pool_length=2))
branch.add(Flatten())
branches.append(branch)
train_xs.append(train_x)
test_xs.append(test_x)
model = Sequential()
model.add(Merge(branches, mode='concat'))
model.add(Dense(cfg.getint('cnn', 'hidden')))
model.add(Dropout(cfg.getfloat('cnn', 'dropout')))
model.add(Activation('relu'))
model.add(Dropout(cfg.getfloat('cnn', 'dropout')))
model.add(Dense(classes))
def m11(num_classes=10):
print('Using Model M11')
m = Sequential()
m.add(
Conv1D(64,
input_shape=[AUDIO_LENGTH, 1],
kernel_size=80,
strides=4,
padding='same',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(l=0.0001)))
m.add(BatchNormalization())
m.add(Activation('relu'))
m.add(MaxPooling1D(pool_size=4, strides=None))
for i in range(2):
m.add(
Conv1D(64,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(l=0.0001)))
m.add(BatchNormalization())
m.add(Activation('relu'))
m.add(MaxPooling1D(pool_size=4, strides=None))
for i in range(2):
m.add(
Conv1D(128,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(l=0.0001)))
m.add(BatchNormalization())
m.add(Activation('relu'))
m.add(MaxPooling1D(pool_size=4, strides=None))
for i in range(3):
m.add(
Conv1D(256,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(l=0.0001)))
m.add(BatchNormalization())
m.add(Activation('relu'))
m.add(MaxPooling1D(pool_size=4, strides=None))
for i in range(2):
m.add(
Conv1D(512,
kernel_size=3,
strides=1,
padding='same',
kernel_initializer='glorot_uniform',
kernel_regularizer=regularizers.l2(l=0.0001)))
m.add(BatchNormalization())
m.add(Activation('relu'))
m.add(Lambda(lambda x: K.mean(x, axis=1)))
m.add(Dense(num_classes, activation='softmax'))
return m
def RNN(X_train, y_train, args):
"""
Purpose -> Define and train the proposed LSTM network
Input -> Data, Labels and model hyperparameters
Output -> Trained LSTM network
"""
# Sets the model hyperparameters
# Embedding hyperparameters
max_features = args[0]
maxlen = args[1]
embedding_size = args[2]
# Convolution hyperparameters
filter_length = args[3]
nb_filter = args[4]
pool_length = args[5]
# LSTM hyperparameters
lstm_output_size = args[6]
# Training hyperparameters
batch_size = args[7]
nb_epoch = args[8]
numclasses = args[9]
test_size = args[10]
# Format conversion for y_train for compatibility with Keras
y_train = np_utils.to_categorical(y_train, numclasses)
print(y_train)
# Train & Validation data splitting
X_train, X_valid, y_train, y_valid = train_test_split(X_train,
y_train,
test_size=test_size,
random_state=42)
# Build the sequential model
# Model Architecture is:
# Input -> Embedding -> Conv1D+Maxpool1D -> LSTM -> LSTM -> FC-1 -> Softmaxloss
print('Build model...')
start = time()
log_dir = datetime.now().strftime('model_%Y%m%d_%H%M')
os.mkdir(log_dir)
es = EarlyStopping(monitor='val_loss', patience=20)
mc = ModelCheckpoint(log_dir +
'\\CIFAR10-EP{epoch:02d}-ACC{val_acc:.4f}.h5',
monitor='val_loss',
save_best_only=True)
tb = TensorBoard(log_dir=log_dir, histogram_freq=0)
sequence = Input(shape=(maxlen, ), dtype='int32')
# embedded = Embedding(input_dim=max_features, output_dim=num_features, input_length=maxlen, mask_zero=True, weights=[W], trainable=False) (sequence)
embedded = Embedding(input_dim=max_features,
output_dim=embedding_size,
input_length=maxlen,
trainable=False)(sequence)
embedded = Dropout(0.25)(embedded)
convolution = Convolution1D(filters=nb_filter,
filter_length=filter_length,
padding='valid',
activation='relu',
strides=1)(embedded)
maxpooling = MaxPooling1D(pool_length=pool_length)(convolution)
lstm = LSTM(lstm_output_size,
dropout_W=0.2,
dropout_U=0.2,
return_sequences=True)(maxpooling)
lstm1 = LSTM(lstm_output_size,
dropout_W=0.2,
dropout_U=0.2,
return_sequences=False)(lstm)
enc = Bidirectional(
GRU(lstm_output_size // 2,
recurrent_dropout=0.25,
return_sequences=True))(maxpooling)
att = AttentionM()(enc)
x = keras.layers.Concatenate(axis=1)([lstm1, att])
fc1 = Dense(128, activation="relu")(x)
fc2 = Dense(64, activation="relu")(fc1)
fc3 = Dense(32, activation="relu")(fc2)
fc4 = Dense(16, activation="relu")(fc3)
fc4_dropout = Dropout(0.25)(fc4)
output = Dense(3, activation='softmax')(fc4_dropout)
model = Model(inputs=sequence, outputs=output)
'''model = Sequential()
model.add(Embedding(max_features, embedding_size, input_length=maxlen))
model.add(Convolution1D(nb_filter=nb_filter,
filter_length=filter_length,
border_mode='valid',
activation='relu',
subsample_length=1))
model.add(MaxPooling1D(pool_length=pool_length))
model.add(SpatialDropout1D(0.1))
model.add(Bidirectional(CuDNNGRU(64, return_sequences=True)))
model.add(Bidirectional(CuDNNGRU(64, return_sequences=True)))
Routings = 5
Num_capsule = 10
Dim_capsule = 32
model.add(Capsule(num_capsule=Num_capsule, dim_capsule=Dim_capsule, routings=Routings, share_weights=True))
model.add(Flatten())
model.add(LSTM(lstm_output_size, dropout_W=0.2, dropout_U=0.2, return_sequences=True))
model.add(LSTM(lstm_output_size, dropout_W=0.2, dropout_U=0.2, return_sequences=True))
model.add(Bidirectional(LSTM(lstm_output_size//2, recurrent_dropout=0.25, return_sequences=False)))
#model.add(AttentionM())
model.add(Dropout(0.25))
model.add(Dense(numclasses,activation='softmax'))'''
# Optimizer is Adamax along with categorical crossentropy loss
model.compile(
loss='categorical_crossentropy',
optimizer='adamax',
metrics=['accuracy'],
)
print(model.summary())
history = LossHistory()
print('Train...')
# Trains model for 50 epochs with shuffling after every epoch for training data and validates on validation data
model.fit(X_train,
y_train,
batch_size=batch_size,
shuffle=True,
nb_epoch=nb_epoch,
validation_data=(X_valid, y_valid),
callbacks=[history, es, mc, tb])
history.loss_plot('epoch')
return model
# load model
sorted_pi_list = np.load('/mnt/sorted_pi_list.out')
model = load_model('best_model.h5')
print("Loaded model")
print(model.summary())
# make model that ends at last neuron pre-softmax
c1 = model.get_layer("conv1d_1")
b1 = model.get_layer("batch_normalization_1")
d1 = model.get_layer("dense_1")
b2 = model.get_layer("batch_normalization_2")
d2 = model.get_layer("dense_2")
model2 = Sequential()
model2.add(Convolution1D(input_shape=(16048,4), filters=128, kernel_size=12, activation="relu", padding="same", weights=c1.get_weights()))
model2.add(MaxPooling1D(pool_size=16048))
model2.add(BatchNormalization(weights=b1.get_weights()))
model2.add(Flatten())
model2.add(Dense(input_dim=128,units=64, weights=d1.get_weights()))
model2.add(Activation("relu"))
model2.add(BatchNormalization(weights=b2.get_weights()))
model2.add(Dense(units=827, weights=d2.get_weights()))
model2.compile(loss='categorical_crossentropy', optimizer='rmsprop', metrics=["accuracy"])
# cycle through each plasmid of interest
for h in range(len(dna_seq_of_interest)):
seq = dna_seq_of_interest[h]
# intialize variables for plasmid disruption scan
total_plasmid_size = len(seq)+disruption_len
probability_for_correct = np.zeros(total_plasmid_size)
def main(args):
if len(args) < 1:
sys.stderr.write("Error - one required argument: <data directory>\n")
sys.exit(-1)
working_dir = args[0]
data_file = os.path.join(working_dir, 'training-data.liblinear')
# learn alphabet from training data
provider = dataset.DatasetProvider(data_file)
# now load training examples and labels
train_x, train_y = provider.load(data_file)
# turn x and y into numpy array among other things
maxlen = max([len(seq) for seq in train_x])
classes = len(set(train_y))
train_x = pad_sequences(train_x, maxlen=maxlen)
train_y = to_categorical(np.array(train_y), classes)
#loading pre-trained embedding file:
embeddings_index = {}
f = open(os.path.join(working_dir, 'mimic.txt'))
values = f.readline().split()
EMBEDDING_WORDNUM = int(values[0])
EMBEDDING_DIM = int(values[1])
for line in f:
values = line.split()
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
embeddings_index[word] = coefs
f.close()
print('load embeddings for %s=%s words.' %
(len(embeddings_index), EMBEDDING_WORDNUM))
# prepare embedding matrix
nb_words = len(provider.word2int)
embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))
for word, i in provider.word2int.items():
embedding_vector = embeddings_index.get(word)
if embedding_vector is not None: # words not found in embedding index will be all-zeros.
embedding_matrix[i] = embedding_vector
pickle.dump(maxlen, open(os.path.join(working_dir, 'maxlen.p'), "wb"))
pickle.dump(provider.word2int,
open(os.path.join(working_dir, 'word2int.p'), "wb"))
pickle.dump(provider.label2int,
open(os.path.join(working_dir, 'label2int.p'), "wb"))
print 'train_x shape:', train_x.shape
print 'train_y shape:', train_y.shape
branches = [] # models to be merged
train_xs = [] # train x for each branch
inflows = [] # placeholder for each input
for filter_len in '2,5'.split(','):
branch = Input(shape=(maxlen, ))
embed = Embedding(len(provider.word2int),
EMBEDDING_DIM,
weights=[embedding_matrix],
trainable=True)(branch)
conv = Conv1D(filters=200,
kernel_size=int(filter_len),
padding='valid',
activation='relu',
strides=1)(embed)
pool = MaxPooling1D(pool_size=2)(conv)
flat = Flatten()(pool)
branches.append(flat)
train_xs.append(train_x)
inflows.append(branch)
concat = concatenate(branches)
drop1 = Dropout(0.25)(concat)
dense = Dense(200, activation='relu')(drop1)
drop2 = Dropout(0.25)(dense)
out = Dense(classes, activation='softmax')(drop2)
model = Model(inputs=inflows, outputs=out)
#optimizer = RMSprop(lr=0.0001, rho=0.9, epsilon=1e-08)
optimizer = Adam(lr=0.001)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
stopper = EarlyStopping(monitor='val_loss',
patience=10,
verbose=0,
mode='auto')
model.fit(train_xs,
train_y,
epochs=20,
batch_size=128,
verbose=1,
validation_split=0.1,
callbacks=[stopper])
json_string = model.to_json()
open(os.path.join(working_dir, 'model_0.json'), 'w').write(json_string)
model.save_weights(os.path.join(working_dir, 'model_0.h5'), overwrite=True)
sys.exit(0)
def lstm_model():
# model
model = Sequential()
# add conv1D layer
model.add(
Conv1D(filters=32,
kernel_size=7,
padding='same',
dilation_rate=2,
activation='relu',
input_shape=(features_num, vec_size)))
#model.add(Dropout(0.2))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.2))
for i in range(4):
# add conv1D layer
model.add(
Conv1D(filters=32,
kernel_size=7,
dilation_rate=2,
padding='same',
activation='relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.2))
for i in range(5):
model.add(
Conv1D(filters=64,
kernel_size=5,
padding='same',
activation='relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.2))
for i in range(5):
model.add(
Conv1D(filters=128,
kernel_size=3,
padding='same',
activation='relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.2))
for i in range(5):
model.add(
Conv1D(filters=256,
kernel_size=3,
padding='same',
activation='relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.2))
# add LSTM layer
model.add(LSTM(600, dropout=0.2, recurrent_dropout=0.2))
# model.add(Dropout(0.2))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')
return model
verbose, epochs, batch_size = 0, 50, 150
n_timesteps, n_features, n_outputs = n_timesteps, X_train.shape[2], y_train.shape[0]
# reshape data into time steps of sub-sequences
n_steps, n_length = 5, 50
X_train = X_train1.reshape((X_train1.shape[0], n_steps, n_length, n_features))
X_test = X_test1.reshape((X_test1.shape[0], n_steps, n_length, n_features))
# Initialising the model
classifier = Sequential()
#Adding the first layer of CNN
classifier.add(TimeDistributed(Conv1D(filters=45, kernel_size=3, activation='relu'),
input_shape=(None,n_length,n_features)))
#Adding the second layer of CNN and some Dropout regularisation
classifier.add(TimeDistributed(Conv1D(filters=45, kernel_size=3, activation='relu')))
classifier.add(TimeDistributed(Dropout(0.5)))
#Adding a layer of MaxPooling and Flatten regularisation
classifier.add(TimeDistributed(MaxPooling1D(pool_size=2)))
classifier.add(TimeDistributed(Flatten()))
# Adding the first LSTM layer and some Dropout regularisation
classifier.add(LSTM(units = 100, return_sequences = True))
classifier.add(Dropout(0.5))
# Adding a second LSTM layer and some Dropout regularisation
classifier.add(LSTM(units = 100, return_sequences = True))
classifier.add(Dropout(0.5))
# Adding a third LSTM layer and some Dropout regularisation
classifier.add(LSTM(units = 100, return_sequences = True))
classifier.add(Dropout(0.5))
# Adding a fourth LSTM layer and some Dropout regularisation
classifier.add(LSTM(units = 100))
classifier.add(Dropout(0.5))
# Adding the output layer
classifier.add(Dense(1))
def distancenet(self,
vocab_size,
output_size,
maxsize=1,
hop_depth=-1,
dropout=False,
d_perc=1,
type="CCE",
shape=0,
q_shape=1):
print(bcolors.UNDERLINE + 'Building nn model...' + bcolors.ENDC)
print(q_shape, shape, "====================")
sentrnn = Sequential()
emb = Embedding(vocab_size,
self.embed_hidden_size,
mask_zero=False,
W_constraint=mx(),
W_regularizer=reg(),
init=init_function,
input_shape=shape)
sentrnn.add(emb)
#emb_bn = bn()
#sentrnn.add(emb_bn)
sentrnn.add(Dropout(0.2))
if (convolution):
#print maxsize, q_shape
#exit()
conv = Convolution1D(self.sent_hidden_size,
maxsize,
subsample_length=1,
activation=act,
border_mode="same")
#conv_bn = bn()
sentrnn.add(conv)
#sentrnn.add(conv_bn)
#sentrnn.add(Dropout(0.1))
sentrnn.add(MaxPooling1D(pool_length=maxsize))
#sentrnn.add(Dropout(d_perc))
else:
sentrnn.add(MaxPooling1D(pool_length=maxsize))
#sentrnn.add(SimpleRNN( self.query_hidden_size, return_sequences=True,activation = "leakyrelu", init = init_function,dropout_W = 0.3, dropout_U=0.3, consume_less = "mem"))
# sentrnn.add(ResettingRNN(maxsize, self.query_hidden_size, return_sequences=True,activation = "tanh", init = init_function,dropout_W = 0.3, dropout_U=0.3, consume_less = "mem"))
# sentrnn.add(bn())
# sentrnn.add(AttentionPooling(pool_length=maxsize))
#sentrnn.add(Convolution1D(self.sent_hidden_size,maxsize, subsample_length = 1, activation=act, border_mode="same"))
#sentrnn.add(bn())
#
#sentrnn.add(Dropout(0.1))
#td = TimeDistributedDense(self.sent_hidden_size, activation = act, init = init_function)
#sentrnn.add(td)
#sentrnn.add(bn())
#sentrnn.add(Dropout(d_perc))
qrnn = Sequential()
#emb = Embedding(vocab_size, self.embed_hidden_size, mask_zero=False,W_constraint=mx(), W_regularizer=reg(), init = init_function, input_shape=shape, input_length=q_shape)
qrnn.add(InputLayer(input_shape=(q_shape, )))
qrnn.add(emb)
#qrnn.add(emb_bn)
qrnn.add(Dropout(0.2))
if (convolution):
#conv = Convolution1D(self.sent_hidden_size,q_shape, subsample_length = 1, activation=act, border_mode="same")
qrnn.add(conv)
#qrnn.add(conv_bn)
#qrnn.add(Dropout(0.1))
qrnn.add(MaxPooling1D(pool_length=q_shape))
qrnn.add(Flatten(1))
else:
qrnn.add(
SimpleRNN(self.query_hidden_size,
return_sequences=False,
activation=act,
init=init_function,
dropout_W=0.1,
dropout_U=0.1,
consume_less="mem"))
#qrnn.add(Convolution1D(self.sent_hidden_size,q_shape, subsample_length = 1, activation=act, border_mode="same"))
#qrnn.add(bn())
#qrnn.add(MaxPooling1D(pool_length=q_shape))
#qrnn.add(Flatten())
init_qa = [sentrnn, qrnn]
past = []
#at = GRU(self.sent_hidden_size, dropout_W = 0.3, dropout_U=0.3, activation=act)
td = TimeDistributedDense(self.sent_hidden_size,
activation=act,
init=init_function)
at = AttentionRecurrent(self.sent_hidden_size, activation="softmax")
for i in range(hop_depth):
hop = Sequential()
hop.add(
AttentionMerge((i == hop_depth - 1),
"abs",
init_qa + past,
mode="sum"))
hop.add(td)
#hop.add(bn())
#hop.add(Dropout(0.1))
hop.add(at)
#hop.add(bn())
#hop.add(Dropout(0.1))
past.append(hop)
# model = Sequential()
# model.add(AttentionMerge(False, "concat", init_qa + past, mode = "concat"))
# model.add(LSTM(self.query_hidden_size, return_sequences=False, init = init_function,dropout_W = 0.2, dropout_U=0.2))
model = hop
#self._adddepth(model, dropout, d_perc)
model.add(
Dense(vocab_size,
W_constraint=mx(),
W_regularizer=reg(),
init=init_function))
#model.add(bn())
model.add(Activation("softmax"))
if (type == "CCE"):
model.compile(optimizer=self._getopt(),
loss='categorical_crossentropy',
metrics=["accuracy"],
class_mode='categorical')
else:
model.compile(optimizer=self._getopt(), loss='mse')
print(model.summary())
plot(model,
show_layer_names=False,
show_shapes=True,
to_file='model.png')
return model
build_tweet.append(X_vecs[token])
vector_list.append(build_tweet)
X_test = np.array(vector_list)
# Keras CNN model
del X_vecs
batch_size = 64
nb_epochs = 50
model = Sequential()
model.add(Conv1D(64, kernel_size=3, activation='elu', padding='same', input_shape=(max_tweet_length, vector_size)))
model.add(Conv1D(128, kernel_size=3, activation='elu', padding='same'))
model.add(Dropout(0.25))
model.add(MaxPooling1D(pool_size=2, strides=2))
model.add(Conv1D(256, kernel_size=3, activation='elu', padding='same'))
model.add(Conv1D(512, kernel_size=3, activation='elu', padding='same'))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(256, activation='tanh'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
# Compile the model
model.compile(
loss='binary_crossentropy',
def build_model(top_words,
embedding_vecor_length,
max_review_length,
show_summaries=False):
input_layer = Embedding(top_words,
embedding_vecor_length,
input_length=max_review_length)
# --- 3 ---
branch_3 = Sequential()
branch_3.add(input_layer)
branch_3.add(
Conv1D(filters=128,
kernel_size=3,
padding='same',
kernel_regularizer=l2(.01)))
branch_3.add(Activation('relu'))
branch_3.add(MaxPooling1D(pool_size=2))
branch_3.add(Dropout(0.5))
branch_3.add(BatchNormalization())
branch_3.add(LSTM(128))
# --- 5 ---
branch_5 = Sequential()
branch_5.add(input_layer)
branch_5.add(
Conv1D(filters=128,
kernel_size=5,
padding='same',
kernel_regularizer=l2(.01)))
branch_5.add(Activation('relu'))
branch_5.add(MaxPooling1D(pool_size=2))
branch_5.add(Dropout(0.5))
branch_5.add(BatchNormalization())
branch_5.add(LSTM(128))
# --- 7 ---
branch_7 = Sequential()
branch_7.add(input_layer)
branch_7.add(
Conv1D(filters=128,
kernel_size=7,
padding='same',
kernel_regularizer=l2(.01)))
branch_7.add(Activation('relu'))
branch_7.add(MaxPooling1D(pool_size=2))
branch_7.add(Dropout(0.5))
branch_7.add(BatchNormalization())
branch_7.add(LSTM(128))
# --- 9 ---
branch_9 = Sequential()
branch_9.add(input_layer)
branch_9.add(
Conv1D(filters=128,
kernel_size=9,
padding='same',
kernel_regularizer=l2(.01)))
branch_9.add(Activation('relu'))
branch_9.add(MaxPooling1D(pool_size=2))
branch_9.add(Dropout(0.5))
branch_9.add(BatchNormalization())
branch_9.add(LSTM(128))
model = Sequential()
model.add(Merge([branch_3, branch_5, branch_7, branch_9], mode='concat'))
model.add(Dense(1, activation='sigmoid'))
opt = keras.optimizers.SGD()
model.compile(loss='binary_crossentropy',
optimizer=opt,
metrics=['accuracy'])
if show_summaries:
print(branch_3.summary())
print(branch_4.summary())
print(branch_5.summary())
print(model.summary())
return model
print("BUILDING MODEL")
embedding_vecor_length = 32
input_layer = Embedding(top_words,
embedding_vecor_length,
input_length=max_review_length)
branch_3 = Sequential()
branch_3.add(input_layer)
branch_3.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
kernel_regularizer=l2(.01)))
branch_3.add(Activation('relu'))
branch_3.add(MaxPooling1D(pool_size=2))
branch_3.add(Dropout(0.5))
branch_3.add(BatchNormalization())
branch_3.add(LSTM(100))
branch_4 = Sequential()
branch_4.add(input_layer)
branch_4.add(
Conv1D(filters=32,
kernel_size=4,
padding='same',
kernel_regularizer=l2(.01)))
branch_4.add(Activation('relu'))
branch_4.add(MaxPooling1D(pool_size=2))
branch_4.add(Dropout(0.5))
branch_4.add(BatchNormalization())
n_steps_in, n_steps_out = 14, 7
X, y = split_sequence(data, n_steps_in, n_steps_out)
# X.shape is [samples, timesteps]
# reshape from [samples, timesteps] into [samples, subsequences, timesteps, features]
n_features = 1
n_seq = 2
n_steps = 7
X = X.reshape((X.shape[0], n_seq, n_steps, n_features))
# define CNN-LSTM model
model = Sequential()
model.add(
TimeDistributed(Conv1D(64, 1, activation='relu'),
input_shape=(None, n_steps, n_features)))
model.add(TimeDistributed(MaxPooling1D()))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(50, activation='relu'))
model.add(Dense(n_steps_out))
model.compile(optimizer='adam', loss='mse', metrics=['accuracy'])
model.fit(X, y, epochs=200, verbose=1)
X_test = np.array(data[-n_steps_in:]).reshape(1, n_seq, n_steps, n_features)
y_hat = model.predict(X_test, verbose=1)
print(f'y = {y[-n_steps_out:]}, y_hat = {y_hat}')
print(model.summary)
def run(train_file, test_file, batch, epochs, embdims, filters, filtlen,
hidden, dropout, learnrt):
"""Train/test with given parameters. Return F1."""
np.random.seed(1337)
print 'train:', train_file
print 'test:', test_file
print 'batch:', batch
print 'epochs:', epochs
print 'embdims:', embdims
print 'filters:', filters
print 'filtlen:', filtlen
print 'hidden:', hidden
print 'dropout:', dropout
print 'learnrt:', learnrt
# learn alphabet from training examples
datset = dataset.DatasetProvider(train_file)
# now load training examples and labels
train_x, train_y = datset.load(train_file)
maxlen = max([len(seq) for seq in train_x])
# now load test examples and labels
test_x, test_y = datset.load(test_file, maxlen=maxlen)
# turn x and y into numpy array among other things
classes = len(set(train_y))
train_x = pad_sequences(train_x, maxlen=maxlen)
train_y = to_categorical(np.array(train_y), classes)
test_x = pad_sequences(test_x, maxlen=maxlen)
test_y = to_categorical(np.array(test_y), classes)
branches = [] # models to be merged
train_xs = [] # train x for each branch
test_xs = [] # test x for each branch
for filter_len in filtlen.split(','):
branch = Sequential()
branch.add(
Embedding(len(datset.word2int),
embdims,
trainable=False,
input_length=maxlen))
branch.add(
Convolution1D(nb_filter=filters,
filter_length=int(filter_len),
border_mode='valid',
activation='relu',
subsample_length=1))
branch.add(MaxPooling1D(pool_length=2))
branch.add(Flatten())
branches.append(branch)
train_xs.append(train_x)
test_xs.append(test_x)
model = Sequential()
model.add(Merge(branches, mode='concat'))
model.add(Dropout(dropout))
model.add(Dense(hidden))
model.add(Activation('relu'))
model.add(Dropout(dropout))
model.add(Dense(classes))
model.add(Activation('softmax'))
optimizer = RMSprop(lr=learnrt, rho=0.9, epsilon=1e-08)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.fit(train_xs,
train_y,
nb_epoch=epochs,
batch_size=batch,
verbose=0,
validation_split=0.0,
class_weight=None)
# probability for each class; (test size, num of classes)
distribution = \
model.predict(test_xs, batch_size=batch)
# class predictions; (test size,)
predictions = np.argmax(distribution, axis=1)
# gold labels; (test size,)
gold = np.argmax(test_y, axis=1)
# f1 scores
label_f1 = f1_score(gold, predictions, average=None)
print
for label, idx in datset.label2int.items():
print 'f1(%s)=%f' % (label, label_f1[idx])
if 'contains' in datset.label2int:
idxs = [datset.label2int['contains'], datset.label2int['contains-1']]
contains_f1 = f1_score(gold, predictions, labels=idxs, average='micro')
print '\nf1(contains average) =', contains_f1
else:
idxs = datset.label2int.values()
average_f1 = f1_score(gold, predictions, labels=idxs, average='micro')
print 'f1(all) =', average_f1
print '******************************************'
from keras.datasets import imdb
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Flatten
from keras.layers.convolutional import Conv1D
from keras.layers.convolutional import MaxPooling1D
from keras.layers.embeddings import Embedding
from keras.preprocessing import sequence
# load the dataset but only keep the top n words, zero the rest
top_words = 5000
(X_train, y_train), (X_test, y_test) = imdb.load_data(num_words=top_words)
# pad dataset to a maximum review length in words
max_words = 500
X_train = sequence.pad_sequences(X_train, maxlen=max_words)
X_test = sequence.pad_sequences(X_test, maxlen=max_words)
# create the model
model = Sequential()
model.add(Embedding(top_words, 32, input_length=max_words))
model.add(Conv1D(64, 3, padding='same', activation='relu'))
model.add(MaxPooling1D())
model.add(Flatten())
model.add(Dense(250, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
# Fit the model
model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=2, batch_size=128, verbose=2)
# Final evaluation of the model
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy: %.2f%%" % (scores[1]*100))
# model.compile(loss='mean_squared_error', optimizer='adam')
# cnnblstm
train_xinitial = train_xinitial.reshape(train_xinitial.shape[0], 2, int(
train_xinitial.shape[1] / 2), train_xinitial.shape[2])
train_x = train_x.reshape(train_x.shape[0], 2, int(
train_x.shape[1] / 2), train_x.shape[2])
test_x = test_x.reshape(test_x.shape[0], 2, int(
test_x.shape[1] / 2), test_x.shape[2])
val_x = val_x.reshape(val_x.shape[0], 2, int(
val_x.shape[1] / 2), val_x.shape[2])
model = Sequential()
model.add(TimeDistributed(Conv1D(filters=128, kernel_size=1, activation='tanh'),
input_shape=(None, train_x.shape[2], train_x.shape[3])))
model.add(TimeDistributed(MaxPooling1D(pool_size=4)))
# # #print(model.summary())
# model.add(TimeDistributed(Conv1D(filters=64, kernel_size=1, activation='tanh'), input_shape=(None, train_x.shape[2],train_x.shape[3])))
# model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
# # #print(model.summary())
# model.add(TimeDistributed(Conv1D(filters=64, kernel_size=2, activation='tanh'), input_shape=(None, train_x.shape[2],train_x.shape[3])))
# model.add(TimeDistributed(MaxPooling1D(pool_size=4)))
# model.add(TimeDistributed(Conv1D(filters=32, kernel_size=2, activation='tanh'), input_shape=(None, train_x.shape[2],train_x.shape[3])))
# model.add(TimeDistributed(MaxPooling1D(pool_size=4)))
model.add(TimeDistributed(Flatten()))
# print(model.summary())
model.add(Bidirectional(LSTM(60, return_sequences=True)))
model.add(Bidirectional(LSTM(48, return_sequences=True)))
model.add(Bidirectional(LSTM(48)))
model.add(Dropout(0.5))
def make_model(num_tfs, num_bws, num_motifs, num_recurrent, num_dense, dropout_rate):
from keras import backend as K
from keras.models import Model
from keras.layers import Dense, Dropout, Activation, Flatten, Layer, merge, Input
from keras.layers.convolutional import Convolution1D, MaxPooling1D
from keras.layers.pooling import GlobalMaxPooling1D
from keras.layers.recurrent import LSTM
from keras.layers.wrappers import Bidirectional, TimeDistributed
'''
import tensorflow as tf
config = tf.ConfigProto(device_count={'gpu':1})
config.gpu_options.allow_growth=True
session = tf.Session(config=config)
'''
forward_input = Input(shape=(L, 4 + num_bws,))
reverse_input = Input(shape=(L, 4 + num_bws,))
if num_recurrent < 0:
hidden_layers = [
Convolution1D(input_dim=4 + num_bws, nb_filter=num_motifs,
filter_length=w, border_mode='valid', activation='relu',
subsample_length=1),
Dropout(0.1),
TimeDistributed(Dense(num_motifs, activation='relu')),
GlobalMaxPooling1D(),
Dropout(dropout_rate),
Dense(num_dense, activation='relu'),
Dropout(dropout_rate),
Dense(num_tfs, activation='sigmoid')
]
elif num_recurrent == 0:
hidden_layers = [
Convolution1D(input_dim=4 + num_bws, nb_filter=num_motifs,
filter_length=w, border_mode='valid', activation='relu',
subsample_length=1),
Dropout(0.1),
TimeDistributed(Dense(num_motifs, activation='relu')),
MaxPooling1D(pool_length=w2, stride=w2),
Dropout(dropout_rate),
Flatten(),
Dense(num_dense, activation='relu'),
Dropout(dropout_rate),
Dense(num_tfs, activation='sigmoid')
]
else:
hidden_layers = [
Convolution1D(input_dim=4 + num_bws, nb_filter=num_motifs,
filter_length=w, border_mode='valid', activation='relu',
subsample_length=1),
Dropout(0.1),
TimeDistributed(Dense(num_motifs, activation='relu')),
MaxPooling1D(pool_length=w2, stride=w2),
Bidirectional(LSTM(num_recurrent, dropout_W=0.1, dropout_U=0.1, return_sequences=True)),
Dropout(dropout_rate),
Flatten(),
Dense(num_dense, activation='relu'),
Dropout(dropout_rate),
Dense(num_tfs, activation='sigmoid')
]
forward_output = get_output(forward_input, hidden_layers)
reverse_output = get_output(reverse_input, hidden_layers)
output = merge([forward_output, reverse_output], mode='ave')
model = Model(input=[forward_input, reverse_input], output=output)
return model
print(X_test.shape[0], 'test samples')
# convert class vectors to binary class matrices
Y_train = np_utils.to_categorical(y_train.astype(int), nb_classes)
Y_test = np_utils.to_categorical(y_test.astype(int), nb_classes)
print('Y_train shape:', Y_train.shape)
model = Sequential()
model.add(
Convolution1D(nb_filters1,
nb_conv,
border_mode='valid',
input_shape=(time, freq)))
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_length=4))
model.add(Dropout(0.75))
model.add(Convolution1D(
nb_filters1,
nb_conv,
border_mode='valid',
))
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_length=2))
#model.add(Dropout(0.25))
model.add(Convolution1D(
nb_filters2,
nb_conv,
border_mode='valid',
def cnn_lstm(train_x, test_x, train_y, test_y):
if not os.path.exists("models"):
os.mkdir("models")
class_weights = \
class_weight.compute_class_weight('balanced',
np.unique(train_y),
train_y)
class_weights = dict(enumerate(class_weights))
scaler = StandardScaler()
scaler.fit(train_x)
train_x = scaler.transform(train_x)
test_x = scaler.transform(test_x)
train_y = to_categorical(train_y)
test_y = to_categorical(test_y)
# define model
verbose, epochs, batch_size = 1, 25, 32
print(train_x.shape)
print(train_y.shape)
n_timesteps, n_outputs = train_x.shape[1], train_y.shape[1]
n_steps = 10
n_features = 1
# reshape data into time steps of sub-sequences
n_length = int(n_timesteps / n_steps)
train_x = train_x.reshape(
(train_x.shape[0], n_steps, n_length, n_features))
test_x = test_x.reshape((test_x.shape[0], n_steps, n_length, n_features))
print(train_x.shape)
print(test_x.shape)
# define model
model = Sequential()
model.add(
TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu'),
input_shape=(None, n_length, n_features)))
model.add(
TimeDistributed(Conv1D(filters=64, kernel_size=3, activation='relu')))
model.add(TimeDistributed(Dropout(0.5)))
model.add(TimeDistributed(MaxPooling1D(pool_size=2)))
model.add(TimeDistributed(Flatten()))
model.add(LSTM(80))
model.add(Dropout(0.5))
model.add(Dense(30, activation='relu'))
model.add(Dense(n_outputs, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# fit network
checkpoint = \
ModelCheckpoint("models/physiological_cnn_model.h5",
monitor='val_accuracy',
verbose=1,
save_best_only=True,
save_weights_only=False,
mode='auto',
period=1)
early_stopping = \
EarlyStopping(monitor='val_loss',
patience=50,
verbose=1,
mode='auto')
model.summary()
model.fit(train_x,
train_y,
epochs=epochs,
batch_size=batch_size,
verbose=verbose,
class_weight=class_weights,
validation_data=(np.array(test_x), np.array(test_y)),
callbacks=[checkpoint, early_stopping])
# evaluate model
_, accuracy = model.evaluate(test_x,
test_y,
batch_size=batch_size,
verbose=0)
physiological_model = load_model("models/physiological_cnn_model.h5")
preds_physiological = physiological_model.predict_proba(np.array(test_x))
print(accuracy)
return preds_physiological
#######################################
# build model
#######################################
print 'building model'
model = Sequential()
model.add(
Convolution1D(input_dim=4,
input_length=maxlen,
nb_filter=nb_filter,
filter_length=filter_length,
border_mode="valid",
activation="relu",
subsample_length=1))
model.add(MaxPooling1D(pool_length=13, stride=13))
#model.add(Dropout(0.2))
'''
model.add(brnn)
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(input_dim=75*640, output_dim=925))
model.add(Activation('relu'))
model.add(Dense(input_dim=925, output_dim=919))
model.add(Activation('sigmoid'))
'''
def build_reference_annotation_1d_model_from_args(args,
conv_width = 6,
conv_layers = [128, 128, 128, 128],
conv_dropout = 0.0,
conv_batch_normalize = False,
spatial_dropout = True,
max_pools = [],
padding='valid',
activation = 'relu',
annotation_units = 16,
annotation_shortcut = False,
annotation_batch_normalize = True,
fc_layers = [64],
fc_dropout = 0.0,
fc_batch_normalize = False,
fc_initializer = 'glorot_normal',
kernel_initializer = 'glorot_normal',
alpha_dropout = False
):
'''Build Reference 1d CNN model for classifying variants.
Architecture specified by parameters.
Dynamically sets input channels based on args via defines.total_input_channels_from_args(args)
Uses the functional API.
Prints out model summary.
Arguments
args.annotations: The variant annotations, perhaps from a HaplotypeCaller VCF.
args.labels: The output labels (e.g. SNP, NOT_SNP, INDEL, NOT_INDEL)
Returns
The keras model
'''
in_channels = tensor_maps.total_input_channels_from_args(args)
concat_axis = -1
x = reference = Input(shape=(args.window_size, in_channels), name=args.tensor_name)
max_pool_diff = len(conv_layers)-len(max_pools)
for i,c in enumerate(conv_layers):
if conv_batch_normalize:
x = Conv1D(filters=c, kernel_size=conv_width, activation='linear', padding=padding, kernel_initializer=kernel_initializer)(x)
x = BatchNormalization(axis=concat_axis)(x)
x = Activation(activation)(x)
else:
x = Conv1D(filters=c, kernel_size=conv_width, activation=activation, padding=padding, kernel_initializer=kernel_initializer)(x)
if conv_dropout > 0 and alpha_dropout:
x = AlphaDropout(conv_dropout)(x)
elif conv_dropout > 0 and spatial_dropout:
x = SpatialDropout1D(conv_dropout)(x)
elif conv_dropout > 0:
x = Dropout(conv_dropout)(x)
if i >= max_pool_diff:
x = MaxPooling1D(max_pools[i-max_pool_diff])(x)
f = Flatten()(x)
annotations = annotations_in = Input(shape=(len(args.annotations),), name=args.annotation_set)
if annotation_batch_normalize:
annotations_in = BatchNormalization(axis=concat_axis)(annotations_in)
annotation_mlp = Dense(units=annotation_units, kernel_initializer=fc_initializer, activation=activation)(annotations_in)
x = layers.concatenate([f, annotation_mlp], axis=1)
for fc in fc_layers:
if fc_batch_normalize:
x = Dense(units=fc, activation='linear', kernel_initializer=fc_initializer)(x)
x = BatchNormalization(axis=1)(x)
x = Activation(activation)(x)
else:
x = Dense(units=fc, activation=activation, kernel_initializer=fc_initializer)(x)
if fc_dropout > 0 and alpha_dropout:
x = AlphaDropout(fc_dropout)(x)
elif fc_dropout > 0:
x = Dropout(fc_dropout)(x)
if annotation_shortcut:
x = layers.concatenate([x, annotations_in], axis=1)
prob_output = Dense(units=len(args.labels), activation='softmax', name='softmax_predictions')(x)
model = Model(inputs=[reference, annotations], outputs=[prob_output])
adam = Adam(lr=0.0001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, clipnorm=1.)
model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=get_metrics(args.labels))
model.summary()
if os.path.exists(args.weights_hd5):
model.load_weights(args.weights_hd5, by_name=True)
print('Loaded model weights from:', args.weights_hd5)
return model
def run(model_name, dataset_name):
"""
run the baseline model
:param model_name: name of baseline models, CNN or LSTM
:param dataset_name: name of datasets, candidate values [bbc, digg, MySpace, rw, Twitter, YouTube]
"""
print "model: %s" % model_name
print "process dataset %s..." % dataset_name
data = cPickle.load(open('./pkl/%s.pkl' % dataset_name, 'rb'))
records, glove_embeddings, vocab, word_to_df = data
dim_emb = len(glove_embeddings[1])
# index of word starts from 1
# initial weights of embedding layer
embeddings = np.zeros((len(vocab) + 1, dim_emb), dtype='float32')
for w in vocab:
wid = vocab[w]
embeddings[wid, :] = glove_embeddings[wid]
train_x, train_y, val_x, val_y, test_x, test_y, test_strength = [], [], [], [], [], [], []
max_len = 0
for r in records:
text = r['text']
y = r['label']
wids = [vocab[w] for w in text.split(' ')]
if len(wids) > max_len:
max_len = len(wids)
if r['type'] == 'train':
train_x.append(wids)
train_y.append(y)
elif r['type'] == 'val':
val_x.append(wids)
val_y.append(y)
elif r['type'] == 'test':
strength = r['strength']
test_x.append(wids)
test_y.append(y)
test_strength.append(strength)
train_x, val_x, test_x = ToArray(train=train_x, val=val_x, test=test_x)
train_y, val_y, test_y = ToArray(train=train_y, val=val_y, test=test_y)
_, _, test_strength = ToArray(train=[], val=[], test=test_strength)
#print train_x.shape, val_x.shape, test_x.shape
train_x, val_x, test_x = Padding(train=train_x,
val=val_x,
test=test_x,
max_len=max_len)
batch_size = 50 if model_name == 'CNN' else 32
if train_x.shape[0] % batch_size:
n_extra = batch_size - train_x.shape[0] % batch_size
x_extra = train_x[:n_extra, :]
y_extra = train_y[:n_extra]
train_x = np.append(train_x, x_extra, axis=0)
train_y = np.append(train_y, y_extra, axis=0)
np.random.seed(38438)
# shuffle the training set
train_set = np.random.permutation(zip(train_x, train_y))
train_x, train_y = [], []
for (x, y) in train_set:
train_x.append(x)
train_y.append(y)
n_labels = 2
train_x = np.array(train_x)
train_y = np.array(train_y)
train_y = to_categorical(train_y)
val_y = to_categorical(val_y)
print "n_train: %s, n_val: %s, n_test: %s" % (
train_x.shape[0], val_x.shape[0], test_x.shape[0])
if model_name == 'CNN':
model = Graph()
model.add_input(name='input', input_shape=(max_len, ), dtype='int')
model.add_node(Embedding(input_dim=len(vocab) + 1,
output_dim=dim_emb,
input_length=max_len,
weights=[embeddings]),
name="emb",
input="input")
filter_hs = [3, 4, 5]
n_filter = 100
dropout_rate = 0.5
n_epoch = 20
for i in xrange(len(filter_hs)):
win_size = filter_hs[i]
conv_name = 'conv%s' % i
pool_name = "pool%s" % i
flatten_name = "flatten%s" % i
pool_size = max_len - win_size + 1
model.add_node(
layer=Convolution1D(nb_filter=n_filter,
filter_length=win_size,
activation='relu',
W_constraint=maxnorm(m=3),
b_constraint=maxnorm(m=3)),
name=conv_name,
input='emb',
)
model.add_node(layer=MaxPooling1D(pool_length=pool_size),
name=pool_name,
input=conv_name)
model.add_node(layer=Flatten(), name=flatten_name, input=pool_name)
model.add_node(layer=Dropout(p=dropout_rate),
name="dropout",
inputs=["flatten0", "flatten1", "flatten2"])
model.add_node(layer=Dense(output_dim=n_labels, activation='softmax'),
name='softmax',
input='dropout')
model.add_output(input='softmax', name="output")
model.compile(loss={'output': 'categorical_crossentropy'},
optimizer='adadelta',
metrics=['accuracy'])
model_path = './model/%s_%s.hdf5' % (model_name, dataset_name)
best_model = ModelCheckpoint(filepath=model_path,
monitor='val_acc',
save_best_only=True,
mode='max')
print "training..."
model.fit(data={
'input': train_x,
'output': train_y
},
batch_size=batch_size,
nb_epoch=n_epoch,
validation_data={
'input': val_x,
'output': val_y
},
callbacks=[best_model],
verbose=0)
else:
model = Sequential()
model.add(
Embedding(input_dim=len(vocab) + 1,
output_dim=dim_emb,
mask_zero=True,
input_length=max_len,
weights=[embeddings]))
model.add(LSTM(output_dim=128, dropout_W=0.2, dropout_U=0.2))
model.add(Dense(n_labels, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model_path = './model/%s_%s.hdf5' % (model_name, dataset_name)
best_model = ModelCheckpoint(filepath=model_path,
monitor='val_acc',
save_best_only=True,
mode='max')
n_epoch = 20
print "training..."
model.fit(x=train_x,
y=train_y,
batch_size=batch_size,
nb_epoch=n_epoch,
validation_data=(val_x, val_y),
callbacks=[best_model],
verbose=0)
pred_strength = []
print "load the best model from disk..."
model.load_weights(model_path)
if model_name == 'LSTM':
pred_strength = model.predict(x=test_x, batch_size=batch_size)
else:
for i in xrange(len(test_x)):
res = model.predict(data={'input': test_x[i:i + 1]}, batch_size=1)
pred_strength.append(res['output'])
pred_strength = np.array(pred_strength)
pred_strength = pred_strength.reshape(
(pred_strength.shape[0], pred_strength.shape[2]))
assert pred_strength.shape == test_strength.shape
print "evaluate performance of the system..."
accu, mae, rmse = evaluate(strength_gold=test_strength,
strength_pred=pred_strength)
print "%s over %s--->accuracy: %s, mae: %s, rmse: %s\n\n" % (
model_name, dataset_name, accu, mae, rmse)
pred_strengths_lines = []
for strength in pred_strength:
pred_strengths_lines.append('%s\n' %
' '.join([str(ele) for ele in strength]))
def VGGLikeModel(weights='ResNetLike-125k.wts.h5',
input_shape=[1024, 2],
classes=24,
**kwargs):
if weights is not None and not (os.path.exists(weights)):
raise ValueError('The `weights` argument should be either '
'`None` (random initialization), '
'or the path to the weights file to be loaded.')
dr = 0.5 # dropout rate (%)
model = models.Sequential()
tap = 8
model.add(
Conv1D(input_shape=input_shape,
filters=64,
kernel_size=tap,
padding='same',
activation="relu",
name="conv1",
kernel_initializer='glorot_uniform'))
model.add(MaxPooling1D(pool_size=2, strides=2))
model.add(
Conv1D(filters=64,
kernel_size=tap,
padding='same',
activation="relu",
name="conv2",
kernel_initializer='glorot_uniform'))
model.add(MaxPooling1D(pool_size=2, strides=2))
model.add(
Conv1D(filters=64,
kernel_size=tap,
padding='same',
activation="relu",
name="conv3",
kernel_initializer='glorot_uniform'))
model.add(MaxPooling1D(pool_size=2, strides=2))
model.add(
Conv1D(filters=64,
kernel_size=tap,
padding='same',
activation="relu",
name="conv4",
kernel_initializer='glorot_uniform'))
model.add(MaxPooling1D(pool_size=2, strides=2))
model.add(
Conv1D(filters=64,
kernel_size=tap,
padding='same',
activation="relu",
name="conv5",
kernel_initializer='glorot_uniform'))
model.add(MaxPooling1D(pool_size=2, strides=2))
model.add(
Conv1D(filters=64,
kernel_size=tap,
padding='same',
activation="relu",
name="conv6",
kernel_initializer='glorot_uniform'))
model.add(MaxPooling1D(pool_size=2, strides=2))
model.add(
Conv1D(filters=64,
kernel_size=tap,
padding='same',
activation="relu",
name="conv7",
kernel_initializer='glorot_uniform'))
model.add(MaxPooling1D(pool_size=2, strides=2))
model.add(Flatten())
model.add(
Dense(128,
activation='selu',
kernel_initializer='he_normal',
name="dense1",
kernel_regularizer=regularizers.l2(0.01)))
model.add(Dropout(dr))
model.add(
Dense(128,
activation='selu',
kernel_initializer='he_normal',
name="dense2"))
model.add(Dropout(dr))
model.add(
Dense(classes,
kernel_initializer='he_normal',
name="dense3",
kernel_regularizer=regularizers.l2(0.01)))
model.add(Activation('softmax'))
# model.add(Reshape((classes,)))
# Load weights.
if weights is not None:
model.load_weights(weights)
return model
def trainModel(self):
'''
Definición y compilación del modelo.
'''
input_size = self.max_length
#Inputs y embeddings
sequence_input = Input(shape=(input_size, ),
dtype='float64',
name="input_data")
sequence_input_lengths = Input(shape=(1, ),
dtype='int32',
name="input_sequence_lengths")
#Definimos función de pérdida -> en función de sequence_input_lengths
self.loss_object = Lossfunction(self.batch_size,
sequence_input_lengths)
embedding_layer = Embedding(self.word_embeddings.shape[0],
self.word_embeddings.shape[1],
weights=[self.word_embeddings],
trainable=False,
input_length=input_size,
name="embeddings") #Trainable false
embedded_sequence = embedding_layer(sequence_input)
#Una primera capa con bi-LSTM
x = Bidirectional(LSTM(units=self.num_units,
return_sequences=True))(embedded_sequence)
#Primera convolución
x = Conv1D(filters=self.filter_1,
kernel_size=self.kernel_1,
padding="same",
activation="tanh",
name="first_convolution")(x)
x = MaxPooling1D(pool_size=2,
strides=1,
padding="same",
name="first_max_pooling")(x)
x = Dropout(self.dropout_1, name="first_dropout")(x)
#Segunda
x = Conv1D(filters=self.filter_2,
kernel_size=self.kernel_2,
padding="same",
activation="tanh",
name="second_convolution")(x)
x = MaxPooling1D(pool_size=2,
strides=1,
padding="same",
name="second_max_pooling")(x)
x = Dropout(self.dropout_2, name="second_dropout")(x)
#Última capa
#preds = TimeDistributed(Dense(4, name = "last_layer"))(x)
#preds = Dense(4, activation = "tanh", name = "last_layer")(x)
preds = Dense(4, name="last_layer")(x)
#Creamos el modelo
self.model = Model(input=[sequence_input, sequence_input_lengths],
output=[preds])
self.model.summary()
self.model.compile(loss=self.loss_object.loss,
optimizer='adam',
metrics=['accuracy'])
NUM_EPOCHS = 20
EMBEDDING_SIZE = 128
HIDDEN_LAYER_SIZE = 64
"""embedding_layer = Embedding(vocab_size + 1 ,
EMBEDDING_DIM,
weights = word_dict,
input_length = max_length,
trainable=False)"""
model = Sequential()
model.add(
Conv1D(filters=32,
kernel_size=3,
padding='same',
activation='relu',
input_shape=input_shape))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.2))
# 1 layer of 100 units in the hidden layers of the LSTM cells
model.add(LSTM(64))
model.add(Dense(5, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# 模型訓練
checkpoint = ModelCheckpoint('sentiment-analysis-model',
verbose=1,
monitor='loss',
save_best_only=True,
mode='min')
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
def build_model(num_classes,
input_dim=(500, 64, 1),
epochs=100,
bs=32,
lr=1e-3,
init='normal'):
#1 CONV => CONV => POOL
model = Sequential()
model.add(
Conv1D(64, 3, padding='same', activation='tanh',
input_shape=input_dim))
model.add(BatchNormalization(axis=-1))
model.add(Conv1D(64, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(MaxPooling2D(pool_size=2))
#2 CONV => CONV => POOL
model.add(Conv1D(128, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(Conv1D(128, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(MaxPooling1D(pool_size=2))
#3 CONV => CONV => CONV => POOL
model.add(Conv1D(256, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(Conv1D(256, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(Conv1D(256, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.1))
#4 CONV => CONV => CONV => POOL
model.add(Conv1D(512, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(Conv1D(512, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(Conv1D(512, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.1))
#5 CONV => CONV => CONV => POOL
model.add(Conv1D(512, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(Conv1D(512, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(Conv1D(512, 3, padding='same', activation='relu'))
model.add(BatchNormalization(axis=-1))
model.add(MaxPooling1D(pool_size=2))
model.add(Dropout(0.1))
#6 DENSE-1
model.add(Dense(14 * 4 * 512))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
#7 DENSE-2
model.add(Dense(64 * 64))
model.add(Activation('relu'))
model.add(BatchNormalization())
model.add(Dropout(0.5))
# DENSE-3 => OUTPUT
model.add(Dense(num_classes))
model.add(Activation('softmax'))
# COMPILE MODEL
opt = Adam(lr=lr, decay=lr / epochs)
model.compile(loss="categorical_crossentropy",
optimizer=opt,
metrics=["accuracy"])
return model
