def build(i):
#input = Input(shape=(sentence_length,))
# embedding
embedding = Embedding(len(vocab), WORD_EMBEDDING_DIM)
input_embedding = embedding(i)
conv = Convolution1D(
nb_filter=NB_FILTER,
filter_length=FILTER_LENGTH,
border_mode='same',
#activation='tanh',
subsample_length=1,
W_constraint = maxnorm(3),
b_constraint = maxnorm(3),
#input_shape=(AMAX_TIMESTAMP, WORD_EMBEDDING_DIM),
#name='conv'
)
# dropout = Dropout(0.5)
# dropout
input_dropout = conv(input_embedding)
# maxpooling
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False),
output_shape=lambda x: (x[0], x[2]))
input_pool = maxpool(input_dropout)
# activation
activation = Activation('tanh')
output = activation(input_pool)
return output
def buildConvolution(self, name):
filters = self.params.get('filters')
nb_filter = self.params.get('nb_filter')
assert filters
assert nb_filter
convs = []
for fsz in filters:
layer_name = '%s-conv-%d' % (name, fsz)
conv = Convolution1D(
nb_filter=nb_filter,
filter_length=fsz,
border_mode='valid',
#activation='relu',
subsample_length=1,
init='glorot_uniform',
#init=init,
#init=lambda shape, name: initializations.uniform(shape, scale=0.01, name=name),
W_constraint=maxnorm(self.params.get('w_maxnorm')),
b_constraint=maxnorm(self.params.get('b_maxnorm')),
#W_regularizer=regularizers.l2(self.params.get('w_l2')),
#b_regularizer=regularizers.l2(self.params.get('b_l2')),
#input_shape=(self.q_length, self.wdim),
name=layer_name
)
convs.append(conv)
self.layers['%s-convolution' % name] = convs
def build_embedding_layer(args):
try:
n_embeddings = args.n_vocab
except AttributeError:
n_embeddings = args.n_embeddings
try:
mask_zero = args.mask_zero
except AttributeError:
mask_zero = False
if hasattr(args, 'embedding_weights') and args.embedding_weights is not None:
W = np.load(args.embedding_weights)
if args.train_embeddings is True or args.train_embeddings == 'true':
return Embedding(n_embeddings, args.n_embed_dims,
weights=[W], input_length=args.input_width,
W_constraint=maxnorm(args.embedding_max_norm),
mask_zero=mask_zero)
else:
return ImmutableEmbedding(n_embeddings, args.n_embed_dims,
weights=[W], mask_zero=mask_zero,
input_length=args.input_width)
else:
if args.train_embeddings is True:
return Embedding(n_embeddings, args.n_embed_dims,
init=args.embedding_init,
W_constraint=maxnorm(args.embedding_max_norm),
mask_zero=mask_zero,
input_length=args.input_width)
else:
return ImmutableEmbedding(n_embeddings, args.n_embed_dims,
init=args.embedding_init,
mask_zero=mask_zero,
input_length=args.input_width)
def create_model(): # create model model = Sequential() model.add(Dropout(0.2, input_shape=(60,))) model.add(Dense(60, init='normal', activation='relu', W_constraint=maxnorm(3))) model.add(Dense(30, init='normal', activation='relu', W_constraint=maxnorm(3))) model.add(Dense(1, init='normal', activation='sigmoid')) # Compile model sgd = SGD(lr=0.1, momentum=0.9, decay=0.0, nesterov=False) model.compile(loss='binary_crossentropy', optimizer=sgd, metrics=['accuracy']) return model
def setup_model_3b(patch_size, in_chan):
inputs = Input((patch_size, patch_size, in_chan))
drop1 = Dropout(0.1)(inputs)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(drop1)
conv1 = Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(conv1)
pool1 = MaxPooling2D(pool_size=(2, 2))(conv1)
drop2 = Dropout(0.05)(pool1)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(drop2)
conv2 = Conv2D(64, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(conv2)
pool2 = MaxPooling2D(pool_size=(2, 2))(conv2)
drop3 = Dropout(0.05)(pool2)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(drop3)
conv3 = Conv2D(128, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3))(conv3)
pool3 = MaxPooling2D(pool_size=(2, 2))(conv3)
flat = Flatten()(pool3)
dense8 = Dense(64, activation='relu', kernel_constraint=maxnorm(3))(flat)
dense9 = Dense(16, activation='relu', kernel_constraint=maxnorm(3))(dense8)
output = Dense(1,activation='sigmoid')(dense9)
model = Model(inputs=inputs, outputs=output)
model.compile(optimizer=Adam(decay=0.0001),
loss='binary_crossentropy',
metrics=['accuracy',
libs.metrics.f1_score,
libs.metrics.precision,
libs.metrics.recall])
print model.summary()
return model
def test_maxnorm():
for m in test_values:
norm_instance = constraints.maxnorm(m)
normed = norm_instance(K.variable(example_array))
assert(np.all(K.eval(normed) < m))
# a more explicit example
norm_instance = constraints.maxnorm(2.0)
x = np.array([[0, 0, 0], [1.0, 0, 0], [3, 0, 0], [3, 3, 3]]).T
x_normed_target = np.array([[0, 0, 0], [1.0, 0, 0],
[2.0, 0, 0],
[2. / np.sqrt(3), 2. / np.sqrt(3), 2. / np.sqrt(3)]]).T
x_normed_actual = K.eval(norm_instance(K.variable(x)))
assert_allclose(x_normed_actual, x_normed_target, rtol=1e-05)
def test_maxnorm(self):
from keras.constraints import maxnorm
for m in self.some_values:
norm_instance = maxnorm(m)
normed = norm_instance(self.example_array)
assert (np.all(normed.eval() < m))
# a more explicit example
norm_instance = maxnorm(2.0)
x = np.array([[0, 0, 0], [1.0, 0, 0], [3, 0, 0], [3, 3, 3]]).T
x_normed_target = np.array([[0, 0, 0], [1.0, 0, 0], [2.0, 0, 0], [2./np.sqrt(3), 2./np.sqrt(3), 2./np.sqrt(3)]]).T
x_normed_actual = norm_instance(x).eval()
assert_allclose(x_normed_actual, x_normed_target)
def build_model():
main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input')
embedding = Embedding(max_features, embedding_dims,
weights=[np.matrix(W)], input_length=maxlen,
name='embedding')(main_input)
embedding = Dropout(0.50)(embedding)
conv4 = Conv1D(filters=nb_filter,
kernel_size=4,
padding='valid',
activation='relu',
strides=1,
name='conv4')(embedding)
maxConv4 = MaxPooling1D(pool_size=2,
name='maxConv4')(conv4)
conv5 = Conv1D(filters=nb_filter,
kernel_size=5,
padding='valid',
activation='relu',
strides=1,
name='conv5')(embedding)
maxConv5 = MaxPooling1D(pool_size=2,
name='maxConv5')(conv5)
# x = merge([maxConv4, maxConv5], mode='concat')
x = keras.layers.concatenate([maxConv4, maxConv5])
x = Dropout(0.15)(x)
x = RNN(rnn_output_size)(x)
x = Dense(hidden_dims, activation='relu', kernel_initializer='he_normal',
kernel_constraint = maxnorm(3), bias_constraint=maxnorm(3),
name='mlp')(x)
x = Dropout(0.10, name='drop')(x)
output = Dense(1, kernel_initializer='he_normal',
activation='sigmoid', name='output')(x)
model = Model(inputs=main_input, outputs=output)
model.compile(loss='binary_crossentropy',
# optimizer=Adadelta(lr=0.95, epsilon=1e-06),
# optimizer=Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0),
# optimizer=Adagrad(lr=0.01, epsilon=1e-08, decay=1e-4),
metrics=["accuracy"])
return model
def build_model():
print('Build model...%d of %d' % (i + 1, folds))
main_input = Input(shape=(maxlen, ), dtype='int32', name='main_input')
embedding = Embedding(max_features, embedding_dims,
weights=[np.matrix(W)], input_length=maxlen,
name='embedding')(main_input)
embedding = Dropout(0.50)(embedding)
conv4 = Convolution1D(nb_filter=nb_filter,
filter_length=4,
border_mode='valid',
activation='relu',
subsample_length=1,
name='conv4')(embedding)
maxConv4 = MaxPooling1D(pool_length=2,
name='maxConv4')(conv4)
conv5 = Convolution1D(nb_filter=nb_filter,
filter_length=5,
border_mode='valid',
activation='relu',
subsample_length=1,
name='conv5')(embedding)
maxConv5 = MaxPooling1D(pool_length=2,
name='maxConv5')(conv5)
x = merge([maxConv4, maxConv5], mode='concat')
x = Dropout(0.15)(x)
x = RNN(rnn_output_size)(x)
x = Dense(hidden_dims, activation='relu', init='he_normal',
W_constraint = maxnorm(3), b_constraint=maxnorm(3),
name='mlp')(x)
x = Dropout(0.10, name='drop')(x)
output = Dense(1, init='he_normal',
activation='sigmoid', name='output')(x)
model = Model(input=main_input, output=output)
model.compile(loss={'output':'binary_crossentropy'},
optimizer=Adadelta(lr=0.95, epsilon=1e-06),
metrics=["accuracy"])
return model
def build_dense_layer(config, n_hidden=None, activation='linear'):
if n_hidden is None:
n_hidden = config.n_hidden
return Dense(n_hidden,
W_regularizer=l2(config.l2_penalty),
W_constraint=maxnorm(config.dense_max_norm),
activation=activation)
def build_model(self, n_classes, max_words, w2v_size, vocab_size, filter_sizes, n_filters,
dense_layer_sizes, activation_function, dropout_p):
positive_node, positive_input = self._build_conv_node(activation_function, max_words, w2v_size, vocab_size, n_filters, filter_sizes)
negative_node, negative_input = self._build_conv_node(activation_function, max_words, w2v_size, vocab_size, n_filters, filter_sizes)
merged_dense_layer = merge([[positive_node, negative_node], 'concat'])
dense_layers = []
first_dense_layer = Dense(dense_layer_sizes.pop(0))(merged_dense_layer)
dense_layers.append(first_dense_layer)
if len(dense_layer_sizes) > 1:
i = 1
for dense_layer_size in dense_layer_sizes:
dense_layer = Dense(dense_layer_size, W_constraint=maxnorm(2))(dense_layer_sizes[i-1])
dense_activation_layer = Activation(activation_function)(dense_layer)
dropout_layer = Dropout(dropout_p)(dense_activation_layer)
dense_layers.append(dropout_layer)
i += 1
# Add last layer
softmax_dense_layer = Dense(n_classes)(dense_layers[-1])
softmax_layer = Activation('softmax')(softmax_dense_layer)
model = Model(input=[positive_input, negative_input], output=softmax_layer)
return model
def make_model(self,len_feature):
##########################Parameters for the model and dataset
#TRAINING_SIZE = len(inputs)
# Try replacing JZS1 with LSTM, GRU, or SimpleRNN
HIDDEN_SIZE = self.node0
RNN = recurrent.LSTM(HIDDEN_SIZE, input_shape=(None, len(self.chars)),
return_sequences=False,kernel_regularizer=l2(self.l2_c),
bias_regularizer=l2(self.l2_c),recurrent_dropout=self.drop_out_c,
dropout=self.drop_out_c)
#len0_hla = 34
#ratio_t = 1
###class number = binder or non-binder (1 = binder, 0 = non-binder)
#classes = [0,1]
##########################start a model##########################
##########fixed part
model_fixed = Sequential()
model_fixed.add(Dense(help_nn,input_dim=len_feature,
activation=act_fun, kernel_constraint=maxnorm(constrain_max)))
model_fixed.add(Dropout(drop_out_c))
##########recurrent part
model_r = Sequential()
if mask0:
model_r.add(Masking(mask_value=0., input_shape=(MAXLEN, len(dict_aa['A']))))
model_r.add(RNN)
####merge
merged = Merge([model_fixed, model_r],mode='concat')
###final
final_model = Sequential()
final_model.add(merged)
for _ in range(0,help_layer0):
final_model.add(Dense(help_nn, kernel_constraint=maxnorm(constrain_max)))
final_model.add(Activation(act_fun))
final_model.add(Dropout(drop_out_c))
final_model.add(Dense(1))
final_model.compile(loss=loss_function0, optimizer="adam")
model = final_model
json_string = model.to_json()
open(path_save+file_name0+out_name+'_model.json', 'w').write(json_string)
return model
def build_convolutional_context_model(config, n_classes):
graph = Graph()
prev_non_word_layer = build_char_model(graph, config)
# Word-level input for the context of the non-word error.
graph.add_input(config.context_input_name,
input_shape=(config.context_input_width,), dtype='int')
context_embedding = build_embedding_layer(config,
input_width=config.context_input_width,
n_embeddings=config.n_context_embeddings,
n_embed_dims=config.n_context_embed_dims)
graph.add_node(context_embedding,
name='context_embedding', input=config.context_input_name)
context_conv = build_convolutional_layer(config,
n_filters=config.n_context_filters,
filter_width=config.context_filter_width)
context_conv.trainable = config.train_filters
graph.add_node(context_conv, name='context_conv', input='context_embedding')
context_prev_layer = add_bn_relu(graph, config, 'context_conv')
context_pool = build_pooling_layer(config,
input_width=config.context_input_width,
filter_width=config.context_filter_width)
graph.add_node(context_pool,
name='context_pool', input=context_prev_layer)
graph.add_node(Flatten(), name='context_flatten', input='context_pool')
prev_context_layer = 'context_flatten'
prev_layer = build_fully_connected_layers(graph, config,
prev_non_word_layer,
prev_context_layer)
prev_layer = build_residual_blocks(graph, config, prev_layer)
if hasattr(config, 'n_hsm_classes'):
graph.add_node(build_hierarchical_softmax_layer(config),
name='softmax', input=prev_layer)
else:
graph.add_node(Dense(n_classes, init=config.softmax_init,
W_constraint=maxnorm(config.softmax_max_norm)),
name='softmax', input=prev_layer)
prev_layer = 'softmax'
if config.batch_normalization:
graph.add_node(BatchNormalization(), name='softmax_bn', input='softmax')
prev_layer = 'softmax_bn'
graph.add_node(Activation('softmax'), name='softmax_activation', input=prev_layer)
graph.add_output(name='multiclass_correction_target', input='softmax_activation')
load_weights(config, graph)
optimizer = build_optimizer(config)
graph.compile(loss={'multiclass_correction_target': config.loss}, optimizer=optimizer)
return graph
def buildConvolution(self, name):
filters = self.params.get('filters')
nb_filter = self.params.get('nb_filter')
assert filters
assert nb_filter
convs = []
for fsz in filters:
layer_name = '%s-conv-%d' % (name, fsz)
conv = Convolution2D(
nb_filter=nb_filter,
nb_row=fsz,
nb_col=self.wdim,
border_mode='valid',
init='glorot_uniform',
W_constraint=maxnorm(self.params.get('w_maxnorm')),
b_constraint=maxnorm(self.params.get('b_maxnorm')),
name=layer_name
)
convs.append(conv)
self.layers['%s-convolution' % name] = convs
def build_embedding_layer(config, input_width=None):
try:
n_embeddings = config.n_vocab
except AttributeError:
n_embeddings = config.n_embeddings
try:
input_width = config.input_width
except AttributeError:
input_width = input_width
try:
mask_zero = config.mask_zero
except AttributeError:
mask_zero = False
if hasattr(config, 'embedding_weights') and config.embedding_weights is not None:
W = np.load(config.embedding_weights)
if config.train_embeddings is True or config.train_embeddings == 'true':
return Embedding(n_embeddings, config.n_embed_dims,
weights=[W], input_length=input_width,
W_constraint=maxnorm(config.embedding_max_norm),
mask_zero=mask_zero)
else:
return ImmutableEmbedding(n_embeddings, config.n_embed_dims,
weights=[W], mask_zero=mask_zero,
input_length=input_width)
else:
if config.train_embeddings is True:
return Embedding(n_embeddings, config.n_embed_dims,
init=config.embedding_init,
W_constraint=maxnorm(config.embedding_max_norm),
mask_zero=mask_zero,
input_length=input_width)
else:
return ImmutableEmbedding(n_embeddings, config.n_embed_dims,
init=config.embedding_init,
mask_zero=mask_zero,
input_length=input_width)
def double_conv_layerunet(x, size, dropout, batch_norm):
kernel_size=(3,3)
conv = Conv2D(size,kernel_size, activation='relu',kernel_constraint=maxnorm(4.),padding='same')(x)
# conv = Conv2D(size,kernel_size,kernel_constraint=maxnorm(4.,axis=-1),padding='same')(x)
if batch_norm == True:
conv = BatchNormalization( )(conv)
# conv = Activation('relu')(conv)
# conv = LeakyReLU(alpha=0.15)(conv)
# if dropout > 0:
# conv = Dropout(dropout)(conv)
conv = Conv2D(size, kernel_size,activation='relu',kernel_constraint=maxnorm(4.), padding='same')(conv)
# conv = Conv2D(size,kernel_size,kernel_constraint=maxnorm(4.,axis=-1),padding='same')(conv)
if batch_norm == True:
conv = BatchNormalization()(conv)
# conv = Activation('relu')(conv)
# conv = LeakyReLU(alpha=0.15)(conv)
if dropout > 0:
conv = Dropout(dropout)(conv)
return conv
def create_model(self):
'''
Create new keras model.
'''
self.class_weight = {0: 0.25, 1: 0.75}
model = Sequential()
model.add(Dense(self.data.shape[1], input_dim=self.data.shape[1],
activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='RMSprop', loss='binary_crossentropy',
metrics=['accuracy'])
return model
def train_ensemble(X_train, X_test, Y_train, n_models=6):
n_categs = Y_train.shape[1]
for i in range(n_models):
print('---' * 20)
print('Training model #: {}'.format(i + 1))
print('---' * 20)
model = Sequential()
dim = random.choice(np.arange(512, 769))
model.add(Dense(output_dim=dim, input_dim=X_train.shape[1],
init="glorot_uniform", W_constraint=maxnorm(1)))
model.add(PReLU())
model.add(Dropout(0.6))
model.add(Dense(output_dim=n_categs, init="glorot_uniform",
W_constraint=maxnorm(1)))
model.add(Activation("softmax"))
model.compile(loss='categorical_crossentropy',
optimizer='adagrad')
print("Training model...")
epoch = random.choice(np.arange(100, 400))
model.fit(X_train, Y_train, nb_epoch=epoch, batch_size=512)
if i == 0:
probs = model.predict_proba(X_test, batch_size=512)
else:
probs += model.predict_proba(X_test, batch_size=512)
probs /= n_models
column_names, index = get_extra()
df_out = pd.DataFrame(data=np.c_[index, probs], columns=column_names)
df_out['VisitNumber'] = np.int32(df_out['VisitNumber'])
df_out.to_csv('nnEnsemble.csv', index=False)
return df_out
def model():
model=Sequential()
model.add(Convolution2D(32,3,3,activation='relu',input_shape=(3,32,32),border_mode='same',W_constraint=maxnorm(3)))
#model.add(Dropout(0.2))
model.add(Convolution2D(32,3,3,activation='relu',input_shape=(3,32,32),border_mode='same',W_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2,2),strides=(2,2)))
model.add(Flatten())
model.add(Dense(512,activation='relu',W_constraint=maxnorm(3)))
#model.add(Dropout(0.5))
model.add(Dense(10,activation='softmax'))
return model
def double_conv_layerunet(x, size, dropout, batch_norm,dim_org,ke_i,kernel_siz):
if K.image_dim_ordering() == 'th':
axis = 1
else:
axis = 3
conv = Conv2D(size,kernel_siz, activation='relu',kernel_initializer=ke_i,
data_format=dim_org,kernel_constraint=maxnorm(4.),padding='same')(x)
if batch_norm == True:
conv = BatchNormalization(axis=axis)(conv)
if dropout > 0:
conv = Dropout(dropout)(conv)
conv = Conv2D(size, kernel_siz,activation='relu',kernel_initializer=ke_i,
data_format=dim_org,kernel_constraint=maxnorm(4.), padding='same')(conv)
if batch_norm == True:
conv = BatchNormalization(axis=axis)(conv)
# conv = LeakyReLU(alpha=0.15)(conv)
return conv
def create_model(self):
'''
Create new keras model.
'''
self.class_weight = {0: 0.25, 1: 0.75}
# Entity branch
entity_inputs = Input(shape=(self.data[0].shape[1],))
entity_x = Dense(self.data[0].shape[1], activation='relu',
kernel_constraint=maxnorm(3))(entity_inputs)
entity_x = Dropout(0.25)(entity_x)
# entity_x = Dense(50, activation='relu',
# self.kernel_constraint=maxnorm(3))(entity_x)
# entity_x = Dropout(0.25)(entity_x)
# Candidate branch
candidate_inputs = Input(shape=(self.data[1].shape[1],))
candidate_x = Dense(self.data[1].shape[1], activation='relu',
kernel_constraint=maxnorm(3))(candidate_inputs)
candidate_x = Dropout(0.25)(candidate_x)
# candidate_x = Dense(50, activation='relu',
# kernel_constraint=maxnorm(3))(candidate_x)
# candidate_x = Dropout(0.25)(candidate_x)
# Cosine proximity
# cos_x = dot([entity_x, candidate_x], axes=1, normalize=False)
# cos_x = concatenate([entity_x, candidate_x])
# cos_output = Dense(1, activation='sigmoid')(cos_x)
# Match branch
match_inputs = Input(shape=(self.data[2].shape[1],))
match_x = Dense(self.data[1].shape[1], activation='relu',
kernel_constraint=maxnorm(3))(match_inputs)
match_x = Dropout(0.25)(match_x)
# Merge
x = concatenate([entity_x, candidate_x, match_x])
x = Dense(32, activation='relu', kernel_constraint=maxnorm(3))(x)
x = Dropout(0.25)(x)
x = Dense(16, activation='relu', kernel_constraint=maxnorm(3))(x)
x = Dropout(0.25)(x)
x = Dense(8, activation='relu', kernel_constraint=maxnorm(3))(x)
x = Dropout(0.25)(x)
predictions = Dense(1, activation='sigmoid')(x)
model = Model(inputs=[entity_inputs, candidate_inputs, match_inputs],
outputs=predictions)
model.compile(optimizer='RMSprop', loss='binary_crossentropy',
metrics=['accuracy'])
return model
def build_model(train_features,
neurons = 100,
layers = 1,
verbose = 0,
lmbda = 1e-5,
learn_rate = 0.01,
b_maxnorm = 0,
batch_norm = 0,
decay = 0,
momentum = 0.5,
dropout_P=0.5,
activation = "relu"):
#Start from empty sequential model where we can stack layers
model = Sequential()
#Add one+ fully-connected layers
for l in range(layers):
if l ==0:
model.add(Dense(output_dim=neurons, input_dim=train_features.shape[1],W_regularizer=l1(lmbda),b_constraint=maxnorm(m=b_maxnorm)))
else:
model.add(Dense(b_constraint=maxnorm(m=b_norm),output_dim=neurons, input_dim=neurons,W_regularizer=l1(lmbda)))
# Add activation function to each neuron
model.add(Activation(activation)) #rectified linear activation
#batch normalization: maintain mean activation ~ 0 and activation standard deviation ~ 1.
if batch_norm:
model.add(BatchNormalization())
#dropout
model.add(Dropout(dropout_P))
#Add fully-connected layer with 2 neurons, one for each class of labels
model.add(Dense(output_dim=2))
#Add softmax layer to force the 2 outputs to sum up to one so that we have a probability representation over the labels.
model.add(Activation("softmax"))
#Compile model with categorical_crossentrotry as loss, and stochastic gradient descent(learning rate=0.001, momentum=0.5 as optimizer)
model.compile(loss='categorical_crossentropy', optimizer=SGD(lr= learn_rate, momentum=momentum,decay = decay, nesterov=True), metrics=['accuracy'])
return model
def sk3(num_class,num_bit,img_rows,img_cols,INP_SHAPE,dim_org):
print INP_SHAPE
kernel_size=(3,3)
print 'sk3 with num_class :',num_class
model = Sequential()
model.add(Conv2D(16, kernel_size, input_shape=INP_SHAPE, padding='same',
activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3)))
model.add(Conv2D(16, kernel_size, padding='same',
activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3)))
# model.add(AveragePooling2D(pool_size=(2, 2),data_format=dim_org))
model.add(Dropout(0.1))
model.add(Conv2D(32, kernel_size, padding='same',
activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3)))
model.add(AveragePooling2D(pool_size=(2, 2),data_format=dim_org))
model.add(Dropout(0.2))
model.add(Conv2D(64, kernel_size, padding='same',
activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3)))
model.add(AveragePooling2D(pool_size=(2, 2),data_format=dim_org))
# model.add(Dropout(0.2))
# model.add(Conv2D(128, kernel_size, padding='same',
# activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3)))
# model.add(MaxPooling2D(pool_size=(2, 2),data_format=dim_org))
model.add(Dropout(0.3))
model.add(Conv2D(128, kernel_size, padding='same',
activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3)))
model.add(UpSampling2D(size=(2, 2),data_format=dim_org))
model.add(Dropout(0.3))
model.add(Conv2DTranspose(64, kernel_size, padding='same',
activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3)))
model.add(UpSampling2D(size=(2, 2),data_format=dim_org))
model.add(Dropout(0.2))
model.add(Conv2DTranspose(32, kernel_size, padding='same',
activation='relu',data_format=dim_org, kernel_constraint=maxnorm(3)))
# model.add(UpSampling2D(size=(2, 2),data_format=dim_org))
model.add(Dropout(0.1))
model.add(Conv2DTranspose(num_class, kernel_size, activation='softmax',
data_format=dim_org,padding='same'))
return model
def build_residual_blocks(graph, config, prev_layer):
# Add sequence of residual blocks.
for i in range(config.n_residual_blocks):
# Add a fixed number of layers per residual block.
block_name = '%02d' % i
graph.add_node(Identity(), name=block_name+'input', input=prev_layer)
prev_layer = block_input_layer = block_name+'input'
try:
n_layers_per_residual_block = config.n_layers_per_residual_block
except AttributeError:
n_layers_per_residual_block = 2
for layer_num in range(n_layers_per_residual_block):
layer_name = 'h%s%02d' % (block_name, layer_num)
l = Dense(config.n_hidden_residual, init=config.residual_init,
W_constraint=maxnorm(config.residual_max_norm))
graph.add_node(l, name=layer_name, input=prev_layer)
prev_layer = layer_name
if config.batch_normalization:
graph.add_node(BatchNormalization(), name=layer_name+'bn', input=prev_layer)
prev_layer = layer_name+'bn'
if i < n_layers_per_residual_block:
graph.add_node(Activation('relu'), name=layer_name+'relu', input=prev_layer)
prev_layer = layer_name+'relu'
if config.dropout_fc_p > 0.:
graph.add_node(Dropout(config.dropout_fc_p), name=layer_name+'do', input=prev_layer)
prev_layer = layer_name+'do'
graph.add_node(Identity(), name=block_name+'output', inputs=[block_input_layer, prev_layer], merge_mode='sum')
graph.add_node(Activation('relu'), name=block_name+'relu', input=block_name+'output')
prev_layer = block_input_layer = block_name+'relu'
return prev_layer
def __init__(self, vocab_size, nb_labels, emb_dim, maxlen,
embedding_weights, filter_hs, nb_filters, dropout_p,
trainable_embeddings, pretrained_embeddings):
self.nb_labels = nb_labels
if pretrained_embeddings is False:
embedding_weights = None
else:
embedding_weights = [embedding_weights]
sentence_input = Input(shape=(maxlen,), dtype='int32')
x = Embedding(input_dim=vocab_size+1, output_dim=emb_dim,
input_length=maxlen, dropout=dropout_p[0],
weights=embedding_weights,
trainable=trainable_embeddings)
x = x(sentence_input)
conv_pools = []
for filter_h in filter_hs:
conv = Convolution1D(nb_filters, filter_h, border_mode='same',
W_constraint=maxnorm(2))
conved = conv(x)
batchnorm = BatchNormalization()(conved)
#conved_relu = Activation('relu')(batchnorm)
conved_relu = PReLU()(batchnorm)
pool = Lambda(max_1d, output_shape=(nb_filters,))
pooled = pool(conved_relu)
conv_pools.append(pooled)
if len(conv_pools) == 1:
merged = conv_pools[0]
else:
merged = merge(conv_pools, mode='concat')
dropout = Dropout(dropout_p[1])(merged)
if nb_labels == 2:
out = Dense(1, activation='sigmoid')
out = out(dropout)
else:
out = Dense(nb_labels, activation='softmax')
out = out(dropout)
self.model = Model(input=sentence_input, output=out)
#model
model = Sequential()
model.add(Embedding(maxDictionary_size, 32,
input_length=maxWordCount)) #to change words to ints
# model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))
# model.add(MaxPooling1D(pool_size=2))
# model.add(Dropout(0.5))
# model.add(Conv1D(filters=32, kernel_size=2, padding='same', activation='relu'))
# model.add(MaxPooling1D(pool_size=2))
#hidden layers
model.add(LSTM(10))
# model.add(Flatten())
model.add(Dropout(0.6))
model.add(Dense(1200, activation='relu', W_constraint=maxnorm(1)))
# model.add(Dropout(0.6))
model.add(Dense(500, activation='relu', W_constraint=maxnorm(1)))
# model.add(Dropout(0.5))
#output layer
model.add(Dense(5, activation='softmax'))
# Compile model
# adam=Adam(lr=learning_rate, beta_1=0.7, beta_2=0.999, epsilon=1e-08, decay=0.0000001)
model.summary()
learning_rate = 0.0001
epochs = 60
batch_size = 32 #32
# #optimizer
# # sgd = optimizers.SGD(lr=0.01,momentum=True)
# # compile model
# model.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])
# return model
left_branch = Sequential()
left_branch.add(
Convolution2D(nb_filter,
nb_row,
nb_col,
border_mode='same',
input_shape=(np.array(dataX).shape[1],
np.array(dataX).shape[2],
np.array(dataX).shape[3]),
W_constraint=maxnorm(1),
init='normal'))
left_branch.add(
MaxPooling2D(pool_size=(pool_size, pool_size), border_mode='same'))
left_branch.add(Activation('relu'))
left_branch.add(Flatten())
right_branch = Sequential()
right_branch.add(Dense(32, input_dim=n_frames * n_hours))
merged = Merge([left_branch, right_branch], mode='concat')
model = Sequential()
model.add(merged)
model.add(Dense(np.array(dataY).shape[1]))
from keras.constraints import maxnorm
from keras.optimizers import SGD
def createSupervisedEncoder(X1,X2,Y,
learning_rate=0.1,
ENCODING_DIM = 5
other_parameters = []):
# Remark : input_shape can be (x,None) to allow input to have a variable dimension in the None part
branch1 = Sequential()
branch1.add(Dense(X1.shape[1], input_shape = (X1.shape[1],), init = 'normal', activation = 'relu'))
branch2 = Sequential()
branch2.add(Dense(X2.shape[1], input_shape = (X2.shape[1],), init = 'normal', activation = 'relu'))
branch2.add(Dense(X2.shape[1], init = 'normal', activation = 'relu', W_constraint = maxnorm(5)))
model = Sequential()
model.add(Merge([branch1, branch2], mode = 'concat'))
model.add(Dense(ENCODING_DIM, init = 'normal', activation = 'sigmoid'))
# Use a concatenate layer instead ?
# TODO Remark : you can also write layers as : LayerType(layer_parms)(previous_layer_object)
model.compile(loss = 'binary_crossentropy',
optimizer = SGD(lr = learning_rate),
metrics = ['accuracy'])
seed(42)
model.fit([X1, X2], Y.values,
batch_size = 2000, nb_epoch = 100, verbose = 1)
X_train = X_train / 255.0
X_test = X_test / 255.0
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
print('Classes - ', num_classes)
model = Sequential()
model.add(
Conv2D(32, (3, 3),
input_shape=(3, 32, 32),
padding='same',
activation='relu',
kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(
Conv2D(32, (3, 3),
activation='relu',
padding='same',
kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(512, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
epochs = 25
lrate = 0.01
decay = lrate / epochs
def catdog():
# 학습 관련 파라메타들
sizeOffm = 4
sizeOfpm = 3
my_maxnorm = 2.
model = Sequential()
# model.add(Convolution2D(16, 5, 5, border_mode='same', input_shape=(3, ROWS, COLS), kernel_constraint=maxnorm(2), activation='relu'))
model.add(
Convolution2D(16,
sizeOffm,
sizeOffm,
border_mode='same',
input_shape=(3, ROWS, COLS),
kernel_constraint=maxnorm(my_maxnorm)))
model.add(BatchNormalization())
#model.add(Activation('relu'))
model.add(Activation(LeakyReLU(alpha=0.1)))
model.add(
Convolution2D(16,
sizeOffm,
sizeOffm,
border_mode='same',
input_shape=(3, ROWS, COLS),
kernel_constraint=maxnorm(my_maxnorm)))
model.add(BatchNormalization())
#model.add(Activation('relu'))
model.add(Activation(LeakyReLU(alpha=0.1)))
model.add(
MaxPooling2D(pool_size=(sizeOfpm, sizeOfpm), dim_ordering="th"))
model.add(
Convolution2D(32,
sizeOffm,
sizeOffm,
border_mode='same',
input_shape=(3, ROWS, COLS),
kernel_constraint=maxnorm(my_maxnorm)))
model.add(BatchNormalization())
#model.add(Activation('relu'))
model.add(Activation(LeakyReLU(alpha=0.1)))
model.add(
Convolution2D(32,
sizeOffm,
sizeOffm,
border_mode='same',
input_shape=(3, ROWS, COLS),
kernel_constraint=maxnorm(my_maxnorm)))
model.add(BatchNormalization())
#model.add(Activation('relu'))
model.add(Activation(LeakyReLU(alpha=0.1)))
model.add(
MaxPooling2D(pool_size=(sizeOfpm, sizeOfpm), dim_ordering="th"))
model.add(
Convolution2D(64,
sizeOffm,
sizeOffm,
border_mode='same',
input_shape=(3, ROWS, COLS),
kernel_constraint=maxnorm(my_maxnorm)))
model.add(BatchNormalization())
#model.add(Activation('relu'))
model.add(Activation(LeakyReLU(alpha=0.1)))
model.add(
Convolution2D(64,
sizeOffm,
sizeOffm,
border_mode='same',
input_shape=(3, ROWS, COLS),
kernel_constraint=maxnorm(my_maxnorm)))
model.add(BatchNormalization())
#model.add(Activation('relu'))
model.add(Activation(LeakyReLU(alpha=0.1)))
model.add(
MaxPooling2D(pool_size=(sizeOfpm, sizeOfpm), dim_ordering="th"))
model.add(
Convolution2D(128,
sizeOffm,
sizeOffm,
border_mode='same',
input_shape=(3, ROWS, COLS),
kernel_constraint=maxnorm(my_maxnorm)))
model.add(BatchNormalization())
#model.add(Activation('relu'))
model.add(Activation(LeakyReLU(alpha=0.2)))
model.add(
Convolution2D(128,
sizeOffm,
sizeOffm,
border_mode='same',
input_shape=(3, ROWS, COLS),
kernel_constraint=maxnorm(my_maxnorm)))
model.add(BatchNormalization())
#model.add(Activation('relu'))
model.add(Activation(LeakyReLU(alpha=0.2)))
model.add(
MaxPooling2D(pool_size=(sizeOfpm, sizeOfpm), dim_ordering="th"))
model.add(Flatten())
# model.add(Dense(256, kernel_constraint=maxnorm(my_maxnorm), activation='relu'))
model.add(
MaxoutDense(output_dim=128, nb_feature=8, init='glorot_uniform'))
#model.add(Dropout(0.3))
# model.add(Dense(256, kernel_constraint=maxnorm(my_maxnorm), activation='relu'))
model.add(
MaxoutDense(output_dim=128, nb_feature=8, init='glorot_uniform'))
#model.add(Dropout(0.3))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# model.compile(loss=objective, optimizer=optimizer, metrics=['accuracy'])
model.compile(loss=objective,
optimizer=optimizer,
metrics=['accuracy'])
return model
# Create the model model = Sequential() model.add(Conv2D(32, (3, 3), activation='linear', padding='same',input_shape=train_data.shape[1:])) model.add(PReLU()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(BatchNormalization()) model.add(Conv2D(48,(3, 3), activation='linear', padding='same')) model.add(PReLU()) model.add(MaxPooling2D(pool_size=(2, 2))) model.add(BatchNormalization()) model.add(Conv2D(64, (3, 3), activation='linear', padding='same')) model.add(PReLU()) model.add(Dropout(0.1)) model.add(Flatten()) model.add(BatchNormalization()) model.add(Dense(1024, activation='linear', kernel_constraint=maxnorm(3))) model.add(PReLU()) model.add(Dropout(0.2)) model.add(Dense(84, activation='linear', kernel_constraint=maxnorm(3))) model.add(PReLU()) model.add(Dense(num_classes, activation='softmax')) model.compile(loss='categorical_crossentropy',optimizer=Adam(lr=lr, decay = lr*0.9),metrics=['accuracy']) #Training the Model datagen = ImageDataGenerator(zoom_range=0.2,horizontal_flip=True) model.fit_generator(datagen.flow(train_data, train_labels, batch_size=128),steps_per_epoch=len(train_data) /128, epochs=epochs,verbose =0) #Testing the Model #Reading test data test_file = unpickle(test) test_data = test_file["data"] test_data = StandardScaler().fit_transform(test_data.astype(float)) test_data = test_data.reshape(test_data.shape[0], 3, 32, 32)
print(X_train[0])
print (len(y_train))
print (len(X_train))
print("Pad sequences (samples x time)")
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
model = Sequential()
model.add(Embedding(max_features, 200, input_length=maxlen))
model.add(LSTM(300))
model.add(Dense(210,init='he_normal',W_constraint = maxnorm(2)))
model.add(Activation('sigmoid'))
model.add(Dense(205,init='he_normal',W_constraint = maxnorm(2)))
model.add(Activation('tanh'))
model.add(Dense(201,init='he_normal',W_constraint = maxnorm(2)))
model.compile(loss=hinge2, optimizer='adam')
print("Train...")
model.fit(X_train, y_train, batch_size=batch_size, nb_epoch=25, validation_data=(X_test, y_test))
json_string = model.to_json()
open('my_model_struct.json', 'w').write(json_string)
print('model structure saved')
model.save_weights('my_model_weights.h5')
print('model weights saved')
test, test_xy, test_x, test_y, test_set_x, test_set_y = read_dataset(
test_dataset, test_data_args)
# Reading dev dataset
dev_dataset, dev_data_args = read_data_args(dev_data_file)
dev, dev_xy, dev_x, dev_y, dev_set_x, dev_set_y = read_dataset(
dev_dataset, dev_data_args)
train_set_y = train_set_y.astype(numpy.int64)
valid_set_y = valid_set_y.astype(numpy.int64)
test_set_y = test_set_y.astype(numpy.int64)
dev_set_y = dev_set_y.astype(numpy.int64)
model = Sequential()
model.add(Dropout(0.2, input_shape=(440, )))
model.add(Dense(1024, init='uniform', W_constraint=maxnorm(4)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1024, init='uniform', W_constraint=maxnorm(4)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1024, init='uniform', W_constraint=maxnorm(4)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1024, init='uniform', W_constraint=maxnorm(4)))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(1940, init='uniform'))
model.add(Activation('softmax'))
model.load_weights('weights3.h5')
def DLS2F_construct_withaa_complex_win_filter_layer_opt(win_array,ktop_node,output_dim,use_bias,hidden_type,nb_filters,nb_layers,opt,hidden_num):
ss_feature_num = 3
aa_feature_num = 20
ktop_node= ktop_node
win_array = list(win_array)
print("Setting hidden models as ",hidden_type)
print("Setting nb_filters as ",nb_filters)
print("Setting nb_layers as ",nb_layers)
print("Setting opt as ",opt)
print("Setting win_array as ",win_array)
print("Setting use_bias as ",use_bias)
########################################## set up model
DLS2F_input_shape =(None,aa_feature_num+ss_feature_num)
filter_sizes=win_array
DLS2F_input = Input(shape=DLS2F_input_shape)
DLS2F_convs = []
for fsz in filter_sizes:
DLS2F_conv = DLS2F_input
for i in range(0,nb_layers):
DLS2F_conv = _conv_bn_relu1D(nb_filter=nb_filters, nb_row=fsz, subsample=1,use_bias=use_bias)(DLS2F_conv)
DLS2F_pool = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv)
DLS2F_flatten = Flatten()(DLS2F_pool)
DLS2F_convs.append(DLS2F_flatten)
if len(filter_sizes)>1:
DLS2F_out = Merge(mode='concat')(DLS2F_convs)
else:
DLS2F_out = DLS2F_convs[0]
DLS2F_dense1 = Dense(output_dim=hidden_num, init='he_normal', activation=hidden_type, W_constraint=maxnorm(3))(DLS2F_out) # changed on 20170314 to check if can visualzie better
DLS2F_dropout1 = Dropout(0.2)(DLS2F_dense1)
DLS2F_output = Dense(output_dim=output_dim, init="he_normal", activation="softmax")(DLS2F_dropout1)
DLS2F_ResCNN = Model(input=[DLS2F_input], output=DLS2F_output)
DLS2F_ResCNN.compile(loss="categorical_crossentropy", metrics=['accuracy'], optimizer=opt)
return DLS2F_ResCNN
def run(model_name, dataset_name):
"""
run the baseline model
:param model_name: name of baseline models, CNN or LSTM
:param dataset_name: name of datasets, candidate values [bbc, digg, MySpace, rw, Twitter, YouTube]
"""
print "model: %s" % model_name
print "process dataset %s..." % dataset_name
data = cPickle.load(open('./pkl/%s.pkl' % dataset_name, 'rb'))
records, glove_embeddings, vocab, word_to_df = data
dim_emb = len(glove_embeddings[1])
# index of word starts from 1
# initial weights of embedding layer
embeddings = np.zeros((len(vocab) + 1, dim_emb), dtype='float32')
for w in vocab:
wid = vocab[w]
embeddings[wid, :] = glove_embeddings[wid]
train_x, train_y, val_x, val_y, test_x, test_y, test_strength = [], [], [], [], [], [], []
max_len = 0
for r in records:
text = r['text']
y = r['label']
wids = [vocab[w] for w in text.split(' ')]
if len(wids) > max_len:
max_len = len(wids)
if r['type'] == 'train':
train_x.append(wids)
train_y.append(y)
elif r['type'] == 'val':
val_x.append(wids)
val_y.append(y)
elif r['type'] == 'test':
strength = r['strength']
test_x.append(wids)
test_y.append(y)
test_strength.append(strength)
train_x, val_x, test_x = ToArray(train=train_x, val=val_x, test=test_x)
train_y, val_y, test_y = ToArray(train=train_y, val=val_y, test=test_y)
_, _, test_strength = ToArray(train=[], val=[], test=test_strength)
#print train_x.shape, val_x.shape, test_x.shape
train_x, val_x, test_x = Padding(train=train_x,
val=val_x,
test=test_x,
max_len=max_len)
batch_size = 50 if model_name == 'CNN' else 32
if train_x.shape[0] % batch_size:
n_extra = batch_size - train_x.shape[0] % batch_size
x_extra = train_x[:n_extra, :]
y_extra = train_y[:n_extra]
train_x = np.append(train_x, x_extra, axis=0)
train_y = np.append(train_y, y_extra, axis=0)
np.random.seed(38438)
# shuffle the training set
train_set = np.random.permutation(zip(train_x, train_y))
train_x, train_y = [], []
for (x, y) in train_set:
train_x.append(x)
train_y.append(y)
n_labels = 2
train_x = np.array(train_x)
train_y = np.array(train_y)
train_y = to_categorical(train_y)
val_y = to_categorical(val_y)
print "n_train: %s, n_val: %s, n_test: %s" % (
train_x.shape[0], val_x.shape[0], test_x.shape[0])
if model_name == 'CNN':
model = Graph()
model.add_input(name='input', input_shape=(max_len, ), dtype='int')
model.add_node(Embedding(input_dim=len(vocab) + 1,
output_dim=dim_emb,
input_length=max_len,
weights=[embeddings]),
name="emb",
input="input")
filter_hs = [3, 4, 5]
n_filter = 100
dropout_rate = 0.5
n_epoch = 20
for i in xrange(len(filter_hs)):
win_size = filter_hs[i]
conv_name = 'conv%s' % i
pool_name = "pool%s" % i
flatten_name = "flatten%s" % i
pool_size = max_len - win_size + 1
model.add_node(
layer=Convolution1D(nb_filter=n_filter,
filter_length=win_size,
activation='relu',
W_constraint=maxnorm(m=3),
b_constraint=maxnorm(m=3)),
name=conv_name,
input='emb',
)
model.add_node(layer=MaxPooling1D(pool_length=pool_size),
name=pool_name,
input=conv_name)
model.add_node(layer=Flatten(), name=flatten_name, input=pool_name)
model.add_node(layer=Dropout(p=dropout_rate),
name="dropout",
inputs=["flatten0", "flatten1", "flatten2"])
model.add_node(layer=Dense(output_dim=n_labels, activation='softmax'),
name='softmax',
input='dropout')
model.add_output(input='softmax', name="output")
model.compile(loss={'output': 'categorical_crossentropy'},
optimizer='adadelta',
metrics=['accuracy'])
model_path = './model/%s_%s.hdf5' % (model_name, dataset_name)
best_model = ModelCheckpoint(filepath=model_path,
monitor='val_acc',
save_best_only=True,
mode='max')
print "training..."
model.fit(data={
'input': train_x,
'output': train_y
},
batch_size=batch_size,
nb_epoch=n_epoch,
validation_data={
'input': val_x,
'output': val_y
},
callbacks=[best_model],
verbose=0)
else:
model = Sequential()
model.add(
Embedding(input_dim=len(vocab) + 1,
output_dim=dim_emb,
mask_zero=True,
input_length=max_len,
weights=[embeddings]))
model.add(LSTM(output_dim=128, dropout_W=0.2, dropout_U=0.2))
model.add(Dense(n_labels, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model_path = './model/%s_%s.hdf5' % (model_name, dataset_name)
best_model = ModelCheckpoint(filepath=model_path,
monitor='val_acc',
save_best_only=True,
mode='max')
n_epoch = 20
print "training..."
model.fit(x=train_x,
y=train_y,
batch_size=batch_size,
nb_epoch=n_epoch,
validation_data=(val_x, val_y),
callbacks=[best_model],
verbose=0)
pred_strength = []
print "load the best model from disk..."
model.load_weights(model_path)
if model_name == 'LSTM':
pred_strength = model.predict(x=test_x, batch_size=batch_size)
else:
for i in xrange(len(test_x)):
res = model.predict(data={'input': test_x[i:i + 1]}, batch_size=1)
pred_strength.append(res['output'])
pred_strength = np.array(pred_strength)
pred_strength = pred_strength.reshape(
(pred_strength.shape[0], pred_strength.shape[2]))
assert pred_strength.shape == test_strength.shape
print "evaluate performance of the system..."
accu, mae, rmse = evaluate(strength_gold=test_strength,
strength_pred=pred_strength)
print "%s over %s--->accuracy: %s, mae: %s, rmse: %s\n\n" % (
model_name, dataset_name, accu, mae, rmse)
pred_strengths_lines = []
for strength in pred_strength:
pred_strengths_lines.append('%s\n' %
' '.join([str(ele) for ele in strength]))
model.add(Conv2D(64, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Conv2D(128, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Flatten())
model.add(Dropout(0.2))
model.add(Dense(256, kernel_constraint=maxnorm(3)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(128, kernel_constraint=maxnorm(3)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(BatchNormalization())
model.add(Dense(class_num))
model.add(Activation('softmax'))
epochs = 25
optimizer = 'adam'
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
# model.add(BatchNormalization())
model.add(Conv2D(128, (3, 3), padding='same'))
model.add(Activation('relu'))
model.add(Dropout(0.2))
# model.add(BatchNormalization())
model.add(Flatten())
model.add(Dropout(0.2))
model.add(
Dense(256,
kernel_regularizer=regularizers.l2(0.0001),
kernel_constraint=maxnorm(3)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
# model.add(BatchNormalization())
model.add(
Dense(128,
kernel_regularizer=regularizers.l2(0.0001),
kernel_constraint=maxnorm(3)))
model.add(Activation('relu'))
model.add(Dropout(0.2))
# model.add(BatchNormalization())
model.add(Dense(num_classes))
model.add(Activation('softmax'))
epochs = 76
optimizer = 'Adam'
def DLS2F_construct_withaa_complex_win_filter_layer_opt(
win_array, ktop_node, output_dim, use_bias, hidden_type, nb_filters,
nb_layers, opt, hidden_num):
ss_feature_num = 3
sa_feature_num = 2
aa_feature_num = 20
pssm_feature_num = 20
ktop_node = ktop_node
print "Setting hidden models as ", hidden_type
print "Setting nb_filters as ", nb_filters
print "Setting nb_layers as ", nb_layers
print "Setting opt as ", opt
print "Setting win_array as ", win_array
print "Setting use_bias as ", use_bias
########################################## set up model
if (two_stream):
# TWO STREAM
DLS2F_input_shape_aa = (None, aa_feature_num)
DLS2F_input_shape_rest = (None, ss_feature_num + sa_feature_num +
pssm_feature_num)
filter_sizes = win_array
DLS2F_input_aa = Input(shape=DLS2F_input_shape_aa, name='input_aa')
DLS2F_input_rest = Input(shape=DLS2F_input_shape_rest,
name='input_rest')
DLS2F_convs_aa = []
DLS2F_convs_rest = []
for fsz in filter_sizes:
DLS2F_conv_aa = DLS2F_input_aa
for i in range(0, nb_layers):
DLS2F_conv_aa = _conv_bn_relu1D(
nb_filter=nb_filters,
nb_row=fsz,
subsample=1,
use_bias=use_bias)(DLS2F_conv_aa)
if extra_fusion_CONV:
DLS2F_pool_aa = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv_aa)
DLS2F_convs_aa.append(DLS2F_pool_aa)
else:
DLS2F_pool_aa = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv_aa)
DLS2F_flatten_aa = Flatten()(DLS2F_pool_aa)
DLS2F_convs_aa.append(DLS2F_flatten_aa)
# TWO STREAM
for fsz in filter_sizes:
DLS2F_conv_rest = DLS2F_input_rest
for i in range(0, nb_layers):
DLS2F_conv_rest = _conv_bn_relu1D(
nb_filter=nb_filters,
nb_row=fsz,
subsample=1,
use_bias=use_bias)(DLS2F_conv_rest)
if extra_fusion_CONV:
DLS2F_pool_rest = K_max_pooling1d(
ktop=ktop_node)(DLS2F_conv_rest)
DLS2F_convs_rest.append(DLS2F_pool_rest)
else:
DLS2F_pool_rest = K_max_pooling1d(
ktop=ktop_node)(DLS2F_conv_rest)
DLS2F_flatten_rest = Flatten()(DLS2F_pool_rest)
DLS2F_convs_rest.append(DLS2F_flatten_rest)
if extra_fusion_FC:
# TWO STREAM
if len(filter_sizes) > 1:
DLS2F_out_aa = Merge(mode='concat')(DLS2F_convs_aa)
DLS2F_out_rest = Merge(mode='concat')(DLS2F_convs_rest)
DLS2F_dense_aa_two_stream = Dense(
output_dim=hidden_num,
init='he_normal',
activation=hidden_type,
W_constraint=maxnorm(3))(
DLS2F_out_aa
) # changed on 20170314 to check if can visualzie better
DLS2F_dense_rest_two_stream = Dense(
output_dim=hidden_num,
init='he_normal',
activation=hidden_type,
W_constraint=maxnorm(3))(
DLS2F_out_rest
) # changed on 20170314 to check if can visualzie better
DLS2F_dropout_aa_two_stream = Dropout(0.2)(
DLS2F_dense_aa_two_stream)
DLS2F_dropout_rest_two_stream = Dropout(0.2)(
DLS2F_dense_rest_two_stream)
DLS2F_out = Merge(mode='concat')(DLS2F_dropout_aa_two_stream +
DLS2F_dropout_rest_two_stream)
else:
DLS2F_out = DLS2F_convs[0]
else:
if len(filter_sizes) > 1:
DLS2F_convs = DLS2F_convs_aa + DLS2F_convs_rest
DLS2F_out = Merge(mode='concat')(DLS2F_convs)
else:
DLS2F_out = DLS2F_convs[0]
if extra_fusion_CONV:
DLS2F_conv_extra = DLS2F_out
for i in range(0, nb_layers):
DLS2F_conv_extra = _conv_bn_relu1D(
nb_filter=nb_filters,
nb_row=6,
subsample=1,
use_bias=use_bias)(DLS2F_conv_extra)
DLS2F_pool_extra = K_max_pooling1d(
ktop=ktop_node)(DLS2F_conv_extra)
DLS2F_flatten_extra = Flatten()(DLS2F_pool_extra)
DLS2F_dense1_two_stream = Dense(
output_dim=hidden_num,
init='he_normal',
activation=hidden_type,
W_constraint=maxnorm(3))(
DLS2F_flatten_extra
) # changed on 20170314 to check if can visualzie better
else:
DLS2F_dense1_two_stream = Dense(
output_dim=hidden_num,
init='he_normal',
activation=hidden_type,
W_constraint=maxnorm(3))(
DLS2F_out
) # changed on 20170314 to check if can visualzie better
# TWO STREAM
DLS2F_dropout1_two_stream = Dropout(0.2)(DLS2F_dense1_two_stream)
DLS2F_output_two_stream = Dense(
output_dim=output_dim, init="he_normal",
activation="softmax")(DLS2F_dropout1_two_stream)
DLS2F_ResCNN_two_stream = Model(
input=[DLS2F_input_aa, DLS2F_input_rest],
output=DLS2F_output_two_stream)
DLS2F_ResCNN_two_stream.compile(loss="categorical_crossentropy",
metrics=['accuracy'],
optimizer=opt)
# TWO STREAM
DLS2F_ResCNN_two_stream.summary()
return DLS2F_ResCNN_two_stream
else:
# DEFAULT DO NOT EDIT
DLS2F_input_shape = (None, aa_feature_num + ss_feature_num +
sa_feature_num + pssm_feature_num)
filter_sizes = win_array
DLS2F_input = Input(shape=DLS2F_input_shape)
DLS2F_convs = []
if pyramid_window_size:
for fsz in filter_sizes:
DLS2F_conv = DLS2F_input
for i in range(0, nb_layers):
if pyramid_nb_filters:
DLS2F_conv = _conv_bn_relu1D(
nb_filter=nb_filters - i,
nb_row=fsz - i,
subsample=1,
use_bias=use_bias)(DLS2F_conv)
else:
DLS2F_conv = _conv_bn_relu1D(
nb_filter=nb_filters,
nb_row=fsz if i == 0 else fsz / 2,
subsample=1,
use_bias=use_bias)(DLS2F_conv)
DLS2F_pool = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv)
DLS2F_flatten = Flatten()(DLS2F_pool)
DLS2F_convs.append(DLS2F_flatten)
else:
for fsz in filter_sizes:
DLS2F_conv = DLS2F_input
for i in range(0, nb_layers):
if pyramid_nb_filters:
DLS2F_conv = _conv_bn_relu1D(
nb_filter=nb_filters - i,
nb_row=fsz,
subsample=1,
use_bias=use_bias)(DLS2F_conv)
else:
DLS2F_conv = _conv_bn_relu1D(
nb_filter=nb_filters,
nb_row=fsz,
subsample=1,
use_bias=use_bias)(DLS2F_conv)
DLS2F_pool = K_max_pooling1d(ktop=ktop_node)(DLS2F_conv)
DLS2F_flatten = Flatten()(DLS2F_pool)
DLS2F_convs.append(DLS2F_flatten)
# DEFAULT DO NOT EDIT
if len(filter_sizes) > 1:
DLS2F_out = Merge(mode='concat')(DLS2F_convs)
else:
DLS2F_out = DLS2F_convs[0]
# DEFAULT DO NOT EDIT
DLS2F_dense1 = Dense(
output_dim=hidden_num,
init='he_normal',
activation=hidden_type,
W_constraint=maxnorm(3)
)(DLS2F_out) # changed on 20170314 to check if can visualzie better
DLS2F_dropout1 = Dropout(0.2)(DLS2F_dense1)
DLS2F_output = Dense(output_dim=output_dim,
init="he_normal",
activation="softmax")(DLS2F_dropout1)
DLS2F_ResCNN = Model(input=[DLS2F_input], output=DLS2F_output)
DLS2F_ResCNN.compile(loss="categorical_crossentropy",
metrics=['accuracy'],
optimizer=opt)
# DEFAULT DO NOT EDIT
DLS2F_ResCNN.summary()
return DLS2F_ResCNN
def model_define(
self
): # Defines the modified sequential class with regularizers defined.
model = Sequential()
l2_param = 0.0
l1_param = 0.0
dropout_param = 0.0
Gaussian_noise = 1
initializer = "uniform"
activation_func = "relu"
adam = Adam(lr=0.0005, beta_1=0.9, beta_2=0.999, epsilon=5e-09)
model.add(
Dense(2048,
input_dim=2048,
kernel_initializer=initializer,
activation=activation_func,
activity_regularizer=l1_l2(l1_param, l2_param),
kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_param))
model.add(BatchNormalization())
model.add(GaussianNoise(Gaussian_noise))
model.add(
Dense(2048,
kernel_initializer=initializer,
activation=activation_func,
activity_regularizer=l1_l2(l1_param, l2_param),
kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_param))
model.add(BatchNormalization())
model.add(GaussianNoise(Gaussian_noise))
model.add(
Dense(512,
kernel_initializer=initializer,
activation=activation_func,
activity_regularizer=l1_l2(l1_param, l2_param),
kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_param))
model.add(BatchNormalization())
model.add(GaussianNoise(Gaussian_noise))
model.add(
Dense(64,
kernel_initializer=initializer,
activation=activation_func,
activity_regularizer=l1_l2(l1_param, l2_param),
kernel_constraint=maxnorm(3)))
model.add(Dropout(dropout_param))
model.add(BatchNormalization())
model.add(Dense(3, activation="softmax", kernel_initializer="normal"))
model.compile(loss="categorical_crossentropy",
optimizer=adam,
metrics=["accuracy"])
return model
def design_dnn(nb_features,
input_shape,
nb_levels,
conv_size,
nb_labels,
feat_mult=1,
pool_size=2,
padding='same',
activation='elu',
final_layer='dense-sigmoid',
conv_dropout=0,
conv_maxnorm=0,
nb_input_features=1,
batch_norm=False,
name=None,
prefix=None,
use_strided_convolution_maxpool=True,
nb_conv_per_level=2):
"""
"deep" cnn with dense or global max pooling layer @ end...
Could use sequential...
"""
model_name = name
if model_name is None:
model_name = 'model_1'
if prefix is None:
prefix = model_name
ndims = len(input_shape)
input_shape = tuple(input_shape)
convL = getattr(KL, 'Conv%dD' % ndims)
maxpool = KL.MaxPooling3D if len(input_shape) == 3 else KL.MaxPooling2D
if isinstance(pool_size, int):
pool_size = (pool_size, ) * ndims
# kwargs for the convolution layer
conv_kwargs = {'padding': padding, 'activation': activation}
if conv_maxnorm > 0:
conv_kwargs['kernel_constraint'] = maxnorm(conv_maxnorm)
# initialize a dictionary
enc_tensors = {}
# first layer: input
name = '%s_input' % prefix
enc_tensors[name] = KL.Input(shape=input_shape + (nb_input_features, ),
name=name)
last_tensor = enc_tensors[name]
# down arm:
# add nb_levels of conv + ReLu + conv + ReLu. Pool after each of first nb_levels - 1 layers
for level in range(nb_levels):
for conv in range(nb_conv_per_level):
if conv_dropout > 0:
name = '%s_dropout_%d_%d' % (prefix, level, conv)
enc_tensors[name] = KL.Dropout(conv_dropout)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_conv_%d_%d' % (prefix, level, conv)
nb_lvl_feats = nb_features * (feat_mult**level)
enc_tensors[name] = convL(nb_lvl_feats,
conv_size,
**conv_kwargs,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
# max pool
if use_strided_convolution_maxpool:
name = '%s_strided_conv_%d' % (prefix, level)
enc_tensors[name] = convL(nb_lvl_feats,
pool_size,
**conv_kwargs,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
else:
name = '%s_maxpool_%d' % (prefix, level)
enc_tensors[name] = maxpool(pool_size=pool_size,
name=name,
padding=padding)(last_tensor)
last_tensor = enc_tensors[name]
# dense layer
if final_layer == 'dense-sigmoid':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(1, name=name,
activation="sigmoid")(last_tensor)
elif final_layer == 'dense-tanh':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(1, name=name)(last_tensor)
last_tensor = enc_tensors[name]
# Omittting BatchNorm for now, it seems to have a cpu vs gpu problem
# https://github.com/tensorflow/tensorflow/pull/8906
# https://github.com/fchollet/keras/issues/5802
# name = '%s_%s_bn' % prefix
# enc_tensors[name] = KL.BatchNormalization(axis=batch_norm, name=name)(last_tensor)
# last_tensor = enc_tensors[name]
name = '%s_%s_tanh' % prefix
enc_tensors[name] = KL.Activation(activation="tanh",
name=name)(last_tensor)
elif final_layer == 'dense-softmax':
name = "%s_flatten" % prefix
enc_tensors[name] = KL.Flatten(name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_dense' % prefix
enc_tensors[name] = KL.Dense(nb_labels,
name=name,
activation="softmax")(last_tensor)
# global max pooling layer
elif final_layer == 'myglobalmaxpooling':
name = '%s_batch_norm' % prefix
enc_tensors[name] = KL.BatchNormalization(axis=batch_norm,
name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool' % prefix
enc_tensors[name] = KL.Lambda(_global_max_nd, name=name)(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool_reshape' % prefix
enc_tensors[name] = KL.Reshape((1, 1), name=name)(last_tensor)
last_tensor = enc_tensors[name]
# cannot do activation in lambda layer. Could code inside, but will do extra lyaer
name = '%s_global_max_pool_sigmoid' % prefix
enc_tensors[name] = KL.Conv1D(1,
1,
name=name,
activation="sigmoid",
use_bias=True)(last_tensor)
elif final_layer == 'globalmaxpooling':
name = '%s_conv_to_featmaps' % prefix
enc_tensors[name] = KL.Conv3D(2, 1, name=name,
activation="relu")(last_tensor)
last_tensor = enc_tensors[name]
name = '%s_global_max_pool' % prefix
enc_tensors[name] = KL.GlobalMaxPooling3D(name=name)(last_tensor)
last_tensor = enc_tensors[name]
# cannot do activation in lambda layer. Could code inside, but will do extra lyaer
name = '%s_global_max_pool_softmax' % prefix
enc_tensors[name] = KL.Activation('softmax', name=name)(last_tensor)
last_tensor = enc_tensors[name]
# create the model
model = Model(inputs=[enc_tensors['%s_input' % prefix]],
outputs=[last_tensor],
name=model_name)
return model
img_data = np.expand_dims(img_data, axis=4)
Y = np_utils.to_categorical(labels, num_classes)
x, y = shuffle(img_data, Y, random_state=2)
X_train, X_test, y_train, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=2)
input_shape = img_data[0].shape
print(input_shape)
model.add(ZeroPadding2D((1, 1), input_shape=input_shape))
model.add(Conv2D(64, (3, 3), activation='relu', kernel_constraint=maxnorm(3)))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(64, (3, 3), activation='relu', kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(128, (3, 3), activation='relu', kernel_constraint=maxnorm(3)))
model.add(ZeroPadding2D((1, 1)))
model.add(Conv2D(128, (3, 3), activation='relu', kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Dropout(0.2))
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
# normalize inputs from 0-255 to 0.0-1.0
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train = X_train / 255.0
X_test = X_test / 255.0
# one hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
# Create the model
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(32, 32,3), padding='same', activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.3))
model.add(BatchNormalization())
model.add(Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3)))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3), input_shape=(32, 32,3), padding='same', activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.5))
model.add(BatchNormalization())
model.add(Conv2D(64, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3)))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Conv2D(128, (3, 3), input_shape=(32, 32,3), padding='valid', activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.7))
model.add(BatchNormalization())
model.add(Conv2D(256, (3, 3), activation='relu', padding='valid', kernel_constraint=maxnorm(3)))
model.add(MaxPool2D(pool_size=(2, 2)))
model.add(Conv2D(512, (2, 2), activation='relu'))
def create_model(neurons=1): # create model model = Sequential() model.add(Dense(neurons, input_dim=8, kernel_initializer='uniform', activation='linear', kernel_constraint=maxnorm(4))) model.add(Dropout(0.2)) model.add(Dense(1, kernel_initializer='uniform', activation='sigmoid')) # Compile model model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy']) return model
# ADD THE FIRST LAYER (CONVOLUTIONAL)
# Params:
# 1. filters: essentially size of image; number of inputs??
# 2. kernel_size: dimensions for kernel (typically 3x3)
# 3. input_shape: 3D dimensions of figure
# 4. activation: type of activation funciton
# 5. padding: 'same' ensures the image doesn't shrink in the convolutional layer
# 6. kernel_constraint: ensures that large numbers aren't being used; scales numbers down in kernel
model.add(
Conv2D(filters=32,
kernel_size=(3, 3),
input_shape=(32, 32, 3),
activation='reLu',
padding='same',
kernel_constraint=maxnorm(3)))
# ADD THE SECOND LAYER (MAX POOLING)
# Params:
# 1. pool_size: finds the mac value in each n x n section of input matrix
model.add(MaxPooling2D(pool_size=(2, 2)))
# BEFORE PUTTING FEATURES INTO DENSE LAYER NEED TO FLATTEN
model.add(Flatten)
# NOW WE CAN PUT THIS INTO A DENSE LAYER
# where the 'thinking' happens
# Params:
# 1. units: number of neurons; directly proportional to how long you want model to train
# 2. activation
# 3. kernel_constraint
def LSTM_network(train_path, test_path):
X_train, Y_train = load_TrainData(train_path)
X_test, X_test_PhraseID = load_TestData(test_path)
print(
'==============================The shape of Training data & Testing data =============================='
)
print("X_train shape is", X_train.shape)
print("Y_train shape is", Y_train.shape)
print("X_test shape is", X_test.shape)
print("X_test_PhraseID shape is", X_test_PhraseID.shape)
tokenizer = Tokenizer()
#To create token dictionary, every element is a document(use train and test data to encure the integrity of token dictionary)
tokenizer.fit_on_texts(np.concatenate((X_train, X_test), axis=0))
#calculate the word dictionary size exclude the same word
tokenizer_vocabulary_size = len(tokenizer.word_index) + 1
print("Vocabulary size", tokenizer_vocabulary_size)
# print(type(X_train))
# print("Word index",Tokenizer.word_index)
#split the data, use 20% of whole data as validation data
Split_number = 31212
Y_val = Y_train[:Split_number]
X_val = X_train[:Split_number]
X_train = X_train[Split_number:]
Y_train = Y_train[Split_number:]
#set the max word and max dictionary size for building LSTM network ( as embedding parameters)
maxWord = 60
Dictionary_size = tokenizer_vocabulary_size
#transform the document to vector shape, the shape type is [the number of document, the length per document]
encoded_X_train = tokenizer.texts_to_sequences(X_train)
encoded_X_val = tokenizer.texts_to_sequences(X_val)
encoded_X_test = tokenizer.texts_to_sequences(X_test)
#padding all text to same size
X_train_encoded = sequence.pad_sequences(encoded_X_train, maxlen=maxWord)
X_val_encoded = sequence.pad_sequences(encoded_X_val, maxlen=maxWord)
X_test_encoded = sequence.pad_sequences(encoded_X_test, maxlen=maxWord)
# One Hot Encoding
Y_train = keras.utils.to_categorical(Y_train, 5)
Y_val = keras.utils.to_categorical(Y_val, 5)
#shuffling the traing Set
shuffle_data(X_train_encoded, Y_train)
#Build the LSTM network
model = Sequential()
#change words to int type
model.add(Embedding(Dictionary_size, 32, input_length=maxWord))
# model.add(Conv1D(filters=32, kernel_size=3, padding='same', activation='relu'))
# model.add(MaxPooling1D(pool_size=2))
# model.add(Dropout(0.5))
# model.add(Conv1D(filters=32, kernel_size=2, padding='same', activation='relu'))
# model.add(MaxPooling1D(pool_size=2))
#hidden layers
model.add(LSTM(64))
# model.add(Flatten())
model.add(Dropout(0.5)) #Freeze some Neuron to avoid overestimate
model.add(Dense(1200, activation='relu', W_constraint=maxnorm(1)))
# model.add(Dropout(0.6))
model.add(Dense(500, activation='relu', W_constraint=maxnorm(1)))
# model.add(Dropout(0.5))
#output layer
model.add(Dense(5, activation='softmax'))
# Compile model
model.summary()
learning_rate = 0.001
epochs = 10
batch_size = 64 #32
sgd = SGD(lr=learning_rate, nesterov=True, momentum=0.7, decay=1e-4)
Nadam = keras.optimizers.Nadam(lr=0.003,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-08,
schedule_decay=0.004)
model.compile(loss='categorical_crossentropy',
optimizer=Nadam,
metrics=['accuracy'])
tensorboard = keras.callbacks.TensorBoard(
log_dir='./logs/log_1',
histogram_freq=0,
write_graph=True,
write_images=False) #Create the daily record
checkpointer = ModelCheckpoint(
filepath="./weights/weights_1.hdf5",
verbose=1,
save_best_only=True,
monitor="val_loss") #For saving the model between the epochs
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.8,
patience=0,
verbose=1,
mode='auto',
cooldown=0,
min_lr=1e-6) # control the learning rate
earlyStopping = EarlyStopping(
monitor='val_loss', min_delta=0, patience=6, verbose=1
) #when early stop is actived, after the number of patience epochs stop training
#Loading best weights
# model.load_weights("./weights/weights_19.hdf5")
print(
"=============================== Training ========================================="
)
# uncommit this to train
# tensorboard --logdir=./logs
history = model.fit(
X_train_encoded,
Y_train,
epochs=epochs,
batch_size=batch_size,
verbose=1,
validation_data=(X_val_encoded, Y_val),
callbacks=[tensorboard, reduce_lr, checkpointer, earlyStopping])
print(
"=============================== Score_Accuarcy ========================================="
)
scores = model.evaluate(X_val_encoded, Y_val, verbose=0)
print("Accuracy: %.2f%%" % (scores[1] * 100))
print(
"=============================== Predicting ========================================="
)
f = open('Submission_result.csv', 'w')
f.write('PhraseId,Sentiment\n')
# Get the predicted data
predicted = model.predict_classes(X_test_encoded,
batch_size=batch_size,
verbose=1)
for i in range(0, X_test_PhraseID.shape[0]):
f.write(str(X_test_PhraseID[i]) + "," + str(predicted[i]) + '\n')
f.close()
print(
"================================Done===============================")
# Count the epoch
epoch_count = range(1, len(history.history['loss']) + 1)
#visulize the training process
plt.plot(epoch_count, history.history['loss'], 'r--')
plt.plot(epoch_count, history.history['val_loss'], 'b-')
plt.plot(epoch_count, history.history['acc'], 'gray')
plt.plot(epoch_count, history.history['val_acc'], 'green')
plt.legend(
['Training Loss', 'Validation Loss', 'Training Acc', 'Validation Acc'])
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
overlap=True))
# Block 6
model.add(Conv2D(512, (3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.3))
model.add(Conv2D(512, (3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.3))
model.add(Conv2D(512, (3, 3), padding='same'))
model.add(LeakyReLU(alpha=0.3))
model.add(
FractionalPooling2D(pool_ratio=(1, 1.25, 1.25, 1),
pseudo_random=True,
overlap=True))
model.add(Reshape((16, 512)))
# fc layer_1
model.add(Dense(1024, kernel_constraint=maxnorm(3)))
model.add(LeakyReLU(alpha=0.3))
# fc_layer_2
model.add(Dense(512, kernel_constraint=maxnorm(3)))
model.add(LeakyReLU(alpha=0.3))
model.add(Flatten())
model.add(Dense(num_classes, activation='softmax'))
opt = keras.optimizers.Adadelta(1, decay=1e-4) #0.1
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
print(model.summary())
def build_convolutional_layer(config):
return Convolution1D(config.n_filters, config.filter_width,
W_constraint=maxnorm(config.filter_max_norm),
border_mode=config.border_mode,
W_regularizer=l2(config.l2_penalty))
# Create first network with Keras
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.constraints import maxnorm
import numpy as np
# load pima indians dataset
X = np.load('../Data/X.npy')
Y = X[:, 0]
X = np.array(X[:, 1::], dtype=np.int32)
X.shape
# create model
model = Sequential()
#model.add(Dropout(0.2, input_shape=(8,)))
model.add(Dense(50, input_dim=8, activation='relu', W_constraint=maxnorm(3)))
#model.add(Dropout(0.2))
model.add(Dense(8, init='normal', activation='relu', W_constraint=maxnorm(3)))
#model.add(Dropout(0.2))
model.add(Dense(1, activation='sigmoid'))
# Compile model
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# Fit the model
model.fit(X, Y, nb_epoch=40, batch_size=10)
# evaluate the model
scores = model.evaluate(X, Y)
print("%s: %.2f%%" % (model.metrics_names[1], scores[1] * 100))
x_train_scaled = np.transpose(
np.asarray(x_train_scaled_list)) # Convert to numpy array
# Convert min and max lists to arrays and save
min_array = np.asarray(min_list)
max_array = np.asarray(max_list)
np.save(path + 'min_array.npy', min_array)
np.save(path + 'max_array.npy', max_array)
### Create encoder ###
# First layer
encoder_initial = Dense(n_nodes,
activation=activation_function,
input_shape=(input_dim, ),
kernel_constraint=maxnorm(kc_max_norm))
# Hidden layers
encoder_list = []
for i in range(0, n_hidden_layers - 1):
encoder_list.append(
Dense(n_nodes,
activation=activation_function,
kernel_constraint=maxnorm(kc_max_norm)))
# Final layer
encoder_final = Dense(bottleneck_size, activation="linear")
### Create decoder ###
# First and hidden layers
decoder_list = []
def create_model(X_vocab_len, X_max_len, y_vocab_len, y_max_len,
n_phonetic_features, y1, n1, y2, n2, y3, n3, y4, n4, y5, n5,
y6, n6, hidden_size, num_layers):
def smart_merge(vectors, **kwargs):
return vectors[0] if len(vectors) == 1 else merge(vectors, **kwargs)
current_word = Input(shape=(X_max_len, ), dtype='float32',
name='input1') # for encoder (shared)
decoder_input = Input(shape=(X_max_len, ), dtype='float32',
name='input3') # for decoder -- attention
right_word1 = Input(shape=(X_max_len, ), dtype='float32', name='input4')
right_word2 = Input(shape=(X_max_len, ), dtype='float32', name='input5')
right_word3 = Input(shape=(X_max_len, ), dtype='float32', name='input6')
right_word4 = Input(shape=(X_max_len, ), dtype='float32', name='input7')
left_word1 = Input(shape=(X_max_len, ), dtype='float32', name='input8')
left_word2 = Input(shape=(X_max_len, ), dtype='float32', name='input9')
left_word3 = Input(shape=(X_max_len, ), dtype='float32', name='input10')
left_word4 = Input(shape=(X_max_len, ), dtype='float32', name='input11')
phonetic_input = Input(shape=(n_phonetic_features, ),
dtype='float32',
name='input12')
emb_layer1 = Embedding(X_vocab_len,
EMBEDDING_DIM,
input_length=X_max_len,
mask_zero=False,
name='Embedding')
list_of_inputs = [
current_word, right_word1, right_word2, right_word3, right_word4,
left_word1, left_word2, left_word3, left_word4
]
current_word_embedding, right_word_embedding1, right_word_embedding2,right_word_embedding3, right_word_embedding4, \
left_word_embedding1, left_word_embedding2, left_word_embedding3, left_word_embedding4 = [emb_layer1(i) for i in list_of_inputs]
print("Type:: ", type(current_word_embedding))
list_of_embeddings1 = [current_word_embedding, right_word_embedding1, right_word_embedding2,right_word_embedding3, right_word_embedding4, \
left_word_embedding1, left_word_embedding2, left_word_embedding3, left_word_embedding4]
list_of_embeddings = [
Dropout(0.50, name='drop1_' + str(j))(i)
for i, j in zip(list_of_embeddings1, range(len(list_of_embeddings1)))
]
list_of_embeddings = [
GaussianNoise(0.05, name='noise1_' + str(j))(i)
for i, j in zip(list_of_embeddings, range(len(list_of_embeddings)))
]
conv4_curr, conv4_right1, conv4_right2, conv4_right3, conv4_right4, conv4_left1, conv4_left2, conv4_left3, conv4_left4 =\
[Conv1D(filters=no_filters,
kernel_size=4, padding='valid',activation='relu',
strides=1, name='conv4_'+str(j))(i) for i,j in zip(list_of_embeddings, range(len(list_of_embeddings)))]
conv4s = [
conv4_curr, conv4_right1, conv4_right2, conv4_right3, conv4_right4,
conv4_left1, conv4_left2, conv4_left3, conv4_left4
]
maxPool4 = [
MaxPooling1D(name='max4_' + str(j))(i)
for i, j in zip(conv4s, range(len(conv4s)))
]
avgPool4 = [
AveragePooling1D(name='avg4_' + str(j))(i)
for i, j in zip(conv4s, range(len(conv4s)))
]
pool4_curr, pool4_right1, pool4_right2, pool4_right3, pool4_right4, pool4_left1, pool4_left2, pool4_left3, pool4_left4 = \
[merge([i,j], name='merge_conv4_'+str(k)) for i,j,k in zip(maxPool4, avgPool4, range(len(maxPool4)))]
conv5_curr, conv5_right1, conv5_right2, conv5_right3, conv5_right4, conv5_left1, conv5_left2, conv5_left3, conv5_left4 = \
[Conv1D(filters=no_filters,
kernel_size=5,
padding='valid',
activation='relu',
strides=1, name='conv5_'+str(j))(i) for i,j in zip(list_of_embeddings, range(len(list_of_embeddings)))]
conv5s = [
conv5_curr, conv5_right1, conv5_right2, conv5_right3, conv5_right4,
conv5_left1, conv5_left2, conv5_left3, conv5_left4
]
maxPool5 = [
MaxPooling1D(name='max5_' + str(j))(i)
for i, j in zip(conv5s, range(len(conv5s)))
]
avgPool5 = [
AveragePooling1D(name='avg5_' + str(j))(i)
for i, j in zip(conv5s, range(len(conv5s)))
]
pool5_curr, pool5_right1, pool5_right2, pool5_right3, pool5_right4, pool5_left1, pool5_left2, pool5_left3, pool5_left4 = \
[merge([i,j], name='merge_conv5_'+str(k)) for i,j,k in zip(maxPool5, avgPool5, range(len(maxPool5)))]
maxPools = [pool4_curr, pool4_right1, pool4_right2, pool4_right3, pool4_right4, \
pool4_left1, pool4_left2, pool4_left3, pool4_left4, \
pool5_curr, pool5_right1, pool5_right2, pool5_right3, pool5_right4, \
pool5_left1, pool5_left2, pool5_left3, pool5_left4]
concat = merge(maxPools, mode='concat', name='main_merge')
x = Dropout(0.15, name='drop_single1')(concat)
x = Bidirectional(RNN(rnn_output_size), name='bidirec1')(x)
total_features = [x, phonetic_input]
concat2 = merge(total_features, mode='concat', name='phonetic_merging')
x = Dense(HIDDEN_DIM,
activation='relu',
kernel_initializer='he_normal',
kernel_constraint=maxnorm(3),
bias_constraint=maxnorm(3),
name='dense1')(concat2)
x = Dropout(0.15, name='drop_single2')(x)
x = Dense(HIDDEN_DIM,
kernel_initializer='he_normal',
activation='tanh',
kernel_constraint=maxnorm(3),
bias_constraint=maxnorm(3),
name='dense2')(x)
x = Dropout(0.15, name='drop_single3')(x)
out1 = Dense(n1,
kernel_initializer='he_normal',
activation='softmax',
name='output1')(x)
out2 = Dense(n2,
kernel_initializer='he_normal',
activation='softmax',
name='output2')(x)
out3 = Dense(n3,
kernel_initializer='he_normal',
activation='softmax',
name='output3')(x)
out4 = Dense(n4,
kernel_initializer='he_normal',
activation='softmax',
name='output4')(x)
out5 = Dense(n5,
kernel_initializer='he_normal',
activation='softmax',
name='output5')(x)
out6 = Dense(n6,
kernel_initializer='he_normal',
activation='softmax',
name='output6')(x)
# Luong et al. 2015 attention model
emb_layer = Embedding(X_vocab_len,
EMBEDDING_DIM,
input_length=X_max_len,
mask_zero=True,
name='Embedding_for_seq2seq')
current_word_embedding, right_word_embedding1, right_word_embedding2,right_word_embedding3, right_word_embedding4, \
left_word_embedding1, left_word_embedding2, left_word_embedding3, left_word_embedding4 = [emb_layer(i) for i in list_of_inputs]
# current_word_embedding = smart_merge([ current_word_embedding, right_word_embedding1, left_word_embedding1])
encoder, state = GRU(rnn_output_size,
return_sequences=True,
unroll=True,
return_state=True,
name='encoder')(current_word_embedding)
encoder_last = encoder[:, -1, :]
decoder = emb_layer(decoder_input)
decoder = GRU(rnn_output_size,
return_sequences=True,
unroll=True,
name='decoder')(decoder, initial_state=[encoder_last])
attention = dot([decoder, encoder], axes=[2, 2], name='dot')
attention = Activation('softmax', name='attention')(attention)
context = dot([attention, encoder], axes=[2, 1], name='dot2')
decoder_combined_context = concatenate([context, decoder],
name='concatenate')
outputs = TimeDistributed(Dense(64, activation='tanh'),
name='td1')(decoder_combined_context)
outputs = TimeDistributed(Dense(X_vocab_len, activation='softmax'),
name='td2')(outputs)
all_inputs = [current_word, decoder_input, right_word1, right_word2, right_word3, right_word4, left_word1, left_word2, left_word3,\
left_word4, phonetic_input]
all_outputs = [outputs, out1, out2, out3, out4, out5, out6]
model = Model(input=all_inputs, output=all_outputs)
opt = Adam()
return model
def build_model(args):
print("args", vars(args))
model = Sequential()
np.random.seed(args.seed)
if hasattr(args, 'embedding_weights') and args.embedding_weights is not None:
W = np.load(args.embedding_weights)
if args.train_embeddings:
model.add(Embedding(args.n_vocab, args.n_word_dims,
weights=[W],
W_constraint=maxnorm(args.embedding_max_norm)))
else:
model.add(ImmutableEmbedding(args.n_vocab, args.n_word_dims,
weights=[W]))
else:
model.add(Embedding(args.n_vocab, args.n_word_dims,
mask_zero=args.mask_zero,
W_constraint=maxnorm(args.embedding_max_norm)))
model.add(LSTM(args.n_word_dims, args.n_units,
truncate_gradient=args.truncate_gradient,
return_sequences=True))
if args.regularization_layer == 'dropout':
model.add(Dropout(0.2))
#elif args.regularization_layer == 'normalization':
# model.add(BatchNormalization((args.n_filters,)))
model.add(LSTM(args.n_units, args.n_units,
truncate_gradient=args.truncate_gradient,
return_sequences=True))
if args.regularization_layer == 'dropout':
model.add(Dropout(0.2))
#elif args.regularization_layer == 'normalization':
# model.add(BatchNormalization((args.n_filters,)))
'''
model.add(LSTM(args.n_units, args.n_units,
truncate_gradient=args.truncate_gradient,
return_sequences=True))
if args.regularization_layer == 'dropout':
model.add(Dropout(0.2))
#elif args.regularization_layer == 'normalization':
# model.add(BatchNormalization((args.n_filters,)))
'''
model.add(LSTM(args.n_units, args.n_units,
truncate_gradient=args.truncate_gradient,
return_sequences=False))
if args.regularization_layer == 'dropout':
model.add(Dropout(0.2))
#elif args.regularization_layer == 'normalization':
# model.add(BatchNormalization((args.n_filters,)))
model.add(Dense(args.n_units, args.n_classes,
W_regularizer=l2(args.l2_penalty)))
model.add(Activation('softmax'))
if args.optimizer == 'SGD':
optimizer = SGD(lr=args.learning_rate,
decay=args.decay, momentum=args.momentum,
clipnorm=args.clipnorm)
elif args.optimizer == 'Adam':
optimizer = Adam(clipnorm=args.clipnorm)
elif args.optimizer == 'RMSprop':
optimizer = RMSprop(clipnorm=args.clipnorm)
elif args.optimizer == 'Adadelta':
optimizer = Adadelta(clipnorm=args.clipnorm)
elif args.optimizer == 'Adagrad':
optimizer = Adagrad(clipnorm=args.clipnorm)
else:
raise ValueError("don't know how to use optimizer {0}".format(args.optimizer))
model.compile(loss=args.loss, optimizer=optimizer)
return model
seed = 7
numpy.random.seed(seed)
# load data
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
# normalize inputs from 0-255 to 0.0-1.0
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train = X_train / 255.0
X_test = X_test / 255.0
# one hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
# Create the model
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=(3, 32, 32), padding='same', activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Conv2D(32, (3, 3), activation='relu', padding='same', kernel_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(512, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
# Compile model
epochs = 25
lrate = 0.01
decay = lrate/epochs
sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)
#tbCallBack= keras.callbacks.TensorBoard(log_dir='./Graph’, histogram_freq=0,write_graph=True, write_images=True)
model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy'])
(X_train, y_train), (X_test, y_test) = cifar10.load_data()
# normalize inputs from 0-255 to 0.0-1.0
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train = X_train / 255.0
X_test = X_test / 255.0
# one hot encode outputs
y_train = np_utils.to_categorical(y_train)
y_test = np_utils.to_categorical(y_test)
num_classes = y_test.shape[1]
# Create the model
model = Sequential()
model.add(Convolution2D(32, 3, 3, input_shape=(3, 32, 32), border_mode='same', activation='relu', W_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Convolution2D(32, 3, 3, activation='relu', border_mode='same', W_constraint=maxnorm(3)))
# plot_filters(model[0, ])
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(512, activation='relu', W_constraint=maxnorm(3)))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
# Compile model
epochs = 1
learning_rate = 0.01
decay = learning_rate / epochs
sgd = SGD(lr=learning_rate, momentum=0.9, decay=decay, nesterov=False)
def DNN_RUN(filename,
Epochs=100,
ROTATION=4,
LAYER_NUM=2,
Node_num=100,
Train_Batch=50):
####################################### SELF SETTING !!!!!
if (filename[0] == "/"):
filename = filename
elif (filename[0] == '~'):
filename = filename.replace("~", os.environ['HOME'])
elif ((filename[0] == "C") & (filename[1] == ":")):
filename = filename
else:
filename = os.getcwd(
) + "/" + filename # get the path included filename
loca = len(filename)
for i in range(1, len(filename) + 1): # find the "/" location
if (filename[-i] == "/"):
loca = i - 1
break
FILENAME = filename.replace(filename[:-loca],
"") # this is the shorten filename
filename_No_Txt = FILENAME.replace(".txt", "")
infile = filename
data = TFile.Open(infile)
dirlist = data.GetListOfKeys()
ITER = dirlist.MakeIterator()
key = ITER.Next()
TREE_NAME = (key.ReadObj()).GetName()
#print(TREE_NAME)
tree = data.Get(TREE_NAME)
TOT_ENTRY = tree.GetEntries()
UPPER_LIMIT = int(TOT_ENTRY * float(ROTATION - 1) / float(ROTATION))
upper_limit = UPPER_LIMIT # 1000
EPOCHS = Epochs
#NODENUM = 128
NODENUM = Node_num
BATCH_SIZE_train = Train_Batch # 500
BATCH_SIZE_test = 1 # 1
OPTIMIZER = 'adam' #'adam', 'Adamax', 'adagrad', 'sgd', 'adadelta', 'RMSprop', 'Nadam', 'TFOptimizer'
KERNAL_INIT = 'he_uniform' #truncated_normal, glorot_uniform, random_uniform, he_uniform, he_normal
L2 = 1e-5 #1e-5
DROP_RATE = 0.0 # 0.2
MAXNORM = 10000000 #5
ACTIVATION = "relu"
####################################### Input DATA Sets !!!!!
b_reco_lep1Pt_ = tree2array(tree, branches="reco_lep1Pt")
b_reco_lep1Eta_ = tree2array(tree, branches="reco_lep1Eta")
b_reco_lep1Phi_ = tree2array(tree, branches="reco_lep1Phi")
b_reco_lep2Pt_ = tree2array(tree, branches="reco_lep2Pt")
b_reco_lep2Eta_ = tree2array(tree, branches="reco_lep2Eta")
b_reco_lep2Phi_ = tree2array(tree, branches="reco_lep2Phi")
b_reco_MET_et_ = tree2array(tree, branches="reco_MET_et")
b_reco_MET_phi_ = tree2array(tree, branches="reco_MET_phi")
###############################################################################################################
##################################### Target DATA !!!!!
b_TARGET_ = tree2array(tree, branches="gen_Nu_phi1")
###############################################################################################################
##################################### Pick valid Input/Target DATA (TARGET is output data) !!!!!
b_reco_lep1Pt = np.zeros(b_reco_lep1Pt_.size)
b_reco_lep1Eta = np.zeros(b_reco_lep1Eta_.size)
b_reco_lep1Phi = np.zeros(b_reco_lep1Phi_.size)
b_reco_lep2Pt = np.zeros(b_reco_lep2Pt_.size)
b_reco_lep2Eta = np.zeros(b_reco_lep2Eta_.size)
b_reco_lep2Phi = np.zeros(b_reco_lep2Phi_.size)
b_reco_MET_et = np.zeros(b_reco_MET_et_.size)
b_reco_MET_phi = np.zeros(b_reco_MET_phi_.size)
b_TARGET = np.zeros(b_TARGET_.size)
for i in range(b_TARGET.size):
b_reco_lep1Pt[i] = b_reco_lep1Pt_[i]
b_reco_lep1Eta[i] = b_reco_lep1Eta_[i]
b_reco_lep1Phi[i] = b_reco_lep1Phi_[i]
b_reco_lep2Pt[i] = b_reco_lep2Pt_[i]
b_reco_lep2Eta[i] = b_reco_lep2Eta_[i]
b_reco_lep2Phi[i] = b_reco_lep2Phi_[i]
b_reco_MET_et[i] = b_reco_MET_et_[i]
b_reco_MET_phi[i] = b_reco_MET_phi_[i]
b_TARGET[i] = b_TARGET_[i]
###############################################################################################################
##################################### ARRAY is input DATA !!!!!
ARRAY = np.stack(
(b_reco_lep1Pt, b_reco_lep1Eta, b_reco_lep1Phi, b_reco_lep2Pt,
b_reco_lep2Eta, b_reco_lep2Phi, b_reco_MET_et, b_reco_MET_phi))
print(ARRAY.shape)
ARRAY = ARRAY.T
print(ARRAY.shape)
print(ARRAY[:2])
print(b_TARGET[:2])
###############################################################################################################
ndim = ARRAY.shape[1]
nEvents = ARRAY.shape[0]
rotation = 0
for i in range(ROTATION):
# print("This is ",i+1,"th rotation, Of total", ROTATION)
rotation = rotation + 1
if (rotation == 1):
LOAD_WEIGHTS = False
if (rotation != 1):
LOAD_WEIGHTS = True
if (rotation == 1):
data_Temp_train = ARRAY[0:upper_limit]
target_Temp_train = b_TARGET[0:upper_limit]
data_Temp_test = ARRAY[(upper_limit + 1):ARRAY.shape[0]]
target_Temp_test = b_TARGET[(upper_limit + 1):ARRAY.shape[0]]
elif (rotation == ROTATION):
data_Temp_train = ARRAY[int(TOT_ENTRY / float(ROTATION)) +
1:TOT_ENTRY]
target_Temp_train = b_TARGET[int(TOT_ENTRY / float(ROTATION)) +
1:TOT_ENTRY]
data_Temp_test = ARRAY[0:int(TOT_ENTRY / float(ROTATION))]
target_Temp_test = b_TARGET[0:int(TOT_ENTRY / float(ROTATION))]
else:
INIT1 = 0
END1 = TOT_ENTRY - int(TOT_ENTRY / float(ROTATION)) * (rotation)
INIT2 = TOT_ENTRY - int(
TOT_ENTRY / float(ROTATION)) * (rotation - 1)
END2 = TOT_ENTRY
data_Temp_train1 = ARRAY[INIT1:END1]
target_Temp_train1 = b_TARGET[INIT1:END1]
data_Temp_train2 = ARRAY[INIT2 + 1:END2]
target_Temp_train2 = b_TARGET[INIT2 + 1:END2]
data_Temp_test = ARRAY[END1 + 1:INIT2]
target_Temp_test = b_TARGET[END1 + 1:INIT2]
data_Temp_train = np.concatenate(
(data_Temp_train1, data_Temp_train2), axis=0)
target_Temp_train = np.concatenate(
(target_Temp_train1, target_Temp_train2), axis=0)
data_train = data_Temp_train
target_train = target_Temp_train
data_test = data_Temp_test
target_test = target_Temp_test
ijkl = 0
modelRegress = Sequential()
#modelRegress.add(Dropout(DROP_RATE, input_shape=(ndim,)))
#modelRegress.add(Dense(NODENUM, kernel_initializer=KERNAL_INIT, activation=ACTIVATION, W_regularizer=l2(L2), kernel_constraint=maxnorm(3)))
modelRegress.add(
Dense(NODENUM,
kernel_initializer=KERNAL_INIT,
activation=ACTIVATION,
W_regularizer=l2(L2),
input_dim=ndim,
kernel_constraint=maxnorm(MAXNORM)))
#modelRegress.add(BatchNormalization())
#modelRegress.add(Activation(ACTIVATION))
modelRegress.add(Dropout(DROP_RATE))
ijkl = ijkl + 1
if (LAYER_NUM >= 2):
modelRegress.add(
Dense(NODENUM,
kernel_initializer=KERNAL_INIT,
activation=ACTIVATION,
W_regularizer=l2(L2),
kernel_constraint=maxnorm(MAXNORM)))
modelRegress.add(Dropout(DROP_RATE))
ijkl = ijkl + 1
if (LAYER_NUM >= 3):
modelRegress.add(
Dense(NODENUM,
kernel_initializer=KERNAL_INIT,
activation=ACTIVATION,
W_regularizer=l2(L2))) #additional layer
modelRegress.add(Dropout(DROP_RATE))
ijkl = ijkl + 1
if (LAYER_NUM >= 4):
modelRegress.add(
Dense(NODENUM,
kernel_initializer=KERNAL_INIT,
activation=ACTIVATION,
W_regularizer=l2(L2))) #additional layer
modelRegress.add(Dropout(DROP_RATE))
ijkl = ijkl + 1
if (LAYER_NUM >= 5):
modelRegress.add(
Dense(NODENUM,
kernel_initializer=KERNAL_INIT,
activation=ACTIVATION,
W_regularizer=l2(L2))) #additional layer
modelRegress.add(Dropout(DROP_RATE))
ijkl = ijkl + 1
modelRegress.add(
Dense(1, kernel_initializer=KERNAL_INIT,
activation='linear')) #sigmoid
modelRegress.compile(loss='mean_squared_error', optimizer=OPTIMIZER)
if (LOAD_WEIGHTS == True):
LOAD_MODEL_WEIGHTS = "MODELs/Model_EN" + str(
upper_limit) + "_LN" + str(ijkl) + "_E" + str(
EPOCHS) + "_M" + str(rotation - 1) + ".h5"
modelRegress.load_weights(LOAD_MODEL_WEIGHTS, by_name=True)
print(
"************************************ MODEL LOADED ************************************"
)
modelRegress.summary()
print("************************************ This is ", i + 1,
"th rotation, Of total", ROTATION,
"************************************")
history_callback = modelRegress.fit(data_train,
target_train,
validation_data=(data_test,
target_test),
epochs=EPOCHS,
batch_size=BATCH_SIZE_train)
predict_train = modelRegress.predict(data_train,
batch_size=BATCH_SIZE_test)
predict_test = modelRegress.predict(data_test,
batch_size=BATCH_SIZE_test)
modelRegress.save_weights("MODELs/Model_EN" + str(upper_limit) +
"_LN" + str(ijkl) + "_E" + str(EPOCHS) +
"_M" + str(rotation) + ".h5")
if (rotation == ROTATION):
f = TFile(
"Rotation_tree_EN" + str(upper_limit) + "_LN" + str(ijkl) +
"_E" + str(EPOCHS) + "_NN" + str(NODENUM) + "_B" +
str(BATCH_SIZE_train) + "_" + str(OPTIMIZER) + "_L" + str(L2) +
"_DR" + str(DROP_RATE) + ".root", "recreate")
hist_target_train = TH1F('TrainData', 'TrainData', 20, -3.5, 3.5)
hist_target_test = TH1F('TestData', 'TestData', 20, -3.5, 3.5)
hist_output_train = TH1F('OutputDataTrain', 'OutputDataTrain', 20,
-3.5, 3.5)
hist_output_test = TH1F('OutputDataTest', 'OutputDataTest', 20,
-3.5, 3.5)
tree_train = TTree('tree_train', 'tree_train')
Ttrain = np.zeros(1, dtype=float)
Otrain = np.zeros(1, dtype=float)
tree_train.Branch('target_train', Ttrain, 'target_train/D')
tree_train.Branch('output_train', Otrain, 'output_train/D')
for ij1 in range(upper_limit - 2):
Ttrain[0] = target_train[ij1]
Otrain[0] = predict_train[ij1]
tree_train.Fill()
tree_test = TTree('tree_test', 'tree_test')
Ttest = np.zeros(1, dtype=float)
Otest = np.zeros(1, dtype=float)
tree_test.Branch('target_test', Ttest, 'target_test/D')
tree_test.Branch('output_test', Otest, 'output_test/D')
for ij2 in range(ARRAY.shape[0] - upper_limit - 3):
Ttest[0] = target_test[ij2]
Otest[0] = predict_test[ij2]
tree_test.Fill()
fill_hist(hist_target_train, target_train)
fill_hist(hist_target_test, target_test)
fill_hist(hist_output_train, predict_train[:, 0])
fill_hist(hist_output_test, predict_test[:, 0])
f.Write()
f.Close()
CP_FNAME = "MODELs/Model_EN" + str(upper_limit) + "_LN" + str(
ijkl) + "_E" + str(EPOCHS) + "_M" + str(ROTATION) + ".h5"
CP_TNAME = "MODELs/FINAL_EN" + str(upper_limit) + "_LN" + str(
ijkl) + "_EP" + str(EPOCHS) + "_M" + str(ROTATION) + ".h5"
CP_FINAL_MODEL = "cp " + CP_FNAME + " " + CP_TNAME
RM_COMMAND = "rm "
RM_NAMES = "MODELs/Model_EN" + str(upper_limit) + "_LN" + str(
ijkl) + "_E" + str(EPOCHS) + "_M" + "*" + ".h5"
RM_COMMAND = RM_COMMAND + RM_NAMES
os.system(CP_FINAL_MODEL)
os.system(RM_COMMAND)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Build model...')
input_sentence = Input(shape=(maxlen,), dtype='int32')
# we start off with an efficient embedding layer which maps
# our vocab indices into embedding_dims dimensions
input_embedding = Embedding(max_features, embedding_dims,
weights=[np.load('mr_word2vec_300_dim_google.embeddings')])(input_sentence)
cnns = [Convolution1D(filter_length=filter_length,
nb_filter=nb_filter,
W_regularizer=l2(0.0001),
W_constraint=maxnorm(3),
activation='relu',
border_mode='same') for filter_length in [2, 3, 5, 7]]
cnn_feature = merge([cnn(input_embedding) for cnn in cnns], mode='concat')
dropout = Dropout(0.25)
sentence_dropout = dropout(cnn_feature)
maxpool = Lambda(lambda x: K.max(x, axis=1, keepdims=False), output_shape=lambda x: (x[0], x[2]))
sentence_pool = maxpool(sentence_dropout)
predict_sentiment = Dense(2, activation='softmax')(sentence_pool)
model = Model(input=[input_sentence], output=[predict_sentiment])
def get_model(self,opt):
text_branch = Sequential()
embedding_layer = Embedding(len(opt.word_index) + 1,opt.embedding_dim,weights=[opt.embedding_matrix],input_length=opt.max_sequence_length,trainable=False)
text_branch.add(embedding_layer)
# text_branch.add(TrigPosEmbedding(
# input_shape=(None,),
# output_dim=EMBEDDING_DIM, # The dimension of embeddings.
# mode=TrigPosEmbedding.MODE_ADD, # Use `add` mode; MODE_CONCAT
# name='Pos-Embd',
# ))
# text_branch.add(Bidirectional(LSTM(units=100,return_sequences=False)))
text_branch.add(self.rnncell(units=self.opt.hidden_unit_num,return_sequences=True, activation='relu' ,kernel_constraint=maxnorm(3)))
text_branch.add(Dropout(self.opt.dropout_rate))
text_branch.add(self.rnncell(units=self.opt.hidden_unit_num,return_sequences=False, activation='relu' ,kernel_constraint=maxnorm(3)))
text_branch.add(Dropout(self.opt.dropout_rate))
# text_branch.add(SeqSelfAttention(attention_width=5,attention_activation='sigmoid'))
text_branch.add(Dense(3,activation="softmax"))
return text_branch
activation='relu',
border_mode='same'))
model.add(Dropout(0.2))
model.add(Convolution2D(32, 3, 3, activation='relu', border_mode='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(64, 3, 3, activation='relu', border_mode='same'))
model.add(Dropout(0.2))
model.add(Convolution2D(64, 3, 3, activation='relu', border_mode='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Convolution2D(128, 3, 3, activation='relu', border_mode='same'))
model.add(Dropout(0.2))
model.add(Convolution2D(128, 3, 3, activation='relu', border_mode='same'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dropout(0.2))
model.add(Dense(1024, activation='relu', W_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Dense(512, activation='relu', W_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Dense(num_classes, activation='softmax'))
# Compile model
epochs = 25
lrate = 0.01
decay = lrate / epochs
sgd = SGD(lr=lrate, momentum=0.9, decay=decay, nesterov=False)
model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
print(model.summary())
def train(save_best=True):
physical_devices = tf.config.list_physical_devices('GPU')
try:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
except:
# Invalid device or cannot modify virtual devices once initialized.
pass
print("Training CNN classifier...")
# fix random seed for reproducibility
seed = 7
np.random.seed(seed)
# TODO:
data_path = "test"
names = os.listdir(data_path)
input_res = (72, 128)
# load data from folders based on folder name:
images = []
labels = []
label_dict = {}
num_images = 0
for i, name in enumerate(names):
# count examples
image_files = os.listdir(data_path + '/' + name)
num_images += len(image_files)
im_path = data_path + '/' + names[0]
# encoded category for name
label_dict.update({name: i})
# load image into example pool
for im_name in image_files:
im = data_path + '/' + name + '/' + im_name
im = cv.imread(im)
image = cv.resize(im, dsize=(input_res))
images.append(image)
labels.append(i)
# Normalize pixel values to be between 0 and 1
images = np.asarray(images)
images = images / 255.0
# encode labels
train_images, test_images, train_labels, test_labels = train_test_split(images, labels, test_size=0.33)
# Create the model
model = Sequential()
model.add(Conv2D(16, kernel_size=3, padding='same', activation='relu', input_shape=(input_res[0], input_res[1], 3)))
model.add(Dropout(0.1))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(Dropout(0.1))
model.add(Conv2D(32, kernel_size=3, padding='same', activation='relu'))
model.add(Dropout(0.1))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(64, kernel_size=3, padding='same', activation='relu'))
model.add(Dropout(0.2))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(64, kernel_size=5, padding='same', activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Flatten())
model.add(Dropout(0.4))
model.add(Dense(2048, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.5))
model.add(Dense(2048, activation='relu', kernel_constraint=maxnorm(3)))
model.add(Dropout(0.5))
model.add(Dense(10))
# Compile model
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model.summary()
callbacks_list = []
if save_best:
filepath = "best_facial_classifier.hdf5"
checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
callbacks_list.append(checkpoint)
history = model.fit(train_images, train_labels, batch_size=64, epochs=500,
validation_data=(test_images, test_labels), callbacks=callbacks_list)
return [model, history]
import cv2
from PIL import Image
# fix random seed for reproducibility
seed = 7
numpy.random.seed(seed)
mode = 'v'
num_classes = 3
output = ['Sad','Happy','Angry']
# Create the model
model = Sequential()
model.add(Convolution2D(32, 3, 3, input_shape=(1, 32, 32), border_mode='same', activation='relu', W_constraint=maxnorm(3)))
model.add(Dropout(0.2))
model.add(Convolution2D(32, 3, 3, activation='relu', border_mode='same', W_constraint=maxnorm(3)))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(512, activation='relu', W_constraint=maxnorm(3)))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
model.load_weights("models/model-0.h5")
# Compile model
epochs = 25
def def_model(self, model_id=0):
input_shape = self.input_shape
if model_id == 0: # vgg 16
input_tensor = Input(shape=input_shape)
x = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv1')(input_tensor)
x = Conv2D(64, (3, 3),
activation='relu',
padding='same',
name='block1_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv1')(x)
x = Conv2D(128, (3, 3),
activation='relu',
padding='same',
name='block2_conv2')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv1')(x)
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv2')(x)
x = Conv2D(256, (3, 3),
activation='relu',
padding='same',
name='block3_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv1')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv2')(x)
x = Conv2D(512, (3, 3),
activation='relu',
padding='same',
name='block4_conv3')(x)
x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x)
# Classification block
x = Flatten(name='flatten')(x)
x = Dense(4096, activation='relu', name='fc1')(x)
x = Dense(4096, activation='relu', name='fc2')(x)
x = Dropout(0.5)(x)
x = Dense(10, activation='softmax', name='predictions')(x)
return_model = Model(inputs=[input_tensor], outputs=[x])
return_model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adam(),
metrics=['accuracy'])
elif model_id == 1:
"""
Convolutional Neural Network: https://github.com/umbertogriffo/Fashion-mnist-cnn-keras/blob/
master/src/convolutional/fashion_mnist_cnn.py
"""
return_model = Sequential()
return_model.add(
Conv2D(32, (5, 5),
input_shape=self.input_shape,
padding='same',
activation='relu'))
return_model.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
return_model.add(
Conv2D(64, (5, 5), padding='same', activation='relu'))
return_model.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
return_model.add(
Conv2D(128, (1, 1), padding='same', activation='relu'))
return_model.add(MaxPooling2D(pool_size=(2, 2), padding='same'))
return_model.add(Flatten())
return_model.add(
Dense(1024, activation='relu', kernel_constraint=maxnorm(3)))
return_model.add(Dropout(0.5))
return_model.add(
Dense(512, activation='relu', kernel_constraint=maxnorm(3)))
return_model.add(Dropout(0.5))
return_model.add(Dense(self.num_classes, activation='softmax'))
# Compile model
lrate = 0.1
decay = lrate / self.epoch
sgd = keras.optimizers.SGD(lr=lrate,
momentum=0.9,
decay=decay,
nesterov=True)
return_model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
elif model_id == 2:
# https://github.com/cmasch/zalando-fashion-mnist/blob/master/Simple_Convolutional_Neural_Network_Fashion-MNIST.ipynb
cnn = Sequential()
cnn.add(InputLayer(input_shape=self.input_shape))
cnn.add(BatchNormalization())
cnn.add(
Convolution2D(64, (4, 4), padding='same', activation='relu'))
cnn.add(MaxPooling2D(pool_size=(2, 2)))
cnn.add(Dropout(0.1))
cnn.add(Convolution2D(64, (4, 4), activation='relu'))
cnn.add(MaxPooling2D(pool_size=(2, 2)))
cnn.add(Dropout(0.3))
cnn.add(Flatten())
cnn.add(Dense(256, activation='relu'))
cnn.add(Dropout(0.5))
cnn.add(Dense(64, activation='relu'))
cnn.add(BatchNormalization())
cnn.add(Dense(self.num_classes, activation='softmax'))
cnn.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(),
metrics=['accuracy'])
return cnn
elif model_id == 3:
pass # https://github.com/markjay4k/Fashion-MNIST-with-Keras/blob/master/pt%204%20-%20Deeper%20CNNs.ipynb
else:
raise Exception("unsupported model")
return return_model
