def ensemble_folds(models, input_shape):
model_input = layers.Input(shape=input_shape)
outputs = [model(model_input) for model in models]
y = layers.Average()(outputs)
model = Model(model_input, y, name='ensemble')
return model
def ensemble(self, models, model_input):
# collect outputs of models in a list
outputs = [model(model_input) for model in models]
# averaging outputs
y = layers.Average()(outputs)
# build model from same input and avg output
model = Model(inputs=model_input, outputs=y, name='ensemble')
return model
def build_model(self):
"""
Build the model
Returns:
Model : Keras model instance
"""
# Checks
if len(self.kernel_sizes) != len(self.feature_maps):
raise Exception('Please define `kernel_sizes` and `feature_maps` with the same amount.')
if not self.embedding_layer and (not self.num_words or not self.embedding_dim):
raise Exception('Please define `num_words` and `embedding_dim` if you not using a pre-trained embedding.')
if self.use_char and (not self.char_max_length or not self.alphabet_size):
raise Exception('Please define `char_max_length` and `alphabet_size` if you are using char.')
# Building word-embeddings from scratch
if self.embedding_layer is None:
self.embedding_layer = layers.Embedding(
input_dim=self.num_words,
output_dim=self.embedding_dim,
input_length=self.max_seq_length,
weights=None, trainable=True,
name="word_embedding"
)
# WORD-level
word_input = layers.Input(shape=(self.max_seq_length,), dtype='int32', name='word_input')
x = self.embedding_layer(word_input)
if self.dropout_rate:
x = layers.Dropout(self.dropout_rate)(x)
x = self.building_block(x, self.kernel_sizes, self.feature_maps)
x = layers.Activation('relu')(x)
prediction = layers.Dense(self.nb_classes, activation='softmax')(x)
# CHAR-level
if self.use_char:
char_input = layers.Input(shape=(self.char_max_length,), dtype='int32', name='char_input')
x_char = layers.Embedding(
input_dim=self.alphabet_size + 1,
output_dim=50,
input_length=self.char_max_length,
name='char_embedding'
)(char_input)
x_char = self.building_block(x_char, self.char_kernel_sizes, self.char_feature_maps)
x_char = layers.Activation('relu')(x_char)
x_char = layers.Dense(self.nb_classes, activation='softmax')(x_char)
prediction = layers.Average()([prediction, x_char])
return keras.Model(inputs=[word_input, char_input], outputs=prediction, name='CNN_Word_Char')
return keras.Model(inputs=word_input, outputs=prediction, name='CNN_Word')
def test_merge_average():
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
o = layers.average([i1, i2])
assert o._keras_shape == (None, 4, 5)
model = models.Model([i1, i2], o)
avg_layer = layers.Average()
o2 = avg_layer([i1, i2])
assert avg_layer.output_shape == (None, 4, 5)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
assert out.shape == (2, 4, 5)
assert_allclose(out, 0.5 * (x1 + x2), atol=1e-4)
def model(reader):
assert reader.channeltypes == [ChannelType.POS_TAGS]
known_in = L.Input(shape=(None, ), dtype='int32')
unknown_in = L.Input(shape=(None, ), dtype='int32')
embedding = L.Embedding(
len(reader.channels[0].vocabulary_above_cutoff) + 2, 1)
known_emb = embedding(known_in)
unknown_emb = embedding(unknown_in)
feature_extractor1 = L.Bidirectional(L.LSTM(100, return_sequences=True))
pool = L.GlobalAvgPool1D()
features_known = pool(feature_extractor1(known_emb))
features_unknown = pool(feature_extractor1(unknown_emb))
abs_diff = L.merge(inputs=[features_known, features_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
# Dense network.
dense1 = L.Dense(300)(abs_diff)
dense2 = L.Dense(200)(dense1)
dense3 = L.Dense(100)(dense2)
output1 = L.Dense(2, activation='softmax')(dense1)
output2 = L.Dense(2, activation='softmax')(dense2)
output3 = L.Dense(2, activation='softmax')(dense3)
output = L.Average()([output1, output2, output3])
model = Model(inputs=[known_in, unknown_in], outputs=output)
optimizer = O.Adam(lr=0.0005)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def create_multiview_model(base_model, multiviews_num, input_shape, output_size, use_lstm):
assert base_model == "voxnet" or base_model == "pointnet"
if base_model == "voxnet":
base_model = create_voxnet_model_homepage(input_shape, output_size)
elif base_model == "pointnet":
base_model = create_point_net(input_shape, output_size)
input = layers.Input(shape=(multiviews_num,) + input_shape)
multiview_outputs = []
for i in range(multiviews_num):
multiview_input = layers.Lambda(lambda x: x[:,i])(input)
multiview_output = base_model(multiview_input)
multiview_outputs.append(multiview_output)
output = layers.Average()(multiview_outputs)
model = models.Model(input, output)
return model
def build_ensemble(net_configs, pop_per_type = 5, merge_type = None):
merge_layers = {
"Average" : nn_layers.Average()
}
inputs = []
outputs = []
for net_conf in net_configs:
for i in range(pop_per_type):
il, ol = build_layers(json.loads(net_conf))
inputs.append(il)
outputs.append(ol)
train_model = Model(inputs=inputs, outputs=outputs)
if len(outputs) > 1:
merge_layer = merge_layers[merge_type] if merge_type != None else None
merge_model = Model(inputs=inputs, outputs=merge_layer(outputs)) if merge_layer != None else None
else: # There is only 1 member in the Ensemble
merge_model = train_model
model_list = [Model(inputs=il,outputs=ol) for il,ol in zip(inputs,outputs)]
return inputs, outputs, train_model, model_list, merge_model
def get_keras_model(self):
inp, bias_counts_input, bias_profile_input = self.get_inputs()
rev_inp = kl.Lambda(lambda x: x[:, ::-1, ::-1])(inp)
countouttaskname, profileouttaskname = self.get_names()
first_conv = kl.Conv1D(self.filters,
kernel_size=self.conv1_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid',
activation='relu')
first_conv_fwd = first_conv(inp)
first_conv_rev = first_conv(rev_inp)
curr_layer_size = self.input_seq_len - (self.conv1_kernel_size - 1)
prev_layers_fwd = first_conv_fwd
prev_layers_rev = first_conv_rev
for i in range(1, self.n_dil_layers + 1):
dilation_rate = 2**i
conv_output = kl.Conv1D(self.filters,
kernel_size=self.dil_kernel_size,
padding='valid',
kernel_initializer=self.kernel_initializer,
activation='relu',
dilation_rate=dilation_rate)
conv_output_fwd = conv_output(prev_layers_fwd)
conv_output_rev = conv_output(prev_layers_rev)
width_to_trim = dilation_rate * (self.dil_kernel_size - 1)
curr_layer_size = (curr_layer_size - width_to_trim)
prev_layers_fwd = self.trim_flanks_of_conv_layer(
conv_layer=prev_layers_fwd,
output_len=curr_layer_size,
width_to_trim=width_to_trim,
filters=self.filters)
prev_layers_rev = self.trim_flanks_of_conv_layer_revcomp(
conv_layer=prev_layers_rev,
output_len=curr_layer_size,
width_to_trim=width_to_trim,
filters=self.filters)
if (self.is_add):
prev_layers_fwd = kl.add([prev_layers_fwd, conv_output_fwd])
prev_layers_rev = kl.add([prev_layers_rev, conv_output_rev])
else:
prev_layers_fwd = kl.average(
[prev_layers_fwd, conv_output_fwd])
prev_layers_rev = kl.average(
[prev_layers_rev, conv_output_rev])
combined_conv_fwd = prev_layers_fwd
combined_conv_rev = prev_layers_rev
# Counts Prediction
counts_dense_layer = kl.Dense(
2,
kernel_initializer=self.kernel_initializer,
)
gap_combined_conv_fwd = kl.GlobalAvgPool1D()(combined_conv_fwd)
gap_combined_conv_rev = kl.GlobalAvgPool1D()(combined_conv_rev)
main_count_out_fwd = counts_dense_layer(
kl.concatenate([gap_combined_conv_fwd, bias_counts_input],
axis=-1))
main_count_out_rev = counts_dense_layer(
kl.concatenate([bias_counts_input, gap_combined_conv_rev],
axis=-1))
rc_rev_count_out = kl.Lambda(lambda x: x[:, ::-1])(main_count_out_rev)
avg_count_out = kl.Average(name=countouttaskname)(
[main_count_out_fwd, rc_rev_count_out])
# Profile Prediction
profile_penultimate_conv = kl.Conv1D(
filters=2,
kernel_size=self.outconv_kernel_size,
kernel_initializer=self.kernel_initializer,
padding='valid')
profile_final_conv = kl.Conv1D(
2,
kernel_size=1,
kernel_initializer=self.kernel_initializer,
)
profile_out_prebias_fwd = profile_penultimate_conv(combined_conv_fwd)
main_profile_out_fwd = profile_final_conv(
kl.concatenate([profile_out_prebias_fwd, bias_profile_input],
axis=-1))
profile_out_prebias_rev = profile_penultimate_conv(combined_conv_rev)
rev_bias_profile_input = kl.Lambda(lambda x: x[:, ::-1, :])(
bias_profile_input)
main_profile_out_rev = profile_final_conv(
kl.concatenate([profile_out_prebias_rev, rev_bias_profile_input],
axis=-1))
rc_rev_profile_out = kl.Lambda(lambda x: x[:, ::-1, ::-1])(
main_profile_out_rev)
avg_profile_out = kl.Average(name=profileouttaskname)(
[main_profile_out_fwd, rc_rev_profile_out])
model = keras.models.Model(
inputs=[inp, bias_counts_input, bias_profile_input],
outputs=[avg_count_out, avg_profile_out])
model.compile(keras.optimizers.Adam(lr=self.lr),
loss=['mse', MultichannelMultinomialNLL(2)],
loss_weights=[self.c_task_weight, self.p_task_weight])
return model
def train(self):
U.save_GPU_mem_keras()
expr = U.ExprCreaterAndResumer(modelDir,
postfix="dr%s_imgOnly" % (str(dropout)))
# x channel: image
x_inputs = L.Input(shape=inputShape)
x = x_inputs # inputs is used by the line "Model(inputs, ... )" below
conv11 = L.Conv2D(16, (3, 3),
strides=1,
dilation_rate=2,
padding='valid')
x = conv11(x)
x = L.Activation('relu')(x)
x = L.BatchNormalization()(x)
# Batch needs to be after relu, otherwise it won't train...
conv12 = L.Conv2D(16, (3, 3),
strides=1,
dilation_rate=2,
padding='valid')
x = conv12(x)
x = L.Activation('relu')(x)
x = L.BatchNormalization()(x)
conv13 = L.Conv2D(16, (3, 3),
strides=1,
dilation_rate=2,
padding='valid')
x = conv13(x)
x = L.Activation('relu')(x)
x_output = L.BatchNormalization()(x)
# z channel: optical flow
z_inputs = L.Input(shape=inputShapeOF)
z = z_inputs # inputs is used by the line "Model(inputs, ... )" below
conv21 = L.Conv2D(16, (3, 3),
strides=1,
dilation_rate=2,
padding='valid')
z = conv21(z)
z = L.Activation('relu')(z)
z = L.BatchNormalization()(z)
conv22 = L.Conv2D(16, (3, 3),
strides=1,
dilation_rate=2,
padding='valid')
z = conv22(z)
z = L.Activation('relu')(z)
z = L.BatchNormalization()(z)
conv23 = L.Conv2D(16, (3, 3),
strides=1,
dilation_rate=2,
padding='valid')
z = conv23(z)
z = L.Activation('relu')(z)
z_output = L.BatchNormalization()(z)
joint = L.Average()([x_output, z_output])
joint = L.Flatten()(joint)
joint = L.Dense(32, activation='relu')(joint)
joint = L.Dropout(dropout)(joint)
output = L.Dense(1, activation='sigmoid')(joint)
model = Model(inputs=[x_inputs, z_inputs], outputs=output)
opt = K.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0)
#opt = K.optimizers.Adam(lr=0.1, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
model.compile(optimizer=opt,
loss='binary_crossentropy',
metrics=[K.metrics.binary_accuracy])
print(model.summary())
# snapshot code before training the model
expr.dump_src_code_and_model_def(sys.argv[0], model)
model.fit([self.trainData, self.trainDataOF],
self.trainLabel,
validation_data=([self.testData,
self.testDataOF], self.testLabel),
shuffle=True,
batch_size=100,
epochs=epoch,
verbose=2,
callbacks=[
K.callbacks.TensorBoard(log_dir=expr.dir),
K.callbacks.ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001),
U.PrintLrCallback()
])
expr.save_weight_and_training_config_state(model)
score = model.evaluate([self.testData, self.testDataOF],
self.testLabel,
batch_size=100,
verbose=0)
expr.printdebug("eval score:" + str(score))
def get_or_create(self, gaze_model, name, reuse, num_actions, layer_norm,
dueling):
if name in self.models:
assert reuse == True
logger.log("QFunc model named %s is reused" % name)
else:
logger.log("QFunc model named %s is created" % name)
assert reuse == False
imgs = L.Input(shape=(84, 84, 4))
g = gaze_model(imgs)
gaze_heatmaps = L.Lambda(lambda x: (x - tf.reduce_min(
x, [1, 2, 3], True)) / (tf.reduce_max(x, [1, 2, 3], True) - tf.
reduce_min(x, [1, 2, 3], True)))(g)
g = gaze_heatmaps
x = imgs
x = L.Multiply(name="img_mul_gaze")([x, g])
c1 = L.Conv2D(32, (8, 8),
strides=4,
padding='same',
activation="relu",
name='mul_c1')
x = c1(x)
c2 = L.Conv2D(64, (4, 4),
strides=2,
padding='same',
activation="relu",
name='mul_c2')
x = c2(x)
c3 = L.Conv2D(64, (3, 3),
strides=1,
padding='same',
activation="relu",
name='mul_c3')
x = c3(x)
# ============================ channel 2 ============================
orig_x = imgs
orig_x = L.Conv2D(32, (8, 8),
strides=4,
padding='same',
activation="relu")(orig_x)
orig_x = L.Conv2D(64, (4, 4),
strides=2,
padding='same',
activation="relu")(orig_x)
orig_x = L.Conv2D(64, (3, 3),
strides=1,
padding='same',
activation="relu")(orig_x)
x = L.Average()([x, orig_x])
x = L.Flatten()(x)
if dueling:
state_score = L.Dense(512)(x)
if layer_norm:
state_score = L.BatchNormalization()(state_score)
state_score = L.Activation('relu')(state_score)
state_score = L.Dense(1)(state_score)
action_score = L.Dense(512)(x)
if layer_norm:
logger.log(
"Warning: layer_norm is set to True, but Keras doesn't have it. Replacing with BatchNorm."
)
action_score = L.BatchNormalization()(action_score)
action_score = L.Activation('relu')(action_score)
last_dense = L.Dense(num_actions, name="logits")
action_score = last_dense(action_score)
if dueling:
def wrapped_tf_ops(s):
action_score, state_score = s
return action_score - tf.reduce_mean(
action_score, 1, keep_dims=True) + state_score
action_score = L.Lambda(wrapped_tf_ops)(
[action_score, state_score])
model = Model(inputs=[imgs], outputs=[action_score, gaze_heatmaps])
model.interesting_layers = [
c1, c2, c3, last_dense
] # export variable interesting_layers for monitoring in train.py
self.models[name] = model
return self.models[name]
def experiment(network_model, reshape_mode='mlp'):
reshape_funs = {
"conv": lambda d: d.reshape(-1, 28, 28, 1),
"mlp": lambda d: d.reshape(-1, 784)
}
xtrain, ytrain, xtest, ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain, xtest = reshape_fun(xtrain), reshape_fun(xtest)
digits_data = utils.load_processed_data('combined_testing_data_more')
digits_data2 = utils.load_processed_data('digits_og_and_optimal')
taus = [25, 27, 30]
digits = list(map(reshape_fun, [digits_data[t] for t in taus]))
digits = list(map(utils.normalize_data, digits))
digits_og = digits_data2['optimal_lw']
d_labels = utils.create_one_hot(digits_data['labels'].astype('uint'))
d2_labels = utils.create_one_hot(digits_data2['labels'].astype('uint'))
ensemble_size = 20
epochs = 50
trials = 10
mnist_correct = []
mnist_wrong = []
digits_wrong = []
digits_correct = []
d2_wrong = []
d2_correct = []
for t in range(trials):
inputs = []
outputs = []
model_list = []
for e in range(ensemble_size):
model = Sequential()
model.add(
layers.Dense(200,
input_dim=784,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(200,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(200,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(
layers.Dense(10,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("softmax"))
es = clb.EarlyStopping(monitor='val_loss',
patience=5,
restore_best_weights=True)
model.compile(optimizer=opt.Adam(),
loss="categorical_crossentropy",
metrics=['acc'])
model.fit(xtrain,
ytrain,
epochs=epochs,
batch_size=100,
validation_split=(1 / 6),
callbacks=[es])
model_list.append(model)
inputs.extend(model.inputs)
outputs.extend(model.outputs)
merge_model = Model(inputs=inputs, outputs=layers.Average()(outputs))
#mnist_preds = merge_model.predict([xtest]*ensemble_size)
#mnist_mem_preds = np.array(train_model.predict([xtest]*ensemble_size)).transpose(1,2,0)
#correct, wrong = bin_entropies(mnist_preds, mnist_mem_preds ,ytest)
#mnist_correct.extend(correct)
#mnist_wrong.extend(wrong)
sp_digits = apply_salt_pepper(digits_og)
for s_d in sp_digits:
s_d = utils.normalize_data(reshape_fun(s_d))
d2_preds = merge_model.predict([s_d] * ensemble_size)
d2_mempreds = np.array(
list(map(lambda m: m.predict(s_d),
model_list))).transpose(1, 2, 0)
correct, wrong = bin_entropies(d2_preds, d2_mempreds, d2_labels)
d2_correct.extend(correct)
d2_wrong.extend(wrong)
for d in digits:
digits_preds = merge_model.predict([d] * ensemble_size)
mempreds = np.array(list(map(lambda m: m.predict(d),
model_list))).transpose(1, 2, 0)
correct, wrong = bin_entropies(digits_preds, mempreds, d_labels)
digits_wrong.extend(wrong)
digits_correct.extend(correct)
ensemble = {
#'mnist_correct' : mnist_correct,
#'mnist_wrong' : mnist_wrong,
'digits_correct': digits_correct,
'digits_wrong': digits_wrong,
'lecunn_correct': d2_correct,
'lecunn_wrong': d2_wrong
}
return ensemble
orig_x=imgs
orig_x=L.Conv2D(32, (8,8), strides=4, padding='same')(orig_x)
orig_x=L.BatchNormalization()(orig_x)
orig_x=L.Activation('relu')(orig_x)
orig_x=L.Dropout(dropout)(orig_x)
orig_x=L.Conv2D(64, (4,4), strides=2, padding='same')(orig_x)
orig_x=L.BatchNormalization()(orig_x)
orig_x=L.Activation('relu')(orig_x)
orig_x=L.Dropout(dropout)(orig_x)
orig_x=L.Conv2D(64, (3,3), strides=1, padding='same')(orig_x)
orig_x=L.BatchNormalization()(orig_x)
orig_x=L.Activation('relu')(orig_x)
x=L.Average()([x,orig_x])
x=L.Dropout(dropout)(x)
x=L.Flatten()(x)
x=L.Dense(512, activation='relu')(x)
x=L.Dropout(dropout)(x)
logits=L.Dense(NUM_CLASSES, name="logits")(x)
prob=L.Activation('softmax', name="prob")(logits)
model=Model(inputs=[imgs, gaze_heatmaps], outputs=[logits, prob, g, x_intermediate])
opt=K.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0)
model.compile(loss={"prob":None, "logits": MU.loss_func},
optimizer=opt,metrics={"logits": MU.acc_})
expr.dump_src_code_and_model_def(sys.argv[0], model)
d=input_utils.Dataset(LABELS_FILE_TRAIN, LABELS_FILE_VAL, SHAPE)
def model_creation(input_shape1,
input_shape2,
model_version,
optim,
problem_type=0):
"""
op_sequence : False, by default. If True, then also uses the num_nodes parameter.
num_nodes : The number that denotes that how many values to predict at the output layer.
"""
print("Inside model_creation")
past_inp = L.Input(shape=(input_shape1))
fut_inp = L.Input(shape=(input_shape2))
if (model_version == "M1V1"):
cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn1 = L.Conv1D(filters=32, kernel_size=2)(cnn1)
cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp)
cnn2 = L.Conv1D(filters=32, kernel_size=2)(cnn2)
lstm_inp = L.Average()([cnn1, cnn2])
lstm_out = L.LSTM(32,
recurrent_dropout=0.2,
return_sequences=True,
bias_initializer='ones')(lstm_inp)
x1 = L.Average()([lstm_out, lstm_inp])
x1 = L.Flatten()(x1)
elif (model_version == "M1V2"):
cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn1 = L.Conv1D(filters=32, kernel_size=3)(cnn1)
cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp)
cnn2 = L.Conv1D(filters=32, kernel_size=3)(cnn2)
x1 = L.Average()([cnn1, cnn2])
x1 = L.Flatten()(x1)
elif (model_version == "M1V3"):
x1 = L.LSTM(32,
recurrent_dropout=0.2,
return_sequences=True,
bias_initializer='ones')(past_inp)
#x1 = L.LSTM(32, recurrent_dropout=0.2, return_sequences=True, bias_initializer='ones')(x1)
x1 = L.LSTM(32, recurrent_dropout=0.2, bias_initializer='ones')(x1)
elif (model_version == "M2V1"):
cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn1 = L.Conv1D(filters=32, kernel_size=2)(cnn1)
lstm_out1 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(cnn1)
lstm_out1 = L.Average()([cnn1, lstm_out1])
cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp)
cnn2 = L.Conv1D(filters=32, kernel_size=2)(cnn2)
lstm_out2 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(cnn2)
lstm_out2 = L.Average()([cnn2, lstm_out2])
x1 = L.Average()([lstm_out1, lstm_out2])
x1 = L.Flatten()(x1)
elif (model_version == "M2V2"):
lstm_out1 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(past_inp)
lstm_out2 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(fut_inp)
x1 = L.Average()([lstm_out1, lstm_out2])
x1 = L.Flatten()(x1)
elif (model_version == "M3V1"):
# cnn_inp = L.Concatenate(axis=1)([past_inp,fut_inp])
cnn = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn = L.Conv1D(filters=32, kernel_size=2)(cnn)
layer = L.Lambda(lambda cnn: K.reverse(cnn, axes=1))
cnn2 = layer(cnn)
lstm1 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(cnn)
lstm2 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(cnn2)
lstm = L.Average()([lstm1, lstm2])
x1 = L.Average()([cnn, lstm])
x1 = L.Flatten()(x1)
elif (model_version == "M3V2"):
cnn_inp = L.Concatenate(axis=1)([past_inp, fut_inp])
cnn = L.Conv1D(filters=32, kernel_size=5)(cnn_inp)
cnn = L.Conv1D(filters=32, kernel_size=2)(cnn)
layer = L.Lambda(lambda cnn: K.reverse(cnn, axes=1))
cnn2 = layer(cnn)
lstm1 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(cnn)
lstm2 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(cnn2)
lstm = L.Average()([lstm1, lstm2])
x1 = L.Average()([cnn, lstm])
x1 = L.Flatten()(x1)
elif (model_version == "M4"):
cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn1 = L.Conv1D(filters=32, kernel_size=2)(cnn1)
cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp)
cnn2 = L.Conv1D(filters=32, kernel_size=2)(cnn2)
lstm1 = L.Bidirectional(
L.LSTM(16,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones'))(past_inp)
lstm2 = L.Bidirectional(
L.LSTM(16,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones'))(fut_inp)
out1 = L.Concatenate(axis=1)([cnn1, lstm1])
out2 = L.Concatenate(axis=1)([cnn2, lstm2])
# out1 = L.Average()([cnn1,cnn2])
# ousst2 = L.Average()([lstm1,lstm2])
# x1 = L.Concatenate(axis=1)([out1,out2])
x1 = L.Average()([out1, out2])
x1 = L.Flatten()(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
#Classification Part.
if (problem_type == 1):
main_out = L.Dense(2, activation='softmax')(x1)
model = M.Model(inputs=[past_inp, fut_inp],
outputs=[main_out],
name=model_version)
model.summary()
model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
#Regression Part.
else:
x1 = L.Dense(1)(x1)
main_out = L.advanced_activations.LeakyReLU(0.2)(x1)
model = M.Model(inputs=[past_inp, fut_inp],
outputs=[main_out],
name=model_version)
model.compile(optimizer=optim,
loss=tf.losses.huber_loss,
metrics=['mae', 'mse', rmse])
model.summary()
return model
model.fit(xtrain,
ytrain,
epochs=epochs,
batch_size=100,
callbacks=[es],
validation_split=(1 / 6))
members.append(model)
i = 0
for ensemble_size in range(2, ensemble_size_top + 1):
inputs = []
outputs = []
members_to_use = members[i:i + ensemble_size]
for m in members_to_use:
inputs.extend(m.inputs)
outputs.extend(m.outputs)
print((outputs))
ensemble = Model(inputs=inputs, outputs=layers.Average()(outputs))
ensemble.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['acc'])
accuracy = ann.test_model(ensemble, [xtest] * ensemble_size, ytest,
'accuracy')
i += ensemble_size
t_accuracies.append(accuracy)
t_accuracies = np.array(t_accuracies)
utils.save_processed_data(t_accuracies,
'ensemble_sizesim-trial-%s' % (t + 1))
def experiment(reshape_mode = 'mlp'):
reshape_funs = {
"conv" : lambda d : d.reshape(-1,28,28,1),
"mlp" : lambda d : d.reshape(-1,784)
}
xtrain,ytrain,xtest,ytest = utils.load_mnist()
reshape_fun = reshape_funs[reshape_mode]
xtrain,xtest = reshape_fun(xtrain),reshape_fun(xtest)
notmnist = load_notMNIST()
notmnist = reshape_fun(notmnist)
print(notmnist.shape)
ensemble_size = 20
epochs = 50
trials = 10
results = {
'A': [],
'B': [],
'C': [],
'D': [],
'E': [],
'F': [],
'G': [],
'H': [],
'I': [],
'J': []
}
results = []
for t in range(trials):
inputs = []
outputs = []
model_list = []
for e in range(ensemble_size):
model = Sequential()
model.add(layers.Dense(200,input_dim=784, kernel_initializer = inits.RandomUniform(maxval=0.5,minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(layers.Dense(200, kernel_initializer = inits.RandomUniform(maxval=0.5,minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(layers.Dense(200, kernel_initializer = inits.RandomUniform(maxval=0.5,minval=-0.5)))
model.add(layers.Activation("relu"))
model.add(layers.BatchNormalization())
model.add(layers.Dense(10, kernel_initializer = inits.RandomUniform(maxval=0.5,minval=-0.5)))
model.add(layers.Activation("softmax"))
es = clb.EarlyStopping(monitor='val_loss',patience=2,restore_best_weights=True)
model.compile(optimizer=opt.Adam(),loss="categorical_crossentropy",metrics=['acc'])
model.fit(xtrain,ytrain,epochs=epochs,batch_size=100,validation_split=(1/6),callbacks=[es])
model_list.append(model)
inputs.extend(model.inputs)
outputs.extend(model.outputs)
merge_model = Model(inputs = inputs, outputs = layers.Average()(outputs))
preds = merge_model.predict([notmnist]*ensemble_size)
mem_preds = np.array(list(map(lambda m : m.predict(notmnist), model_list))).transpose(1,2,0)
print(mem_preds.shape)
bits = list(map(stats.entropy,preds))
s_q = list(map(calc_pred_vars,mem_preds))
results.extend(list(zip(bits,s_q)))
return results
layers.Dense(10,
kernel_initializer=inits.RandomUniform(
maxval=0.5, minval=-0.5)))
model.add(layers.Activation("softmax"))
es = clb.EarlyStopping(monitor='val_loss',
patience=10,
restore_best_weights=True)
model.compile(optimizer=opt.Adam(),
loss="categorical_crossentropy",
metrics=['acc'])
model.fit(xtrain,
ytrain,
epochs=epochs,
batch_size=100,
validation_split=(1 / 6),
callbacks=[es])
model_list.append(model)
inputs.extend(model.inputs)
outputs.extend(model.outputs)
merge_layer = layers.Average()(
outputs) if ensemble_size > 1 else outputs
ensemble = Model(inputs=inputs, outputs=merge_layer)
pred = ensemble.predict([notmnist] * ensemble_size)
h = list(map(stats.entropy, pred))
results[ensemble_size].extend(h)
utils.save_processed_data(results, 'notmnist_sim-trial-%s' % (t + 1))
y = L.Dropout(dropout)(y)
deconv22 = L.Conv2DTranspose(32, (4, 4), strides=2, padding='valid')
y = deconv22(y)
y = L.Activation('relu')(y)
y = L.BatchNormalization()(y)
y = L.Dropout(dropout)(y)
deconv23 = L.Conv2DTranspose(1, (8, 8), strides=4, padding='valid')
y = deconv23(y)
print deconv23.output_shape
y = L.Activation('relu')(y)
y_output = L.BatchNormalization()(y)
# Merge outputs from 2 channels
outputs = L.Average()([x_output, y_output])
outputs = L.Activation(MU.my_softmax)(outputs)
model = Model(inputs=[x_inputs, y_inputs], outputs=outputs)
opt = K.optimizers.Adadelta(lr=1.0, rho=0.95, epsilon=1e-08, decay=0.0)
# opt=K.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
model.compile(loss=MU.my_kld, optimizer=opt, metrics=[MU.NSS])
d = IU.DatasetWithHeatmap_PastKFrames(LABELS_FILE_TRAIN, LABELS_FILE_VAL,
SHAPE, heatmap_shape, GAZE_POS_ASC_FILE,
k, stride)
of = IU.Dataset_OpticalFlow_PastKFrames(LABELS_FILE_TRAIN, LABELS_FILE_VAL,
SHAPE, k, stride)
def model_creation(input_shape1, input_shape2, model_version):
"""
op_sequence : False, by default. If True, then also uses the num_nodes parameter.
num_nodes : The number that denotes that how many values to predict at the output layer.
"""
past_inp = L.Input(shape=(input_shape1))
fut_inp = L.Input(shape=(input_shape2))
if (model_version == "M1V1"):
cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn1 = L.Conv1D(filters=32, kernel_size=3)(cnn1)
# cnn1 = L.Dense(32)(cnn1)
# cnn1 = L.advanced_activations.LeakyReLU(0.2)(cnn1)
cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp)
cnn2 = L.Conv1D(filters=32, kernel_size=3)(cnn2)
# cnn2 = L.Dense(32)(cnn2)
# cnn2 = L.advanced_activations.LeakyReLU(0.2)(cnn2)
lstm_inp = L.Average()([cnn1, cnn2])
lstm_out = L.LSTM(32,
recurrent_dropout=0.2,
return_sequences=True,
bias_initializer='ones')(lstm_inp)
x1 = L.Average()([lstm_out, lstm_inp])
x1 = L.Flatten()(x1)
elif (model_version == "M1V2"):
cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn1 = L.Conv1D(filters=32, kernel_size=3)(cnn1)
cnn1 = L.Dense(32)(cnn1)
cnn1 = L.advanced_activations.LeakyReLU(0.2)(cnn1)
cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp)
cnn2 = L.Conv1D(filters=32, kernel_size=3)(cnn2)
cnn2 = L.Dense(32)(cnn2)
cnn2 = L.advanced_activations.LeakyReLU(0.2)(cnn2)
x1 = L.Average()([cnn1, cnn2])
x1 = L.Flatten()(x1)
elif (model_version == "M1V3"):
x1 = L.LSTM(32,
recurrent_dropout=0.2,
return_sequences=True,
bias_initializer='ones')(past_inp)
x1 = L.LSTM(32,
recurrent_dropout=0.2,
return_sequences=True,
bias_initializer='ones')(x1)
x1 = L.LSTM(32, recurrent_dropout=0.2, bias_initializer='ones')(x1)
elif (model_version == "M2V1"):
cnn1 = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn1 = L.Conv1D(filters=32, kernel_size=2)(cnn1)
cnn1 = L.Dense(32)(cnn1)
cnn1 = L.advanced_activations.LeakyReLU(0.2)(cnn1)
lstm_out1 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(cnn1)
lstm_out1 = L.Average()([cnn1, lstm_out1])
cnn2 = L.Conv1D(filters=32, kernel_size=5)(fut_inp)
cnn2 = L.Conv1D(filters=32, kernel_size=2)(cnn2)
cnn2 = L.Dense(32)(cnn2)
cnn2 = L.advanced_activations.LeakyReLU(0.2)(cnn2)
lstm_out2 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(cnn2)
lstm_out2 = L.Average()([cnn2, lstm_out2])
x1 = L.Average()([lstm_out1, lstm_out2])
x1 = L.Flatten()(x1)
elif (model_version == "M2V2"):
lstm_out1 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(past_inp)
lstm_out2 = L.LSTM(32,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones')(fut_inp)
x1 = L.Average()([lstm_out1, lstm_out2])
x1 = L.Flatten()(x1)
elif (model_version == "M3V1"):
# cnn_inp = L.Concatenate(axis=1)([past_inp,fut_inp])
cnn = L.Conv1D(filters=32, kernel_size=5)(past_inp)
cnn = L.Conv1D(filters=32, kernel_size=2)(cnn)
# cnn = L.Dense(32)(cnn)
# cnn = L.advanced_activations.LeakyReLU(0.2)(cnn)
lstm = L.Bidirectional(
L.LSTM(16,
recurrent_dropout=0.3,
return_sequences=True,
bias_initializer='ones'))(cnn)
x1 = L.Average()([cnn, lstm])
x1 = L.Flatten()(x1)
elif (model_version == "M3V2"):
cnn_inp = L.Concatenate(axis=1)([past_inp, fut_inp])
cnn = L.Conv1D(filters=32, kernel_size=5)(cnn_inp)
cnn = L.Conv1D(filters=32, kernel_size=2)(cnn)
cnn = L.Dense(32)(cnn)
cnn = L.advanced_activations.LeakyReLU(0.2)(cnn)
x1 = L.Flatten()(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(256)(x1)
x1 = L.advanced_activations.LeakyReLU(0.2)(x1)
x1 = L.Dense(1)(x1)
main_out = L.advanced_activations.LeakyReLU(0.2)(x1)
model = M.Model(inputs=[past_inp, fut_inp],
outputs=[main_out],
name=model_version)
model.summary()
model.compile(optimizer='adam',
loss=tf.losses.huber_loss,
metrics=['mae', rmse])
return model
models = []
inputs = []
outputs = []
setup_keras()
for i in range(ensemble_size):
net_input, net_output = create_model()
inputs.append(net_input)
outputs.append(net_output)
model = Model(inputs=net_input, outputs=net_output)
models.append(model)
ensemble_model = nn_layers.Average()(outputs)
ensemble_model = Model(inputs=inputs, outputs=ensemble_model)
multi_model = Model(inputs=inputs, outputs=outputs)
multi_model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
multi_model.fit([train_x, train_x, train_x], [train_y, train_y, train_y],
verbose=1,
validation_data=([val_x, val_x, val_x], [val_y, val_y, val_y]),
epochs=5)
ntest_x, test_y = process_data(x_test, y_test)
predictions = ensemble_model.predict([ntest_x, ntest_x, ntest_x])
correct = np.equal(np.argmax(predictions, 1), np.argmax(test_y, 1))
accuracy = np.mean(correct)
