def design_model_nn(lstm_data_dim, nn_data_dim, timesteps):
model_B = Sequential()
# NN Part
nn_hidden_size = [50, 50]
nn_drop_rate = [0.4, 0.4]
nn_reg = [0.01, 0.01, 0.01]
nn_areg = [0.01, 0.01, 0.01]
model_B.add(
Dense(nn_hidden_size[0],
input_dim=nn_data_dim,
W_regularizer=l2(nn_reg[0]),
activity_regularizer=activity_l2(nn_areg[0])))
model_B.add(Dropout(nn_drop_rate[0]))
model_B.add(
Dense(nn_hidden_size[1],
W_regularizer=l2(nn_reg[1]),
activity_regularizer=activity_l2(nn_areg[1])))
model_B.add(Dropout(nn_drop_rate[1]))
model_B.add(
Dense(1,
activation='linear',
W_regularizer=l2(nn_reg[2]),
activity_regularizer=activity_l2(nn_areg[2])))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model_B, show_shape=True)
graph.write_png("model.png")
return model_B
def design_model(lstm_data_dim, nn_data_dim, timesteps):
model_A = Sequential()
model_B = Sequential()
model_Combine = Sequential()
# LSTM Part
lstm_hidden_size = [20, 100]
drop_out_rate = [0.5, 0.5]
model_A.add(LSTM(lstm_hidden_size[0], return_sequences=True, input_shape=(timesteps, lstm_data_dim)))
model_A.add(Dropout(drop_out_rate[0])) # return_sequences=True means output cell state C at each LSTM sequence
model_A.add(LSTM(lstm_hidden_size[1], return_sequences=False))
model_A.add(Dropout(drop_out_rate[1])) # return_sequence=False means output only last cell state C in LSTM sequence
model_A.add(Dense(1, activation='linear'))
# NN Part
nn_hidden_size = [20, 20]
nn_drop_rate = [0.2, 0.2]
model_B.add(Dense(nn_hidden_size[0], input_dim=nn_data_dim))
model_B.add(Dropout(nn_drop_rate[0]))
model_B.add(Dense(nn_hidden_size[1]))
model_B.add(Dropout(nn_drop_rate[1]))
model_B.add(Dense(1, activation='linear'))
# Merge and Final Layer
model_Combine.add(Merge([model_A, model_B], mode='concat'))
model_Combine.add(Dense(1, activation='linear'))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model_Combine, show_shape=True)
graph.write_png("model.png")
return model_Combine
def draw_model(model):
from IPython.display import SVG
from keras.utils.visualize_util import to_graph
from keras.utils.visualize_util import plot
SVG(to_graph(model).create(prog='dot', format='svg'))
plot(model, to_file='UFCNN/UFCNN_1.png')
def design_model(data_dim, timesteps):
model = Sequential()
output_hidden_size = [100, 100]
drop_out_rate = [0.5, 0.5]
model.add(
LSTM(output_hidden_size[0],
return_sequences=True,
input_shape=(timesteps, data_dim)))
model.add(
Dropout(drop_out_rate[0])
) # return_sequences=True means output cell state C at each LSTM sequence
model.add(LSTM(output_hidden_size[1], return_sequences=False))
model.add(
Dropout(drop_out_rate[1])
) # return_sequence=False means output only last cell state C in LSTM sequence
# final dense layer
model.add(Dense(1, activation='linear'))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model, show_shape=True)
graph.write_png("model.png")
return model
def design_model_A(lstm_data_dim, nn_data_dim, timesteps):
model_A = Sequential()
model_B = Sequential()
model_Combine = Sequential()
# LSTM Part
lstm_hidden_size = [40, 100]
drop_out_rate = [0.6, 0.5]
reg = [0.01]
areg = [0.01]
# unfortunately regularization is not implemented for LSTMs
model_A.add(
LSTM(lstm_hidden_size[0],
return_sequences=True,
input_shape=(timesteps, lstm_data_dim)))
model_A.add(
Dropout(drop_out_rate[0])
) # return_sequences=True means output cell state C at each LSTM sequence
model_A.add(LSTM(lstm_hidden_size[1], return_sequences=False))
model_A.add(
Dropout(drop_out_rate[1])
) # return_sequence=False means output only last cell state C in LSTM sequence
model_A.add(
Dense(1,
activation='linear',
W_regularizer=l2(reg[0]),
activity_regularizer=activity_l2(areg[0])))
# NN Part
nn_hidden_size = [40, 40]
nn_drop_rate = [0.5, 0.5]
nn_reg = [0.01, 0.01, 0.01]
nn_areg = [0.01, 0.01, 0.01]
model_B.add(
Dense(nn_hidden_size[0],
input_dim=nn_data_dim,
W_regularizer=l2(nn_reg[0]),
activity_regularizer=activity_l2(nn_areg[0])))
model_B.add(Dropout(nn_drop_rate[0]))
model_B.add(
Dense(nn_hidden_size[1],
W_regularizer=l2(nn_reg[1]),
activity_regularizer=activity_l2(nn_areg[1])))
model_B.add(Dropout(nn_drop_rate[1]))
model_B.add(
Dense(1,
activation='linear',
W_regularizer=l2(nn_reg[2]),
activity_regularizer=activity_l2(nn_areg[2])))
# Merge and Final Layer
model_Combine.add(Merge([model_A, model_B], mode='concat'))
model_Combine.add(Dense(1, activation='linear'))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model_Combine, show_shape=True)
graph.write_png("model.png")
return model_Combine
def design_model_lstm(lstm_data_dim, nn_data_dim, timesteps):
model_A = Sequential()
# LSTM Part
lstm_hidden_size = [20, 100]
drop_out_rate = [0.5, 0.5]
reg = [0.01]
areg = [0.01]
# unfortunately regularization is not implemented for LSTMs
model_A.add(
LSTM(lstm_hidden_size[0],
return_sequences=True,
input_shape=(timesteps, lstm_data_dim)))
model_A.add(
Dropout(drop_out_rate[0])
) # return_sequences=True means output cell state C at each LSTM sequence
model_A.add(LSTM(lstm_hidden_size[1], return_sequences=False))
model_A.add(
Dropout(drop_out_rate[1])
) # return_sequence=False means output only last cell state C in LSTM sequence
model_A.add(
Dense(1,
activation='linear',
W_regularizer=l2(reg[0]),
activity_regularizer=activity_l2(areg[0])))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model_A, show_shape=True)
graph.write_png("model.png")
return model_A
def draw_model(model):
from IPython.display import SVG
from keras.utils.visualize_util import to_graph
from keras.utils.visualize_util import plot
#graph = to_graph(model, show_shape=True)
#graph.write_png("UFCNN_1.png")
SVG(to_graph(model).create(prog='dot', format='svg'))
plot(model, to_file='UFCNN_1.png')
def draw_model(model):
from IPython.display import SVG
from keras.utils.visualize_util import to_graph
from keras.utils.visualize_util import plot
#graph = to_graph(model, show_shape=True)
#graph.write_png("UFCNN_1.png")
SVG(to_graph(model).create(prog='dot', format='svg'))
plot(model, to_file='UFCNN_1.png')
def design_model_nn(lstm_data_dim, nn_data_dim, timesteps):
model_B = Sequential()
# NN Part
nn_hidden_size = [50, 50]
nn_drop_rate = [0.4, 0.4]
nn_reg = [0.01, 0.01, 0.01]
nn_areg = [0.01, 0.01, 0.01]
model_B.add(Dense(nn_hidden_size[0], input_dim=nn_data_dim, W_regularizer=l2(nn_reg[0]), activity_regularizer=activity_l2(nn_areg[0])))
model_B.add(Dropout(nn_drop_rate[0]))
model_B.add(Dense(nn_hidden_size[1], W_regularizer=l2(nn_reg[1]), activity_regularizer=activity_l2(nn_areg[1])))
model_B.add(Dropout(nn_drop_rate[1]))
model_B.add(Dense(1, activation='linear', W_regularizer=l2(nn_reg[2]), activity_regularizer=activity_l2(nn_areg[2])))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model_B, show_shape=True)
graph.write_png("model.png")
return model_B
def design_model_lstm(lstm_data_dim, nn_data_dim, timesteps):
model_A = Sequential()
# LSTM Part
lstm_hidden_size = [20, 100]
drop_out_rate = [0.5, 0.5]
reg = [0.01]
areg = [0.01]
# unfortunately regularization is not implemented for LSTMs
model_A.add(LSTM(lstm_hidden_size[0], return_sequences=True, input_shape=(timesteps, lstm_data_dim)))
model_A.add(Dropout(drop_out_rate[0])) # return_sequences=True means output cell state C at each LSTM sequence
model_A.add(LSTM(lstm_hidden_size[1], return_sequences=False))
model_A.add(Dropout(drop_out_rate[1])) # return_sequence=False means output only last cell state C in LSTM sequence
model_A.add(Dense(1, activation='linear', W_regularizer=l2(reg[0]), activity_regularizer=activity_l2(areg[0])))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model_A, show_shape=True)
graph.write_png("model.png")
return model_A
def design_model(lstm_data_dim, nn_data_dim, timesteps):
model_A = Sequential()
model_B = Sequential()
model_Combine = Sequential()
# LSTM Part
lstm_hidden_size = [40, 100]
drop_out_rate = [0.6, 0.5]
reg = [0.01]
areg = [0.01]
# unfortunately regularization is not implemented for LSTMs
model_A.add(LSTM(lstm_hidden_size[0], return_sequences=True, input_shape=(timesteps, lstm_data_dim)))
model_A.add(Dropout(drop_out_rate[0])) # return_sequences=True means output cell state C at each LSTM sequence
model_A.add(LSTM(lstm_hidden_size[1], return_sequences=False))
model_A.add(Dropout(drop_out_rate[1])) # return_sequence=False means output only last cell state C in LSTM sequence
model_A.add(Dense(1, activation='linear', W_regularizer=l2(reg[0]), activity_regularizer=activity_l2(areg[0])))
# NN Part
nn_hidden_size = [40, 40]
nn_drop_rate = [0.5, 0.5]
nn_reg = [0.01, 0.01, 0.01]
nn_areg = [0.01, 0.01, 0.01]
model_B.add(Dense(nn_hidden_size[0], input_dim=nn_data_dim, W_regularizer=l2(nn_reg[0]), activity_regularizer=activity_l2(nn_areg[0])))
model_B.add(Dropout(nn_drop_rate[0]))
model_B.add(Dense(nn_hidden_size[1], W_regularizer=l2(nn_reg[1]), activity_regularizer=activity_l2(nn_areg[1])))
model_B.add(Dropout(nn_drop_rate[1]))
model_B.add(Dense(1, activation='linear', W_regularizer=l2(nn_reg[2]), activity_regularizer=activity_l2(nn_areg[2])))
# Merge and Final Layer
model_Combine.add(Merge([model_A, model_B], mode='concat'))
model_Combine.add(Dense(1, activation='linear'))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model_Combine, show_shape=True)
graph.write_png("model.png")
return model_Combine
return_sequences=True,
input_shape=(timesteps, data_dim)))
model_A.add(
Dropout(drop_out_rate[0])
) # return_sequences=True means output cell state C at each LSTM sequence
model_A.add(LSTM(lstm_hidden_size[1], return_sequences=False))
model_A.add(
Dropout(drop_out_rate[1])
) # return_sequence=False means output only last cell state C in LSTM sequence
model_A.add(Dense(1, activation='linear'))
# NN Part
in_dimension = 3
nn_hidden_size = [100, 100]
nn_drop_rate = [0.2, 0.2]
model_B.add(Dense(nn_hidden_size[0], input_dim=in_dimension))
model_B.add(Dropout(nn_drop_rate[0]))
model_B.add(Dense(nn_hidden_size[1]))
model_B.add(Dropout(nn_drop_rate[1]))
model_B.add(Dense(1, activation='linear'))
model_Combine.add(Merge([model_A, model_B], mode='concat'))
model_Combine.add(Dense(1, activation='linear'))
model_Combine.compile(loss="mse", optimizer="rmsprop")
# output the model
from keras.utils.visualize_util import plot, to_graph
graph = to_graph(model_Combine, show_shape=True)
graph.write_png("rnnandnn.png")
def draw_model(model, model_name):
# Plot the network (requires pydot and graphviz)
from keras.utils.visualize_util import to_graph
open('model.svg',
'w').write(to_graph(model).create(prog='dot', format='svg'))
def run_network_keras(config, output_file_name=None):
for line in config:
print line, config[line]
print
# Unpack configs required
learning_params = config['LearningParameters']
net_structure = config['NetworkStructure']
fitting_params = config['TrainingParameters']
file_config = config['FileConfig']
out_directory = file_config['output_directory']
if output_file_name is None: # Use the config files name only if we're not provided one here.
output_file_name = file_config['output_file_name']
train_data = get_training_data(config['DatabaseConfig'], config['PreprocessingConfig'], config['FileConfig'])
# Training data comes packed in to dictionaries to avoid needing to return a whole pile of values.
train_in = train_data['train_in']
train_out = train_data['train_out']
test_in = train_data['test_in']
test_out = train_data['test_out']
galaxy_ids_test = train_data['galaxy_ids_test']
out_normaliser = train_data['out_normaliser']
# Some general statistics of the dataset
mean_in = train_data['mean_in']
mean_out = train_data['mean_out']
stddev_in = train_data['stddev_in']
stddev_out = train_data['stddev_out']
min_out = train_data['min_out']
max_out = train_data['max_out']
LOG.info('Dataset shape train')
LOG.info('{0}'.format(np.shape(train_in)))
LOG.info('{0}'.format(np.shape(train_out)))
LOG.info('Dataset shape test')
LOG.info('{0}'.format(np.shape(test_in)))
LOG.info('{0}'.format(np.shape(test_out)))
LOG.info('Compiling neural network model')
if net_structure['optimiser'] == 'sgd':
optimiser = SGD(lr=learning_params['learning_rate'], momentum=learning_params['momentum'], decay=learning_params['decay'], nesterov=True)
else:
optimiser = net_structure['optimiser']
print optimiser
input_dim = len(train_in[0])
try:
output_dim = len(train_out[0])
except:
output_dim = 1
model = build_network(net_structure, input_dim, output_dim, optimiser)
LOG.info("Compiled.")
# Train the model each generation and show predictions against the validation dataset
history_log = History_Log()
trained = False
total_epoch = 0
epoch_history = []
differences = []
lowest_val_loss = 99999
lowest_val_loss_weights = None # Best weight configuration with lowest validation loss
lowest_val_loss_epoch = 0
while not trained:
LOG.info('epoch {0} / {1}'.format(total_epoch, fitting_params['max_epochs']))
model.fit(train_in, train_out, batch_size=fitting_params['batch_size'],
nb_epoch=fitting_params['epochs_per_fit'],
validation_split=fitting_params['validation_split'],
show_accuracy=True, verbose=False, callbacks=[history_log])
LOG.info('{0}'.format(history_log.epoch_data))
total_epoch += fitting_params['epochs_per_fit']
if np.isnan(history_log.epoch_data['val_loss']):
raise FuckingNaN("Nan'd")
# If the val loss is lower, save the weights
if history_log.epoch_data['val_loss'] < lowest_val_loss:
# We have something with lower validation loss.
lowest_val_loss = history_log.epoch_data['val_loss']
lowest_val_loss_weights = weights_to_list(model)
lowest_val_loss_epoch = total_epoch
# Predict a test sample (change 1 to any other value to test more than 1)
# and use it to track how the network's output for this/these test(s) changes over time.
prediction = model.predict(np.array(test_in[:3]))
differences.append(prediction)
epoch_history.append(history_log.epoch_data)
if history_log.epoch_data['val_loss'] < 0.001 or total_epoch >= fitting_params['max_epochs']:
trained = True
differences = np.array(differences) # Need special np indexing on this later
if not out_directory:
out_directory = os.getcwd()
if not os.path.exists(out_directory):
os.makedirs(out_directory)
if not os.path.exists('{0}/graph'.format(out_directory)):
os.mkdir('{0}/graph'.format(out_directory))
print '{0}/graph'.format(out_directory)
graph_out = '{0}/graph/{1}'.format(out_directory, output_file_name)
# Save network weights, graph of network and loss over epoch graph.
to_graph(model).write_svg("{0}_Graph.svg".format(graph_out))
model.save_weights('{0}_weights.h5'.format(graph_out), overwrite=True)
save_mean_convergence_graph('{0}_convergence'.format(graph_out),differences, mean_out, min_out, max_out, fitting_params['epochs_per_fit'])
save_graph(epoch_history, '{0}_loss'.format(graph_out), fitting_params['epochs_per_fit'])
# do 30 tests on the network's final weights.
test_results, test_means, test_std = do_tests(model, test_in, fitting_params['num_tests'])
# evaluate the network on its final weights
evaluation = {'At end of training:': model.evaluate(test_in, test_out, 1000, True, True)}
# do 30 tests on the lowest validation loss weights.
model = weight_from_list(model, lowest_val_loss_weights)
model.save_weights('{0}_best_weights.h5'.format(graph_out), overwrite=True)
val_test_results, val_test_means, val_test_std = do_tests(model, test_in, fitting_params['num_tests'])
# evaluate the network on the lowest validation loss weights
evaluation['Best validation loss:'] = model.evaluate(test_in, test_out, 1000, True, True)
with open('{0}/{1}.txt'.format(out_directory, output_file_name), 'w') as f:
write_summary_data(f,
config,
evaluation,
epoch_history,
mean_in,
stddev_in,
mean_out,
stddev_out,
(lowest_val_loss_epoch, lowest_val_loss)
)
f.write('\n\n\n\n----------TEST DATA FOR FINAL MODEL----------\n\n\n\n')
write_test_data(f,
test_results,
test_in,
test_out,
test_means,
test_std,
galaxy_ids_test,
out_normaliser,
config['PreprocessingConfig']['single_output']
)
f.write('\n\n\n\n----------TEST DATA FOR BEST VALIDATION LOSS MODEL----------\n\n\n\n')
write_test_data(f,
val_test_results,
test_in,
test_out,
val_test_means,
val_test_std,
galaxy_ids_test,
out_normaliser,
config['PreprocessingConfig']['single_output'])
def run_network_keras(config, output_file_name=None):
for line in config:
print line, config[line]
print
# Unpack configs required
learning_params = config['LearningParameters']
net_structure = config['NetworkStructure']
fitting_params = config['TrainingParameters']
file_config = config['FileConfig']
out_directory = file_config['output_directory']
if output_file_name is None: # Use the config files name only if we're not provided one here.
output_file_name = file_config['output_file_name']
train_data = get_training_data(config['DatabaseConfig'],
config['PreprocessingConfig'],
config['FileConfig'])
# Training data comes packed in to dictionaries to avoid needing to return a whole pile of values.
train_in = train_data['train_in']
train_out = train_data['train_out']
test_in = train_data['test_in']
test_out = train_data['test_out']
galaxy_ids_test = train_data['galaxy_ids_test']
out_normaliser = train_data['out_normaliser']
# Some general statistics of the dataset
mean_in = train_data['mean_in']
mean_out = train_data['mean_out']
stddev_in = train_data['stddev_in']
stddev_out = train_data['stddev_out']
min_out = train_data['min_out']
max_out = train_data['max_out']
LOG.info('Dataset shape train')
LOG.info('{0}'.format(np.shape(train_in)))
LOG.info('{0}'.format(np.shape(train_out)))
LOG.info('Dataset shape test')
LOG.info('{0}'.format(np.shape(test_in)))
LOG.info('{0}'.format(np.shape(test_out)))
LOG.info('Compiling neural network model')
if net_structure['optimiser'] == 'sgd':
optimiser = SGD(lr=learning_params['learning_rate'],
momentum=learning_params['momentum'],
decay=learning_params['decay'],
nesterov=True)
else:
optimiser = net_structure['optimiser']
print optimiser
input_dim = len(train_in[0])
try:
output_dim = len(train_out[0])
except:
output_dim = 1
model = build_network(net_structure, input_dim, output_dim, optimiser)
LOG.info("Compiled.")
# Train the model each generation and show predictions against the validation dataset
history_log = History_Log()
trained = False
total_epoch = 0
epoch_history = []
differences = []
lowest_val_loss = 99999
lowest_val_loss_weights = None # Best weight configuration with lowest validation loss
lowest_val_loss_epoch = 0
while not trained:
LOG.info('epoch {0} / {1}'.format(total_epoch,
fitting_params['max_epochs']))
model.fit(train_in,
train_out,
batch_size=fitting_params['batch_size'],
nb_epoch=fitting_params['epochs_per_fit'],
validation_split=fitting_params['validation_split'],
show_accuracy=True,
verbose=False,
callbacks=[history_log])
LOG.info('{0}'.format(history_log.epoch_data))
total_epoch += fitting_params['epochs_per_fit']
if np.isnan(history_log.epoch_data['val_loss']):
raise FuckingNaN("Nan'd")
# If the val loss is lower, save the weights
if history_log.epoch_data['val_loss'] < lowest_val_loss:
# We have something with lower validation loss.
lowest_val_loss = history_log.epoch_data['val_loss']
lowest_val_loss_weights = weights_to_list(model)
lowest_val_loss_epoch = total_epoch
# Predict a test sample (change 1 to any other value to test more than 1)
# and use it to track how the network's output for this/these test(s) changes over time.
prediction = model.predict(np.array(test_in[:3]))
differences.append(prediction)
epoch_history.append(history_log.epoch_data)
if history_log.epoch_data[
'val_loss'] < 0.001 or total_epoch >= fitting_params[
'max_epochs']:
trained = True
differences = np.array(
differences) # Need special np indexing on this later
if not out_directory:
out_directory = os.getcwd()
if not os.path.exists(out_directory):
os.makedirs(out_directory)
if not os.path.exists('{0}/graph'.format(out_directory)):
os.mkdir('{0}/graph'.format(out_directory))
print '{0}/graph'.format(out_directory)
graph_out = '{0}/graph/{1}'.format(out_directory, output_file_name)
# Save network weights, graph of network and loss over epoch graph.
to_graph(model).write_svg("{0}_Graph.svg".format(graph_out))
model.save_weights('{0}_weights.h5'.format(graph_out), overwrite=True)
save_mean_convergence_graph('{0}_convergence'.format(graph_out),
differences, mean_out, min_out, max_out,
fitting_params['epochs_per_fit'])
save_graph(epoch_history, '{0}_loss'.format(graph_out),
fitting_params['epochs_per_fit'])
# do 30 tests on the network's final weights.
test_results, test_means, test_std = do_tests(model, test_in,
fitting_params['num_tests'])
# evaluate the network on its final weights
evaluation = {
'At end of training:': model.evaluate(test_in, test_out, 1000, True,
True)
}
# do 30 tests on the lowest validation loss weights.
model = weight_from_list(model, lowest_val_loss_weights)
model.save_weights('{0}_best_weights.h5'.format(graph_out), overwrite=True)
val_test_results, val_test_means, val_test_std = do_tests(
model, test_in, fitting_params['num_tests'])
# evaluate the network on the lowest validation loss weights
evaluation['Best validation loss:'] = model.evaluate(
test_in, test_out, 1000, True, True)
with open('{0}/{1}.txt'.format(out_directory, output_file_name), 'w') as f:
write_summary_data(f, config, evaluation, epoch_history, mean_in,
stddev_in, mean_out, stddev_out,
(lowest_val_loss_epoch, lowest_val_loss))
f.write(
'\n\n\n\n----------TEST DATA FOR FINAL MODEL----------\n\n\n\n')
write_test_data(f, test_results, test_in, test_out, test_means,
test_std, galaxy_ids_test, out_normaliser,
config['PreprocessingConfig']['single_output'])
f.write(
'\n\n\n\n----------TEST DATA FOR BEST VALIDATION LOSS MODEL----------\n\n\n\n'
)
write_test_data(f, val_test_results, test_in, test_out, val_test_means,
val_test_std, galaxy_ids_test, out_normaliser,
config['PreprocessingConfig']['single_output'])
from keras.layers.advanced_activations import PReLU
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.normalization import BatchNormalization
from keras.models import Sequential
from keras.utils import visualize_util
model = Sequential()
model.add(Dense(32, input_shape=(10,)))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(32))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(32))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(2))
model.add(Activation('softmax'))
graph = visualize_util.to_graph(model, show_shape=True)
graph.write_png('/tmp/model.png')
def mnistCae():
#-----------------------------------------------------------------------------------------------
# Params
batch_size = 128
nb_epoch = 20
conv_num_filters = 16
filter_size = 3
pool_size = 2
encode_size = 16
dense_mid_size = 64
#-----------------------------------------------------------------------------------------------
# input image dimensions
img_rows, img_cols = 28, 28
# the data, shuffled and split between tran and test sets
(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train = X_train.reshape(X_train.shape[0], 1, img_rows, img_cols)
X_test = X_test.reshape(X_test.shape[0], 1, img_rows, img_cols)
X_train = X_train.astype('float32')
X_test = X_test.astype('float32')
X_train /= 255
X_test /= 255
print('X_train shape:', X_train.shape)
print(X_train.shape[0], 'train samples')
print(X_test.shape[0], 'test samples')
# Build the model
model = Sequential()
act = 'relu'
model.add(Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='valid', activation=act, input_shape=(1, img_rows, img_cols)))
model.add(Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='valid', activation=act))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Convolution2D(2*conv_num_filters, filter_size, filter_size, border_mode='valid', activation=act))
model.add(MaxPooling2D(pool_size=(pool_size, pool_size)))
model.add(Flatten())
model.add(Dense(dense_mid_size, activation=act))
model.add(Dense(encode_size, activation=act))
model.add(Dense(dense_mid_size, activation=act))
model.add(Dense(800, activation=act))
model.add(Reshape((2*conv_num_filters, 5, 5)))
model.add(UpSampling2D(size=(pool_size, pool_size)))
model.add(Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same', activation=act))
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(UpSampling2D(size=(pool_size, pool_size)))
model.add(Convolution2D(conv_num_filters, filter_size, filter_size, border_mode='same', activation=act))
model.add(ZeroPadding2D(padding=(1, 1)))
model.add(Convolution2D(1, filter_size, filter_size, border_mode='same', activation=act))
model.add(ZeroPadding2D(padding=(1, 1)))
# Show summary of model
model.summary()
model.compile(loss='mse', optimizer='adadelta')
# Show graph of model
outputDir = datetime.datetime.now().strftime("%A, %d. %B %Y %I.%M%p")
os.mkdir(outputDir)
import keras.utils.visualize_util as vutil
t = vutil.to_graph(model, recursive=True, show_shape=True).create(prog='dot', format="png")
fid = open(os.path.join(outputDir, 'graph.png'), 'wb'); fid.write(t); fid.close()
history = LossHistory(outputDir)
# Train up model
model.fit(X_train, X_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=2, validation_split=0.1, callbacks=[history])
# Show examples of reconstruction
for k in np.random.randint(0, 1000, size=10):
img = np.squeeze(X_test[k:k+1,:,:,:])
reconstruction = np.squeeze(model.predict(X_test[k:k+1,:,:,:]))
plt.figure()
plt.subplot(121)
plt.imshow(img); plt.title('Sample'); plt.colorbar()
plt.subplot(122)
plt.imshow(reconstruction); plt.title('Reconstruction'); plt.colorbar()
plt.savefig(os.path.join(outputDir, 'reconstruction example - %d.png'%k))
return
from keras.layers.advanced_activations import PReLU
from keras.layers.core import Dense, Dropout, Activation
from keras.layers.normalization import BatchNormalization
from keras.models import Sequential
from keras.utils import visualize_util
model = Sequential()
model.add(Dense(32, input_shape=(10, )))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(32))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(32))
model.add(PReLU())
model.add(BatchNormalization())
model.add(Dropout(0.5))
model.add(Dense(2))
model.add(Activation('softmax'))
graph = visualize_util.to_graph(model, show_shape=True)
graph.write_png('/tmp/model.png')
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('sigmoid'))
# model.compile(loss='binary_crossentropy', optimizer='rmsprop', class_mode='binary')
model_b = Sequential()
model_b.add(Dense(287, init='uniform', input_shape=(sequence_length,)))
model_b.add(Dense(32, init='uniform'))
model_b.add(Activation('relu'))
model_b.add(Dense(2, init='uniform'))
model_b.compile(loss='binary_crossentropy', optimizer='rmsprop', class_mode='binary')
decoder = Sequential()
decoder.add(Merge([model, model_b], mode='concat'))
decoder.add(Dense(2, activation='softmax'))
decoder.compile(loss='binary_crossentropy', optimizer='rmsprop', class_mode='binary')
# Training model
# ==================================================
print ("Drawing graph")
graph = to_graph(decoder, show_shape=True)
graph.write_png("model.png")
print ("Training model")
decoder.fit([x_shuffled, x_pos], y_shuffled, batch_size=batch_size,
nb_epoch=num_epochs, show_accuracy=True,
validation_split=val_split, verbose=2)
