def run(gParameters):
print('Params:', gParameters)
file_train = gParameters['train_data']
file_test = gParameters['test_data']
url = gParameters['data_url']
train_file = data_utils.get_file(file_train,
url + file_train,
cache_subdir='Pilot1')
test_file = data_utils.get_file(file_test,
url + file_test,
cache_subdir='Pilot1')
X_train, Y_train, X_test, Y_test = load_data(train_file, test_file,
gParameters)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Y_train shape:', Y_train.shape)
print('Y_test shape:', Y_test.shape)
x_train_len = X_train.shape[1]
# this reshaping is critical for the Conv1D to work
X_train = np.expand_dims(X_train, axis=2)
X_test = np.expand_dims(X_test, axis=2)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
model = Sequential()
layer_list = list(range(0, len(gParameters['conv']), 3))
for l, i in enumerate(layer_list):
filters = gParameters['conv'][i]
filter_len = gParameters['conv'][i + 1]
stride = gParameters['conv'][i + 2]
print(int(i / 3), filters, filter_len, stride)
if gParameters['pool']:
pool_list = gParameters['pool']
if type(pool_list) != list:
pool_list = list(pool_list)
if filters <= 0 or filter_len <= 0 or stride <= 0:
break
if 'locally_connected' in gParameters:
model.add(
LocallyConnected1D(filters,
filter_len,
strides=stride,
padding='valid',
input_shape=(x_train_len, 1)))
else:
#input layer
if i == 0:
model.add(
Conv1D(filters=filters,
kernel_size=filter_len,
strides=stride,
padding='valid',
input_shape=(x_train_len, 1)))
else:
model.add(
Conv1D(filters=filters,
kernel_size=filter_len,
strides=stride,
padding='valid'))
model.add(Activation(gParameters['activation']))
if gParameters['pool']:
model.add(MaxPooling1D(pool_size=pool_list[int(i / 3)]))
model.add(Flatten())
for layer in gParameters['dense']:
if layer:
model.add(Dense(layer))
model.add(Activation(gParameters['activation']))
# This has to be disabled for tensorrt otherwise I am getting an error
if False and gParameters['drop']:
model.add(Dropout(gParameters['drop']))
#model.add(Dense(gParameters['classes']))
#model.add(Activation(gParameters['out_act']), name='activation_5')
model.add(
Dense(gParameters['classes'],
activation=gParameters['out_act'],
name='activation_5'))
#Reference case
#model.add(Conv1D(filters=128, kernel_size=20, strides=1, padding='valid', input_shape=(P, 1)))
#model.add(Activation('relu'))
#model.add(MaxPooling1D(pool_size=1))
#model.add(Conv1D(filters=128, kernel_size=10, strides=1, padding='valid'))
#model.add(Activation('relu'))
#model.add(MaxPooling1D(pool_size=10))
#model.add(Flatten())
#model.add(Dense(200))
#model.add(Activation('relu'))
#model.add(Dropout(0.1))
#model.add(Dense(20))
#model.add(Activation('relu'))
#model.add(Dropout(0.1))
#model.add(Dense(CLASSES))
#model.add(Activation('softmax'))
kerasDefaults = p1_common.keras_default_config()
# Define optimizer
optimizer = p1_common_keras.build_optimizer(gParameters['optimizer'],
gParameters['learning_rate'],
kerasDefaults)
model.summary()
for layer in model.layers:
print(layer.name)
print([x.op.name for x in model.outputs])
model.compile(loss=gParameters['loss'],
optimizer=optimizer,
metrics=[gParameters['metrics']])
output_dir = gParameters['save']
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# calculate trainable and non-trainable params
gParameters.update(compute_trainable_params(model))
# set up a bunch of callbacks to do work during model training..
model_name = gParameters['model_name']
path = '{}/{}.autosave.model.h5'.format(output_dir, model_name)
# checkpointer = ModelCheckpoint(filepath=path, verbose=1, save_weights_only=False, save_best_only=True)
csv_logger = CSVLogger('{}/training.log'.format(output_dir))
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=10,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
candleRemoteMonitor = CandleRemoteMonitor(params=gParameters)
timeoutMonitor = TerminateOnTimeOut(TIMEOUT)
history = model.fit(
X_train,
Y_train,
batch_size=gParameters['batch_size'],
epochs=2, #gParameters['epochs'],
verbose=1,
validation_data=(X_test, Y_test),
callbacks=[csv_logger, reduce_lr, candleRemoteMonitor, timeoutMonitor])
score = model.evaluate(X_test, Y_test, verbose=0)
#Begin tensorrt code
config = {
# Where to save models (Tensorflow + TensorRT)
"graphdef_file":
"/gpfs/jlse-fs0/users/pbalapra/tensorrt/Benchmarks/Pilot1/NT3/nt3.pb",
"frozen_model_file":
"/gpfs/jlse-fs0/users/pbalapra/tensorrt/Benchmarks/Pilot1/NT3/nt3_frozen_model.pb",
"snapshot_dir":
"/gpfs/jlse-fs0/users/pbalapra/tensorrt/Benchmarks/Pilot1/NT3/snapshot",
"engine_save_dir":
"/gpfs/jlse-fs0/users/pbalapra/tensorrt/Benchmarks/Pilot1/NT3",
# Needed for TensorRT
"inference_batch_size": 1, # inference batch size
"input_layer":
"conv1d_1", # name of the input tensor in the TF computational graph
"out_layer":
"activation_5/Softmax", # name of the output tensorf in the TF conputational graph
"output_size": 2, # number of classes in output (5)
"precision":
"fp32" # desired precision (fp32, fp16) "test_image_path" : "/home/data/val/roses"
}
# Now, let's use the Tensorflow backend to get the TF graphdef and frozen graph
K.set_learning_phase(0)
sess = K.get_session()
saver = saver_lib.Saver(write_version=saver_pb2.SaverDef.V2)
# save model weights in TF checkpoint
checkpoint_path = saver.save(sess,
config['snapshot_dir'],
global_step=0,
latest_filename='checkpoint_state')
# remove nodes not needed for inference from graph def
train_graph = sess.graph
inference_graph = tf.graph_util.remove_training_nodes(
train_graph.as_graph_def())
#print(len([n.name for n in tf.get_default_graph().as_graph_def().node]))
# write the graph definition to a file.
# You can view this file to see your network structure and
# to determine the names of your network's input/output layers.
graph_io.write_graph(inference_graph, '.', config['graphdef_file'])
# specify which layer is the output layer for your graph.
# In this case, we want to specify the softmax layer after our
# last dense (fully connected) layer.
out_names = config['out_layer']
# freeze your inference graph and save it for later! (Tensorflow)
freeze_graph.freeze_graph(config['graphdef_file'], '', False,
checkpoint_path, out_names, "save/restore_all",
"save/Const:0", config['frozen_model_file'],
False, "")
if False:
print('Test score:', score[0])
print('Test accuracy:', score[1])
# serialize model to JSON
model_json = model.to_json()
with open("{}/{}.model.json".format(output_dir, model_name),
"w") as json_file:
json_file.write(model_json)
# serialize model to YAML
model_yaml = model.to_yaml()
with open("{}/{}.model.yaml".format(output_dir, model_name),
"w") as yaml_file:
yaml_file.write(model_yaml)
# serialize weights to HDF5
model.save_weights("{}/{}.weights.h5".format(output_dir, model_name))
print("Saved model to disk")
# load json and create model
json_file = open('{}/{}.model.json'.format(output_dir, model_name),
'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model_json = model_from_json(loaded_model_json)
# load yaml and create model
yaml_file = open('{}/{}.model.yaml'.format(output_dir, model_name),
'r')
loaded_model_yaml = yaml_file.read()
yaml_file.close()
loaded_model_yaml = model_from_yaml(loaded_model_yaml)
# load weights into new model
loaded_model_json.load_weights('{}/{}.weights.h5'.format(
output_dir, model_name))
print("Loaded json model from disk")
# evaluate json loaded model on test data
loaded_model_json.compile(loss=gParameters['loss'],
optimizer=gParameters['optimizer'],
metrics=[gParameters['metrics']])
score_json = loaded_model_json.evaluate(X_test, Y_test, verbose=0)
print('json Test score:', score_json[0])
print('json Test accuracy:', score_json[1])
print("json %s: %.2f%%" %
(loaded_model_json.metrics_names[1], score_json[1] * 100))
# load weights into new model
loaded_model_yaml.load_weights('{}/{}.weights.h5'.format(
output_dir, model_name))
print("Loaded yaml model from disk")
# evaluate loaded model on test data
loaded_model_yaml.compile(loss=gParameters['loss'],
optimizer=gParameters['optimizer'],
metrics=[gParameters['metrics']])
score_yaml = loaded_model_yaml.evaluate(X_test, Y_test, verbose=0)
print('yaml Test score:', score_yaml[0])
print('yaml Test accuracy:', score_yaml[1])
print("yaml %s: %.2f%%" %
(loaded_model_yaml.metrics_names[1], score_yaml[1] * 100))
return history
def run(gParameters):
print ('Params:', gParameters)
file_train = gParameters['train_data']
file_test = gParameters['test_data']
url = gParameters['data_url']
'''path = '/home/orlandomelchor/Desktop/Research 2018-2019/CANDLE/'
tr_file = 'nt_train2.csv'
te_file = 'nt_train2.csv'
train_file = path + tr_file
test_file = path + te_file
X_train, Y_train, X_test, Y_test = load_data(train_file, test_file, gParameters)'''
path = '../data-05-31-2018/'
full_data_file = 'formatted_full_data.csv'
X_train, Y_train, X_test, Y_test = load_data(path+full_data_file, gParameters)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Y_train shape:', Y_train.shape)
print('Y_test shape:', Y_test.shape)
x_train_len = X_train.shape[1]
# this reshaping is critical for the Conv1D to work
model = Sequential()
for layer in gParameters['dense']:
if layer:
model.add(Dense(layer,input_shape=(x_train_len,)))
model.add(Activation(gParameters['activation']))
if gParameters['drop']:
model.add(Dropout(gParameters['drop']))
model.add(Dense(gParameters['classes']))
model.add(Activation(gParameters['out_act']))
#Reference case
#model.add(Conv1D(filters=128, kernel_size=20, strides=1, padding='valid', input_shape=(P, 1)))
#model.add(Activation('relu'))
#model.add(MaxPooling1D(pool_size=1))
#model.add(Conv1D(filters=128, kernel_size=10, strides=1, padding='valid'))
#model.add(Activation('relu'))
#model.add(MaxPooling1D(pool_size=10))
#model.add(Flatten())
#model.add(Dense(200))
#model.add(Activation('relu'))
#model.add(Dropout(0.1))
#model.add(Dense(20))
#model.add(Activation('relu'))
#model.add(Dropout(0.1))
#model.add(Dense(CLASSES))
#model.add(Activation('softmax'))
kerasDefaults = p1_common.keras_default_config()
# Define optimizer
optimizer = p1_common_keras.build_optimizer(gParameters['optimizer'],
gParameters['learning_rate'],
kerasDefaults)
model.summary()
model.compile(loss=gParameters['loss'],
optimizer=optimizer,
metrics=[gParameters['metrics']])
output_dir = gParameters['save']
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# calculate trainable and non-trainable params
gParameters.update(compute_trainable_params(model))
# set up a bunch of callbacks to do work during model training..
model_name = gParameters['model_name']
path = '{}/{}.autosave.model.h5'.format(output_dir, model_name)
# checkpointer = ModelCheckpoint(filepath=path, verbose=1, save_weights_only=False, save_best_only=True)
csv_logger = CSVLogger('{}/training.log'.format(output_dir))
reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0)
candleRemoteMonitor = CandleRemoteMonitor(params=gParameters)
timeoutMonitor = TerminateOnTimeOut(TIMEOUT)
history = model.fit(X_train, Y_train,
batch_size=gParameters['batch_size'],
epochs=gParameters['epochs'],
verbose=1,
validation_data=(X_test, Y_test),
callbacks = [csv_logger, reduce_lr, candleRemoteMonitor, timeoutMonitor])
score = model.evaluate(X_test, Y_test, verbose=0)
if True:
print('Test score:', score[0])
print('Test accuracy:', score[1])
# serialize model to JSON
model_json = model.to_json()
with open("{}/{}.model.json".format(output_dir, model_name), "w") as json_file:
json_file.write(model_json)
# serialize model to YAML
model_yaml = model.to_yaml()
with open("{}/{}.model.yaml".format(output_dir, model_name), "w") as yaml_file:
yaml_file.write(model_yaml)
# serialize model to HDF5
model.save('{}/{}_network{}.h5'.format(output_dir, model_name, i))
print("Saved model to disk")
# load json and create model
json_file = open('{}/{}.model.json'.format(output_dir, model_name), 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model_json = model_from_json(loaded_model_json)
# load yaml and create model
yaml_file = open('{}/{}.model.yaml'.format(output_dir, model_name), 'r')
loaded_model_yaml = yaml_file.read()
yaml_file.close()
loaded_model_yaml = model_from_yaml(loaded_model_yaml)
# load into new model
loaded_model_json.load_weights('{}/{}_network{}.h5'.format(output_dir, model_name, i))
print("Loaded json model from disk")
# evaluate json loaded model on test data
loaded_model_json.compile(loss=gParameters['loss'],
optimizer=gParameters['optimizer'],
metrics=[gParameters['metrics']])
score_json = loaded_model_json.evaluate(X_test, Y_test, verbose=0)
print('json Test score:', score_json[0])
print('json Test accuracy:', score_json[1])
print("json %s: %.2f%%" % (loaded_model_json.metrics_names[1], score_json[1]*100))
# load weights into new model
loaded_model_yaml.load_weights('{}/{}_network{}.h5'.format(output_dir, model_name, i))
print("Loaded yaml model from disk")
# evaluate loaded model on test data
loaded_model_yaml.compile(loss=gParameters['loss'],
optimizer=gParameters['optimizer'],
metrics=[gParameters['metrics']])
score_yaml = loaded_model_yaml.evaluate(X_test, Y_test, verbose=0)
print('yaml Test score:', score_yaml[0])
print('yaml Test accuracy:', score_yaml[1])
print("yaml %s: %.2f%%" % (loaded_model_yaml.metrics_names[1], score_yaml[1]*100))
acc_file = open('{}/{}_accuracy.txt'.format(output_dir, model_name),'w')
acc_file.write(str(round(score_yaml[1],4)*100))
acc_file.close()
return history
def main():
# Get command-line parameters
parser = get_p1b2_parser()
args = parser.parse_args()
#print('Args:', args)
# Get parameters from configuration file
fileParameters = p1b2.read_config_file(args.config_file)
#print ('Params:', fileParameters)
# Consolidate parameter set. Command-line parameters overwrite file configuration
gParameters = p1_common.args_overwrite_config(args, fileParameters)
print('Params:', gParameters)
# Construct extension to save model
ext = p1b2.extension_from_parameters(gParameters, '.keras')
logfile = args.logfile if args.logfile else args.save + ext + '.log'
p1b2.logger.info('Params: {}'.format(gParameters))
# Get default parameters for initialization and optimizer functions
kerasDefaults = p1_common.keras_default_config()
seed = gParameters['rng_seed']
# Load dataset
#(X_train, y_train), (X_test, y_test) = p1b2.load_data(gParameters, seed)
(X_train,
y_train), (X_val,
y_val), (X_test,
y_test) = p1b2.load_data_one_hot(gParameters, seed)
print("Shape X_train: ", X_train.shape)
print("Shape X_val: ", X_val.shape)
print("Shape X_test: ", X_test.shape)
print("Shape y_train: ", y_train.shape)
print("Shape y_val: ", y_val.shape)
print("Shape y_test: ", y_test.shape)
print("Range X_train --> Min: ", np.min(X_train), ", max: ",
np.max(X_train))
print("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val))
print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test))
print("Range y_train --> Min: ", np.min(y_train), ", max: ",
np.max(y_train))
print("Range y_val --> Min: ", np.min(y_val), ", max: ", np.max(y_val))
print("Range y_test --> Min: ", np.min(y_test), ", max: ", np.max(y_test))
input_dim = X_train.shape[1]
input_vector = Input(shape=(input_dim, ))
output_dim = y_train.shape[1]
# Initialize weights and learning rule
initializer_weights = p1_common_keras.build_initializer(
gParameters['initialization'], kerasDefaults, seed)
initializer_bias = p1_common_keras.build_initializer(
'constant', kerasDefaults, 0.)
activation = gParameters['activation']
# Define MLP architecture
layers = gParameters['dense']
if layers != None:
if type(layers) != list:
layers = list(layers)
for i, l in enumerate(layers):
if i == 0:
x = Dense(l,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias,
kernel_regularizer=l2(gParameters['penalty']),
activity_regularizer=l2(
gParameters['penalty']))(input_vector)
else:
x = Dense(l,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias,
kernel_regularizer=l2(gParameters['penalty']),
activity_regularizer=l2(gParameters['penalty']))(x)
if gParameters['drop']:
x = Dropout(gParameters['drop'])(x)
output = Dense(output_dim,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias)(x)
else:
output = Dense(output_dim,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias)(input_vector)
# Build MLP model
mlp = Model(outputs=output, inputs=input_vector)
p1b2.logger.debug('Model: {}'.format(mlp.to_json()))
# Define optimizer
optimizer = p1_common_keras.build_optimizer(gParameters['optimizer'],
gParameters['learning_rate'],
kerasDefaults)
# Compile and display model
mlp.compile(loss=gParameters['loss'],
optimizer=optimizer,
metrics=['accuracy'])
mlp.summary()
# Seed random generator for training
np.random.seed(seed)
mlp.fit(X_train,
y_train,
batch_size=gParameters['batch_size'],
epochs=gParameters['epochs'],
validation_data=(X_val, y_val))
# model save
#save_filepath = "model_mlp_W_" + ext
#mlp.save_weights(save_filepath)
# Evalute model on test set
y_pred = mlp.predict(X_test)
scores = p1b2.evaluate_accuracy_one_hot(y_pred, y_test)
print('Evaluation on test data:', scores)
def run(gParameters):
print('Params:', gParameters)
file_train = gParameters['train_data']
file_test = gParameters['test_data']
url = gParameters['data_url']
train_file = data_utils.get_file(file_train,
url + file_train,
cache_subdir='Pilot1')
test_file = data_utils.get_file(file_test,
url + file_test,
cache_subdir='Pilot1')
X_train, Y_train, X_test, Y_test = load_data(train_file, test_file,
gParameters)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
print('Y_train shape:', Y_train.shape)
print('Y_test shape:', Y_test.shape)
x_train_len = X_train.shape[1]
# this reshaping is critical for the Conv1D to work
X_train = np.expand_dims(X_train, axis=2)
X_test = np.expand_dims(X_test, axis=2)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
model = Sequential()
layer_list = list(range(0, len(gParameters['conv']), 3))
for l, i in enumerate(layer_list):
filters = gParameters['conv'][i]
filter_len = gParameters['conv'][i + 1]
stride = gParameters['conv'][i + 2]
print(int(i / 3), filters, filter_len, stride)
if gParameters['pool']:
pool_list = gParameters['pool']
if type(pool_list) != list:
pool_list = list(pool_list)
if filters <= 0 or filter_len <= 0 or stride <= 0:
break
if 'locally_connected' in gParameters:
model.add(
LocallyConnected1D(filters,
filter_len,
strides=stride,
padding='valid',
input_shape=(x_train_len, 1)))
else:
#input layer
if i == 0:
model.add(
Conv1D(filters=filters,
kernel_size=filter_len,
strides=stride,
padding='valid',
input_shape=(x_train_len, 1)))
else:
model.add(
Conv1D(filters=filters,
kernel_size=filter_len,
strides=stride,
padding='valid'))
model.add(Activation(gParameters['activation']))
if gParameters['pool']:
model.add(MaxPooling1D(pool_size=pool_list[int(i / 3)]))
model.add(Flatten())
for layer in gParameters['dense']:
if layer:
model.add(Dense(layer))
model.add(Activation(gParameters['activation']))
if gParameters['drop']:
model.add(Dropout(gParameters['drop']))
model.add(Dense(gParameters['classes']))
model.add(Activation(gParameters['out_act']))
#Reference case
#model.add(Conv1D(filters=128, kernel_size=20, strides=1, padding='valid', input_shape=(P, 1)))
#model.add(Activation('relu'))
#model.add(MaxPooling1D(pool_size=1))
#model.add(Conv1D(filters=128, kernel_size=10, strides=1, padding='valid'))
#model.add(Activation('relu'))
#model.add(MaxPooling1D(pool_size=10))
#model.add(Flatten())
#model.add(Dense(200))
#model.add(Activation('relu'))
#model.add(Dropout(0.1))
#model.add(Dense(20))
#model.add(Activation('relu'))
#model.add(Dropout(0.1))
#model.add(Dense(CLASSES))
#model.add(Activation('softmax'))
kerasDefaults = p1_common.keras_default_config()
# Define optimizer
optimizer = p1_common_keras.build_optimizer(gParameters['optimizer'],
gParameters['learning_rate'],
kerasDefaults)
model.summary()
model.compile(loss=gParameters['loss'],
optimizer=optimizer,
metrics=[gParameters['metrics']])
output_dir = gParameters['save']
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# calculate trainable and non-trainable params
gParameters.update(compute_trainable_params(model))
# set up a bunch of callbacks to do work during model training..
model_name = gParameters['model_name']
path = '{}/{}.autosave.model.h5'.format(output_dir, model_name)
# checkpointer = ModelCheckpoint(filepath=path, verbose=1, save_weights_only=False, save_best_only=True)
csv_logger = CSVLogger('{}/training.log'.format(output_dir))
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=10,
verbose=1,
mode='auto',
epsilon=0.0001,
cooldown=0,
min_lr=0)
candleRemoteMonitor = CandleRemoteMonitor(params=gParameters)
timeoutMonitor = TerminateOnTimeOut(TIMEOUT)
history = model.fit(
X_train,
Y_train,
batch_size=gParameters['batch_size'],
epochs=gParameters['epochs'],
verbose=1,
validation_data=(X_test, Y_test),
callbacks=[csv_logger, reduce_lr, candleRemoteMonitor, timeoutMonitor])
score = model.evaluate(X_test, Y_test, verbose=0)
if False:
print('Test score:', score[0])
print('Test accuracy:', score[1])
# serialize model to JSON
model_json = model.to_json()
with open("{}/{}.model.json".format(output_dir, model_name),
"w") as json_file:
json_file.write(model_json)
# serialize model to YAML
model_yaml = model.to_yaml()
with open("{}/{}.model.yaml".format(output_dir, model_name),
"w") as yaml_file:
yaml_file.write(model_yaml)
# serialize weights to HDF5
model.save_weights("{}/{}.weights.h5".format(output_dir, model_name))
print("Saved model to disk")
# load json and create model
json_file = open('{}/{}.model.json'.format(output_dir, model_name),
'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model_json = model_from_json(loaded_model_json)
# load yaml and create model
yaml_file = open('{}/{}.model.yaml'.format(output_dir, model_name),
'r')
loaded_model_yaml = yaml_file.read()
yaml_file.close()
loaded_model_yaml = model_from_yaml(loaded_model_yaml)
# load weights into new model
loaded_model_json.load_weights('{}/{}.weights.h5'.format(
output_dir, model_name))
print("Loaded json model from disk")
# evaluate json loaded model on test data
loaded_model_json.compile(loss=gParameters['loss'],
optimizer=gParameters['optimizer'],
metrics=[gParameters['metrics']])
score_json = loaded_model_json.evaluate(X_test, Y_test, verbose=0)
print('json Test score:', score_json[0])
print('json Test accuracy:', score_json[1])
print("json %s: %.2f%%" %
(loaded_model_json.metrics_names[1], score_json[1] * 100))
# load weights into new model
loaded_model_yaml.load_weights('{}/{}.weights.h5'.format(
output_dir, model_name))
print("Loaded yaml model from disk")
# evaluate loaded model on test data
loaded_model_yaml.compile(loss=gParameters['loss'],
optimizer=gParameters['optimizer'],
metrics=[gParameters['metrics']])
score_yaml = loaded_model_yaml.evaluate(X_test, Y_test, verbose=0)
print('yaml Test score:', score_yaml[0])
print('yaml Test accuracy:', score_yaml[1])
print("yaml %s: %.2f%%" %
(loaded_model_yaml.metrics_names[1], score_yaml[1] * 100))
return history
def run(gParameters):
"""
Runs the model using the specified set of parameters
Args:
gParameters: a python dictionary containing the parameters (e.g. epoch)
to run the model with.
"""
#
if 'dense' in gParameters:
dval = gParameters['dense']
if type(dval) != list:
res = list(dval)
#try:
#is_str = isinstance(dval, basestring)
#except NameError:
#is_str = isinstance(dval, str)
#if is_str:
#res = str2lst(dval)
gParameters['dense'] = res
print(gParameters['dense'])
if 'conv' in gParameters:
#conv_list = p1_common.parse_conv_list(gParameters['conv'])
#cval = gParameters['conv']
#try:
#is_str = isinstance(cval, basestring)
#except NameError:
#is_str = isinstance(cval, str)
#if is_str:
#res = str2lst(cval)
#gParameters['conv'] = res
print('Conv input', gParameters['conv'])
# print('Params:', gParameters)
# Construct extension to save model
ext = p1b3.extension_from_parameters(gParameters, '.keras')
logfile = gParameters['logfile'] if gParameters[
'logfile'] else gParameters['save'] + ext + '.log'
fh = logging.FileHandler(logfile)
fh.setFormatter(
logging.Formatter("[%(asctime)s %(process)d] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"))
fh.setLevel(logging.DEBUG)
sh = logging.StreamHandler()
sh.setFormatter(logging.Formatter(''))
sh.setLevel(logging.DEBUG if gParameters['verbose'] else logging.INFO)
p1b3.logger.setLevel(logging.DEBUG)
p1b3.logger.addHandler(fh)
p1b3.logger.addHandler(sh)
p1b3.logger.info('Params: {}'.format(gParameters))
# Get default parameters for initialization and optimizer functions
kerasDefaults = p1_common.keras_default_config()
seed = gParameters['rng_seed']
# Build dataset loader object
loader = p1b3.DataLoader(
seed=seed,
dtype=gParameters['datatype'],
val_split=gParameters['validation_split'],
test_cell_split=gParameters['test_cell_split'],
cell_features=gParameters['cell_features'],
drug_features=gParameters['drug_features'],
feature_subsample=gParameters['feature_subsample'],
scaling=gParameters['scaling'],
scramble=gParameters['scramble'],
min_logconc=gParameters['min_logconc'],
max_logconc=gParameters['max_logconc'],
subsample=gParameters['subsample'],
category_cutoffs=gParameters['category_cutoffs'])
# Initialize weights and learning rule
initializer_weights = p1_common_keras.build_initializer(
gParameters['initialization'], kerasDefaults, seed)
initializer_bias = p1_common_keras.build_initializer(
'constant', kerasDefaults, 0.)
activation = gParameters['activation']
# Define model architecture
gen_shape = None
out_dim = 1
model = Sequential()
if 'dense' in gParameters: # Build dense layers
for layer in gParameters['dense']:
if layer:
model.add(
Dense(layer,
input_dim=loader.input_dim,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias))
if gParameters['batch_normalization']:
model.add(BatchNormalization())
model.add(Activation(gParameters['activation']))
if gParameters['drop']:
model.add(Dropout(gParameters['drop']))
else: # Build convolutional layers
gen_shape = 'add_1d'
layer_list = list(range(0, len(gParameters['conv'])))
lc_flag = False
if 'locally_connected' in gParameters:
lc_flag = True
for l, i in enumerate(layer_list):
if i == 0:
add_conv_layer(model,
gParameters['conv'][i],
input_dim=loader.input_dim,
locally_connected=lc_flag)
else:
add_conv_layer(model,
gParameters['conv'][i],
locally_connected=lc_flag)
if gParameters['batch_normalization']:
model.add(BatchNormalization())
model.add(Activation(gParameters['activation']))
if gParameters['pool']:
model.add(MaxPooling1D(pool_size=gParameters['pool']))
model.add(Flatten())
model.add(Dense(out_dim))
# Define optimizer
optimizer = p1_common_keras.build_optimizer(gParameters['optimizer'],
gParameters['learning_rate'],
kerasDefaults)
# Compile and display model
model.compile(loss=gParameters['loss'], optimizer=optimizer)
model.summary()
p1b3.logger.debug('Model: {}'.format(model.to_json()))
train_gen = p1b3.DataGenerator(
loader,
batch_size=gParameters['batch_size'],
shape=gen_shape,
name='train_gen',
cell_noise_sigma=gParameters['cell_noise_sigma']).flow()
val_gen = p1b3.DataGenerator(loader,
partition='val',
batch_size=gParameters['batch_size'],
shape=gen_shape,
name='val_gen').flow()
val_gen2 = p1b3.DataGenerator(loader,
partition='val',
batch_size=gParameters['batch_size'],
shape=gen_shape,
name='val_gen2').flow()
test_gen = p1b3.DataGenerator(loader,
partition='test',
batch_size=gParameters['batch_size'],
shape=gen_shape,
name='test_gen').flow()
train_steps = int(loader.n_train / gParameters['batch_size'])
val_steps = int(loader.n_val / gParameters['batch_size'])
test_steps = int(loader.n_test / gParameters['batch_size'])
if 'train_steps' in gParameters:
train_steps = gParameters['train_steps']
if 'val_steps' in gParameters:
val_steps = gParameters['val_steps']
if 'test_steps' in gParameters:
test_steps = gParameters['test_steps']
checkpointer = ModelCheckpoint(filepath=gParameters['save'] + '.model' +
ext + '.h5',
save_best_only=True)
progbar = MyProgbarLogger(train_steps * gParameters['batch_size'])
loss_history = MyLossHistory(
progbar=progbar,
val_gen=val_gen2,
test_gen=test_gen,
val_steps=val_steps,
test_steps=test_steps,
metric=gParameters['loss'],
category_cutoffs=gParameters['category_cutoffs'],
ext=ext,
pre=gParameters['save'])
# Seed random generator for training
np.random.seed(seed)
candleRemoteMonitor = CandleRemoteMonitor(params=gParameters)
history = model.fit_generator(
train_gen,
train_steps,
epochs=gParameters['epochs'],
validation_data=val_gen,
validation_steps=val_steps,
verbose=0,
callbacks=[checkpointer, loss_history, progbar, candleRemoteMonitor],
pickle_safe=True,
workers=gParameters['workers'])
p1b3.logger.removeHandler(fh)
p1b3.logger.removeHandler(sh)
return history