def run(gParameters): # Construct extension to save model ext = p1b2.extension_from_parameters(gParameters, '.keras') candle.verify_path(gParameters['save_path']) prefix = '{}{}'.format(gParameters['save_path'], ext) logfile = gParameters['logfile'] if gParameters[ 'logfile'] else prefix + '.log' #candle.set_up_logger(logfile, p1b2.logger, gParameters['verbose']) #p1b2.logger.info('Params: {}'.format(gParameters)) # Get default parameters for initialization and optimizer functions kerasDefaults = candle.keras_default_config() seed = gParameters['rng_seed'] # Load dataset (X_train, y_train), (X_test, y_test) = p1b2.load_data2(gParameters, seed) print("Shape X_test: ", X_test.shape) print("Shape y_test: ", y_test.shape) print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test)) print("Range y_test --> Min: ", np.min(y_test), ", max: ", np.max(y_test)) # Define optimizer optimizer = candle.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) # load json and create model json_file = open('p1b2.model.json', 'r') loaded_model_json = json_file.read() json_file.close() loaded_model_json = model_from_json(loaded_model_json) # load weights into new model loaded_model_json.load_weights('p1b2.model.h5') print("Loaded model from disk") # evaluate loaded model on test data loaded_model_json.compile(loss=gParameters['loss'], optimizer=optimizer, metrics=['accuracy']) y_pred = loaded_model_json.predict(X_test) scores = p1b2.evaluate_accuracy_one_hot(y_pred, y_test) print('Evaluation on test data:', scores)
def run(gParameters): print('Params:', gParameters) file_train = gParameters['train_data'] file_test = gParameters['test_data'] url = gParameters['data_url'] train_file = candle.get_file(file_train, url + file_train, cache_subdir='Pilot1') test_file = candle.get_file(file_test, url + file_test, cache_subdir='Pilot1') X_train, Y_train, X_test, Y_test = load_data(train_file, test_file, gParameters) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) print('Y_train shape:', Y_train.shape) print('Y_test shape:', Y_test.shape) x_train_len = X_train.shape[1] # this reshaping is critical for the Conv1D to work X_train = np.expand_dims(X_train, axis=2) X_test = np.expand_dims(X_test, axis=2) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) # Have to add this line or else ALW on 2020-11-15 finds Supervisor jobs using canonically CANDLE-compliant model scripts die as soon as a particular task is used a second time: # EXCEPTION: # InvalidArgumentError() ... # File "<string>", line 23, in <module> # File "/gpfs/alpine/med106/world-shared/candle/2020-11-11/checkouts/Supervisor/workflows/common/python/model_runner.py", line 241, in run_model # result, history = run(hyper_parameter_map, obj_return) # File "/gpfs/alpine/med106/world-shared/candle/2020-11-11/checkouts/Supervisor/workflows/common/python/model_runner.py", line 169, in run # history = pkg.run(params) # File "/gpfs/alpine/med106/world-shared/candle/2020-11-11/checkouts/Benchmarks/Pilot1/NT3/nt3_candle_wrappers_baseline_keras2.py", line 211, in run # callbacks=[csv_logger, reduce_lr, candleRemoteMonitor, timeoutMonitor]) # File "/gpfs/alpine/world-shared/med106/sw/condaenv-200408/lib/python3.6/site-packages/keras/engine/training.py", line 1178, in fit # validation_freq=validation_freq) # File "/gpfs/alpine/world-shared/med106/sw/condaenv-200408/lib/python3.6/site-packages/keras/engine/training_arrays.py", line 204, in fit_loop # outs = fit_function(ins_batch) # File "/gpfs/alpine/world-shared/med106/sw/condaenv-200408/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2979, in __call__ # return self._call(inputs) # File "/gpfs/alpine/world-shared/med106/sw/condaenv-200408/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2933, in _call # session) # File "/gpfs/alpine/world-shared/med106/sw/condaenv-200408/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2885, in _make_callable # callable_fn = session._make_callable_from_options(callable_opts) # File "/gpfs/alpine/world-shared/med106/sw/condaenv-200408/lib/python3.6/site-packages/tensorflow_core/python/client/session.py", line 1505, in _make_callable_from_options # return BaseSession._Callable(self, callable_options) # File "/gpfs/alpine/world-shared/med106/sw/condaenv-200408/lib/python3.6/site-packages/tensorflow_core/python/client/session.py", line 1460, in __init__ # session._session, options_ptr) K.clear_session() model = Sequential() layer_list = list(range(0, len(gParameters['conv']), 3)) for _, i in enumerate(layer_list): filters = gParameters['conv'][i] filter_len = gParameters['conv'][i + 1] stride = gParameters['conv'][i + 2] print(int(i / 3), filters, filter_len, stride) if gParameters['pool']: pool_list = gParameters['pool'] if type(pool_list) != list: pool_list = list(pool_list) if filters <= 0 or filter_len <= 0 or stride <= 0: break if 'locally_connected' in gParameters: model.add( LocallyConnected1D(filters, filter_len, strides=stride, padding='valid', input_shape=(x_train_len, 1))) else: # input layer if i == 0: model.add( Conv1D(filters=filters, kernel_size=filter_len, strides=stride, padding='valid', input_shape=(x_train_len, 1))) else: model.add( Conv1D(filters=filters, kernel_size=filter_len, strides=stride, padding='valid')) model.add(Activation(gParameters['activation'])) if gParameters['pool']: model.add(MaxPooling1D(pool_size=pool_list[int(i / 3)])) model.add(Flatten()) for layer in gParameters['dense']: if layer: model.add(Dense(layer)) model.add(Activation(gParameters['activation'])) if gParameters['dropout']: model.add(Dropout(gParameters['dropout'])) model.add(Dense(gParameters['classes'])) model.add(Activation(gParameters['out_activation'])) # Reference case # model.add(Conv1D(filters=128, kernel_size=20, strides=1, padding='valid', input_shape=(P, 1))) # model.add(Activation('relu')) # model.add(MaxPooling1D(pool_size=1)) # model.add(Conv1D(filters=128, kernel_size=10, strides=1, padding='valid')) # model.add(Activation('relu')) # model.add(MaxPooling1D(pool_size=10)) # model.add(Flatten()) # model.add(Dense(200)) # model.add(Activation('relu')) # model.add(Dropout(0.1)) # model.add(Dense(20)) # model.add(Activation('relu')) # model.add(Dropout(0.1)) # model.add(Dense(CLASSES)) # model.add(Activation('softmax')) kerasDefaults = candle.keras_default_config() # Define optimizer optimizer = candle.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) model.summary() model.compile(loss=gParameters['loss'], optimizer=optimizer, metrics=[gParameters['metrics']]) output_dir = gParameters['output_dir'] if not os.path.exists(output_dir): os.makedirs(output_dir) # calculate trainable and non-trainable params gParameters.update(candle.compute_trainable_params(model)) # set up a bunch of callbacks to do work during model training.. model_name = gParameters['model_name'] path = '{}/{}.autosave.model.h5'.format(output_dir, model_name) # checkpointer = ModelCheckpoint(filepath=path, verbose=1, save_weights_only=False, save_best_only=True) csv_logger = CSVLogger('{}/training.log'.format(output_dir)) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) candleRemoteMonitor = candle.CandleRemoteMonitor(params=gParameters) timeoutMonitor = candle.TerminateOnTimeOut(gParameters['timeout']) history = model.fit( X_train, Y_train, batch_size=gParameters['batch_size'], epochs=gParameters['epochs'], verbose=1, validation_data=(X_test, Y_test), callbacks=[csv_logger, reduce_lr, candleRemoteMonitor, timeoutMonitor]) score = model.evaluate(X_test, Y_test, verbose=0) if False: print('Test score:', score[0]) print('Test accuracy:', score[1]) # serialize model to JSON model_json = model.to_json() with open("{}/{}.model.json".format(output_dir, model_name), "w") as json_file: json_file.write(model_json) # serialize model to YAML model_yaml = model.to_yaml() with open("{}/{}.model.yaml".format(output_dir, model_name), "w") as yaml_file: yaml_file.write(model_yaml) # serialize weights to HDF5 model.save_weights("{}/{}.weights.h5".format(output_dir, model_name)) print("Saved model to disk") # load json and create model json_file = open('{}/{}.model.json'.format(output_dir, model_name), 'r') loaded_model_json = json_file.read() json_file.close() loaded_model_json = model_from_json(loaded_model_json) # load yaml and create model yaml_file = open('{}/{}.model.yaml'.format(output_dir, model_name), 'r') loaded_model_yaml = yaml_file.read() yaml_file.close() loaded_model_yaml = model_from_yaml(loaded_model_yaml) # load weights into new model loaded_model_json.load_weights('{}/{}.weights.h5'.format( output_dir, model_name)) print("Loaded json model from disk") # evaluate json loaded model on test data loaded_model_json.compile(loss=gParameters['loss'], optimizer=gParameters['optimizer'], metrics=[gParameters['metrics']]) score_json = loaded_model_json.evaluate(X_test, Y_test, verbose=0) print('json Test score:', score_json[0]) print('json Test accuracy:', score_json[1]) print("json %s: %.2f%%" % (loaded_model_json.metrics_names[1], score_json[1] * 100)) # load weights into new model loaded_model_yaml.load_weights('{}/{}.weights.h5'.format( output_dir, model_name)) print("Loaded yaml model from disk") # evaluate loaded model on test data loaded_model_yaml.compile(loss=gParameters['loss'], optimizer=gParameters['optimizer'], metrics=[gParameters['metrics']]) score_yaml = loaded_model_yaml.evaluate(X_test, Y_test, verbose=0) print('yaml Test score:', score_yaml[0]) print('yaml Test accuracy:', score_yaml[1]) print("yaml %s: %.2f%%" % (loaded_model_yaml.metrics_names[1], score_yaml[1] * 100)) return history
def run(gParameters): print('gParameters: ', gParameters) EPOCH = gParameters['epochs'] BATCH = gParameters['batch_size'] nb_classes = gParameters['classes'] DR = gParameters['drop'] ACTIVATION = gParameters['activation'] kerasDefaults = candle.keras_default_config() kerasDefaults['momentum_sgd'] = gParameters['momentum'] OPTIMIZER = candle.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) PL = 6213 # 38 + 60483 PS = 6212 # 60483 X_train, Y_train, X_test, Y_test = load_data(nb_classes, PL, gParameters) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) print('Y_train shape:', Y_train.shape) print('Y_test shape:', Y_test.shape) inputs = Input(shape=(PS, )) x = Dense(2000, activation=ACTIVATION)(inputs) x = Dense(1000, activation=ACTIVATION)(x) for i in range(gParameters['connections']): x = f(x, gParameters, distance=gParameters['distance']) x = Dropout(DR)(x) x = Dense(500, activation=ACTIVATION)(x) x = Dropout(DR)(x) x = Dense(250, activation=ACTIVATION)(x) x = Dropout(DR)(x) x = Dense(125, activation=ACTIVATION)(x) x = Dropout(DR)(x) x = Dense(62, activation=ACTIVATION)(x) x = Dropout(DR)(x) x = Dense(30, activation=ACTIVATION)(x) x = Dropout(DR)(x) outputs = Dense(2, activation='softmax')(x) model = Model(inputs=inputs, outputs=outputs) model.summary() model.compile(loss='categorical_crossentropy', optimizer=OPTIMIZER, metrics=['accuracy']) # set up a bunch of callbacks to do work during model training. checkpointer = ModelCheckpoint(filepath='t29res.autosave.model.h5', verbose=0, save_weights_only=False, save_best_only=True) csv_logger = CSVLogger('t29res.training.log') reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.4, patience=10, verbose=1, mode='auto', epsilon=0.0001, cooldown=3, min_lr=0.000000001) callbacks = [checkpointer, csv_logger, reduce_lr] def warmup_scheduler(epoch): lr = gParameters['learning_rate'] if epoch <= 4: K.set_value(model.optimizer.lr, (lr * (epoch + 1) / 5)) print('Epoch {}: lr={}'.format(epoch, K.get_value(model.optimizer.lr))) return K.get_value(model.optimizer.lr) if 'warmup_lr' in gParameters: warmup_lr = LearningRateScheduler(warmup_scheduler) print("adding LearningRateScheduler") callbacks.append(warmup_lr) history = model.fit(X_train, Y_train, batch_size=BATCH, epochs=EPOCH, verbose=1, validation_data=(X_test, Y_test), callbacks=callbacks) score = model.evaluate(X_test, Y_test, verbose=0) # summarize history for accuracy plt.plot(history.history['acc']) plt.plot(history.history['val_acc']) plt.title('Model Accuracy') plt.ylabel('accuracy') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.savefig('t29res.accuracy.png', bbox_inches='tight') plt.savefig('t29res.accuracy.pdf', bbox_inches='tight') plt.close() # summarize history for loss plt.plot(history.history['loss']) plt.plot(history.history['val_loss']) plt.title('Model Loss') plt.ylabel('loss') plt.xlabel('epoch') plt.legend(['train', 'test'], loc='upper left') plt.savefig('t29res.loss.png', bbox_inches='tight') plt.savefig('t29res.loss.pdf', bbox_inches='tight') print('Test val_loss:', score[0]) print('Test accuracy:', score[1]) # serialize model to JSON model_json = model.to_json() with open("t29res.model.json", "w") as json_file: json_file.write(model_json) # serialize model to YAML model_yaml = model.to_yaml() with open("t29res.model.yaml", "w") as yaml_file: yaml_file.write(model_yaml) # serialize weights to HDF5 model.save_weights("t29res.model.h5") print("Saved model to disk") # load json and create model json_file = open('t29res.model.json', 'r') loaded_model_json = json_file.read() json_file.close() loaded_model_json = model_from_json(loaded_model_json) # load yaml and create model yaml_file = open('t29res.model.yaml', 'r') loaded_model_yaml = yaml_file.read() yaml_file.close() loaded_model_yaml = model_from_yaml(loaded_model_yaml) # load weights into new model loaded_model_json.load_weights("t29res.model.h5") print("Loaded json model from disk") # evaluate json loaded model on test data loaded_model_json.compile(loss='binary_crossentropy', optimizer=gParameters['optimizer'], metrics=['accuracy']) score_json = loaded_model_json.evaluate(X_test, Y_test, verbose=0) print('json Validation loss:', score_json[0]) print('json Validation accuracy:', score_json[1]) print("json %s: %.2f%%" % (loaded_model_json.metrics_names[1], score_json[1] * 100)) # load weights into new model loaded_model_yaml.load_weights("t29res.model.h5") print("Loaded yaml model from disk") # evaluate loaded model on test data loaded_model_yaml.compile(loss='binary_crossentropy', optimizer=gParameters['optimizer'], metrics=['accuracy']) score_yaml = loaded_model_yaml.evaluate(X_test, Y_test, verbose=0) print('yaml Validation loss:', score_yaml[0]) print('yaml Validation accuracy:', score_yaml[1]) print("yaml %s: %.2f%%" % (loaded_model_yaml.metrics_names[1], score_yaml[1] * 100)) # predict using loaded yaml model on test and training data predict_yaml_train = loaded_model_yaml.predict(X_train) predict_yaml_test = loaded_model_yaml.predict(X_test) print('Yaml_train_shape:', predict_yaml_train.shape) print('Yaml_test_shape:', predict_yaml_test.shape) predict_yaml_train_classes = np.argmax(predict_yaml_train, axis=1) predict_yaml_test_classes = np.argmax(predict_yaml_test, axis=1) np.savetxt("predict_yaml_train.csv", predict_yaml_train, delimiter=",", fmt="%.3f") np.savetxt("predict_yaml_test.csv", predict_yaml_test, delimiter=",", fmt="%.3f") np.savetxt("predict_yaml_train_classes.csv", predict_yaml_train_classes, delimiter=",", fmt="%d") np.savetxt("predict_yaml_test_classes.csv", predict_yaml_test_classes, delimiter=",", fmt="%d") return history
def run(GP): # set the seed if GP['rng_seed']: np.random.seed(GP['rng_seed']) else: np.random.seed(np.random.randint(10000)) # Set paths if not os.path.isdir(GP['home_dir']): print('Keras home directory not set') sys.exit(0) sys.path.append(GP['home_dir']) # Setup loggin args = candle.ArgumentStruct(**GP) # set_seed(args.rng_seed) # ext = extension_from_parameters(args) candle.verify_path(args.save_path) prefix = args.save_path # + ext logfile = args.logfile if args.logfile else prefix + '.log' candle.set_up_logger(logfile, logger, False) # args.verbose logger.info('Params: {}'.format(GP)) import p2b1 as hf reload(hf) # import keras_model_utils as KEU # reload(KEU) # reload(p2ck) # reload(p2ck.optimizers) maps = hf.autoencoder_preprocess() from keras.optimizers import SGD, RMSprop, Adam from keras.datasets import mnist from keras.callbacks import LearningRateScheduler, ModelCheckpoint from keras import callbacks from keras.layers.advanced_activations import ELU from keras.preprocessing.image import ImageDataGenerator # GP=hf.ReadConfig(opts.config_file) batch_size = GP['batch_size'] learning_rate = GP['learning_rate'] kerasDefaults = candle.keras_default_config() # #### Read Data ######## import helper (data_files, fields) = p2b1.get_list_of_data_files(GP) # Read from local directoy # (data_files, fields) = helper.get_local_files('/p/gscratchr/brainusr/datasets/cancer/pilot2/3k_run16_10us.35fs-DPPC.20-DIPC.60-CHOL.20.dir/') # (data_files, fields) = helper.get_local_files('3k_run16', '/p/lscratchf/brainusr/datasets/cancer/pilot2/') # Define datagenerator datagen = hf.ImageNoiseDataGenerator(corruption_level=GP['noise_factor']) # get data dimension ## num_samples = 0 for f in data_files: # Seperate different arrays from the data (X, nbrs, resnums) = helper.get_data_arrays(f) num_samples += X.shape[0] (X, nbrs, resnums) = helper.get_data_arrays(data_files[0]) print('\nData chunk shape: ', X.shape) molecular_hidden_layers = GP['molecular_num_hidden'] if not molecular_hidden_layers: X_train = hf.get_data(X, case=GP['case']) input_dim = X_train.shape[1] else: # computing input dimension for outer AE input_dim = X.shape[1] * molecular_hidden_layers[-1] print('\nState AE input/output dimension: ', input_dim) # get data dimension for molecular autoencoder molecular_nbrs = np.int(GP['molecular_nbrs']) num_molecules = X.shape[1] num_beads = X.shape[2] if GP['nbr_type'] == 'relative': # relative x, y, z positions num_loc_features = 3 loc_feat_vect = ['rel_x', 'rel_y', 'rel_z'] elif GP['nbr_type'] == 'invariant': # relative distance and angle num_loc_features = 2 loc_feat_vect = ['rel_dist', 'rel_angle'] else: print('Invalid nbr_type!!') exit() if not GP['type_bool']: # only consider molecular location coordinates num_type_features = 0 type_feat_vect = [] else: num_type_features = 5 type_feat_vect = list(fields.keys())[3:8] num_features = num_loc_features + num_type_features + num_beads dim = np.prod([num_beads, num_features, molecular_nbrs + 1]) bead_kernel_size = num_features molecular_input_dim = dim mol_kernel_size = num_beads feature_vector = loc_feat_vect + type_feat_vect + list(fields.keys())[8:] print('\nMolecular AE input/output dimension: ', molecular_input_dim) print( '\nData Format:\n[Frames (%s), Molecules (%s), Beads (%s), %s (%s)]' % (num_samples, num_molecules, num_beads, feature_vector, num_features)) # ## Define Model, Solver and Compile ########## print('\nDefine the model and compile') opt = candle.build_optimizer(GP['optimizer'], learning_rate, kerasDefaults) model_type = 'mlp' memo = '%s_%s' % (GP['base_memo'], model_type) # ####### Define Molecular Model, Solver and Compile ######### molecular_nonlinearity = GP['molecular_nonlinearity'] len_molecular_hidden_layers = len(molecular_hidden_layers) conv_bool = GP['conv_bool'] full_conv_bool = GP['full_conv_bool'] if conv_bool: molecular_model, molecular_encoder = AE_models.conv_dense_mol_auto( bead_k_size=bead_kernel_size, mol_k_size=mol_kernel_size, weights_path=None, input_shape=(1, molecular_input_dim, 1), nonlinearity=molecular_nonlinearity, hidden_layers=molecular_hidden_layers, l2_reg=GP['l2_reg'], drop=float(GP['dropout'])) elif full_conv_bool: molecular_model, molecular_encoder = AE_models.full_conv_mol_auto( bead_k_size=bead_kernel_size, mol_k_size=mol_kernel_size, weights_path=None, input_shape=(1, molecular_input_dim, 1), nonlinearity=molecular_nonlinearity, hidden_layers=molecular_hidden_layers, l2_reg=GP['l2_reg'], drop=float(GP['dropout'])) else: molecular_model, molecular_encoder = AE_models.dense_auto( weights_path=None, input_shape=(molecular_input_dim, ), nonlinearity=molecular_nonlinearity, hidden_layers=molecular_hidden_layers, l2_reg=GP['l2_reg'], drop=float(GP['dropout'])) if GP['loss'] == 'mse': loss_func = 'mse' elif GP['loss'] == 'custom': loss_func = helper.combined_loss molecular_model.compile( optimizer=opt, loss=loss_func, metrics=['mean_squared_error', 'mean_absolute_error']) print('\nModel Summary: \n') molecular_model.summary() # #### set up callbacks and cooling for the molecular_model ########## drop = GP['dropout'] mb_epochs = GP['epochs'] initial_lrate = GP['learning_rate'] epochs_drop = 1 + int(np.floor(mb_epochs / 3)) def step_decay(epoch): global initial_lrate, epochs_drop, drop lrate = initial_lrate * np.power(drop, np.floor((1 + epoch) / epochs_drop)) return lrate lr_scheduler = LearningRateScheduler(step_decay) history = callbacks.History() # callbacks=[history,lr_scheduler] history_logger = candle.LoggingCallback(logger.debug) candleRemoteMonitor = candle.CandleRemoteMonitor(params=GP) timeoutMonitor = candle.TerminateOnTimeOut(TIMEOUT) callbacks = [history, history_logger, candleRemoteMonitor, timeoutMonitor] loss = 0. # ### Save the Model to disk if GP['save_path'] is not None: save_path = GP['save_path'] if not os.path.exists(save_path): os.makedirs(save_path) else: save_path = '.' model_json = molecular_model.to_json() with open(save_path + '/model.json', "w") as json_file: json_file.write(model_json) encoder_json = molecular_encoder.to_json() with open(save_path + '/encoder.json', "w") as json_file: json_file.write(encoder_json) print('Saved model to disk') # ### Train the Model if GP['train_bool']: ct = hf.Candle_Molecular_Train( molecular_model, molecular_encoder, data_files, mb_epochs, callbacks, batch_size=batch_size, nbr_type=GP['nbr_type'], save_path=GP['save_path'], len_molecular_hidden_layers=len_molecular_hidden_layers, molecular_nbrs=molecular_nbrs, conv_bool=conv_bool, full_conv_bool=full_conv_bool, type_bool=GP['type_bool'], sampling_density=GP['sampling_density']) frame_loss, frame_mse = ct.train_ac() else: frame_mse = [] frame_loss = [] return frame_loss, frame_mse
def build_model(gParameters, kerasDefaults, shared_nnet_spec, individual_nnet_spec, input_dim, Y_train, Y_test, verbose=False): labels_train = [] labels_test = [] n_out_nodes = [] for l in range( len( Y_train ) ): truth_train = np.array( Y_train[l], dtype='int32' ) truth_test = np.array( Y_test[l], dtype='int32' ) mv = int( np.max( truth_train ) ) label_train = np.zeros( ( len( truth_train ), mv + 1 ) ) for i in range( len( truth_train ) ): label_train[ i, truth_train[ i ] ] = 1 label_test = np.zeros( ( len( truth_test ), mv + 1 ) ) for i in range( len(truth_test) ): label_test[ i, truth_test[ i ] ] = 1 labels_train.append( label_train ) labels_test.append( label_test ) n_out_nodes.append( mv + 1 ) shared_layers = [] # input layer layer = Input( shape = ( input_dim, ), name= 'input' ) shared_layers.append( layer ) # shared layers for k in range( len( shared_nnet_spec ) ): layer = Dense( shared_nnet_spec[ k ], activation=gParameters['activation'], name= 'shared_layer_' + str( k ) )( shared_layers[ -1 ] ) if gParameters['drop'] > 0: layer = Dropout( gParameters['drop'] )( shared_layers[ -1 ] ) shared_layers.append( layer ) # individual layers indiv_layers_arr = [] models = [] trainable_count = 0 non_trainable_count = 0 for l in range( len( individual_nnet_spec ) ): indiv_layers = [ shared_layers[-1] ] for k in range( len( individual_nnet_spec[l] ) + 1 ): if k < len( individual_nnet_spec[l] ): layer = Dense( individual_nnet_spec[l][k], activation=gParameters['activation'], name= 'indiv_layer_' + str( l ) + '_' + str( k ) )( indiv_layers[-1] ) indiv_layers.append( layer ) if gParameters['drop'] > 0: layer = Dropout( gParameters['drop'] )( indiv_layers[-1] ) indiv_layers.append( layer ) else: layer = Dense( n_out_nodes[l], activation=gParameters['out_activation'], name= 'out_' + str( l ) )( indiv_layers[-1] ) indiv_layers.append( layer ) indiv_layers_arr.append( indiv_layers ) model = Model( inputs=[shared_layers[0]], outputs=[indiv_layers[-1]] ) # calculate trainable/non-trainable param count for each model param_counts = candle.compute_trainable_params(model) trainable_count += param_counts['trainable_params'] non_trainable_count += param_counts['non_trainable_params'] models.append( model ) # capture total param counts gParameters['trainable_params'] = trainable_count gParameters['non_trainable_params'] = non_trainable_count gParameters['total_params'] = trainable_count + non_trainable_count # Define optimizer optimizer = candle.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) # DEBUG - verify if verbose: for k in range( len( models ) ): model = models[k] print('Model: ', k) model.summary() for k in range( len( models ) ): model = models[ k ] model.compile( loss=gParameters['loss'], optimizer=optimizer, metrics=[gParameters['metrics']] ) return models, labels_train, labels_test
def run(gParameters): """ Runs the model using the specified set of parameters Args: gParameters: a python dictionary containing the parameters (e.g. epoch) to run the model with. """ # if 'dense' in gParameters: dval = gParameters['dense'] if type(dval) != list: res = list(dval) #try: #is_str = isinstance(dval, basestring) #except NameError: #is_str = isinstance(dval, str) #if is_str: #res = str2lst(dval) gParameters['dense'] = res print(gParameters['dense']) if 'conv' in gParameters: #conv_list = p1_common.parse_conv_list(gParameters['conv']) #cval = gParameters['conv'] #try: #is_str = isinstance(cval, basestring) #except NameError: #is_str = isinstance(cval, str) #if is_str: #res = str2lst(cval) #gParameters['conv'] = res print('Conv input', gParameters['conv']) # print('Params:', gParameters) # Construct extension to save model ext = benchmark.extension_from_parameters(gParameters, '.keras') logfile = gParameters['logfile'] if gParameters[ 'logfile'] else gParameters['output_dir'] + ext + '.log' fh = logging.FileHandler(logfile) fh.setFormatter( logging.Formatter("[%(asctime)s %(process)d] %(message)s", datefmt="%Y-%m-%d %H:%M:%S")) fh.setLevel(logging.DEBUG) sh = logging.StreamHandler() sh.setFormatter(logging.Formatter('')) sh.setLevel(logging.DEBUG if gParameters['verbose'] else logging.INFO) benchmark.logger.setLevel(logging.DEBUG) benchmark.logger.addHandler(fh) benchmark.logger.addHandler(sh) benchmark.logger.info('Params: {}'.format(gParameters)) # Get default parameters for initialization and optimizer functions kerasDefaults = candle.keras_default_config() seed = gParameters['rng_seed'] # Build dataset loader object loader = benchmark.DataLoader( seed=seed, dtype=gParameters['data_type'], val_split=gParameters['val_split'], test_cell_split=gParameters['test_cell_split'], cell_features=gParameters['cell_features'], drug_features=gParameters['drug_features'], feature_subsample=gParameters['feature_subsample'], scaling=gParameters['scaling'], scramble=gParameters['scramble'], min_logconc=gParameters['min_logconc'], max_logconc=gParameters['max_logconc'], subsample=gParameters['subsample'], category_cutoffs=gParameters['category_cutoffs']) # Initialize weights and learning rule initializer_weights = candle.build_initializer( gParameters['initialization'], kerasDefaults, seed) initializer_bias = candle.build_initializer('constant', kerasDefaults, 0.) activation = gParameters['activation'] # Define model architecture gen_shape = None out_dim = 1 model = Sequential() if 'dense' in gParameters: # Build dense layers for layer in gParameters['dense']: if layer: model.add( Dense(layer, input_dim=loader.input_dim, kernel_initializer=initializer_weights, bias_initializer=initializer_bias)) if gParameters['batch_normalization']: model.add(BatchNormalization()) model.add(Activation(gParameters['activation'])) if gParameters['dropout']: model.add(Dropout(gParameters['dropout'])) else: # Build convolutional layers gen_shape = 'add_1d' layer_list = list(range(0, len(gParameters['conv']))) lc_flag = False if 'locally_connected' in gParameters: lc_flag = True for l, i in enumerate(layer_list): if i == 0: add_conv_layer(model, gParameters['conv'][i], input_dim=loader.input_dim, locally_connected=lc_flag) else: add_conv_layer(model, gParameters['conv'][i], locally_connected=lc_flag) if gParameters['batch_normalization']: model.add(BatchNormalization()) model.add(Activation(gParameters['activation'])) if gParameters['pool']: model.add(MaxPooling1D(pool_size=gParameters['pool'])) model.add(Flatten()) model.add(Dense(out_dim)) # Define optimizer optimizer = candle.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) # Compile and display model model.compile(loss=gParameters['loss'], optimizer=optimizer) model.summary() benchmark.logger.debug('Model: {}'.format(model.to_json())) train_gen = benchmark.DataGenerator( loader, batch_size=gParameters['batch_size'], shape=gen_shape, name='train_gen', cell_noise_sigma=gParameters['cell_noise_sigma']).flow() val_gen = benchmark.DataGenerator(loader, partition='val', batch_size=gParameters['batch_size'], shape=gen_shape, name='val_gen').flow() val_gen2 = benchmark.DataGenerator(loader, partition='val', batch_size=gParameters['batch_size'], shape=gen_shape, name='val_gen2').flow() test_gen = benchmark.DataGenerator(loader, partition='test', batch_size=gParameters['batch_size'], shape=gen_shape, name='test_gen').flow() train_steps = int(loader.n_train / gParameters['batch_size']) val_steps = int(loader.n_val / gParameters['batch_size']) test_steps = int(loader.n_test / gParameters['batch_size']) if 'train_steps' in gParameters: train_steps = gParameters['train_steps'] if 'val_steps' in gParameters: val_steps = gParameters['val_steps'] if 'test_steps' in gParameters: test_steps = gParameters['test_steps'] checkpointer = ModelCheckpoint(filepath=gParameters['output_dir'] + '.model' + ext + '.h5', save_best_only=True) progbar = MyProgbarLogger(train_steps * gParameters['batch_size']) loss_history = MyLossHistory( progbar=progbar, val_gen=val_gen2, test_gen=test_gen, val_steps=val_steps, test_steps=test_steps, metric=gParameters['loss'], category_cutoffs=gParameters['category_cutoffs'], ext=ext, pre=gParameters['output_dir']) # Seed random generator for training np.random.seed(seed) candleRemoteMonitor = candle.CandleRemoteMonitor(params=gParameters) history = model.fit_generator( train_gen, train_steps, epochs=gParameters['epochs'], validation_data=val_gen, validation_steps=val_steps, verbose=0, callbacks=[checkpointer, loss_history, progbar, candleRemoteMonitor], ) benchmark.logger.removeHandler(fh) benchmark.logger.removeHandler(sh) return history
#model.add(MaxPooling1D(pool_size=10)) #model.add(Flatten()) #model.add(Dense(200)) #model.add(Activation('relu')) #model.add(Dropout(0.1)) #model.add(Dense(20)) #model.add(Activation('relu')) #model.add(Dropout(0.1)) #model.add(Dense(CLASSES)) #model.add(Activation('softmax')) kerasDefaults = candle.keras_default_config() # Define optimizer optimizer = candle.build_optimizer(candle_params['optimizer'], candle_params['learning_rate'], kerasDefaults) model.summary() model.compile(loss=candle_params['loss'], optimizer=optimizer, metrics=[candle_params['metrics']]) output_dir = candle_params['save'] if not os.path.exists(output_dir): os.makedirs(output_dir) # calculate trainable and non-trainable params candle_params.update(candle.compute_trainable_params(model))
def run(params): args = candle.ArgumentStruct(**params) seed = args.rng_seed candle.set_seed(seed) # Construct extension to save model ext = extension_from_parameters(params, 'keras') candle.verify_path(params['save_path']) prefix = '{}{}'.format(params['save_path'], ext) logfile = params['logfile'] if params['logfile'] else prefix+'.log' root_fname = 'Agg_attn_abs_bin' candle.set_up_logger(logfile, attn.logger, params['verbose']) attn.logger.info('Params: {}'.format(params)) # Get default parameters for initialization and optimizer functions keras_defaults = candle.keras_default_config() ## X_train, _Y_train, X_val, _Y_val, X_test, _Y_test = attn.load_data(params, seed) # move this inside the load_data function Y_train = _Y_train['AUC'] Y_test = _Y_test['AUC'] Y_val = _Y_val['AUC'] Y_train_neg, Y_train_pos = np.bincount(Y_train) Y_test_neg, Y_test_pos = np.bincount(Y_test) Y_val_neg, Y_val_pos = np.bincount(Y_val) Y_train_total = Y_train_neg + Y_train_pos Y_test_total = Y_test_neg + Y_test_pos Y_val_total = Y_val_neg + Y_val_pos total = Y_train_total + Y_test_total + Y_val_total neg = Y_train_neg + Y_test_neg + Y_val_neg pos = Y_train_pos + Y_test_pos + Y_val_pos print('Examples:\n Total: {}\n Positive: {} ({:.2f}% of total)\n'.format( total, pos, 100 * pos / total)) nb_classes = params['dense'][-1] # Convert classes to categorical with an extra slot for the abstaining class Y_train, Y_test, Y_val = candle.modify_labels(nb_classes+1, Y_train, Y_test, Y_val) # Disable class weight (for initial testing of the abstention classifier) #y_integers = np.argmax(Y_train, axis=1) #class_weights = compute_class_weight('balanced', np.unique(y_integers), y_integers) #d_class_weights = dict(enumerate(class_weights)) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) print('Y_train shape:', Y_train.shape) print('Y_test shape:', Y_test.shape) PS = X_train.shape[1] model = build_attention_model(params, PS) model = candle.add_model_output(model, mode='abstain', num_add=1, activation='sigmoid') print('Model after modifying layer for abstention') model.summary() # Configure abstention model mask_ = np.zeros(nb_classes+1) mask_[-1] = 1 mu0 = 0.5 # In the long term this is not as important since mu auto tunes, however it may require a large number of epochs to converge if set far away from target candle.abstention_variable_initialization(mu0, mask_, nb_classes) #parallel_model = multi_gpu_model(model, gpus=4) #parallel_model.compile(loss='mean_squared_error', # optimizer=SGD(lr=0.0001, momentum=0.9), # metrics=['mae',r2]) kerasDefaults = candle.keras_default_config() if params['momentum']: kerasDefaults['momentum_sgd'] = params['momentum'] optimizer = candle.build_optimizer(params['optimizer'], params['learning_rate'], kerasDefaults) # compile model with abstention loss model.compile(loss=candle.abstention_loss, optimizer=optimizer, metrics=['acc',tf_auc,candle.abs_acc,candle.acc_class1,candle.abs_acc_class1]) # set up a bunch of callbacks to do work during model training.. checkpointer = ModelCheckpoint(filepath=params['save_path'] + root_fname + '.autosave.model.h5', verbose=1, save_weights_only=False, save_best_only=True) csv_logger = CSVLogger('{}/{}.training.log'.format(params['save_path'], root_fname)) reduce_lr = ReduceLROnPlateau(monitor='val_tf_auc', factor=0.20, patience=40, verbose=1, mode='auto', min_delta=0.0001, cooldown=3, min_lr=0.000000001) early_stop = EarlyStopping(monitor='val_tf_auc', patience=200, verbose=1, mode='auto') candle_monitor = candle.CandleRemoteMonitor(params=params) candle_monitor = candle.CandleRemoteMonitor(params=params) timeout_monitor = candle.TerminateOnTimeOut(params['timeout']) tensorboard = TensorBoard(log_dir="tb/tb{}".format(ext)) history_logger = candle.LoggingCallback(attn.logger.debug) abstention_cbk = candle.AbstentionAdapt_Callback(monitor='val_abs_acc_class1', scale_factor=params['abs_scale_factor'], target_acc=params['target_abs_acc']) callbacks = [candle_monitor, timeout_monitor, csv_logger, history_logger, abstention_cbk] if params['reduce_lr']: callbacks.append(reduce_lr) if params['use_cp']: callbacks.append(checkpointer) if params['use_tb']: callbacks.append(tensorboard) if params['early_stop']: callbacks.append(early_stop) epochs = params['epochs'] batch_size=params['batch_size'] history = model.fit(X_train, Y_train, #class_weight=d_class_weights, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(X_val, Y_val), callbacks = callbacks) # diagnostic plots if 'loss' in history.history.keys(): candle.plot_history(params['save_path'] + root_fname, history, 'loss') if 'acc' in history.history.keys(): candle.plot_history(params['save_path'] + root_fname, history, 'acc') if 'abs_acc' in history.history.keys(): candle.plot_history(params['save_path'] + root_fname, history, 'abs_acc') # Plot mu evolution fname = params['save_path'] + root_fname + '.mu.png' xlabel='Epochs' ylabel='Abstention Weight mu' title='mu Evolution' attnviz.plot_array(abstention_cbk.muvalues, xlabel, ylabel, title, fname) # Evaluate model score = model.evaluate(X_test, Y_test, verbose=0) Y_predict = model.predict(X_test) evaluate_abstention(params, root_fname, nb_classes, Y_test, _Y_test, Y_predict, pos, total, score) save_and_test_saved_model(params, model, root_fname, X_train, X_test, Y_test) attn.logger.handlers = [] return history
def run(gParameters): # Construct extension to save model ext = p1b2.extension_from_parameters(gParameters, '.keras') candle.verify_path(gParameters['save_path']) prefix = '{}{}'.format(gParameters['save_path'], ext) logfile = gParameters['logfile'] if gParameters[ 'logfile'] else prefix + '.log' candle.set_up_logger(logfile, p1b2.logger, gParameters['verbose']) p1b2.logger.info('Params: {}'.format(gParameters)) # Get default parameters for initialization and optimizer functions kerasDefaults = candle.keras_default_config() seed = gParameters['rng_seed'] # Load dataset #(X_train, y_train), (X_test, y_test) = p1b2.load_data(gParameters, seed) #(X_train, y_train), (X_val, y_val), (X_test, y_test) = p1b2.load_data_one_hot(gParameters, seed) (X_train, y_train), (X_test, y_test) = p1b2.load_data2(gParameters, seed) print("Shape X_train: ", X_train.shape) #print ("Shape X_val: ", X_val.shape) print("Shape X_test: ", X_test.shape) print("Shape y_train: ", y_train.shape) #print ("Shape y_val: ", y_val.shape) print("Shape y_test: ", y_test.shape) print("Range X_train --> Min: ", np.min(X_train), ", max: ", np.max(X_train)) #print ("Range X_val --> Min: ", np.min(X_val), ", max: ", np.max(X_val)) print("Range X_test --> Min: ", np.min(X_test), ", max: ", np.max(X_test)) print("Range y_train --> Min: ", np.min(y_train), ", max: ", np.max(y_train)) #print ("Range y_val --> Min: ", np.min(y_val), ", max: ", np.max(y_val)) print("Range y_test --> Min: ", np.min(y_test), ", max: ", np.max(y_test)) input_dim = X_train.shape[1] input_vector = Input(shape=(input_dim, )) output_dim = y_train.shape[1] # Initialize weights and learning rule initializer_weights = candle.build_initializer( gParameters['initialization'], kerasDefaults, seed) initializer_bias = candle.build_initializer('constant', kerasDefaults, 0.) activation = gParameters['activation'] # Define MLP architecture layers = gParameters['dense'] if layers != None: if type(layers) != list: layers = list(layers) for i, l in enumerate(layers): if i == 0: x = Dense(l, activation=activation, kernel_initializer=initializer_weights, bias_initializer=initializer_bias, kernel_regularizer=l2(gParameters['reg_l2']), activity_regularizer=l2( gParameters['reg_l2']))(input_vector) else: x = Dense(l, activation=activation, kernel_initializer=initializer_weights, bias_initializer=initializer_bias, kernel_regularizer=l2(gParameters['reg_l2']), activity_regularizer=l2(gParameters['reg_l2']))(x) if gParameters['dropout']: x = Dropout(gParameters['dropout'])(x) output = Dense(output_dim, activation=activation, kernel_initializer=initializer_weights, bias_initializer=initializer_bias)(x) else: output = Dense(output_dim, activation=activation, kernel_initializer=initializer_weights, bias_initializer=initializer_bias)(input_vector) # Build MLP model mlp = Model(outputs=output, inputs=input_vector) p1b2.logger.debug('Model: {}'.format(mlp.to_json())) # Define optimizer optimizer = candle.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) # Compile and display model mlp.compile(loss=gParameters['loss'], optimizer=optimizer, metrics=['accuracy']) mlp.summary() # Seed random generator for training np.random.seed(seed) history = mlp.fit(X_train, y_train, batch_size=gParameters['batch_size'], epochs=gParameters['epochs'], validation_split=gParameters['val_split'] #validation_data=(X_val, y_val) ) best_val_loss = "{:.5f}".format(min(history.history['val_loss'])) best_val_acc = "{:.5f}".format(max(history.history['val_acc'])) print('best_val_loss =', best_val_loss, 'best_val_acc =', best_val_acc) # model save # save_filepath = "model_mlp_W_" + ext # mlp.save_weights(save_filepath) model_json = mlp.to_json() with open(prefix + '.model.json', 'w') as f: print(model_json, file=f) mlp.save_weights(prefix + '.weights.h5') # Evalute model on test set y_pred = mlp.predict(X_test) scores = p1b2.evaluate_accuracy_one_hot(y_pred, y_test) print('Evaluation on test data:', scores)
def run(gParameters): origin = gParameters['data_url'] train_data = gParameters['train_data'] data_loc = candle.fetch_file(origin + train_data, untar=True, md5_hash=None, subdir='Pilot3') print('Data downloaded and stored at: ' + data_loc) data_path = os.path.dirname(data_loc) print(data_path) kerasDefaults = candle.keras_default_config() rnn_size = gParameters['rnn_size'] n_layers = gParameters['n_layers'] learning_rate = gParameters['learning_rate'] dropout = gParameters['dropout'] recurrent_dropout = gParameters['recurrent_dropout'] n_epochs = gParameters['epochs'] data_train = data_path + '/data.pkl' verbose = gParameters['verbose'] savedir = gParameters['output_dir'] do_sample = gParameters['do_sample'] temperature = gParameters['temperature'] primetext = gParameters['primetext'] length = gParameters['length'] # load data from pickle f = open(data_train, 'rb') if (sys.version_info > (3, 0)): classes = pickle.load(f, encoding='latin1') chars = pickle.load(f, encoding='latin1') char_indices = pickle.load(f, encoding='latin1') indices_char = pickle.load(f, encoding='latin1') maxlen = pickle.load(f, encoding='latin1') step = pickle.load(f, encoding='latin1') X_ind = pickle.load(f, encoding='latin1') y_ind = pickle.load(f, encoding='latin1') else: classes = pickle.load(f) chars = pickle.load(f) char_indices = pickle.load(f) indices_char = pickle.load(f) maxlen = pickle.load(f) step = pickle.load(f) X_ind = pickle.load(f) y_ind = pickle.load(f) f.close() [s1, s2] = X_ind.shape print(X_ind.shape) print(y_ind.shape) print(maxlen) print(len(chars)) X = np.zeros((s1, s2, len(chars)), dtype=np.bool) y = np.zeros((s1, len(chars)), dtype=np.bool) for i in range(s1): for t in range(s2): X[i, t, X_ind[i, t]] = 1 y[i, y_ind[i]] = 1 # build the model: a single LSTM if verbose: print('Build model...') model = Sequential() # for rnn_size in rnn_sizes: for k in range(n_layers): if k < n_layers - 1: ret_seq = True else: ret_seq = False if k == 0: model.add( LSTM(rnn_size, input_shape=(maxlen, len(chars)), return_sequences=ret_seq, dropout=dropout, recurrent_dropout=recurrent_dropout)) else: model.add( LSTM(rnn_size, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=ret_seq)) model.add(Dense(len(chars))) model.add(Activation(gParameters['activation'])) optimizer = candle.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) model.compile(loss=gParameters['loss'], optimizer=optimizer) if verbose: model.summary() for iteration in range(1, n_epochs + 1): if verbose: print() print('-' * 50) print('Iteration', iteration) history = LossHistory() model.fit(X, y, batch_size=100, epochs=1, callbacks=[history]) loss = history.losses[-1] if verbose: print(loss) dirname = savedir if len(dirname) > 0 and not dirname.endswith('/'): dirname = dirname + '/' if not os.path.exists(dirname): os.makedirs(dirname) # serialize model to JSON model_json = model.to_json() with open( dirname + "/model_" + str(iteration) + "_" + "{:f}".format(loss) + ".json", "w") as json_file: json_file.write(model_json) # serialize weights to HDF5 model.save_weights(dirname + "/model_" + str(iteration) + "_" + "{:f}".format(loss) + ".h5") if verbose: print("Checkpoint saved.") if do_sample: outtext = open(dirname + "/example_" + str(iteration) + "_" + "{:f}".format(loss) + ".txt", "w", encoding='utf-8') diversity = temperature outtext.write('----- diversity:' + str(diversity) + "\n") generated = '' seedstr = primetext outtext.write('----- Generating with seed: "' + seedstr + '"' + "\n") sentence = " " * maxlen # class_index = 0 generated += sentence outtext.write(generated) for c in seedstr: sentence = sentence[1:] + c x = np.zeros((1, maxlen, len(chars))) for t, char in enumerate(sentence): x[0, t, char_indices[char]] = 1. preds = model.predict(x, verbose=verbose)[0] next_index = sample(preds, diversity) next_char = indices_char[next_index] generated += c outtext.write(c) for i in range(length): x = np.zeros((1, maxlen, len(chars))) for t, char in enumerate(sentence): x[0, t, char_indices[char]] = 1. preds = model.predict(x, verbose=verbose)[0] next_index = sample(preds, diversity) next_char = indices_char[next_index] generated += next_char sentence = sentence[1:] + next_char if (sys.version_info > (3, 0)): outtext.write(generated + '\n') else: outtext.write(generated.decode('utf-8').encode('utf-8') + '\n') outtext.close()
def run(gParameters): print ('Params:', gParameters) file_train = gParameters['train_data'] file_test = gParameters['test_data'] url = gParameters['data_url'] train_file = candle.get_file(file_train, url+file_train, cache_subdir='Pilot1') test_file = candle.get_file(file_test, url+file_test, cache_subdir='Pilot1') X_train, Y_train, X_test, Y_test = bmk.load_data(train_file, test_file, gParameters) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) print('Y_train shape:', Y_train.shape) print('Y_test shape:', Y_test.shape) x_train_len = X_train.shape[1] # this reshaping is critical for the Conv1D to work X_train = np.expand_dims(X_train, axis=2) X_test = np.expand_dims(X_test, axis=2) print('X_train shape:', X_train.shape) print('X_test shape:', X_test.shape) model = Sequential() layer_list = list(range(0, len(gParameters['conv']), 3)) for l, i in enumerate(layer_list): filters = gParameters['conv'][i] filter_len = gParameters['conv'][i+1] stride = gParameters['conv'][i+2] print(int(i/3), filters, filter_len, stride) if gParameters['pool']: pool_list=gParameters['pool'] if type(pool_list) != list: pool_list=list(pool_list) if filters <= 0 or filter_len <= 0 or stride <= 0: break if 'locally_connected' in gParameters: model.add(LocallyConnected1D(filters, filter_len, strides=stride, padding='valid', input_shape=(x_train_len, 1))) else: #input layer if i == 0: model.add(Conv1D(filters=filters, kernel_size=filter_len, strides=stride, padding='valid', input_shape=(x_train_len, 1))) else: model.add(Conv1D(filters=filters, kernel_size=filter_len, strides=stride, padding='valid')) model.add(Activation(gParameters['activation'])) if gParameters['pool']: model.add(MaxPooling1D(pool_size=pool_list[int(i/3)])) model.add(Flatten()) for layer in gParameters['dense']: if layer: model.add(Dense(layer)) model.add(Activation(gParameters['activation'])) if gParameters['dropout']: model.add(Dropout(gParameters['dropout'])) model.add(Dense(gParameters['classes'])) model.add(Activation(gParameters['out_activation'])) #Reference case #model.add(Conv1D(filters=128, kernel_size=20, strides=1, padding='valid', input_shape=(P, 1))) #model.add(Activation('relu')) #model.add(MaxPooling1D(pool_size=1)) #model.add(Conv1D(filters=128, kernel_size=10, strides=1, padding='valid')) #model.add(Activation('relu')) #model.add(MaxPooling1D(pool_size=10)) #model.add(Flatten()) #model.add(Dense(200)) #model.add(Activation('relu')) #model.add(Dropout(0.1)) #model.add(Dense(20)) #model.add(Activation('relu')) #model.add(Dropout(0.1)) #model.add(Dense(CLASSES)) #model.add(Activation('softmax')) kerasDefaults = candle.keras_default_config() # Define optimizer optimizer = candle.build_optimizer(gParameters['optimizer'], gParameters['learning_rate'], kerasDefaults) model.summary() model.compile(loss=gParameters['loss'], optimizer=optimizer, metrics=[gParameters['metrics']]) output_dir = gParameters['output_dir'] if not os.path.exists(output_dir): os.makedirs(output_dir) # calculate trainable and non-trainable params gParameters.update(candle.compute_trainable_params(model)) # set up a bunch of callbacks to do work during model training.. model_name = gParameters['model_name'] path = '{}/{}.autosave.model.h5'.format(output_dir, model_name) checkpointer = ModelCheckpoint(filepath=path, verbose=1, save_weights_only=False, save_best_only=True) csv_logger = CSVLogger('{}/training.log'.format(output_dir)) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=10, verbose=1, mode='auto', epsilon=0.0001, cooldown=0, min_lr=0) candleRemoteMonitor = candle.CandleRemoteMonitor(params=gParameters) timeoutMonitor = candle.TerminateOnTimeOut(gParameters['timeout']) history = model.fit(X_train, Y_train, batch_size=gParameters['batch_size'], epochs=gParameters['epochs'], verbose=1, validation_data=(X_test, Y_test), callbacks = [csv_logger, reduce_lr, candleRemoteMonitor, timeoutMonitor]) score = model.evaluate(X_test, Y_test, verbose=0) print('Test score:', score[0]) print('Test accuracy:', score[1]) # serialize model to JSON model_json = model.to_json() with open("{}/{}.model.json".format(output_dir, model_name), "w") as json_file: json_file.write(model_json) # serialize model to YAML model_yaml = model.to_yaml() with open("{}/{}.model.yaml".format(output_dir, model_name), "w") as yaml_file: yaml_file.write(model_yaml) # serialize weights to HDF5 model.save_weights("{}/{}.weights.h5".format(output_dir, model_name)) print("Saved model to disk") # load json and create model json_file = open('{}/{}.model.json'.format(output_dir, model_name), 'r') loaded_model_json = json_file.read() json_file.close() loaded_model_json = model_from_json(loaded_model_json) # load yaml and create model yaml_file = open('{}/{}.model.yaml'.format(output_dir, model_name), 'r') loaded_model_yaml = yaml_file.read() yaml_file.close() loaded_model_yaml = model_from_yaml(loaded_model_yaml) # load weights into new model loaded_model_json.load_weights('{}/{}.weights.h5'.format(output_dir, model_name)) print("Loaded json model from disk") # evaluate json loaded model on test data loaded_model_json.compile(loss=gParameters['loss'], optimizer=gParameters['optimizer'], metrics=[gParameters['metrics']]) score_json = loaded_model_json.evaluate(X_test, Y_test, verbose=0) print('json Test score:', score_json[0]) print('json Test accuracy:', score_json[1]) print("json %s: %.2f%%" % (loaded_model_json.metrics_names[1], score_json[1]*100)) # load weights into new model loaded_model_yaml.load_weights('{}/{}.weights.h5'.format(output_dir, model_name)) print("Loaded yaml model from disk") # evaluate loaded model on test data loaded_model_yaml.compile(loss=gParameters['loss'], optimizer=gParameters['optimizer'], metrics=[gParameters['metrics']]) score_yaml = loaded_model_yaml.evaluate(X_test, Y_test, verbose=0) print('yaml Test score:', score_yaml[0]) print('yaml Test accuracy:', score_yaml[1]) print("yaml %s: %.2f%%" % (loaded_model_yaml.metrics_names[1], score_yaml[1]*100)) return history
def run(params): args = candle.ArgumentStruct(**params) seed = args.rng_seed candle.set_seed(seed) # Construct extension to save model ext = attn.extension_from_parameters(params, "keras") candle.verify_path(params["save_path"]) prefix = "{}{}".format(params["save_path"], ext) logfile = params["logfile"] if params["logfile"] else prefix + ".log" root_fname = "Agg_attn_bin" candle.set_up_logger(logfile, attn.logger, params["verbose"]) attn.logger.info("Params: {}".format(params)) # Get default parameters for initialization and optimizer functions keras_defaults = candle.keras_default_config() ## X_train, _Y_train, X_val, _Y_val, X_test, _Y_test = attn.load_data( params, seed) # move this inside the load_data function Y_train = _Y_train["AUC"] Y_test = _Y_test["AUC"] Y_val = _Y_val["AUC"] Y_train_neg, Y_train_pos = np.bincount(Y_train) Y_test_neg, Y_test_pos = np.bincount(Y_test) Y_val_neg, Y_val_pos = np.bincount(Y_val) Y_train_total = Y_train_neg + Y_train_pos Y_test_total = Y_test_neg + Y_test_pos Y_val_total = Y_val_neg + Y_val_pos total = Y_train_total + Y_test_total + Y_val_total neg = Y_train_neg + Y_test_neg + Y_val_neg pos = Y_train_pos + Y_test_pos + Y_val_pos print("Examples:\n Total: {}\n Positive: {} ({:.2f}% of total)\n". format(total, pos, 100 * pos / total)) nb_classes = params["dense"][-1] Y_train = np_utils.to_categorical(Y_train, nb_classes) Y_test = np_utils.to_categorical(Y_test, nb_classes) Y_val = np_utils.to_categorical(Y_val, nb_classes) y_integers = np.argmax(Y_train, axis=1) class_weights = compute_class_weight("balanced", np.unique(y_integers), y_integers) d_class_weights = dict(enumerate(class_weights)) print("X_train shape:", X_train.shape) print("X_test shape:", X_test.shape) print("Y_train shape:", Y_train.shape) print("Y_test shape:", Y_test.shape) PS = X_train.shape[1] model = build_attention_model(params, PS) # parallel_model = multi_gpu_model(model, gpus=4) # parallel_model.compile(loss='mean_squared_error', # optimizer=SGD(lr=0.0001, momentum=0.9), # metrics=['mae',r2]) kerasDefaults = candle.keras_default_config() if params["momentum"]: kerasDefaults["momentum_sgd"] = params["momentum"] optimizer = candle.build_optimizer(params["optimizer"], params["learning_rate"], kerasDefaults) model.compile( loss=params["loss"], optimizer=optimizer, # SGD(lr=0.00001, momentum=0.9), # optimizer=Adam(lr=0.00001), # optimizer=RMSprop(lr=0.0001), # optimizer=Adadelta(), metrics=[ "acc", tf.keras.metrics.AUC(name="auroc", curve="ROC"), tf.keras.metrics.AUC(name="aucpr", curve="PR"), ], ) # set up a bunch of callbacks to do work during model training.. checkpointer = ModelCheckpoint( filepath=params["save_path"] + root_fname + ".autosave.model.h5", verbose=1, save_weights_only=False, save_best_only=True, ) csv_logger = CSVLogger("{}/{}.training.log".format(params["save_path"], root_fname)) reduce_lr = ReduceLROnPlateau( monitor="val_auroc", factor=0.20, patience=40, verbose=1, mode="auto", min_delta=0.0001, cooldown=3, min_lr=0.000000001, ) early_stop = EarlyStopping(monitor="val_auroc", patience=200, verbose=1, mode="auto") candle_monitor = candle.CandleRemoteMonitor(params=params) candle_monitor = candle.CandleRemoteMonitor(params=params) timeout_monitor = candle.TerminateOnTimeOut(params["timeout"]) tensorboard = TensorBoard(log_dir="tb/tb{}".format(ext)) history_logger = LoggingCallback(attn.logger.debug) callbacks = [candle_monitor, timeout_monitor, csv_logger, history_logger] if params["reduce_lr"]: callbacks.append(reduce_lr) if params["use_cp"]: callbacks.append(checkpointer) if params["use_tb"]: callbacks.append(tensorboard) if params["early_stop"]: callbacks.append(early_stop) epochs = params["epochs"] batch_size = params["batch_size"] history = model.fit( X_train, Y_train, class_weight=d_class_weights, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(X_val, Y_val), callbacks=callbacks, ) # diagnostic plots if "loss" in history.history.keys(): candle.plot_history(params["save_path"] + root_fname, history, "loss") if "acc" in history.history.keys(): candle.plot_history(params["save_path"] + root_fname, history, "acc") if "auroc" in history.history.keys(): candle.plot_history(params["save_path"] + root_fname, history, "auroc") if "auprc" in history.history.keys(): candle.plot_history(params["save_path"] + root_fname, history, "aucpr") # Evaluate model score = model.evaluate(X_test, Y_test, verbose=0) Y_predict = model.predict(X_test) evaluate_model(params, root_fname, nb_classes, Y_test, _Y_test, Y_predict, pos, total, score) save_and_test_saved_model(params, model, root_fname, X_train, X_test, Y_test) attn.logger.handlers = [] return history
def run_cnn(GP, train_x, train_y, test_x, test_y, learning_rate=0.01, batch_size=10, epochs=10, dropout=0.5, optimizer='adam', wv_len=300, filter_sizes=[3, 4, 5], num_filters=[300, 300, 300], emb_l2=0.001, w_l2=0.01): max_vocab = np.max(train_x) max_vocab2 = np.max(test_x) if max_vocab2 > max_vocab: max_vocab = max_vocab2 wv_mat = np.random.randn(max_vocab + 1, wv_len).astype('float32') * 0.1 num_classes = [] num_classes.append(np.max(train_y[:, 0]) + 1) num_classes.append(np.max(train_y[:, 1]) + 1) num_classes.append(np.max(train_y[:, 2]) + 1) num_classes.append(np.max(train_y[:, 3]) + 1) kerasDefaults = candle.keras_default_config() optimizer_run = candle.build_optimizer(optimizer, learning_rate, kerasDefaults) cnn = keras_mt_shared_cnn.init_export_network(num_classes=num_classes, in_seq_len=1500, vocab_size=len(wv_mat), wv_space=wv_len, filter_sizes=filter_sizes, num_filters=num_filters, concat_dropout_prob=dropout, emb_l2=emb_l2, w_l2=w_l2, optimizer=optimizer_run) print(cnn.summary()) validation_data = ({ 'Input': test_x }, { 'Dense0': test_y[:, 0], 'Dense1': test_y[:, 1], 'Dense2': test_y[:, 2], 'Dense3': test_y[:, 3] }) # candleRemoteMonitor = CandleRemoteMonitor(params= GP) # timeoutMonitor = TerminateOnTimeOut(TIMEOUT) candleRemoteMonitor = candle.CandleRemoteMonitor(params=GP) timeoutMonitor = candle.TerminateOnTimeOut(GP['timeout']) history = cnn.fit(x=np.array(train_x), y=[ np.array(train_y[:, 0]), np.array(train_y[:, 1]), np.array(train_y[:, 2]), np.array(train_y[:, 3]) ], batch_size=batch_size, epochs=epochs, verbose=2, validation_data=validation_data, callbacks=[candleRemoteMonitor, timeoutMonitor]) return history
def run(params): args = candle.ArgumentStruct(**params) seed = args.rng_seed candle.set_seed(seed) # Construct extension to save model ext = adrp.extension_from_parameters(params, ".keras") candle.verify_path(params["save_path"]) prefix = "{}{}".format(params["save_path"], ext) logfile = params["logfile"] if params["logfile"] else prefix + ".log" candle.set_up_logger(logfile, adrp.logger, params["verbose"]) adrp.logger.info("Params: {}".format(params)) # Get default parameters for initialization and optimizer functions keras_defaults = candle.keras_default_config() ## X_train, Y_train, X_test, Y_test, PS = adrp.load_data(params, seed) print("X_train shape:", X_train.shape) print("X_test shape:", X_test.shape) print("Y_train shape:", Y_train.shape) print("Y_test shape:", Y_test.shape) # Initialize weights and learning rule initializer_weights = candle.build_initializer(params["initialization"], keras_defaults, seed) initializer_bias = candle.build_initializer("constant", keras_defaults, 0.0) activation = params["activation"] # TODO: set output_dim output_dim = 1 # TODO: Use dense_layers for creating inputs/outputs dense_layers = params["dense"] inputs = Input(shape=(PS, )) if dense_layers != None: if type(dense_layers) != list: dense_layers = list(dense_layers) for i, l in enumerate(dense_layers): if i == 0: x = Dense( l, activation=activation, kernel_initializer=initializer_weights, bias_initializer=initializer_bias, )(inputs) else: x = Dense( l, activation=activation, kernel_initializer=initializer_weights, bias_initializer=initializer_bias, )(x) if params["dropout"]: x = Dropout(params["dropout"])(x) output = Dense( output_dim, activation=activation, kernel_initializer=initializer_weights, bias_initializer=initializer_bias, )(x) else: output = Dense( output_dim, activation=activation, kernel_initializer=initializer_weights, bias_initializer=initializer_bias, )(inputs) model = Model(inputs=inputs, outputs=output) model.summary() kerasDefaults = candle.keras_default_config() if params["momentum"]: kerasDefaults["momentum_sgd"] = params["momentum"] optimizer = candle.build_optimizer(params["optimizer"], params["learning_rate"], kerasDefaults) model.compile( loss=params["loss"], optimizer=optimizer, metrics=["mae", r2], ) # set up a bunch of callbacks to do work during model training.. checkpointer = ModelCheckpoint( filepath=params["save_path"] + "agg_adrp.autosave.model.h5", verbose=1, save_weights_only=False, save_best_only=True, ) csv_logger = CSVLogger(params["save_path"] + "agg_adrp.training.log") reduce_lr = ReduceLROnPlateau( monitor="val_loss", factor=0.75, patience=20, mode="auto", epsilon=0.0001, cooldown=3, min_lr=0.000000001, ) early_stop = EarlyStopping(monitor="val_loss", patience=100, verbose=1, mode="auto") # history = parallel_model.fit(X_train, Y_train, epochs = params["epochs"] batch_size = params["batch_size"] history = model.fit( X_train, Y_train, batch_size=batch_size, epochs=epochs, verbose=1, validation_data=(X_test, Y_test), callbacks=[checkpointer, csv_logger, reduce_lr, early_stop], ) score = model.evaluate(X_test, Y_test, verbose=0) print(score) print(history.history.keys()) # see big fuction below, creates plots etc. # TODO: Break post_process into multiple functions post_process(params, X_train, X_test, Y_test, score, history, model) adrp.logger.handlers = [] return history