def optimizer_from_config(config): if 'class_name' not in config: raise ValueError('class_name is needed to define optimizer') if 'config' not in config: # auto add empty config for optimizer with only class_name config['config'] = {} return deserialize(config, custom_objects=Optimizers().optimizers)
def run_model_posttraining(config): data = load_data_source() x_train_list = [data[f'x_train_{i}'] for i in range(4)] y_train = data['y_train'] x_val_list = [data[f'x_val_{i}'] for i in range(4)] y_val = data['y_val'] num_epochs = config['hyperparameters']['num_epochs'] batch_size = 32 # config['hyperparameters']['batch_size'] config['create_structure']['func'] = util.load_attr_from( config['create_structure']['func']) input_shape = [np.shape(a)[1:] for a in x_train_list] output_shape = (1, ) cs_kwargs = config['create_structure'].get('kwargs') if cs_kwargs is None: structure = config['create_structure']['func'](input_shape, output_shape) else: structure = config['create_structure']['func'](input_shape, output_shape, **cs_kwargs) arch_seq = config['arch_seq'] structure.set_ops(arch_seq) structure.draw_graphviz('nas_model_uno.dot') model = structure.create_model() from keras.utils import plot_model plot_model(model, 'keras_model_uno.png', show_shapes=True) n_params = model.count_params() optimizer = optimizers.deserialize({'class_name': 'adam', 'config': {}}) model.compile(loss='mse', optimizer=optimizer, metrics=[mae, r2]) t1 = time.time() history = model.fit(x_train_list, y_train, batch_size=batch_size, epochs=num_epochs, validation_data=(x_val_list, y_val)) t2 = time.time() data = history.history data['n_parameters'] = n_params data['training_time'] = t2 - t1 print(data) return data
def build(self): from tensorflow.keras.optimizers import deserialize opt_config = {'class_name': self.name, 'config': self.config} opt = deserialize(opt_config) if self.horovod_wrapper: import horovod.keras as hvd if hasattr(opt, 'lr'): opt.lr *= hvd.size() opt = hvd.DistributedOptimizer(opt) return opt
def _test_optimizer(optimizer, target=0.75): x_train, y_train = get_test_data() model = get_model(x_train.shape[1], 10, y_train.shape[1]) model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy']) history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0) assert history.history['acc'][-1] >= target config = optimizers.serialize(optimizer) custom_objects = {optimizer.__class__.__name__: optimizer.__class__} optim = optimizers.deserialize(config, custom_objects) new_config = optimizers.serialize(optim) assert config == new_config
def clone_optimizer(optimizer): if type(optimizer) is str: return optimizers.get(optimizer) # Requires Keras 1.0.7 since get_config has breaking changes. params = dict([(k, v) for k, v in optimizer.get_config().items()]) config = { 'class_name': optimizer.__class__.__name__, 'config': params, } if hasattr(optimizers, 'optimizer_from_config'): # COMPATIBILITY: Keras < 2.0 clone = optimizers.optimizer_from_config(config) else: clone = optimizers.deserialize(config) return clone
def run_model(config): params = initialize_parameters() args = Struct(**params) data = load_data2() x_train_list = [data[f'x_train_{i}'] for i in range(4)] y_train = data['y_train'] x_val_list = [data[f'x_val_{i}'] for i in range(4)] y_val = data['y_val'] num_epochs = config['hyperparameters']['num_epochs'] batch_size = args.batch_size # config['hyperparameters']['batch_size'] config['create_structure']['func'] = util.load_attr_from( config['create_structure']['func']) input_shape = [np.shape(a)[1:] for a in x_train_list] print('input_shape: ', input_shape) output_shape = (1, ) print('output_shape: ', output_shape) cs_kwargs = config['create_structure'].get('kwargs') if cs_kwargs is None: structure = config['create_structure']['func'](input_shape, output_shape) else: structure = config['create_structure']['func'](input_shape, output_shape, **cs_kwargs) arch_seq = config['arch_seq'] print(f'actions list: {arch_seq}') structure.set_ops(arch_seq) #structure.draw_graphviz('model_global_uno.dot') model = structure.create_model() #from keras.utils import plot_model #plot_model(model, 'model_global_combo.png', show_shapes=True) model.summary() optimizer = optimizers.deserialize({'class_name': 'adam', 'config': {}}) model.compile(loss='mse', optimizer=optimizer, metrics=[mae, r2]) stop_if_unfeasible = StopIfUnfeasible(time_limit=900) history = model.fit(x_train_list, y_train, batch_size=batch_size, epochs=num_epochs, callbacks=[stop_if_unfeasible], validation_data=(x_val_list, y_val)) print('avr_batch_timing :', stop_if_unfeasible.avr_batch_time) print('avr_timing: ', stop_if_unfeasible.estimate_training_time) print('stopped: ', stop_if_unfeasible.stopped) print(history.history) try: return history.history['val_r2'][0] except: return -1.0
def run(params): args = Struct(**params) set_seed(args.rng_seed) ext = extension_from_parameters(args) verify_path(args.save) prefix = args.save + ext logfile = args.logfile if args.logfile else prefix + '.log' set_up_logger(logfile, args.verbose) logger.info('Params: {}'.format(params)) loader = CombinedDataLoader(seed=args.rng_seed) loader.load( cache=args.cache, ncols=args.feature_subsample, agg_dose=args.agg_dose, cell_features=args.cell_features, drug_features=args.drug_features, drug_median_response_min=args.drug_median_response_min, drug_median_response_max=args.drug_median_response_max, use_landmark_genes=args.use_landmark_genes, use_filtered_genes=args.use_filtered_genes, cell_feature_subset_path=args.cell_feature_subset_path or args.feature_subset_path, drug_feature_subset_path=args.drug_feature_subset_path or args.feature_subset_path, preprocess_rnaseq=args.preprocess_rnaseq, single=args.single, train_sources=args.train_sources, test_sources=args.test_sources, embed_feature_source=not args.no_feature_source, encode_response_source=not args.no_response_source, ) target = args.agg_dose or 'Growth' val_split = args.validation_split train_split = 1 - val_split if args.export_data: fname = args.export_data loader.partition_data(cv_folds=args.cv, train_split=train_split, val_split=val_split, cell_types=args.cell_types, by_cell=args.by_cell, by_drug=args.by_drug, cell_subset_path=args.cell_subset_path, drug_subset_path=args.drug_subset_path) train_gen = CombinedDataGenerator(loader, batch_size=args.batch_size, shuffle=args.shuffle) val_gen = CombinedDataGenerator(loader, partition='val', batch_size=args.batch_size, shuffle=args.shuffle) x_train_list, y_train = train_gen.get_slice(size=train_gen.size, dataframe=True, single=args.single) x_val_list, y_val = val_gen.get_slice(size=val_gen.size, dataframe=True, single=args.single) df_train = pd.concat([y_train] + x_train_list, axis=1) df_val = pd.concat([y_val] + x_val_list, axis=1) df = pd.concat([df_train, df_val]).reset_index(drop=True) if args.growth_bins > 1: df = uno_data.discretize(df, 'Growth', bins=args.growth_bins) df.to_csv(fname, sep='\t', index=False, float_format="%.3g") return loader.partition_data(cv_folds=args.cv, train_split=train_split, val_split=val_split, cell_types=args.cell_types, by_cell=args.by_cell, by_drug=args.by_drug, cell_subset_path=args.cell_subset_path, drug_subset_path=args.drug_subset_path) model = build_model(loader, args) logger.info('Combined model:') model.summary(print_fn=logger.info) plot_model(model, to_file=prefix + '.model.png', show_shapes=True) if args.cp: model_json = model.to_json() with open(prefix + '.model.json', 'w') as f: print(model_json, file=f) def warmup_scheduler(epoch): lr = args.learning_rate or base_lr * args.batch_size / 100 if epoch <= 5: K.set_value(model.optimizer.lr, (base_lr * (5 - epoch) + lr * epoch) / 5) logger.debug('Epoch {}: lr={:.5g}'.format( epoch, K.get_value(model.optimizer.lr))) return K.get_value(model.optimizer.lr) df_pred_list = [] cv_ext = '' cv = args.cv if args.cv > 1 else 1 for fold in range(cv): if args.cv > 1: logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv)) cv_ext = '.cv{}'.format(fold + 1) model = build_model(loader, args, silent=True) optimizer = optimizers.deserialize({ 'class_name': args.optimizer, 'config': {} }) base_lr = args.base_lr or K.get_value(optimizer.lr) if args.learning_rate: K.set_value(optimizer.lr, args.learning_rate) model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2]) # calculate trainable and non-trainable params params.update(candle.compute_trainable_params(model)) candle_monitor = candle.CandleRemoteMonitor(params=params) timeout_monitor = candle.TerminateOnTimeOut(params['timeout']) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=0.00001) warmup_lr = LearningRateScheduler(warmup_scheduler) checkpointer = ModelCheckpoint(prefix + cv_ext + '.weights.h5', save_best_only=True, save_weights_only=True) tensorboard = TensorBoard(log_dir="tb/tb{}{}".format(ext, cv_ext)) history_logger = LoggingCallback(logger.debug) model_recorder = ModelRecorder() # callbacks = [history_logger, model_recorder] callbacks = [ candle_monitor, timeout_monitor, history_logger, model_recorder ] # callbacks = [candle_monitor, history_logger, model_recorder] # if args.reduce_lr: callbacks.append(reduce_lr) if args.warmup_lr: callbacks.append(warmup_lr) if args.cp: callbacks.append(checkpointer) if args.tb: callbacks.append(tensorboard) train_gen = CombinedDataGenerator(loader, fold=fold, batch_size=args.batch_size, shuffle=args.shuffle) val_gen = CombinedDataGenerator(loader, partition='val', fold=fold, batch_size=args.batch_size, shuffle=args.shuffle) df_val = val_gen.get_response(copy=True) y_val = df_val[target].values y_shuf = np.random.permutation(y_val) log_evaluation(evaluate_prediction(y_val, y_shuf), description='Between random pairs in y_val:') if args.no_gen: x_train_list, y_train = train_gen.get_slice(size=train_gen.size, single=args.single) x_val_list, y_val = val_gen.get_slice(size=val_gen.size, single=args.single) history = model.fit(x_train_list, y_train, batch_size=args.batch_size, epochs=args.epochs, callbacks=callbacks, validation_data=(x_val_list, y_val)) else: logger.info('Data points per epoch: train = %d, val = %d', train_gen.size, val_gen.size) logger.info('Steps per epoch: train = %d, val = %d', train_gen.steps, val_gen.steps) history = model.fit_generator( train_gen.flow(single=args.single), train_gen.steps, epochs=args.epochs, callbacks=callbacks, validation_data=val_gen.flow(single=args.single), validation_steps=val_gen.steps) if args.cp: model = model_recorder.best_model model.save(prefix + '.model.h5') # model.load_weights(prefix+cv_ext+'.weights.h5') if args.no_gen: y_val_pred = model.predict(x_val_list, batch_size=args.batch_size) else: val_gen.reset() y_val_pred = model.predict_generator( val_gen.flow(single=args.single), val_gen.steps) y_val_pred = y_val_pred[:val_gen.size] y_val_pred = y_val_pred.flatten() scores = evaluate_prediction(y_val, y_val_pred) log_evaluation(scores) # df_val = df_val.assign(PredictedGrowth=y_val_pred, GrowthError=y_val_pred-y_val) df_val['Predicted' + target] = y_val_pred df_val[target + 'Error'] = y_val_pred - y_val df_pred_list.append(df_val) plot_history(prefix, history, 'loss') plot_history(prefix, history, 'r2') pred_fname = prefix + '.predicted.tsv' df_pred = pd.concat(df_pred_list) if args.agg_dose: df_pred.sort_values(['Source', 'Sample', 'Drug1', 'Drug2', target], inplace=True) else: df_pred.sort_values( ['Source', 'Sample', 'Drug1', 'Drug2', 'Dose1', 'Dose2', 'Growth'], inplace=True) df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g') if args.cv > 1: scores = evaluate_prediction(df_pred[target], df_pred['Predicted' + target]) log_evaluation(scores, description='Combining cross validation folds:') for test_source in loader.test_sep_sources: test_gen = CombinedDataGenerator(loader, partition='test', batch_size=args.batch_size, source=test_source) df_test = test_gen.get_response(copy=True) y_test = df_test[target].values n_test = len(y_test) if n_test == 0: continue if args.no_gen: x_test_list, y_test = test_gen.get_slice(size=test_gen.size, single=args.single) y_test_pred = model.predict(x_test_list, batch_size=args.batch_size) else: y_test_pred = model.predict_generator( test_gen.flow(single=args.single), test_gen.steps) y_test_pred = y_test_pred[:test_gen.size] y_test_pred = y_test_pred.flatten() scores = evaluate_prediction(y_test, y_test_pred) log_evaluation(scores, description='Testing on data from {} ({})'.format( test_source, n_test)) if K.backend() == 'tensorflow': K.clear_session() logger.handlers = [] return history
def from_config(cls, config): optimizer = optimizers.deserialize(config.pop('optimizer')) return cls(optimizer, **config)
def run_model(config): t1 = time.time() num_epochs = config['hyperparameters']['num_epochs'] config['create_structure']['func'] = util.load_attr_from( config['create_structure']['func']) input_shape = [(942, ), (3820, ), (3820, )] output_shape = (1, ) cs_kwargs = config['create_structure'].get('kwargs') if cs_kwargs is None: structure = config['create_structure']['func'](input_shape, output_shape) else: structure = config['create_structure']['func'](input_shape, output_shape, **cs_kwargs) arch_seq = config['arch_seq'] print(f'actions list: {arch_seq}') structure.set_ops(arch_seq) # structure.draw_graphviz('model_global_combo.dot') model = structure.create_model() # from keras.utils import plot_model # plot_model(model, 'model_global_combo.png', show_shapes=True) model.summary() t2 = time.time() t_model_create = t2 - t1 print('Time model creation: ', t_model_create) import sys t1 = time.time() params = initialize_parameters() args = Struct(**params) set_seed(args.rng_seed) optimizer = optimizers.deserialize({ 'class_name': args.optimizer, 'config': {} }) base_lr = args.base_lr or K.get_value(optimizer.lr) if args.learning_rate: K.set_value(optimizer.lr, args.learning_rate) model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2]) if config.get('load_data') is None: data = combo_ld_numpy(args) else: if not (config['load_data'].get('prop') is None): print('Data prop: ', config['load_data']['prop']) data = combo_ld_numpy(args, prop=config['load_data']['prop']) else: data = combo_ld_numpy(args) x_train_list = [data['x_train_0'], data['x_train_1'], data['x_train_2']] y_train = data['y_train'] x_val_list = [data['x_val_0'], data['x_val_1'], data['x_val_2']] y_val = data['y_val'] print('y_val shape: ', np.shape(y_val)) t2 = time.time() t_data_loading = t2 - t1 print('Time data loading: ', t_data_loading) stop_if_unfeasible = StopIfUnfeasible(time_limit=1200) t1 = time.time() history = model.fit(x_train_list, y_train, batch_size=args.batch_size, shuffle=args.shuffle, epochs=num_epochs, callbacks=[stop_if_unfeasible], validation_data=(x_val_list, y_val)) t2 = time.time() t_training = t2 - t1 print('Time training: ', t_training) print('avr_batch_timing :', stop_if_unfeasible.avr_batch_time) print('avr_timing: ', stop_if_unfeasible.estimate_training_time) print('stopped: ', stop_if_unfeasible.stopped) print(history.history) try: return history.history['val_r2'][0] except: return -1.0
def run(params): args = Struct(**params) set_seed(args.rng_seed) ext = extension_from_parameters(args) verify_path(args.save) prefix = args.save + ext logfile = args.logfile if args.logfile else prefix + '.log' set_up_logger(logfile, args.verbose) logger.info('Params: {}'.format(params)) loader = ComboDataLoader(seed=args.rng_seed, val_split=args.validation_split, cell_features=args.cell_features, drug_features=args.drug_features, response_url=args.response_url, use_landmark_genes=args.use_landmark_genes, preprocess_rnaseq=args.preprocess_rnaseq, exclude_cells=args.exclude_cells, exclude_drugs=args.exclude_drugs, use_combo_score=args.use_combo_score, cv_partition=args.cv_partition, cv=args.cv) # test_loader(loader) # test_generator(loader) train_gen = ComboDataGenerator(loader, batch_size=args.batch_size).flow() val_gen = ComboDataGenerator(loader, partition='val', batch_size=args.batch_size).flow() train_steps = int(loader.n_train / args.batch_size) val_steps = int(loader.n_val / args.batch_size) model = build_model(loader, args, verbose=True) print('Creating model PNG') from keras.utils import plot_model plot_model(model, 'model_global_combo.png', show_shapes=True) print('Model PNG has been created successfuly!') model.summary() # plot_model(model, to_file=prefix+'.model.png', show_shapes=True) if args.cp: model_json = model.to_json() with open(prefix + '.model.json', 'w') as f: print(model_json, file=f) def warmup_scheduler(epoch): lr = args.learning_rate or base_lr * args.batch_size / 100 if epoch <= 5: K.set_value(model.optimizer.lr, (base_lr * (5 - epoch) + lr * epoch) / 5) logger.debug('Epoch {}: lr={}'.format(epoch, K.get_value(model.optimizer.lr))) return K.get_value(model.optimizer.lr) df_pred_list = [] cv_ext = '' cv = args.cv if args.cv > 1 else 1 fold = 0 while fold < cv: if args.cv > 1: logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv)) cv_ext = '.cv{}'.format(fold + 1) model = build_model(loader, args) optimizer = optimizers.deserialize({ 'class_name': args.optimizer, 'config': {} }) base_lr = args.base_lr or K.get_value(optimizer.lr) if args.learning_rate: K.set_value(optimizer.lr, args.learning_rate) model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2]) # calculate trainable and non-trainable params params.update(candle.compute_trainable_params(model)) candle_monitor = candle.CandleRemoteMonitor(params=params) timeout_monitor = candle.TerminateOnTimeOut(params['timeout']) reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=0.00001) warmup_lr = LearningRateScheduler(warmup_scheduler) checkpointer = ModelCheckpoint(prefix + cv_ext + '.weights.h5', save_best_only=True, save_weights_only=True) tensorboard = TensorBoard(log_dir="tb/tb{}{}".format(ext, cv_ext)) history_logger = LoggingCallback(logger.debug) model_recorder = ModelRecorder() # callbacks = [history_logger, model_recorder] callbacks = [ candle_monitor, timeout_monitor, history_logger, model_recorder ] if args.reduce_lr: callbacks.append(reduce_lr) if args.warmup_lr: callbacks.append(warmup_lr) if args.cp: callbacks.append(checkpointer) if args.tb: callbacks.append(tensorboard) if args.gen: history = model.fit_generator(train_gen, train_steps, epochs=args.epochs, callbacks=callbacks, validation_data=val_gen, validation_steps=val_steps) fold += 1 else: if args.cv > 1: x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data_cv( fold) else: x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data( ) y_shuf = np.random.permutation(y_val) log_evaluation(evaluate_prediction(y_val, y_shuf), description='Between random pairs in y_val:') history = model.fit(x_train_list, y_train, batch_size=args.batch_size, shuffle=args.shuffle, epochs=args.epochs, callbacks=callbacks, validation_data=(x_val_list, y_val)) if args.cp: model.load_weights(prefix + cv_ext + '.weights.h5') if not args.gen: y_val_pred = model.predict(x_val_list, batch_size=args.batch_size).flatten() scores = evaluate_prediction(y_val, y_val_pred) if args.cv > 1 and scores[args.loss] > args.max_val_loss: logger.warn( 'Best val_loss {} is greater than {}; retrain the model...' .format(scores[args.loss], args.max_val_loss)) continue else: fold += 1 log_evaluation(scores) df_val.is_copy = False df_val['GROWTH_PRED'] = y_val_pred df_val['GROWTH_ERROR'] = y_val_pred - y_val df_pred_list.append(df_val) if args.cp: # model.save(prefix+'.model.h5') model_recorder.best_model.save(prefix + '.model.h5') # test reloadded model prediction # new_model = keras.models.load_model(prefix+'.model.h5') # new_model.load_weights(prefix+cv_ext+'.weights.h5') # new_pred = new_model.predict(x_val_list, batch_size=args.batch_size).flatten() # print('y_val:', y_val[:10]) # print('old_pred:', y_val_pred[:10]) # print('new_pred:', new_pred[:10]) plot_history(prefix, history, 'loss') plot_history(prefix, history, 'r2') if K.backend() == 'tensorflow': K.clear_session() if not args.gen: pred_fname = prefix + '.predicted.growth.tsv' if args.use_combo_score: pred_fname = prefix + '.predicted.score.tsv' df_pred = pd.concat(df_pred_list) df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g') logger.handlers = [] return history