def optimizer_from_config(config):
if 'class_name' not in config:
raise ValueError('class_name is needed to define optimizer')
if 'config' not in config:
# auto add empty config for optimizer with only class_name
config['config'] = {}
return deserialize(config, custom_objects=Optimizers().optimizers)
def run_model_posttraining(config):
data = load_data_source()
x_train_list = [data[f'x_train_{i}'] for i in range(4)]
y_train = data['y_train']
x_val_list = [data[f'x_val_{i}'] for i in range(4)]
y_val = data['y_val']
num_epochs = config['hyperparameters']['num_epochs']
batch_size = 32 # config['hyperparameters']['batch_size']
config['create_structure']['func'] = util.load_attr_from(
config['create_structure']['func'])
input_shape = [np.shape(a)[1:] for a in x_train_list]
output_shape = (1, )
cs_kwargs = config['create_structure'].get('kwargs')
if cs_kwargs is None:
structure = config['create_structure']['func'](input_shape,
output_shape)
else:
structure = config['create_structure']['func'](input_shape,
output_shape,
**cs_kwargs)
arch_seq = config['arch_seq']
structure.set_ops(arch_seq)
structure.draw_graphviz('nas_model_uno.dot')
model = structure.create_model()
from keras.utils import plot_model
plot_model(model, 'keras_model_uno.png', show_shapes=True)
n_params = model.count_params()
optimizer = optimizers.deserialize({'class_name': 'adam', 'config': {}})
model.compile(loss='mse', optimizer=optimizer, metrics=[mae, r2])
t1 = time.time()
history = model.fit(x_train_list,
y_train,
batch_size=batch_size,
epochs=num_epochs,
validation_data=(x_val_list, y_val))
t2 = time.time()
data = history.history
data['n_parameters'] = n_params
data['training_time'] = t2 - t1
print(data)
return data
def build(self):
from tensorflow.keras.optimizers import deserialize
opt_config = {'class_name': self.name, 'config': self.config}
opt = deserialize(opt_config)
if self.horovod_wrapper:
import horovod.keras as hvd
if hasattr(opt, 'lr'):
opt.lr *= hvd.size()
opt = hvd.DistributedOptimizer(opt)
return opt
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = get_model(x_train.shape[1], 10, y_train.shape[1])
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
config = optimizers.serialize(optimizer)
custom_objects = {optimizer.__class__.__name__: optimizer.__class__}
optim = optimizers.deserialize(config, custom_objects)
new_config = optimizers.serialize(optim)
assert config == new_config
def clone_optimizer(optimizer):
if type(optimizer) is str:
return optimizers.get(optimizer)
# Requires Keras 1.0.7 since get_config has breaking changes.
params = dict([(k, v) for k, v in optimizer.get_config().items()])
config = {
'class_name': optimizer.__class__.__name__,
'config': params,
}
if hasattr(optimizers, 'optimizer_from_config'):
# COMPATIBILITY: Keras < 2.0
clone = optimizers.optimizer_from_config(config)
else:
clone = optimizers.deserialize(config)
return clone
def run_model(config):
params = initialize_parameters()
args = Struct(**params)
data = load_data2()
x_train_list = [data[f'x_train_{i}'] for i in range(4)]
y_train = data['y_train']
x_val_list = [data[f'x_val_{i}'] for i in range(4)]
y_val = data['y_val']
num_epochs = config['hyperparameters']['num_epochs']
batch_size = args.batch_size # config['hyperparameters']['batch_size']
config['create_structure']['func'] = util.load_attr_from(
config['create_structure']['func'])
input_shape = [np.shape(a)[1:] for a in x_train_list]
print('input_shape: ', input_shape)
output_shape = (1, )
print('output_shape: ', output_shape)
cs_kwargs = config['create_structure'].get('kwargs')
if cs_kwargs is None:
structure = config['create_structure']['func'](input_shape,
output_shape)
else:
structure = config['create_structure']['func'](input_shape,
output_shape,
**cs_kwargs)
arch_seq = config['arch_seq']
print(f'actions list: {arch_seq}')
structure.set_ops(arch_seq)
#structure.draw_graphviz('model_global_uno.dot')
model = structure.create_model()
#from keras.utils import plot_model
#plot_model(model, 'model_global_combo.png', show_shapes=True)
model.summary()
optimizer = optimizers.deserialize({'class_name': 'adam', 'config': {}})
model.compile(loss='mse', optimizer=optimizer, metrics=[mae, r2])
stop_if_unfeasible = StopIfUnfeasible(time_limit=900)
history = model.fit(x_train_list,
y_train,
batch_size=batch_size,
epochs=num_epochs,
callbacks=[stop_if_unfeasible],
validation_data=(x_val_list, y_val))
print('avr_batch_timing :', stop_if_unfeasible.avr_batch_time)
print('avr_timing: ', stop_if_unfeasible.estimate_training_time)
print('stopped: ', stop_if_unfeasible.stopped)
print(history.history)
try:
return history.history['val_r2'][0]
except:
return -1.0
def run(params):
args = Struct(**params)
set_seed(args.rng_seed)
ext = extension_from_parameters(args)
verify_path(args.save)
prefix = args.save + ext
logfile = args.logfile if args.logfile else prefix + '.log'
set_up_logger(logfile, args.verbose)
logger.info('Params: {}'.format(params))
loader = CombinedDataLoader(seed=args.rng_seed)
loader.load(
cache=args.cache,
ncols=args.feature_subsample,
agg_dose=args.agg_dose,
cell_features=args.cell_features,
drug_features=args.drug_features,
drug_median_response_min=args.drug_median_response_min,
drug_median_response_max=args.drug_median_response_max,
use_landmark_genes=args.use_landmark_genes,
use_filtered_genes=args.use_filtered_genes,
cell_feature_subset_path=args.cell_feature_subset_path
or args.feature_subset_path,
drug_feature_subset_path=args.drug_feature_subset_path
or args.feature_subset_path,
preprocess_rnaseq=args.preprocess_rnaseq,
single=args.single,
train_sources=args.train_sources,
test_sources=args.test_sources,
embed_feature_source=not args.no_feature_source,
encode_response_source=not args.no_response_source,
)
target = args.agg_dose or 'Growth'
val_split = args.validation_split
train_split = 1 - val_split
if args.export_data:
fname = args.export_data
loader.partition_data(cv_folds=args.cv,
train_split=train_split,
val_split=val_split,
cell_types=args.cell_types,
by_cell=args.by_cell,
by_drug=args.by_drug,
cell_subset_path=args.cell_subset_path,
drug_subset_path=args.drug_subset_path)
train_gen = CombinedDataGenerator(loader,
batch_size=args.batch_size,
shuffle=args.shuffle)
val_gen = CombinedDataGenerator(loader,
partition='val',
batch_size=args.batch_size,
shuffle=args.shuffle)
x_train_list, y_train = train_gen.get_slice(size=train_gen.size,
dataframe=True,
single=args.single)
x_val_list, y_val = val_gen.get_slice(size=val_gen.size,
dataframe=True,
single=args.single)
df_train = pd.concat([y_train] + x_train_list, axis=1)
df_val = pd.concat([y_val] + x_val_list, axis=1)
df = pd.concat([df_train, df_val]).reset_index(drop=True)
if args.growth_bins > 1:
df = uno_data.discretize(df, 'Growth', bins=args.growth_bins)
df.to_csv(fname, sep='\t', index=False, float_format="%.3g")
return
loader.partition_data(cv_folds=args.cv,
train_split=train_split,
val_split=val_split,
cell_types=args.cell_types,
by_cell=args.by_cell,
by_drug=args.by_drug,
cell_subset_path=args.cell_subset_path,
drug_subset_path=args.drug_subset_path)
model = build_model(loader, args)
logger.info('Combined model:')
model.summary(print_fn=logger.info)
plot_model(model, to_file=prefix + '.model.png', show_shapes=True)
if args.cp:
model_json = model.to_json()
with open(prefix + '.model.json', 'w') as f:
print(model_json, file=f)
def warmup_scheduler(epoch):
lr = args.learning_rate or base_lr * args.batch_size / 100
if epoch <= 5:
K.set_value(model.optimizer.lr,
(base_lr * (5 - epoch) + lr * epoch) / 5)
logger.debug('Epoch {}: lr={:.5g}'.format(
epoch, K.get_value(model.optimizer.lr)))
return K.get_value(model.optimizer.lr)
df_pred_list = []
cv_ext = ''
cv = args.cv if args.cv > 1 else 1
for fold in range(cv):
if args.cv > 1:
logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv))
cv_ext = '.cv{}'.format(fold + 1)
model = build_model(loader, args, silent=True)
optimizer = optimizers.deserialize({
'class_name': args.optimizer,
'config': {}
})
base_lr = args.base_lr or K.get_value(optimizer.lr)
if args.learning_rate:
K.set_value(optimizer.lr, args.learning_rate)
model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2])
# calculate trainable and non-trainable params
params.update(candle.compute_trainable_params(model))
candle_monitor = candle.CandleRemoteMonitor(params=params)
timeout_monitor = candle.TerminateOnTimeOut(params['timeout'])
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001)
warmup_lr = LearningRateScheduler(warmup_scheduler)
checkpointer = ModelCheckpoint(prefix + cv_ext + '.weights.h5',
save_best_only=True,
save_weights_only=True)
tensorboard = TensorBoard(log_dir="tb/tb{}{}".format(ext, cv_ext))
history_logger = LoggingCallback(logger.debug)
model_recorder = ModelRecorder()
# callbacks = [history_logger, model_recorder]
callbacks = [
candle_monitor, timeout_monitor, history_logger, model_recorder
]
# callbacks = [candle_monitor, history_logger, model_recorder] #
if args.reduce_lr:
callbacks.append(reduce_lr)
if args.warmup_lr:
callbacks.append(warmup_lr)
if args.cp:
callbacks.append(checkpointer)
if args.tb:
callbacks.append(tensorboard)
train_gen = CombinedDataGenerator(loader,
fold=fold,
batch_size=args.batch_size,
shuffle=args.shuffle)
val_gen = CombinedDataGenerator(loader,
partition='val',
fold=fold,
batch_size=args.batch_size,
shuffle=args.shuffle)
df_val = val_gen.get_response(copy=True)
y_val = df_val[target].values
y_shuf = np.random.permutation(y_val)
log_evaluation(evaluate_prediction(y_val, y_shuf),
description='Between random pairs in y_val:')
if args.no_gen:
x_train_list, y_train = train_gen.get_slice(size=train_gen.size,
single=args.single)
x_val_list, y_val = val_gen.get_slice(size=val_gen.size,
single=args.single)
history = model.fit(x_train_list,
y_train,
batch_size=args.batch_size,
epochs=args.epochs,
callbacks=callbacks,
validation_data=(x_val_list, y_val))
else:
logger.info('Data points per epoch: train = %d, val = %d',
train_gen.size, val_gen.size)
logger.info('Steps per epoch: train = %d, val = %d',
train_gen.steps, val_gen.steps)
history = model.fit_generator(
train_gen.flow(single=args.single),
train_gen.steps,
epochs=args.epochs,
callbacks=callbacks,
validation_data=val_gen.flow(single=args.single),
validation_steps=val_gen.steps)
if args.cp:
model = model_recorder.best_model
model.save(prefix + '.model.h5')
# model.load_weights(prefix+cv_ext+'.weights.h5')
if args.no_gen:
y_val_pred = model.predict(x_val_list, batch_size=args.batch_size)
else:
val_gen.reset()
y_val_pred = model.predict_generator(
val_gen.flow(single=args.single), val_gen.steps)
y_val_pred = y_val_pred[:val_gen.size]
y_val_pred = y_val_pred.flatten()
scores = evaluate_prediction(y_val, y_val_pred)
log_evaluation(scores)
# df_val = df_val.assign(PredictedGrowth=y_val_pred, GrowthError=y_val_pred-y_val)
df_val['Predicted' + target] = y_val_pred
df_val[target + 'Error'] = y_val_pred - y_val
df_pred_list.append(df_val)
plot_history(prefix, history, 'loss')
plot_history(prefix, history, 'r2')
pred_fname = prefix + '.predicted.tsv'
df_pred = pd.concat(df_pred_list)
if args.agg_dose:
df_pred.sort_values(['Source', 'Sample', 'Drug1', 'Drug2', target],
inplace=True)
else:
df_pred.sort_values(
['Source', 'Sample', 'Drug1', 'Drug2', 'Dose1', 'Dose2', 'Growth'],
inplace=True)
df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g')
if args.cv > 1:
scores = evaluate_prediction(df_pred[target],
df_pred['Predicted' + target])
log_evaluation(scores, description='Combining cross validation folds:')
for test_source in loader.test_sep_sources:
test_gen = CombinedDataGenerator(loader,
partition='test',
batch_size=args.batch_size,
source=test_source)
df_test = test_gen.get_response(copy=True)
y_test = df_test[target].values
n_test = len(y_test)
if n_test == 0:
continue
if args.no_gen:
x_test_list, y_test = test_gen.get_slice(size=test_gen.size,
single=args.single)
y_test_pred = model.predict(x_test_list,
batch_size=args.batch_size)
else:
y_test_pred = model.predict_generator(
test_gen.flow(single=args.single), test_gen.steps)
y_test_pred = y_test_pred[:test_gen.size]
y_test_pred = y_test_pred.flatten()
scores = evaluate_prediction(y_test, y_test_pred)
log_evaluation(scores,
description='Testing on data from {} ({})'.format(
test_source, n_test))
if K.backend() == 'tensorflow':
K.clear_session()
logger.handlers = []
return history
def from_config(cls, config):
optimizer = optimizers.deserialize(config.pop('optimizer'))
return cls(optimizer, **config)
def run_model(config):
t1 = time.time()
num_epochs = config['hyperparameters']['num_epochs']
config['create_structure']['func'] = util.load_attr_from(
config['create_structure']['func'])
input_shape = [(942, ), (3820, ), (3820, )]
output_shape = (1, )
cs_kwargs = config['create_structure'].get('kwargs')
if cs_kwargs is None:
structure = config['create_structure']['func'](input_shape,
output_shape)
else:
structure = config['create_structure']['func'](input_shape,
output_shape,
**cs_kwargs)
arch_seq = config['arch_seq']
print(f'actions list: {arch_seq}')
structure.set_ops(arch_seq)
# structure.draw_graphviz('model_global_combo.dot')
model = structure.create_model()
# from keras.utils import plot_model
# plot_model(model, 'model_global_combo.png', show_shapes=True)
model.summary()
t2 = time.time()
t_model_create = t2 - t1
print('Time model creation: ', t_model_create)
import sys
t1 = time.time()
params = initialize_parameters()
args = Struct(**params)
set_seed(args.rng_seed)
optimizer = optimizers.deserialize({
'class_name': args.optimizer,
'config': {}
})
base_lr = args.base_lr or K.get_value(optimizer.lr)
if args.learning_rate:
K.set_value(optimizer.lr, args.learning_rate)
model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2])
if config.get('load_data') is None:
data = combo_ld_numpy(args)
else:
if not (config['load_data'].get('prop') is None):
print('Data prop: ', config['load_data']['prop'])
data = combo_ld_numpy(args, prop=config['load_data']['prop'])
else:
data = combo_ld_numpy(args)
x_train_list = [data['x_train_0'], data['x_train_1'], data['x_train_2']]
y_train = data['y_train']
x_val_list = [data['x_val_0'], data['x_val_1'], data['x_val_2']]
y_val = data['y_val']
print('y_val shape: ', np.shape(y_val))
t2 = time.time()
t_data_loading = t2 - t1
print('Time data loading: ', t_data_loading)
stop_if_unfeasible = StopIfUnfeasible(time_limit=1200)
t1 = time.time()
history = model.fit(x_train_list,
y_train,
batch_size=args.batch_size,
shuffle=args.shuffle,
epochs=num_epochs,
callbacks=[stop_if_unfeasible],
validation_data=(x_val_list, y_val))
t2 = time.time()
t_training = t2 - t1
print('Time training: ', t_training)
print('avr_batch_timing :', stop_if_unfeasible.avr_batch_time)
print('avr_timing: ', stop_if_unfeasible.estimate_training_time)
print('stopped: ', stop_if_unfeasible.stopped)
print(history.history)
try:
return history.history['val_r2'][0]
except:
return -1.0
def run(params):
args = Struct(**params)
set_seed(args.rng_seed)
ext = extension_from_parameters(args)
verify_path(args.save)
prefix = args.save + ext
logfile = args.logfile if args.logfile else prefix + '.log'
set_up_logger(logfile, args.verbose)
logger.info('Params: {}'.format(params))
loader = ComboDataLoader(seed=args.rng_seed,
val_split=args.validation_split,
cell_features=args.cell_features,
drug_features=args.drug_features,
response_url=args.response_url,
use_landmark_genes=args.use_landmark_genes,
preprocess_rnaseq=args.preprocess_rnaseq,
exclude_cells=args.exclude_cells,
exclude_drugs=args.exclude_drugs,
use_combo_score=args.use_combo_score,
cv_partition=args.cv_partition,
cv=args.cv)
# test_loader(loader)
# test_generator(loader)
train_gen = ComboDataGenerator(loader, batch_size=args.batch_size).flow()
val_gen = ComboDataGenerator(loader,
partition='val',
batch_size=args.batch_size).flow()
train_steps = int(loader.n_train / args.batch_size)
val_steps = int(loader.n_val / args.batch_size)
model = build_model(loader, args, verbose=True)
print('Creating model PNG')
from keras.utils import plot_model
plot_model(model, 'model_global_combo.png', show_shapes=True)
print('Model PNG has been created successfuly!')
model.summary()
# plot_model(model, to_file=prefix+'.model.png', show_shapes=True)
if args.cp:
model_json = model.to_json()
with open(prefix + '.model.json', 'w') as f:
print(model_json, file=f)
def warmup_scheduler(epoch):
lr = args.learning_rate or base_lr * args.batch_size / 100
if epoch <= 5:
K.set_value(model.optimizer.lr,
(base_lr * (5 - epoch) + lr * epoch) / 5)
logger.debug('Epoch {}: lr={}'.format(epoch,
K.get_value(model.optimizer.lr)))
return K.get_value(model.optimizer.lr)
df_pred_list = []
cv_ext = ''
cv = args.cv if args.cv > 1 else 1
fold = 0
while fold < cv:
if args.cv > 1:
logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv))
cv_ext = '.cv{}'.format(fold + 1)
model = build_model(loader, args)
optimizer = optimizers.deserialize({
'class_name': args.optimizer,
'config': {}
})
base_lr = args.base_lr or K.get_value(optimizer.lr)
if args.learning_rate:
K.set_value(optimizer.lr, args.learning_rate)
model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2])
# calculate trainable and non-trainable params
params.update(candle.compute_trainable_params(model))
candle_monitor = candle.CandleRemoteMonitor(params=params)
timeout_monitor = candle.TerminateOnTimeOut(params['timeout'])
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001)
warmup_lr = LearningRateScheduler(warmup_scheduler)
checkpointer = ModelCheckpoint(prefix + cv_ext + '.weights.h5',
save_best_only=True,
save_weights_only=True)
tensorboard = TensorBoard(log_dir="tb/tb{}{}".format(ext, cv_ext))
history_logger = LoggingCallback(logger.debug)
model_recorder = ModelRecorder()
# callbacks = [history_logger, model_recorder]
callbacks = [
candle_monitor, timeout_monitor, history_logger, model_recorder
]
if args.reduce_lr:
callbacks.append(reduce_lr)
if args.warmup_lr:
callbacks.append(warmup_lr)
if args.cp:
callbacks.append(checkpointer)
if args.tb:
callbacks.append(tensorboard)
if args.gen:
history = model.fit_generator(train_gen,
train_steps,
epochs=args.epochs,
callbacks=callbacks,
validation_data=val_gen,
validation_steps=val_steps)
fold += 1
else:
if args.cv > 1:
x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data_cv(
fold)
else:
x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data(
)
y_shuf = np.random.permutation(y_val)
log_evaluation(evaluate_prediction(y_val, y_shuf),
description='Between random pairs in y_val:')
history = model.fit(x_train_list,
y_train,
batch_size=args.batch_size,
shuffle=args.shuffle,
epochs=args.epochs,
callbacks=callbacks,
validation_data=(x_val_list, y_val))
if args.cp:
model.load_weights(prefix + cv_ext + '.weights.h5')
if not args.gen:
y_val_pred = model.predict(x_val_list,
batch_size=args.batch_size).flatten()
scores = evaluate_prediction(y_val, y_val_pred)
if args.cv > 1 and scores[args.loss] > args.max_val_loss:
logger.warn(
'Best val_loss {} is greater than {}; retrain the model...'
.format(scores[args.loss], args.max_val_loss))
continue
else:
fold += 1
log_evaluation(scores)
df_val.is_copy = False
df_val['GROWTH_PRED'] = y_val_pred
df_val['GROWTH_ERROR'] = y_val_pred - y_val
df_pred_list.append(df_val)
if args.cp:
# model.save(prefix+'.model.h5')
model_recorder.best_model.save(prefix + '.model.h5')
# test reloadded model prediction
# new_model = keras.models.load_model(prefix+'.model.h5')
# new_model.load_weights(prefix+cv_ext+'.weights.h5')
# new_pred = new_model.predict(x_val_list, batch_size=args.batch_size).flatten()
# print('y_val:', y_val[:10])
# print('old_pred:', y_val_pred[:10])
# print('new_pred:', new_pred[:10])
plot_history(prefix, history, 'loss')
plot_history(prefix, history, 'r2')
if K.backend() == 'tensorflow':
K.clear_session()
if not args.gen:
pred_fname = prefix + '.predicted.growth.tsv'
if args.use_combo_score:
pred_fname = prefix + '.predicted.score.tsv'
df_pred = pd.concat(df_pred_list)
df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g')
logger.handlers = []
return history
