def test_keras_module_ensemble(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' subblock1 = MLPBlock(layers=[2, 1]) subblock2 = MLPBlock(layers=[2, 1]) model1 = EnsembleModel(models=[subblock1, subblock2], prefix='test', ensemble_type='linear', individual_loss=True) model2 = EnsembleModel( models=[subblock1, subblock2], prefix='test', ensemble_type='softmax', ) input_tensor = np.random.normal(size=(3, 2)) model1(input_tensor) model2(input_tensor) with pytest.raises(ValueError): EnsembleModel( models=[subblock1, subblock2], prefix='test', ensemble_type='dummy_type', )
def test_keras_module_conv2d_invalid_layer(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' with pytest.raises(ValueError): Conv2DBlock(layers_conv2d=[('dummy_layers', {})])
def test_agent(): env0 = MyTask() env1 = MyTask() env2 = MyTask() env3 = MyTask() hps_dict = {'hp_layer': [5, 10, 15], 'hp_epoch': [128, 256, 512]} hps = Hyperparameters(hps_dict) task_scheduler = TaskScheduler(['step0', 'step1']) task_scheduler.add_subtask('step0', 'task0', env=env0, hps=hps) task_scheduler.add_subtask('step0', 'task1', env=env1) task_scheduler.add_subtask('step1', 'task2', env=env2, hps=hps) task_scheduler.add_subtask('step1', 'task3', env=env3) task_scheduler.show_info() task_scheduler.get_sorted_task_ids() assert task_scheduler.get_sorted_task_ids() == ['step0', 'step1'] assert task_scheduler.get_subtask_ids('step0') == ['task0', 'task1'] assert task_scheduler.get_children_task_ids('step0') == ['step1'] assert task_scheduler.get_parents_task_ids('step1') == ['step0'] assert task_scheduler.get_subtask('step0', 'task0').env == env0 storegate = StoreGate(backend='numpy', data_id='test_agent') saver = Saver() metric = RandomMetric() logger.set_level(logger.DEBUG) agent = GridSearchAgent(saver=saver, storegate=storegate, task_scheduler=task_scheduler, metric=metric, dump_all_results=True) assert agent._storegate is storegate assert agent._saver is saver assert agent._task_scheduler is task_scheduler assert agent._metric is metric agent.storegate = storegate agent.saver = saver agent.task_scheduler = task_scheduler agent.metric = metric assert agent.storegate is storegate assert agent.saver is saver assert agent.task_scheduler is task_scheduler assert agent.metric is metric agent.execute() agent.finalize() best_result = agent.get_best_result() assert best_result.metric_value > 0
def test_mytask(): logger.set_level(logger.DEBUG) task = MyTask() task.set_hps({'hp_layer': 5, 'hp_epoch': 256}) assert task._hp_layer == 5 assert task._hp_epoch == 256 task.execute() task.finalize()
def test_keras_base(): KerasBaseTask.__abstractmethods__ = set() logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' saver = Saver() task = KerasBaseTask(saver=saver, ) task.load_metadata() # Do nothing
def test_keras_module_mlp(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' block = MLPBlock( layers=[2, 1], batch_norm=True, ) input_tensor = np.random.normal(size=(3, 2)) block(input_tensor)
def test_keras_module_connection_model(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' saver = Saver() storegate = build_storegate() args_mlptask = { # BaseTask 'saver': saver, 'storegate': storegate, # KerasBaseTask 'optimizer': 'adam', 'num_epochs': 2, 'max_patience': 1, 'loss': 'mse', 'run_eagerly': True, # MLPTask 'activation': 'relu', 'activation_last': 'sigmoid', } task0 = MLPTask(input_var_names=['var0', 'var1'], output_var_names=['output0', 'output1'], true_var_names=['var2', 'var3'], layers=[4, 2], **args_mlptask) task1 = MLPTask(input_var_names=['output0', 'output1'], output_var_names=['output2'], true_var_names=['var4'], layers=[4, 1], **args_mlptask) task2 = MLPTask(input_var_names=['output2'], output_var_names=['output3'], true_var_names=['label'], layers=[4, 1], **args_mlptask) task0.execute() task1.execute() task2.execute() model = ConnectionModel( models=[task0.ml.model, task1.ml.model, task2.ml.model], input_var_index=[[0, 1], [-1, -2], [-3]], output_var_index=[[0, 1], [2], [3]]) trainalbe_variables = model._get_variables() assert len(trainalbe_variables) > 0 input_tensor = np.random.normal(size=(3, 2)) model(input_tensor)
def test_keras_module_softmax_dense_layer(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' layer1 = SoftMaxDenseLayer() layer2 = SoftMaxDenseLayer(dropout_rate=0.3) layer1.get_config() input_tensor = np.random.normal(size=(3, 2)) layer1(input_tensor, training=None) layer2(input_tensor, training=True)
def test_keras_module_darts_model_sum_tensor(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' sum_tensor = SumTensor(name='test_sum') with pytest.raises(ValueError): sum_tensor.result() sum_tensor(0.1) sum_tensor(0.2) sum_tensor.result()
def test_mypytorchtask(): logger.set_level(logger.DEBUG) saver = Saver() task = MyPytorchTask(saver=saver, model=MyPytorchModel, optimizer='SGD', optimizer_args=dict(lr=0.1), loss='CrossEntropyLoss', metrics=['acc', 'lrs']) task.set_hps({'num_epochs': 5}) task.execute() task.finalize() assert saver.load_ml(key='mypytorch')['_num_epochs'] == 5
def preprocessing(tau4vec_tasks=['MLP', 'conv2D', 'SF', 'zero', 'noise'], higgsId_tasks=['mlp', 'lstm', 'mass', 'zero', 'noise'], truth_intermediate_inputs=True): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' load_weights = False from multiml.saver import Saver saver = Saver() # Storegate storegate = get_storegate(max_events=100) # Task scheduler from multiml.task_scheduler import TaskScheduler from my_tasks import get_higgsId_subtasks, get_tau4vec_subtasks task_scheduler = TaskScheduler() subtask1 = get_higgsId_subtasks(saver, subtask_names=higgsId_tasks, truth_input=truth_intermediate_inputs, load_weights=load_weights, run_eagerly=run_eagerly) task_scheduler.add_task(task_id='higgsId', parents=['tau4vec'], children=[], subtasks=subtask1) subtask2 = get_tau4vec_subtasks(saver, subtask_names=tau4vec_tasks, load_weights=load_weights, run_eagerly=run_eagerly) task_scheduler.add_task(task_id='tau4vec', parents=[], children=['higgsId'], subtasks=subtask2) # Metric from multiml.agent.metric import AUCMetric metric = AUCMetric(pred_var_name='probability', true_var_name='label', phase='test') return saver, storegate, task_scheduler, metric
def test_keras_util(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' saver = Saver() model = MLPBlock( layers=[2, 1], batch_norm=True, ) model.compile(optimizer='adam', loss='binary_crossentropy') x_train = np.random.normal(size=(100, 2)) x_valid = np.random.normal(size=(100, 2)) y_train = np.random.binomial(n=1, p=0.5, size=(100, )) y_valid = np.random.binomial(n=1, p=0.5, size=(100, )) chpt_path = f"{saver.save_dir}/test_keras_util" from multiml.task.keras.keras_util import training_keras_model training_keras_model(model=model, num_epochs=2, batch_size=10, max_patience=2, x_train=x_train, y_train=y_train, x_valid=x_valid, y_valid=y_valid, chpt_path=chpt_path, tensorboard_path=f'{saver.save_dir}/test_keras_util') from multiml.task.keras.keras_util import get_optimizer get_optimizer('adam', dict(learning_rate=None)) get_optimizer('adam', dict(learning_rate=0.001)) get_optimizer('sgd', dict(learning_rate=None)) get_optimizer('sgd', dict(learning_rate=0.1)) from tensorflow.keras import optimizers get_optimizer(optimizers.Adam()) with pytest.raises(ValueError): get_optimizer(None) with pytest.raises(NotImplementedError): get_optimizer('dummyoptimizer')
def test_keras_module_conv2d(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' block = Conv2DBlock( layers_conv2d=[ ('conv2d', { 'filters': 4, 'kernel_size': (2, 2) }), ('maxpooling2d', { 'pool_size': (2, 2) }), ('upsampling2d', { 'size': (2, 2) }), ], conv2d_padding='valid', ) input_tensor = np.random.normal(size=(3, 3, 3, 2)) block(input_tensor)
def main(opts): logger.set_level(opts.loglevel) global DEVICE from utils import load_config from run_utils import get_multi_loss, set_seed config = load_config(opts.config) verbose = 1 if opts.seed is not None: config['seed'] = opts.seed if opts.gpu_index is not None and DEVICE == device('cuda'): DEVICE = device(f'cuda:{opts.gpu_index}') if opts.data_path is not None: config['dataset']['params']['data_path'] = opts.data_path if opts.event is not None: config['dataset']['params']['max_events'] = int(opts.event) if opts.clip_value is not None : config['ASNG']['clip'] = opts.clip_value if opts.alpha is not None : config['ASNG']['alpha'] = opts.alpha if opts.lam is not None : config['ASNG']['lam'] = opts.lam if opts.delta is not None : config['ASNG']['delta'] = opts.delta if opts.epochs is not None : config['ASNG']['epochs'] = opts.epochs set_seed(config.seed) if opts.do_pretrain : jobid = 'pretrain_' + opts.jobid else : jobid = 'no_train_' + opts.jobid save_dir = f'output/{os.path.basename(__file__)[:-3]}_{opts.event}evt_weight{opts.weight}_{jobid}' use_multi_loss, loss_weights = get_multi_loss(opts.weight) from run_utils import preprocessing saver, storegate, task_scheduler, metric = preprocessing( save_dir=save_dir, config=config, device=DEVICE, tau4vec_tasks=['conv2D', 'MLP', 'SF'], higgsId_tasks=['lstm', 'mlp', 'mass'], ) # Time measurements from timer import timer timer_reg = {} phases = ['test'] if opts.load_weights else ['train', 'valid', 'test'] # Agent logger.info(f'lambda / alpha / delta is {config.ASNG.lam} / {config.ASNG.alpha} / {config.ASNG.delta}') from multiml.agent.pytorch import PytorchASNGNASAgent with timer(timer_reg, "initialize"): from my_tasks import mapping_truth_corr config['ASNG']['connectiontask_args']['phases'] = phases config['ASNG']['connectiontask_args']['variable_mapping'] = mapping_truth_corr config['ASNG']['connectiontask_args']['device'] = DEVICE config['ASNG']['connectiontask_args']['loss_weights'] = loss_weights agent = PytorchASNGNASAgent( verbose = verbose, num_epochs = config.ASNG.epochs, max_patience = config.ASNG.patience, batch_size = config.ASNG.batch_size, asng_args = config.ASNG.asng_args, optimizer = config.ASNG.optimizer.name, optimizer_args = config.ASNG.optimizer.params, scheduler = config.ASNG.scheduler, # BaseAgent saver=saver, storegate=storegate, task_scheduler=task_scheduler, metric=metric, # EnsembleAgent # ConnectionSimpleAgent freeze_model_weights=False, do_pretraining = opts.do_pretrain, connectiontask_args= config.ASNG.connectiontask_args, ) with timer(timer_reg, "execute"): agent.execute() with timer(timer_reg, "finalize"): agent.finalize() results = agent.results_json results['walltime'] = timer_reg['execute'][1] results['timer_reg'] = timer_reg results['seed'] = opts.seed results['nevents'] = opts.event*2 def print_dict(key, val) : if type(val) is dict : for k, v in val.items(): print_dict( f'{key} {k}', v) else : logger.info(f'{key: <30} : {val}') for key, val in results.items() : print_dict(key, val) with open(f'{saver.save_dir}/result.run_connection_asngnas_{opts.event}evt_weight{opts.weight}.json', 'w') as fo : json.dump([results], fo, indent=2) if not opts.load_weights: with open(f"{saver.save_dir}/timer.pkl", 'wb') as f: import pickle pickle.dump(timer_reg, f) ### post processing variables = [] from my_tasks import corr_tau_4vec variables.extend(corr_tau_4vec) variables.extend(['probability']) for phase in phases : # dump prediction storegate.set_data_id("") y_pred = np.array( storegate.get_data(phase = phase, var_names = variables ) ) os.makedirs(f'{saver.save_dir}/pred/{phase}', exist_ok = True ) for i, v in enumerate(variables): np.save(f'{saver.save_dir}/pred/{phase}/{v}', y_pred[i])
def preprocessing(save_dir, args, tau4vec_tasks=['MLP', 'conv2D', 'SF', 'zero', 'noise'], higgsId_tasks=['mlp', 'lstm', 'mass', 'zero', 'noise'], truth_intermediate_inputs=True): from multiml import logger logger.set_level(args.log_level) from setup_tensorflow import setup_tensorflow setup_tensorflow(args.seed, args.igpu) load_weights = args.load_weights from multiml.saver import Saver saver = Saver(save_dir, serial_id=args.seed) saver.add("seed", args.seed) # Storegate from my_storegate import get_storegate storegate = get_storegate( data_path=args.data_path, max_events=args.max_events, ) # Task scheduler from multiml.task_scheduler import TaskScheduler from my_tasks import get_higgsId_subtasks, get_tau4vec_subtasks task_scheduler = TaskScheduler() if args.remove_dummy_models: tau4vec_tasks = [ v for v in tau4vec_tasks if v not in ['zero', 'noise'] ] higgsId_tasks = [ v for v in higgsId_tasks if v not in ['zero', 'noise'] ] if len(tau4vec_tasks) > 0 and len(higgsId_tasks) > 0: subtask1 = get_higgsId_subtasks(saver, subtask_names=higgsId_tasks, truth_input=truth_intermediate_inputs, load_weights=load_weights, run_eagerly=args.run_eagerly) task_scheduler.add_task(task_id='higgsId', parents=['tau4vec'], children=[], subtasks=subtask1) subtask2 = get_tau4vec_subtasks(saver, subtask_names=tau4vec_tasks, load_weights=load_weights, run_eagerly=args.run_eagerly) task_scheduler.add_task(task_id='tau4vec', parents=[], children=['higgsId'], subtasks=subtask2) elif len(higgsId_tasks) > 0: subtask = get_higgsId_subtasks(saver, subtask_names=higgsId_tasks, load_weights=load_weights, run_eagerly=args.run_eagerly) task_scheduler.add_task(task_id='higgsId', subtasks=subtask) elif len(tau4vec_tasks) > 0: subtask = get_tau4vec_subtasks(saver, subtask_names=tau4vec_tasks, load_weights=load_weights, run_eagerly=args.run_eagerly) task_scheduler.add_task(task_id='tau4vec', subtasks=subtask) else: raise ValueError("Strange task combination...") # Metric if len(tau4vec_tasks) > 0 and len(higgsId_tasks) == 0: from multiml_htautau.task.metrics import CustomMSEMetric from my_tasks import corr_tau_4vec, truth_tau_4vec metric = CustomMSEMetric(pred_var_name=corr_tau_4vec, true_var_name=truth_tau_4vec, phase='test') else: from multiml.agent.metric import AUCMetric metric = AUCMetric(pred_var_name='probability', true_var_name='label', phase='test') return saver, storegate, task_scheduler, metric
def test_keras_mlp(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' storegate = build_storegate() saver = Saver() args_task = { # BaseTask 'saver': saver, 'storegate': storegate, # MLBaseTask 'phases': None, 'save_weights': True, # KerasBaseTask 'input_var_names': ['var0', 'var1'], 'output_var_names': 'output0', 'true_var_names': 'label', 'optimizer': 'adam', 'num_epochs': 2, 'max_patience': 1, 'loss': 'binary_crossentropy', 'run_eagerly': True, # MLPTask 'layers': [8, 1], 'activation': 'relu', 'activation_last': 'sigmoid', 'batch_norm': True, } task = MLPTask(**args_task) assert task._layers == [8, 1] task.set_hps({ 'layers': [4, 1], 'input_shapes': None, 'output_var_names': 'output0' }) assert task._layers == [4, 1] assert task._input_shapes == [len(['var0', 'var1'])] assert task.get_inputs()[0].shape[1] == len(['var0', 'var1']) task.execute() task.finalize() # Validate model save/load args_task['phases'] = ['test'] args_task['save_weights'] = False args_task['load_weights'] = saver.load_ml(task._name)['model_path'] args_task['layers'] = [4, 1] task2 = MLPTask(**args_task) # Fail due to call models before defining the model with pytest.raises(ValueError): y_pred_load = task2.predict(phase='test') task2.execute() task2.finalize() y_pred = task.predict(phase='test') y_pred_load = task2.predict(phase='test') # assert (np.sum(np.square(y_pred - y_pred_load)) < 1e-10) assert (np.array_equal(y_pred, y_pred_load))
def test_keras_ensemble(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' saver = Saver() storegate = build_storegate() args_mlptask = { # BaseTask 'saver': saver, 'storegate': storegate, # KerasBaseTask 'optimizer': 'adam', 'num_epochs': 2, 'max_patience': 1, 'loss': 'mse', 'run_eagerly': True, # MLPTask 'activation': 'relu', 'activation_last': 'sigmoid', } subtask0 = MLPTask(name='subtask0', input_var_names=['var0', 'var1'], output_var_names=['output0'], true_var_names=['label'], layers=[4, 1], **args_mlptask) subtask1 = MLPTask(name='subtask1', input_var_names=['var0', 'var1'], output_var_names=['output0'], true_var_names=['label'], layers=[4, 1], **args_mlptask) subtask0.execute() subtask1.execute() task = EnsembleTask( subtasks=[subtask0, subtask1], dropout_rate=None, individual_loss=True, individual_loss_weights=1.0, saver=saver, storegate=storegate, save_weights=True, # do_training=False, phases=["train", "valid", "test"], ) assert task.get_inputs()[0].shape[1] == len(['var0', 'var1']) assert task.input_var_names == ['var0', 'var1'] assert task.output_var_names == ['output0'] task.execute() task.finalize() task2 = EnsembleTask( name='EnsembleTask2', subtasks=[subtask0, subtask1], dropout_rate=None, individual_loss=True, individual_loss_weights=1.0, saver=saver, storegate=storegate, phases=["test"], load_weights=saver.load_ml(task.get_unique_id())['model_path'], ) task2.execute() task2.finalize() y_pred = task.predict(phase='test') y_pred_load = task2.predict(phase='test') assert (np.array_equal(y_pred, y_pred_load))
import os save_dir = f'output/{os.path.basename(__file__)[:-3]}' from run_utils import common_parser parser = common_parser() parser.add_argument("--model", dest="model", help="model", type=str, default='MLP') args = parser.parse_args() from multiml import logger logger.set_level(args.log_level) from setup_tensorflow import setup_tensorflow setup_tensorflow(args.seed, args.igpu) from run_utils import add_suffix save_dir = add_suffix(save_dir, args) save_dir += f'_{args.model}' from multiml.saver import Saver saver = Saver(save_dir, serial_id=args.seed) saver.add("seed", args.seed) # Storegate from my_storegate import get_storegate storegate = get_storegate( data_path=args.data_path,
def test_keras_model_connection(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' saver = Saver() storegate = build_storegate() args_mlptask = { # BaseTask 'saver': saver, 'storegate': storegate, # KerasBaseTask 'optimizer': 'adam', 'num_epochs': 2, 'max_patience': 1, 'loss': 'mse', 'run_eagerly': True, # MLPTask 'activation': 'relu', 'activation_last': 'sigmoid', } subtask0 = MLPTask(name='subtask0', input_var_names=['var0', 'var1'], output_var_names=['output0', 'output1'], true_var_names=['var2', 'var3'], layers=[4, 2], **args_mlptask) subtask1 = MLPTask(name='subtask1', input_var_names=['var2', 'var3'], output_var_names=['output2'], true_var_names=['label'], layers=[4, 1], **args_mlptask) subtask0.execute() subtask1.execute() chpt_path = f"{saver.save_dir}/test_keras_model_connection" task = ModelConnectionTask( # ConnectionModel subtasks=[subtask0, subtask1], use_multi_loss=True, loss_weights=[0.5, 0.5], variable_mapping=[('var2', 'output0'), ('var3', 'output1')], # KerasBaseTask saver=saver, storegate=storegate, optimizer='adam', num_epochs=2, max_patience=1, loss='binary_crossentropy', run_eagerly=True, load_weights=False, save_weights=chpt_path, phases=['train', 'valid', 'test'], ) task.execute() task.finalize() # Load model weights task2 = ModelConnectionTask( # ConnectionModel subtasks=[subtask0, subtask1], use_multi_loss=True, loss_weights=[0.5, 0.5], variable_mapping=[('var2', 'output0'), ('var3', 'output1')], # KerasBaseTask saver=saver, storegate=storegate, optimizer='adam', num_epochs=2, max_patience=1, loss='binary_crossentropy', run_eagerly=True, save_weights=False, load_weights=chpt_path, phases=['test'], ) task2.execute() task2.finalize() y_pred = task.predict(phase='test') y_pred_load = task2.predict(phase='test') assert (np.array_equal(y_pred[0], y_pred_load[0])) assert (np.array_equal(y_pred[1], y_pred_load[1])) # Connect as series dag task3 = ModelConnectionTask( # ConnectionModel subtasks=[subtask0, subtask1], use_multi_loss=True, loss_weights=[0.5, 0.5], variable_mapping=[('var2', 'output0'), ('var3', 'output1')], # KerasBaseTask saver=Saver(), storegate=storegate, optimizer='adam', num_epochs=2, max_patience=1, loss='binary_crossentropy', run_eagerly=True, load_weights=False, save_weights=chpt_path, phases=['train', 'valid', 'test'], ) task3.execute() task3.finalize()
def test_agent_keras_connection_simple(): os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' logger.set_level(logger.DEBUG) saver = Saver() storegate = build_storegate() args_mlptask = { # BaseTask 'saver': saver, 'storegate': storegate, # KerasBaseTask 'optimizer': 'adam', 'num_epochs': 2, 'max_patience': 1, 'loss': 'mse', 'run_eagerly': True, # MLPTask 'activation': 'relu', 'activation_last': 'sigmoid', } subtask0 = MLPTask(name='subtask0', input_var_names=['var0', 'var1'], output_var_names=['output0', 'output1'], true_var_names=['var2', 'var3'], layers=[4, 2], batch_norm=True, **args_mlptask) subtask1 = MLPTask(name='subtask1', input_var_names=['var2', 'var3'], output_var_names=['output2'], true_var_names=['label'], layers=[4, 1], **args_mlptask) task_scheduler = TaskScheduler() task_scheduler.add_task('step0') task_scheduler.add_task('step1', parents=['step0']) task_scheduler.add_subtask('step0', 'subtask0', env=subtask0) task_scheduler.add_subtask('step1', 'subtask1', env=subtask1) task_scheduler.show_info() import copy task_scheduler0 = copy.deepcopy(task_scheduler) task_scheduler1 = copy.deepcopy(task_scheduler) metric = RandomMetric() agent_args = { # BaseAgent 'saver': saver, 'storegate': storegate, 'task_scheduler': task_scheduler0, 'metric': metric, # KerasConnectionSimpleAgent 'freeze_model_weights': False, 'do_pretraining': False, 'connectiontask_name': 'connectiontask', 'connectiontask_args': { 'num_epochs': 2, 'max_patience': 1, 'batch_size': 1200, 'save_weights': True, 'phases': None, 'use_multi_loss': True, 'loss_weights': [0.5, 0.5], 'optimizer': 'adam', 'optimizer_args': dict(learning_rate=1e-3), 'variable_mapping': [('var2', 'output0'), ('var3', 'output1')], 'run_eagerly': True, }, } agent = KerasConnectionRandomSearchAgent(**agent_args) agent.execute() agent.finalize() # Get the 1st task 1st model subtask = agent._task_prod[0][0] # Apply pretraining agent._execute_subtask(subtask, is_pretraining=True) # Set all variables in keras model to trainable KerasConnectionRandomSearchAgent._set_trainable_flags(subtask.env._model, do_trainable=True) # Build connect-model agent._build_connected_models( subtasks=[agent._task_prod[0][0].env, agent._task_prod[1][0].env], job_id='dummy', use_task_scheduler=False) # Load saved model for (task_id, subtask_id) in [ ('step0', 'subtask0'), ('step1', 'subtask1'), ]: task = task_scheduler1.get_subtask(task_id, subtask_id).env task._save_weights = False task._load_weights = False task._phases = ['test'] agent_args['connectiontask_args']['save_weights'] = False agent_args['connectiontask_args']['load_weights'] = True #agent_args['saver'] = Saver() agent_args['saver'] = saver agent_args['task_scheduler'] = task_scheduler1 agent_args['connectiontask_args']['phases'] = ['test'] agent2 = KerasConnectionRandomSearchAgent(**agent_args) agent2.execute() agent2.finalize() task_scheduler0.show_info() task_scheduler1.show_info() task0 = task_scheduler0.get_subtask('connection', 'connectiontask').env task1 = task_scheduler1.get_subtask('connection', 'connectiontask').env y_pred = task0.predict(phase='test') y_pred_load = task1.predict(phase='test') assert (np.array_equal(y_pred[0], y_pred_load[0])) assert (np.array_equal(y_pred[1], y_pred_load[1]))
def test_agent_keras_ensemble(): os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' logger.set_level(logger.DEBUG) saver = Saver() storegate = build_storegate() args_mlptask = { # BaseTask 'saver': saver, 'storegate': storegate, # KerasBaseTask 'optimizer': 'adam', 'num_epochs': 2, 'max_patience': 1, 'loss': 'mse', 'run_eagerly': True, 'save_weights': True, # MLPTask 'activation': 'relu', 'activation_last': 'sigmoid', } subtask0 = MLPTask(name='subtask0', input_var_names=['var0', 'var1'], output_var_names=['output0', 'output1'], true_var_names=['var2', 'var3'], layers=[4, 2], batch_norm=True, **args_mlptask) subtask1 = MLPTask(name='subtask1', input_var_names=['var0', 'var1'], output_var_names=['output0', 'output1'], true_var_names=['var2', 'var3'], layers=[4, 2], **args_mlptask) subtask2 = MLPTask(name='subtask2', input_var_names=['var2', 'var3'], output_var_names=['output2'], true_var_names=['label'], layers=[4, 1], **args_mlptask) task_scheduler = TaskScheduler() task_scheduler.add_task('step0') task_scheduler.add_task('step1', parents=['step0']) task_scheduler.add_subtask('step0', 'subtask0', env=subtask0) task_scheduler.add_subtask('step0', 'subtask1', env=subtask1) task_scheduler.add_subtask('step1', 'subtask2', env=subtask2) task_scheduler.show_info() import copy task_scheduler0 = copy.deepcopy(task_scheduler) task_scheduler1 = copy.deepcopy(task_scheduler) metric = RandomMetric() agent_args = { # BaseAgent 'saver': saver, 'storegate': storegate, 'task_scheduler': task_scheduler0, 'metric': metric, # KerasConnectionSimpleAgent 'freeze_model_weights': False, 'do_pretraining': True, 'connectiontask_name': 'connectiontask', 'connectiontask_args': { 'num_epochs': 2, 'max_patience': 1, 'batch_size': 1200, 'save_weights': True, 'phases': None, 'use_multi_loss': True, 'loss_weights': [0.5, 0.5], 'optimizer': 'adam', 'optimizer_args': dict(learning_rate=1e-3), 'variable_mapping': [('var2', 'output0'), ('var3', 'output1')], 'run_eagerly': True, }, # KerasEnsembleAgent 'ensembletask_args': { 'dropout_rate': None, 'individual_loss': False, 'individual_loss_weights': 0.0, 'phases': ['test'], 'save_weights': True, 'run_eagerly': True, }, } agent = KerasEnsembleAgent(**agent_args) agent.execute() agent.finalize() # Not implemented the case of phses = ['train'] for ensemble task with pytest.raises(ValueError): agent_args['ensembletask_args']['phases'] = ['train', 'valid', 'test'] KerasEnsembleAgent(**agent_args) # Load saved model for (task_id, subtask_id) in [ ('step0', 'subtask0'), ('step0', 'subtask1'), ('step1', 'subtask2'), ]: task = task_scheduler1.get_subtask(task_id, subtask_id).env task._save_weights = False task._load_weights = True task._phases = ['test'] agent_args['ensembletask_args']['phases'] = ['test'] agent_args['connectiontask_args']['save_weights'] = False agent_args['connectiontask_args']['load_weights'] = True agent_args['saver'] = saver agent_args['task_scheduler'] = task_scheduler1 agent_args['connectiontask_args']['phases'] = ['test'] agent2 = KerasEnsembleAgent(**agent_args) agent2.execute() agent2.finalize() task_scheduler0.show_info() task_scheduler1.show_info() for (task_id, subtask_id) in [ ('step0', 'subtask0'), ('step0', 'subtask1'), ('step1', 'subtask2'), ('step0', 'step0_ensemble'), ('connection', 'connectiontask'), ]: task0 = task_scheduler0.get_subtask(task_id, subtask_id).env task1 = task_scheduler1.get_subtask(task_id, subtask_id).env y_pred = task0.predict(phase='test') y_pred_load = task1.predict(phase='test') for y, y_load in zip(y_pred, y_pred_load): assert (np.array_equal(y, y_load))
def test_keras_module_darts_model(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' saver = Saver() storegate = build_storegate() args_mlptask = { # BaseTask 'saver': saver, 'storegate': storegate, # KerasBaseTask 'optimizer': 'adam', 'num_epochs': 2, 'max_patience': 1, 'loss': 'mse', 'run_eagerly': True, # MLPTask 'activation': 'relu', 'activation_last': 'sigmoid', } subtask0 = MLPTask(input_var_names=['var0', 'var1'], output_var_names=['output0', 'output1'], true_var_names=['var2', 'var3'], layers=[4, 2], **args_mlptask) subtask1 = MLPTask(input_var_names=['var0', 'var1'], output_var_names=['output0', 'output1'], true_var_names=['var2', 'var3'], layers=[4, 2], **args_mlptask) subtask2 = MLPTask(input_var_names=['output0', 'output1'], output_var_names=['output2'], true_var_names=['label'], layers=[4, 1], **args_mlptask) subtask3 = MLPTask(input_var_names=['output0', 'output1'], output_var_names=['output2'], true_var_names=['label'], layers=[4, 1], **args_mlptask) subtask0.execute() subtask1.execute() subtask2.execute() subtask3.execute() task0 = EnsembleTask( subtasks=[subtask0, subtask1], dropout_rate=None, saver=saver, storegate=storegate, phases=['test'], ) task1 = EnsembleTask( subtasks=[subtask2, subtask3], dropout_rate=None, saver=saver, storegate=storegate, phases=['test'], ) task0.execute() task1.execute() model = DARTSModel( # DARTSModel optimizer_alpha='adam', optimizer_weight='adam', learning_rate_alpha=1e-3, learning_rate_weight=1e-3, zeta=1e-3, # ConnectionModel models=[task0.ml.model, task1.ml.model], input_var_index=[[0, 1], [-1, -2]], output_var_index=[[0, 1], [2]]) # Directly use objest for optimizer instead of str DARTSModel( # DARTSModel optimizer_alpha=Adam(learning_rate=0.001), optimizer_weight=Adam(learning_rate=0.001), learning_rate_alpha=-1, # dummy learning_rate_weight=-1, # dummy zeta=1e-3, # ConnectionModel models=[task0.ml.model, task1.ml.model], input_var_index=[[0, 1], [-1, -2]], output_var_index=[[0, 1], [2]]) model.compile( optimizer='adam', loss='binary_crossentropy', run_eagerly=True, ) input_tensor_train = [ np.random.normal(size=(10, 1)), np.random.normal(size=(10, 1)) ] input_tensor_test = [ np.random.normal(size=(10, 1)), np.random.normal(size=(10, 1)) ] output_tensor_train = [ np.random.normal(size=(10, 2)), np.random.normal(size=(10, 1)) ] output_tensor_test = [ np.random.normal(size=(10, 2)), np.random.normal(size=(10, 1)) ] model._batch_size_train.assign(2) model.fit( x=input_tensor_train, y=output_tensor_train, validation_data=(input_tensor_test, output_tensor_test), epochs=2, ) alphas = model._get_variable_numpy(model.alpha_vars) assert alphas[0].shape == (2, 1) # alphas of step0 assert alphas[1].shape == (2, 1) # alphas of step1 best_submodel_index = model.get_index_of_best_submodels() assert len(best_submodel_index) == 2
def test_keras_conv2d(): logger.set_level(logger.DEBUG) os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1' storegate = build_storegate() saver = Saver() args_task = { # BaseTask 'saver': saver, 'storegate': storegate, # MLBaseTask 'phases': None, 'save_weights': True, # KerasBaseTask 'input_var_names': ['var0'], 'output_var_names': ['output0'], 'optimizer': 'adam', 'num_epochs': 2, 'max_patience': 1, 'loss': 'binary_crossentropy', 'run_eagerly': True, # MLPTask 'true_var_names': ['label'], 'layers': [4, 1], 'input_shapes': (1, 3, 3, 1), 'activation': 'relu', 'activation_last': 'sigmoid', # Conv2DTask 'conv2d_layers': [ ('conv2d', { 'filters': 4, 'kernel_size': (2, 2) }), ('maxpooling2d', { 'pool_size': (2, 2) }), ('upsampling2d', { 'size': (2, 2) }), ], } from multiml.task.keras import Conv2DTask task = Conv2DTask(**args_task) task.execute() task.finalize() # Validate model save/load args_task['phases'] = ['test'] args_task['save_weights'] = False args_task['load_weights'] = saver.load_ml(task._name)['model_path'] task2 = Conv2DTask(**args_task) task2.execute() task2.finalize() y_pred = task.predict(phase='test') y_pred_load = task2.predict(phase='test') assert (np.array_equal(y_pred, y_pred_load))