def test_dmlscheduler_arbitrary_scheduling(scheduler): """ Manually schedule events and check that all jobs are completed. """ first = make_initialize_job() second = make_initialize_job() scheduler.add_job(first) scheduler.add_job(second) while len(scheduler.processed) == 0: scheduler.runners_run_next_jobs() third = make_initialize_job() fourth = make_initialize_job() scheduler.add_job(third) scheduler.add_job(fourth) while len(scheduler.processed) < 4: scheduler.runners_run_next_jobs() fifth = make_initialize_job() scheduler.add_job(fifth) while len(scheduler.processed) < 5: scheduler.runners_run_next_jobs() assert len(scheduler.processed) == 5, \ "Jobs {} failed/not completed in time!".format([ result.job.job_type for result in scheduler.processed]) while scheduler.processed: output = scheduler.processed.pop(0) initial_weights = output.results['weights'] assert type(initial_weights) == list assert type(initial_weights[0]) == np.ndarray
def test_dmlrunner_uniform_initialization(runner): initialize_job = make_initialize_job(small_filepath) result = runner.run_job(initialize_job).results first_weights = result['weights'] initialize_job = make_initialize_job(small_filepath) result = runner.run_job(initialize_job).results second_weights = result['weights'] assert all(np.allclose(arr1, arr2) for arr1,arr2 in zip(first_weights, second_weights))
def test_communication_manager_can_inform_new_job_to_the_optimizer( config_manager, ipfs_client): """ Ensures that Communication Manager can tell the optimizer of something, and that the job will transfer correctly. """ communication_manager = CommunicationManager() scheduler = DMLScheduler(config_manager) communication_manager.configure(scheduler) scheduler.configure(communication_manager, ipfs_client) true_job = make_initialize_job(make_model_json()) true_job.hyperparams['epochs'] = 10 true_job.hyperparams['batch_size'] = 128 true_job.hyperparams['split'] = .05 serialized_job = serialize_job(true_job) new_session_event = { TxEnum.KEY.name: None, TxEnum.CONTENT.name: { "optimizer_params": {}, "serialized_job": serialized_job } } communication_manager.inform(MessageEventTypes.NEW_SESSION.name, new_session_event) optimizer_job = communication_manager.optimizer.job assert optimizer_job.weights == true_job.weights assert optimizer_job.serialized_model == true_job.serialized_model assert optimizer_job.framework_type == true_job.framework_type assert optimizer_job.hyperparams == true_job.hyperparams assert optimizer_job.label_column_name == true_job.label_column_name
def test_dmlrunner_same_train_job_with_split_1( \ runner, mnist_filepath): split_job = make_split_job(mnist_filepath) split_job.hyperparams['split'] = 1 job_results = runner.run_job(split_job) session_filepath = job_results.results['session_filepath'] datapoint_count = job_results.results['datapoint_count'] initialize_job = make_initialize_job() initial_weights = runner.run_job(initialize_job).results['weights'] train_job = make_train_job( initial_weights, session_filepath, datapoint_count ) result = runner.run_job(train_job) assert result.status == 'successful' results = result.results new_weights = results['weights'] omega = results['omega'] train_stats = results['train_stats'] assert result.job.job_type is JobTypes.JOB_TRAIN.name assert type(new_weights) == list assert type(new_weights[0]) == np.ndarray assert type(omega) == int or type(omega) == float assert type(train_stats) == dict
def test_dmlscheduler_sanity(scheduler): """ Check that the scheduling/running functionality is maintained. """ initialize_job = make_initialize_job() scheduler.add_job(initialize_job) scheduler.runners_run_next_jobs() while not scheduler.processed: time.sleep(0.1) scheduler.runners_run_next_jobs() output = scheduler.processed.pop(0) initial_weights = output.results['weights'] assert type(initial_weights) == list assert type(initial_weights[0]) == np.ndarray
def test_dmlscheduler_cron(scheduler): """ Test that the scheduler's cron works. """ m = 2 for _ in range(m): initialize_job = make_initialize_job() scheduler.add_job(initialize_job) scheduler.start_cron(period_in_mins = 0.01) timeout = time.time() + 6 while time.time() < timeout and len(scheduler.processed) != m: time.sleep(1) scheduler.stop_cron() assert len(scheduler.processed) == m while scheduler.processed: output = scheduler.processed.pop(0) initial_weights = output.results['weights'] assert type(initial_weights) == list assert type(initial_weights[0]) == np.ndarray
def test_dmlrunner_same_train_job_with_split_1( \ config_manager, mnist_filepath): model_json = make_model_json() hyperparams = make_hyperparams(split=1) runner = DMLRunner(config_manager) initialize_job = make_initialize_job(model_json) initial_weights = runner.run_job(initialize_job).results['weights'] split_job = make_split_job(model_json, mnist_filepath) job_results = runner.run_job(split_job) session_filepath = job_results.results['session_filepath'] datapoint_count = job_results.results['datapoint_count'] train_job = make_train_job(model_json, initial_weights, hyperparams, session_filepath, datapoint_count) result = runner.run_job(train_job) assert result.status == 'successful' results = result.results new_weights = results['weights'] omega = results['omega'] train_stats = results['train_stats'] assert result.job.job_type is JobTypes.JOB_TRAIN.name assert type(new_weights) == list assert type(new_weights[0]) == np.ndarray assert type(omega) == int or type(omega) == float assert type(train_stats) == dict
def init_dmlresult_obj(config_manager, small_filepath): runner = DMLRunner(config_manager) initialize_job = make_initialize_job(make_model_json(), small_filepath) result = runner.run_job(initialize_job) return result
def init_dmlresult_obj(runner, small_filepath): initialize_job = make_initialize_job(small_filepath) result = runner.run_job(initialize_job) return result
def init_dmlresult_obj(config_manager, small_uuid, dataset_manager): runner = DMLRunner(config_manager) small_filepath = dataset_manager.get_mappings()[small_uuid] initialize_job = make_initialize_job(make_model_json(), small_filepath) result = runner.run_job(initialize_job) return result