def test_engine_submit_to_multiple_resources(num_resources=3, num_jobs=50): """Test job spread across multiple resources.""" # sanity check for parameters assert num_jobs > 10*(num_resources*(num_resources-1)/2), \ "There must be enough jobs to fill the first N-1 resources" assert num_jobs < 10*(num_resources*(num_resources+1)/2), \ "Too many jobs: would fill all resources" # set up cfg = gc3libs.config.Configuration() cfg.TYPE_CONSTRUCTOR_MAP['noop'] = ('gc3libs.backends.noop', 'NoOpLrms') for n in range(num_resources): name = 'test{nr}'.format(nr=n+1) cfg.resources[name].update( name=name, type='noop', auth='none', transport='local', max_cores_per_job=1, max_memory_per_core=1*GB, max_walltime=8*hours, max_cores=((n+1)*10), architecture=Run.Arch.X86_64, ) core = Core(cfg) engine = Engine(core) # generate 50 no-op tasks for n in range(num_jobs): name = 'app{nr}'.format(nr=n) engine.add( Application( ['/bin/true'], inputs=[], outputs=[], output_dir='/tmp', jobname=name, requested_cores=1, ) ) # submit them all engine.progress() # get handles to the actual backend objects rscs = [ core.get_backend('test{nr}'.format(nr=n+1)) for n in range(num_resources) ] num_jobs_per_resource = [ len([task for task in engine._in_flight if task.execution.resource_name == rsc.name]) for rsc in rscs ] # check that all jobs have been submitted and that each # resource got at least one job assert_equal(sum(num_jobs_per_resource), num_jobs) for num in num_jobs_per_resource: assert num > 0 # since TYPE_CONSTRUCTOR_MAP is a class-level variable, we # need to clean up otherwise other tests will see the No-Op # backend del cfg.TYPE_CONSTRUCTOR_MAP['noop']
def test_engine_progress(num_jobs=1, transition_graph=None, max_iter=100): cfg = gc3libs.config.Configuration() cfg.TYPE_CONSTRUCTOR_MAP['noop'] = ('gc3libs.backends.noop', 'NoOpLrms') name = 'test' cfg.resources[name].update( name=name, type='noop', auth='none', transport='local', max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=8 * hours, max_cores=2, architecture=Run.Arch.X86_64, ) core = Core(cfg) rsc = core.get_backend(name) if transition_graph: rsc.transition_graph = transition_graph else: # give each job a 50% chance of moving from one state to the # next one rsc.transition_graph = { Run.State.SUBMITTED: { 0.50: Run.State.RUNNING }, Run.State.RUNNING: { 0.50: Run.State.TERMINATING }, } engine = Engine(core) # generate some no-op tasks for n in range(num_jobs): name = 'app{nr}'.format(nr=n + 1) engine.add( Application( ['/bin/true'], inputs=[], outputs=[], output_dir='/tmp', jobname=name, requested_cores=1, )) # run them all current_iter = 0 done = engine.stats()[Run.State.TERMINATED] while done < num_jobs and current_iter < max_iter: engine.progress() done = engine.stats()[Run.State.TERMINATED] current_iter += 1 # since TYPE_CONSTRUCTOR_MAP is a class-level variable, we # need to clean up otherwise other tests will see the No-Op # backend del cfg.TYPE_CONSTRUCTOR_MAP['noop']
def test_engine_find_task_by_id(): """ Test that saved tasks are can be retrieved from the Engine given their ID only. """ with temporary_core() as core: with temporary_directory() as tmpdir: store = FilesystemStore(tmpdir) engine = Engine(core, store=store) task = SuccessfulApp() store.save(task) engine.add(task) task_id = task.persistent_id assert engine.find_task_by_id(task_id) == task
def test_engine_progress(num_jobs=1, transition_graph=None, max_iter=100): cfg = gc3libs.config.Configuration() cfg.TYPE_CONSTRUCTOR_MAP['noop'] = ('gc3libs.backends.noop', 'NoOpLrms') name = 'test' cfg.resources[name].update( name=name, type='noop', auth='none', transport='local', max_cores_per_job=1, max_memory_per_core=1*GB, max_walltime=8*hours, max_cores=2, architecture=Run.Arch.X86_64, ) core = Core(cfg) rsc = core.get_backend(name) if transition_graph: rsc.transition_graph = transition_graph else: # give each job a 50% chance of moving from one state to the # next one rsc.transition_graph = { Run.State.SUBMITTED: {0.50: Run.State.RUNNING}, Run.State.RUNNING: {0.50: Run.State.TERMINATING}, } engine = Engine(core) # generate some no-op tasks for n in range(num_jobs): name = 'app{nr}'.format(nr=n+1) engine.add( Application( ['/bin/true'], inputs=[], outputs=[], output_dir='/tmp', jobname=name, requested_cores=1, ) ) # run them all current_iter = 0 done = engine.stats()[Run.State.TERMINATED] while done < num_jobs and current_iter < max_iter: engine.progress() done = engine.stats()[Run.State.TERMINATED] current_iter += 1 # since TYPE_CONSTRUCTOR_MAP is a class-level variable, we # need to clean up otherwise other tests will see the No-Op # backend del cfg.TYPE_CONSTRUCTOR_MAP['noop']
def test_engine_submit_to_multiple_resources(num_resources=3, num_jobs=50): """Test job spread across multiple resources.""" # sanity check for parameters assert num_jobs > 10*(num_resources*(num_resources-1)/2), \ "There must be enough jobs to fill the first N-1 resources" assert num_jobs < 10*(num_resources*(num_resources+1)/2), \ "Too many jobs: would fill all resources" # set up cfg = gc3libs.config.Configuration() cfg.TYPE_CONSTRUCTOR_MAP['noop'] = ('gc3libs.backends.noop', 'NoOpLrms') for n in range(num_resources): name = 'test{nr}'.format(nr=n + 1) cfg.resources[name].update( name=name, type='noop', auth='none', transport='local', max_cores_per_job=1, max_memory_per_core=1 * GB, max_walltime=8 * hours, max_cores=((n + 1) * 10), architecture=Run.Arch.X86_64, ) core = Core(cfg) engine = Engine(core) # generate 50 no-op tasks for n in range(num_jobs): name = 'app{nr}'.format(nr=n) engine.add( Application( ['/bin/true'], inputs=[], outputs=[], output_dir='/tmp', jobname=name, requested_cores=1, )) # submit them all engine.progress() # get handles to the actual backend objects rscs = [ core.get_backend('test{nr}'.format(nr=n + 1)) for n in range(num_resources) ] num_jobs_per_resource = [ len([ task for task in engine._in_flight if task.execution.resource_name == rsc.name ]) for rsc in rscs ] # check that all jobs have been submitted and that each # resource got at least one job assert_equal(sum(num_jobs_per_resource), num_jobs) for num in num_jobs_per_resource: assert num > 0 # since TYPE_CONSTRUCTOR_MAP is a class-level variable, we # need to clean up otherwise other tests will see the No-Op # backend del cfg.TYPE_CONSTRUCTOR_MAP['noop']
def test_engine_cannot_find_task_by_id_if_not_saved(): """ Test that *unsaved* tasks are cannot be retrieved from the Engine given their ID only. """ with temporary_core() as core: with temporary_directory() as tmpdir: store = FilesystemStore(tmpdir) engine = Engine(core, store=store) task = SuccessfulApp() engine.add(task) store.save(task) # guarantee it has a `.persistent_id` task_id = task.persistent_id with pytest.raises(KeyError): engine.find_task_by_id(task_id)
def temporary_engine(transition_graph=None, **kw): with temporary_core(transition_graph, **kw) as core: yield Engine(core)