def create_gc3pie_engine(store): '''Creates an `Engine` instance for submitting jobs for parallel processing. Parameters ---------- store: gc3libs.persistence.store.Store GC3Pie store object Returns ------- gc3libs.core.Engine engine ''' logger.debug('create GC3Pie engine') n = cfg.resource.max_cores * 2 logger.debug('set maximum number of submitted jobs to %d', n) engine = gc3libs.create_engine(store=store, max_in_flight=n, max_submitted=n, forget_terminated=True) # Put all output files in the same directory logger.debug('store stdout/stderr in common output directory') engine.retrieve_overwrites = True return engine
def complete(self): """ Complete a bulk job submission. Create engine, and progress it until all jobs have terminated. """ # create an instance of `Engine` using the list of configuration files try: self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True) except gc3libs.exceptions.Error as err: raise EasyBuildError("Failed to create GC3Pie engine: %s", err) # make sure that all job log files end up in the same directory, rather than renaming the output directory # see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine self._engine.retrieve_overwrites = True # `Engine.stats()` (which is used later on in `_print_status_report()`) # changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible # with both try: self._engine.init_stats_for(Application) except AttributeError: _log.debug("No `init_stats_for` method in the Engine class;" " assuming pre-2.5.0 GC3Pie and ignoring error.") # Add your application to the engine. This will NOT submit # your application yet, but will make the engine *aware* of # the application. self._engine.add(self.jobs) # in case you want to select a specific resource, call target_resource = build_option('job_target_resource') if target_resource: res = self._engine.select_resource(target_resource) if res == 0: raise EasyBuildError( "Failed to select target resource '%s' in GC3Pie", target_resource) # Periodically check the status of your application. while self.jobs.execution.state != Run.State.TERMINATED: # `Engine.progress()` will do the GC3Pie magic: # submit new jobs, update status of submitted jobs, get # results of terminating jobs etc... self._engine.progress() # report progress self._print_status_report() # Wait a few seconds... time.sleep(self.poll_interval) # final status report print_msg("Done processing jobs", log=self.log, silent=build_option('silent')) self._print_status_report()
def test_create_engine_non_default2(): """Test `create_engine` with several non-default arguments.""" with temporary_config_file() as cfgfile: engine = create_engine(cfgfile.name, can_submit=False, max_in_flight=1234) assert_equal(engine.can_submit, False) assert_equal(engine.max_in_flight, 1234)
def test_create_engine_non_default2(): """Test `create_engine` with several non-default arguments.""" with temporary_config_file() as cfgfile: engine = create_engine(cfgfile.name, can_submit=False, max_in_flight=1234) assert engine.can_submit == False assert engine.max_in_flight == 1234
def complete(self): """ Complete a bulk job submission. Create engine, and progress it until all jobs have terminated. """ # create an instance of `Engine` using the list of configuration files try: self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True) except gc3libs.exceptions.Error as err: raise EasyBuildError("Failed to create GC3Pie engine: %s", err) # make sure that all job log files end up in the same directory, rather than renaming the output directory # see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine self._engine.retrieve_overwrites = True # some sites may not be happy with flooding the cluster with build jobs... self._engine.max_in_flight = build_option('job_max_jobs') # `Engine.stats()` (which is used later on in `_print_status_report()`) # changed between 2.4.2 and 2.5.0.dev -- make sure we stay compatible # with both try: self._engine.init_stats_for(Application) except AttributeError: _log.debug("No `init_stats_for` method in the Engine class;" " assuming pre-2.5.0 GC3Pie and ignoring error.") # Add your application to the engine. This will NOT submit # your application yet, but will make the engine *aware* of # the application. self._engine.add(self.jobs) # in case you want to select a specific resource, call target_resource = build_option('job_target_resource') if target_resource: res = self._engine.select_resource(target_resource) if res == 0: raise EasyBuildError("Failed to select target resource '%s' in GC3Pie", target_resource) # Periodically check the status of your application. while self.jobs.execution.state != Run.State.TERMINATED: # `Engine.progress()` will do the GC3Pie magic: # submit new jobs, update status of submitted jobs, get # results of terminating jobs etc... self._engine.progress() # report progress self._print_status_report() # Wait a few seconds... time.sleep(self.poll_interval) # final status report print_msg("Done processing jobs", log=self.log, silent=build_option('silent')) self._print_status_report()
def test_create_engine_with_core_options(): """Test `create_engine` with a mix of Engine and Core options.""" with temporary_config_file() as cfgfile: # use a specific MatchMaker instance for equality testing mm = MatchMaker() engine = create_engine(cfgfile.name, can_submit=False, matchmaker=mm, auto_enable_auth=False) assert_equal(engine.can_submit, False) assert_equal(engine._core.matchmaker, mm) assert_equal(engine._core.auto_enable_auth, False)
def test_create_engine_with_core_options(): """Test `create_engine` with a mix of Engine and Core options.""" with temporary_config_file() as cfgfile: # use a specific MatchMaker instance for equality testing mm = MatchMaker() engine = create_engine(cfgfile.name, can_submit=False, matchmaker=mm, auto_enable_auth=False) assert engine.can_submit == False assert engine._core.matchmaker == mm assert engine._core.auto_enable_auth == False
def complete(self): """ Complete a bulk job submission. Create engine, and progress it until all jobs have terminated. """ # create an instance of `Engine` using the list of configuration files try: self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True) except gc3libs.exceptions.Error as err: raise EasyBuildError("Failed to create GC3Pie engine: %s", err) # make sure that all job log files end up in the same directory, rather than renaming the output directory # see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine self._engine.retrieve_overwrites = True # some sites may not be happy with flooding the cluster with build jobs... self._engine.max_in_flight = build_option('job_max_jobs') # Add your application to the engine. This will NOT submit # your application yet, but will make the engine *aware* of # the application. self._engine.add(self.jobs) # select a specific execution resource? target_resource = build_option('job_target_resource') if target_resource: res = self._engine.select_resource(target_resource) if res == 0: raise EasyBuildError( "Failed to select target resource '%s' in GC3Pie", target_resource) # Periodically check the status of your application. while self.jobs.execution.state != Run.State.TERMINATED: # `Engine.progress()` will do the GC3Pie magic: # submit new jobs, update status of submitted jobs, get # results of terminating jobs etc... self._engine.progress() # report progress self._print_status_report() # Wait a few seconds... time.sleep(self.poll_interval) # final status report print_msg("Done processing jobs", log=self.log, silent=build_option('silent')) self._print_status_report()
def complete(self): """ Complete a bulk job submission. Create engine, and progress it until all jobs have terminated. """ # create an instance of `Engine` using the list of configuration files try: self._engine = create_engine(*self.config_files, resource_errors_are_fatal=True) except gc3libs.exceptions.Error as err: raise EasyBuildError("Failed to create GC3Pie engine: %s", err) # make sure that all job log files end up in the same directory, rather than renaming the output directory # see https://gc3pie.readthedocs.org/en/latest/programmers/api/gc3libs/core.html#gc3libs.core.Engine self._engine.retrieve_overwrites = True # Add your application to the engine. This will NOT submit # your application yet, but will make the engine *aware* of # the application. self._engine.add(self.jobs) # in case you want to select a specific resource, call target_resource = build_option("job_target_resource") if target_resource: res = self._engine.select_resource(target_resource) if res == 0: raise EasyBuildError("Failed to select target resource '%s' in GC3Pie", target_resource) # Periodically check the status of your application. while self.jobs.execution.state != Run.State.TERMINATED: # `Engine.progress()` will do the GC3Pie magic: # submit new jobs, update status of submitted jobs, get # results of terminating jobs etc... self._engine.progress() # report progress self._print_status_report() # Wait a few seconds... time.sleep(self.poll_interval) # final status report print_msg("Done processing jobs", log=self.log, silent=build_option("silent")) self._print_status_report()
def test_issue(self): """Test that SequentialTasksCollection goes in terminated state when all of its tasks are in TERMINATED state.""" self.ptasks = 5 task = MySequentialCollection( [ Application( ['echo','test1'], [],[], os.path.join(self.tmpdir, 'test.%d.d' % i)) for i in range(self.ptasks) ] ) engine = create_engine(self.cfgfile, auto_enable_auth=True) engine.add(task) while True: engine.progress() if len([t for t in task.tasks if t.execution.state == Run.State.TERMINATED]) == self.ptasks: engine.progress() assert_equal(task.execution.state, Run.State.TERMINATED) break assert_equal(task.next_called_n_times, self.ptasks)
def test_issue_335(self): """Test that SequentialTasksCollection goes in TERMINATED state when all of its tasks are in TERMINATED state.""" num_tasks_in_seq = 5 seq = MySequentialCollection([ Application(['echo', 'test1'], [], [], os.path.join(self.tmpdir, 'test.%d.d' % i)) for i in range(num_tasks_in_seq) ]) engine = create_engine(self.cfgfile, auto_enable_auth=True) engine.add(seq) while True: engine.progress() if (len([ task for task in seq.tasks if task.execution.state == Run.State.TERMINATED ]) == num_tasks_in_seq): engine.progress() # check that final SequentialCollection state is TERMINATED assert seq.execution.state == Run.State.TERMINATED break # check that next() has been called once per each task assert seq.next_called_n_times == num_tasks_in_seq
def test_issue_335(self): """Test that SequentialTasksCollection goes in TERMINATED state when all of its tasks are in TERMINATED state.""" num_tasks_in_seq = 5 seq = MySequentialCollection([ Application( ['echo', 'test1'], [], [], os.path.join(self.tmpdir, 'test.%d.d' % i)) for i in range(num_tasks_in_seq) ]) engine = create_engine(self.cfgfile, auto_enable_auth=True) engine.add(seq) while True: engine.progress() if (len([task for task in seq.tasks if task.execution.state == Run.State.TERMINATED]) == num_tasks_in_seq): engine.progress() # check that final SequentialCollection state is TERMINATED assert seq.execution.state == Run.State.TERMINATED break # check that next() has been called once per each task assert seq.next_called_n_times == num_tasks_in_seq
def create_gc3pie_engine(store): '''Creates an `Engine` instance for submitting jobs for parallel processing. Parameters ---------- store: gc3libs.persistence.store.Store GC3Pie store object Returns ------- gc3libs.core.Engine engine ''' logger.debug('create GC3Pie engine') n = cfg.resource.max_cores * 2 logger.debug('set maximum number of submitted jobs to %d', n) engine = gc3libs.create_engine( store=store, max_in_flight=n, max_submitted=n, forget_terminated=True ) # Put all output files in the same directory logger.debug('store stdout/stderr in common output directory') engine.retrieve_overwrites = True return engine
# the following arguments are mandatory: arguments=["/bin/hostname"], inputs=[], outputs=[], output_dir="./GdemoSimpleApp_output", # the rest is optional and has reasonable defaults: stdout="stdout.txt", ) # Create an instance of GdemoSimpleApp app = GdemoSimpleApp() # Create an instance of `Engine` using the configuration file present # in your home directory. engine = gc3libs.create_engine() # Add your application to the engine. This will NOT submit your # application yet, but will make the engine awere *aware* of the # application. engine.add(app) # in case you want to select a specific resource, call # `Engine.select_resource(<resource_name>)` if len(sys.argv) > 1: engine.select_resource(sys.argv[1]) # Periodically check the status of your application. while app.execution.state != gc3libs.Run.State.TERMINATED: print "Job in status %s " % app.execution.state # `Engine.progress()` will do the GC3Pie magic:
# the following arguments are mandatory: arguments=args, inputs=[], outputs=[], output_dir=self.__class__.__name__, # the rest is optional and has reasonable defaults: stdout="stdout.txt",) def terminated(self): logger.info('program was terminated') print self.execution.state print self.execution.returncode self.execution.returncode = 1 app = CommandApp('/bin/hostname') engine = gc3libs.create_engine() engine.add(app) if len(sys.argv) > 1: engine.select_resource(sys.argv[1]) while app.execution.state != gc3libs.Run.State.TERMINATED: print "Job in status %s " % app.execution.state engine.progress() time.sleep(1) print("Job is now terminated.") print("The output of the application is in `%s`." % app.output_dir)
def run_jobs(jobs, argv, interval=1, verbose=True, max_concurrent=0): """ Create and run jobs, each executing the command specified by `argv`. If any item in sequence `argv` is equal to the (single-character string) ``#``, it is substituted with the current job index. """ engine = create_engine(max_in_flight=max_concurrent) tasks = [] # convenience for extracting stats later for n in xrange(jobs): jobname = ('worker{n}'.format(n=n)) job_argv = [(arg if arg != '#' else n) for arg in argv] task = Application( job_argv, inputs=[], outputs=[], output_dir=os.path.join(os.getcwd(), jobname), stdout=(jobname + '.log'), join=True, jobname = jobname, ) engine.add(task) tasks.append(task) # loop until all jobs are done stats = engine.stats() done = stats['TERMINATED'] while done < jobs: time.sleep(interval) engine.progress() stats = engine.stats() done = stats['TERMINATED'] logging.info( "%d jobs terminated (of which %d successfully)," " %d running, %d queued.", done, stats['ok'], stats['RUNNING'], stats['SUBMITTED'] + stats['NEW'] ) if verbose: fields = [ # description field name type zero value ('duration', 'duration', Duration, (0, seconds)), ('CPU time', 'used_cpu_time', Duration, (0, seconds)), ('RAM', 'max_used_memory', Memory, (0, Memory.MB)), ] # initialize counters to 0 totals = {} totals_ok = {} for desc, name, init, zero in fields: totals[name] = init(*zero) totals_ok[name] = init(*zero) # compute totals for task in tasks: for desc, name, _, _ in fields: value = getattr(task.execution, name) totals[name] += value if (task.execution.state == 'TERMINATED' and task.execution.returncode == 0): totals_ok[name] += value # print totals and averages print("Resource consumption statistics:") for desc, name, _, _ in fields: print ("- Average {0} per job: {1}".format(desc, totals[name] / jobs)) print ("- Average {0} per *successful* job: {1}".format(desc, totals_ok[name] / jobs))
def test_create_engine_default(): """Test `create_engine` with factory defaults.""" with temporary_config_file() as cfgfile: # std factory params engine = create_engine(cfgfile.name) assert_is_instance(engine, Engine)
def test_create_engine_non_default1(): """Test `create_engine` with one non-default argument.""" with temporary_config_file() as cfgfile: engine = create_engine(cfgfile.name, can_submit=False) assert engine.can_submit == False
def test_create_engine_default(): """Test `create_engine` with factory defaults.""" with temporary_config_file() as cfgfile: # std factory params engine = create_engine(cfgfile.name) assert isinstance(engine, Engine)
def test_create_engine_non_default1(): """Test `create_engine` with one non-default argument.""" with temporary_config_file() as cfgfile: engine = create_engine(cfgfile.name, can_submit=False) assert_equal(engine.can_submit, False)
import gc3libs gc3libs.configure_logger(30) gc3conffile = 'config/samples/gc3pie_localhost.conf' engine = gc3libs.create_engine(gc3conffile) core = engine._core