def new_tasks(self, extra, epath=None, emask=0): app = None if not epath: # Startup: just submit an Hello World application extra['jobname'] = 'EchoApp' app = gc3libs.Application(['/bin/echo', 'first run'], [], gc3libs.ANY_OUTPUT, stdout='stdout.txt', stderr='stderr.txt', **extra) return [app] else: # A new file has been created. Process it. extra['jobname'] = 'LSApp.%s' % os.path.basename(epath.path) # inputs = [epath] if epath.scheme == 'file' else [] inputs = {epath: 'foo'} if emask & inotifyx.IN_CLOSE_WRITE: app = gc3libs.Application(['/bin/echo', epath], inputs, gc3libs.ANY_OUTPUT, stdout='stdout.txt', stderr='stderr.txt', **extra) return [app] # No app created, return empty list return []
def new_tasks(self, extra, epath=None, emask=0): app = None if not epath: app = gc3libs.Application( ['/bin/echo', 'first run'], [], gc3libs.ANY_OUTPUT, stdout='stdout.txt', stderr='stderr.txt', jobname='EchoApp', **extra) return [app] else: # A new file has been created. Process it. if emask & inotifyx.IN_CLOSE_WRITE: app = gc3libs.Application( ['/bin/echo', epath], inputs={epath:'foo'}, outputs=gc3libs.ANY_OUTPUT, stdout='stdout.txt', stderr='stderr.txt', jobname=('LSApp.' + os.path.basename(epath.path)), **extra) return [app] # No app created, return empty list return []
def stage1(self): # Check exit status of previous task if self.tasks[-1].execution.returncode % 2 == 0: return gc3libs.Application( arguments=[ "echo", "previous application exited with an even exit code (%d)" % self.tasks[-1].execution.returncode ], inputs=[], outputs=[], output_dir='TwoStageWorkflow.stage1', stdout='stdout.txt', ) else: return gc3libs.Application( arguments=[ "echo", "previous application exited with an odd exit code (%d)" % self.tasks[-1].execution.returncode ], inputs=[], outputs=[], output_dir='TwoStageWorkflow.stage1', stdout='stdout.txt', )
def test_stdout_in_directory(self): """Check that `Application.stdout` can include a full path""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) app = gc3libs.Application( arguments=['/bin/echo', 'OK'], inputs=[], outputs=[], output_dir=tmpdir, stdout='logs/stdout.txt') self.core.submit(app) self.apps_to_kill.append(app) self.run_until_terminating(app) self.core.fetch_output(app) stdout_dir = os.path.join(app.output_dir, 'logs') stdout_file = os.path.join(stdout_dir, 'stdout.txt') assert os.path.exists(stdout_dir) assert os.path.isdir(stdout_dir) assert os.path.exists(stdout_file) assert os.path.isfile(stdout_file) stdout_contents = open(stdout_file, 'r').read() assert stdout_contents == 'OK\n'
def test_pid_list_is_string(self): tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') (fd, cfgfile) = tempfile.mkstemp() f = os.fdopen(fd, 'w+') f.write(TestBackendShellcmdCFG.CONF % ("False", cfgfile + '.d')) f.close() self.files_to_remove = [cfgfile, cfgfile + '.d'] self.cfg = gc3libs.config.Configuration() self.cfg.merge_file(cfgfile) self.core = gc3libs.core.Core(self.cfg) self.backend = self.core.get_backend('localhost_test') # Update resource status app = gc3libs.Application( arguments=['/bin/echo', 'Hello', 'World'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=1, requested_memory=10 * Memory.MiB, ) try: self.core.submit(app) for pid in list(self.backend._job_infos.keys()): assert isinstance(pid,str) finally: self.core.kill(app) self.core.free(app)
def test_resource_usage(self): """Check book-keeping of core and memory resources""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) app = gc3libs.Application( arguments=['/bin/echo', 'Hello', 'World'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=2, requested_memory=10 * Memory.MB, ) cores_before = self.backend.free_slots mem_before = self.backend.available_memory self.core.submit(app) self.apps_to_kill.append(app) cores_after = self.backend.free_slots mem_after = self.backend.available_memory assert cores_before == cores_after + 2 assert mem_before == mem_after + app.requested_memory self.run_until_terminating(app) assert self.backend.free_slots == cores_before assert self.backend.available_memory == mem_before
def test_slots_usage(self): """Test slots (but no memory) book-keeping on the backend.""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) app = gc3libs.Application(arguments=['/bin/echo', 'Hello', 'World'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=2) cores_before = self.backend.free_slots mem_before = self.backend.available_memory # app in state SUBMITTED, resources are allocated self.core.submit(app) cores_after = self.backend.free_slots mem_after = self.backend.available_memory assert cores_before == cores_after + 2 assert mem_before == mem_after # app in state RUNNING, no change self.core.update_job_state(app) assert app.execution.state == gc3libs.Run.State.RUNNING assert cores_before == cores_after + 2 assert mem_before == mem_after # app in state TERMINATED, resources are released self.core.update_job_state(app) assert app.execution.state == gc3libs.Run.State.TERMINATING assert self.backend.free_slots == cores_before assert self.backend.available_memory == mem_before
def test_check_app_after_reloading_session(self): """Check that the job status is still available the end of the starter script""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) app = gc3libs.Application( arguments=['/usr/bin/env'], inputs=[], outputs=[], output_dir=tmpdir, stdout="stdout.txt", stderr="stderr.txt", requested_cores=1, ) self.core.submit(app) self.apps_to_kill.append(app) self.cleanup_file(app.execution.lrms_execdir) # The wrapper process should die and write the final status # and the output to a file, so that `Core` will be able to # retrieve it. self.run_until_terminating(app) assert app.execution.state == gc3libs.Run.State.TERMINATING assert app.execution.returncode == 0
def stage0(self): return gc3libs.Application( arguments = ["bash", "-c", "exit $RANDOM"], inputs = [], outputs = [], output_dir = 'TwoStageWorkflow.stage0', stdout = 'stdout.txt', stderr = 'stderr.txt', )
def new_tasks(self, extra): yield ('Dice App', DiceApplication, [ gc3libs.Application(arguments=["bash", "-c", "exit $[$RANDOM%6]"], inputs=[], outputs=[], output_dir='DiceApp', stderr='stderr.txt', stdout='stdout.txt', **extra) ], extra)
def test_not_enough_memory_usage(self): tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) bigapp = gc3libs.Application( arguments=['/bin/echo', 'Hello', 'World'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=1, requested_memory=self.backend.available_memory + Memory.B, ) self.core.submit(bigapp)
def test_not_enough_cores_usage(self): tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) bigapp = gc3libs.Application( arguments=['/bin/echo', 'Hello', 'World'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=self.backend.free_slots + 1, requested_memory=10 * Memory.MiB, ) with pytest.raises(gc3libs.exceptions.NoResources): self.core.submit(bigapp)
def test_not_enough_cores_usage(self): """Check that a `NoResources` exception is raised if more cores are requested than available""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) bigapp = gc3libs.Application( arguments=['/bin/echo', 'Hello', 'World'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=self.backend.free_slots + 1, requested_memory=10 * Memory.MiB, ) self.core.submit(bigapp)
def test_not_enough_memory_usage(self): """Check that a `NoResources` exception is raised if more memory is requested than available""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) bigapp = gc3libs.Application( arguments=['/bin/echo', 'Hello', 'World'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=1, requested_memory=self.backend.total_memory + Memory.B, ) with pytest.raises(gc3libs.exceptions.NoResources): self.core.submit(bigapp)
def test_persist_Application_with_no_job_name(self): app = gc3libs.Application(arguments=['/bin/true'], inputs=[], outputs=[], output_dir='/tmp') app.execution.state = Run.State.NEW app.execution.lrms_jobid = 1 id_ = self.store.save(app) q = sql.select([self.store._tables.c.state ]).where(self.store._tables.c.id == id_) result = self.conn.execute(q) row = result.fetchone() assert row[0] == app.execution.state
def test_submission_ok(self): """ Test a successful submission cycle and the backends' resource book-keeping. """ tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') app = gc3libs.Application( arguments=['/usr/bin/env'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=1, ) self.core.submit(app) self.cleanup_file(tmpdir) # app must be in SUBMITTED state here assert app.execution.state == gc3libs.Run.State.SUBMITTED assert self.backend.free_slots == 122 assert self.backend.queued == 1 assert self.backend.user_queued == 1 assert self.backend.user_run == 0 # transition to RUNNING self.core.update_job_state(app) assert app.execution.state == gc3libs.Run.State.RUNNING assert self.backend.free_slots == 122 assert self.backend.queued == 0 assert self.backend.user_queued == 0 assert self.backend.user_run == 1 # transition to TERMINATING self.core.update_job_state(app) assert app.execution.state == gc3libs.Run.State.TERMINATING assert self.backend.free_slots == 123 assert self.backend.queued == 0 assert self.backend.user_queued == 0 assert self.backend.user_run == 0 # transition to TERMINATED self.core.fetch_output(app) assert app.execution.state == gc3libs.Run.State.TERMINATED assert self.backend.free_slots == 123 assert self.backend.queued == 0 assert self.backend.user_queued == 0 assert self.backend.user_run == 0
def new_tasks(self, extra): """ Populate session with initial tasks. """ jobname = 'EchoApp' extra['output_dir'] = os.path.join(self.params.working_dir, jobname) return [ gc3libs.Application( ['/bin/echo', 'first run'], inputs=[], outputs=gc3libs.ANY_OUTPUT, stdout='stdout.txt', stderr='stderr.txt', jobname=jobname, **extra) ]
def test_submission_ok(self): """Test a successful submission cycle and the backends' resource book-keeping""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') app = gc3libs.Application( arguments=['/usr/bin/env'], inputs=[], outputs=[], output_dir=tmpdir, stdout="stdout.txt", stderr="stderr.txt", requested_cores=1, ) self.core.submit(app) self.apps_to_kill.append(app) self.cleanup_file(tmpdir) self.cleanup_file(app.execution.lrms_execdir) # there's no SUBMITTED state here: jobs go immediately into # RUNNING state assert app.execution.state == gc3libs.Run.State.SUBMITTED assert self.backend.free_slots == 122 assert self.backend.user_queued == 0 assert self.backend.user_run == 1 self.run_until_terminating(app) try: assert app.execution.state == gc3libs.Run.State.TERMINATING assert self.backend.free_slots == 123 assert self.backend.user_queued == 0 assert self.backend.user_run == 0 except: self.core.fetch_output(app) self.core.free(app) raise self.core.fetch_output(app) try: assert app.execution.state == gc3libs.Run.State.TERMINATED assert self.backend.free_slots == 123 assert self.backend.user_queued == 0 assert self.backend.user_run == 0 except: self.core.free(app) raise
def created(self, inbox, subject): """ A new file has been created. Process it. """ path = subject.path jobname = ('LSApp.' + os.path.basename(path)) extra = self.extra.copy() extra['output_dir'] = os.path.join(self.params.working_dir, jobname) self.add( gc3libs.Application( ['/bin/echo', path], inputs={path:'foo'}, outputs=gc3libs.ANY_OUTPUT, stdout='stdout.txt', stderr='stderr.txt', jobname=jobname, **extra) )
def test_app_argument_with_spaces(self): """Check that arguments with spaces are not split""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) app = gc3libs.Application( arguments=['/bin/ls', '-d', '/ /'], inputs=[], outputs=[], output_dir=tmpdir, stdout="stdout.txt", stderr="stderr.txt", requested_cores=1, ) self.core.submit(app) self.apps_to_kill.append(app) self.cleanup_file(app.execution.lrms_execdir) self.run_until_terminating(app) assert app.execution.state == gc3libs.Run.State.TERMINATING assert app.execution.returncode != 0
def test_resource_sharing_w_multiple_backends(self): tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') (fd, cfgfile) = tempfile.mkstemp() f = os.fdopen(fd, 'w+') f.write(TestBackendShellcmdCFG.CONF % ("False", cfgfile + '.d')) f.close() self.files_to_remove = [cfgfile, cfgfile + '.d', tmpdir] cfg1 = gc3libs.config.Configuration() cfg1.merge_file(cfgfile) cfg2 = gc3libs.config.Configuration() cfg2.merge_file(cfgfile) core1 = gc3libs.core.Core(cfg1) core2 = gc3libs.core.Core(cfg2) backend1 = core1.get_backend('localhost_test') backend2 = core2.get_backend('localhost_test') app = gc3libs.Application( arguments=['/bin/echo', 'Hello', 'World'], inputs=[], outputs=[], output_dir=tmpdir, requested_cores=1, requested_memory=10 * Memory.MiB, ) try: core1.submit(app) assert (backend1.free_slots == backend1.max_cores - app.requested_cores) assert backend2.free_slots == backend2.max_cores backend2.get_resource_status() assert (backend2.free_slots == backend2.max_cores - app.requested_cores) finally: core1.kill(app) core1.free(app)
def test_env_vars_definition(self): """Check that `Application.environment` settings are correctly propagated""" tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d') self.cleanup_file(tmpdir) app = gc3libs.Application(arguments=['/bin/echo', '$MSG'], inputs=[], outputs=[], output_dir=tmpdir, stdout='stdout.txt', environment={'MSG': 'OK'}) self.core.submit(app) self.apps_to_kill.append(app) self.run_until_terminating(app) self.core.fetch_output(app) stdout_file = os.path.join(app.output_dir, app.stdout) assert os.path.exists(stdout_file) assert os.path.isfile(stdout_file) stdout_contents = open(stdout_file, 'r').read() assert stdout_contents == 'OK\n'