def new_tasks(self, extra, epath=None, emask=0):
app = None
if not epath:
# Startup: just submit an Hello World application
extra['jobname'] = 'EchoApp'
app = gc3libs.Application(['/bin/echo', 'first run'], [],
gc3libs.ANY_OUTPUT,
stdout='stdout.txt',
stderr='stderr.txt',
**extra)
return [app]
else:
# A new file has been created. Process it.
extra['jobname'] = 'LSApp.%s' % os.path.basename(epath.path)
# inputs = [epath] if epath.scheme == 'file' else []
inputs = {epath: 'foo'}
if emask & inotifyx.IN_CLOSE_WRITE:
app = gc3libs.Application(['/bin/echo', epath],
inputs,
gc3libs.ANY_OUTPUT,
stdout='stdout.txt',
stderr='stderr.txt',
**extra)
return [app]
# No app created, return empty list
return []
def new_tasks(self, extra, epath=None, emask=0):
app = None
if not epath:
app = gc3libs.Application(
['/bin/echo', 'first run'],
[],
gc3libs.ANY_OUTPUT,
stdout='stdout.txt',
stderr='stderr.txt',
jobname='EchoApp',
**extra)
return [app]
else:
# A new file has been created. Process it.
if emask & inotifyx.IN_CLOSE_WRITE:
app = gc3libs.Application(
['/bin/echo', epath],
inputs={epath:'foo'},
outputs=gc3libs.ANY_OUTPUT,
stdout='stdout.txt',
stderr='stderr.txt',
jobname=('LSApp.' + os.path.basename(epath.path)),
**extra)
return [app]
# No app created, return empty list
return []
def stage1(self):
# Check exit status of previous task
if self.tasks[-1].execution.returncode % 2 == 0:
return gc3libs.Application(
arguments=[
"echo",
"previous application exited with an even exit code (%d)" %
self.tasks[-1].execution.returncode
],
inputs=[],
outputs=[],
output_dir='TwoStageWorkflow.stage1',
stdout='stdout.txt',
)
else:
return gc3libs.Application(
arguments=[
"echo",
"previous application exited with an odd exit code (%d)" %
self.tasks[-1].execution.returncode
],
inputs=[],
outputs=[],
output_dir='TwoStageWorkflow.stage1',
stdout='stdout.txt',
)
def test_stdout_in_directory(self):
"""Check that `Application.stdout` can include a full path"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
app = gc3libs.Application(
arguments=['/bin/echo', 'OK'],
inputs=[],
outputs=[],
output_dir=tmpdir,
stdout='logs/stdout.txt')
self.core.submit(app)
self.apps_to_kill.append(app)
self.run_until_terminating(app)
self.core.fetch_output(app)
stdout_dir = os.path.join(app.output_dir, 'logs')
stdout_file = os.path.join(stdout_dir, 'stdout.txt')
assert os.path.exists(stdout_dir)
assert os.path.isdir(stdout_dir)
assert os.path.exists(stdout_file)
assert os.path.isfile(stdout_file)
stdout_contents = open(stdout_file, 'r').read()
assert stdout_contents == 'OK\n'
def test_pid_list_is_string(self):
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
(fd, cfgfile) = tempfile.mkstemp()
f = os.fdopen(fd, 'w+')
f.write(TestBackendShellcmdCFG.CONF % ("False", cfgfile + '.d'))
f.close()
self.files_to_remove = [cfgfile, cfgfile + '.d']
self.cfg = gc3libs.config.Configuration()
self.cfg.merge_file(cfgfile)
self.core = gc3libs.core.Core(self.cfg)
self.backend = self.core.get_backend('localhost_test')
# Update resource status
app = gc3libs.Application(
arguments=['/bin/echo', 'Hello', 'World'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=1,
requested_memory=10 * Memory.MiB, )
try:
self.core.submit(app)
for pid in list(self.backend._job_infos.keys()):
assert isinstance(pid,str)
finally:
self.core.kill(app)
self.core.free(app)
def test_resource_usage(self):
"""Check book-keeping of core and memory resources"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
app = gc3libs.Application(
arguments=['/bin/echo', 'Hello', 'World'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=2,
requested_memory=10 * Memory.MB,
)
cores_before = self.backend.free_slots
mem_before = self.backend.available_memory
self.core.submit(app)
self.apps_to_kill.append(app)
cores_after = self.backend.free_slots
mem_after = self.backend.available_memory
assert cores_before == cores_after + 2
assert mem_before == mem_after + app.requested_memory
self.run_until_terminating(app)
assert self.backend.free_slots == cores_before
assert self.backend.available_memory == mem_before
def test_slots_usage(self):
"""Test slots (but no memory) book-keeping on the backend."""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
app = gc3libs.Application(arguments=['/bin/echo', 'Hello', 'World'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=2)
cores_before = self.backend.free_slots
mem_before = self.backend.available_memory
# app in state SUBMITTED, resources are allocated
self.core.submit(app)
cores_after = self.backend.free_slots
mem_after = self.backend.available_memory
assert cores_before == cores_after + 2
assert mem_before == mem_after
# app in state RUNNING, no change
self.core.update_job_state(app)
assert app.execution.state == gc3libs.Run.State.RUNNING
assert cores_before == cores_after + 2
assert mem_before == mem_after
# app in state TERMINATED, resources are released
self.core.update_job_state(app)
assert app.execution.state == gc3libs.Run.State.TERMINATING
assert self.backend.free_slots == cores_before
assert self.backend.available_memory == mem_before
def test_check_app_after_reloading_session(self):
"""Check that the job status is still available the end of the starter script"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
app = gc3libs.Application(
arguments=['/usr/bin/env'],
inputs=[],
outputs=[],
output_dir=tmpdir,
stdout="stdout.txt",
stderr="stderr.txt",
requested_cores=1, )
self.core.submit(app)
self.apps_to_kill.append(app)
self.cleanup_file(app.execution.lrms_execdir)
# The wrapper process should die and write the final status
# and the output to a file, so that `Core` will be able to
# retrieve it.
self.run_until_terminating(app)
assert app.execution.state == gc3libs.Run.State.TERMINATING
assert app.execution.returncode == 0
def stage0(self):
return gc3libs.Application(
arguments = ["bash", "-c", "exit $RANDOM"],
inputs = [],
outputs = [],
output_dir = 'TwoStageWorkflow.stage0',
stdout = 'stdout.txt',
stderr = 'stderr.txt',
)
def new_tasks(self, extra):
yield ('Dice App', DiceApplication, [
gc3libs.Application(arguments=["bash", "-c", "exit $[$RANDOM%6]"],
inputs=[],
outputs=[],
output_dir='DiceApp',
stderr='stderr.txt',
stdout='stdout.txt',
**extra)
], extra)
def test_not_enough_memory_usage(self):
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
bigapp = gc3libs.Application(
arguments=['/bin/echo', 'Hello', 'World'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=1,
requested_memory=self.backend.available_memory + Memory.B,
)
self.core.submit(bigapp)
def test_not_enough_cores_usage(self):
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
bigapp = gc3libs.Application(
arguments=['/bin/echo', 'Hello', 'World'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=self.backend.free_slots + 1,
requested_memory=10 * Memory.MiB, )
with pytest.raises(gc3libs.exceptions.NoResources):
self.core.submit(bigapp)
def test_not_enough_cores_usage(self):
"""Check that a `NoResources` exception is raised if more cores are requested than available"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
bigapp = gc3libs.Application(
arguments=['/bin/echo', 'Hello', 'World'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=self.backend.free_slots + 1,
requested_memory=10 * Memory.MiB,
)
self.core.submit(bigapp)
def test_not_enough_memory_usage(self):
"""Check that a `NoResources` exception is raised if more memory is requested than available"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
bigapp = gc3libs.Application(
arguments=['/bin/echo', 'Hello', 'World'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=1,
requested_memory=self.backend.total_memory + Memory.B, )
with pytest.raises(gc3libs.exceptions.NoResources):
self.core.submit(bigapp)
def test_persist_Application_with_no_job_name(self):
app = gc3libs.Application(arguments=['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp')
app.execution.state = Run.State.NEW
app.execution.lrms_jobid = 1
id_ = self.store.save(app)
q = sql.select([self.store._tables.c.state
]).where(self.store._tables.c.id == id_)
result = self.conn.execute(q)
row = result.fetchone()
assert row[0] == app.execution.state
def test_submission_ok(self):
"""
Test a successful submission cycle and the backends' resource
book-keeping.
"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
app = gc3libs.Application(
arguments=['/usr/bin/env'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=1,
)
self.core.submit(app)
self.cleanup_file(tmpdir)
# app must be in SUBMITTED state here
assert app.execution.state == gc3libs.Run.State.SUBMITTED
assert self.backend.free_slots == 122
assert self.backend.queued == 1
assert self.backend.user_queued == 1
assert self.backend.user_run == 0
# transition to RUNNING
self.core.update_job_state(app)
assert app.execution.state == gc3libs.Run.State.RUNNING
assert self.backend.free_slots == 122
assert self.backend.queued == 0
assert self.backend.user_queued == 0
assert self.backend.user_run == 1
# transition to TERMINATING
self.core.update_job_state(app)
assert app.execution.state == gc3libs.Run.State.TERMINATING
assert self.backend.free_slots == 123
assert self.backend.queued == 0
assert self.backend.user_queued == 0
assert self.backend.user_run == 0
# transition to TERMINATED
self.core.fetch_output(app)
assert app.execution.state == gc3libs.Run.State.TERMINATED
assert self.backend.free_slots == 123
assert self.backend.queued == 0
assert self.backend.user_queued == 0
assert self.backend.user_run == 0
def new_tasks(self, extra):
"""
Populate session with initial tasks.
"""
jobname = 'EchoApp'
extra['output_dir'] = os.path.join(self.params.working_dir, jobname)
return [
gc3libs.Application(
['/bin/echo', 'first run'],
inputs=[],
outputs=gc3libs.ANY_OUTPUT,
stdout='stdout.txt',
stderr='stderr.txt',
jobname=jobname,
**extra)
]
def test_submission_ok(self):
"""Test a successful submission cycle and the backends' resource book-keeping"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
app = gc3libs.Application(
arguments=['/usr/bin/env'],
inputs=[],
outputs=[],
output_dir=tmpdir,
stdout="stdout.txt",
stderr="stderr.txt",
requested_cores=1,
)
self.core.submit(app)
self.apps_to_kill.append(app)
self.cleanup_file(tmpdir)
self.cleanup_file(app.execution.lrms_execdir)
# there's no SUBMITTED state here: jobs go immediately into
# RUNNING state
assert app.execution.state == gc3libs.Run.State.SUBMITTED
assert self.backend.free_slots == 122
assert self.backend.user_queued == 0
assert self.backend.user_run == 1
self.run_until_terminating(app)
try:
assert app.execution.state == gc3libs.Run.State.TERMINATING
assert self.backend.free_slots == 123
assert self.backend.user_queued == 0
assert self.backend.user_run == 0
except:
self.core.fetch_output(app)
self.core.free(app)
raise
self.core.fetch_output(app)
try:
assert app.execution.state == gc3libs.Run.State.TERMINATED
assert self.backend.free_slots == 123
assert self.backend.user_queued == 0
assert self.backend.user_run == 0
except:
self.core.free(app)
raise
def created(self, inbox, subject):
"""
A new file has been created. Process it.
"""
path = subject.path
jobname = ('LSApp.' + os.path.basename(path))
extra = self.extra.copy()
extra['output_dir'] = os.path.join(self.params.working_dir, jobname)
self.add(
gc3libs.Application(
['/bin/echo', path],
inputs={path:'foo'},
outputs=gc3libs.ANY_OUTPUT,
stdout='stdout.txt',
stderr='stderr.txt',
jobname=jobname,
**extra)
)
def test_app_argument_with_spaces(self):
"""Check that arguments with spaces are not split"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
app = gc3libs.Application(
arguments=['/bin/ls', '-d', '/ /'],
inputs=[],
outputs=[],
output_dir=tmpdir,
stdout="stdout.txt",
stderr="stderr.txt",
requested_cores=1, )
self.core.submit(app)
self.apps_to_kill.append(app)
self.cleanup_file(app.execution.lrms_execdir)
self.run_until_terminating(app)
assert app.execution.state == gc3libs.Run.State.TERMINATING
assert app.execution.returncode != 0
def test_resource_sharing_w_multiple_backends(self):
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
(fd, cfgfile) = tempfile.mkstemp()
f = os.fdopen(fd, 'w+')
f.write(TestBackendShellcmdCFG.CONF % ("False", cfgfile + '.d'))
f.close()
self.files_to_remove = [cfgfile, cfgfile + '.d', tmpdir]
cfg1 = gc3libs.config.Configuration()
cfg1.merge_file(cfgfile)
cfg2 = gc3libs.config.Configuration()
cfg2.merge_file(cfgfile)
core1 = gc3libs.core.Core(cfg1)
core2 = gc3libs.core.Core(cfg2)
backend1 = core1.get_backend('localhost_test')
backend2 = core2.get_backend('localhost_test')
app = gc3libs.Application(
arguments=['/bin/echo', 'Hello', 'World'],
inputs=[],
outputs=[],
output_dir=tmpdir,
requested_cores=1,
requested_memory=10 * Memory.MiB,
)
try:
core1.submit(app)
assert (backend1.free_slots == backend1.max_cores -
app.requested_cores)
assert backend2.free_slots == backend2.max_cores
backend2.get_resource_status()
assert (backend2.free_slots == backend2.max_cores -
app.requested_cores)
finally:
core1.kill(app)
core1.free(app)
def test_env_vars_definition(self):
"""Check that `Application.environment` settings are correctly propagated"""
tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='.d')
self.cleanup_file(tmpdir)
app = gc3libs.Application(arguments=['/bin/echo', '$MSG'],
inputs=[],
outputs=[],
output_dir=tmpdir,
stdout='stdout.txt',
environment={'MSG': 'OK'})
self.core.submit(app)
self.apps_to_kill.append(app)
self.run_until_terminating(app)
self.core.fetch_output(app)
stdout_file = os.path.join(app.output_dir, app.stdout)
assert os.path.exists(stdout_file)
assert os.path.isfile(stdout_file)
stdout_contents = open(stdout_file, 'r').read()
assert stdout_contents == 'OK\n'
