def test_pack_call_local_fail(tmpdir, basic_localfs_state, localproc_pack_fail,
default_async):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
with pytest.raises(RuntimeError):
localproc_pack_fail(parameters=pars, state=basic_localfs_state)
assert tmpdir.join('helloworld.txt').check()
def __init__(self, name, spec, state, parameters=None, inputs=None):
self.metadata = {"name": name}
self.inputs = inputs or []
self.parameters = datamodel.create(parameters or {},
state.datamodel if state else None)
self.spec = spec
self.state = state
def test_docker_parmounts(tmpdir,basic_localfs_state, docker_env_parmounts):
state = basic_localfs_state
environment = docker_env_parmounts.spec['environment']
parameters, state = finalize_inputs(pdm.create({'outputfile': '{workdir}/hello.txt'}), state)
env = handlers[environment['environment_type']]['default'](environment,parameters,state)
assert env['par_mounts'][0]['mountcontent'] == '"{}"'.format(parameters['outputfile'])
def test_fromparjq_pub(tmpdir,basic_localfs_state):
tmpdir.join('hello_myvalue_1.txt').ensure(file = True)
tmpdir.join('hello_myvalue_2.txt').ensure(file = True)
pub = {
'publisher_type': 'fromparjq-pub',
'script': '{hello: ["hello_myvalue_2.txt","hello_myvalue_1.txt"]}',
'relative_paths': True,
'tryExact': True,
'glob': False
}
pars = pdm.create({
'mypar': 'myvalue'
})
pars = pdm.create(pars)
pubbed = handlers['fromparjq-pub']['default'](pub,pars,basic_localfs_state)
filelist = map(str,[tmpdir.join('hello_myvalue_1.txt'),tmpdir.join('hello_myvalue_2.txt')])
assert set(pubbed['hello']) == set(filelist)
def test_multiproc(tmpdir,basic_localfs_state,localproc_packspec):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
backend = MultiProcBackend(2)
proxy = backend.submit(localproc_packspec,pars,basic_localfs_state)
while not backend.ready(proxy):
pass
assert backend.successful(proxy)
assert backend.result(proxy).json() == {'output': str(tmpdir.join('helloworld.txt'))}
assert tmpdir.join('helloworld.txt').check() == True
def test_multiproc_fail(tmpdir, basic_localfs_state, localproc_pack_fail):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
backend = MultiProcBackend(2)
proxy = backend.submit(localproc_pack_fail, pars, basic_localfs_state)
while not backend.ready(proxy):
pass
assert backend.successful(proxy) == False
backend.fail_info(proxy)
def test_multiproc_fail(tmpdir,basic_localfs_state,localproc_pack_fail):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
backend = MultiProcBackend(2)
proxy = backend.submit(localproc_pack_fail,pars,basic_localfs_state)
while not backend.ready(proxy):
pass
assert backend.successful(proxy) == False
backend.fail_info(proxy)
def test_yml_pub(tmpdir,basic_localfs_state):
tmpdir.join('hello.yml').write('hello: world\n')
pub = {
'publisher_type': 'fromyaml-pub',
'yamlfile': 'hello.yml',
}
pars = pdm.create({
'mypar': 'myvalue'
})
pubbed = handlers['fromyaml-pub']['default'](pub,pars,basic_localfs_state)
assert pubbed == {'hello': 'world'}
def test_pack_call_local(tmpdir, basic_localfs_state):
pars = pdm.create({
'parcard': ['{workdir}/parcard.dat'],
'banner_file': '{workdir}/banner.txt'
})
newpars = basic_localfs_state.model(pars)
assert newpars['banner_file'] == os.path.join(str(tmpdir), 'banner.txt')
assert newpars['parcard'][0] == os.path.join(str(tmpdir), 'parcard.dat')
def test_build_oneline_job(default_handler_config, basic_localfs_state):
job = packtivity.syncbackends.build_job(
{
'process_type': 'string-interpolated-cmd',
'cmd': 'hello {one} {two}'
}, pdm.create({
'one': 'ONE',
'two': 'TWO'
}), basic_localfs_state, default_handler_config)
assert 'command' in job
assert job['command'] == 'hello ONE TWO'
def test_docker_parmounts(tmpdir, basic_localfs_state, docker_env_parmounts):
state = basic_localfs_state
environment = docker_env_parmounts.spec['environment']
parameters, state = finalize_inputs(
pdm.create({'outputfile': '{workdir}/hello.txt'}), state)
env = handlers[environment['environment_type']]['default'](environment,
parameters,
state)
assert env['par_mounts'][0]['mountcontent'] == '"{}"'.format(
parameters['outputfile'])
def test_build_oneline_job(default_handler_config,basic_localfs_state):
job = packtivity.syncbackends.build_job(
{
'process_type':'string-interpolated-cmd',
'cmd': 'hello {one} {two}'},
pdm.create({'one':'ONE', 'two':'TWO'}),
basic_localfs_state,
default_handler_config
)
assert 'command' in job
assert job['command'] == 'hello ONE TWO'
def fromJSON(cls, data, deserialization_opts = None):
if data['task']['type'] == 'packtivity_task':
task = packtivity_task.fromJSON(data['task'], deserialization_opts)
result = datamodel.create(data['result'],getattr(task.state,'datamodel',{})) if data['result'] else None
instance = cls(data['name'], task, data['id'],result)
adage.serialize.set_generic_data(instance,data)
instance.resultproxy = load_proxy(data['proxy'],deserialization_opts, best_effort_backend = False) if data['proxy'] else None
return instance
else:
raise RuntimeError('unknown task type',data['task']['type'])
def test_build_script_job(default_handler_config, basic_localfs_state):
job = packtivity.syncbackends.build_job(
{
'process_type': 'interpolated-script-cmd',
'interpreter': 'sh',
'script': 'hello {one} {two}\n echo another line {two}'
}, pdm.create({
'one': 'ONE',
'two': 'TWO'
}), basic_localfs_state, default_handler_config)
assert 'script' in job
assert 'interpreter' in job
def test_multiproc(tmpdir, basic_localfs_state, localproc_packspec):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
backend = MultiProcBackend(2)
proxy = backend.submit(localproc_packspec, pars, basic_localfs_state)
while not backend.ready(proxy):
pass
assert backend.successful(proxy)
assert backend.result(proxy).json() == {
'output': str(tmpdir.join('helloworld.txt'))
}
assert tmpdir.join('helloworld.txt').check() == True
def test_build_script_job(default_handler_config,basic_localfs_state):
job = packtivity.syncbackends.build_job(
{
'process_type':'interpolated-script-cmd',
'interpreter':'sh',
'script': 'hello {one} {two}\n echo another line {two}'},
pdm.create({'one':'ONE', 'two':'TWO'}),
basic_localfs_state,
default_handler_config
)
assert 'script' in job
assert 'interpreter' in job
def test_parpub(tmpdir,basic_localfs_state):
pub = {
'publisher_type': 'frompar-pub',
'outputmap': {
'hello': 'mypar'
}
}
pars = pdm.create({
'mypar': 'myvalue'
})
pubbed = handlers['frompar-pub']['default'](pub,pars,basic_localfs_state)
assert pubbed == {'hello':'myvalue'}
def test_interp_pub(tmpdir,basic_localfs_state):
pub = {
'publisher_type': 'interpolated-pub',
'publish': {
'hello': 'hello_{mypar}_world',
},
'relative_paths': False,
'glob': False
}
pars = pdm.create({
'mypar': 'myvalue'
})
pubbed = handlers['interpolated-pub']['default'](pub,pars,basic_localfs_state)
assert pubbed == {'hello': 'hello_myvalue_world'}
def test_glob_pub(tmpdir,basic_localfs_state):
tmpdir.join('hello_1.txt').ensure(file = True)
tmpdir.join('hello_2.txt').ensure(file = True)
pub = {
'publisher_type': 'fromglob-pub',
'outputkey': 'hello',
'globexpression': 'hello_*.txt',
}
pars = pdm.create({
'mypar': 'myvalue'
})
pubbed = handlers['fromglob-pub']['default'](pub,pars,basic_localfs_state)
filelist = map(str,[tmpdir.join('hello_1.txt'),tmpdir.join('hello_2.txt')])
assert set(pubbed['hello']) == set(filelist)
def test_interp_pub_glob_relative(tmpdir,basic_localfs_state):
tmpdir.join('hello_myvalue_1.txt').ensure(file = True)
tmpdir.join('hello_myvalue_2.txt').ensure(file = True)
pub = {
'publisher_type': 'interpolated-pub',
'publish': {
'hello': ['hello_myvalue_2.txt','hello_myvalue_1.txt'],
},
'relative_paths': True,
'glob': False
}
pars = pdm.create({
'mypar': 'myvalue'
})
pubbed = handlers['interpolated-pub']['default'](pub,pars,basic_localfs_state)
filelist = map(str,[tmpdir.join('hello_myvalue_1.txt'),tmpdir.join('hello_myvalue_2.txt')])
assert set(pubbed['hello']) == set(filelist)
def fromJSON(cls, data, deserialization_opts=None):
if data["task"]["type"] == "packtivity_task":
task = packtivity_task.fromJSON(data["task"], deserialization_opts)
result = (
datamodel.create(data["result"], getattr(task.state, "datamodel", {}))
if data["result"]
else None
)
instance = cls(data["name"], task, data["id"], result)
adage.serialize.set_generic_data(instance, data)
instance.resultproxy = (
load_proxy(
data["proxy"], deserialization_opts, best_effort_backend=False
)
if data["proxy"]
else None
)
return instance
else:
raise RuntimeError("unknown task type", data["task"]["type"])
def test_pack_workdir(tmpdir, basic_localfs_state, docker_touchfile_workdir,
default_async):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
with pytest.raises(RuntimeError):
docker_touchfile_workdir(parameters=pars, state=basic_localfs_state)
def test_pack_call_docker_script(tmpdir, basic_localfs_state,
dockeproc_script_pack):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
dockeproc_script_pack(parameters=pars, state=basic_localfs_state)
assert tmpdir.join('helloworld.txt').check()
def __init__(self, name, spec, state, parameters = None, inputs = None):
self.metadata = {'name': name}
self.inputs = inputs or []
self.parameters = datamodel.create(parameters or {}, state.datamodel if state else None)
self.spec = spec
self.state = state
def test_pack_call_local_fail(tmpdir,basic_localfs_state,localproc_pack_fail,default_async):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
with pytest.raises(RuntimeError):
localproc_pack_fail(parameters = pars, state = basic_localfs_state)
assert tmpdir.join('helloworld.txt').check()
def test_pack_call_docker_script(tmpdir,basic_localfs_state,dockeproc_script_pack):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
dockeproc_script_pack(parameters = pars, state = basic_localfs_state)
assert tmpdir.join('helloworld.txt').check()
def test_pack_workdir(tmpdir,basic_localfs_state,docker_touchfile_workdir,default_async):
basic_localfs_state.ensure()
pars = pdm.create({'outputfile': '{workdir}/helloworld.txt'})
with pytest.raises(RuntimeError):
docker_touchfile_workdir(parameters = pars, state = basic_localfs_state)