def test_env_import(tmpdir, monkeypatch):
monkeypatch.setenv(
'PACKTIVITY_ASYNCBACKEND',
'packtivity.asyncbackends:ForegroundBackend:ForegroundProxy')
b = backend_from_string('fromenv')
optfile = tmpdir.join("opt.yml")
optfile.write('{}')
monkeypatch.setenv('PACKTIVITY_ASYNCBACKEND_OPTS', str(optfile))
b = backend_from_string('fromenv')
assert b
def test_known():
for known_backend in [
'celery', 'multiproc:4', 'multiproc:auto', 'foregroundasync',
'externalasync:default'
]:
b = backend_from_string(known_backend)
assert b
def pubtest(
spec,
parfiles,
state,
parameter,
read,
write,
toplevel,
schemasource,
validate,
verbosity,
backend,
):
logging.basicConfig(level=getattr(logging, verbosity))
spec = utils.load_packtivity(spec, toplevel, schemasource, validate)
state = yaml.safe_load(open(state)) if state else {}
if not state:
state.setdefault("readwrite", []).extend(map(os.path.realpath, write))
state.setdefault("readonly", []).extend(map(os.path.realpath, read))
state = LocalFSState(state["readwrite"], state["readonly"])
parameters = getinit_data(parfiles, parameter)
is_sync, backend = bkutils.backend_from_string(backend)
publish = backend.prepublish(spec, parameters, state)
click.echo(str(publish))
def runcli(
spec,
parfiles,
state,
parameter,
read,
write,
toplevel,
schemasource,
asyncwait,
validate,
verbosity,
backend,
proxyfile,
outfile,
):
logging.basicConfig(level=getattr(logging, verbosity))
spec = utils.load_packtivity(spec, toplevel, schemasource, validate)
parameters = getinit_data(parfiles, parameter)
state = yaml.safe_load(open(state)) if state else {}
if not state:
state.setdefault("readwrite", []).extend(map(os.path.realpath, write))
state.setdefault("readonly", []).extend(map(os.path.realpath, read))
state = LocalFSState(state["readwrite"], state["readonly"])
state.ensure()
is_sync, backend = bkutils.backend_from_string(backend)
backend_kwargs = ({
"syncbackend": backend
} if is_sync else {
"asyncbackend": backend,
"asyncwait": asyncwait
})
prepub = backend.prepublish(spec, parameters, state)
if prepub:
click.echo(str(prepub) + (" (prepublished)"))
pack = packtivity.pack_object(spec)
result = pack(parameters, state, **backend_kwargs)
if not is_sync and not asyncwait:
click.secho("proxy-json {}".format(json.dumps(result.json())))
with open(proxyfile, "w") as p:
json.dump(result.json(), p)
else:
click.echo(str(result) + (" (post-run)" if prepub else ""))
if outfile:
with open(outfile, "w") as out:
out.write(json.dumps(result.json()))
def pubtest(spec,parfiles,state,parameter,read,write,toplevel,schemasource,validate,verbosity,backend):
logging.basicConfig(level = getattr(logging,verbosity))
spec = utils.load_packtivity(spec,toplevel,schemasource,validate)
state = yaml.load(open(state)) if state else {}
if not state:
state.setdefault('readwrite',[]).extend(map(os.path.realpath,write))
state.setdefault('readonly',[]).extend(map(os.path.realpath,read))
state = LocalFSState(state['readwrite'],state['readonly'])
parameters = getinit_data(parfiles,parameter)
is_sync, backend = bkutils.backend_from_string(backend)
publish = backend.prepublish(spec,parameters,state)
click.echo(str(publish))
def __init__(self, packtivity_backendstring = None, packtivity_backend = None, backendopts = None):
purepubopts = backendopts.pop('purepubopts',{})
if packtivity_backendstring:
is_sync, backend = backend_from_string(packtivity_backendstring, backendopts = backendopts)
assert not is_sync
elif packtivity_backend:
backend = packtivity_backend
else:
raise RuntimeError('need backend or backendstring')
self.cached = False
super(PacktivityBackend, self).__init__({
'purepub': ForegroundBackend(purepubopts),
'packtivity': backend
})
def __init__(self,
packtivity_backendstring=None,
packtivity_backend=None,
backendopts=None):
purepubopts = backendopts.pop("purepubopts", {})
if packtivity_backendstring:
is_sync, backend = backend_from_string(packtivity_backendstring,
backendopts=backendopts)
assert not is_sync
elif packtivity_backend:
backend = packtivity_backend
else:
raise RuntimeError("need backend or backendstring")
self.cached = False
super(PacktivityBackend, self).__init__({
"purepub":
ForegroundBackend(purepubopts),
"packtivity":
backend
})
def runcli(spec,parfiles,state,parameter,read,write,toplevel,schemasource,asyncwait,validate,verbosity,backend,proxyfile,outfile):
logging.basicConfig(level = getattr(logging,verbosity))
spec = utils.load_packtivity(spec,toplevel,schemasource,validate)
parameters = getinit_data(parfiles,parameter)
state = yaml.load(open(state)) if state else {}
if not state:
state.setdefault('readwrite',[]).extend(map(os.path.realpath,write))
state.setdefault('readonly',[]).extend(map(os.path.realpath,read))
state = LocalFSState(state['readwrite'],state['readonly'])
state.ensure()
is_sync, backend = bkutils.backend_from_string(backend)
backend_kwargs = {
'syncbackend': backend
} if is_sync else {
'asyncbackend':backend,
'asyncwait': asyncwait
}
prepub = backend.prepublish(spec,parameters,state)
if prepub:
click.echo(str(prepub)+(' (prepublished)'))
pack = packtivity.pack_object(spec)
result = pack(parameters,state,**backend_kwargs)
if not is_sync and not asyncwait:
click.secho('proxy-json {}'.format(json.dumps(result.json())))
with open(proxyfile,'w') as p:
json.dump(result.json(),p)
else:
click.echo(str(result)+(' (post-run)' if prepub else ''))
if outfile:
with open(outfile,'w') as out:
out.write(json.dumps(result.json()))
def test_unknown():
with pytest.raises(RuntimeError):
backend_from_string('doesnotexist')
def test_python_import():
b = backend_from_string('py:packtivity.asyncbackends:MultiProcBackend',
{'poolsize': 1})
assert b