def test_env_import(tmpdir, monkeypatch): monkeypatch.setenv( 'PACKTIVITY_ASYNCBACKEND', 'packtivity.asyncbackends:ForegroundBackend:ForegroundProxy') b = backend_from_string('fromenv') optfile = tmpdir.join("opt.yml") optfile.write('{}') monkeypatch.setenv('PACKTIVITY_ASYNCBACKEND_OPTS', str(optfile)) b = backend_from_string('fromenv') assert b
def test_known(): for known_backend in [ 'celery', 'multiproc:4', 'multiproc:auto', 'foregroundasync', 'externalasync:default' ]: b = backend_from_string(known_backend) assert b
def pubtest( spec, parfiles, state, parameter, read, write, toplevel, schemasource, validate, verbosity, backend, ): logging.basicConfig(level=getattr(logging, verbosity)) spec = utils.load_packtivity(spec, toplevel, schemasource, validate) state = yaml.safe_load(open(state)) if state else {} if not state: state.setdefault("readwrite", []).extend(map(os.path.realpath, write)) state.setdefault("readonly", []).extend(map(os.path.realpath, read)) state = LocalFSState(state["readwrite"], state["readonly"]) parameters = getinit_data(parfiles, parameter) is_sync, backend = bkutils.backend_from_string(backend) publish = backend.prepublish(spec, parameters, state) click.echo(str(publish))
def runcli( spec, parfiles, state, parameter, read, write, toplevel, schemasource, asyncwait, validate, verbosity, backend, proxyfile, outfile, ): logging.basicConfig(level=getattr(logging, verbosity)) spec = utils.load_packtivity(spec, toplevel, schemasource, validate) parameters = getinit_data(parfiles, parameter) state = yaml.safe_load(open(state)) if state else {} if not state: state.setdefault("readwrite", []).extend(map(os.path.realpath, write)) state.setdefault("readonly", []).extend(map(os.path.realpath, read)) state = LocalFSState(state["readwrite"], state["readonly"]) state.ensure() is_sync, backend = bkutils.backend_from_string(backend) backend_kwargs = ({ "syncbackend": backend } if is_sync else { "asyncbackend": backend, "asyncwait": asyncwait }) prepub = backend.prepublish(spec, parameters, state) if prepub: click.echo(str(prepub) + (" (prepublished)")) pack = packtivity.pack_object(spec) result = pack(parameters, state, **backend_kwargs) if not is_sync and not asyncwait: click.secho("proxy-json {}".format(json.dumps(result.json()))) with open(proxyfile, "w") as p: json.dump(result.json(), p) else: click.echo(str(result) + (" (post-run)" if prepub else "")) if outfile: with open(outfile, "w") as out: out.write(json.dumps(result.json()))
def pubtest(spec,parfiles,state,parameter,read,write,toplevel,schemasource,validate,verbosity,backend): logging.basicConfig(level = getattr(logging,verbosity)) spec = utils.load_packtivity(spec,toplevel,schemasource,validate) state = yaml.load(open(state)) if state else {} if not state: state.setdefault('readwrite',[]).extend(map(os.path.realpath,write)) state.setdefault('readonly',[]).extend(map(os.path.realpath,read)) state = LocalFSState(state['readwrite'],state['readonly']) parameters = getinit_data(parfiles,parameter) is_sync, backend = bkutils.backend_from_string(backend) publish = backend.prepublish(spec,parameters,state) click.echo(str(publish))
def __init__(self, packtivity_backendstring = None, packtivity_backend = None, backendopts = None): purepubopts = backendopts.pop('purepubopts',{}) if packtivity_backendstring: is_sync, backend = backend_from_string(packtivity_backendstring, backendopts = backendopts) assert not is_sync elif packtivity_backend: backend = packtivity_backend else: raise RuntimeError('need backend or backendstring') self.cached = False super(PacktivityBackend, self).__init__({ 'purepub': ForegroundBackend(purepubopts), 'packtivity': backend })
def __init__(self, packtivity_backendstring=None, packtivity_backend=None, backendopts=None): purepubopts = backendopts.pop("purepubopts", {}) if packtivity_backendstring: is_sync, backend = backend_from_string(packtivity_backendstring, backendopts=backendopts) assert not is_sync elif packtivity_backend: backend = packtivity_backend else: raise RuntimeError("need backend or backendstring") self.cached = False super(PacktivityBackend, self).__init__({ "purepub": ForegroundBackend(purepubopts), "packtivity": backend })
def runcli(spec,parfiles,state,parameter,read,write,toplevel,schemasource,asyncwait,validate,verbosity,backend,proxyfile,outfile): logging.basicConfig(level = getattr(logging,verbosity)) spec = utils.load_packtivity(spec,toplevel,schemasource,validate) parameters = getinit_data(parfiles,parameter) state = yaml.load(open(state)) if state else {} if not state: state.setdefault('readwrite',[]).extend(map(os.path.realpath,write)) state.setdefault('readonly',[]).extend(map(os.path.realpath,read)) state = LocalFSState(state['readwrite'],state['readonly']) state.ensure() is_sync, backend = bkutils.backend_from_string(backend) backend_kwargs = { 'syncbackend': backend } if is_sync else { 'asyncbackend':backend, 'asyncwait': asyncwait } prepub = backend.prepublish(spec,parameters,state) if prepub: click.echo(str(prepub)+(' (prepublished)')) pack = packtivity.pack_object(spec) result = pack(parameters,state,**backend_kwargs) if not is_sync and not asyncwait: click.secho('proxy-json {}'.format(json.dumps(result.json()))) with open(proxyfile,'w') as p: json.dump(result.json(),p) else: click.echo(str(result)+(' (post-run)' if prepub else '')) if outfile: with open(outfile,'w') as out: out.write(json.dumps(result.json()))
def test_unknown(): with pytest.raises(RuntimeError): backend_from_string('doesnotexist')
def test_python_import(): b = backend_from_string('py:packtivity.asyncbackends:MultiProcBackend', {'poolsize': 1}) assert b