import yaml
import packtivity.utils as utils
import glob2
import json
import click
import copy
import logging
log = logging.getLogger(__name__)
handlers, publisher = utils.handler_decorator()
@publisher('frompar-pub')
def process_attr_pub_handler(publisher, attributes, context):
outputs = copy.deepcopy(publisher['outputmap'])
for path, value in utils.leaf_iterator(publisher['outputmap']):
actualval = attributes[value]
path.set(outputs, actualval)
return outputs
@publisher('interpolated-pub')
def interpolated_pub_handler(publisher, attributes, context):
forinterp = attributes.copy()
forinterp.update(workdir=context['readwrite'][0])
result = copy.deepcopy(publisher['publish'])
for path, value in utils.leaf_iterator(publisher['publish']):
resultval = value.format(**forinterp)
resultval = glob2.glob(resultval)
path.set(result, resultval)
return result
import os
import subprocess
import sys
import time
import psutil
import shlex
import pipes
import click
import yaml
import json
import packtivity.utils as utils
import packtivity.logutils as logutils
handlers, executor = utils.handler_decorator()
def sourcepath(config, path):
workdir_location = config.container_config.workdir_location()
if workdir_location:
old, new = workdir_location.split(":")
dockerpath = new + path.rsplit(old, 1)[1]
return dockerpath
else:
return path
def state_context_to_mounts(config, state):
readwrites = state.readwrite
readonlies = state.readonly
import packtivity.utils as utils
import click
import yaml
handlers,process = utils.handler_decorator()
@process('string-interpolated-cmd')
def stringinterp_handler(process_spec,attributes):
flattened = {k:v if not (type(v)==list) else ' '.join([str(x) for x in v]) for k,v in attributes.iteritems()}
return {
'command':process_spec['cmd'].format(**flattened)
}
@process('interpolated-script-cmd')
def interp_script(process_spec,attributes):
flattened = {k:v if not (type(v)==list) else ' '.join([str(x) for x in v]) for k,v in attributes.iteritems()}
return {
'script':process_spec['script'].format(**flattened),
'interpreter':process_spec['interpreter']
}
@process('manual-instructions-proc')
def manual_proc(process_spec,attributes):
instructions = process_spec['instructions']
attrs = yaml.safe_dump(attributes,default_flow_style = False)
click.secho(instructions, fg = 'blue')
click.secho(attrs, fg = 'cyan')
@process('grid-transform')
def grid_transform(process_spec,attributes):
import grid_handlers
return grid_handlers.build_grid_job(process_spec,attributes)
import packtivity.utils as utils
import jq
import copy
handlers, environment = utils.handler_decorator()
@environment("docker-encapsulated")
def docker(environment, parameters, state):
environment = copy.deepcopy(environment)
jsonpars = parameters.json()
for p, v in parameters.leafs():
if p.path == "":
jsonpars = v
break
p.set(jsonpars, v)
for i, x in enumerate(environment["par_mounts"]):
script = x.pop("jqscript")
x["mountcontent"] = jq.jq(script).transform(jsonpars, text_output=True)
if environment["workdir"] is not None:
environment["workdir"] = state.contextualize_value(
environment["workdir"])
return environment
@environment("default")
def default(environment, parameters, state):
return environment
import os
import importlib
import yaml
import logging
import packtivity.asyncbackends as asyncbackends
import packtivity.syncbackends as syncbackends
import packtivity.utils as utils
log = logging.getLogger(__name__)
proxyhandlers, proxyloader = utils.handler_decorator()
@proxyloader('ForegroundProxy')
def foreground_loader(jsondata, deserialization_opts = None, best_effort_backend = False):
from .asyncbackends import ForegroundProxy
proxy = ForegroundProxy.fromJSON(jsondata)
if best_effort_backend:
_, backend = backend_from_string('foregroundasync')
return proxy, backend
return proxy
@proxyloader('ExternalAsyncProxy')
def external_loader(jsondata, deserialization_opts = None, best_effort_backend = False):
from .asyncbackends import ExternalAsyncProxy
proxy = ExternalAsyncProxy.fromJSON(jsondata)
if best_effort_backend:
_, backend = backend_from_string('externalasync')
return proxy, backend
return proxy
import click
import yaml
import packtivity.utils as utils
handlers,process = utils.handler_decorator()
@process('string-interpolated-cmd')
def stringinterp_handler(process_spec,parameters, state):
if isinstance(parameters.typed(),dict):
flattened_kwargs = {k:v if not (type(v)==list) else ' '.join([str(x) for x in v]) for k,v in parameters.typed().items()}
command = process_spec['cmd'].format(**flattened_kwargs)
elif isinstance(parameters.typed(),list):
flattened_args = [v if not (type(v)==list) else ' '.join([str(x) for x in v]) for v in parameters.typed()]
command = process_spec['cmd'].format(*flattened_args)
else:
command = process_spec['cmd'].format(value = parameters.typed())
return {'command': command}
@process('interpolated-script-cmd')
def interp_script(process_spec,parameters, state):
if isinstance(parameters.typed(),dict):
flattened_kwargs = {k:v if not (type(v)==list) else ' '.join([str(x) for x in v]) for k,v in parameters.typed().items()}
script = process_spec['script'].format(**flattened_kwargs)
elif isinstance(parameters.typed(),list):
flattened_args = [v if not (type(v)==list) else ' '.join([str(x) for x in v]) for v in parameters.typed()]
script = process_spec['script'].format(*flattened_args)
else:
script = process_spec['script'].format(value = parameters.typed())
return {'script':script, 'interpreter':process_spec['interpreter']}
import os
import importlib
import yaml
import logging
import packtivity.asyncbackends as asyncbackends
import packtivity.syncbackends as syncbackends
import packtivity.utils as utils
log = logging.getLogger(__name__)
proxyhandlers, proxyloader = utils.handler_decorator()
@proxyloader("ForegroundProxy")
def foreground_loader(jsondata,
deserialization_opts=None,
best_effort_backend=False):
from .asyncbackends import ForegroundProxy
proxy = ForegroundProxy.fromJSON(jsondata)
if best_effort_backend:
_, backend = backend_from_string("foregroundasync")
return proxy, backend
return proxy
@proxyloader("ExternalAsyncProxy")
def external_loader(jsondata,
deserialization_opts=None,
best_effort_backend=False):
import packtivity.utils as utils
import jq
import copy
handlers,environment = utils.handler_decorator()
@environment('docker-encapsulated')
def docker(environment,parameters,state):
environment = copy.deepcopy(environment)
jsonpars = parameters.json()
for p,v in parameters.leafs():
if p.path == '':
jsonpars = v
break
p.set(jsonpars, v)
for i,x in enumerate(environment['par_mounts']):
script = x.pop('jqscript')
x['mountcontent'] = jq.jq(script).transform(jsonpars, text_output = True)
if environment['workdir'] is not None:
environment['workdir'] = state.contextualize_value(environment['workdir'])
return environment
@environment('default')
def default(environment,parameters,state):
return environment