def _convert_val(self, param_type, val):
if param_type["type"] == "native":
switch = {
"str" : lambda x: str(x),
"list" : lambda x: list(x),
"tuple" : lambda x: tuple(x),
"dict" : lambda x: dict(x),
"int" : lambda x: int(x),
"float" : lambda x: float(x),
"unicode" : lambda x: unicode(x)
}
return switch[param_type["name"]](val)
elif param_type["type"] == "fileset":
val = FileSet(val)
return val
elif param_type["type"] == "component":
# val should be like this:
# { "class": "SpecificComponent", "params": {} }
#
# allow for inheritance by letting the json specify the
# specific class that will be used
component_cls = self._get_component_cls(val["class"])
component_args = self._convert_params(val["params"], component_cls)
val = component_cls(parent_log = self._log, job = self)
val.init(**component_args)
return val
def __init__(self, id, idx, params, tool, fileset_id, progress_callback, results_callback):
"""TODO: to be defined1.
:idx: TODO
:params: TODO
:tool: TODO
:fileset_id: The id of the default fileset that files should be added to
"""
self._id = id
self._idx = idx
self._params = params
self._tool = tool
self._fileset_id = fileset_id
self._fileset = FileSet(self._fileset_id)
self._progress_callback = progress_callback
self._results_callback = results_callback
self._log = logging.getLogger("JOB:{}".format(self._id))
def __init__(self, id, idx, params, tool, fileset_id, progress_callback, results_callback):
"""TODO: to be defined1.
:idx: TODO
:params: TODO
:tool: TODO
:fileset_id: The id of the default fileset that files should be added to
"""
self._id = id
self._idx = idx
self._params = params
self._tool = tool
self._fileset_id = fileset_id
self._fileset = FileSet(self._fileset_id)
self._progress_callback = progress_callback
self._results_callback = results_callback
self._log = logging.getLogger("JOB:{}".format(self._id))
class Job(object):
"""This is the class that will run a task."""
def __init__(self, id, idx, params, tool, fileset_id, progress_callback, results_callback):
"""TODO: to be defined1.
:idx: TODO
:params: TODO
:tool: TODO
:fileset_id: The id of the default fileset that files should be added to
"""
self._id = id
self._idx = idx
self._params = params
self._tool = tool
self._fileset_id = fileset_id
self._fileset = FileSet(self._fileset_id)
self._progress_callback = progress_callback
self._results_callback = results_callback
self._log = logging.getLogger("JOB:{}".format(self._id))
def add_file(self, contents, content_type="application/octet-stream", filename=None, **metadata):
"""Add a file to the default result fileset
"""
return self._fileset.add(
contents=contents,
filename=filename,
content_type=content_type,
**metadata
)
def run(self):
self._log.debug("preparing to run job")
try:
tool_cls = self._get_tool_cls()
real_params = self._convert_params(self._params, tool_cls)
tool = tool_cls(
idx=self._idx,
progress_cb=self._progress_callback,
results_cb=self._results_callback,
parent_log=self._log,
job=self
)
self._log.debug("RUNNING TOOL")
tool.run(**real_params)
except TalusError as e:
self._log.error(e.message)
self._log.debug("FINISHED RUNNING TOOL")
def _camel_to_under(self, name):
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def _get_tool_cls(self):
mod_name = self._camel_to_under(self._tool)
mod = __import__("talus.tools." + mod_name, globals(), locals(), fromlist=[str(self._tool)])
return getattr(mod, self._tool)
def _get_component_cls(self, cls_name):
mod_name = self._camel_to_under(cls_name)
mod_base = __import__("talus.components", globals(), locals(), fromlist=[str(mod_name)])
mod = getattr(mod_base, mod_name)
return getattr(mod, cls_name)
def _convert_params(self, params, code_cls):
filename = inspect.getfile(code_cls)
param_types = self._get_param_types(code_cls)
real_params = {}
for name, val in params.iteritems():
if name not in param_types:
raise TalusError("unmapped argument: {!r}".format(name))
real_params[name] = self._convert_val(param_types[name]["type"], val)
return real_params
def _to_bool(self, val):
if type(val) is bool:
return val
if type(val) is str:
return val.lower() == "true"
return not (not val)
def _convert_val(self, param_type, val):
if param_type["type"] == "native":
switch = {
"str": lambda x: str(x),
"list": lambda x: list(x),
"tuple": lambda x: tuple(x),
"dict": lambda x: dict(x),
"int": lambda x: int(x),
"float": lambda x: float(x),
"unicode": lambda x: unicode(x)
"bool" : self._to_bool,
}
return switch[param_type["name"]](val)
def _convert_val(self, param_type, val):
if param_type["type"] == "native":
switch = {
"str": lambda x: str(x),
"list": lambda x: list(x),
"tuple": lambda x: tuple(x),
"dict": lambda x: dict(x),
"int": lambda x: int(x),
"float": lambda x: float(x),
"unicode": lambda x: unicode(x)
"bool" : self._to_bool,
}
return switch[param_type["name"]](val)
elif param_type["type"] == "fileset":
val = FileSet(val)
return val
elif param_type["type"] == "component":
# val should be like this:
# { "class": "SpecificComponent", "params": {} }
#
# allow for inheritance by letting the json specify the
# specific class that will be used
component_cls = self._get_component_cls(val["class"])
component_args = self._convert_params(val["params"], component_cls)
val = component_cls(parent_log=self._log, job=self)
val.init(**component_args)
return val
class Job(object):
"""This is the class that will run a task."""
def __init__(self, id, idx, params, tool, fileset_id, progress_callback, results_callback):
"""TODO: to be defined1.
:idx: TODO
:params: TODO
:tool: TODO
:fileset_id: The id of the default fileset that files should be added to
"""
self._id = id
self._idx = idx
self._params = params
self._tool = tool
self._fileset_id = fileset_id
self._fileset = FileSet(self._fileset_id)
self._progress_callback = progress_callback
self._results_callback = results_callback
self._log = logging.getLogger("JOB:{}".format(self._id))
def add_file(self, contents, content_type="application/octet-stream", filename=None, **metadata):
"""Add a file to the default result fileset
"""
return self._fileset.add(
contents = contents,
filename = filename,
content_type = content_type,
**metadata
)
def run(self):
self._log.debug("preparing to run job")
try:
tool_cls = self._get_tool_cls()
real_params = self._convert_params(self._params, tool_cls)
tool = tool_cls(
idx = self._idx,
progress_cb = self._progress_callback,
results_cb = self._results_callback,
parent_log = self._log,
job = self
)
self._log.debug("RUNNING TOOL")
tool.run(**real_params)
except TalusError as e:
self._log.error(e.message)
self._log.debug("FINISHED RUNNING TOOL")
def _camel_to_under(self, name):
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def _get_tool_cls(self):
mod_name = self._camel_to_under(self._tool)
mod = __import__("talus.tools." + mod_name, globals(), locals(), fromlist=[str(self._tool)])
return getattr(mod, self._tool)
def _get_component_cls(self, cls_name):
mod_name = self._camel_to_under(cls_name)
mod_base = __import__("talus.components", globals(), locals(), fromlist=[str(mod_name)])
mod = getattr(mod_base, mod_name)
return getattr(mod, cls_name)
def _convert_params(self, params, code_cls):
filename = inspect.getfile(code_cls)
param_types = self._get_param_types(code_cls)
real_params = {}
for name,val in params.iteritems():
if name not in param_types:
raise TalusError("unmapped argument: {!r}".format(name))
real_params[name] = self._convert_val(param_types[name]["type"], val)
return real_params
def _convert_val(self, param_type, val):
if param_type["type"] == "native":
switch = {
"str" : lambda x: str(x),
"list" : lambda x: list(x),
"tuple" : lambda x: tuple(x),
"dict" : lambda x: dict(x),
"int" : lambda x: int(x),
"float" : lambda x: float(x),
"unicode" : lambda x: unicode(x)
}
return switch[param_type["name"]](val)
elif param_type["type"] == "fileset":
val = FileSet(val)
return val
elif param_type["type"] == "component":
# val should be like this:
# { "class": "SpecificComponent", "params": {} }
#
# allow for inheritance by letting the json specify the
# specific class that will be used
component_cls = self._get_component_cls(val["class"])
component_args = self._convert_params(val["params"], component_cls)
val = component_cls(parent_log = self._log, job = self)
val.init(**component_args)
return val
def _get_param_types(self, cls):
cls_name = cls.__name__
filename = inspect.getfile(cls)
with open(filename, "r") as f:
source = f.read()
pyclass = None
mod = ast.parse(source)
for node in mod.body:
if isinstance(node,ast.ClassDef):
cls = PyClass(self._log, filename, node)
if cls.name == cls_name:
pyclass = cls
break
# make the query func always return true - this is assuming the
# pre-receive hook has done its job and prevented invalid components
# from slipping in
return pyclass.get_run_params(lambda x: True)