def run(self, task, job, **jobargs):
# Entry point into the executing pipeline worker task. This
# initializes the task environment, sets up the current stage,
# and then executes it.
worker.active_task = task
for key in self:
self[key] = self.getitem(key, job, jobargs)
sys_version = '{0[0]}.{0[1]}'.format(sys.version_info[:2])
assert self['version'] == sys_version, "Python version mismatch"
# Set up the task environment.
globals_ = globals().copy()
for module in self['required_modules']:
name = module[0] if util.iskv(module) else module
globals_[name.split('.')[-1]] = __import__(name, fromlist=[name])
for obj in util.flatten(self.values()):
util.globalize(obj, globals_)
# Set up the stage.
params = self.getitem('params', job, jobargs, worker.Params())
pipeline = dict([(s.name, (idx, s))
for idx, (g, s) in enumerate(self['pipeline'])])
pipe_idx, stage = pipeline[task.stage]
stage.taskinfo = TaskInfo(jobname=task.jobname,
host=task.host,
stage=task.stage,
group=task.group,
label=task.group_label)
if not stage.input_chain:
stage.input_chain = Stage.default_input_chain(pipe_idx)
if not stage.output_chain:
stage.output_chain = Stage.default_output_chain
# And now run it.
self.run_stage(task, stage, params)
def run(self, task, job, **jobargs):
# Entry point into the executing pipeline worker task. This
# initializes the task environment, sets up the current stage,
# and then executes it.
for key in self:
self[key] = self.getitem(key, job, jobargs)
sys_version = '{0[0]}.{0[1]}'.format(sys.version_info[:2])
assert self['version'] == sys_version, "Python version mismatch"
# Set up the task environment.
globals_ = globals().copy()
for module in self['required_modules']:
name = module[0] if util.iskv(module) else module
globals_[name.split('.')[-1]] = __import__(name, fromlist=[name])
for obj in util.flatten(self.values()):
util.globalize(obj, globals_)
# Set up the stage.
params = self.getitem('params', job, jobargs, worker.Params())
pipeline = dict([(s.name, (idx, s))
for idx, (g, s) in enumerate(self['pipeline'])])
pipe_idx, stage = pipeline[task.stage]
stage.taskinfo = TaskInfo(jobname=task.jobname, host=task.host,
stage=task.stage, group=task.group,
label=task.group_label)
if not stage.input_chain:
stage.input_chain = Stage.default_input_chain(pipe_idx)
if not stage.output_chain:
stage.output_chain = Stage.default_output_chain
# And now run it.
self.run_stage(task, stage, params)
def run(self, task, job, **jobargs):
global Task
Task = task
for key in self:
self[key] = self.getitem(key, job, jobargs)
assert self['version'] == '%s.%s' % sys.version_info[:2], "Python version mismatch"
params = self['params']
if isinstance(self[task.mode], dict):
params = self['ext_params']
self[task.mode] = external.prepare(params, task.mode)
globals_ = globals().copy()
for module in self['required_modules']:
name = module[0] if util.iskv(module) else module
globals_[name.split('.')[-1]] = __import__(name, fromlist=[name])
for obj in util.flatten(self.values()):
util.globalize(obj, globals_)
getattr(self, task.mode)(task, params)
external.close()
def run(self, task, job, **jobargs):
global Task
Task = task
for key in self:
self[key] = self.getitem(key, job, jobargs)
assert self['version'] == '{0[0]}.{0[1]}'.format(sys.version_info[:2]), "Python version mismatch"
params = self['params']
if isinstance(self[task.stage], dict):
params = self['ext_params']
self[task.stage] = external.prepare(params, task.stage)
globals_ = globals().copy()
for module in self['required_modules']:
name = module[0] if util.iskv(module) else module
globals_[name.split('.')[-1]] = __import__(name, fromlist=[name])
for obj in util.flatten(self.values()):
util.globalize(obj, globals_)
getattr(self, task.stage)(task, params)
external.close()
def input_stream(stream, size, url, params, globals=globals()):
input_stream = import_scheme(url).input_stream
util.globalize(input_stream, globals)
return input_stream(stream, size, url, params)
def input_stream(stream, size, url, params, globals=globals()):
input_stream = import_scheme(url).input_stream
util.globalize(input_stream, globals)
return input_stream(stream, size, url, params)