def _filter_dep_attributes_to_collect(dep_attributes, filter_deps):
if not filter_deps:
return dep_attributes
filter_deps = re.sub(r'[\s,|]+', ' ', filter_deps).strip()
if not filter_deps:
return dep_attributes
else:
dep_names = list(dep_attributes)
dep_names.extend(['all', 'none'])
dep_attributes_out = {}
for f_dep in filter_deps.lower().split():
try:
full_dep = _match_prefix(dep_names, f_dep)
except ValueError as ex:
raise InvalidCommand("graph-type %s" % ex.args[0])
else:
if not full_dep:
msg = "Unsupported dependency-type '%s'; should be one: %s"
raise InvalidCommand(msg %
(full_dep, sorted(dep_names)))
if 'all' == full_dep:
return dep_attributes
elif 'none' == full_dep:
return {}
dep_attributes_out[full_dep] = dep_attributes[full_dep]
return dep_attributes_out
def _select_graph_func(graph, graph_type):
graph_names = sorted(SUPPORTED_GRAPH_TYPES)
try:
matched_graph_type = _match_prefix(graph_names, graph_type.lower())
except ValueError as ex:
raise InvalidCommand("graph-type %s" % ex.args[0])
else:
if not matched_graph_type:
msg = "Unsupported graph-type '%s'; should be one: %s"
raise InvalidCommand(msg % (graph_type, graph_names))
else:
func = SUPPORTED_GRAPH_TYPES[matched_graph_type]
return matched_graph_type, func
def execute(self, params, args):
"""execute cmd 'help' """
cmds = self.doit_app.sub_cmds
if len(args) == 0 or len(args) > 2:
self.print_usage(cmds)
elif args[0] == 'task':
self.print_task_help()
elif args == ['pipeline']:
six.print_(cmds['pipeline'].help())
elif args[0] == 'pipeline':
cls = PipelineLoader._import(args[1])
print_pipeline_help(cls)
elif args[0] in cmds:
# help on command
six.print_(cmds[args[0]].help())
else:
# help of specific task
try:
if not DoitCmdBase.execute(self, params, args):
self.print_usage(cmds)
except InvalidDodoFile as e:
self.print_usage(cmds)
raise InvalidCommand("Unable to retrieve task help: " +
e.message)
return 0
def _check_task_names(all_task_names, task_names):
"""repost if task 'task_names' """
# Note: simpler and user-friendlier than cmd_base.check_tasks_exist()
if not set(task_names).issubset(all_task_names):
bad_tasks = set(task_names) - all_task_names
msg = "Task(s) not found: %s" % str(bad_tasks)
raise InvalidCommand(msg)
def execute(self, opt_values, pos_args, *args, **kwargs):
if not pos_args:
raise InvalidCommand("No pipeline specified. Try pipeline -h")
pipeline_name = pos_args.pop()
self._loader.pipeline_cls = pipeline_name
return super(RunPipeline, self).execute(opt_values, pos_args, *args,
**kwargs)
def load_tasks(self, cmd, opt_values, pos_args):
project_path = os.path.realpath(opt_values["project"])
project = models.Project.from_path(project_path)
if not project.exists():
raise InvalidCommand("project %s doesn't exist" %(project_path))
pipeline = self.get_tasks(project)
config = pipeline.configure()
return pipeline.tasks(), config
def _discover_runner_class(self, num_process, par_type):
if num_process == 0:
return Runner
else:
if par_type == 'process':
if MRunner.available():
return partial(MRunner, num_process=num_process)
else:
sys.stderr.write(
"WARNING: multiprocessing module not available, " +
"running in parallel using threads.")
elif par_type == 'thread':
return partial(MThreadRunner, num_process=num_process)
else:
msg = "Invalid parallel type %s"
raise InvalidCommand(msg % par_type)
def make_pipeline_skeleton(pipeline_name,
optional_pipelines=list(),
verbose=True,
template=None):
log = logger_init(verbose)
PipelineClass = PipelineLoader._import(pipeline_name)
optpipe_classes = map(PipelineLoader._import, optional_pipelines)
here = os.getcwd()
input_dir = join(here, "input")
options_dir = join(input_dir, OPTIONS_DIR)
def _combine(attr):
orig = list(getattr(PipelineClass, attr).items())
for p in optpipe_classes:
orig += list(getattr(p, attr).items())
return dict(orig)
attrs_to_combine = ("products", "default_options", "workflows")
allprods, allopts, allworks = map(_combine, attrs_to_combine)
opt_import_stmts, append_stmts = str(), str()
if optpipe_classes:
opt_import_stmts = format_optimports(optpipe_classes)
append_stmts = format_optappends(optpipe_classes)
if os.path.exists(input_dir):
raise InvalidCommand("Input directory already exists: " + input_dir)
log("Constructing input skeleton at {}.\n", input_dir)
os.mkdir(input_dir)
os.mkdir(options_dir)
if not template:
template = default_template()
product_dirs = list()
for name, prod in allprods.iteritems():
skel_func = skel_funcs.get(type(prod))
if not skel_func:
msg = "Unable to handle products of type {}"
raise ValueError(msg.format(skel_func))
skel_dir = join(input_dir, name)
log("Creating input directory {} for {}...", skel_dir, str(type(prod)))
skel_func(PipelineClass, skel_dir)
log("Done.\n")
product_dirs.append(skel_dir)
for name, opt_dict in allopts.iteritems():
options_fname = join(options_dir, name + ".txt")
log("Writing default options for {}.{} into {}...", pipeline_name,
name, options_fname)
workflow_func = allworks.get(name)
write_options(opt_dict, options_fname, workflow_func)
log("Done.\n")
log("Writing dodo.py file...")
dodo_fname = "dodo.py"
with open(dodo_fname, 'w') as dodo_f:
print >> dodo_f, template.format(
pipeline_class=PipelineClass,
known_input_directories=pformat(product_dirs),
options_dir=repr(options_dir),
append_imports=opt_import_stmts,
append_statements=append_stmts)
log("Done.\n")
help_fname = "README.rst"
log("Writing help file to {}...", help_fname)
with open(help_fname, 'w') as help_f:
print_pipeline_help(PipelineClass, optpipe_classes, stream=help_f)
log("Done.\n")
log("Complete.\n")
return allprods, allopts, allworks
def _execute(self,
outfile=sys.stdout,
verbosity=None,
always=False,
continue_=False,
reporter='default',
num_process=0,
par_type='process',
single=False,
pipeline_name="Custom Pipeline"):
"""
@param reporter: (str) one of provided reporters or ...
(class) user defined reporter class
(can only be specified
from DOIT_CONFIG - never from command line)
(reporter instance) - only used in unittests
"""
# get tasks to be executed
# self.control is saved on instance to be used by 'auto' command
self.control = TaskControl(self.task_list)
self.control.process(self.sel_tasks)
if single:
for task_name in self.sel_tasks:
task = self.control.tasks[task_name]
if task.has_subtask:
for task_name in task.task_dep:
sub_task = self.control.tasks[task_name]
sub_task.task_dep = []
else:
task.task_dep = []
# reporter
if isinstance(reporter, six.string_types):
if reporter not in REPORTERS:
msg = ("No reporter named '%s'."
" Type 'doit help run' to see a list "
"of available reporters.")
raise InvalidCommand(msg % reporter)
reporter_cls = REPORTERS[reporter]
else:
# user defined class
reporter_cls = reporter
# verbosity
if verbosity is None:
use_verbosity = Task.DEFAULT_VERBOSITY
else:
use_verbosity = verbosity
show_out = use_verbosity < 2 # show on error report
# outstream
if isinstance(outfile, six.string_types):
outstream = codecs.open(outfile, 'w', encoding='utf-8')
else: # outfile is a file-like object (like StringIO or sys.stdout)
outstream = outfile
# run
try:
# FIXME stderr will be shown twice in case of task error/failure
if isinstance(reporter_cls, type):
reporter_obj = reporter_cls(
outstream, {
'show_out': show_out,
'show_err': True,
'reporter_url': self.opt_values['reporter_url'],
'auth_info': self.opt_values['auth_info']
})
else: # also accepts reporter instances
reporter_obj = reporter_cls
run_args = [
self.dep_class, self.dep_file, reporter_obj, continue_, always,
verbosity
]
run_kwargs = {}
RunnerClass = RUNNER_MAP.get(self.opt_values["runner"])
if not RunnerClass:
RunnerClass = self._discover_runner_class(
num_process, par_type)
elif self.opt_values['runner'] in GRID_RUNNER_MAP:
if not self.opt_values.get('partition', None):
raise InvalidCommand("--partition option is required "
"when using a grid runner")
run_args = [
self.opt_values['partition'], self.opt_values['perf_url'],
self.opt_values['tmpfiledir'], self.opt_values['grid_args']
] + run_args
run_kwargs['num_process'] = num_process if num_process else 1
runner = RunnerClass(*run_args, **run_kwargs)
runner.pipeline_name = pipeline_name
return runner.run_all(self.control.task_dispatcher())
finally:
if isinstance(outfile, str):
outstream.close()
def execute(self, opt_values, pos_args, *args, **kwargs):
if not pos_args:
raise InvalidCommand("No pipeline specified")
pipeline_name = pos_args.pop()
make_pipeline_skeleton(
pipeline_name, optional_pipelines=opt_values['append_pipeline'])