def _filter_dep_attributes_to_collect(dep_attributes, filter_deps): if not filter_deps: return dep_attributes filter_deps = re.sub(r'[\s,|]+', ' ', filter_deps).strip() if not filter_deps: return dep_attributes else: dep_names = list(dep_attributes) dep_names.extend(['all', 'none']) dep_attributes_out = {} for f_dep in filter_deps.lower().split(): try: full_dep = _match_prefix(dep_names, f_dep) except ValueError as ex: raise InvalidCommand("graph-type %s" % ex.args[0]) else: if not full_dep: msg = "Unsupported dependency-type '%s'; should be one: %s" raise InvalidCommand(msg % (full_dep, sorted(dep_names))) if 'all' == full_dep: return dep_attributes elif 'none' == full_dep: return {} dep_attributes_out[full_dep] = dep_attributes[full_dep] return dep_attributes_out
def _select_graph_func(graph, graph_type): graph_names = sorted(SUPPORTED_GRAPH_TYPES) try: matched_graph_type = _match_prefix(graph_names, graph_type.lower()) except ValueError as ex: raise InvalidCommand("graph-type %s" % ex.args[0]) else: if not matched_graph_type: msg = "Unsupported graph-type '%s'; should be one: %s" raise InvalidCommand(msg % (graph_type, graph_names)) else: func = SUPPORTED_GRAPH_TYPES[matched_graph_type] return matched_graph_type, func
def execute(self, params, args): """execute cmd 'help' """ cmds = self.doit_app.sub_cmds if len(args) == 0 or len(args) > 2: self.print_usage(cmds) elif args[0] == 'task': self.print_task_help() elif args == ['pipeline']: six.print_(cmds['pipeline'].help()) elif args[0] == 'pipeline': cls = PipelineLoader._import(args[1]) print_pipeline_help(cls) elif args[0] in cmds: # help on command six.print_(cmds[args[0]].help()) else: # help of specific task try: if not DoitCmdBase.execute(self, params, args): self.print_usage(cmds) except InvalidDodoFile as e: self.print_usage(cmds) raise InvalidCommand("Unable to retrieve task help: " + e.message) return 0
def _check_task_names(all_task_names, task_names): """repost if task 'task_names' """ # Note: simpler and user-friendlier than cmd_base.check_tasks_exist() if not set(task_names).issubset(all_task_names): bad_tasks = set(task_names) - all_task_names msg = "Task(s) not found: %s" % str(bad_tasks) raise InvalidCommand(msg)
def execute(self, opt_values, pos_args, *args, **kwargs): if not pos_args: raise InvalidCommand("No pipeline specified. Try pipeline -h") pipeline_name = pos_args.pop() self._loader.pipeline_cls = pipeline_name return super(RunPipeline, self).execute(opt_values, pos_args, *args, **kwargs)
def load_tasks(self, cmd, opt_values, pos_args): project_path = os.path.realpath(opt_values["project"]) project = models.Project.from_path(project_path) if not project.exists(): raise InvalidCommand("project %s doesn't exist" %(project_path)) pipeline = self.get_tasks(project) config = pipeline.configure() return pipeline.tasks(), config
def _discover_runner_class(self, num_process, par_type): if num_process == 0: return Runner else: if par_type == 'process': if MRunner.available(): return partial(MRunner, num_process=num_process) else: sys.stderr.write( "WARNING: multiprocessing module not available, " + "running in parallel using threads.") elif par_type == 'thread': return partial(MThreadRunner, num_process=num_process) else: msg = "Invalid parallel type %s" raise InvalidCommand(msg % par_type)
def make_pipeline_skeleton(pipeline_name, optional_pipelines=list(), verbose=True, template=None): log = logger_init(verbose) PipelineClass = PipelineLoader._import(pipeline_name) optpipe_classes = map(PipelineLoader._import, optional_pipelines) here = os.getcwd() input_dir = join(here, "input") options_dir = join(input_dir, OPTIONS_DIR) def _combine(attr): orig = list(getattr(PipelineClass, attr).items()) for p in optpipe_classes: orig += list(getattr(p, attr).items()) return dict(orig) attrs_to_combine = ("products", "default_options", "workflows") allprods, allopts, allworks = map(_combine, attrs_to_combine) opt_import_stmts, append_stmts = str(), str() if optpipe_classes: opt_import_stmts = format_optimports(optpipe_classes) append_stmts = format_optappends(optpipe_classes) if os.path.exists(input_dir): raise InvalidCommand("Input directory already exists: " + input_dir) log("Constructing input skeleton at {}.\n", input_dir) os.mkdir(input_dir) os.mkdir(options_dir) if not template: template = default_template() product_dirs = list() for name, prod in allprods.iteritems(): skel_func = skel_funcs.get(type(prod)) if not skel_func: msg = "Unable to handle products of type {}" raise ValueError(msg.format(skel_func)) skel_dir = join(input_dir, name) log("Creating input directory {} for {}...", skel_dir, str(type(prod))) skel_func(PipelineClass, skel_dir) log("Done.\n") product_dirs.append(skel_dir) for name, opt_dict in allopts.iteritems(): options_fname = join(options_dir, name + ".txt") log("Writing default options for {}.{} into {}...", pipeline_name, name, options_fname) workflow_func = allworks.get(name) write_options(opt_dict, options_fname, workflow_func) log("Done.\n") log("Writing dodo.py file...") dodo_fname = "dodo.py" with open(dodo_fname, 'w') as dodo_f: print >> dodo_f, template.format( pipeline_class=PipelineClass, known_input_directories=pformat(product_dirs), options_dir=repr(options_dir), append_imports=opt_import_stmts, append_statements=append_stmts) log("Done.\n") help_fname = "README.rst" log("Writing help file to {}...", help_fname) with open(help_fname, 'w') as help_f: print_pipeline_help(PipelineClass, optpipe_classes, stream=help_f) log("Done.\n") log("Complete.\n") return allprods, allopts, allworks
def _execute(self, outfile=sys.stdout, verbosity=None, always=False, continue_=False, reporter='default', num_process=0, par_type='process', single=False, pipeline_name="Custom Pipeline"): """ @param reporter: (str) one of provided reporters or ... (class) user defined reporter class (can only be specified from DOIT_CONFIG - never from command line) (reporter instance) - only used in unittests """ # get tasks to be executed # self.control is saved on instance to be used by 'auto' command self.control = TaskControl(self.task_list) self.control.process(self.sel_tasks) if single: for task_name in self.sel_tasks: task = self.control.tasks[task_name] if task.has_subtask: for task_name in task.task_dep: sub_task = self.control.tasks[task_name] sub_task.task_dep = [] else: task.task_dep = [] # reporter if isinstance(reporter, six.string_types): if reporter not in REPORTERS: msg = ("No reporter named '%s'." " Type 'doit help run' to see a list " "of available reporters.") raise InvalidCommand(msg % reporter) reporter_cls = REPORTERS[reporter] else: # user defined class reporter_cls = reporter # verbosity if verbosity is None: use_verbosity = Task.DEFAULT_VERBOSITY else: use_verbosity = verbosity show_out = use_verbosity < 2 # show on error report # outstream if isinstance(outfile, six.string_types): outstream = codecs.open(outfile, 'w', encoding='utf-8') else: # outfile is a file-like object (like StringIO or sys.stdout) outstream = outfile # run try: # FIXME stderr will be shown twice in case of task error/failure if isinstance(reporter_cls, type): reporter_obj = reporter_cls( outstream, { 'show_out': show_out, 'show_err': True, 'reporter_url': self.opt_values['reporter_url'], 'auth_info': self.opt_values['auth_info'] }) else: # also accepts reporter instances reporter_obj = reporter_cls run_args = [ self.dep_class, self.dep_file, reporter_obj, continue_, always, verbosity ] run_kwargs = {} RunnerClass = RUNNER_MAP.get(self.opt_values["runner"]) if not RunnerClass: RunnerClass = self._discover_runner_class( num_process, par_type) elif self.opt_values['runner'] in GRID_RUNNER_MAP: if not self.opt_values.get('partition', None): raise InvalidCommand("--partition option is required " "when using a grid runner") run_args = [ self.opt_values['partition'], self.opt_values['perf_url'], self.opt_values['tmpfiledir'], self.opt_values['grid_args'] ] + run_args run_kwargs['num_process'] = num_process if num_process else 1 runner = RunnerClass(*run_args, **run_kwargs) runner.pipeline_name = pipeline_name return runner.run_all(self.control.task_dispatcher()) finally: if isinstance(outfile, str): outstream.close()
def execute(self, opt_values, pos_args, *args, **kwargs): if not pos_args: raise InvalidCommand("No pipeline specified") pipeline_name = pos_args.pop() make_pipeline_skeleton( pipeline_name, optional_pipelines=opt_values['append_pipeline'])