def doit(line): """ Run *doit* with `task_creators` from all interactive variables (IPython's global namespace). Examples: >>> %doit --help ## Show help for options and arguments. >>> def task_foo(): return {'actions': ['echo hi IPython'], 'verbosity': 2} >>> %doit list ## List any tasks discovered. foo >>> %doit ## Run any tasks. . foo hi IPython """ # Override db-files location inside ipython-profile dir, # which is certainly writable. prof_dir = ip.profile_dir.location opt_vals = {'dep_file': os.path.join(prof_dir, 'db', '.doit.db')} commander = DoitMain(ModuleTaskLoader(ip.user_module), extra_config={'GLOBAL': opt_vals}) commander.BIN_NAME = 'doit' commander.run(line.split())
def start(self): """Begin executing tasks.""" if self.log_filename: print('Output will be logged to `%s`.' % self.log_filename) start_time = time.strftime(self.timestamp_fmt) print('Started %s' % start_time) if self.log_filename: orig_stdout = sys.stdout orig_stderr = sys.stderr sys.stdout = self.log_file sys.stderr = self.log_file print('Started %s' % start_time) doit_main = DoitMain(self) doit_main.run(['run']) stop_time = time.strftime(self.timestamp_fmt) if self.log_filename: print('Stopped %s' % stop_time) print() sys.stdout = orig_stdout sys.stderr = orig_stderr self.log_file.close() print('Stopped %s' % stop_time)
def run(self, doit_args=None, verbose=True): '''Run the pipeline. Movees to the directory, loads the tasks into doit, and executes that tasks that are not up-to-date. Args: doit_args (list): Args that would be passed to the doit shell command. By default, just run. verbose (bool): If True, print UI stuff. Returns: int: Exit status of the doit command. ''' if verbose: print(ui.header('Run Tasks', level=4)) if doit_args is None: doit_args = ['run'] if self.n_threads > 1: doit_args.extend(['-n', str(self.n_threads)]) runner = DoitMain(self) with Move(self.directory): if self.profile is True: profile_fn = path.join(self.directory, 'profile.csv') with StartProfiler(filename=profile_fn): return runner.run(doit_args) else: return runner.run(doit_args)
def doit(line): """ Run *doit* with `task_creators` from all interactive variables (IPython's global namespace). Examples: >>> %doit --help ## Show help for options and arguments. >>> def task_foo(): return {'actions': ['echo hi IPython'], 'verbosity': 2} >>> %doit list ## List any tasks discovered. foo >>> %doit ## Run any tasks. . foo hi IPython """ ip = get_ipython() # Override db-files location inside ipython-profile dir, # which is certainly writable. prof_dir = ip.profile_dir.location opt_vals = {'dep_file': os.path.join(prof_dir, 'db', '.doit.db')} commander = DoitMain(ModuleTaskLoader(ip.user_module), extra_config={'GLOBAL': opt_vals}) commander.run(line.split())
def main(): configure_test() load_exps() doit_main = DoitMain(CustomTaskLoader()) return doit_main.run(['run'])
def main(argv=None): myname = path.basename(sys.argv[0]) # @UnusedVariable mydir = path.dirname(__file__) # @UnusedVariable if argv is None: argv = sys.argv[1:] opt_vals = {} # 'dep_file': path.abspath(path.join(mydir, '.doit.db'))} commander = DoitMain(ModuleTaskLoader(pndlcmd), extra_config={'GLOBAL': opt_vals}) commander.run(argv)
def test_help_plugin_name(self, capsys): plugin = {'XXX': 'tests.sample_plugin:MyCmd'} main = DoitMain(extra_config={'COMMAND': plugin}) main.BIN_NAME = 'doit' returned = main.run(["help"]) assert returned == 0 out, err = capsys.readouterr() assert "doit XXX " in out assert "test extending doit commands" in out, out
def test_help_plugin_name(self, capsys): plugin = {'XXX': 'tests.sample_plugin:MyCmd'} main = DoitMain(extra_config={'COMMAND':plugin}) main.BIN_NAME = 'doit' returned = main.run(["help"]) assert returned == 0 out, err = capsys.readouterr() assert "doit XXX " in out assert "test extending doit commands" in out, out
def run(self, cmds): tasks = {} for v in self.loaders: for name, l in v.list_tasks(): f = l tasks[name] = f ml = ModuleTaskLoader(tasks) main = DoitMain(ml) main.config['default_tasks'] = cmds return main.run([])
def run(self, doit_args=None, move=False, profile_fn=None): if doit_args is None: doit_args = ['run'] runner = DoitMain(self) print('\n--- Begin Task Execution ---') if profile_fn is not False and doit_args[0] == 'run': with StartProfiler(filename=profile_fn): return runner.run(doit_args) else: return runner.run(doit_args)
def _doit_prepare(self, builder, task): miner.dodo.builder = builder # create build directory for storing doit database if not os.path.exists(builder.build_dir): os.makedirs(builder.build_dir) opt_vals = {'dep_file': os.path.join(builder.build_dir, '.doit.db')} commander = DoitMain(ModuleTaskLoader(miner.dodo), extra_config={'GLOBAL': opt_vals}) commander.BIN_NAME = 'doit' logging.info('Preparing LEDE build system...') commander.run(['--verbosity', '2', task])
def get_commands(self): # core doit commands cmds = DoitMain.get_commands(self) # load nikola commands for name, cmd in self.nikola._commands.items(): cmds[name] = cmd return cmds
def process_pipeline(tasks, *args, version=None, workdir=None, patterns=None, num_processes=None, output=None, **kwargs): doit_args = ['-n', num_processes, '--continue'] task_names = load_tasks(tasks) if not task_names: domains = get_domains(patterns) task_names = load_tasks([ gen_task(workdir, domains) for gen_task in (gen_dig, gen_host, gen_ssl) ]) def task_setup(): return { 'actions': [ f'rm -rf {workdir}', ], } globals()[task_setup.__name__] = task_setup exitcode = DoitMain(ModuleTaskLoader(globals())).run(doit_args + task_names) create_result(workdir, output) sys.exit(exitcode)
def get_commands(self): # core doit commands cmds = DoitMain.get_commands(self) # load nikola commands for name, cmd in self.nikola.commands.items(): cmds[name] = cmd return cmds
def test_doit_coverage(cookies): result = cookies.bake() with inside_dir(result.project): with poetryenv_in_project(): importlib.reload(dodo) dodo.webbrowser = mock.MagicMock() assert DoitMain(ModuleTaskLoader(dodo)).run(["coverage"]) == 0 importlib.reload(dodo)
def main(): # Fixes issues with multiprocessing with cx_freeze on windows freeze_support() """ Start doit. """ import sys from doit.doit_cmd import DoitMain sys.exit(DoitMain().run(sys.argv[1:]))
def execute(self, cmds=None, spec=None): cmds = cmds or self.cmds spec = spec or self.spec if spec.otto.tasks: loader = OttoTaskLoader(spec.otto.tasks) else: loader = OttoTaskLoader({'tasks': spec.otto}) if cmds: sys.exit(DoitMain(loader).run(cmds))
def get_commands(self): # core doit commands cmds = DoitMain.get_commands(self) # load nikola commands for name, cmd in self.nikola.commands.items(): cmds[name] = cmd # Hide run because Nikola uses build cmds.pop('run') return cmds
def get_commands(self): cmds = DoitMain.get_commands(self) for extra_cmd_cls in self._extra_cmds: if issubclass(extra_cmd_cls, DoitCmdBase): extra_cmd = extra_cmd_cls(task_loader=self.task_loader) extra_cmd.doit_app = self else: extra_cmd = extra_cmd_cls() cmds[extra_cmd.name] = extra_cmd return cmds
def run_task(module, task): """ run_task - Have doit run the named task :param module module: module containing tasks :param str task: task to run """ start = time.time() DoitMain(ModuleTaskLoader(module)).run([task]) print("%.2f seconds" % (time.time() - start))
def run_tasks(tasks, args, config={'verbosity': 2}): tasks = list(tasks) class Loader(TaskLoader): @staticmethod def load_tasks(cmd, opt_values, pos_args): return tasks, config return DoitMain(Loader()).run(args)
def run(doit_args): code = DoitMain(DoitLoader()).run(doit_args) if code == 0: print 'Doit: all tasks were executed.' elif code == 1: print 'Doit: some tasks were failed.' elif code == 2: print 'Doit: error when executing tasks.' elif code == 3: print 'Doit: error before task execution starts.' return code
def run_tasks(tasks, args, config={'verbosity': 0}): if type(tasks) is not list: raise TypeError('tasks must be a list') class Loader(TaskLoader): @staticmethod def load_tasks(cmd, opt_values, pos_args): return tasks, config return DoitMain(Loader()).run(args)
def run_doit(self, task_loader, configuration): arguments = [] extra_configuration = {} if 'threads' in configuration and configuration.get('threads') > 1: arguments.extend(['-n', str(configuration.get('threads'))]) if 'doitConfig' in configuration: extra_configuration = configuration.get('doitConfig') return DoitMain(task_loader=task_loader, extra_config=extra_configuration).run(arguments)
def run_tasks(tasks, args, config={'verbosity': 0}): '''Given a list of `Task` objects, a list of arguments, and a config dictionary, execute the tasks. ''' if type(tasks) is not list: raise TypeError('tasks must be of type list.') class Loader(TaskLoader): @staticmethod def load_tasks(cmd, opt_values, pos_args): return tasks, config return DoitMain(Loader()).run(args)
def test_doit_docs(cookies, docs_generator): extra_context = {"docs_generator": docs_generator} result = cookies.bake(extra_context=extra_context) project = result.project with inside_dir(project): with poetryenv_in_project(): importlib.reload(dodo) dodo.webbrowser = mock.MagicMock() project.mkdir("docs", "htmlcov") with project.join("docs", "htmlcov", "index.html").open("w") as fo: fo.write("") assert DoitMain(ModuleTaskLoader(dodo)).run(["docs"]) == 0 assert project.join("site", "htmlcov").check(dir=1) importlib.reload(dodo)
def run_tasks(tasks, args, doit_config=None): '''Given a list of `Task` objects, a list of arguments, and a config dictionary, execute the tasks. ''' doit_config = {} if doit_config is None else doit_config doit_config.setdefault('verbosity', 0) doit_config.setdefault('action_string_formatting', 'new') class Loader(TaskLoader): @staticmethod def load_tasks(cmd, opt_values, pos_args): return tasks, doit_config return DoitMain(Loader()).run(args)
def main(output, build_dir, elm_path, mount_at, exclude_modules, exclude_source_directories, force_exclusion, fake_user, fake_project, fake_version, fake_summary, fake_license, validate, doit_args, project_path, include_paths): """Generate static documentation for your Elm project""" if not shutil.which('rsync'): raise click.UsageError('this program requires rsync') if not check_rsync_version(): raise click.UsageError( 'this program requires rsync version {} or greater'.format( '.'.join(REQUIRED_RSYNC_VERSION))) if not validate and output is None: raise click.BadParameter('please specify --output directory') resolved_include_paths = [_resolve_path(path) for path in include_paths] exclude_modules = exclude_modules.split(',') if exclude_modules else [] exclude_source_directories = exclude_source_directories.split( ',') if exclude_source_directories else [] project_config = ProjectConfig( include_paths=resolved_include_paths, exclude_modules=exclude_modules, exclude_source_directories=exclude_source_directories, force_exclusion=force_exclusion, fake_user=fake_user, fake_project=fake_project, fake_version=fake_version, fake_summary=fake_summary, fake_license=fake_license, ) task_creators = build_task_creators( _resolve_path(project_path), project_config, _resolve_path(elm_path) if elm_path else None, _resolve_path(output) if output is not None else None, build_path=_resolve_path(build_dir) if build_dir is not None else None, mount_point=mount_at, validate=validate) extra_config = {'GLOBAL': {'outfile': LazyOutfile()}} result = DoitMain(ModuleTaskLoader(task_creators), extra_config=extra_config).run( doit_args.split(' ') if doit_args else []) if result is not None and result > 0: raise DoitException('see output above', result)
def test_config_simple(capsys): DOIT_CONFIG = doit_config(backend='r') loader = ModuleTaskLoader(locals()) loader.setup({}) config = loader.load_doit_config() assert config == {'backend': 'r'} res = DoitMain(ModuleTaskLoader(locals())).run(()) assert res == 3 captured = capsys.readouterr() with capsys.disabled(): assert "TypeError: 'NoneType' object is not callable" in captured.err
def run(clean=False, number_of_processes=None, db_file_name=None): doit_args = [] if clean: doit_args.append('clean') if number_of_processes: doit_args += ['-n', str(number_of_processes)] if db_file_name: doit_args += ['--db-file', db_file_name] code = DoitMain(DoitLoader()).run(doit_args) if code == 0: logger.debug('Doit: all tasks were executed.') elif code == 1: logger.debug('Doit: some tasks were failed.') elif code == 2: logger.debug('Doit: error when executing tasks.') elif code == 3: logger.debub('Doit: error before task execution starts.') return code
def _doIt(self, *args, **kwargs): ''' DoIt wrapper @param args [in] (list) arguments @param kwargs [in] (dict) keyword arguments ''' members = dict(kwargs.get('tasks', self._tasks)) members.update( DOIT_CONFIG={ 'backend': 'json', 'dep_file': self._dep_file, 'reporter': self._reporter, 'verbosity': self._verbosity, 'minversion': '0.27.0' }) status = DoitMain(ModuleTaskLoader(members)).run(args) if status: sys.exit(status)
def cmd_main(args, extra_config=None, bin_name='doit'): if extra_config: extra_config = {'GLOBAL': extra_config} main = DoitMain(extra_config=extra_config) main.BIN_NAME = bin_name return main.run(args)
def doit_app(request): app = DoitMain() app.sub_cmds['tabcompletion'] = TabCompletion() app.sub_cmds['help'] = Help() return app
def task_report(): """ renders the template in reports\report_template.docx with all figures and numbers.yaml mapped""" figure_files = glob.glob(r'reports\figures\*.png') outfile = r'reports\report_analysis-double-pendulum.docx' infile = r'reports\report_template.docx' context_file = r'reports\numbers.yaml' src_file = 'dodo_utils.py' return { 'actions': [(template_renderer(figure_files, context_file), (infile, outfile))], 'targets': [outfile], 'file_dep': [infile] + figure_files + [context_file, src_file], 'clean': True } from doit_xtended.linkedtasks import _generated_linked_tasks if __name__ == '__main__': logging.basicConfig(level=logging.INFO) from doit.cmd_base import ModuleTaskLoader from doit.doit_cmd import DoitMain d = DoitMain(ModuleTaskLoader(globals())) d.run(['-s', 'models:prediction'])
def cmd_main(args, extra_config=None): if extra_config: extra_config = {'GLOBAL': extra_config} return DoitMain(extra_config=extra_config).run(args)
def task_report(): """ renders the template in reports\report_template.docx with all figures and numbers.yaml mapped""" figure_files = glob.glob(r'reports\figures\*.png') outfile = r'reports\report_ThreadImageProcessor.docx' infile = r'reports\report_template.docx' context_file = r'reports\numbers.yaml' src_file = 'dodo_utils.py' return { 'actions': [(template_renderer(figure_files, context_file), (infile, outfile))], 'targets': [outfile], 'file_dep': [infile] + figure_files + [context_file, src_file], 'clean': True } from doit_xtended.linkedtasks import _generated_linked_tasks if __name__ == '__main__': logging.basicConfig(level=logging.INFO) from doit.cmd_base import ModuleTaskLoader from doit.doit_cmd import DoitMain d = DoitMain(ModuleTaskLoader(globals())) d.run(['list', '--all'])
def run(task_creators): """run doit using task_creators @param task_creators: module or dict containing task creators """ sys.exit(DoitMain(ModuleTaskLoader(task_creators)).run(sys.argv[1:]))
def get_commands(self): cmds = DoitMain.get_commands(self) my_cmd = MyCmd() cmds[my_cmd.name] = my_cmd return cmds