def doit(line):
"""
Run *doit* with `task_creators` from all interactive variables
(IPython's global namespace).
Examples:
>>> %doit --help ## Show help for options and arguments.
>>> def task_foo():
return {'actions': ['echo hi IPython'],
'verbosity': 2}
>>> %doit list ## List any tasks discovered.
foo
>>> %doit ## Run any tasks.
. foo
hi IPython
"""
# Override db-files location inside ipython-profile dir,
# which is certainly writable.
prof_dir = ip.profile_dir.location
opt_vals = {'dep_file': os.path.join(prof_dir, 'db', '.doit.db')}
commander = DoitMain(ModuleTaskLoader(ip.user_module),
extra_config={'GLOBAL': opt_vals})
commander.BIN_NAME = 'doit'
commander.run(line.split())
def start(self):
"""Begin executing tasks."""
if self.log_filename:
print('Output will be logged to `%s`.' % self.log_filename)
start_time = time.strftime(self.timestamp_fmt)
print('Started %s' % start_time)
if self.log_filename:
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = self.log_file
sys.stderr = self.log_file
print('Started %s' % start_time)
doit_main = DoitMain(self)
doit_main.run(['run'])
stop_time = time.strftime(self.timestamp_fmt)
if self.log_filename:
print('Stopped %s' % stop_time)
print()
sys.stdout = orig_stdout
sys.stderr = orig_stderr
self.log_file.close()
print('Stopped %s' % stop_time)
def run(self, doit_args=None, verbose=True):
'''Run the pipeline. Movees to the directory, loads the tasks into doit,
and executes that tasks that are not up-to-date.
Args:
doit_args (list): Args that would be passed to the doit shell
command. By default, just run.
verbose (bool): If True, print UI stuff.
Returns:
int: Exit status of the doit command.
'''
if verbose:
print(ui.header('Run Tasks', level=4))
if doit_args is None:
doit_args = ['run']
if self.n_threads > 1:
doit_args.extend(['-n', str(self.n_threads)])
runner = DoitMain(self)
with Move(self.directory):
if self.profile is True:
profile_fn = path.join(self.directory, 'profile.csv')
with StartProfiler(filename=profile_fn):
return runner.run(doit_args)
else:
return runner.run(doit_args)
def doit(line):
"""
Run *doit* with `task_creators` from all interactive variables
(IPython's global namespace).
Examples:
>>> %doit --help ## Show help for options and arguments.
>>> def task_foo():
return {'actions': ['echo hi IPython'],
'verbosity': 2}
>>> %doit list ## List any tasks discovered.
foo
>>> %doit ## Run any tasks.
. foo
hi IPython
"""
ip = get_ipython()
# Override db-files location inside ipython-profile dir,
# which is certainly writable.
prof_dir = ip.profile_dir.location
opt_vals = {'dep_file': os.path.join(prof_dir, 'db', '.doit.db')}
commander = DoitMain(ModuleTaskLoader(ip.user_module),
extra_config={'GLOBAL': opt_vals})
commander.run(line.split())
def start(self):
"""Begin executing tasks."""
if self.log_filename:
print('Output will be logged to `%s`.' % self.log_filename)
start_time = time.strftime(self.timestamp_fmt)
print('Started %s' % start_time)
if self.log_filename:
orig_stdout = sys.stdout
orig_stderr = sys.stderr
sys.stdout = self.log_file
sys.stderr = self.log_file
print('Started %s' % start_time)
doit_main = DoitMain(self)
doit_main.run(['run'])
stop_time = time.strftime(self.timestamp_fmt)
if self.log_filename:
print('Stopped %s' % stop_time)
print()
sys.stdout = orig_stdout
sys.stderr = orig_stderr
self.log_file.close()
print('Stopped %s' % stop_time)
def main():
configure_test()
load_exps()
doit_main = DoitMain(CustomTaskLoader())
return doit_main.run(['run'])
def main(argv=None):
myname = path.basename(sys.argv[0]) # @UnusedVariable
mydir = path.dirname(__file__) # @UnusedVariable
if argv is None:
argv = sys.argv[1:]
opt_vals = {} # 'dep_file': path.abspath(path.join(mydir, '.doit.db'))}
commander = DoitMain(ModuleTaskLoader(pndlcmd),
extra_config={'GLOBAL': opt_vals})
commander.run(argv)
def test_help_plugin_name(self, capsys):
plugin = {'XXX': 'tests.sample_plugin:MyCmd'}
main = DoitMain(extra_config={'COMMAND': plugin})
main.BIN_NAME = 'doit'
returned = main.run(["help"])
assert returned == 0
out, err = capsys.readouterr()
assert "doit XXX " in out
assert "test extending doit commands" in out, out
def test_help_plugin_name(self, capsys):
plugin = {'XXX': 'tests.sample_plugin:MyCmd'}
main = DoitMain(extra_config={'COMMAND':plugin})
main.BIN_NAME = 'doit'
returned = main.run(["help"])
assert returned == 0
out, err = capsys.readouterr()
assert "doit XXX " in out
assert "test extending doit commands" in out, out
def run(self, cmds):
tasks = {}
for v in self.loaders:
for name, l in v.list_tasks():
f = l
tasks[name] = f
ml = ModuleTaskLoader(tasks)
main = DoitMain(ml)
main.config['default_tasks'] = cmds
return main.run([])
def run(self, doit_args=None, move=False, profile_fn=None):
if doit_args is None:
doit_args = ['run']
runner = DoitMain(self)
print('\n--- Begin Task Execution ---')
if profile_fn is not False and doit_args[0] == 'run':
with StartProfiler(filename=profile_fn):
return runner.run(doit_args)
else:
return runner.run(doit_args)
def _doit_prepare(self, builder, task):
miner.dodo.builder = builder
# create build directory for storing doit database
if not os.path.exists(builder.build_dir):
os.makedirs(builder.build_dir)
opt_vals = {'dep_file': os.path.join(builder.build_dir, '.doit.db')}
commander = DoitMain(ModuleTaskLoader(miner.dodo),
extra_config={'GLOBAL': opt_vals})
commander.BIN_NAME = 'doit'
logging.info('Preparing LEDE build system...')
commander.run(['--verbosity', '2', task])
def get_commands(self):
# core doit commands
cmds = DoitMain.get_commands(self)
# load nikola commands
for name, cmd in self.nikola._commands.items():
cmds[name] = cmd
return cmds
def process_pipeline(tasks,
*args,
version=None,
workdir=None,
patterns=None,
num_processes=None,
output=None,
**kwargs):
doit_args = ['-n', num_processes, '--continue']
task_names = load_tasks(tasks)
if not task_names:
domains = get_domains(patterns)
task_names = load_tasks([
gen_task(workdir, domains)
for gen_task in (gen_dig, gen_host, gen_ssl)
])
def task_setup():
return {
'actions': [
f'rm -rf {workdir}',
],
}
globals()[task_setup.__name__] = task_setup
exitcode = DoitMain(ModuleTaskLoader(globals())).run(doit_args +
task_names)
create_result(workdir, output)
sys.exit(exitcode)
def get_commands(self):
# core doit commands
cmds = DoitMain.get_commands(self)
# load nikola commands
for name, cmd in self.nikola.commands.items():
cmds[name] = cmd
return cmds
def test_doit_coverage(cookies):
result = cookies.bake()
with inside_dir(result.project):
with poetryenv_in_project():
importlib.reload(dodo)
dodo.webbrowser = mock.MagicMock()
assert DoitMain(ModuleTaskLoader(dodo)).run(["coverage"]) == 0
importlib.reload(dodo)
def main():
# Fixes issues with multiprocessing with cx_freeze on windows
freeze_support()
""" Start doit.
"""
import sys
from doit.doit_cmd import DoitMain
sys.exit(DoitMain().run(sys.argv[1:]))
def execute(self, cmds=None, spec=None):
cmds = cmds or self.cmds
spec = spec or self.spec
if spec.otto.tasks:
loader = OttoTaskLoader(spec.otto.tasks)
else:
loader = OttoTaskLoader({'tasks': spec.otto})
if cmds:
sys.exit(DoitMain(loader).run(cmds))
def get_commands(self):
# core doit commands
cmds = DoitMain.get_commands(self)
# load nikola commands
for name, cmd in self.nikola.commands.items():
cmds[name] = cmd
# Hide run because Nikola uses build
cmds.pop('run')
return cmds
def get_commands(self):
cmds = DoitMain.get_commands(self)
for extra_cmd_cls in self._extra_cmds:
if issubclass(extra_cmd_cls, DoitCmdBase):
extra_cmd = extra_cmd_cls(task_loader=self.task_loader)
extra_cmd.doit_app = self
else:
extra_cmd = extra_cmd_cls()
cmds[extra_cmd.name] = extra_cmd
return cmds
def run_task(module, task):
"""
run_task - Have doit run the named task
:param module module: module containing tasks
:param str task: task to run
"""
start = time.time()
DoitMain(ModuleTaskLoader(module)).run([task])
print("%.2f seconds" % (time.time() - start))
def run_tasks(tasks, args, config={'verbosity': 2}):
tasks = list(tasks)
class Loader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
return tasks, config
return DoitMain(Loader()).run(args)
def run(doit_args):
code = DoitMain(DoitLoader()).run(doit_args)
if code == 0:
print 'Doit: all tasks were executed.'
elif code == 1:
print 'Doit: some tasks were failed.'
elif code == 2:
print 'Doit: error when executing tasks.'
elif code == 3:
print 'Doit: error before task execution starts.'
return code
def run_tasks(tasks, args, config={'verbosity': 0}):
if type(tasks) is not list:
raise TypeError('tasks must be a list')
class Loader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
return tasks, config
return DoitMain(Loader()).run(args)
def run_doit(self, task_loader, configuration):
arguments = []
extra_configuration = {}
if 'threads' in configuration and configuration.get('threads') > 1:
arguments.extend(['-n', str(configuration.get('threads'))])
if 'doitConfig' in configuration:
extra_configuration = configuration.get('doitConfig')
return DoitMain(task_loader=task_loader,
extra_config=extra_configuration).run(arguments)
def run_tasks(tasks, args, config={'verbosity': 0}):
'''Given a list of `Task` objects, a list of arguments,
and a config dictionary, execute the tasks.
'''
if type(tasks) is not list:
raise TypeError('tasks must be of type list.')
class Loader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
return tasks, config
return DoitMain(Loader()).run(args)
def test_doit_docs(cookies, docs_generator):
extra_context = {"docs_generator": docs_generator}
result = cookies.bake(extra_context=extra_context)
project = result.project
with inside_dir(project):
with poetryenv_in_project():
importlib.reload(dodo)
dodo.webbrowser = mock.MagicMock()
project.mkdir("docs", "htmlcov")
with project.join("docs", "htmlcov", "index.html").open("w") as fo:
fo.write("")
assert DoitMain(ModuleTaskLoader(dodo)).run(["docs"]) == 0
assert project.join("site", "htmlcov").check(dir=1)
importlib.reload(dodo)
def run_tasks(tasks, args, doit_config=None):
'''Given a list of `Task` objects, a list of arguments,
and a config dictionary, execute the tasks.
'''
doit_config = {} if doit_config is None else doit_config
doit_config.setdefault('verbosity', 0)
doit_config.setdefault('action_string_formatting', 'new')
class Loader(TaskLoader):
@staticmethod
def load_tasks(cmd, opt_values, pos_args):
return tasks, doit_config
return DoitMain(Loader()).run(args)
def main(output, build_dir, elm_path, mount_at, exclude_modules,
exclude_source_directories, force_exclusion, fake_user, fake_project,
fake_version, fake_summary, fake_license, validate, doit_args,
project_path, include_paths):
"""Generate static documentation for your Elm project"""
if not shutil.which('rsync'):
raise click.UsageError('this program requires rsync')
if not check_rsync_version():
raise click.UsageError(
'this program requires rsync version {} or greater'.format(
'.'.join(REQUIRED_RSYNC_VERSION)))
if not validate and output is None:
raise click.BadParameter('please specify --output directory')
resolved_include_paths = [_resolve_path(path) for path in include_paths]
exclude_modules = exclude_modules.split(',') if exclude_modules else []
exclude_source_directories = exclude_source_directories.split(
',') if exclude_source_directories else []
project_config = ProjectConfig(
include_paths=resolved_include_paths,
exclude_modules=exclude_modules,
exclude_source_directories=exclude_source_directories,
force_exclusion=force_exclusion,
fake_user=fake_user,
fake_project=fake_project,
fake_version=fake_version,
fake_summary=fake_summary,
fake_license=fake_license,
)
task_creators = build_task_creators(
_resolve_path(project_path),
project_config,
_resolve_path(elm_path) if elm_path else None,
_resolve_path(output) if output is not None else None,
build_path=_resolve_path(build_dir) if build_dir is not None else None,
mount_point=mount_at,
validate=validate)
extra_config = {'GLOBAL': {'outfile': LazyOutfile()}}
result = DoitMain(ModuleTaskLoader(task_creators),
extra_config=extra_config).run(
doit_args.split(' ') if doit_args else [])
if result is not None and result > 0:
raise DoitException('see output above', result)
def test_config_simple(capsys):
DOIT_CONFIG = doit_config(backend='r')
loader = ModuleTaskLoader(locals())
loader.setup({})
config = loader.load_doit_config()
assert config == {'backend': 'r'}
res = DoitMain(ModuleTaskLoader(locals())).run(())
assert res == 3
captured = capsys.readouterr()
with capsys.disabled():
assert "TypeError: 'NoneType' object is not callable" in captured.err
def run(clean=False, number_of_processes=None, db_file_name=None):
doit_args = []
if clean:
doit_args.append('clean')
if number_of_processes:
doit_args += ['-n', str(number_of_processes)]
if db_file_name:
doit_args += ['--db-file', db_file_name]
code = DoitMain(DoitLoader()).run(doit_args)
if code == 0:
logger.debug('Doit: all tasks were executed.')
elif code == 1:
logger.debug('Doit: some tasks were failed.')
elif code == 2:
logger.debug('Doit: error when executing tasks.')
elif code == 3:
logger.debub('Doit: error before task execution starts.')
return code
def _doIt(self, *args, **kwargs):
'''
DoIt wrapper
@param args [in] (list) arguments
@param kwargs [in] (dict) keyword arguments
'''
members = dict(kwargs.get('tasks', self._tasks))
members.update(
DOIT_CONFIG={
'backend': 'json',
'dep_file': self._dep_file,
'reporter': self._reporter,
'verbosity': self._verbosity,
'minversion': '0.27.0'
})
status = DoitMain(ModuleTaskLoader(members)).run(args)
if status:
sys.exit(status)
def cmd_main(args, extra_config=None, bin_name='doit'):
if extra_config:
extra_config = {'GLOBAL': extra_config}
main = DoitMain(extra_config=extra_config)
main.BIN_NAME = bin_name
return main.run(args)
def cmd_main(args, extra_config=None, bin_name='doit'):
if extra_config:
extra_config = {'GLOBAL': extra_config}
main = DoitMain(extra_config=extra_config)
main.BIN_NAME = bin_name
return main.run(args)
def doit_app(request):
app = DoitMain()
app.sub_cmds['tabcompletion'] = TabCompletion()
app.sub_cmds['help'] = Help()
return app
def task_report():
""" renders the template in reports\report_template.docx with all figures and numbers.yaml mapped"""
figure_files = glob.glob(r'reports\figures\*.png')
outfile = r'reports\report_analysis-double-pendulum.docx'
infile = r'reports\report_template.docx'
context_file = r'reports\numbers.yaml'
src_file = 'dodo_utils.py'
return {
'actions': [(template_renderer(figure_files,
context_file), (infile, outfile))],
'targets': [outfile],
'file_dep': [infile] + figure_files + [context_file, src_file],
'clean':
True
}
from doit_xtended.linkedtasks import _generated_linked_tasks
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
from doit.cmd_base import ModuleTaskLoader
from doit.doit_cmd import DoitMain
d = DoitMain(ModuleTaskLoader(globals()))
d.run(['-s', 'models:prediction'])
def cmd_main(args, extra_config=None):
if extra_config:
extra_config = {'GLOBAL': extra_config}
return DoitMain(extra_config=extra_config).run(args)
def task_report():
""" renders the template in reports\report_template.docx with all figures and numbers.yaml mapped"""
figure_files = glob.glob(r'reports\figures\*.png')
outfile = r'reports\report_ThreadImageProcessor.docx'
infile = r'reports\report_template.docx'
context_file = r'reports\numbers.yaml'
src_file = 'dodo_utils.py'
return {
'actions': [(template_renderer(figure_files,
context_file), (infile, outfile))],
'targets': [outfile],
'file_dep': [infile] + figure_files + [context_file, src_file],
'clean':
True
}
from doit_xtended.linkedtasks import _generated_linked_tasks
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO)
from doit.cmd_base import ModuleTaskLoader
from doit.doit_cmd import DoitMain
d = DoitMain(ModuleTaskLoader(globals()))
d.run(['list', '--all'])
def run(task_creators):
"""run doit using task_creators
@param task_creators: module or dict containing task creators
"""
sys.exit(DoitMain(ModuleTaskLoader(task_creators)).run(sys.argv[1:]))
def get_commands(self):
cmds = DoitMain.get_commands(self)
my_cmd = MyCmd()
cmds[my_cmd.name] = my_cmd
return cmds
