def run_dodocker_cli(args):
# helper function for external programs like tests
parser = create_parser()
parsed = parser.parse_known_args(args)
dodocker_config.clear()
dodocker_config.update(load_dodocker_config())
try:
process_args(*parsed)
doit.run(globals())
except SystemExit as e:
# catch normal zero exit, but re-raise other
if e.code != 0:
raise
for key, target in P.ENVENTURES.items():
if not meta_lock_exists and target != P.THIS_META_ENV_LOCK:
continue
yield _make_lock_task(key, target)
def task_publish():
"""publish to pypi"""
def _check_hash():
on_disk = P.SHA256SUMS.read_text()
calculated = _calc_hash()
print("\n--\n".join(
["on-disk:", on_disk, "calculated", calculated]))
if calculated != on_disk:
raise RuntimeError("SHA256SUMS do not match:")
print("SHA256SUMS are OK")
return dict(actions=[
_check_hash,
InteractiveAction(
[*P.RUN_IN["meta"], "twine", "upload", P.SDIST, P.WHEEL],
shell=False,
),
])
if __name__ == "__main__":
doit.run(globals())
res_plots = [os.path.join(plotdir, 'results-%d.svg' % (i+1))
for i in range(5)]
yield {'name': 'models',
'actions': [CmdAction(
'py.test -p fnme.options --simulator nengo.Simulator '
'--seed 5 --plot plots --noprofile -- fnme/benchmarks.py',
cwd=root)],
'file_dep': [plotfile, benchfile],
'targets': res_plots}
yield {'name': 'accuracy',
'actions': [(fnme.plots.accuracy,)],
'file_dep': [plotfile],
'targets': acc_plots}
yield {'name': 'speed',
'actions': [fnme.plots.speed],
'targets': [os.path.join(figdir, 'fig%d.%s' % (i, ext))
for ext in figexts for i in range(5, 7)]}
yield {'name': 'combine',
'actions': [(fnme.plots.fig1,),
(fnme.plots.fig2,),
(fnme.plots.fig3,),
(fnme.plots.fig4,)],
'file_dep': acc_plots + res_plots,
'targets': [os.path.join(figdir, 'fig%d.%s' % (i, ext))
for ext in figexts for i in range(1, 5)]}
if __name__ == '__main__':
doit.run(locals())
def main_entry_point(argv=None):
import doit
doit.run(globals())
def test_valid_config(monkeypatch, depfile_name):
monkeypatch.setattr(sys, 'argv', ['did'])
# create custom config with various settings
DOIT_CONFIG = doit_config(
default_tasks=['a', 'b'], # task 'ignored' should not be run
continue_=True, # a will fail but b will still run
single=True, # task 'ignored' should not be run even if a dep of a
always=True, # a will run even if it is up to date
# database
db_file="tmp_db.json", # the db file will be this one
dep_file=None, # alias for db_file, ok
backend="json", # and it will contain json text
# verbosity
verbosity=2, # 'echo hi' should be captured in capsys
failure_verbosity=2, # a report should be issued at the end
# output reporter and cwd
outfile=
"tmp_out.txt", # all output except 'echo hi' should be in that file
reporter='console', # default
# dir='./tmp',
# parallel
num_process=
2, # the output is slightly different in parallel mode we can check this
parallel_type='thread',
)
ignored = task(name="ignored", actions=["ignored"])
a = task(name="a", task_dep=[ignored], actions=["ech hi"], uptodate=[True])
b = task(name="b", actions=["echo hi"])
try:
# run
run(locals())
except SystemExit as err:
# error code seems to be 1 on windows and 2 on linux (travis)
assert err.code >= 1, "doit execution error"
else: # pragma: no cover
assert False, "Did not receive SystemExit - should not happen"
# Validate that the outcome takes all config options into account
with open("tmp_db.json", mode="rt") as f:
resdict = json.loads(f.read())
os.remove("tmp_db.json")
assert len(resdict) == 1
result_str = resdict['b']['result:']
if sys.version_info > (3, 0):
assert isinstance(result_str, str)
assert len(result_str) > 0
# Validate output
with open("tmp_out.txt", mode="rt") as f:
out_str = f.read()
os.remove("tmp_out.txt")
win = platform.system() == "Windows"
if sys.version_info > (3, 0):
assert out_str == """. a => Cmd: ech hi
. b => Cmd: echo hi
%(failed_type)s - taskid:a
Command %(failed)s: 'ech hi' returned %(failed_code)s
########################################
%(failed_type)s - taskid:a
Command %(failed)s: 'ech hi' returned %(failed_code)s
a <stderr>:
%(errmsg)s
a <stdout>:
""" % dict(failed_type='TaskFailed' if win else 'TaskError',
failed='failed' if win else 'error',
failed_code=1 if win else 127,
errmsg="'ech' is not recognized as an internal or external command,\n"
"operable program or batch file."
if win else "/bin/sh: 1: ech: not found")
else:
assert out_str == """. a => Cmd: ech hi
. b => Cmd: echo hi
########################################
%(failed_type)s - taskid:a
Command %(failed)s: 'ech hi' returned %(failed_code)s
%(errmsg)s
""" % dict(failed_type='TaskFailed' if win else 'TaskError',
failed='failed' if win else 'error',
failed_code=1 if win else 127,
errmsg=
"'ech' is not recognized as an internal or external command, \n"
"operable program or batch file."
if win else "/bin/sh: 1: ech: not found")
for reference_case, scenario_path in REFERENCE_CASES.items():
yield {
'name': '%(reference_case)s' % locals(),
'actions': [(cea.plots.scenario_plots.run_as_script, [], {
'scenario_folders': [scenario_path],
'output_file': None # use default
})],
'verbosity': 1,
}
def task_run_unit_tests():
"""run the unittests"""
def run_unit_tests():
import unittest
import os
os.environ['REFERENCE_CASE'] = REFERENCE_CASES['open']
testsuite = unittest.defaultTestLoader.discover('.')
result = unittest.TextTestRunner(verbosity=1).run(testsuite)
return result.wasSuccessful()
return {
'actions': [run_unit_tests],
'verbosity': 1
}
if __name__ == '__main__':
import doit
doit.run(globals())
def main():
parser = create_parser()
parsed = parser.parse_known_args()
dodocker_config.update(load_dodocker_config())
process_args(*parsed)
doit.run(globals())
def main():
doit.run(globals())
def doit_run():
import doit
doit.run(globals())
def test_task(monkeypatch, depfile_name, capsys):
"""Tests that our various task generation mechanisms work"""
@pytask(title="custom title")
def a():
""" hey """
print("hello !")
@pytask(task_dep=[a])
def b():
""" hey! """
print("hello !!")
@cmdtask(task_dep=[b])
def d():
""" hey!d """
return """
echo ola
# skipped comment
echo hey
"""
@taskgen
def c():
""" hey!!! """
# shell commands
yield task(name="echo",
actions=["echo hi"],
doc="my echo doc",
targets=["hoho.txt"])
# python task - non-decorator style
def c_():
"""here is a doc"""
print("hello")
yield pytask(c_)
for i in range(2):
# python task - decorator style
@pytask(name="subtask %i" % i,
doc="a subtask %s" % i,
title="this is %s running" % i)
def c_():
print("hello sub")
yield c_
# python task - non-decorator style
def d_():
print("hello variant")
yield pytask(name="subtask %i variant" % i,
doc="a subtask %s variant" % i,
title="this is %s running variant" % i)(d_)
if sys.version_info < (3, 0):
# doit version is 0.29 on python 2. Internal api is different.
pass
else:
# manually check that loading the tasks works
loader = ModuleTaskLoader(locals())
loader.setup({})
config = loader.load_doit_config()
task_list = loader.load_tasks(Command(), [])
assert len(task_list) == 10
# Note: unfortunately on python 3.5 and 3.6 the order does not seem guaranteed with this api
# task a
task_a = [t for t in task_list if t.name == 'a']
assert len(task_a) == 1
task_a = task_a[0]
assert task_a.name == 'a'
assert task_a.doc == a.__doc__.strip()
assert task_a.actions[0].py_callable == why_am_i_running
assert task_a.actions[1].py_callable == a
assert task_a.title() == 'a => custom title'
# task b dependency
task_b = [t for t in task_list if t.name == 'b']
assert len(task_b) == 1
task_b = task_b[0]
assert task_b.task_dep == ['a']
# task c with 2 subtasks
# todo
# task d
task_d = [t for t in task_list if t.name == 'd']
assert len(task_d) == 1
task_d = task_d[0]
assert len(task_d.actions) == 2
_sep = '& ' if platform.system() == 'Windows' else '; '
assert task_d.actions[1]._action.split(_sep) == [
"echo ola", "echo hey"
]
# ---- checks : list
monkeypatch.setattr(sys, 'argv',
['did', 'list', '--all', '--db-file', depfile_name])
try:
# run
run(locals())
except SystemExit as err:
assert err.code == 0, "doit execution error"
else: # pragma: no cover
assert False, "Did not receive SystemExit - should not happen"
captured = capsys.readouterr()
with capsys.disabled():
assert captured.out == """a hey
b hey!
c hey!!!
c:c_ here is a doc
c:echo my echo doc
c:subtask 0 a subtask 0
c:subtask 0 variant a subtask 0 variant
c:subtask 1 a subtask 1
c:subtask 1 variant a subtask 1 variant
d hey!d
"""
# -- checks : execution
monkeypatch.setattr(sys, 'argv',
['did', '--verbosity', '2', '--db-file', depfile_name])
try:
# run
run(locals())
except SystemExit as err:
assert err.code == 0, "doit execution error"
else: # pragma: no cover
assert False, "Did not receive SystemExit - should not happen"
captured = capsys.readouterr()
with capsys.disabled():
assert captured.out.replace("\r", "") == """hello !
hello !!
ola
hey
Running <Task: c:echo> because one of its targets does not exist: 'hoho.txt'
hi
hello
hello sub
hello variant
hello sub
hello variant
"""
if sys.version_info < (3, 0):
# doit version is 0.29 on python 2. Internal api is different.
pass
else:
# formal checks: equivalent of doit list --all
output = StringIO()
cmd_list = CmdFactory(List, outstream=output, task_list=task_list)
cmd_list._execute(subtasks=True, quiet=False)
assert output.getvalue() == """a hey
b hey!
c hey!!!
c:c_ here is a doc
c:echo my echo doc
c:subtask 0 a subtask 0
c:subtask 0 variant a subtask 0 variant
c:subtask 1 a subtask 1
c:subtask 1 variant a subtask 1 variant
d hey!d
"""
# formal checks: equivalent of doit (execution)
output = StringIO()
cmd_run = CmdFactory(Run,
backend='dbm',
dep_file=depfile_name,
task_list=task_list)
result = cmd_run._execute(output, verbosity=2)
assert 0 == result
assert output.getvalue() == """. a => custom title
. b => Python: function test_task.<locals>.b
. d => Cmd: echo ola%secho hey
. c:echo => Cmd: echo hi
. c:c_ => Python: function test_task.<locals>.c.<locals>.c_
. c:subtask 0 => this is 0 running
. c:subtask 0 variant => this is 0 running variant
. c:subtask 1 => this is 1 running
. c:subtask 1 variant => this is 1 running variant
""" % _sep
captured = capsys.readouterr()
with capsys.disabled():
assert captured.out.replace("\r", "") == """hello !
def test_doit():
with patch('sys.exit') as _exit:
with patch('sys.argv'):
doit.run(dodo)
_exit.assert_called_with(0)
def run(filename, **kw):
check_versions(kw.get('verbose', False))
cfg = Config(filename, **kw)
with cfg.prepare(task_md2pdf) as locals:
sys.argv = [sys.argv[0]]
doit.run(locals)
def main():
doit.run(globals())
