def test_cycle():
d = DAG()
d.add_vertex("a")
d.add_vertex("b")
d.update_vertex("a", predecessors=["b"])
d.add_vertex("c", predecessors=["b"])
d.update_vertex("b", predecessors=["c"], enable_checks=False)
with pytest.raises(DAGError):
d.check()
with pytest.raises(DAGError):
d.get_context("b")
def test_predecessor_with_no_fingerprint(setup_sbx):
actions = DAG()
actions.add_vertex("1")
actions.add_vertex("2.no_fingerprint", predecessors=["1"])
actions.add_vertex("3", predecessors=["2.no_fingerprint"])
actions.add_vertex("4", predecessors=["3"])
# Execute our planned actions for the first time...
r1 = FingerprintWalk(actions)
for uid in ("1", "2.no_fingerprint", "3", "4"):
job = r1.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r1.job_status == {
"1": ReturnValue.success,
"2.no_fingerprint": ReturnValue.success,
"3": ReturnValue.success,
"4": ReturnValue.success,
}
assert r1.requeued == {}
# Re-execute the plan a second time. Because '2.no_fingerprint'
# has no fingerprint, both '2.no_fingerprint', but also the node
# that depends directly on it should be re-executed. Node '4,
# on the other hand, should only be re-executed if Node "3"'s
# fingerprint changed. The way things are set up in this testsuite
# is such that the fingerprint remained the same, so '4' is not
# expected to be re-run.
r2 = FingerprintWalk(actions)
for uid in ("1", "4"):
job = r2.saved_jobs[uid]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
for uid in ("2.no_fingerprint", "3"):
job = r2.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r2.job_status == {
"1": ReturnValue.skip,
"2.no_fingerprint": ReturnValue.success,
"3": ReturnValue.success,
"4": ReturnValue.skip,
}
assert r2.requeued == {}
def test_reverse_dag():
d = DAG()
d.add_vertex("a")
d.add_vertex("b", predecessors=["a"])
d.add_vertex("c", predecessors=["b"])
d.add_vertex("d", predecessors=["c"])
it = DAGIterator(d)
assert [k for k, _ in it] == ["a", "b", "c", "d"]
reverse_d = d.reverse_graph()
reverse_it = DAGIterator(reverse_d)
assert [k for k, _ in reverse_it] == ["d", "c", "b", "a"]
def test_reverse_dag():
d = DAG()
d.add_vertex('a')
d.add_vertex('b', predecessors=['a'])
d.add_vertex('c', predecessors=['b'])
d.add_vertex('d', predecessors=['c'])
it = DAGIterator(d)
assert [k for k, _ in it] == ['a', 'b', 'c', 'd']
reverse_d = d.reverse_graph()
reverse_it = DAGIterator(reverse_d)
assert [k for k, _ in reverse_it] == ['d', 'c', 'b', 'a']
def test_ordering(self):
"""Test that jobs are ordered correctly."""
results = []
def collect(job):
results.append(job.uid)
dag = DAG()
dag.add_vertex("3")
dag.add_vertex("0")
dag.add_vertex("1")
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=1, collect=collect)
s.run(dag)
assert tuple(results) == ("0", "1", "3")
def test_collect_feedback_scheme(self):
"""Collect feedback construction.
Scheme in which if a job predecessor "fails" then job is skipped
In order to do that get_job and collect should have access to
common data. Note that scheduler ensure that these functions
are called sequentially.
"""
class SchedulerContext(object):
def __init__(self):
# Save in results tuples with first element being a bool
# indicating success or failure and the second the job itself
self.results = {}
def get_job(self, uid, data, predecessors, notify_end):
result = NopJob(uid, data, notify_end)
# If any of the predecessor failed skip the job
for k in predecessors:
if not self.results[k][0]:
result.should_skip = True
return result
def collect(self, job):
if job.should_skip:
# Skipped jobs are considered failed
self.results[job.uid] = [False, job]
else:
# Job '2' is always failing
if job.uid == "2":
self.results[job.uid] = [False, job]
else:
self.results[job.uid] = [True, job]
dag = DAG()
dag.add_vertex("1")
dag.add_vertex("2")
dag.add_vertex("3", predecessors=["1", "2"])
dag.add_vertex("4", predecessors=["3"])
c = SchedulerContext()
s = Scheduler(c.get_job, tokens=2, collect=c.collect)
s.run(dag)
assert (
not c.results["2"][1].should_skip and not c.results["2"][0]
), 'job "2" is run and should be marked as failed'
assert c.results["3"][1].should_skip, 'job "3" should be skipped'
assert c.results["4"][1].should_skip, 'job "4" should be skipped'
def test_cycle_detection():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
with pytest.raises(DAGError):
d.update_vertex('b', data='newb', predecessors=['a'])
# Ensure that DAG is still valid and that previous
# update_vertex has no effect
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert data is None
assert result == ['b', 'a']
def test_add_vertex():
d = DAG()
# add_vertex should fail in case a dep does not exist
with pytest.raises(DAGError):
d.add_vertex("a", predecessors=["b"])
# check order of a iteration with simple dependency
d.add_vertex("b")
d.add_vertex("a", predecessors=["b"])
result = []
for vertex_id, _ in d:
result.append(vertex_id)
assert result == ["b", "a"]
# check that add_vertex fails on attempt to add already existing nodde
with pytest.raises(DAGError):
d.add_vertex("a")
# check update with new dependency
d.add_vertex("c")
d.update_vertex("b", predecessors=["c"])
assert d.get_predecessors("b") == frozenset(["c"])
assert d.vertex_predecessors == {
"a": frozenset(["b"]),
"b": frozenset(["c"]),
"c": frozenset([]),
}
result = []
for vertex_id, _ in d:
result.append(vertex_id)
assert result == ["c", "b", "a"]
d.update_vertex("a", data="datafora_")
d.update_vertex("c", data="dataforc_")
result = []
compound_data = ""
for vertex_id, data in d:
if data is not None:
compound_data += data
result.append(vertex_id)
assert result == ["c", "b", "a"]
assert compound_data == "dataforc_datafora_"
def test_add_vertex():
d = DAG()
# add_vertex should fail in case a dep does not exist
with pytest.raises(DAGError):
d.add_vertex('a', predecessors=['b'])
# check order of a iteration with simple dependency
d.add_vertex('b')
d.add_vertex('a', predecessors=['b'])
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert result == ['b', 'a']
# check that add_vertex fails on attempt to add already existing nodde
with pytest.raises(DAGError):
d.add_vertex('a')
# check update with new dependency
d.add_vertex('c')
d.update_vertex('b', predecessors=['c'])
assert d.get_predecessors('b') == frozenset(['c'])
assert d.vertex_predecessors == {
'a': frozenset(['b']),
'b': frozenset(['c']),
'c': frozenset([])
}
result = []
for vertex_id, data in d:
result.append(vertex_id)
assert result == ['c', 'b', 'a']
d.update_vertex('a', data='datafora_')
d.update_vertex('c', data='dataforc_')
result = []
compound_data = ''
for vertex_id, data in d:
if data is not None:
compound_data += data
result.append(vertex_id)
assert result == ['c', 'b', 'a']
assert compound_data == 'dataforc_datafora_'
def test_do_nothing_job(self, walk_class, setup_sbx):
"""Test DAG leading us to create a DoNothingJob object."""
actions = DAG()
actions.add_vertex("1.do-nothing")
actions.add_vertex("2", predecessors=["1.do-nothing"])
c = walk_class(actions)
job = c.saved_jobs["1.do-nothing"]
assert isinstance(job, DoNothingJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
job = c.saved_jobs["2"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert c.job_status == {
"1.do-nothing": ReturnValue.success,
"2": ReturnValue.success,
}
assert c.requeued == {}
# In the situation where we are using fingerprints,
# verify the behavior when re-doing a walk with
# the same DAG.
if walk_class == FingerprintWalk:
r2 = walk_class(actions)
job = r2.saved_jobs["1.do-nothing"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
job = r2.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
assert r2.job_status == {
"1.do-nothing": ReturnValue.skip,
"2": ReturnValue.skip,
}
assert r2.requeued == {}
def test_failed_predecessor(self, walk_class, setup_sbx):
"""Simulate the scenarior when a predecessor failed."""
actions = DAG()
actions.add_vertex("1.bad")
actions.add_vertex("2", predecessors=["1.bad"])
c = walk_class(actions)
job = c.saved_jobs["1.bad"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue(1)
job = c.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_fail
assert c.job_status == {
"1.bad": ReturnValue(1),
"2": ReturnValue.force_fail
}
assert c.requeued == {}
# In the situation where we are using fingerprints,
# verify the behavior when re-doing a walk with
# the same DAG.
if walk_class == FingerprintWalk:
r2 = walk_class(actions)
job = r2.saved_jobs["1.bad"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue(1)
job = r2.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_fail
assert r2.job_status == {
"1.bad": ReturnValue(1),
"2": ReturnValue.force_fail,
}
assert r2.requeued == {}
def test_timeout(self):
"""Ensure that jobs are interrupted correctly on timeout."""
results = {}
pytest.importorskip('psutil')
def get_job(uid, data, predecessors, notify_end):
return SleepJob(uid, data, notify_end)
def collect(job):
results[job.uid] = job
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect, job_timeout=2)
s.run(dag)
for k, v in results.items():
assert v.interrupted
def test_pruned_dag():
d = DAG()
d.add_vertex('a')
d.add_vertex('a1')
d.add_vertex('a2')
d.add_vertex('b', predecessors=['a', 'a1'])
d.add_vertex('c', predecessors=['b', 'a2'])
d.add_vertex('d', predecessors=['c'])
d.add_tag('d', 'tag')
def f(dg, node):
if node in ('b', 'c'):
return True
return False
d2 = d.prune(f)
for node, _ in d2:
preds = d2.get_predecessors(node)
if node == 'd':
assert len(preds) == 3
assert 'a' in preds
assert 'a1' in preds
assert 'a2' in preds
assert d2.get_tag('d') == 'tag'
elif node.startswith('a'):
assert len(preds) == 0
else:
assert False, 'invalid node'
# By default prune ensure we keep context unchanged.
# If we add a tag to 'b' (that would be suppressed) then we expect a
# DAGError
d.add_tag('b', 'b_tag')
with pytest.raises(DAGError):
d.prune(f)
# In that last test, check that no error is raised but that tag are
# preserved
d4 = d.prune(f, preserve_context=False)
assert d4.get_tag('d') == 'tag'
def test_pruned_dag():
d = DAG()
d.add_vertex("a")
d.add_vertex("a1")
d.add_vertex("a2")
d.add_vertex("b", predecessors=["a", "a1"])
d.add_vertex("c", predecessors=["b", "a2"])
d.add_vertex("d", predecessors=["c"])
d.add_tag("d", "tag")
def f(dg, node):
if node in ("b", "c"):
return True
return False
d2 = d.prune(f)
for node, _ in d2:
preds = d2.get_predecessors(node)
if node == "d":
assert len(preds) == 3
assert "a" in preds
assert "a1" in preds
assert "a2" in preds
assert d2.get_tag("d") == "tag"
elif node.startswith("a"):
assert len(preds) == 0
else:
raise AssertionError("invalid node")
# By default prune ensure we keep context unchanged.
# If we add a tag to 'b' (that would be suppressed) then we expect a
# DAGError
d.add_tag("b", "b_tag")
with pytest.raises(DAGError):
d.prune(f)
# In that last test, check that no error is raised but that tag are
# preserved
d4 = d.prune(f, preserve_context=False)
assert d4.get_tag("d") == "tag"
def test_skip(self):
"""Simple example in which all the tests are skipped."""
results = {}
def get_job(uid, data, predecessors, notify_end):
result = NopJob(uid, data, notify_end)
result.should_skip = True
return result
def collect(job):
results[job.uid] = job.timing_info
# This time test with two interdependent jobs
dag = DAG()
dag.add_vertex('1')
dag.add_vertex('2')
s = Scheduler(get_job, tokens=2, collect=collect)
s.run(dag)
# Check start_time end_time to be sure tests have not been run
for k, v in results.items():
assert v.start_time is None
assert v.stop_time is None
def __init__(self, spec_repository, default_env=None):
"""Initialize a new context.
:param spec_repository: an Anod repository
:type spec_repository: e3.anod.AnodSpecRepository
:param default_env: an env that should be considered as the
default for the current context. Mainly useful to simulate
another server context. If None then we assume that the
context if the local server
:type default_env: BaseEnv | None
"""
self.repo = spec_repository
if default_env is None:
self.default_env = BaseEnv()
else:
self.default_env = default_env.copy()
self.tree = DAG()
self.root = Root()
self.add(self.root)
self.cache = {}
self.sources = {}
def test_requeue(self):
"""Requeue test.
Same as previous example except that all tests are requeued
once.
"""
results = {}
def collect(job):
if job.uid not in results:
results[job.uid] = True
return True
else:
return False
# This time test with two interdependent jobs
dag = DAG()
dag.add_vertex("1")
dag.add_vertex("2")
s = Scheduler(Scheduler.simple_provider(NopJob), tokens=2, collect=collect)
s.run(dag)
assert s.max_active_jobs == 2
assert results["1"]
assert results["2"]
def testsuite_main(self, args=None):
"""Main for the main testsuite script.
:param args: command line arguments. If None use sys.argv
:type args: list[str] | None
"""
self.main = Main(platform_args=self.CROSS_SUPPORT)
# Add common options
parser = self.main.argument_parser
parser.add_argument("-o",
"--output-dir",
metavar="DIR",
default="./out",
help="select output dir")
parser.add_argument("-t",
"--temp-dir",
metavar="DIR",
default=Env().tmp_dir)
parser.add_argument(
"--max-consecutive-failures",
default=0,
help="If there are more than N consecutive failures, the testsuite"
" is aborted. If set to 0 (default) then the testsuite will never"
" be stopped")
parser.add_argument(
"--keep-old-output-dir",
default=False,
action="store_true",
help="This is default with this testsuite framework. The option"
" is kept only to keep backward compatibility of invocation with"
" former framework (gnatpython.testdriver)")
parser.add_argument("--disable-cleanup",
dest="enable_cleanup",
action="store_false",
default=True,
help="disable cleanup of working space")
parser.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
metavar="N",
default=Env().build.cpu.cores,
help="Specify the number of jobs to run simultaneously")
parser.add_argument(
"--show-error-output",
action="store_true",
help="When testcases fail, display their output. This is for"
" convenience for interactive use.")
parser.add_argument(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.")
parser.add_argument('sublist',
metavar='tests',
nargs='*',
default=[],
help='test')
# Add user defined options
self.add_options()
# parse options
self.main.parse_args(args)
self.env = BaseEnv.from_env()
self.env.root_dir = self.root_dir
self.env.test_dir = self.test_dir
# At this stage compute commonly used paths
# Keep the working dir as short as possible, to avoid the risk
# of having a path that's too long (a problem often seen on
# Windows, or when using WRS tools that have their own max path
# limitations).
# Note that we do make sure that working_dir is an absolute
# path, as we are likely to be changing directories when
# running each test. A relative path would no longer work
# under those circumstances.
d = os.path.abspath(self.main.args.output_dir)
self.output_dir = os.path.join(d, 'new')
self.old_output_dir = os.path.join(d, 'old')
if not os.path.isdir(self.main.args.temp_dir):
logging.critical("temp dir '%s' does not exist",
self.main.args.temp_dir)
return 1
self.working_dir = tempfile.mkdtemp(
'', 'tmp', os.path.abspath(self.main.args.temp_dir))
# Create the new output directory that will hold the results
self.setup_result_dir()
# Store in global env: target information and common paths
self.env.output_dir = self.output_dir
self.env.working_dir = self.working_dir
self.env.options = self.main.args
# User specific startup
self.tear_up()
# Retrieve the list of test
self.test_list = self.get_test_list(self.main.args.sublist)
# Launch the mainloop
self.total_test = len(self.test_list)
self.run_test = 0
self.scheduler = Scheduler(job_provider=self.job_factory,
collect=self.collect_result,
tokens=self.main.args.jobs)
actions = DAG()
for test in self.test_list:
self.parse_test(actions, test)
with open(os.path.join(self.output_dir, 'tests.dot'), 'wb') as fd:
fd.write(actions.as_dot())
self.scheduler.run(actions)
self.dump_testsuite_result()
# Clean everything
self.tear_down()
return 0
def test_dot():
d = DAG()
d.add_vertex('a')
d.add_vertex('b', predecessors=['a'])
assert '"b" -> "a"' in d.as_dot()
def test_dag_str():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
assert str(d)
def test_dag_len():
d = DAG()
d.add_vertex('a')
d.add_vertex('b')
d.update_vertex('a', predecessors=['b'])
assert len(d) == 2
def test_tagged_dag():
r"""Test add_tag/get_tag/get_context.
With the following DAG::
A
/ \
B C*
/ \ /
D* E
/ \ / \
F G H*
"""
d = DAG()
d.add_vertex("a")
d.add_vertex("b", predecessors=["a"])
d.add_vertex("c", predecessors=["a"])
d.add_vertex("d", predecessors=["b"])
d.add_vertex("e", predecessors=["b", "c"])
d.add_vertex("f", predecessors=["d"])
d.add_vertex("g", predecessors=["d", "e"])
d.add_vertex("h", predecessors=["e"])
d.add_tag("c", data="tagc")
d.add_tag("d", data="tagd")
d.add_tag("h", data="tagh")
assert d.get_tag("a") is None
assert d.get_tag("b") is None
assert d.get_tag("c") == "tagc"
assert d.get_tag("e") is None
assert d.get_tag("h") == "tagh"
assert d.get_context("d") == [(0, "d", "tagd")]
assert d.get_context("g") == [(1, "d", "tagd"), (2, "c", "tagc")]
assert d.get_context("f") == [(1, "d", "tagd")]
assert d.get_context("b") == []
assert d.get_context("a") == []
assert d.get_context("c") == [(0, "c", "tagc")]
assert d.get_context("e") == [(1, "c", "tagc")]
assert d.get_context("h") == [(0, "h", "tagh")]
assert d.get_context("e", reverse_order=True) == [(1, "h", "tagh")]
assert d.get_context("h", reverse_order=True) == [(0, "h", "tagh")]
assert d.get_context("a", reverse_order=True) == [
(1, "c", "tagc"),
(2, "d", "tagd"),
(3, "h", "tagh"),
]
assert d.get_context("a", reverse_order=True) == [
(1, "c", "tagc"),
(2, "d", "tagd"),
(3, "h", "tagh"),
]
assert d.get_context(vertex_id="a", max_distance=2,
reverse_order=True) == [
(1, "c", "tagc"),
(2, "d", "tagd"),
]
assert d.get_context(vertex_id="a", max_element=2, reverse_order=True) == [
(1, "c", "tagc"),
(2, "d", "tagd"),
]
def test_dot():
d = DAG()
d.add_vertex("a")
d.add_vertex("b", predecessors=["a"])
assert '"b" -> "a"' in d.as_dot()
def test_dry_run(setup_sbx):
"""Simulate the use actions with "dry run" behavior."""
actions = DAG()
actions.add_vertex("1")
actions.add_vertex("2", predecessors=["1"])
# First run in dry-run mode. Both actions are turned into
# empty jobs with a force_skip status.
r1 = FingerprintWalkDryRun(actions)
job = r1.saved_jobs["1"]
assert isinstance(job, DryRunJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_skip
job = r1.saved_jobs["2"]
assert isinstance(job, DryRunJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_skip
assert r1.job_status == {"1": ReturnValue.force_skip, "2": ReturnValue.force_skip}
assert r1.requeued == {}
# Try it again in dry-mode; we should get the same result.
r2 = FingerprintWalkDryRun(actions)
job = r2.saved_jobs["1"]
assert isinstance(job, DryRunJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_skip
job = r2.saved_jobs["2"]
assert isinstance(job, DryRunJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_skip
assert r2.job_status == {"1": ReturnValue.force_skip, "2": ReturnValue.force_skip}
assert r2.requeued == {}
# Now, get action '1' done in normal (non-dry-run) mode.
one_only = DAG()
one_only.add_vertex("1")
r3 = FingerprintWalk(one_only)
job = r3.saved_jobs["1"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r3.job_status == {"1": ReturnValue.success}
assert r3.requeued == {}
# Try again the original plam in dry-run mode.
#
# This time, since '1' has been actually completed in a previous
# run, action '1' should be skipped, and action '2' should be
# a dry-run job...
r4 = FingerprintWalkDryRun(actions)
job = r4.saved_jobs["1"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
job = r4.saved_jobs["2"]
assert isinstance(job, DryRunJob)
assert job.should_skip is True
assert job.status == ReturnValue.force_skip
assert r4.job_status == {"1": ReturnValue.skip, "2": ReturnValue.force_skip}
assert r4.requeued == {}
# Run it again, this time in normal (non-dry-run) mode.
#
# This time, action '2' gets scheduled for real...
r5 = FingerprintWalk(actions)
job = r5.saved_jobs["1"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
job = r5.saved_jobs["2"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r5.job_status == {"1": ReturnValue.skip, "2": ReturnValue.success}
assert r5.requeued == {}
# One more time, in dry-run mode again.
#
# There is nothing to be done, so the results should be
# both actions are skipped.
r6 = FingerprintWalkDryRun(actions)
job = r6.saved_jobs["1"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
job = r6.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
assert r6.job_status == {"1": ReturnValue.skip, "2": ReturnValue.skip}
assert r6.requeued == {}
def test_computing_fingerprint_after_job_done(setup_sbx):
"""Test case where the fingerprint for one job has to be computed late."""
download_uid = DOWNLOAD_JOB_UID_PREFIX + "fingerprint_after_job"
actions = DAG()
actions.add_vertex(download_uid)
actions.add_vertex("2", predecessors=[download_uid])
# Create the contents of the file that the download_uid job
# will be "downloading" from the store.
with open(source_store_fullpath(download_uid), "w") as f:
f.write("Hello world")
# Now, execute the plan for the first time.
# All actions should get executed.
r1 = FingerprintWalk(actions)
for uid in (download_uid, "2"):
job = r1.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r1.job_status == {
download_uid: ReturnValue.success,
"2": ReturnValue.success,
}
assert r1.requeued == {}
# Now, rerun the plan.
#
# Because we set things up so that the fingerprint of the download_uid
# job cannot be computed before the job is executed, we should see
# that job being rerun (despite the fact that, in the end, the file
# it downloads is the same). However, because the source it downloads
# hasn't changed, job '2' should be skipped.
r2 = FingerprintWalk(actions)
job = r2.saved_jobs[download_uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
job = r2.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
assert r2.job_status == {download_uid: ReturnValue.success, "2": ReturnValue.skip}
assert r2.requeued == {}
# Simulate the case where we re-run the plan after the source
# being downloaded has changed. This time, we expect job '2'
# to be re-executed.
with open(source_store_fullpath(download_uid), "w") as f:
f.write("Hello brave new world")
r3 = FingerprintWalk(actions)
for uid in (download_uid, "2"):
job = r3.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r3.job_status == {
download_uid: ReturnValue.success,
"2": ReturnValue.success,
}
assert r3.requeued == {}
# One more time, just for kicks, where we re-execute the plan
# where the file being downloaded hasn't changed. Just to make
# things slightly different, we'll remove the source file
# already downloaded. It shouldn't prevent us from realizing
# that the sources have not change, and so '2' can be skipped.
rm(source_fullpath(download_uid))
assert not os.path.exists(source_fullpath(download_uid))
r4 = FingerprintWalk(actions)
job = r4.saved_jobs[download_uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
job = r4.saved_jobs["2"]
assert isinstance(job, EmptyJob)
assert job.should_skip is True
assert job.status == ReturnValue.skip
assert r4.job_status == {download_uid: ReturnValue.success, "2": ReturnValue.skip}
assert r4.requeued == {}
def test_dag_str():
d = DAG()
d.add_vertex("a")
d.add_vertex("b")
d.update_vertex("a", predecessors=["b"])
assert str(d)
def test_corrupted_fingerprint(setup_sbx):
"""Test the case where a fingerprint somehow got corrupted."""
actions = DAG()
actions.add_vertex("1")
actions.add_vertex("2", predecessors=["1"])
actions.add_vertex("3")
actions.add_vertex("4", predecessors=["2", "3"])
actions.add_vertex("5", predecessors=["4"])
actions.add_vertex("6")
# Now, execute the plan a first time; everything should be run
# and finish succesfullly.
r1 = FingerprintWalk(actions)
for uid in ("1", "2", "3", "4", "5", "6"):
job = r1.saved_jobs[uid]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
assert r1.job_status == {
"1": ReturnValue.success,
"2": ReturnValue.success,
"3": ReturnValue.success,
"4": ReturnValue.success,
"5": ReturnValue.success,
"6": ReturnValue.success,
}
assert r1.requeued == {}
# Now, corrupt the fingerprint of node '3', and then rerun
# the scheduler... We expect the following:
# - The scheduler does _not_ crash ;-)
# - The fingerprint of node '3' gets discarded, and as a result
# it should be re-run again.
# - Since nothing changed in node "3"'s predecessors, the end
# result for node '3' should be the same, which means
# its fingerprint should be the same as before the corruption.
# As a result of that, nodes '4' and '5', which directly
# or indirectly depend on node '3', do not need to be rerun.
with open(r1.fingerprint_filename("3"), "w") as f:
f.write("{")
r2 = FingerprintWalk(actions)
job = r2.saved_jobs["3"]
assert isinstance(job, ControlledJob)
assert job.should_skip is False
assert job.status == ReturnValue.success
for uid in ("1", "2", "4", "5", "6"):
job = r2.saved_jobs[uid]
assert isinstance(job, EmptyJob), "job %s should be EmptyJob" % uid
assert job.should_skip is True
assert job.status == ReturnValue.skip
# Verify also that the fingerprint corruption is gone.
f3 = Fingerprint.load_from_file(r2.fingerprint_filename("3"))
assert isinstance(f3, Fingerprint)
def testsuite_main(self, args: Optional[List[str]] = None) -> int:
"""Main for the main testsuite script.
:param args: Command line arguments. If None, use `sys.argv`.
:return: The testsuite status code (0 for success, a positive for
failure).
"""
self.main = Main(platform_args=True)
# Add common options
parser = self.main.argument_parser
temp_group = parser.add_argument_group(
title="temporaries handling arguments")
temp_group.add_argument("-t",
"--temp-dir",
metavar="DIR",
default=Env().tmp_dir)
temp_group.add_argument(
"--no-random-temp-subdir",
dest="random_temp_subdir",
action="store_false",
help="Disable the creation of a random subdirectory in the"
" temporary directory. Use this when you know that you have"
" exclusive access to the temporary directory (needed in order to"
" avoid name clashes there) to get a deterministic path for"
" testsuite temporaries.")
temp_group.add_argument(
"-d",
"--dev-temp",
metavar="DIR",
nargs="?",
default=None,
const="tmp",
help="Convenience shortcut for dev setups: forces `-t DIR"
" --no-random-temp-subdir --cleanup-mode=none` and cleans up `DIR`"
' first. If no directory is provided, use the local "tmp"'
" directory.")
cleanup_mode_map = enum_to_cmdline_args_map(CleanupMode)
temp_group.add_argument(
"--cleanup-mode",
choices=list(cleanup_mode_map),
help="Control the cleanup of working spaces.\n" +
"\n".join(f"{name}: {CleanupMode.descriptions()[value]}"
for name, value in cleanup_mode_map.items()))
temp_group.add_argument(
"--disable-cleanup",
action="store_true",
help="Disable cleanup of working spaces. This option is deprecated"
" and will disappear in a future version of e3-testsuite. Please"
" use --cleanup-mode instead.")
output_group = parser.add_argument_group(
title="results output arguments")
output_group.add_argument(
"-o",
"--output-dir",
metavar="DIR",
default="./out",
help="Select the output directory, where test results are to be"
" stored (default: './out'). If --old-output-dir=DIR2 is passed,"
" the new results are stored in DIR while DIR2 contains results"
" from a previous run. Otherwise, the new results are stored in"
" DIR/new/ while the old ones are stored in DIR/old. In both"
" cases, the testsuite cleans the directory for new results"
" first.",
)
output_group.add_argument(
"--old-output-dir",
metavar="DIR",
help="Select the old output directory, for baseline comparison."
" See --output-dir.",
)
output_group.add_argument(
"--rotate-output-dirs",
default=False,
action="store_true",
help="Rotate testsuite results: move the new results directory to"
" the old results one before running testcases (this removes the"
" old results directory first). If not passed, we just remove the"
" new results directory before running testcases (i.e. just ignore"
" the old results directory).",
)
output_group.add_argument(
"--show-error-output",
"-E",
action="store_true",
help="When testcases fail, display their output. This is for"
" convenience for interactive use.",
)
output_group.add_argument(
"--show-time-info",
action="store_true",
help="Display time information for test results, if available",
)
output_group.add_argument(
"--xunit-output",
dest="xunit_output",
metavar="FILE",
help="Output testsuite report to the given file in the standard"
" XUnit XML format. This is useful to display results in"
" continuous build systems such as Jenkins.",
)
output_group.add_argument(
"--gaia-output",
action="store_true",
help="Output a GAIA-compatible testsuite report next to the YAML"
" report.",
)
output_group.add_argument(
"--status-update-interval",
default=1.0,
type=float,
help="Minimum number of seconds between status file updates. The"
" more often we update this file, the more often one will read"
" garbage.")
auto_gen_default = ("enabled"
if self.auto_generate_text_report else "disabled")
output_group.add_argument(
"--generate-text-report",
action="store_true",
dest="generate_text_report",
default=self.auto_generate_text_report,
help=(
f"When the testsuite completes, generate a 'report' text file"
f" in the output directory ({auto_gen_default} by default)."),
)
output_group.add_argument(
"--no-generate-text-report",
action="store_false",
dest="generate_text_report",
help="Disable the generation of a 'report' text file (see"
"--generate-text-report).",
)
output_group.add_argument(
"--truncate-logs",
"-T",
metavar="N",
type=int,
default=200,
help="When outputs (for instance subprocess outputs) exceed 2*N"
" lines, only include the first and last N lines in logs. This is"
" necessary when storage for testsuite results have size limits,"
" and the useful information is generally either at the beginning"
" or the end of such outputs. If 0, never truncate logs.",
)
output_group.add_argument(
"--dump-environ",
dest="dump_environ",
action="store_true",
default=False,
help="Dump all environment variables in a file named environ.sh,"
" located in the output directory (see --output-dir). This"
" file can then be sourced from a Bourne shell to recreate"
" the environement that existed when this testsuite was run"
" to produce a given testsuite report.",
)
exec_group = parser.add_argument_group(
title="execution control arguments")
exec_group.add_argument(
"--max-consecutive-failures",
"-M",
metavar="N",
type=int,
default=self.default_max_consecutive_failures,
help="Number of test failures (FAIL or ERROR) that trigger the"
" abortion of the testuite. If zero, this behavior is disabled. In"
" some cases, aborting the testsuite when there are just too many"
" failures saves time and costs: the software to test/environment"
" is too broken, there is no point to continue running the"
" testsuite.",
)
exec_group.add_argument(
"-j",
"--jobs",
dest="jobs",
type=int,
metavar="N",
default=Env().build.cpu.cores,
help="Specify the number of jobs to run simultaneously",
)
exec_group.add_argument(
"--failure-exit-code",
metavar="N",
type=int,
default=self.default_failure_exit_code,
help="Exit code the testsuite must use when at least one test"
" result shows a failure/error. By default, this is"
f" {self.default_failure_exit_code}. This option is useful when"
" running a testsuite in a continuous integration setup, as this"
" can make the testing process stop when there is a regression.",
)
exec_group.add_argument(
"--force-multiprocessing",
action="store_true",
help="Force the use of subprocesses to execute tests, for"
" debugging purposes. This is normally automatically enabled when"
" both the level of requested parallelism is high enough (to make"
" it profitable regarding the contention of Python's GIL) and no"
" test fragment has dependencies on other fragments. This flag"
" forces the use of multiprocessing even if any of these two"
" conditions is false.")
parser.add_argument("sublist",
metavar="tests",
nargs="*",
default=[],
help="test")
# Add user defined options
self.add_options(parser)
# Parse options
self.main.parse_args(args)
assert self.main.args is not None
# If there is a chance for the logging to end up in a non-tty stream,
# disable colors. If not, be user-friendly and automatically show error
# outputs.
if (self.main.args.log_file or not isatty(sys.stdout)
or not isatty(sys.stderr)):
enable_colors = False
else: # interactive-only
enable_colors = True
self.main.args.show_error_output = True
self.colors = ColorConfig(enable_colors)
self.Fore = self.colors.Fore
self.Style = self.colors.Style
self.env = Env()
self.env.enable_colors = enable_colors
self.env.root_dir = self.root_dir
self.env.test_dir = self.test_dir
# Setup output directories and create an index for the results we are
# going to produce.
self.output_dir: str
self.old_output_dir: Optional[str]
self.setup_result_dirs()
self.report_index = ReportIndex(self.output_dir)
# Set the cleanup mode from command-line arguments
if self.main.args.cleanup_mode is not None:
self.env.cleanup_mode = (
cleanup_mode_map[self.main.args.cleanup_mode])
elif self.main.args.disable_cleanup:
logger.warning(
"--disable-cleanup is deprecated and will disappear in a"
" future version of e3-testsuite. Please use --cleanup-mode"
" instead.")
self.env.cleanup_mode = CleanupMode.NONE
else:
self.env.cleanup_mode = CleanupMode.default()
# Settings for temporary directory creation
temp_dir: str = self.main.args.temp_dir
random_temp_subdir: bool = self.main.args.random_temp_subdir
# The "--dev-temp" option forces several settings
if self.main.args.dev_temp:
self.env.cleanup_mode = CleanupMode.NONE
temp_dir = self.main.args.dev_temp
random_temp_subdir = False
# Now actually setup the temporary directory: make sure we start from a
# clean directory if we use a deterministic directory.
#
# Note that we do make sure that working_dir is an absolute path, as we
# are likely to be changing directories when running each test. A
# relative path would no longer work under those circumstances.
temp_dir = os.path.abspath(temp_dir)
if not random_temp_subdir:
self.working_dir = temp_dir
rm(self.working_dir, recursive=True)
mkdir(self.working_dir)
elif not os.path.isdir(temp_dir):
# If the temp dir is supposed to be randomized, we need to create a
# subdirectory, so check that the parent directory exists first.
logger.critical("temp dir '%s' does not exist", temp_dir)
return 1
else:
self.working_dir = tempfile.mkdtemp("", "tmp", temp_dir)
# Create the exchange directory (to exchange data between the testsuite
# main and the subprocesses running test fragments). Compute the name
# of the file to pass environment data to subprocesses.
self.exchange_dir = os.path.join(self.working_dir, "exchange")
self.env_filename = os.path.join(self.exchange_dir, "_env.bin")
mkdir(self.exchange_dir)
# Make them both available to test fragments
self.env.exchange_dir = self.exchange_dir
self.env.env_filename = self.env_filename
self.gaia_result_files: Dict[str, GAIAResultFiles] = {}
"""Mapping from test names to files for results in the GAIA report."""
# Store in global env: target information and common paths
self.env.output_dir = self.output_dir
self.env.working_dir = self.working_dir
self.env.options = self.main.args
# Create an object to report testsuite execution status to users
from e3.testsuite.running_status import RunningStatus
self.running_status = RunningStatus(
os.path.join(self.output_dir, "status"),
self.main.args.status_update_interval,
)
# User specific startup
self.set_up()
# Retrieve the list of test
self.test_list = self.get_test_list(self.main.args.sublist)
# Create a DAG to constraint the test execution order
dag = DAG()
for parsed_test in self.test_list:
self.add_test(dag, parsed_test)
self.adjust_dag_dependencies(dag)
dag.check()
self.running_status.set_dag(dag)
# Determine whether to use multiple processes for fragment execution
# parallelism.
self.use_multiprocessing = self.compute_use_multiprocessing()
self.env.use_multiprocessing = self.use_multiprocessing
# Record modules lookup path, including for the file corresponding to
# the __main__ module. Subprocesses will need it to have access to the
# same modules.
main_module = sys.modules["__main__"]
self.env.modules_search_path = [
os.path.dirname(os.path.abspath(main_module.__file__))
] + sys.path
# Now that the env is supposed to be complete, dump it for the test
# fragments to pick it up.
self.env.store(self.env_filename)
# For debugging purposes, dump the final DAG to a DOT file
with open(os.path.join(self.output_dir, "tests.dot"), "w") as fd:
fd.write(dag.as_dot())
if self.use_multiprocessing:
self.run_multiprocess_mainloop(dag)
else:
self.run_standard_mainloop(dag)
self.report_index.write()
self.dump_testsuite_result()
if self.main.args.xunit_output:
dump_xunit_report(self, self.main.args.xunit_output)
if self.main.args.gaia_output:
dump_gaia_report(self, self.output_dir, self.gaia_result_files)
# Clean everything
self.tear_down()
# If requested, generate a text report
if self.main.args.generate_text_report:
# Use the previous testsuite results for comparison, if available
old_index = (ReportIndex.read(self.old_output_dir)
if self.old_output_dir else None)
# Include all information, except logs for successful tests, which
# is just too verbose.
with open(os.path.join(self.output_dir, "report"),
"w",
encoding="utf-8") as f:
generate_report(
output_file=f,
new_index=self.report_index,
old_index=old_index,
colors=ColorConfig(colors_enabled=False),
show_all_logs=False,
show_xfail_logs=True,
show_error_output=True,
show_time_info=True,
)
# Return the appropriate status code: 1 when there is a framework
# issue, the failure status code from the --failure-exit-code=N option
# when there is a least one testcase failure, or 0.
statuses = {
s
for s, count in self.report_index.status_counters.items() if count
}
if TestStatus.FAIL in statuses or TestStatus.ERROR in statuses:
return self.main.args.failure_exit_code
else:
return 0
def schedule(self, resolver):
"""Compute a DAG of scheduled actions.
:param resolver: a function that helps the scheduler resolve cases
for which a decision should be taken
:type resolver: (Action, Decision) -> bool
"""
rev = self.tree.reverse_graph()
uploads = []
dag = DAG()
# Retrieve existing tags
dag.tags = self.tree.tags
# Note that schedule perform a pruning on the DAG, thus no cycle can
# be introduced. That's why checks are disabled when creating the
# result graph.
for uid, action in rev:
if uid == "root":
# Root node is always in the final DAG
dag.update_vertex(uid, action, enable_checks=False)
elif isinstance(action, Decision):
# Decision node does not appears in the final DAG but we need
# to apply the triggers based on the current list of scheduled
# actions.
action.apply_triggers(dag)
elif isinstance(action, Upload):
uploads.append((action, self.tree.get_predecessors(uid)))
else:
# Compute the list of successors for the current node (i.e:
# predecessors in the reversed graph). Ignore Upload
# nodes as they will be processed only once the scheduling
# is done.
preds = list([
k for k in rev.get_predecessors(uid)
if not isinstance(rev[k], Upload)
])
if len(preds) == 1 and isinstance(rev[preds[0]], Decision):
decision = rev[preds[0]]
# The current node addition is driven by a decision
# First check that the parent of the decision is
# scheduled. If not discard the item.
if decision.initiator not in dag:
continue
# Now check decision made. If the decision cannot be made
# delegate to the resolve function.
choice = decision.get_decision()
if choice == uid:
dag.update_vertex(uid, action, enable_checks=False)
dag.update_vertex(decision.initiator,
predecessors=[uid],
enable_checks=False)
elif choice is None:
# delegate to resolver
try:
if resolver(action, decision):
dag.update_vertex(uid,
action,
enable_checks=False)
dag.update_vertex(
decision.initiator,
predecessors=[uid],
enable_checks=False,
)
except SchedulingError as e:
# In order to help the analysis of a scheduling
# error compute the explicit initiators of that
# action
dag.update_vertex(uid, action, enable_checks=False)
dag.update_vertex(
decision.initiator,
predecessors=[action.uid],
enable_checks=False,
)
rev_graph = dag.reverse_graph()
# Initiators are explicit actions (connected to
# 'root') that are in the closure of the failing
# node.
initiators = [
iuid for iuid in rev_graph.get_closure(uid)
if "root" in rev_graph.get_predecessors(iuid)
]
raise SchedulingError(e.messages,
uid=uid,
initiators=initiators)
else:
# An action is scheduled only if one of its successors is
# scheduled.
successors = [k for k in preds if k in dag]
if successors:
dag.update_vertex(uid, action, enable_checks=False)
for a in successors:
dag.update_vertex(a,
predecessors=[uid],
enable_checks=False)
# Handle Upload nodes. Add the node only if all predecessors
# are scheduled.
for action, predecessors in uploads:
if len([p for p in predecessors if p not in dag]) == 0:
dag.update_vertex(action.uid,
action,
predecessors=predecessors,
enable_checks=False)
# connect upload to the root node
dag.update_vertex("root",
predecessors=[action.uid],
enable_checks=False)
return dag
def test_dag_len():
d = DAG()
d.add_vertex("a")
d.add_vertex("b")
d.update_vertex("a", predecessors=["b"])
assert len(d) == 2