def nested():
j11, j12, j13, j14, j15 = [job(i) for i in range(11, 16)]
s2 = Scheduler(Sequence(j11, j12, j13), label="nested internal")
j12.requires(j14)
j13.requires(j15)
j1, j2, j3, j4, j5 = [job(i) for i in range(1, 6)]
s1 = Scheduler(Sequence(j1, s2, j3), label="nested top")
j1.requires(j4)
j1.requires(j11)
s2.requires(j13)
# j2 not included in sched, untouched
j2.requires(j1)
self.assertEqual(len(j12.required), 2)
self.assertEqual(len(j13.required), 2)
self.assertEqual(len(j1.required), 2)
self.assertEqual(len(s2.required), 2)
self.assertEqual(len(j3.required), 1)
s1.sanitize()
self.assertEqual(len(j12.required), 1)
self.assertEqual(len(j13.required), 1)
self.assertEqual(len(j1.required), 0)
self.assertEqual(len(s2.required), 1)
self.assertEqual(len(j3.required), 1)
def test_nested_cycles(self):
watch = Watch()
def job(i):
return Job(co_print_sleep(watch, .2, f"job {i}"),
label=f"job{i}")
js1, js2, js3 = [job(i) for i in range(11, 14)]
s2 = Scheduler(Sequence(js1, js2, js3))
j1, j3 = job(1), job(3)
s1 = Scheduler(Sequence(j1, s2, j3))
self.assertTrue(s1.check_cycles())
# create cycle in subgraph
js1.requires(js3)
self.assertFalse(s1.check_cycles())
# restore in OK state
js1.requires(js3, remove=True)
self.assertTrue(s1.check_cycles())
# add cycle in toplevel
j1.requires(j3)
self.assertFalse(s1.check_cycles())
# restore in OK state
j1.requires(j3, remove=True)
self.assertTrue(s1.check_cycles())
# add one level down
s3 = Scheduler()
jss1, jss2, jss3 = [job(i) for i in range(111, 114)]
Sequence(jss1, jss2, jss3, scheduler=s3)
# surgery in s2; no cycles
s2.remove(js2)
s2.sanitize()
s2.add(s3)
s3.requires(js1)
js3.requires(s3)
self.assertTrue(s1.check_cycles())
# add cycle in s3
js1.requires(js3)
self.assertFalse(s1.check_cycles())
def prepare_testbed_scheduler( # pylint: disable=r0913, r0914
gateway: SshNode,
load_flag: bool,
experiment_scheduler: Scheduler,
images_mapping,
nodes_left_alone=None,
sdrs_left_alone=None,
phones_left_alone=None,
verbose_jobs=False):
"""
This function is designed as a standard way for experiments to warm up.
Experimenters only need to write a scheduler that defines the behaviour
of their core experiment, this function will add additional steps that
take care of a) checking for a valid lease, b) load images on nodes, and
c) turn off unused devices.
It is generally desirable to write an experiment script that has a
`--load/-l` boolean flag; typically, one would use the ``--load`` flag the
first time that an experiment is launched during a given timeslot, while
subsequent calls won't. That is the purpose of the ``load_flag`` below;
when set to False, only step a) is performed, otherwise the resulting
scheduler will go for the full monty.
Parameters:
gateway_sshnode: the ssh handle to the gateway
load_flag(bool): if not set, only the lease is checked
experiment_scheduler: core scheduler for the experiment
images_mapping: a dictionary that specifies images to be loaded on nodes;
see examples below
nodes_left_alone: a list of node numbers that should be left intact,
neither loaded nor turned off;
phones_left_alone: a list of node numbers that should be left intact,
i.e. not switched to airplane mode.
Return :
The overall scheduler where the input ``experiment_scheduler`` is embedded.
Examples:
Specify a mapping like the following::
images_mapping = { "ubuntu" : [1, 4, 5], "gnuradio": [16]}
Note that the format for ``images_mapping``, is flexible;
if only one node is to be loaded, the iterable level is optional;
also each node can be specified as an ``int``, a ``bytes``, a ``str``,
in which case non numeric characters are ignored. So this is a legitimate
requirement as well::
images_mapping = { 'openair-cn': 12 + 4,
'openair-enodeb': ('fit32',),
'ubuntu': {12, 'reboot1', '004',
'you-get-the-picture-34'}
}
"""
# handle default mutable args
nodes_left_alone = set(nodes_left_alone) if nodes_left_alone else set()
sdrs_left_alone = set(sdrs_left_alone) if sdrs_left_alone else set()
phones_left_alone = set(phones_left_alone) if phones_left_alone else set()
scheduler = Scheduler(label="Preparation")
check_lease = SshJob(
scheduler=scheduler,
node=gateway,
verbose=verbose_jobs,
label="Check lease {}".format(gateway.username),
command=Run("rhubarbe leases --check", label="rlease"),
)
# if no image loading is requested, we're done here
if not load_flag:
scheduler.add(experiment_scheduler)
experiment_scheduler.requires(check_lease)
return scheduler
# otherwise, we want to do in parallel
# (*) as many image-loading jobs as we have entries in images_mapping
# (*) one job to turn off phones, nodes and usrps
# as parallelizing brings no speed up at all
# todo ideally we could also probe the testbed to figure out which nodes
# are currently unavailable, and let them alone as well; but well.
# the jobs that we need to wait for before going on with the real stuff
octopus = []
loaded_nodes = set()
for image, nodes in images_mapping.items():
# let's be as flexible as possible
# (1) empty node list should be fine
if not nodes:
continue
# (2) atomic types should be allowed
if isinstance(nodes, (int, str, bytes)):
nodes = [nodes]
# (3) accept all forms of inputs
nodes = {r2lab_id(node) for node in nodes}
duplicates = loaded_nodes & nodes
if duplicates:
print("WARNING - nodes in {} have been assigned several images".
format(duplicates))
loaded_nodes.update(nodes)
# for there on we need strings
node_args = " ".join(str(node) for node in nodes)
octopus.append(
SshJob(
gateway,
scheduler=scheduler,
required=check_lease,
label=("loading {} on {}".format(image, node_args)),
commands=[
Run("rhubarbe load -i {} {}".format(image, node_args)),
Run("rhubarbe wait {}".format(node_args)),
],
verbose=verbose_jobs,
))
### turn off stuff
# nodes
dont_off_nodes = nodes_left_alone | loaded_nodes
# do turn off usrp device even on loaded nodes
dont_off_sdrs = sdrs_left_alone
# phones - there's no equivalent of --all ~ notation with phones
off_phones = set(range(1, PHONES+1)) \
- {r2lab_id(ph) for ph in phones_left_alone}
r2lab_includes = [
find_local_embedded_script(x) for x in ("faraday.sh", "r2labutils.sh")
]
if off_phones:
octopus.append(
SshJob(gateway,
scheduler=scheduler,
required=check_lease,
critical=False,
commands=[
RunScript(find_local_embedded_script("faraday.sh"),
"macphone{}".format(phone),
"r2lab-embedded/shell/macphone.sh",
"phone-off",
label="turn off phone {}".format(phone),
includes=r2lab_includes)
for phone in off_phones
],
verbose=verbose_jobs))
octopus.append(
SshJob(
gateway,
scheduler=scheduler,
required=check_lease,
label="Turn off unused devices",
commands=[
Run(_rhubarbe_command(verb="off", left_alone=dont_off_nodes)),
Run(_rhubarbe_command(verb="usrpoff",
left_alone=dont_off_sdrs)),
],
verbose=verbose_jobs,
))
# embed experiment scheduler
experiment_scheduler.requires(octopus)
scheduler.add(experiment_scheduler)
return scheduler
