def test_creation_scheduler(self): sched = Scheduler() s = Seq(J(sl(1)), J(sl(2)), scheduler=sched) j = J(sl(3), required=s, scheduler=sched) # make sure that jobs appended in the sequence # even later on are also added to the scheduler s.append(J(sl(.5))) self.assertEqual(len(sched.jobs), 4) self.assertTrue(sched.rain_check()) self.assertTrue(sched.orchestrate())
def test_cycle(self): """a simple loop with 3 jobs - cannot handle that""" a1, a2, a3 = J(sl(1.1)), J(sl(1.2)), J(sl(1.3)) a1.requires(a2) a2.requires(a3) a3.requires(a1) sched = Scheduler(a1, a2, a3) # these lines seem to trigger a nasty message about a coro not being # waited self.assertFalse(sched.rain_check())
def test_simple(self): """a simple topology, that should work""" jobs = SLJ(0.1), SLJ(0.2), SLJ(0.3), SLJ( 0.4), SLJ(0.5), J(sl(0.6)), J(sl(0.7)) a1, a2, a3, a4, a5, a6, a7 = jobs a4.requires(a1, a2, a3) a5.requires(a4) a6.requires(a4) a7.requires(a5) a7.requires(a6) sched = Scheduler(*jobs) list_sep(sched, sep + "LIST BEFORE") self.assertTrue(sched.rain_check()) self.assertTrue(sched.orchestrate(loop=asyncio.get_event_loop())) for j in jobs: self.assertFalse(j.raised_exception()) list_sep(sched, sep + "LIST AFTER") print(sep + "DEBRIEF") sched.debrief()
def run( *, # the pieces to use slice, hss, epc, enb, phones, e3372_ues, oai_ues, gnuradios, e3372_ue_xterms, oai_ue_xterms, gnuradio_xterms, # boolean flags load_nodes, skip_reset_usb, oscillo, # the images to load image_gw, image_enb, image_oai_ue, image_e3372_ue, image_gnuradio, # miscell n_rb, verbose, dry_run): """ ########## # 3 methods to get nodes ready # (*) load images # (*) reset nodes that are known to have the right image # (*) do nothing, proceed to experiment expects e.g. * slice : s.t like [email protected] * hss : 04 * epc : 03 * enb : 23 * phones: list of indices of phones to use * e3372_ues : list of nodes to use as a UE using e3372 * oai_ues : list of nodes to use as a UE using OAI * gnuradios : list of nodes to load with a gnuradio image * image_* : the name of the images to load on the various nodes Plus * load_nodes: whether to load images or not - in which case image_gw, image_enb and image_* are used to tell the image names * skip_reset_usb : the USRP board will be reset as well unless this is set """ # what argparse knows as a slice actually is a gateway (user + host) gwuser, gwhost = r2lab_parse_slice(slice) gwnode = SshNode(hostname=gwhost, username=gwuser, formatter=TimeColonFormatter(verbose=verbose), debug=verbose) hostnames = hssname, epcname, enbname = [ r2lab_hostname(x) for x in (hss, epc, enb) ] optional_ids = e3372_ues + oai_ues + gnuradios + \ e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms hssnode, epcnode, enbnode = [ SshNode(gateway=gwnode, hostname=hostname, username='******', formatter=TimeColonFormatter(verbose=verbose), debug=verbose) for hostname in hostnames ] sched = Scheduler(verbose=verbose) ########## preparation job_check_for_lease = SshJob( node=gwnode, command=["rhubarbe", "leases", "--check"], label="check we have a current lease", scheduler=sched, ) # turn off all nodes turn_off_command = ["rhubarbe", "off", "-a"] # except our 3 nodes and the optional ones turn_off_command += [ "~{}".format(x) for x in [hss, epc, enb] + optional_ids ] # only do the turn-off thing if load_nodes if load_nodes: job_off_nodes = SshJob( node=gwnode, # switch off all nodes but the ones we use command=turn_off_command, label="turn off unused nodes", required=job_check_for_lease, scheduler=sched, ) # actually run this in the gateway, not on the macphone # the ssh keys are stored in the gateway and we do not yet have # the tools to leverage such remote keys job_stop_phones = [ SshJob( node=gwnode, command=RunScript( # script find_local_embedded_script("faraday.sh"), # arguments "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-off", # options includes=includes), label="put phone{} in airplane mode".format(id), required=job_check_for_lease, scheduler=sched, ) for id in phones ] ########## prepare the image-loading phase # this will be a dict of items imagename -> ids to_load = defaultdict(list) to_load[image_gw] += [hss, epc] to_load[image_enb] += [enb] if e3372_ues: to_load[image_e3372_ue] += e3372_ues if e3372_ue_xterms: to_load[image_e3372_ue] += e3372_ue_xterms if oai_ues: to_load[image_oai_ue] += oai_ues if oai_ue_xterms: to_load[image_oai_ue] += oai_ue_xterms if gnuradios: to_load[image_gnuradio] += gnuradios if gnuradio_xterms: to_load[image_gnuradio] += gnuradio_xterms prep_job_by_node = {} for image, nodes in to_load.items(): commands = [] if load_nodes: commands.append(Run("rhubarbe", "usrpoff", *nodes)) commands.append(Run("rhubarbe", "load", "-i", image, *nodes)) commands.append(Run("rhubarbe", "usrpon", *nodes)) # always do this commands.append(Run("rhubarbe", "wait", "-t", 120, *nodes)) job = SshJob( node=gwnode, commands=commands, label="Prepare node(s) {}".format(nodes), required=job_check_for_lease, scheduler=sched, ) for node in nodes: prep_job_by_node[node] = job # start services job_service_hss = SshJob( node=hssnode, command=RunScript(find_local_embedded_script("oai-hss.sh"), "run-hss", epc, includes=includes), label="start HSS service", required=prep_job_by_node[hss], scheduler=sched, ) delay = 15 job_service_epc = SshJob( node=epcnode, commands=[ Run("echo giving HSS a headstart {delay}s to warm up; sleep {delay}" .format(delay=delay)), RunScript(find_local_embedded_script("oai-epc.sh"), "run-epc", hss, includes=includes), ], label="start EPC services", required=prep_job_by_node[epc], scheduler=sched, ) ########## enodeb job_warm_enb = SshJob( node=enbnode, commands=[ RunScript(find_local_embedded_script("oai-enb.sh"), "warm-enb", epc, n_rb, not skip_reset_usb, includes=includes), ], label="Warm eNB", required=prep_job_by_node[enb], scheduler=sched, ) enb_requirements = (job_warm_enb, job_service_hss, job_service_epc) # wait for everything to be ready, and add an extra grace delay grace = 30 if load_nodes else 10 grace_delay = SshJob( node = LocalNode(formatter=TimeColonFormatter()), command = "echo Allowing grace of {grace} seconds; sleep {grace}"\ .format(grace=grace), required = enb_requirements, scheduler = sched, ) # start services job_service_enb = SshJob( node=enbnode, # run-enb expects the id of the epc as a parameter # n_rb means number of resource blocks for DL, set to either 25 or 50. commands=[ RunScript(find_local_embedded_script("oai-enb.sh"), "run-enb", oscillo, includes=includes, x11=oscillo), ], label="start softmodem on eNB", required=grace_delay, scheduler=sched, ) ########## run experiment per se # Manage phone(s) # this starts at the same time as the eNB, but some # headstart is needed so that eNB actually is ready to serve delay = 12 msg = "wait for {delay}s for enodeb to start up"\ .format(delay=delay) wait_command = "echo {msg}; sleep {delay}".format(msg=msg, delay=delay) job_start_phones = [ SshJob( node=gwnode, commands=[ Run(wait_command), RunScript(find_local_embedded_script("faraday.sh"), "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-on", includes=includes), RunScript(find_local_embedded_script("faraday.sh"), "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-start-app", includes=includes), ], label="start Nexus phone and speedtest app", required=grace_delay, scheduler=sched, ) for id in phones ] job_ping_phones_from_epc = [ SshJob( node=epcnode, commands=[ Run("sleep 10"), Run("ping -c 100 -s 100 -i .05 172.16.0.{ip} &> /root/ping-phone" .format(ip=id + 1)), ], label="ping Nexus phone from EPC", critical=False, required=job_start_phones, ) for id in phones ] ########## xterm nodes colors = ["wheat", "gray", "white", "darkolivegreen"] xterms = e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms for xterm, color in zip(xterms, itertools.cycle(colors)): xterm_node = SshNode(gateway=gwnode, hostname=r2lab_hostname(xterm), username='******', formatter=TimeColonFormatter(verbose=verbose), debug=verbose) SshJob( node=xterm_node, command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*" " -bg {} -geometry 90x10".format(color), x11=True), label="xterm on node {}".format(xterm_node.hostname), required=prep_job_by_node[xterm], scheduler=sched, # don't set forever; if we do, then these xterms get killed # when all other tasks have completed # forever = True, ) # # remove dangling requirements - if any - should not be needed but won't hurt either sched.sanitize() print(20 * "*", "nodes usage summary") if load_nodes: for image, nodes in to_load.items(): for node in nodes: print("node {node} : {image}".format(node=node, image=image)) else: print("NODES ARE USED AS IS (no image loaded, no reset)") print(10 * "*", "phones usage summary") if phones: for phone in phones: print("Using phone{phone}".format(phone=phone)) else: print("No phone involved") sched.rain_check() # Update the .dot and .png file for illustration purposes if verbose or dry_run: sched.list() name = "scenario-load" if load_nodes else \ "scenario" sched.export_as_dotfile("{name}.dot".format(name=name)) os.system("dot -Tpng {name}.dot -o {name}.png".format(name=name)) print("(Over)wrote {name}.png".format(name=name)) if dry_run: return False if verbose: input('OK ? - press control C to abort ? ') if not sched.orchestrate(): print("RUN KO : {}".format(sched.why())) sched.debrief() return False else: print("RUN OK") return True
def run(slice, hss, epc, enb, extras, load_nodes, image_gw, image_enb, image_extra, reset_nodes, reset_usrp, spawn_xterms, verbose): """ ########## # 3 methods to get nodes ready # (*) load images # (*) reset nodes that are known to have the right image # (*) do nothing, proceed to experiment expects e.g. * slice : s.t like [email protected] * hss : 04 * epc : 03 * enb : 23 * extras : a list of ids that will be loaded with the gnuradio image Plus * load_nodes: whether to load images or not - in which case image_gw, image_enb and image_extra are used to tell the image names * reset_nodes: if load_nodes is false and reset_nodes is true, the nodes are reset - i.e. rebooted * otherwise (both False): do nothing * reset_usrp : if not False, the USRP board won't be reset * spawn_xterms : if set, starts xterm on all extra nodes * image_* : the name of the images to load on the various nodes """ # what argparse knows as a slice actually is a gateway (user + host) gwuser, gwhost = parse_slice(slice) gwnode = SshNode(hostname=gwhost, username=gwuser, formatter=ColonFormatter(verbose=verbose), debug=verbose) hostnames = hssname, epcname, enbname = [ r2lab_hostname(x) for x in (hss, epc, enb) ] extra_hostnames = [r2lab_hostname(x) for x in extras] hssnode, epcnode, enbnode = [ SshNode(gateway=gwnode, hostname=hostname, username='******', formatter=ColonFormatter(verbose=verbose), debug=verbose) for hostname in hostnames ] extra_nodes = [ SshNode(gateway=gwnode, hostname=hostname, username='******', formatter=ColonFormatter(verbose=verbose), debug=verbose) for hostname in extra_hostnames ] ########## preparation job_check_for_lease = SshJob( node=gwnode, command=["rhubarbe", "leases", "--check"], label="check we have a current lease", ) # turn off all nodes turn_off_command = ["rhubarbe", "off", "-a"] # except our 3 nodes and the optional extras turn_off_command += [ "~{}".format(x) for x in [hss, epc, enb] + extras + [20] ] job_off_nodes = SshJob( node=gwnode, # switch off all nodes but the ones we use command=turn_off_command, label="turn off unused nodes", required=job_check_for_lease, ) # actually run this in the gateway, not on the mac # the ssh keys are stored in the gateway and we do not yet have # the tools to leverage such remote keys job_stop_phone = SshJob( node=gwnode, command=RunScript(locate_local_script("faraday.sh"), "macphone", "r2lab/infra/user-env/macphone.sh", "phone-off", includes=includes), label="stop phone", required=job_check_for_lease, ) jobs_prepare = [job_check_for_lease, job_stop_phone] # turn off nodes only when --load or --reset is set if load_nodes or reset_nodes: jobs_prepare.append(job_off_nodes) ########## infra nodes hss + epc # prepare nodes commands = [] if load_nodes: commands.append( Run("rhubarbe", "load", "-i", image_gw, hssname, epcname)) elif reset_nodes: commands.append(Run("rhubarbe", "reset", hssname, epcname)) # always do this commands.append(Run("rhubarbe", "wait", "-t", 120, hssname, epcname)) job_load_infra = SshJob( node=gwnode, commands=commands, label="load and wait HSS and EPC nodes", required=jobs_prepare, ) # start services job_service_hss = SshJob( node=hssnode, command=RunScript(locate_local_script("oai-hss.sh"), "run-hss", epc, includes=includes), label="start HSS service", required=job_load_infra, ) msg = "wait for HSS to warm up" job_service_epc = Sequence( # let 15 seconds to HSS Job( verbose_delay(15, msg), label=msg, ), SshJob( node=epcnode, command=RunScript(locate_local_script("oai-epc.sh"), "run-epc", hss, includes=includes), label="start EPC services", ), required=job_load_infra, ) jobs_infra = job_load_infra, job_service_hss, job_service_epc ########## enodeb # prepare node commands = [] if load_nodes: commands.append(Run("rhubarbe", "usrpoff", enb)) commands.append(Run("rhubarbe", "load", "-i", image_enb, enb)) elif reset_nodes: commands.append(Run("rhubarbe", "reset", enb)) commands.append(Run("rhubarbe", "wait", "-t", "120", enb)) job_load_enb = SshJob( node=gwnode, commands=commands, label="load and wait ENB", required=jobs_prepare, ) # start service msg = "wait for EPC to warm up" job_service_enb = Sequence( Job(verbose_delay(15, msg), label=msg), SshJob( node=enbnode, # run-enb expects the id of the epc as a parameter command=RunScript(locate_local_script("oai-enb.sh"), "run-enb", epc, reset_usrp, includes=includes), label="start softmodem on ENB", ), required=(job_load_enb, job_service_hss, job_service_epc), ) jobs_enb = job_load_enb, job_service_enb ########## run experiment per se # the phone # we need to wait for the USB firmware to be loaded duration = 30 if reset_usrp is not False else 8 msg = "wait for enodeb firmware to load on USRP".format(duration) job_wait_enb = Job(verbose_delay(duration, msg), label=msg, required=job_service_enb) job_start_phone = SshJob( node=gwnode, commands=[ RunScript(locate_local_script("faraday.sh"), "macphone", "r2lab/infra/user-env/macphone.sh", "phone-on", includes=includes), RunScript(locate_local_script("faraday.sh"), "macphone", "r2lab/infra/user-env/macphone.sh", "phone-start-app", includes=includes), ], label="start phone 4g and speedtest app", required=job_wait_enb, ) job_ping_phone_from_epc = SshJob( node=epcnode, commands=[ Run("sleep 10"), Run("ping -c 100 -s 100 -i .05 172.16.0.2 &> /root/ping-phone"), ], label="ping phone from EPC", critical=False, required=job_wait_enb, ) jobs_exp = job_wait_enb, job_start_phone, job_ping_phone_from_epc ########## extra nodes # ssh -X not yet supported in apssh, so one option is to start them using # a local process # xxx to update: The following code kind of works, but it needs to be # turned off, because the process in question would be killed # at the end of the Scheduler orchestration (at the end of the run function) # which is the exact time where it would be useful :) # however the code for LocalJob appears to work fine, it would be nice to # move it around - maybe in apssh ? commands = [] if not extras: commands.append(Run("echo no extra nodes specified - ignored")) else: if load_nodes: commands.append(Run("rhubarbe", "usrpoff", *extra_hostnames)) commands.append( Run("rhubarbe", "load", "-i", image_extra, *extra_hostnames)) commands.append( Run("rhubarbe", "wait", "-t", 120, *extra_hostnames)) commands.append(Run("rhubarbe", "usrpon", *extra_hostnames)) elif reset_nodes: commands.append(Run("rhubarbe", "reset", extra_hostnames)) commands.append(Run("rhubarbe", "wait", "-t", "120", *extra_hostnames)) job_load_extras = SshJob( node=gwnode, commands=commands, label="load and wait extra nodes", required=job_check_for_lease, ) jobs_extras = [job_load_extras] colors = ["wheat", "gray", "white"] if spawn_xterms: jobs_xterms_extras = [ SshJob( node=extra_node, command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*" " -bg {} -geometry 90x10".format(color), x11=True), label="xterm on node {}".format(extra_node.hostname), required=job_load_extras, # don't set forever; if we do, then these xterms get killed # when all other tasks have completed # forever = True, ) for extra_node, color in zip(extra_nodes, itertools.cycle(colors)) ] jobs_extras += jobs_xterms_extras # schedule the load phases only if required sched = Scheduler(verbose=verbose) # this is just a way to add a collection of jobs to the scheduler sched.update(jobs_prepare) sched.update(jobs_infra) sched.update(jobs_enb) sched.update(jobs_exp) sched.update(jobs_extras) # remove dangling requirements - if any - should not be needed but won't hurt either sched.sanitize() print(40 * "*") if load_nodes: print("LOADING IMAGES: (gw->{}, enb->{}, extras->{})".format( load_nodes, image_gw, image_enb, image_extra)) elif reset_nodes: print("RESETTING NODES") else: print("NODES ARE USED AS IS (no image loaded, no reset)") sched.rain_check() # Update the .dot and .png file for illustration purposes if verbose: sched.list() name = "scenario-load" if load_nodes else \ "scenario-reset" if reset_nodes else \ "scenario" sched.export_as_dotfile("{}.dot".format(name)) os.system("dot -Tpng {}.dot -o {}.png".format(name, name)) sched.list() if not sched.orchestrate(): print("RUN KO : {}".format(sched.why())) sched.debrief() return False else: print("RUN OK") return True