def run( *, # the pieces to use slice, hss, epc, enb, phones, e3372_ues, oai_ues, gnuradios, e3372_ue_xterms, oai_ue_xterms, gnuradio_xterms, # boolean flags load_nodes, skip_reset_usb, oscillo, # the images to load image_gw, image_enb, image_oai_ue, image_e3372_ue, image_gnuradio, # miscell n_rb, verbose, dry_run): """ ########## # 3 methods to get nodes ready # (*) load images # (*) reset nodes that are known to have the right image # (*) do nothing, proceed to experiment expects e.g. * slice : s.t like [email protected] * hss : 04 * epc : 03 * enb : 23 * phones: list of indices of phones to use * e3372_ues : list of nodes to use as a UE using e3372 * oai_ues : list of nodes to use as a UE using OAI * gnuradios : list of nodes to load with a gnuradio image * image_* : the name of the images to load on the various nodes Plus * load_nodes: whether to load images or not - in which case image_gw, image_enb and image_* are used to tell the image names * skip_reset_usb : the USRP board will be reset as well unless this is set """ # what argparse knows as a slice actually is a gateway (user + host) gwuser, gwhost = r2lab_parse_slice(slice) gwnode = SshNode(hostname=gwhost, username=gwuser, formatter=TimeColonFormatter(verbose=verbose), debug=verbose) hostnames = hssname, epcname, enbname = [ r2lab_hostname(x) for x in (hss, epc, enb) ] optional_ids = e3372_ues + oai_ues + gnuradios + \ e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms hssnode, epcnode, enbnode = [ SshNode(gateway=gwnode, hostname=hostname, username='******', formatter=TimeColonFormatter(verbose=verbose), debug=verbose) for hostname in hostnames ] sched = Scheduler(verbose=verbose) ########## preparation job_check_for_lease = SshJob( node=gwnode, command=["rhubarbe", "leases", "--check"], label="check we have a current lease", scheduler=sched, ) # turn off all nodes turn_off_command = ["rhubarbe", "off", "-a"] # except our 3 nodes and the optional ones turn_off_command += [ "~{}".format(x) for x in [hss, epc, enb] + optional_ids ] # only do the turn-off thing if load_nodes if load_nodes: job_off_nodes = SshJob( node=gwnode, # switch off all nodes but the ones we use command=turn_off_command, label="turn off unused nodes", required=job_check_for_lease, scheduler=sched, ) # actually run this in the gateway, not on the macphone # the ssh keys are stored in the gateway and we do not yet have # the tools to leverage such remote keys job_stop_phones = [ SshJob( node=gwnode, command=RunScript( # script find_local_embedded_script("faraday.sh"), # arguments "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-off", # options includes=includes), label="put phone{} in airplane mode".format(id), required=job_check_for_lease, scheduler=sched, ) for id in phones ] ########## prepare the image-loading phase # this will be a dict of items imagename -> ids to_load = defaultdict(list) to_load[image_gw] += [hss, epc] to_load[image_enb] += [enb] if e3372_ues: to_load[image_e3372_ue] += e3372_ues if e3372_ue_xterms: to_load[image_e3372_ue] += e3372_ue_xterms if oai_ues: to_load[image_oai_ue] += oai_ues if oai_ue_xterms: to_load[image_oai_ue] += oai_ue_xterms if gnuradios: to_load[image_gnuradio] += gnuradios if gnuradio_xterms: to_load[image_gnuradio] += gnuradio_xterms prep_job_by_node = {} for image, nodes in to_load.items(): commands = [] if load_nodes: commands.append(Run("rhubarbe", "usrpoff", *nodes)) commands.append(Run("rhubarbe", "load", "-i", image, *nodes)) commands.append(Run("rhubarbe", "usrpon", *nodes)) # always do this commands.append(Run("rhubarbe", "wait", "-t", 120, *nodes)) job = SshJob( node=gwnode, commands=commands, label="Prepare node(s) {}".format(nodes), required=job_check_for_lease, scheduler=sched, ) for node in nodes: prep_job_by_node[node] = job # start services job_service_hss = SshJob( node=hssnode, command=RunScript(find_local_embedded_script("oai-hss.sh"), "run-hss", epc, includes=includes), label="start HSS service", required=prep_job_by_node[hss], scheduler=sched, ) delay = 15 job_service_epc = SshJob( node=epcnode, commands=[ Run("echo giving HSS a headstart {delay}s to warm up; sleep {delay}" .format(delay=delay)), RunScript(find_local_embedded_script("oai-epc.sh"), "run-epc", hss, includes=includes), ], label="start EPC services", required=prep_job_by_node[epc], scheduler=sched, ) ########## enodeb job_warm_enb = SshJob( node=enbnode, commands=[ RunScript(find_local_embedded_script("oai-enb.sh"), "warm-enb", epc, n_rb, not skip_reset_usb, includes=includes), ], label="Warm eNB", required=prep_job_by_node[enb], scheduler=sched, ) enb_requirements = (job_warm_enb, job_service_hss, job_service_epc) # wait for everything to be ready, and add an extra grace delay grace = 30 if load_nodes else 10 grace_delay = SshJob( node = LocalNode(formatter=TimeColonFormatter()), command = "echo Allowing grace of {grace} seconds; sleep {grace}"\ .format(grace=grace), required = enb_requirements, scheduler = sched, ) # start services job_service_enb = SshJob( node=enbnode, # run-enb expects the id of the epc as a parameter # n_rb means number of resource blocks for DL, set to either 25 or 50. commands=[ RunScript(find_local_embedded_script("oai-enb.sh"), "run-enb", oscillo, includes=includes, x11=oscillo), ], label="start softmodem on eNB", required=grace_delay, scheduler=sched, ) ########## run experiment per se # Manage phone(s) # this starts at the same time as the eNB, but some # headstart is needed so that eNB actually is ready to serve delay = 12 msg = "wait for {delay}s for enodeb to start up"\ .format(delay=delay) wait_command = "echo {msg}; sleep {delay}".format(msg=msg, delay=delay) job_start_phones = [ SshJob( node=gwnode, commands=[ Run(wait_command), RunScript(find_local_embedded_script("faraday.sh"), "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-on", includes=includes), RunScript(find_local_embedded_script("faraday.sh"), "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-start-app", includes=includes), ], label="start Nexus phone and speedtest app", required=grace_delay, scheduler=sched, ) for id in phones ] job_ping_phones_from_epc = [ SshJob( node=epcnode, commands=[ Run("sleep 10"), Run("ping -c 100 -s 100 -i .05 172.16.0.{ip} &> /root/ping-phone" .format(ip=id + 1)), ], label="ping Nexus phone from EPC", critical=False, required=job_start_phones, ) for id in phones ] ########## xterm nodes colors = ["wheat", "gray", "white", "darkolivegreen"] xterms = e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms for xterm, color in zip(xterms, itertools.cycle(colors)): xterm_node = SshNode(gateway=gwnode, hostname=r2lab_hostname(xterm), username='******', formatter=TimeColonFormatter(verbose=verbose), debug=verbose) SshJob( node=xterm_node, command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*" " -bg {} -geometry 90x10".format(color), x11=True), label="xterm on node {}".format(xterm_node.hostname), required=prep_job_by_node[xterm], scheduler=sched, # don't set forever; if we do, then these xterms get killed # when all other tasks have completed # forever = True, ) # # remove dangling requirements - if any - should not be needed but won't hurt either sched.sanitize() print(20 * "*", "nodes usage summary") if load_nodes: for image, nodes in to_load.items(): for node in nodes: print("node {node} : {image}".format(node=node, image=image)) else: print("NODES ARE USED AS IS (no image loaded, no reset)") print(10 * "*", "phones usage summary") if phones: for phone in phones: print("Using phone{phone}".format(phone=phone)) else: print("No phone involved") sched.rain_check() # Update the .dot and .png file for illustration purposes if verbose or dry_run: sched.list() name = "scenario-load" if load_nodes else \ "scenario" sched.export_as_dotfile("{name}.dot".format(name=name)) os.system("dot -Tpng {name}.dot -o {name}.png".format(name=name)) print("(Over)wrote {name}.png".format(name=name)) if dry_run: return False if verbose: input('OK ? - press control C to abort ? ') if not sched.orchestrate(): print("RUN KO : {}".format(sched.why())) sched.debrief() return False else: print("RUN OK") return True
# the command we want to run in node1 is as simple as it gets ping = SshJob( node=node1, required=(init_node_01, init_node_02), command=Run( 'ping', '-c', '20', '10.0.0.2', '-I', wireless_interface, # verbose=True, ), scheduler=scheduler, ) ########## # run the scheduler ok = scheduler.orchestrate() # give details if it failed ok or scheduler.debrief() success = ok and ping.result() == 0 # producing a dot file for illustration scheduler.export_as_dotfile("B1.dot") # return something useful to your OS exit(0 if success else 1)
def one_run(tx_power, phy_rate, antenna_mask, channel, interference, protocol, *, run_name=default_run_name, slicename=default_slicename, load_images=False, node_ids=None, verbose_ssh=False, verbose_jobs=False, dry_run=False, tshark=False, map=False, warmup=False, exp=default_exp, dest=default_node_ids, ping_number=default_ping_number, route_sampling=False): """ Performs data acquisition on all nodes with the following settings Arguments: tx_power: in dBm, a string like 5, 10 or 14. Correspond to the transmission power. phy_rate: a string among 1, 54. Correspond to the wifi rate. antenna_mask: a string among 1, 3, 7. channel: a string like e.g. 1 or 40. Correspond to the channel. protocol: a string among batman , olsr. Correspond to the protocol interference : in dBm, a string like 60 or 50. Correspond to the power of the noise generated in the root. run_name: the name for a subdirectory where all data will be kept successive runs should use the same name for further visualization slicename: the Unix login name (slice name) to enter the gateway load_images: a boolean specifying whether nodes should be re-imaged first node_ids: a list of node ids to run the scenario against; strings or ints are OK; defaults to the nodes [1, 4, 5, 12, 19, 22,27 ,31, 33, 37] tshark: a boolean specifying wether we should format/parse the .pcap. map: a boolean specifying wether we should fetch/parse the route tables of the nodes. warmup: a boolean specifying wether we should run a ping before the experiment to be certain of the stabilisation on the network. exp: a list of nodes from which we will launch the ping from. strings or ints are OK. default to the node [1] ping_number : The number of pings that will be generated """ # set default for the nodes parameter node_ids = [int(id) for id in node_ids ] if node_ids is not None else default_node_ids exp_ids = [int(id) for id in exp] if exp is not None else default_exp dest_ids = [int(id) for id in dest] if dest is not None else default_node_ids # # dry-run mode # just display a one-liner with parameters # if dry_run: print("************************************") print("\n") run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=False) load_msg = "" if not load_images else " LOAD" nodes = " ".join(str(n) for n in node_ids) exps = " ".join(str(n) for n in exp) pingst = [ "PING{}-->{}".format(e, j) for e in exp_ids # and on the destination for j in node_ids if e != j #and not #(j in exp_ids and j < e) ] print( "dry-run:{protocol} {run_name}{load_msg} -" " t{tx_power} r{phy_rate} a{antenna_mask} ch{channel} I{interference}-" "nodes {nodes}" " exp {exps}".format(**locals())) print( "\nNodes from which the experiment will be launched : \n{}\nList of pings generated:\n" .format(exps)) print(pingst) print("\n") if warmup: print("Will do warmup pings\n") if tshark: print( "Will format data using tshark and will agregate the RSSI into one RSSI.txt file" ) if map: print( "Will fetch the routing tables of the node (when stabilited) and will agregate the results\n" ) if route_sampling: print("Will launch route sampling services on nodes") #print("Test creation of ROUTES files") #post_processor= ProcessRoutes(run_root, exp_ids, node_ids) #post_processor.run() #print("\nList of tracepaths generated:\n{}".format(tracepathst)) # in dry-run mode we are done ### # create the logs directory based on input parameters run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=False) if (run_root.is_dir()): purgedir(run_root) run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=True) exp_info_file_name = run_root / "info.txt" with exp_info_file_name.open("w") as info_file: info_file.write("Selected nodes : \n") for node in node_ids[:-1]: info_file.write(f"{node} ") info_file.write(f"{node_ids[-1]}") info_file.write("\nSources : \n") for src in exp_ids[:-1]: info_file.write(f"{src} ") info_file.write(f"{exp_ids[-1]}") info_file.write("\nDestinations : \n") for dest in dest_ids[:-1]: info_file.write(f"{dest} ") info_file.write(f"{dest_ids[-1]}" + "\n") # the nodes involved faraday = SshNode(hostname=default_gateway, username=slicename, formatter=TimeColonFormatter(), verbose=verbose_ssh) # this is a python dictionary that allows to retrieve a node object # from an id node_index = { id: SshNode(gateway=faraday, hostname=fitname(id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) for id in node_ids } if interference != "None": node_scrambler = SshNode(gateway=faraday, hostname=fitname(scrambler_id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) # the global scheduler scheduler = Scheduler(verbose=verbose_jobs) # if tshark: #scheduler_monitoring = Scheduler(verbose=verbose_jobs) #if interference != "None": #scheduler_interferences = Scheduler(verbose=verbose_jobs) ########## check_lease = SshJob( scheduler=scheduler, node=faraday, verbose=verbose_jobs, critical=True, label="rhubarbe check lease", command=Run("rhubarbe leases --check", label="rlease"), #keep_connection = True ) # load images if requested green_light = check_lease if load_images: # the nodes that we **do not** use should be turned off # so if we have selected e.g. nodes 10 12 and 15, we will do # rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15 negated_node_ids = ["~{}".format(id) for id in node_ids] #Add the id of the scrambler in the list and load the gnuradio image negated_node_ids.append("~{}".format(scrambler_id)) load_ids = [int(id) for id in node_ids ] if node_ids is not None else default_node_ids load_ids.append(scrambler_id) # replace green_light in this case #We use a modified image of gnuradio where uhd_siggen handle the signal SIGTERM in order to finish properly green_light = SshJob( node=faraday, required=check_lease, #critical=True, scheduler=scheduler, verbose=verbose_jobs, label="rhubarbe load/wait on nodes {}".format(load_ids), commands=[ Run("rhubarbe", "off", "-a", *negated_node_ids, label="roff {}".format(negated_node_ids)), Run("rhubarbe", "load", *node_ids, label="rload {}".format(node_ids)), Run("rhubarbe", "load", "-i", "gnuradio_batman", scrambler_id, label="load gnuradio batman on {}".format(scrambler_id)), Run("rhubarbe", "wait", *load_ids, label="rwait") ], #keep_connection = True ) ########## # setting up the wireless interface on all nodes # # this is a python feature known as a list comprehension # we just create as many SshJob instances as we have # (id, SshNode) couples in node_index # and gather them all in init_wireless_jobs # they all depend on green_light # # provide node-utilities with the ranges/units it expects frequency = channel_frequency[int(channel)] # tx_power_in_mBm not in dBm tx_power_driver = tx_power * 100 init_wireless_sshjobs = [ SshJob( #scheduler=scheduler, #required=green_light, node=node, verbose=verbose_jobs, label="init {}".format(id), command=RunScript("node-utilities.sh", "init-ad-hoc-network-{}".format(wireless_driver), wireless_driver, "foobar", frequency, phy_rate, antenna_mask, tx_power_driver, label="init add-hoc network"), #keep_connection = True ) for id, node in node_index.items() ] init_wireless_jobs = Scheduler( *init_wireless_sshjobs, scheduler=scheduler, required=green_light, #critical = True, verbose=verbose_jobs, label="Initialisation of wireless chips") green_light_prot = init_wireless_jobs if interference != "None": #Run uhd_siggen with the chosen power frequency_str = frequency / 1000 frequency_str = str(frequency_str) + "G" init_scrambler_job = [ SshJob( forever=True, node=node_scrambler, verbose=verbose_jobs, label="init scrambler on node {}".format(scrambler_id), command=RunScript("node-utilities.sh", "init-scrambler", interference, frequency_str, label="init scambler"), #keep_connection = True ) ] init_scrambler = Scheduler( *init_scrambler_job, scheduler=scheduler, required=green_light, #forever = True, #critical = True, verbose=verbose_jobs, label="Running interference") # then install and run batman on fit nodes run_protocol_job = [ SshJob( #scheduler=scheduler, node=node, #required=green_light_prot, label="init and run {} on fit node {}".format(protocol, i), verbose=verbose_jobs, command=RunScript("node-utilities.sh", "run-{}".format(protocol), label="run {}".format(protocol)), #keep_connection = True ) for i, node in node_index.items() ] run_protocol = Scheduler( *run_protocol_job, scheduler=scheduler, required=green_light_prot, #critical = True, verbose=verbose_jobs, label="init and run routing protocols") # after that, run tcpdump on fit nodes, this job never ends... if tshark: run_tcpdump_job = [ SshJob( #scheduler=scheduler_monitoring, node=node, forever=True, label="run tcpdump on fit node".format(i), verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "run-tcpdump", wireless_driver, i, label="run tcpdump") ], #keep_connection = True ) for i, node in node_index.items() ] run_tcpdump = Scheduler( *run_tcpdump_job, scheduler=scheduler, required=run_protocol, forever=True, #critical = True, verbose=verbose_jobs, label="Monitoring (tcpdum) Jobs") # let the wireless network settle settle_wireless_job = PrintJob( "Let the wireless network settle", sleep=settle_delay, scheduler=scheduler, required=run_protocol, label="settling for {} sec".format(settle_delay)) green_light_experiment = settle_wireless_job if warmup: warmup_pings_job = [ SshJob( node=nodei, #required=green_light_experiment, label="warmup ping {} -> {}".format(i, j), verbose=verbose_jobs, commands=[ Run("echo {} '->' {}".format(i, j), label="ping {} '->' {}".format(i, j)), RunScript("node-utilities.sh", "my-ping", "10.0.0.{}".format(j), ping_timeout, ping_interval, ping_size, ping_number, label="") ], #keep_connection = True ) #for each selected experiment nodes for e in exp_ids # looping on the source (to get the correct sshnodes) for i, nodei in node_index.items() # and on the destination for j, nodej in node_index.items() # and keep only sources that are in the selected experiment nodes and remove destination that are themselves # and remove the couples that have already be done # print("i {index} exp {expe}".format(index = i, expe= exp)) if (i == e) and e != j and not (j in exp_ids and j < e) ] warmup_pings = Scheduler( Sequence(*warmup_pings_job), scheduler=scheduler, required=green_light_experiment, #critical = True, verbose=verbose_jobs, label="Warmup ping") settle_wireless_job2 = PrintJob( "Let the wireless network settle", sleep=settle_delay / 2, scheduler=scheduler, required=warmup_pings, label="settling-warmup for {} sec".format(settle_delay / 2)) green_light_experiment = settle_wireless_job2 ########## # create all the tracepath jobs from the first node in the list # if map: routes_job = [ SshJob( node=nodei, #scheduler=scheduler, #required=green_light_experiment, label="Generating ROUTE file for prot {} on node {}".format( protocol, i), verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "route-{}".format(protocol), ">", "ROUTE-TABLE-{:02d}".format(i), label="get route table"), Pull(remotepaths="ROUTE-TABLE-{:02d}".format(i), localpath=str(run_root), label="") ], #keep_connection = True ) for i, nodei in node_index.items() ] routes = Scheduler( *routes_job, scheduler=scheduler, required=green_light_experiment, #critical = True, verbose=verbose_jobs, label="Snapshoting route files") green_light_experiment = routes if route_sampling: routes_sampling_job2 = [ SshJob( node=nodei, label="Route sampling service for prot {} on node {}".format( protocol, i), verbose=False, #forever = True, commands=[ Push(localpaths=["route_sample_service.sh"], remotepath=".", label=""), Run("source", "route_sample_service.sh;", "route-sample", "ROUTE-TABLE-{:02d}-SAMPLED".format(i), "{}".format(protocol), label="run route sampling service"), ], #keep_connection = True ) for i, nodei in node_index.items() ] routes_sampling_job = [ SshJob( node=nodei, label="Route sampling service for prot {} on node {}".format( protocol, i), verbose=False, forever=True, #critical = True, #required = green_light_experiment, #scheduler = scheduler, commands=[ RunScript("route_sample_service.sh", "route-sample", "ROUTE-TABLE-{:02d}-SAMPLED".format(i), "{}".format(protocol), label="run route sampling service"), ], #keep_connection = True ) for i, nodei in node_index.items() ] routes_sampling = Scheduler( *routes_sampling_job, scheduler=scheduler, verbose=False, forever=True, #critical = True, label="Route Sampling services launch", required=green_light_experiment) ########## # create all the ping jobs, i.e. max*(max-1)/2 # this again is a python list comprehension # see the 2 for instructions at the bottom # # notice that these SshJob instances are not yet added # to the scheduler, we will add them later on # depending on the sequential/parallel strategy pings_job = [ SshJob( node=nodei, #required=green_light_experiment, label="ping {} -> {}".format(i, j), verbose=verbose_jobs, commands=[ Run("echo {} '->' {}".format(i, j), label="ping {}'->' {}".format(i, j)), RunScript("node-utilities.sh", "my-ping", "10.0.0.{}".format(j), ping_timeout, ping_interval, ping_size, ping_number, ">", "PING-{:02d}-{:02d}".format(i, j), label=""), Pull(remotepaths="PING-{:02d}-{:02d}".format(i, j), localpath=str(run_root), label=""), ], #keep_connection = True ) #for each selected experiment nodes for e in exp_ids # looping on the source (to get the correct sshnodes) for i, nodei in node_index.items() # and on the destination for j in dest_ids # and keep only sources that are in the selected experiment nodes and remove destination that are themselves # and remove the couples that have already be done if (i == e) and e != j and not (j in exp_ids and j < e) ] pings = Scheduler( scheduler=scheduler, label="PINGS", #critical = True, verbose=verbose_jobs, required=green_light_experiment) # retrieve all pcap files from fit nodes stop_protocol_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="kill routing protocol on fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ RunScript("node-utilities.sh", "kill-{}".format(protocol), label="kill-{}".format(protocol)), ], #keep_connection = False ) for i, nodei in node_index.items() ] stop_protocol = Scheduler( *stop_protocol_job, scheduler=scheduler, required=pings, #critical = True, label="Stop routing protocols", ) if tshark: retrieve_tcpdump_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="retrieve pcap trace from fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ # RunScript("node-utilities.sh", "kill-{}".format(protocol), label = "kill-{}".format(protocol)), RunScript("node-utilities.sh", "kill-tcpdump", label="kill-tcpdump"), #Run("sleep 1"), Run("echo retrieving pcap trace and result-{i}.txt from fit{i:02d}" .format(i=i), label=""), Pull(remotepaths=["/tmp/fit{}.pcap".format(i)], localpath=str(run_root), label=""), ], #keep_connection = True ) for i, nodei in node_index.items() ] retrieve_tcpdump = Scheduler( *retrieve_tcpdump_job, scheduler=scheduler, required=pings, #critical = True, label="Retrieve tcpdump", ) if route_sampling: retrieve_sampling_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="retrieve sampling trace from fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ #RunScript("node-utilities.sh", "kill-route-sample", protocol, # label = "kill route sample"), RunScript("route_sample_service.sh", "kill-route-sample", label="kill route sample"), Run("echo retrieving sampling trace from fit{i:02d}". format(i=i), label=""), Pull(remotepaths=["ROUTE-TABLE-{:02d}-SAMPLED".format(i)], localpath=str(run_root), label=""), ], #keep_connection = True ) for i, nodei in node_index.items() ] retrieve_sampling = Scheduler( *retrieve_sampling_job, scheduler=scheduler, required=pings, #critical=True, verbose=verbose_jobs, label="Retrieve & stopping route sampling", ) if tshark: parse_pcaps_job = [ SshJob( #scheduler=scheduler, node=LocalNode(), #required=retrieve_tcpdump, label="parse pcap trace {path}/fit{node}.pcap".format( path=run_root, node=i), verbose=verbose_jobs, #commands = [RunScript("parsepcap.sh", run_root, i)] commands=[ Run("tshark", "-2", "-r", "{path}/fit{node}.pcap".format(path=run_root, node=i), "-R", "'(ip.dst==10.0.0.{node} && icmp) && radiotap.dbm_antsignal'" .format(node=i), "-Tfields", "-e", "'ip.src'", "-e" "'ip.dst'", "-e", "'radiotap.dbm_antsignal'", ">", "{path}/result-{node}.txt".format(path=run_root, node=i), label="parse pcap locally") ], #keep_connection = True ) for i in node_ids ] parse_pcaps = Scheduler( *parse_pcaps_job, scheduler=scheduler, required=retrieve_tcpdump, #critical=True, label="Parse pcap", ) #TODO: TURN OFF USRP if interference != "None": kill_uhd_siggen = SshJob( scheduler=scheduler, node=node_scrambler, required=pings, label="killing uhd_siggen on the scrambler node {}".format( scrambler_id), verbose=verbose_jobs, #critical = True, commands=[Run("pkill", "uhd_siggen")], #keep_connection = True ) kill_2_uhd_siggen = SshJob( scheduler=scheduler, node=faraday, required=kill_uhd_siggen, label="turning off usrp on the scrambler node {}".format( scrambler_id), verbose=verbose_jobs, commands=[ Run("rhubarbe", "usrpoff", "fit{}".format(scrambler_id)) ], #keep_connection = True ) #if map: #scheduler.add(Sequence(*tracepaths, scheduler=scheduler)) #if warmup: # scheduler.add(Sequence(*warmup_pings_job, scheduler=scheduler)) pings.add(Sequence(*pings_job)) # for running sequentially we impose no limit on the scheduler # that will be limitied anyways by the very structure # of the required graph #jobs_window = None if dry_run: scheduler.export_as_pngfile(run_root / "experiment_graph") return True # if not in dry-run mode, let's proceed to the actual experiment ok = scheduler.orchestrate() #jobs_window=jobs_window) scheduler.shutdown() dot_file = run_root / "experiment_graph" if not dot_file.is_file(): scheduler.export_as_dotfile(dot_file) #TODO : Is it necessary? if the user want to see it he can just do it? #call(["dot", "-Tpng", dot_file, "-o", run_root / "experitment_graph.png"]) #ok=True #ok = False # give details if it failed if not ok: scheduler.debrief() scheduler.export_as_dotfile("debug") if ok and map: print("Creation of ROUTES files") post_processor = ProcessRoutes(run_root, exp_ids, node_ids) post_processor.run() if ok and route_sampling: post_processor = ProcessRoutes(run_root, exp_ids, node_ids) post_processor.run_sampled() print("END of creation for ROUTES FILES") # data acquisition is done, let's aggregate results # i.e. compute averages if ok and tshark: post_processor = Aggregator(run_root, node_ids, antenna_mask) post_processor.run() return ok
def main(nodename1, nodename2, *, verbose=True): # show ssh outputs on stdout as they appear # together with corresponding hostname formatter = ColonFormatter(verbose=verbose) ########## declare the needed ssh connections # our main ssh connection gateway = SshNode(hostname=gwname, username=slice, formatter=formatter) # the ssh connections to each of the 2 nodes node1, node2 = [ SshNode( hostname=nodename, username="******", # this is how we create a 2-hop # ssh connection behind a gateway gateway=gateway, formatter=formatter, debug=verbose) for nodename in (nodename1, nodename2) ] ########## job_warmup = SshJob( node=gateway, # with just Run() # you can run a command already available on the remote command=[ Run("rhubarbe leases --check"), Run("rhubarbe on", nodename1, nodename2), Run("rhubarbe wait", nodename1, nodename2), ]) job_prep_send = SshJob( node=node1, command=[ # an example of a compound job # with RunScript, we run a command whose source is local here RunScript("demo.sh", "prepare-sender"), Run("ip address show control"), ], # run this only once this job is done required=job_warmup, ) job_prep_recv = SshJob( node=node2, command=RunScript("demo.sh", "prepare-receiver"), required=job_warmup, ) job_run_send = SshJob( node=node1, command=[ RunScript("demo.sh", "run-sender"), Pull("PREP", "PREP-SEND"), Pull("RUN", "RUN-SEND"), ], # start when both nodes are ready required=(job_prep_send, job_prep_recv), ) job_run_recv = SshJob( node=node2, command=[ RunScript("demo.sh", "run-receiver"), Pull("PREP", "PREP-RECV"), Pull("RUN", "RUN-RECV"), ], required=(job_prep_send, job_prep_recv), ) scheduler = Scheduler(job_warmup, job_prep_send, job_prep_recv, job_run_send, job_run_recv, verbose=verbose) scheduler.export_as_dotfile('demo.dot') print("# produce .png file with the following command") print("# install dot with e.g. brew install graphviz on macos") print("dot -Tpng demo.dot -o demo.png") print(20 * '=') ok = scheduler.orchestrate() if not ok: scheduler.debrief()
]), required = turn_on_datas, scheduler = scheduler, ) SshJob( node = node2, # this job won't finish on its own forever = True, # see above label = "infinite monitor", commands = [ RunString(receiver_manager_script, "monitor", remote_name="receiver-manager"), ], scheduler = scheduler, ) ########## # run the scheduler ok = scheduler.orchestrate() # give details if it failed ok or scheduler.debrief() # producing a dot file for illustration scheduler.export_as_dotfile("C3bis.dot") # return something useful to your OS exit(0 if ok else 1)
# dry-run mode # show the scheduler using list(details=True) # also generate a .dot file, and attempt to # transform it into a .png - should work if graphviz is installed # but don't run anything of course # if args.dry_run: print("==================== COMPLETE SCHEDULER") # -n + -v = max details scheduler.list(details=verbose_jobs) suffix = "par" if args.parallel else "seq" if args.load_images: suffix += "-load" filename = "multi-ping-{}-{}".format(suffix, args.max) print("Creating dot file: {filename}.dot".format(filename=filename)) scheduler.export_as_dotfile(filename+".dot") # try to run dot command = "dot -Tpng -o {filename}.png {filename}.dot".format(filename=filename) print("Trying to run dot to create {filename}.png".format(filename=filename)) retcod = os.system(command) if retcod == 0: print("{filename}.png OK".format(filename=filename)) else: print("Could not create {filename}.png - do you have graphviz installed ?" .format(filename=filename)) # in dry-run mode we are done exit(0) ok = scheduler.orchestrate(jobs_window=jobs_window) # give details if it failed ok or scheduler.debrief()
def run(slice, hss, epc, enb, extras, load_nodes, image_gw, image_enb, image_extra, reset_nodes, reset_usrp, spawn_xterms, verbose): """ ########## # 3 methods to get nodes ready # (*) load images # (*) reset nodes that are known to have the right image # (*) do nothing, proceed to experiment expects e.g. * slice : s.t like [email protected] * hss : 04 * epc : 03 * enb : 23 * extras : a list of ids that will be loaded with the gnuradio image Plus * load_nodes: whether to load images or not - in which case image_gw, image_enb and image_extra are used to tell the image names * reset_nodes: if load_nodes is false and reset_nodes is true, the nodes are reset - i.e. rebooted * otherwise (both False): do nothing * reset_usrp : if not False, the USRP board won't be reset * spawn_xterms : if set, starts xterm on all extra nodes * image_* : the name of the images to load on the various nodes """ # what argparse knows as a slice actually is a gateway (user + host) gwuser, gwhost = parse_slice(slice) gwnode = SshNode(hostname=gwhost, username=gwuser, formatter=ColonFormatter(verbose=verbose), debug=verbose) hostnames = hssname, epcname, enbname = [ r2lab_hostname(x) for x in (hss, epc, enb) ] extra_hostnames = [r2lab_hostname(x) for x in extras] hssnode, epcnode, enbnode = [ SshNode(gateway=gwnode, hostname=hostname, username='******', formatter=ColonFormatter(verbose=verbose), debug=verbose) for hostname in hostnames ] extra_nodes = [ SshNode(gateway=gwnode, hostname=hostname, username='******', formatter=ColonFormatter(verbose=verbose), debug=verbose) for hostname in extra_hostnames ] ########## preparation job_check_for_lease = SshJob( node=gwnode, command=["rhubarbe", "leases", "--check"], label="check we have a current lease", ) # turn off all nodes turn_off_command = ["rhubarbe", "off", "-a"] # except our 3 nodes and the optional extras turn_off_command += [ "~{}".format(x) for x in [hss, epc, enb] + extras + [20] ] job_off_nodes = SshJob( node=gwnode, # switch off all nodes but the ones we use command=turn_off_command, label="turn off unused nodes", required=job_check_for_lease, ) # actually run this in the gateway, not on the mac # the ssh keys are stored in the gateway and we do not yet have # the tools to leverage such remote keys job_stop_phone = SshJob( node=gwnode, command=RunScript(locate_local_script("faraday.sh"), "macphone", "r2lab/infra/user-env/macphone.sh", "phone-off", includes=includes), label="stop phone", required=job_check_for_lease, ) jobs_prepare = [job_check_for_lease, job_stop_phone] # turn off nodes only when --load or --reset is set if load_nodes or reset_nodes: jobs_prepare.append(job_off_nodes) ########## infra nodes hss + epc # prepare nodes commands = [] if load_nodes: commands.append( Run("rhubarbe", "load", "-i", image_gw, hssname, epcname)) elif reset_nodes: commands.append(Run("rhubarbe", "reset", hssname, epcname)) # always do this commands.append(Run("rhubarbe", "wait", "-t", 120, hssname, epcname)) job_load_infra = SshJob( node=gwnode, commands=commands, label="load and wait HSS and EPC nodes", required=jobs_prepare, ) # start services job_service_hss = SshJob( node=hssnode, command=RunScript(locate_local_script("oai-hss.sh"), "run-hss", epc, includes=includes), label="start HSS service", required=job_load_infra, ) msg = "wait for HSS to warm up" job_service_epc = Sequence( # let 15 seconds to HSS Job( verbose_delay(15, msg), label=msg, ), SshJob( node=epcnode, command=RunScript(locate_local_script("oai-epc.sh"), "run-epc", hss, includes=includes), label="start EPC services", ), required=job_load_infra, ) jobs_infra = job_load_infra, job_service_hss, job_service_epc ########## enodeb # prepare node commands = [] if load_nodes: commands.append(Run("rhubarbe", "usrpoff", enb)) commands.append(Run("rhubarbe", "load", "-i", image_enb, enb)) elif reset_nodes: commands.append(Run("rhubarbe", "reset", enb)) commands.append(Run("rhubarbe", "wait", "-t", "120", enb)) job_load_enb = SshJob( node=gwnode, commands=commands, label="load and wait ENB", required=jobs_prepare, ) # start service msg = "wait for EPC to warm up" job_service_enb = Sequence( Job(verbose_delay(15, msg), label=msg), SshJob( node=enbnode, # run-enb expects the id of the epc as a parameter command=RunScript(locate_local_script("oai-enb.sh"), "run-enb", epc, reset_usrp, includes=includes), label="start softmodem on ENB", ), required=(job_load_enb, job_service_hss, job_service_epc), ) jobs_enb = job_load_enb, job_service_enb ########## run experiment per se # the phone # we need to wait for the USB firmware to be loaded duration = 30 if reset_usrp is not False else 8 msg = "wait for enodeb firmware to load on USRP".format(duration) job_wait_enb = Job(verbose_delay(duration, msg), label=msg, required=job_service_enb) job_start_phone = SshJob( node=gwnode, commands=[ RunScript(locate_local_script("faraday.sh"), "macphone", "r2lab/infra/user-env/macphone.sh", "phone-on", includes=includes), RunScript(locate_local_script("faraday.sh"), "macphone", "r2lab/infra/user-env/macphone.sh", "phone-start-app", includes=includes), ], label="start phone 4g and speedtest app", required=job_wait_enb, ) job_ping_phone_from_epc = SshJob( node=epcnode, commands=[ Run("sleep 10"), Run("ping -c 100 -s 100 -i .05 172.16.0.2 &> /root/ping-phone"), ], label="ping phone from EPC", critical=False, required=job_wait_enb, ) jobs_exp = job_wait_enb, job_start_phone, job_ping_phone_from_epc ########## extra nodes # ssh -X not yet supported in apssh, so one option is to start them using # a local process # xxx to update: The following code kind of works, but it needs to be # turned off, because the process in question would be killed # at the end of the Scheduler orchestration (at the end of the run function) # which is the exact time where it would be useful :) # however the code for LocalJob appears to work fine, it would be nice to # move it around - maybe in apssh ? commands = [] if not extras: commands.append(Run("echo no extra nodes specified - ignored")) else: if load_nodes: commands.append(Run("rhubarbe", "usrpoff", *extra_hostnames)) commands.append( Run("rhubarbe", "load", "-i", image_extra, *extra_hostnames)) commands.append( Run("rhubarbe", "wait", "-t", 120, *extra_hostnames)) commands.append(Run("rhubarbe", "usrpon", *extra_hostnames)) elif reset_nodes: commands.append(Run("rhubarbe", "reset", extra_hostnames)) commands.append(Run("rhubarbe", "wait", "-t", "120", *extra_hostnames)) job_load_extras = SshJob( node=gwnode, commands=commands, label="load and wait extra nodes", required=job_check_for_lease, ) jobs_extras = [job_load_extras] colors = ["wheat", "gray", "white"] if spawn_xterms: jobs_xterms_extras = [ SshJob( node=extra_node, command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*" " -bg {} -geometry 90x10".format(color), x11=True), label="xterm on node {}".format(extra_node.hostname), required=job_load_extras, # don't set forever; if we do, then these xterms get killed # when all other tasks have completed # forever = True, ) for extra_node, color in zip(extra_nodes, itertools.cycle(colors)) ] jobs_extras += jobs_xterms_extras # schedule the load phases only if required sched = Scheduler(verbose=verbose) # this is just a way to add a collection of jobs to the scheduler sched.update(jobs_prepare) sched.update(jobs_infra) sched.update(jobs_enb) sched.update(jobs_exp) sched.update(jobs_extras) # remove dangling requirements - if any - should not be needed but won't hurt either sched.sanitize() print(40 * "*") if load_nodes: print("LOADING IMAGES: (gw->{}, enb->{}, extras->{})".format( load_nodes, image_gw, image_enb, image_extra)) elif reset_nodes: print("RESETTING NODES") else: print("NODES ARE USED AS IS (no image loaded, no reset)") sched.rain_check() # Update the .dot and .png file for illustration purposes if verbose: sched.list() name = "scenario-load" if load_nodes else \ "scenario-reset" if reset_nodes else \ "scenario" sched.export_as_dotfile("{}.dot".format(name)) os.system("dot -Tpng {}.dot -o {}.png".format(name, name)) sched.list() if not sched.orchestrate(): print("RUN KO : {}".format(sched.why())) sched.debrief() return False else: print("RUN OK") return True