def test_local_command2(self): self.run_one_job( # details = True, job=SshJob( node=LocalNode(), commands=[ Run("for i in $(seq 3); do echo line $i; sleep 1; done"), ]))
def test_local_command(self): # create random file in python rather than with /dev/random # that is not working in virtualbox random_full = "RANDOM-full" random_head = "RANDOM" self.random_file(random_full, size=19) print("DONE") self.run_one_job( # details = True, job=SshJob( node=LocalNode(), commands=[ Run(f"head -c {2**18} < {random_full} > {random_head}"), Run(f"ls -l {random_head}"), Run(f"shasum {random_head}"), ]))
def test_commands_verbose(self): dummy_path = "tests/dummy-10" dummy_file = Path(dummy_path).name scheduler = Scheduler() Sequence(SshJob( node=self.gateway(), verbose=True, commands=[ Run("hostname"), RunScript("tests/script-with-args.sh", "arg1", "arg2"), RunString("for i in $(seq 3); do echo verbose$i; done"), Push(localpaths=dummy_path, remotepath="."), Pull(remotepaths=dummy_file, localpath=dummy_path + ".loop"), ]), SshJob(node=LocalNode(), critical=True, commands=Run("diff {x} {x}.loop".format(x=dummy_path), verbose=True)), scheduler=scheduler) ok = scheduler.run() ok or scheduler.debrief() self.assertTrue(ok)
nodes = (node1, node2) ########## create the scheduler instance upfront scheduler = Scheduler() check_lease = SshJob( node=faraday, critical=True, command=Run("rhubarbe leases --check"), scheduler=scheduler, ) ########## 1 step, generate a random data file of 1 M bytes create_random_job = SshJob( node=LocalNode(), commands=[ Run("head", "-c", random_size, "<", "/dev/random", ">", "RANDOM"), Run("ls", "-l", "RANDOM"), Run("shasum", "RANDOM"), ], required=check_lease, scheduler=scheduler, ) ########## 2nd step : push this over to node1 push_job = SshJob( node=node1, commands=[ Push(localpaths=["RANDOM"], remotepath="."),
def main() -> bool: parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-U", "--url", default=default_topurl, dest='topurl', help="url to reach nbhosting server") parser.add_argument("-c", "--course-gitdir", default=default_course_gitdir, help="""location of a git repo where to fetch notebooks; needed in order to generate relevant URLs""") parser.add_argument("-i", "--indices", default=[0], action=IntsRanges, help="(cumulative) ranges of indices in the list of known notebooks" " - run nbhtest with -l to see list") parser.add_argument("-u", "--users", default=[1], action=IntsRanges, help="(cumulative) ranges of students indexes; e.g. -u 101-400 -u 501-600") parser.add_argument("-m", "--random", action='store_true', help="if set, a random notebook index is used for each student") parser.add_argument("-b", "--base", default='student', help="basename for students name") parser.add_argument("-p", "--period", default=20, type=float, help="delay between 2 triggers of nbhtest") parser.add_argument("-s", "--sleep", default=default_sleep_internal, type=float, help="delay in seconds to sleep between actions inside nbhtest") parser.add_argument("-w", "--window", default=default_window, type=int, help="window depth for spawning the nbhtest instances") parser.add_argument("-n", "--dry-run", action='store_true') args = parser.parse_args() course_gitdir = args.course_gitdir course, notebooks = list_notebooks(course_gitdir) # in random mode; what are the choices that we randomize on if args.random: if len(args.indices) > 1: choices = args.indices else: choices = list(range(len(notebooks))) local = LocalNode( formatter=TerminalFormatter( format="%H-%M-%S:@line@", verbose=True )) scheduler = Scheduler() jobs = [] for user in args.users: student_name = "{}-{:04d}".format(args.base, user) if args.random: indices = [ random.choice(choices) ] else: indices = args.indices for index in indices: command = "nbhtest.py -U {} -c {} -i {} -u {} -s {} &"\ .format(args.topurl, course_gitdir, index, student_name, args.sleep) if args.dry_run: print("dry-run:", command) else: # schule this command to run job = Sequence( SshJob(scheduler=scheduler, node=local, command = command, ), Job(asyncio.sleep(args.period)) ) jobs.append(job) if args.dry_run: return True overall = scheduler.orchestrate( jobs_window = args.window ) if not overall: scheduler.debrief() print("nbhtests DONE") return overall
def run(*, gateway, slicename, disaggregated_cn, operator_version, nodes, node_master, node_enb, quectel_nodes, phones, flexran, drone, verbose, dry_run, load_images, master_image, worker_image, quectel_image): """ Install K8S on R2lab Arguments: slicename: the Unix login name (slice name) to enter the gateway quectel_nodes: list of indices of quectel UE nodes to use phones: list of indices of phones to use nodes: a list of node ids to run the scenario on; strings or ints are OK; node_master: the master node id, must be part of selected nodes node_enb: the node id for the enb, which is connected to usrp/duplexer disaggregated_cn: Boolean; True for the disaggregated CN scenario. False for all-in-one CN. operator_version: str, either "none" or "v1" or "v2". """ if operator_version == "none": only_kube5g = True else: only_kube5g = False if node_master not in nodes: print(f"master node {node_master} must be part of selected fit nodes {nodes}") exit(1) if node_enb not in nodes: print(f"eNB worker node {node_enb} must be part of selected fit nodes {nodes}") exit(1) # Check if the browser can be automatically run to display the Drone app if drone: run_browser = True if platform == "linux": cmd_open = "xdg-open" elif platform == "darwin": cmd_open = "open" else: run_browser = False if run_browser: print(f"**** Will run the browser with command {cmd_open}") else: print(f"**** Will not be able to run the browser as platform is {platform}") worker_ids = nodes[:] worker_ids.remove(node_master) quectel_ids = quectel_nodes[:] quectel = len(quectel_ids) > 0 faraday = SshNode(hostname=default_gateway, username=slicename, verbose=verbose, formatter=TimeColonFormatter()) master = SshNode(gateway=faraday, hostname=fitname(node_master), username="******", verbose=verbose, formatter=TimeColonFormatter()) node_index = { id: SshNode(gateway=faraday, hostname=fitname(id), username="******",formatter=TimeColonFormatter(), verbose=verbose) for id in nodes } nodes_quectel_index = { id: SshNode(gateway=faraday, hostname=fitname(id), username="******",formatter=TimeColonFormatter(), verbose=verbose) for id in quectel_nodes } worker_index = dict(node_index) del worker_index[node_master] fit_master = fitname(node_master) fit_enb = fitname(node_enb) # the global scheduler scheduler = Scheduler(verbose=verbose) ########## check_lease = SshJob( scheduler=scheduler, node = faraday, critical = True, verbose=verbose, command = Run("rhubarbe leases --check"), ) green_light = check_lease if load_images: green_light = [ SshJob( scheduler=scheduler, required=check_lease, node=faraday, critical=True, verbose=verbose, label = f"Load image {master_image} on master {fit_master}", commands=[ Run(f"rhubarbe load {node_master} -i {master_image}"), Run(f"rhubarbe wait {node_master}"), ] ), SshJob( scheduler=scheduler, required=check_lease, node=faraday, critical=True, verbose=verbose, label = f"Load image {worker_image} on worker nodes", commands=[ Run(f"rhubarbe usrpoff {node_enb}"), # if usrp is on, load could be problematic... Run("rhubarbe", "load", *worker_ids, "-i", worker_image), Run("rhubarbe", "wait", *worker_ids), Run(f"rhubarbe usrpon {node_enb}"), # ensure a reset of the USRP on the enB node ], ), SshJob( scheduler=scheduler, required=check_lease, node=faraday, critical=False, verbose=verbose, label="turning off unused nodes", command=[ Run("rhubarbe bye --all " + "".join(f"~{x} " for x in nodes)) ] ) ] if quectel: prepare_quectel = SshJob( scheduler=scheduler, required=green_light, node=faraday, critical=True, verbose=verbose, label = f"Load image {quectel_image} on quectel UE nodes", commands=[ Run("rhubarbe", "usrpoff", *quectel_ids), Run("rhubarbe", "load", *quectel_ids, "-i", quectel_image), Run("rhubarbe", "wait", *quectel_ids), Run("rhubarbe", "usrpon", *quectel_ids), ], ), ########## if quectel: # wait 30s for Quectel modules show up wait_quectel_ready = PrintJob( "Let Quectel modules show up", scheduler=scheduler, required=prepare_quectel, sleep=30, label="sleep 30s for the Quectel modules to show up" ) # run the Quectel Connection Manager as a service on each Quectel UE node quectelCM_service = Service( command="quectel-CM -s oai.ipv4 -4", service_id="QuectelCM", verbose=verbose, ) init_quectel_nodes = [ SshJob( scheduler=scheduler, required=wait_quectel_ready, node=node, critical=True, verbose=verbose, label=f"Init Quectel UE on fit node {id}", commands = [ RunScript(find_local_embedded_script("nodes.sh"), "check-quectel-on", includes=INCLUDES), quectelCM_service.start_command(), ], ) for id, node in nodes_quectel_index.items() ] # wait 20s for Quectel Connection Manager to start up wait_quectelCM_ready = PrintJob( "Let QuectelCM start up", scheduler=scheduler, required=init_quectel_nodes, sleep=20, label="Sleep 20s for the Quectel Connection Manager(s) to start up" ) detach_quectel_nodes = [ SshJob( scheduler=scheduler, required=wait_quectelCM_ready, node=node, critical=True, verbose=verbose, label=f"Detach Quectel UE on fit node {id}", command = RunScript(find_local_embedded_script("nodes.sh"), "quectel-detach", includes=INCLUDES), ) for id, node in nodes_quectel_index.items() ] ########## # Initialize k8s on the master node init_master = SshJob( scheduler=scheduler, required=green_light, node=master, critical=True, verbose=verbose, label = f"Install and launch k8s on the master {node_master}", commands = [ Run("swapoff -a"), Run("hostnamectl set-hostname master-node"), Run("kubeadm init --pod-network-cidr=10.244.0.0/16 > /tmp/join_msg.txt"), Run("tail -2 /tmp/join_msg.txt > /tmp/join_msg"), Run("mkdir -p $HOME/.kube"), Run("cp -i /etc/kubernetes/admin.conf $HOME/.kube/config"), Run("chown $(id -u):$(id -g) $HOME/.kube/config"), Run("kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"), Run("kubectl get pods --all-namespaces"), ], ) init_workers = [ SshJob( scheduler=scheduler, required=init_master, node=node, critical=True, verbose=verbose, label=f"Init k8s on fit node {id} and join the cluster", commands = [ Run("swapoff -a"), Run("increase-control-mtu"), Run(f"scp -o 'StrictHostKeyChecking no' {fit_master}:/tmp/join_msg /tmp/join_msg"), Run("chmod a+x /tmp/join_msg"), Run("/tmp/join_msg"), ], ) for id, node in worker_index.items() ] # wait 10s for K8S nodes setup wait_k8nodes_ready = PrintJob( "Let k8s set up", scheduler=scheduler, required=init_workers, sleep=10, label="sleep 10s for the k8s nodes to settle" ) init_kube5g = SshJob( scheduler=scheduler, required = wait_k8nodes_ready, node = master, verbose=verbose, label = f"Add oai:ran label to oai-ran pod on {node_enb} and start 5GOperator pod", commands = [ Run("kubectl get nodes"), # add label to the eNB node to help k8s scheduler selects the right fit node Run(f"kubectl label nodes fit{node_enb} oai=ran"), Run("kubectl get nodes -Loai"), ## retrieve the kube5g operator #Run("git clone -b develop [email protected]:mosaic5g/kube5g.git"), # install a few dependencies Run("apt install -y python3-pip"), Run("pip3 install --upgrade pip"), Run("pip3 install ruamel.yaml==0.16.12 colorlog==4.6.2"), Run("sed -i 's/oairan:v1-1.0-1/oairan:v1-1.0-3/g' /root/kube5g/common/config-manager/conf_global_default.yaml"), # specify the R2lab specific configuration Run("cd /root/kube5g/common/config-manager; ./conf-manager.py -s conf_short_r2lab.yaml"), # apply the R2lab CRD Run("cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh -n"), # start the kube5g operator pod Run("cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh container start"), Run("kubectl get pods"), ], ) # wait 30s for K8S 5G Operator setup wait_k8_5GOp_ready = PrintJob( "Let 5G Operator set up", scheduler=scheduler, required=init_kube5g, sleep=30, label="wait 30s for the 5G Operator pod to settle" ) if only_kube5g: finish = SshJob( scheduler=scheduler, required = wait_k8_5GOp_ready, node = master, verbose=verbose, label = f"showing nodes and pods before leaving", commands = [ Run("kubectl get nodes -Loai"), Run("kubectl get pods"), ], ) else: if disaggregated_cn: cn_type="disaggregated-cn" # setup_time = 120 setup_time = 200 else: cn_type="all-in-one" # setup_time = 60 setup_time = 140 if flexran: flexran_opt="flexran" else: flexran_opt="" run_kube5g = SshJob( scheduler=scheduler, required = wait_k8_5GOp_ready, node = master, verbose=verbose, label = f"deploy {operator_version} {cn_type} {flexran_opt} pods", commands = [ Run("kubectl get nodes -Loai"), Run(f"cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh deploy {operator_version} {cn_type} {flexran_opt}"), Run("kubectl get pods"), ], ) # Coffee Break -- wait 1 or 2mn for K8S 5G pods setup wait_k8_5Gpods_ready = PrintJob( "Let all 5G pods set up", scheduler=scheduler, required=run_kube5g, sleep=setup_time, label=f"waiting {setup_time}s for all 5G pods to settle" ) check_kube5g = SshJob( scheduler=scheduler, required = wait_k8_5Gpods_ready, node = master, verbose=verbose, label = "Check which pods are deployed", commands = [ Run("kubectl get nodes -Loai"), Run("kubectl get pods"), ], ) if drone: # the place where runtime variables get stored env = Variables() # # Define and run all the services to launch the Drone app locally on a firefox browser # drone_service = Service( command=f"python /root/mosaic5g/store/sdk/frontend/drone/drone.py --port=8088 --tasks --address=192.168.3.{node_enb}", service_id="drone_app", verbose=verbose, ) k8s_port9999_fwd_service = Service( command=Deferred("kubectl port-forward {{flexran_pod}} 9999:9999 --address 0.0.0.0", env), service_id="k8s-port9999-fwd", verbose=verbose, # somehow this is required for kubectl to run properly environ={'KUBECONFIG': '/etc/kubernetes/admin.conf'} ) # can't use a Service instance on the local box if it's not a Linux # and we have macs... local_port_fwd = (f"ssh -f -N -4" f" -L9999:192.168.3.{node_master}:9999" f" -L8088:192.168.3.{node_enb}:8088" f" -o ExitOnForwardFailure=yes" f" {slicename}@faraday.inria.fr") browser_service = Service( command=f"sleep 10; {cmd_open} http://127.0.0.1:8088/", service_id="drone_browser", verbose=verbose, ) run_drone=SshJob( scheduler=scheduler, required=check_kube5g, node=worker_index[node_enb], verbose=verbose, label=f"Run the drone app on worker node {node_enb} as a service", commands=[ drone_service.start_command(), ], ) get_flexran_podname=SshJob( scheduler=scheduler, required=check_kube5g, node=master, verbose=verbose, label=f"Retrieve the name of the FlexRAN pod", commands=[ # xxx here Run("kubectl get --no-headers=true pods -l app=flexran -o custom-columns=:metadata.name", capture=Capture('flexran_pod', env)), ], ) run_k8s_port9999_fwd=SshJob( scheduler=scheduler, required=get_flexran_podname, node=master, verbose=verbose, label=f"Run port forwarding on the master node as a service", commands=[ k8s_port9999_fwd_service.start_command(), ], ) # On the local machine, impossible to use Services as the latter uses systemd-run, only available on Linux run_local_ports_fwd = SshJob( scheduler=scheduler, required = check_kube5g, node = LocalNode(), verbose=verbose, label = f"Forward local ports 8088 and 9999", command=Run(local_port_fwd + "&", ignore_outputs=True), ) if run_browser: run_local_browser = SshJob( scheduler=scheduler, required = (run_drone, run_k8s_port9999_fwd, run_local_ports_fwd), node = LocalNode(), verbose=verbose, label = f"Run the browser on the local node in background", command=browser_service.command+"&", ) phones_requirements=run_local_browser else: phones_requirements=run_k8s_port9999_fwd else: phones_requirements=check_kube5g ########## Test phone(s) connectivity sleeps_ran = (20, 25) phone_msgs = [f"wait for {sleep}s for eNB to start up before waking up phone{id}" for sleep, id in zip(sleeps_ran, phones)] wait_commands = [f"echo {msg}; sleep {sleep}" for msg, sleep in zip(phone_msgs, sleeps_ran)] sleeps_phone = (15, 20) phone2_msgs = [f"wait for {sleep}s for phone{id} before starting tests" for sleep, id in zip(sleeps_phone, phones)] wait2_commands = [f"echo {msg}; sleep {sleep}" for msg, sleep in zip(phone2_msgs, sleeps_phone)] job_start_phones = [ SshJob( node=faraday, commands=[ Run(wait_command), RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}", "r2lab-embedded/shell/macphone.sh", "phone-on", includes=INCLUDES), Run(wait2_command), RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}", "r2lab-embedded/shell/macphone.sh", "phone-check-cx", includes=INCLUDES), RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}", "r2lab-embedded/shell/macphone.sh", "phone-start-app", includes=INCLUDES), ], label=f"turn off airplane mode on phone {id}", required=phones_requirements, scheduler=scheduler) for id, wait_command, wait2_command in zip(phones, wait_commands, wait2_commands) ] if quectel: # wait 60s for Quectel connection(s) to set up wait_before_attach_quectel = PrintJob( "Wait again 30s before attaching Quectel device(s)", scheduler=scheduler, required=(job_start_phones,check_kube5g,detach_quectel_nodes), sleep=30, label="Sleep 30s before attaching Quectel device(s)" ) job_attach_quectel = [ SshJob( scheduler=scheduler, required=wait_before_attach_quectel, node=node, critical=True, verbose=verbose, label=f"Attach Quectel UE on fit node {id}", command = RunScript(find_local_embedded_script("nodes.sh"), "quectel-attach", includes=INCLUDES), ) for id, node in nodes_quectel_index.items() ] # wait 30s for Quectel connection(s) to set up wait_quectel_cx_ready = PrintJob( "Let the Quectel connection(s) set up", scheduler=scheduler, required=job_attach_quectel, sleep=30, label="Sleep 30s for the Quectel connection(s) to set up" ) test_quectel_cx = [ SshJob( scheduler=scheduler, required=wait_quectel_cx_ready, node=node, critical=False, verbose=verbose, label=f"Check the Quectel cx on fit node {id}", command = RunScript(find_local_embedded_script("nodes.sh"), "check-quectel-cx", includes=INCLUDES), ) for id, node in nodes_quectel_index.items() ] ########## # Update the .dot and .png file for illustration purposes scheduler.check_cycles() name = "deploy-kube5g" print(10*'*', 'See main scheduler in', scheduler.export_as_pngfile(name)) # orchestration scheduler jobs if verbose: scheduler.list() if dry_run: return True if not scheduler.orchestrate(): print(f"RUN KO : {scheduler.why()}") scheduler.debrief() return False print(f"RUN OK, you can log now on master node {fit_master} to manually change the scenario") print(80*'*')
def one_run(*, protocol, interference, run_name=default_run_name, slicename=default_slicename, tx_power, phy_rate, antenna_mask, channel, load_images=False, node_ids=DEFAULT_NODE_IDS, src_ids=DEFAULT_SRC_IDS, dest_ids=DEFAULT_DEST_IDS, scrambler_id=DEFAULT_SCRAMBLER_ID, tshark=False, map=False, warmup=False, route_sampling=False, iperf=False, verbose_ssh=False, verbose_jobs=False, dry_run=False, run_number=None): """ Performs data acquisition on all nodes with the following settings Arguments: tx_power: in dBm, a string like 5, 10 or 14. Corresponds to the transmission power. phy_rate: a string among 1, 54. Correspond to the wifi rate. antenna_mask: a string among 1, 3, 7. channel: a string like e.g. 1 or 40. Correspond to the channel. protocol: a string among batman , olsr. Correspond to the protocol interference : in amplitude percentage, a string like 15 or 20. Correspond to the power of the noise generated in the spectrum. Can be either None or "None" to mean no interference. run_name: the name for a subdirectory where all data will be kept successive runs should use the same name for further visualization slicename: the Unix login name (slice name) to enter the gateway load_images: a boolean specifying whether nodes should be re-imaged first node_ids: a list of node ids to run the scenario against; strings or ints are OK; tshark: a boolean specifying wether we should format/parse the .pcap. map: a boolean specifying wether we should fetch/parse the route tables of the nodes. warmup: a boolean specifying whether we should run a ping before the experiment to be certain of the stabilisation on the network. src_ids: a list of nodes from which we will launch the ping from. strings or ints are OK. ping_messages : the number of ping packets that will be generated """ # set default for the nodes parameter node_ids = ([int(id) for id in node_ids] if node_ids is not None else DEFAULT_NODE_IDS) src_ids = ([int(id) for id in src_ids] if src_ids is not None else DEFAULT_SRC_IDS) dest_ids = ([int(id) for id in dest_ids] if dest_ids is not None else DEFAULT_NODE_IDS) # all nodes - i.e. including sources and destinations - # need to run the protocol node_ids = list(set(node_ids).union(set(src_ids).union(set(dest_ids)))) if interference == "None": interference = None # open result dir no matter what run_root = naming_scheme( run_name=run_name, protocol=protocol, interference=interference, autocreate=True) # fix me trace = run_root / f"trace-{%m-%d-%H-%M}" ref_time = apssh_time() trace = run_root / f"trace-{ref_time}" try: with trace.open('w') as feed: def log_line(line): time_line(line, file=feed) load_msg = f"{'WITH' if load_images else 'NO'} image loading" interference_msg = (f"interference={interference} " f"from scrambler={scrambler_id}") nodes = " ".join(str(n) for n in node_ids) srcs = " ".join(str(n) for n in src_ids) dests = " ".join(str(n) for n in dest_ids) ping_labels = [ f"PING {s} ➡︎ {d}" for s in src_ids # and on the destination for d in dest_ids if d != s ] log_line(f"output in {run_root}") log_line(f"trace in {trace}") log_line(f"protocol={protocol}") log_line(f"{load_msg}") log_line(f"{interference_msg}") log_line("----") log_line(f"Selected nodes : {nodes}") log_line(f"Sources : {srcs}") log_line(f"Destinations : {dests}") for label in ping_labels: log_line(f"{label}") log_line("----") for feature in ('warmup', 'tshark', 'map', 'route_sampling', 'iperf'): log_line(f"Feature {feature}: {locals()[feature]}") except Exception as exc: print(f"Cannot write into {trace} - aborting this run") print(f"Found exception {type(exc)} - {exc}") return False # # dry-run mode # just display a one-liner with parameters # prelude = "" if not dry_run else "dry_run:" with trace.open() as feed: print(f"**************** {ref_time} one_run #{run_number}:") for line in feed: print(prelude, line, sep='', end='') if dry_run: return True # the nodes involved faraday = SshNode(hostname=default_gateway, username=slicename, formatter=TimeColonFormatter(), verbose=verbose_ssh) # this is a python dictionary that allows to retrieve a node object # from an id node_index = { id: SshNode(gateway=faraday, hostname=fitname(id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) for id in node_ids } # extracts for sources and destinations src_index = {id:node for (id, node) in node_index.items() if id in src_ids} dest_index = {id:node for (id, node) in node_index.items() if id in dest_ids} if interference: node_scrambler = SshNode( gateway=faraday, hostname=fitname(scrambler_id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) # the global scheduler scheduler = Scheduler(verbose=verbose_jobs) ########## check_lease = SshJob( scheduler=scheduler, node=faraday, verbose=verbose_jobs, label="rhubarbe check lease", command=Run("rhubarbe leases --check", label="rlease"), ) # load images if requested green_light = check_lease # at some point we did not load the scrambler if interference was None # and that was a way to run faster loads with no interference # but now we always load the scrambler node with gnuradio # this is because when we do runs.py -i None 15 30 ... # then the first call to one_run is with interference being None # but it is still important to load the scrambler if load_images: # copy node_ids load_ids = node_ids[:] load_ids.append(scrambler_id) # the nodes that we **do not** use should be turned off # so if we have selected e.g. nodes 10 12 and 15, we will do # rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15 negated_node_ids = [f"~{id}" for id in load_ids] # we can do these three things in parallel ready_jobs = [ SshJob(node=faraday, required=green_light, scheduler=scheduler, verbose=verbose_jobs, command=Run("rhubarbe", "off", "-a", *negated_node_ids, label="turn off unused nodes")), SshJob(node=faraday, required=green_light, scheduler=scheduler, verbose=verbose_jobs, label="load batman image", command=Run("rhubarbe", "load", "-i", "batman-olsr", *node_ids, label=f"load ubuntu on {node_ids}")), SshJob( node=faraday, required=green_light, scheduler=scheduler, verbose=verbose_jobs, label="load gnuradio image", command=Run("rhubarbe", "load", "-i", "batman-olsr-gnuradio", scrambler_id, label=f"load gnuradio on {scrambler_id}")), ] # replace green_light in this case green_light = SshJob( node=faraday, required=ready_jobs, scheduler=scheduler, verbose=verbose_jobs, label="wait for nodes to come up", command=Run("rhubarbe", "wait", *load_ids)) ########## # setting up the wireless interface on all nodes # # provide node-utilities with the ranges/units it expects frequency = channel_frequency[int(channel)] # tx_power_in_mBm not in dBm tx_power_driver = tx_power * 100 #just in case somme services failed in the previous experiment reset_failed_services_job = [ SshJob( node=node, verbose=verbose_jobs, label="reset failed services", command=Run("systemctl reset-failed", label="reset-failed services")) for id, node in node_index.items() ] reset_failed_services = Scheduler( *reset_failed_services_job, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="Reset failed services") init_wireless_sshjobs = [ SshJob( node=node, verbose=verbose_jobs, label=f"init {id}", command=RunScript( "node-utilities.sh", f"init-ad-hoc-network-{WIRELESS_DRIVER}", WIRELESS_DRIVER, "foobar", frequency, phy_rate, antenna_mask, tx_power_driver, label="init add-hoc network"), ) for id, node in node_index.items()] init_wireless_jobs = Scheduler( *init_wireless_sshjobs, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="Initialisation of wireless chips") if interference: # Run uhd_siggen with the chosen power init_scrambler_job = SshJob( scheduler=scheduler, required=green_light, forever=True, node=node_scrambler, verbose=verbose_jobs, #TODO : If exit-signal patch is done add exit-signal=["TERM"] # to this run object and call uhd_siggen directly commands=[RunScript("node-utilities.sh", "init-scrambler", label="init scrambler"), Run(f"systemd-run --unit=uhd_siggen -t ", f"uhd_siggen -a usrp -f {frequency}M", f"--sine --amplitude 0.{interference}", label="systemctl start uhd_siggen") ] ) green_light = [init_wireless_jobs, reset_failed_services] # then install and run batman on fit nodes run_protocol_job = [ SshJob( # scheduler=scheduler, node=node, label=f"init and run {protocol} on fit node {id}", verbose=verbose_jobs, # CAREFUL : These ones use sytemd-run # with the ----service-type=forking option! command=RunScript("node-utilities.sh", f"run-{protocol}", label=f"run {protocol}"), ) for id, node in node_index.items()] run_protocol = Scheduler( *run_protocol_job, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="init and run routing protocols") green_light = run_protocol # after that, run tcpdump on fit nodes, this job never ends... if tshark: run_tcpdump_job = [ SshJob( # scheduler=scheduler_monitoring, node=node, forever=True, label=f"run tcpdump on fit node {id}", verbose=verbose_jobs, command=[ Run("systemd-run -t --unit=tcpdump", f"tcpdump -U -i moni-{WIRELESS_DRIVER}", f"-y ieee802_11_radio -w /tmp/fit{id}.pcap", label=f"tcpdump {id}") ] ) for id, node in node_index.items() ] run_tcpdump = Scheduler( *run_tcpdump_job, scheduler=scheduler, required=green_light, forever=True, verbose=verbose_jobs, label="Monitoring - tcpdumps") # let the wireless network settle settle_scheduler = Scheduler( scheduler=scheduler, required=green_light, ) if warmup: # warmup pings don't need to be sequential, so let's # do all the nodes at the same time # on a given node though, we'll ping the other ends sequentially # see the graph for more warmup_jobs = [ SshJob( node=node_s, verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "my-ping", f"10.0.0.{d}", warmup_ping_timeout, warmup_ping_interval, warmup_ping_size, warmup_ping_messages, f"warmup {s} ➡︎ {d}", label=f"warmup {s} ➡︎ {d}") for d in dest_index.keys() if s != d ] ) # for each selected experiment nodes for s, node_s in src_index.items() ] warmup_scheduler = Scheduler( *warmup_jobs, scheduler=settle_scheduler, verbose=verbose_jobs, label="Warmup pings") settle_wireless_job2 = PrintJob( "Let the wireless network settle after warmup", sleep=settle_delay_shorter, scheduler=settle_scheduler, required=warmup_scheduler, label=f"settling-warmup for {settle_delay_shorter} sec") # this is a little cheating; could have gone before the bloc above # but produces a nicer graphical output # we might want to help asynciojobs if it offered a means # to specify entry and exit jobs in a scheduler settle_wireless_job = PrintJob( "Let the wireless network settle", sleep=settle_delay_long, scheduler=settle_scheduler, label=f"settling for {settle_delay_long} sec") green_light = settle_scheduler if iperf: iperf_service_jobs = [ SshJob( node=node_d, verbose=verbose_jobs, forever=True, commands=[ Run("systemd-run -t --unit=iperf", "iperf -s -p 1234 -u", label=f"iperf serv on {d}"), ], ) for d, node_d in dest_index.items() ] iperf_serv_sched = Scheduler( *iperf_service_jobs, verbose=verbose_jobs, label="Iperf Servers", # for a nicer graphical output # otherwise the exit arrow # from scheduler 'iperf mode' # to job 'settling for 60s' # gets to start from this box forever=True, ) iperf_cli = [ SshJob( node=node_s, verbose=verbose_jobs, commands=[ Run("sleep 7", label=""), Run(f"iperf", f"-c 10.0.0.{d} -p 1234", f"-u -b {phy_rate}M -t 60", f"-l 1024 > IPERF-{s:02d}-{d:02d}", label=f"run iperf {s} ➡︎ {d}") ] ) for s, node_s in src_index.items() for d, node_d in dest_index.items() if s != d ] iperf_cli_sched = Scheduler( Sequence(*iperf_cli), verbose=verbose_jobs, label="Iperf Clients") iperf_stop = [ SshJob(node=node_d, verbose=verbose_jobs, label=f"Stop iperf on {d}", command=Run("systemctl stop iperf")) for d, node_d in dest_index.items() ] iperf_stop_sched = Scheduler( *iperf_stop, required=iperf_cli_sched, verbose=verbose_jobs, label="Iperf server stop") iperf_fetch = [ SshJob(node=node_s, verbose=verbose_jobs, command=Pull( remotepaths=[f"IPERF-{s:02d}-{d:02d}"], localpath=str(run_root), label="fetch iperf {s} ➡︎ {d}") ) for s, node_s in src_index.items() for d, node_d in dest_index.items() if s != d ] iperf_fetch_sched = Scheduler( *iperf_fetch, required=iperf_stop_sched, verbose=verbose_jobs, label="Iperf fetch report") iperf_jobs = [iperf_serv_sched, iperf_cli_sched, iperf_stop_sched, iperf_fetch_sched] iperf_sched = Scheduler( *iperf_jobs, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="Iperf Module") settle_wireless_job_iperf = PrintJob( "Let the wireless network settle", sleep=settle_delay_shorter, scheduler=scheduler, required=iperf_sched, label=f"settling-iperf for {settle_delay_shorter} sec") green_light = settle_wireless_job_iperf # create all the tracepath jobs from the first node in the list if map: map_jobs = [ SshJob( node=node, label=f"Generating ROUTE file for proto {protocol} on node {id}", verbose=verbose_jobs, commands=[ RunScript(f"node-utilities.sh", f"route-{protocol}", f"> ROUTE-TABLE-{id:02d}", label="get route table"), Pull(remotepaths=[f"ROUTE-TABLE-{id:02d}"], localpath=str(run_root), label="") ], ) for id, node in node_index.items() ] map_scheduler = Scheduler( *map_jobs, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="Snapshoting route files") green_light = map_scheduler if route_sampling: route_sampling_jobs = [ SshJob( node=node, label=f"Route sampling service for proto {protocol} on node {id}", verbose=False, forever=True, commands=[ Push(localpaths=["route-sample-service.sh"], remotepath=".", label=""), Run("chmod +x route-sample-service.sh", label=""), Run("systemd-run -t --unit=route-sample", "/root/route-sample-service.sh", "route-sample", f"ROUTE-TABLE-{id:02d}-SAMPLED", protocol, label="start route-sampling"), ], ) for id, node in node_index.items() ] route_sampling_scheduler = Scheduler( *route_sampling_jobs, scheduler=scheduler, verbose=False, forever=True, label="Route Sampling services launch", required=green_light) ########## # create all the ping jobs, i.e. max*(max-1)/2 # this again is a python list comprehension # see the 2 for instructions at the bottom # # notice that these SshJob instances are not yet added # to the scheduler, we will add them later on # depending on the sequential/parallel strategy pings_job = [ SshJob( node=node_s, verbose=verbose_jobs, commands=[ Run(f"echo actual ping {s} ➡︎ {d} using {protocol}", label=f"ping {s} ➡︎ {d}"), RunScript("node-utilities.sh", "my-ping", f"10.0.0.{d}", ping_timeout, ping_interval, ping_size, ping_messages, f"actual {s} ➡︎ {d}", ">", f"PING-{s:02d}-{d:02d}", label=""), Pull(remotepaths=[f"PING-{s:02d}-{d:02d}"], localpath=str(run_root), label=""), ], ) # for each selected experiment nodes for s, node_s in src_index.items() for d, node_d in dest_index.items() if s != d ] pings = Scheduler( scheduler=scheduler, label="PINGS", verbose=verbose_jobs, required=green_light) # retrieve all pcap files from fit nodes stop_protocol_job = [ SshJob( # scheduler=scheduler, node=node, # required=pings, label=f"kill routing protocol on {id}", verbose=verbose_jobs, command=RunScript(f"node-utilities.sh", f"kill-{protocol}", label=f"kill-{protocol}"), ) for id, node in node_index.items() ] stop_protocol = Scheduler( *stop_protocol_job, scheduler=scheduler, required=pings, label="Stop routing protocols", ) if tshark: retrieve_tcpdump_job = [ SshJob( # scheduler=scheduler, node=nodei, # required=pings, label=f"retrieve pcap trace from fit{i:02d}", verbose=verbose_jobs, commands=[ Run("systemctl stop tcpdump", label="stop tcpdump"), #Run("systemctl reset-failed tcpdump"), #RunScript("node-utilities.sh", "kill-tcpdump", # label="kill-tcpdump"), Run( f"echo retrieving pcap trace and result-{i}.txt from fit{i:02d}", label=""), Pull(remotepaths=[f"/tmp/fit{i}.pcap"], localpath=str(run_root), label=""), ], ) for i, nodei in node_index.items() ] retrieve_tcpdump = Scheduler( *retrieve_tcpdump_job, scheduler=scheduler, required=pings, label="Retrieve tcpdump", ) if route_sampling: retrieve_sampling_job = [ SshJob( # scheduler=scheduler, node=nodei, # required=pings, label=f"retrieve sampling trace from fit{i:02d}", verbose=verbose_jobs, commands=[ # RunScript("node-utilities.sh", "kill-route-sample", protocol, # label = "kill route sample"), #RunScript("route-sample-service.sh", "kill-route-sample", # label="kill route sample"), Run("systemctl stop route-sample", label="stop route-sample"), Run( f"echo retrieving sampling trace from fit{i:02d}", label=""), Pull(remotepaths=[f"ROUTE-TABLE-{i:02d}-SAMPLED"], localpath=str(run_root), label=""), ], ) for i, nodei in node_index.items() ] retrieve_sampling = Scheduler( *retrieve_sampling_job, scheduler=scheduler, required=pings, verbose=verbose_jobs, label="Stop & retrieve route sampling", ) if tshark: parse_pcaps_job = [ SshJob( # scheduler=scheduler, node=LocalNode(), # required=retrieve_tcpdump, label=f"parse pcap trace {run_root}/fit{i}.pcap", verbose=verbose_jobs, #commands = [RunScript("parsepcap.sh", run_root, i)] command=Run("tshark", "-2", "-r", f"{run_root}/fit{i}.pcap", "-R", f"'(ip.dst==10.0.0.{i} && icmp) && radiotap.dbm_antsignal'", "-Tfields", "-e", "'ip.src'", "-e" "'ip.dst'", "-e", "'radiotap.dbm_antsignal'", ">", f"{run_root}/result-{i}.txt", label=f"parsing pcap from {i}"), ) for i in node_ids ] parse_pcaps = Scheduler( *parse_pcaps_job, scheduler=scheduler, required=retrieve_tcpdump, label="Parse pcap", ) if interference: kill_uhd_siggen = SshJob( scheduler=scheduler, node=node_scrambler, required=pings, label=f"killing uhd_siggen on the scrambler node {scrambler_id}", verbose=verbose_jobs, commands=[Run("systemctl", "stop", "uhd_siggen"), #Run("systemctl reset-failed tcpdump"), ], ) kill_2_uhd_siggen = SshJob( scheduler=scheduler, node=faraday, required=kill_uhd_siggen, label=f"turning off usrp on the scrambler node {scrambler_id}", verbose=verbose_jobs, command=Run("rhubarbe", "usrpoff", scrambler_id), ) pings.add(Sequence(*pings_job)) # for running sequentially we impose no limit on the scheduler # that will be limitied anyways by the very structure # of the required graph # safety check scheduler.export_as_pngfile(run_root / "experiment-graph") if dry_run: scheduler.list() return True # if not in dry-run mode, let's proceed to the actual experiment ok = scheduler.run() # jobs_window=jobs_window) # close all ssh connections close_ssh_in_scheduler(scheduler) # give details if it failed if not ok: scheduler.debrief() scheduler.export_as_pngfile("debug") if ok and map: time_line("Creation of MAP files") post_processor = ProcessRoutes(run_root, src_ids, node_ids) post_processor.run() if ok and route_sampling: time_line("Creation of ROUTE SAMPLING files") post_processor = ProcessRoutes(run_root, src_ids, node_ids) post_processor.run_sampled() # data acquisition is done, let's aggregate results # i.e. compute averages #if ok and tshark: #post_processor = Aggregator(run_root, node_ids, antenna_mask) #post_processor.run() time_line("one_run done") return ok
def run( *, # the pieces to use slice, hss, epc, enb, phones, e3372_ues, oai_ues, gnuradios, e3372_ue_xterms, oai_ue_xterms, gnuradio_xterms, # boolean flags load_nodes, skip_reset_usb, oscillo, # the images to load image_gw, image_enb, image_oai_ue, image_e3372_ue, image_gnuradio, # miscell n_rb, verbose, dry_run): """ ########## # 3 methods to get nodes ready # (*) load images # (*) reset nodes that are known to have the right image # (*) do nothing, proceed to experiment expects e.g. * slice : s.t like [email protected] * hss : 04 * epc : 03 * enb : 23 * phones: list of indices of phones to use * e3372_ues : list of nodes to use as a UE using e3372 * oai_ues : list of nodes to use as a UE using OAI * gnuradios : list of nodes to load with a gnuradio image * image_* : the name of the images to load on the various nodes Plus * load_nodes: whether to load images or not - in which case image_gw, image_enb and image_* are used to tell the image names * skip_reset_usb : the USRP board will be reset as well unless this is set """ # what argparse knows as a slice actually is a gateway (user + host) gwuser, gwhost = r2lab_parse_slice(slice) gwnode = SshNode(hostname=gwhost, username=gwuser, formatter=TimeColonFormatter(verbose=verbose), debug=verbose) hostnames = hssname, epcname, enbname = [ r2lab_hostname(x) for x in (hss, epc, enb) ] optional_ids = e3372_ues + oai_ues + gnuradios + \ e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms hssnode, epcnode, enbnode = [ SshNode(gateway=gwnode, hostname=hostname, username='******', formatter=TimeColonFormatter(verbose=verbose), debug=verbose) for hostname in hostnames ] sched = Scheduler(verbose=verbose) ########## preparation job_check_for_lease = SshJob( node=gwnode, command=["rhubarbe", "leases", "--check"], label="check we have a current lease", scheduler=sched, ) # turn off all nodes turn_off_command = ["rhubarbe", "off", "-a"] # except our 3 nodes and the optional ones turn_off_command += [ "~{}".format(x) for x in [hss, epc, enb] + optional_ids ] # only do the turn-off thing if load_nodes if load_nodes: job_off_nodes = SshJob( node=gwnode, # switch off all nodes but the ones we use command=turn_off_command, label="turn off unused nodes", required=job_check_for_lease, scheduler=sched, ) # actually run this in the gateway, not on the macphone # the ssh keys are stored in the gateway and we do not yet have # the tools to leverage such remote keys job_stop_phones = [ SshJob( node=gwnode, command=RunScript( # script find_local_embedded_script("faraday.sh"), # arguments "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-off", # options includes=includes), label="put phone{} in airplane mode".format(id), required=job_check_for_lease, scheduler=sched, ) for id in phones ] ########## prepare the image-loading phase # this will be a dict of items imagename -> ids to_load = defaultdict(list) to_load[image_gw] += [hss, epc] to_load[image_enb] += [enb] if e3372_ues: to_load[image_e3372_ue] += e3372_ues if e3372_ue_xterms: to_load[image_e3372_ue] += e3372_ue_xterms if oai_ues: to_load[image_oai_ue] += oai_ues if oai_ue_xterms: to_load[image_oai_ue] += oai_ue_xterms if gnuradios: to_load[image_gnuradio] += gnuradios if gnuradio_xterms: to_load[image_gnuradio] += gnuradio_xterms prep_job_by_node = {} for image, nodes in to_load.items(): commands = [] if load_nodes: commands.append(Run("rhubarbe", "usrpoff", *nodes)) commands.append(Run("rhubarbe", "load", "-i", image, *nodes)) commands.append(Run("rhubarbe", "usrpon", *nodes)) # always do this commands.append(Run("rhubarbe", "wait", "-t", 120, *nodes)) job = SshJob( node=gwnode, commands=commands, label="Prepare node(s) {}".format(nodes), required=job_check_for_lease, scheduler=sched, ) for node in nodes: prep_job_by_node[node] = job # start services job_service_hss = SshJob( node=hssnode, command=RunScript(find_local_embedded_script("oai-hss.sh"), "run-hss", epc, includes=includes), label="start HSS service", required=prep_job_by_node[hss], scheduler=sched, ) delay = 15 job_service_epc = SshJob( node=epcnode, commands=[ Run("echo giving HSS a headstart {delay}s to warm up; sleep {delay}" .format(delay=delay)), RunScript(find_local_embedded_script("oai-epc.sh"), "run-epc", hss, includes=includes), ], label="start EPC services", required=prep_job_by_node[epc], scheduler=sched, ) ########## enodeb job_warm_enb = SshJob( node=enbnode, commands=[ RunScript(find_local_embedded_script("oai-enb.sh"), "warm-enb", epc, n_rb, not skip_reset_usb, includes=includes), ], label="Warm eNB", required=prep_job_by_node[enb], scheduler=sched, ) enb_requirements = (job_warm_enb, job_service_hss, job_service_epc) # wait for everything to be ready, and add an extra grace delay grace = 30 if load_nodes else 10 grace_delay = SshJob( node = LocalNode(formatter=TimeColonFormatter()), command = "echo Allowing grace of {grace} seconds; sleep {grace}"\ .format(grace=grace), required = enb_requirements, scheduler = sched, ) # start services job_service_enb = SshJob( node=enbnode, # run-enb expects the id of the epc as a parameter # n_rb means number of resource blocks for DL, set to either 25 or 50. commands=[ RunScript(find_local_embedded_script("oai-enb.sh"), "run-enb", oscillo, includes=includes, x11=oscillo), ], label="start softmodem on eNB", required=grace_delay, scheduler=sched, ) ########## run experiment per se # Manage phone(s) # this starts at the same time as the eNB, but some # headstart is needed so that eNB actually is ready to serve delay = 12 msg = "wait for {delay}s for enodeb to start up"\ .format(delay=delay) wait_command = "echo {msg}; sleep {delay}".format(msg=msg, delay=delay) job_start_phones = [ SshJob( node=gwnode, commands=[ Run(wait_command), RunScript(find_local_embedded_script("faraday.sh"), "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-on", includes=includes), RunScript(find_local_embedded_script("faraday.sh"), "macphone{}".format(id), "r2lab-embedded/shell/macphone.sh", "phone-start-app", includes=includes), ], label="start Nexus phone and speedtest app", required=grace_delay, scheduler=sched, ) for id in phones ] job_ping_phones_from_epc = [ SshJob( node=epcnode, commands=[ Run("sleep 10"), Run("ping -c 100 -s 100 -i .05 172.16.0.{ip} &> /root/ping-phone" .format(ip=id + 1)), ], label="ping Nexus phone from EPC", critical=False, required=job_start_phones, ) for id in phones ] ########## xterm nodes colors = ["wheat", "gray", "white", "darkolivegreen"] xterms = e3372_ue_xterms + oai_ue_xterms + gnuradio_xterms for xterm, color in zip(xterms, itertools.cycle(colors)): xterm_node = SshNode(gateway=gwnode, hostname=r2lab_hostname(xterm), username='******', formatter=TimeColonFormatter(verbose=verbose), debug=verbose) SshJob( node=xterm_node, command=Run("xterm -fn -*-fixed-medium-*-*-*-20-*-*-*-*-*-*-*" " -bg {} -geometry 90x10".format(color), x11=True), label="xterm on node {}".format(xterm_node.hostname), required=prep_job_by_node[xterm], scheduler=sched, # don't set forever; if we do, then these xterms get killed # when all other tasks have completed # forever = True, ) # # remove dangling requirements - if any - should not be needed but won't hurt either sched.sanitize() print(20 * "*", "nodes usage summary") if load_nodes: for image, nodes in to_load.items(): for node in nodes: print("node {node} : {image}".format(node=node, image=image)) else: print("NODES ARE USED AS IS (no image loaded, no reset)") print(10 * "*", "phones usage summary") if phones: for phone in phones: print("Using phone{phone}".format(phone=phone)) else: print("No phone involved") sched.rain_check() # Update the .dot and .png file for illustration purposes if verbose or dry_run: sched.list() name = "scenario-load" if load_nodes else \ "scenario" sched.export_as_dotfile("{name}.dot".format(name=name)) os.system("dot -Tpng {name}.dot -o {name}.png".format(name=name)) print("(Over)wrote {name}.png".format(name=name)) if dry_run: return False if verbose: input('OK ? - press control C to abort ? ') if not sched.orchestrate(): print("RUN KO : {}".format(sched.why())) sched.debrief() return False else: print("RUN OK") return True
def one_run(tx_power, phy_rate, antenna_mask, channel, interference, protocol, *, run_name=default_run_name, slicename=default_slicename, load_images=False, node_ids=None, verbose_ssh=False, verbose_jobs=False, dry_run=False, tshark=False, map=False, warmup=False, exp=default_exp, dest=default_node_ids, ping_number=default_ping_number, route_sampling=False): """ Performs data acquisition on all nodes with the following settings Arguments: tx_power: in dBm, a string like 5, 10 or 14. Correspond to the transmission power. phy_rate: a string among 1, 54. Correspond to the wifi rate. antenna_mask: a string among 1, 3, 7. channel: a string like e.g. 1 or 40. Correspond to the channel. protocol: a string among batman , olsr. Correspond to the protocol interference : in dBm, a string like 60 or 50. Correspond to the power of the noise generated in the root. run_name: the name for a subdirectory where all data will be kept successive runs should use the same name for further visualization slicename: the Unix login name (slice name) to enter the gateway load_images: a boolean specifying whether nodes should be re-imaged first node_ids: a list of node ids to run the scenario against; strings or ints are OK; defaults to the nodes [1, 4, 5, 12, 19, 22,27 ,31, 33, 37] tshark: a boolean specifying wether we should format/parse the .pcap. map: a boolean specifying wether we should fetch/parse the route tables of the nodes. warmup: a boolean specifying wether we should run a ping before the experiment to be certain of the stabilisation on the network. exp: a list of nodes from which we will launch the ping from. strings or ints are OK. default to the node [1] ping_number : The number of pings that will be generated """ # set default for the nodes parameter node_ids = [int(id) for id in node_ids ] if node_ids is not None else default_node_ids exp_ids = [int(id) for id in exp] if exp is not None else default_exp dest_ids = [int(id) for id in dest] if dest is not None else default_node_ids # # dry-run mode # just display a one-liner with parameters # if dry_run: print("************************************") print("\n") run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=False) load_msg = "" if not load_images else " LOAD" nodes = " ".join(str(n) for n in node_ids) exps = " ".join(str(n) for n in exp) pingst = [ "PING{}-->{}".format(e, j) for e in exp_ids # and on the destination for j in node_ids if e != j #and not #(j in exp_ids and j < e) ] print( "dry-run:{protocol} {run_name}{load_msg} -" " t{tx_power} r{phy_rate} a{antenna_mask} ch{channel} I{interference}-" "nodes {nodes}" " exp {exps}".format(**locals())) print( "\nNodes from which the experiment will be launched : \n{}\nList of pings generated:\n" .format(exps)) print(pingst) print("\n") if warmup: print("Will do warmup pings\n") if tshark: print( "Will format data using tshark and will agregate the RSSI into one RSSI.txt file" ) if map: print( "Will fetch the routing tables of the node (when stabilited) and will agregate the results\n" ) if route_sampling: print("Will launch route sampling services on nodes") #print("Test creation of ROUTES files") #post_processor= ProcessRoutes(run_root, exp_ids, node_ids) #post_processor.run() #print("\nList of tracepaths generated:\n{}".format(tracepathst)) # in dry-run mode we are done ### # create the logs directory based on input parameters run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=False) if (run_root.is_dir()): purgedir(run_root) run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=True) exp_info_file_name = run_root / "info.txt" with exp_info_file_name.open("w") as info_file: info_file.write("Selected nodes : \n") for node in node_ids[:-1]: info_file.write(f"{node} ") info_file.write(f"{node_ids[-1]}") info_file.write("\nSources : \n") for src in exp_ids[:-1]: info_file.write(f"{src} ") info_file.write(f"{exp_ids[-1]}") info_file.write("\nDestinations : \n") for dest in dest_ids[:-1]: info_file.write(f"{dest} ") info_file.write(f"{dest_ids[-1]}" + "\n") # the nodes involved faraday = SshNode(hostname=default_gateway, username=slicename, formatter=TimeColonFormatter(), verbose=verbose_ssh) # this is a python dictionary that allows to retrieve a node object # from an id node_index = { id: SshNode(gateway=faraday, hostname=fitname(id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) for id in node_ids } if interference != "None": node_scrambler = SshNode(gateway=faraday, hostname=fitname(scrambler_id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) # the global scheduler scheduler = Scheduler(verbose=verbose_jobs) # if tshark: #scheduler_monitoring = Scheduler(verbose=verbose_jobs) #if interference != "None": #scheduler_interferences = Scheduler(verbose=verbose_jobs) ########## check_lease = SshJob( scheduler=scheduler, node=faraday, verbose=verbose_jobs, critical=True, label="rhubarbe check lease", command=Run("rhubarbe leases --check", label="rlease"), #keep_connection = True ) # load images if requested green_light = check_lease if load_images: # the nodes that we **do not** use should be turned off # so if we have selected e.g. nodes 10 12 and 15, we will do # rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15 negated_node_ids = ["~{}".format(id) for id in node_ids] #Add the id of the scrambler in the list and load the gnuradio image negated_node_ids.append("~{}".format(scrambler_id)) load_ids = [int(id) for id in node_ids ] if node_ids is not None else default_node_ids load_ids.append(scrambler_id) # replace green_light in this case #We use a modified image of gnuradio where uhd_siggen handle the signal SIGTERM in order to finish properly green_light = SshJob( node=faraday, required=check_lease, #critical=True, scheduler=scheduler, verbose=verbose_jobs, label="rhubarbe load/wait on nodes {}".format(load_ids), commands=[ Run("rhubarbe", "off", "-a", *negated_node_ids, label="roff {}".format(negated_node_ids)), Run("rhubarbe", "load", *node_ids, label="rload {}".format(node_ids)), Run("rhubarbe", "load", "-i", "gnuradio_batman", scrambler_id, label="load gnuradio batman on {}".format(scrambler_id)), Run("rhubarbe", "wait", *load_ids, label="rwait") ], #keep_connection = True ) ########## # setting up the wireless interface on all nodes # # this is a python feature known as a list comprehension # we just create as many SshJob instances as we have # (id, SshNode) couples in node_index # and gather them all in init_wireless_jobs # they all depend on green_light # # provide node-utilities with the ranges/units it expects frequency = channel_frequency[int(channel)] # tx_power_in_mBm not in dBm tx_power_driver = tx_power * 100 init_wireless_sshjobs = [ SshJob( #scheduler=scheduler, #required=green_light, node=node, verbose=verbose_jobs, label="init {}".format(id), command=RunScript("node-utilities.sh", "init-ad-hoc-network-{}".format(wireless_driver), wireless_driver, "foobar", frequency, phy_rate, antenna_mask, tx_power_driver, label="init add-hoc network"), #keep_connection = True ) for id, node in node_index.items() ] init_wireless_jobs = Scheduler( *init_wireless_sshjobs, scheduler=scheduler, required=green_light, #critical = True, verbose=verbose_jobs, label="Initialisation of wireless chips") green_light_prot = init_wireless_jobs if interference != "None": #Run uhd_siggen with the chosen power frequency_str = frequency / 1000 frequency_str = str(frequency_str) + "G" init_scrambler_job = [ SshJob( forever=True, node=node_scrambler, verbose=verbose_jobs, label="init scrambler on node {}".format(scrambler_id), command=RunScript("node-utilities.sh", "init-scrambler", interference, frequency_str, label="init scambler"), #keep_connection = True ) ] init_scrambler = Scheduler( *init_scrambler_job, scheduler=scheduler, required=green_light, #forever = True, #critical = True, verbose=verbose_jobs, label="Running interference") # then install and run batman on fit nodes run_protocol_job = [ SshJob( #scheduler=scheduler, node=node, #required=green_light_prot, label="init and run {} on fit node {}".format(protocol, i), verbose=verbose_jobs, command=RunScript("node-utilities.sh", "run-{}".format(protocol), label="run {}".format(protocol)), #keep_connection = True ) for i, node in node_index.items() ] run_protocol = Scheduler( *run_protocol_job, scheduler=scheduler, required=green_light_prot, #critical = True, verbose=verbose_jobs, label="init and run routing protocols") # after that, run tcpdump on fit nodes, this job never ends... if tshark: run_tcpdump_job = [ SshJob( #scheduler=scheduler_monitoring, node=node, forever=True, label="run tcpdump on fit node".format(i), verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "run-tcpdump", wireless_driver, i, label="run tcpdump") ], #keep_connection = True ) for i, node in node_index.items() ] run_tcpdump = Scheduler( *run_tcpdump_job, scheduler=scheduler, required=run_protocol, forever=True, #critical = True, verbose=verbose_jobs, label="Monitoring (tcpdum) Jobs") # let the wireless network settle settle_wireless_job = PrintJob( "Let the wireless network settle", sleep=settle_delay, scheduler=scheduler, required=run_protocol, label="settling for {} sec".format(settle_delay)) green_light_experiment = settle_wireless_job if warmup: warmup_pings_job = [ SshJob( node=nodei, #required=green_light_experiment, label="warmup ping {} -> {}".format(i, j), verbose=verbose_jobs, commands=[ Run("echo {} '->' {}".format(i, j), label="ping {} '->' {}".format(i, j)), RunScript("node-utilities.sh", "my-ping", "10.0.0.{}".format(j), ping_timeout, ping_interval, ping_size, ping_number, label="") ], #keep_connection = True ) #for each selected experiment nodes for e in exp_ids # looping on the source (to get the correct sshnodes) for i, nodei in node_index.items() # and on the destination for j, nodej in node_index.items() # and keep only sources that are in the selected experiment nodes and remove destination that are themselves # and remove the couples that have already be done # print("i {index} exp {expe}".format(index = i, expe= exp)) if (i == e) and e != j and not (j in exp_ids and j < e) ] warmup_pings = Scheduler( Sequence(*warmup_pings_job), scheduler=scheduler, required=green_light_experiment, #critical = True, verbose=verbose_jobs, label="Warmup ping") settle_wireless_job2 = PrintJob( "Let the wireless network settle", sleep=settle_delay / 2, scheduler=scheduler, required=warmup_pings, label="settling-warmup for {} sec".format(settle_delay / 2)) green_light_experiment = settle_wireless_job2 ########## # create all the tracepath jobs from the first node in the list # if map: routes_job = [ SshJob( node=nodei, #scheduler=scheduler, #required=green_light_experiment, label="Generating ROUTE file for prot {} on node {}".format( protocol, i), verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "route-{}".format(protocol), ">", "ROUTE-TABLE-{:02d}".format(i), label="get route table"), Pull(remotepaths="ROUTE-TABLE-{:02d}".format(i), localpath=str(run_root), label="") ], #keep_connection = True ) for i, nodei in node_index.items() ] routes = Scheduler( *routes_job, scheduler=scheduler, required=green_light_experiment, #critical = True, verbose=verbose_jobs, label="Snapshoting route files") green_light_experiment = routes if route_sampling: routes_sampling_job2 = [ SshJob( node=nodei, label="Route sampling service for prot {} on node {}".format( protocol, i), verbose=False, #forever = True, commands=[ Push(localpaths=["route_sample_service.sh"], remotepath=".", label=""), Run("source", "route_sample_service.sh;", "route-sample", "ROUTE-TABLE-{:02d}-SAMPLED".format(i), "{}".format(protocol), label="run route sampling service"), ], #keep_connection = True ) for i, nodei in node_index.items() ] routes_sampling_job = [ SshJob( node=nodei, label="Route sampling service for prot {} on node {}".format( protocol, i), verbose=False, forever=True, #critical = True, #required = green_light_experiment, #scheduler = scheduler, commands=[ RunScript("route_sample_service.sh", "route-sample", "ROUTE-TABLE-{:02d}-SAMPLED".format(i), "{}".format(protocol), label="run route sampling service"), ], #keep_connection = True ) for i, nodei in node_index.items() ] routes_sampling = Scheduler( *routes_sampling_job, scheduler=scheduler, verbose=False, forever=True, #critical = True, label="Route Sampling services launch", required=green_light_experiment) ########## # create all the ping jobs, i.e. max*(max-1)/2 # this again is a python list comprehension # see the 2 for instructions at the bottom # # notice that these SshJob instances are not yet added # to the scheduler, we will add them later on # depending on the sequential/parallel strategy pings_job = [ SshJob( node=nodei, #required=green_light_experiment, label="ping {} -> {}".format(i, j), verbose=verbose_jobs, commands=[ Run("echo {} '->' {}".format(i, j), label="ping {}'->' {}".format(i, j)), RunScript("node-utilities.sh", "my-ping", "10.0.0.{}".format(j), ping_timeout, ping_interval, ping_size, ping_number, ">", "PING-{:02d}-{:02d}".format(i, j), label=""), Pull(remotepaths="PING-{:02d}-{:02d}".format(i, j), localpath=str(run_root), label=""), ], #keep_connection = True ) #for each selected experiment nodes for e in exp_ids # looping on the source (to get the correct sshnodes) for i, nodei in node_index.items() # and on the destination for j in dest_ids # and keep only sources that are in the selected experiment nodes and remove destination that are themselves # and remove the couples that have already be done if (i == e) and e != j and not (j in exp_ids and j < e) ] pings = Scheduler( scheduler=scheduler, label="PINGS", #critical = True, verbose=verbose_jobs, required=green_light_experiment) # retrieve all pcap files from fit nodes stop_protocol_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="kill routing protocol on fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ RunScript("node-utilities.sh", "kill-{}".format(protocol), label="kill-{}".format(protocol)), ], #keep_connection = False ) for i, nodei in node_index.items() ] stop_protocol = Scheduler( *stop_protocol_job, scheduler=scheduler, required=pings, #critical = True, label="Stop routing protocols", ) if tshark: retrieve_tcpdump_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="retrieve pcap trace from fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ # RunScript("node-utilities.sh", "kill-{}".format(protocol), label = "kill-{}".format(protocol)), RunScript("node-utilities.sh", "kill-tcpdump", label="kill-tcpdump"), #Run("sleep 1"), Run("echo retrieving pcap trace and result-{i}.txt from fit{i:02d}" .format(i=i), label=""), Pull(remotepaths=["/tmp/fit{}.pcap".format(i)], localpath=str(run_root), label=""), ], #keep_connection = True ) for i, nodei in node_index.items() ] retrieve_tcpdump = Scheduler( *retrieve_tcpdump_job, scheduler=scheduler, required=pings, #critical = True, label="Retrieve tcpdump", ) if route_sampling: retrieve_sampling_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="retrieve sampling trace from fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ #RunScript("node-utilities.sh", "kill-route-sample", protocol, # label = "kill route sample"), RunScript("route_sample_service.sh", "kill-route-sample", label="kill route sample"), Run("echo retrieving sampling trace from fit{i:02d}". format(i=i), label=""), Pull(remotepaths=["ROUTE-TABLE-{:02d}-SAMPLED".format(i)], localpath=str(run_root), label=""), ], #keep_connection = True ) for i, nodei in node_index.items() ] retrieve_sampling = Scheduler( *retrieve_sampling_job, scheduler=scheduler, required=pings, #critical=True, verbose=verbose_jobs, label="Retrieve & stopping route sampling", ) if tshark: parse_pcaps_job = [ SshJob( #scheduler=scheduler, node=LocalNode(), #required=retrieve_tcpdump, label="parse pcap trace {path}/fit{node}.pcap".format( path=run_root, node=i), verbose=verbose_jobs, #commands = [RunScript("parsepcap.sh", run_root, i)] commands=[ Run("tshark", "-2", "-r", "{path}/fit{node}.pcap".format(path=run_root, node=i), "-R", "'(ip.dst==10.0.0.{node} && icmp) && radiotap.dbm_antsignal'" .format(node=i), "-Tfields", "-e", "'ip.src'", "-e" "'ip.dst'", "-e", "'radiotap.dbm_antsignal'", ">", "{path}/result-{node}.txt".format(path=run_root, node=i), label="parse pcap locally") ], #keep_connection = True ) for i in node_ids ] parse_pcaps = Scheduler( *parse_pcaps_job, scheduler=scheduler, required=retrieve_tcpdump, #critical=True, label="Parse pcap", ) #TODO: TURN OFF USRP if interference != "None": kill_uhd_siggen = SshJob( scheduler=scheduler, node=node_scrambler, required=pings, label="killing uhd_siggen on the scrambler node {}".format( scrambler_id), verbose=verbose_jobs, #critical = True, commands=[Run("pkill", "uhd_siggen")], #keep_connection = True ) kill_2_uhd_siggen = SshJob( scheduler=scheduler, node=faraday, required=kill_uhd_siggen, label="turning off usrp on the scrambler node {}".format( scrambler_id), verbose=verbose_jobs, commands=[ Run("rhubarbe", "usrpoff", "fit{}".format(scrambler_id)) ], #keep_connection = True ) #if map: #scheduler.add(Sequence(*tracepaths, scheduler=scheduler)) #if warmup: # scheduler.add(Sequence(*warmup_pings_job, scheduler=scheduler)) pings.add(Sequence(*pings_job)) # for running sequentially we impose no limit on the scheduler # that will be limitied anyways by the very structure # of the required graph #jobs_window = None if dry_run: scheduler.export_as_pngfile(run_root / "experiment_graph") return True # if not in dry-run mode, let's proceed to the actual experiment ok = scheduler.orchestrate() #jobs_window=jobs_window) scheduler.shutdown() dot_file = run_root / "experiment_graph" if not dot_file.is_file(): scheduler.export_as_dotfile(dot_file) #TODO : Is it necessary? if the user want to see it he can just do it? #call(["dot", "-Tpng", dot_file, "-o", run_root / "experitment_graph.png"]) #ok=True #ok = False # give details if it failed if not ok: scheduler.debrief() scheduler.export_as_dotfile("debug") if ok and map: print("Creation of ROUTES files") post_processor = ProcessRoutes(run_root, exp_ids, node_ids) post_processor.run() if ok and route_sampling: post_processor = ProcessRoutes(run_root, exp_ids, node_ids) post_processor.run_sampled() print("END of creation for ROUTES FILES") # data acquisition is done, let's aggregate results # i.e. compute averages if ok and tshark: post_processor = Aggregator(run_root, node_ids, antenna_mask) post_processor.run() return ok
def run(self, verbose, no_load, no_save): """ can skip the load or save phases """ print("Using node {} through gateway {}".format( self.node, self.gateway)) print("In order to produce {} from {}".format(self.to_image, self.from_image)) print("The following scripts will be run:") for i, script in enumerate(self.scripts, 1): print("{:03d}:{}".format(i, " ".join(script))) items = [] if no_load: items.append("skip load") if no_save: items.append("skip save") if items: print("WARNING: using fast-track mode {}".format( ' & '.join(items))) self.locate_companion_shell() if verbose: print("Located companion in {}".format(self.companion)) if verbose: print("Preparing tar of input shell scripts .. ", end="") tarfile = self.prepare_tar(self.to_image) if verbose: print("Done in {}".format(tarfile)) keys = load_agent_keys() if verbose: print("We have found {} keys in the ssh agent".format(len(keys))) #################### the 2 nodes we need to talk to gateway_proxy = None gwuser, gwname = self.user_host(self.gateway) gateway_proxy = None if not gwuser else SshNode( hostname=gwname, username=gwuser, keys=keys, formatter=ColonFormatter(verbose=verbose), ) # really not sure it makes sense to use username other than root username, nodename = self.user_host(self.node) node_proxy = SshNode( gateway=gateway_proxy, hostname=nodename, username=username, keys=keys, formatter=ColonFormatter(verbose=verbose), ) banner = 20 * '=' # now that node_proxy is initialized, we need to # have a valid gateway_proxy for when we run all this from inside # the gateway if gateway_proxy is None: print("WARNING: build-image is designed to be run on your laptop") # best-effort, not even tested.... gateway_proxy = LocalNode() #################### the little pieces sequence = Sequence( PrintJob("Checking for a valid lease"), # bail out if we don't have a valid lease SshJob(node = gateway_proxy, command = "rhubarbe leases --check", critical = True), PrintJob("loading image {}".format(self.from_image) if not no_load else "fast-track: skipping image load", banner = banner, # label = "welcome message", ), SshJob( node = gateway_proxy, commands = [ Run("rhubarbe", "load", "-i", self.from_image, nodename) \ if not no_load else None, Run("rhubarbe", "wait", "-v", "-t", "240", nodename), ], # label = "load and wait image {}".format(self.from_image), ), SshJob( node = node_proxy, commands = [ Run("rm", "-rf", "/etc/rhubarbe-history/{}".format(self.to_image)), Run("mkdir", "-p", "/etc/rhubarbe-history"), Push(localpaths = tarfile, remotepath = "/etc/rhubarbe-history"), RunScript(self.companion, nodename, self.from_image, self.to_image), Pull(localpath = "{}/logs/".format(self.to_image), remotepaths = "/etc/rhubarbe-history/{}/logs/".format(self.to_image), recurse = True), ], label = "set up and run scripts in /etc/rhubarbe-history/{}".format(self.to_image)), ) # avoid creating an SshJob with void commands if self.extra_logs: sequence.append( SshJob( node=node_proxy, label="collecting extra logs", critical=False, commands=[ Pull(localpath="{}/logs/".format(self.to_image), remotepaths=extra_log, recurse=True) for extra_log in self.extra_logs ], )) # creating these as critical = True means the whole # scenario will fail if these are not found for binary in self.expected_binaries: check_with = "ls" if os.path.isabs(binary) else ("type -p") sequence.append( Sequence( PrintJob( "Checking for expected binaries", # label = "message about checking" ), SshJob( node=node_proxy, command=[check_with, binary], # label = "Checking for {}".format(binary) ))) # xxx some flag if no_save: sequence.append( PrintJob("fast-track: skipping image save", banner=banner)) else: sequence.append( Sequence( PrintJob("saving image {} ...".format(self.to_image), banner=banner), # make sure we capture all the logs and all that # mostly to test RunString SshJob( node=node_proxy, command=RunString("sync ; sleep $1; sync; sleep $1", 1), # label = 'sync', ), SshJob( node=gateway_proxy, command=Run("rhubarbe", "save", "-o", self.to_image, nodename), # label = "save image {}".format(self.to_image), ), SshJob( node=gateway_proxy, command="rhubarbe images -d", # label = "list current images", ), )) sched = Scheduler(sequence, verbose=verbose) # sanitizing for the cases where some pieces are left out sched.sanitize() print(20 * '+', "before run") sched.list(details=verbose) print(20 * 'x') if sched.orchestrate(): if verbose: print(20 * '+', "after run") sched.list() print(20 * 'x') print("image {} OK".format(self.to_image)) return True else: print("Something went wrong with image {}".format(self.to_image)) print(20 * '+', "after run - KO") sched.debrief() print(20 * 'x') return False
def main() -> bool: parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-U", "--url", default=default_topurl, dest='topurl', help="url to reach nbhosting server") parser.add_argument( "-i", "--indices", default=[0], action=IntsRanges, help="(cumulative) ranges of indices in the list of known notebooks" " - run nbhtest with -l to see list") parser.add_argument( "-u", "--users", default=[1], action=IntsRanges, help= "(cumulative) ranges of students indexes; e.g. -u 101-400 -u 501-600") parser.add_argument("-b", "--base", default='student', help="basename for students name") parser.add_argument("-p", "--period", default=default_period, type=float, help="delay between 2 triggers of nbhtest") parser.add_argument( "-s", "--sleep", default=default_sleep_internal, type=float, help="delay in seconds to sleep between actions inside nbhtest") parser.add_argument( "-g", "--go", default=default_go_between_notebooks, type=float, help="go/wait duration between 2 consecutive notebooks") parser.add_argument( "-c", "--cut", default=False, action='store_true', help="""just load the urls, don't do any further processing""") parser.add_argument("-w", "--window", default=default_window, type=int, help="window depth for spawning the nbhtest instances") parser.add_argument("--idle", default=None, help="monitor idle setting") parser.add_argument("-n", "--dry-run", action='store_true') parser.add_argument( "coursedirs", default=[default_course_gitdir], nargs='*', help="""a list of git repos where to fetch notebooks""") signature = "".join(sys.argv[1:]) args = parser.parse_args() local = LocalNode(formatter=TerminalFormatter( custom_format="%H-%M-%S:@line@", verbose=True)) if args.idle is not None: hostname = urlparse(args.topurl).netloc command = f"ssh root@{hostname} nbh test-set-monitor-idle {args.idle}" os.system(command) scheduler = Scheduler() for user in args.users: student_name = f"{args.base}-{user:04d}" for coursedir in args.coursedirs: command = (f"nbhtest.py -U {args.topurl} -u {student_name} " f"-s {args.sleep} -g {args.go} ") if args.cut: command += "-c " for index in args.indices: command += f"{coursedir}:{index} " command += " &" if args.dry_run: print("dry-run:", command) else: # schedule this command to run _job = SshJob(scheduler=scheduler, node=local, commands=[command, f"sleep {args.period}"]) if args.dry_run: return True scheduler.jobs_window = args.window overall = scheduler.orchestrate() if not overall: scheduler.debrief() untagged = Path("artefacts") tagged = Path(f"artefacts{signature}") if tagged.exists(): print(f"NOT RENAMING because {tagged} exists; command to run is") print(f"mv {untagged} {tagged}") else: print(f"renaming {untagged} into {tagged}") untagged.rename(tagged) print("nbhtests DONE") return overall