def test_requires_sequence(self): # we leave these untouched (no req. added) a1 = J(sl(0.1), label="a1") a2 = J(sl(0.1), label="a2") a3 = J(sl(0.1), label="a3") a4 = J(sl(0.1), label="a4") a5 = J(sl(0.1), label="a5") # re-create these each time to have fresh data def bs(): b1 = J(sl(0.1), label="b1") b2 = J(sl(0.1), label="b2") b3 = J(sl(0.1), label="b3") return b1, b2, b3 b1, b2, b3, *_ = bs() s1 = Seq(b1, b2, b3, required=[a1, a2]) self.assertEqual(len(b1.required), 2) self.assertEqual(len(b2.required), 1) b1, b2, b3, *_ = bs() s1 = Seq(b1, b2, b3) s1.requires([a1, a2]) self.assertEqual(len(b1.required), 2) self.assertEqual(len(b2.required), 1)
def test_nesting_sequence(self): expected_duration = 1. watch = Watch('test_nesting_sequence') subjob = Scheduler( Sequence( Job(co_print_sleep(watch, .2, "one")), Job(co_print_sleep(watch, .2, "two")), Job(co_print_sleep(watch, .2, "three")), ), watch=watch, label="sub-scheduler\non several lines", critical=True, forever=True, ) main = Scheduler(Sequence( Job(co_print_sleep(watch, .2, "BEGIN"), label="job-label"), subjob, Job(co_print_sleep(watch, .2, "END")), ), watch=watch) print("===== test_nesting_sequence", "LIST with details") main.list(details=True) self.assertTrue(main.run()) self.assertAlmostEqual(watch.seconds(), expected_duration, delta=.05) produce_png(main, "test_nesting_sequence")
def nested(): j11, j12, j13, j14, j15 = [job(i) for i in range(11, 16)] s2 = Scheduler(Sequence(j11, j12, j13), label="nested internal") j12.requires(j14) j13.requires(j15) j1, j2, j3, j4, j5 = [job(i) for i in range(1, 6)] s1 = Scheduler(Sequence(j1, s2, j3), label="nested top") j1.requires(j4) j1.requires(j11) s2.requires(j13) # j2 not included in sched, untouched j2.requires(j1) self.assertEqual(len(j12.required), 2) self.assertEqual(len(j13.required), 2) self.assertEqual(len(j1.required), 2) self.assertEqual(len(s2.required), 2) self.assertEqual(len(j3.required), 1) s1.sanitize() self.assertEqual(len(j12.required), 1) self.assertEqual(len(j13.required), 1) self.assertEqual(len(j1.required), 0) self.assertEqual(len(s2.required), 1) self.assertEqual(len(j3.required), 1)
def test_shutdown_nested_timeout(self): # so here we create 16 jobs for which the shutdown # durations will be # 0.0 0.1 0.2 0.3 - 1.0 1.1 1.2 1.3 # 2.0 2.1 2.2 2.3 - 3.0 3.1 3.2 3.3 # so if we set shutdown_timeout = 0.9s, we should # still find counter == 12 cardinal = 4 # same to the square top = CounterScheduler(label="TOP", shutdown_timeout=0.9) subs = [] for i in range(cardinal): sub = Scheduler(label=f"SUB {i}") subs.append(sub) sub.add( Sequence(*[ CounterJob(top, 10 * i + j, aprint('ok'), label=10 * i + j) for j in range(cardinal) ])) top.add(Sequence(*subs)) self.assertEqual(top.counter, 0) self.assertTrue(top.run()) self.assertEqual(top.counter, cardinal * cardinal) self.assertFalse(top.shutdown()) self.assertEqual(top.counter, cardinal * (cardinal - 1))
def test_creation_scheduler(self): sched = Scheduler() s = Seq(J(sl(1)), J(sl(2)), scheduler=sched) j = J(sl(3), required=s, scheduler=sched) # make sure that jobs appended in the sequence # even later on are also added to the scheduler s.append(J(sl(.5))) self.assertEqual(len(sched.jobs), 4) self.assertTrue(sched.rain_check()) self.assertTrue(sched.orchestrate())
def check_lease(experiment_scheduler, sshnode): """ re-usable function that acts a bit like a python decorator on schedulers. Given an experiment described as a scheduler, this function returns a higher-level scheduler that first checks for the lease, and then proceeds with the experiment. """ check_lease_job = SshJob( # checking the lease is done on the gateway node=faraday, # this means that a failure in any of the commands # will cause the scheduler to bail out immediately critical=True, command=Run("rhubarbe leases --check"), ) return Scheduler( Sequence( check_lease_job, # here we create a nested scheduler # by inserting the experiment_scheduler # as a regular job in the main scheduler experiment_scheduler, ))
def test_deferred_service(self): """ a service can be defined from a deferred instance rather than a plain string """ s = Scheduler() env = Variables() echo_service = Service(Deferred("echo {{run1}}", env), service_id='echo', verbose=True) n = SshNode(localhostname(), username=localuser()) Sequence(SshJob(n, commands=Run("echo from-first-run", capture=Capture('run1', env))), SshJob(n, commands=Run(echo_service.start_command())), SshJob(n, commands=Run(echo_service.journal_command( since="10 second ago"), capture=Capture('journal', env))), scheduler=s) print('STARTING', 20 * '-', echo_service.start_command()) s.run() print('DONE', 20 * '-', echo_service.start_command()) #print(f"env={env}") obtained = env.journal expected = "from-first-run" found = expected in obtained self.assertTrue(found)
def test_deferred_chain(self): """ one command computes a string that gets passed to another one this is analogous to run1=$(ssh localhost echo from-first-run) final=$(ssh localhost echo ${run1}) the 'final' variable is only needed for checking everything went well """ s = Scheduler() env = Variables() n = SshNode(localhostname(), username=localuser()) Sequence(SshJob(n, commands=Run("echo from-first-run", capture=Capture('run1', env))), SshJob(n, commands=Run(Deferred("echo {{run1}}", env), capture=Capture('final', env))), scheduler=s) s.run() #print(f"env={env}") obtained = env.final expected = "from-first-run" self.assertEqual(obtained, expected)
def test_png_styles2(self): """ trying the rendering of critical and forever jobs """ watch = Watch() j1 = pipes(watch, .5, "none", nb_pipes=6) j1.critical = False j1.forever = False j1.label = "label-none" j2 = diamond_scheduler(watch, .5, "critical") j2.critical = True j2.forever = False j2.label = "label-critical" j3 = diamond_scheduler(watch, .5, "forever") j3.critical = False j3.forever = True j3.label = "label-forever" j4 = diamond_scheduler(watch, .5, "both") j4.critical = True j4.forever = True j4.label = "label-both" sched = Scheduler( Sequence(j1, j2, j3, j4), watch=watch, ) produce_png(sched, "test_png_styles2")
def test_nesting1(self): """ one main scheduler in sequence one job one subscheduler that runs 2 jobs in parallel one job """ watch = Watch('test_nesting1') # sub-scheduler - total approx 1 s sub_sched = PureScheduler(watch=watch) Job(co_print_sleep(watch, 0.5, "sub short"), scheduler=sub_sched) Job(co_print_sleep(watch, 1, "sub longs"), scheduler=sub_sched) # main scheduler - total approx 2 s main_sched = PureScheduler(watch=watch) Sequence( Job(co_print_sleep(watch, 0.5, "main begin")), # this is where the subscheduler is merged Job(sub_sched.co_run(), label='subscheduler'), Job(co_print_sleep(watch, 0.5, "main end")), scheduler=main_sched) print("===== test_nesting1", "LIST with details") main_sched.list(details=True) ok = main_sched.run() self.assertTrue(ok) # allow for a small variation around 2s of course duration = watch.seconds() self.assertAlmostEqual(duration, 2, delta=0.05)
def test_order1(self): async def aprint(x): print(x) def job(n): return Job(aprint(n), label=n) sub1, sub2, sub3, sub4 = Scheduler(), Scheduler(), Scheduler(), Scheduler() sched = Scheduler( Sequence( job('top'), sub1, job('middle'), sub2, sub3, sub4)) for i in range(3): sub1.add(job(i+1)) sub2.add(job(i+4)) sub3.add(job(i+7)) sub4.add(job(i+10)) sub4.add(job(13)) produce_png(sched, "test_png_order1")
def test_nested_cycles(self): watch = Watch() def job(i): return Job(co_print_sleep(watch, .2, f"job {i}"), label=f"job{i}") js1, js2, js3 = [job(i) for i in range(11, 14)] s2 = Scheduler(Sequence(js1, js2, js3)) j1, j3 = job(1), job(3) s1 = Scheduler(Sequence(j1, s2, j3)) self.assertTrue(s1.check_cycles()) # create cycle in subgraph js1.requires(js3) self.assertFalse(s1.check_cycles()) # restore in OK state js1.requires(js3, remove=True) self.assertTrue(s1.check_cycles()) # add cycle in toplevel j1.requires(j3) self.assertFalse(s1.check_cycles()) # restore in OK state j1.requires(j3, remove=True) self.assertTrue(s1.check_cycles()) # add one level down s3 = Scheduler() jss1, jss2, jss3 = [job(i) for i in range(111, 114)] Sequence(jss1, jss2, jss3, scheduler=s3) # surgery in s2; no cycles s2.remove(js2) s2.sanitize() s2.add(s3) s3.requires(js1) js3.requires(s3) self.assertTrue(s1.check_cycles()) # add cycle in s3 js1.requires(js3) self.assertFalse(s1.check_cycles())
def test_shutdown_nested(self): cardinal = 4 # same to the square top = CounterScheduler(label="TOP") subs = [] for i in range(cardinal): sub = Scheduler(label=f"SUB {i}") subs.append(sub) sub.add( Sequence(*[ CounterJob(top, 0, aprint('ok'), label=10 * i + j) for j in range(cardinal) ])) top.add(Sequence(*subs)) self.assertEqual(top.counter, 0) self.assertTrue(top.run()) self.assertEqual(top.counter, cardinal * cardinal) self.assertTrue(top.shutdown()) self.assertEqual(top.counter, 0)
def test_shutdown_simple(self): cardinal = 5 sched = CounterScheduler() sched.add( Sequence(*[ CounterJob(sched, i / 10, aprint(i)) for i in range(cardinal) ])) self.assertEqual(sched.counter, 0) self.assertTrue(sched.run()) self.assertEqual(sched.counter, cardinal) self.assertTrue(sched.shutdown()) self.assertEqual(sched.counter, 0)
def test_png_easy2(self): """ same but with a multi-entry/multi-exit sub-sched """ watch = Watch() sched = Scheduler( Sequence( Job(co_print_sleep(watch, .2, "beg"), label="test_png_easy2"), pipes(watch, .6, "middle"), Job(co_print_sleep(watch, .2, "end")), ), watch=watch ) produce_png(sched, "test_png_easy2")
def test_png_easy(self): """ start with an easy one, a sequence that has a diamond inside """ watch = Watch() sched = Scheduler( Sequence( Job(co_print_sleep(watch, .2, "beg"), label="test_easy"), diamond_scheduler(watch, .6, "middle"), Job(co_print_sleep(watch, .2, "end")), ), watch=watch ) produce_png(sched, "test_png_easy")
def test_graph2(self): class TextJob(Job): def __init__(self, text, *args, **kwds): self.text = text super().__init__(*args, **kwds) def text_label(self): return "[[TextJob {}]]".format(self.text[::-1]) def details(self): return f"TextJob details say\n" \ f"that initial text was {self.text}" class GraphJob(Job): def __init__(self, graph, *args, **kwds): self.graph = graph super().__init__(*args, **kwds) def graph_label(self): return "[[GraphJob\n{}]]".format(self.graph[::-1]) def details(self): return f"GraphJob details\nare even more verbose and say\n" \ f"that initial graph message\nwas {self.graph}" watch = Watch() s = PureScheduler() s.add( Sequence( TextJob('textjob-with', co_print_sleep(watch, 0.1, 'textjob, no label ')), TextJob('textjob-without', co_print_sleep(watch, 0.1, 'textjob, with label '), label='TextLabel'), GraphJob('graphjob-with', co_print_sleep(watch, 0.1, 'graphjob, no label ')), GraphJob('graphjob-without', co_print_sleep(watch, 0.1, 'graphjob, with label '), label='GraphLabel'), )) print("test_graph2 NO DETAILS") s.list() print("test_graph2 WITH DETAILS") s.list(details=True) print("GRAPH") self.assertEqual(len(s), 4) produce_png(s, "test_graph2")
def test_png_simple(self): """ a sequence that has 2 subscheds in a row """ watch = Watch() sched = Scheduler( Sequence( Job(co_print_sleep(watch, .2, "beg"), label="test_png_simple start"), diamond_scheduler(watch, .6, "middle-up"), pipes(watch, .6, "middle-down"), Job(co_print_sleep(watch, .2, "end"), label="test_png_simple end"), ), watch=watch ) produce_png(sched, "test_png_simple")
def test_variables(self): """ check how variables are rendered """ v1 = Variables() v2 = Variables() v2.defined = 'OK-defined' template = "defined: {{defined}} undefined: {{undefined}}" scheduler = Scheduler(critical=False) gateway = SshNode(hostname=localhostname(), username=localuser()) Sequence(SshJob(gateway, command=Deferred(template, v1)), SshJob(gateway, command=Deferred(template, v2)), scheduler=scheduler) produce_png(scheduler, "test_graphics_variables")
def test_graph1(self): watch = Watch() s = PureScheduler() s.add( Sequence( Job(co_print_sleep(watch, .25, 'begin')), Job(co_print_sleep(watch, 1., 'middle'), label='middle'), Job(co_print_sleep(watch, .25, 'end')), )) print("test_graph1 NO DETAILS") s.list() print("test_graph1 WITH DETAILS") s.list(details=True) print("GRAPH") self.assertEqual(len(s), 3) s.run() self.assertAlmostEqual(watch.seconds(), 1.5, delta=0.05) produce_png(s, "test_graph1")
def all_off(slice, verbose, debug): """ expects a slice name, and turns off faraday completely """ # what argparse knows as a slice actually is a gateway (user + host) try: gwuser, gwhost = slice.split('@') except: gwuser, gwhost = slice, "faraday.inria.fr" gwnode = SshNode(hostname=gwhost, username=gwuser, formatter=ColonFormatter(verbose=verbose), debug=debug) scheduler = Scheduler( Sequence( SshJob( node=gwnode, command=Run("rhubarbe", "leases", "--check"), label="check we have a current lease", ), SshJob( node=gwnode, command=Run("rhubarbe", "bye"), label="turn off", ))) result = scheduler.orchestrate() if not result: if check_for_lease.raised_exception(): print( "slice {} does not appear to hold a valid lease".format(slice)) else: print("RUN KO : {}".format(scheduler.why())) sched.debrief() else: print("faraday turned off OK") return 0 if result else 1
def test_commands_verbose(self): dummy_path = "tests/dummy-10" dummy_file = Path(dummy_path).name scheduler = Scheduler() Sequence(SshJob( node=self.gateway(), verbose=True, commands=[ Run("hostname"), RunScript("tests/script-with-args.sh", "arg1", "arg2"), RunString("for i in $(seq 3); do echo verbose$i; done"), Push(localpaths=dummy_path, remotepath="."), Pull(remotepaths=dummy_file, localpath=dummy_path + ".loop"), ]), SshJob(node=LocalNode(), critical=True, commands=Run("diff {x} {x}.loop".format(x=dummy_path), verbose=True)), scheduler=scheduler) ok = scheduler.run() ok or scheduler.debrief() self.assertTrue(ok)
def test_png_styles1(self): """ trying the rendering of critical and forever jobs """ watch = Watch() sched = Scheduler( Sequence( Job(co_print_sleep(watch, .1, "regular"), label="regular", critical=False, forever=False), Job(co_print_sleep(watch, .1, "critical"), label="critical", critical=True, forever=False), Job(co_print_sleep(watch, .1, "forever"), label="forever", critical=False, forever=True), Job(co_print_sleep(watch, .1, "both"), label="both", critical=True, forever=True), ), watch=watch, ) produce_png(sched, "test_png_styles1")
def one_run(tx_power, phy_rate, antenna_mask, channel, *, run_name=default_run_name, slicename=default_slicename, load_images=False, node_ids=None, parallel=None, verbose_ssh=False, verbose_jobs=False, dry_run=False): """ Performs data acquisition on all nodes with the following settings Arguments: tx_power: in dBm, a string like 5, 10 or 14 phy_rate: a string among 1, 54 antenna_mask: a string among 1, 3, 7 channel: a string like e.g. 1 or 40 run_name: the name for a subdirectory where all data will be kept successive runs should use the same name for further visualization slicename: the Unix login name (slice name) to enter the gateway load_images: a boolean specifying whether nodes should be re-imaged first node_ids: a list of node ids to run the scenario on; strings or ints are OK; defaults to the all 37 nodes i.e. the whole testbed parallel: a number of simulataneous jobs to run 1 means all data acquisition is sequential (default) 0 means maximum parallel """ # # dry-run mode # just display a one-liner with parameters # if dry_run: load_msg = "" if not load_images else " LOAD" nodes = " ".join(str(n) for n in node_ids) print("dry-run: {run_name}{load_msg} -" " t{tx_power} r{phy_rate} a{antenna_mask} ch{channel} -" "nodes {nodes}" .format(**locals())) # in dry-run mode we are done return True # set default for the nodes parameter node_ids = [int(id) for id in node_ids] if node_ids is not None else default_node_ids ### # create the logs directory based on input parameters run_root = naming_scheme(run_name, tx_power, phy_rate, antenna_mask, channel, autocreate=True) # the nodes involved faraday = SshNode(hostname=default_gateway, username=slicename, formatter=TimeColonFormatter(), verbose=verbose_ssh) # this is a python dictionary that allows to retrieve a node object # from an id node_index = { id: SshNode(gateway=faraday, hostname=fitname(id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) for id in node_ids } # the global scheduler scheduler = Scheduler(verbose=verbose_jobs) ########## check_lease = SshJob( scheduler=scheduler, node=faraday, verbose=verbose_jobs, critical=True, command=Run("rhubarbe leases --check"), ) # load images if requested green_light = check_lease if load_images: # the nodes that we **do not** use should be turned off # so if we have selected e.g. nodes 10 12 and 15, we will do # rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15 negated_node_ids = ["~{}".format(id) for id in node_ids] # replace green_light in this case green_light = SshJob( node=faraday, required=check_lease, critical=True, scheduler=scheduler, verbose=verbose_jobs, commands=[ Run("rhubarbe", "off", "-a", *negated_node_ids), Run("rhubarbe", "load", "-i", "u16-ath-noreg", *node_ids), Run("rhubarbe", "wait", *node_ids) ] ) ########## # setting up the wireless interface on all nodes # # this is a python feature known as a list comprehension # we just create as many SshJob instances as we have # (id, SshNode) couples in node_index # and gather them all in init_wireless_jobs # they all depend on green_light # # provide node-utilities with the ranges/units it expects frequency = channel_frequency[int(channel)] # tx_power_in_mBm not in dBm tx_power_driver = tx_power * 100 if load_images: # The first init_wireless_jobs always has troubles... Do it twice the first time (nasty hack) init_wireless_jobs = [ SshJob( scheduler=scheduler, required=green_light, node=node, verbose=verbose_jobs, label="init {}".format(id), commands=[ RunScript("node-utilities.sh", "init-ad-hoc-network", wireless_driver, "foobar", frequency, phy_rate, antenna_mask, tx_power_driver), RunScript("node-utilities.sh", "init-ad-hoc-network", wireless_driver, "foobar", frequency, phy_rate, antenna_mask, tx_power_driver) ] ) for id, node in node_index.items()] else: init_wireless_jobs = [ SshJob( scheduler=scheduler, required=green_light, node=node, verbose=verbose_jobs, label="init {}".format(id), command=RunScript("node-utilities.sh", "init-ad-hoc-network", wireless_driver, "foobar", frequency, phy_rate, antenna_mask, tx_power_driver) ) for id, node in node_index.items()] # then install and run olsr on fit nodes run_olsr = [ SshJob( scheduler=scheduler, node=node, required=init_wireless_jobs, label="init and run olsr on fit nodes", verbose=verbose_jobs, command=RunScript("node-utilities.sh", "run-olsr") ) for i, node in node_index.items()] # after that, run tcpdump on fit nodes, this job never ends... run_tcpdump = [ SshJob( scheduler=scheduler, node=node, required=run_olsr, label="run tcpdump on fit nodes", verbose=verbose_jobs, commands=[ Run("echo run tcpdump on fit{:02d}".format(i)), Run("tcpdump -U -i moni-{} -y ieee802_11_radio -w /tmp/fit{}.pcap".format(wireless_driver, i)) ] ) for i, node in node_index.items()] # let the wireless network settle settle_wireless_job = PrintJob( "Let the wireless network settle", sleep=settle_delay, scheduler=scheduler, required=run_olsr, label="settling") ########## # create all the ping jobs, i.e. max*(max-1)/2 # this again is a python list comprehension # see the 2 for instructions at the bottom # # notice that these SshJob instances are not yet added # to the scheduler, we will add them later on # depending on the sequential/parallel strategy pings = [ SshJob( node=nodei, required=settle_wireless_job, label="ping {} -> {}".format(i, j), verbose=verbose_jobs, commands=[ Run("echo {} '->' {}".format(i, j)), RunScript("node-utilities.sh", "my-ping", "10.0.0.{}".format(j), ping_timeout, ping_interval, ping_size, ping_number, ">", "PING-{:02d}-{:02d}".format(i, j)), Pull(remotepaths="PING-{:02d}-{:02d}".format(i, j), localpath=str(run_root)), ] ) # looping on the source, now only fit01 is source for i, nodei in node_index.items() # and on the destination for j, nodej in node_index.items() # and keep only half of the couples if (j > i) and (i==1) ] # retrieve all pcap files from fit nodes retrieve_tcpdump = [ SshJob( scheduler=scheduler, node=nodei, required=pings, label="retrieve pcap trace from fit{:02d}".format(i), verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "kill-olsr"), Run("sleep 1;pkill tcpdump; sleep 1"), RunScript("node-utilities.sh", "process-pcap", i), Run( "echo retrieving pcap trace and result-{i}.txt from fit{i:02d}".format(i=i)), Pull(remotepaths=["/tmp/fit{}.pcap".format(i), "/tmp/result-{}.txt".format(i)], localpath=str(run_root)), ] ) for i, nodei in node_index.items() ] # xxx this is a little fishy # should we not just consider that the default is parallel=1 ? if parallel is None: # with the sequential strategy, we just need to # create a Sequence out of the list of pings # Sequence will add the required relationships scheduler.add(Sequence(*pings, scheduler=scheduler)) # for running sequentially we impose no limit on the scheduler # that will be limitied anyways by the very structure # of the required graph jobs_window = None else: # with the parallel strategy # we just need to insert all the ping jobs # as each already has its required OK scheduler.update(pings) # this time the value in parallel is the one # to use as the jobs_limit; if 0 then inch'allah jobs_window = parallel # if not in dry-run mode, let's proceed to the actual experiment ok = scheduler.orchestrate(jobs_window=jobs_window) # give details if it failed if not ok: scheduler.debrief() # data acquisition is done, let's aggregate results # i.e. compute averages if ok: post_processor = Aggregator(run_root, node_ids, antenna_mask) post_processor.run() return ok
def test_graphics1(self): scheduler = Scheduler(critical=False) gateway = SshNode(hostname=localhostname(), username=localuser()) Sequence( SshJob( node=gateway, command='hostname', ), SshJob(node=gateway, command=[ Run('ls /etc/passwd'), Run('wc -l /etc/passwd'), ]), SshJob(node=gateway, commands=[ RunString( "#!/usr/bin/env bash\n" "echo with RunString on $(hostname) at $(date)"), ]), SshJob(node=gateway, commands=[ RunScript("tests/testbasic.sh"), ]), SshJob(node=gateway, commands=[ Run('wc -l /etc/passwd'), RunString( "#!/usr/bin/env bash\n" "echo with RunsString on $(hostname) at $(date)", remote_name="show-host-date"), RunScript("tests/testbasic.sh"), ]), SshJob(node=gateway, commands=[ RunString( "#!/usr/bin/env bash\n" "echo first arg is $1\n", 10) ]), SshJob(node=gateway, commands=[ RunString( "#!/usr/bin/env bash\n" "echo first arg is $1\n", 10, remote_name='short-show-args') ]), SshJob(node=gateway, commands=[ RunString( "#!/usr/bin/env bash\n" "echo first arg is $1\n" "echo second arg is $2\n" "echo third arg is $3\n" "echo fourth arg is $4\n", 100, 200, 300, 400) ]), SshJob(node=gateway, commands=[ RunString( "#!/usr/bin/env bash\n" "echo first arg is $1\n" "echo second arg is $2\n" "echo third arg is $3\n" "echo fourth arg is $4\n", 1000, 2000, 3000, 4000, remote_name='long-show-args') ]), SshJob(node=gateway, commands=[ RunString( "#!/usr/bin/env bash\n" "echo first arg is $1\n" "echo second arg is $2\n" "echo third arg is $3\n" "echo fourth arg is $4\n", 1000, 2000, 3000, 4000, remote_name='long-show-args', label='snip') ]), SshJob(node=gateway, commands=[ Run("hostname", label="Run()"), RunScript("foobar", label="RunScript()"), RunString("foobar", label="RunString()"), Push("foobar", remotepath="remote", label="Push()"), Pull("remote", localpath="foobar", label="Pull()"), Run("hostname", label=None), RunScript("foobar", label=[]), RunString("foobar", label=0), Push("foobar", remotepath="remote", label={}), Pull("remote", localpath="foobar", label=""), ]), scheduler=scheduler, ) print("NO DETAILS") scheduler.list() print("WITH DETAILS") scheduler.list(details=True) produce_png(scheduler, "test_graphics1") ok = scheduler.run() self.assertFalse(ok)
transfer_job = Sequence( # start the receiver - this of course returns immediately SshJob( node = node2, commands = [ Run("netcat", "-l", "data02", netcat_port, ">", "RANDOM", "&"), ], ), # start the sender SshJob( node = node1, # ignore netcat result critical = False, commands = [ # let the server warm up just in case Run("sleep 1"), Run("netcat", "data02", netcat_port, "<", "RANDOM"), Run("echo SENDER DONE"), ], ), # check contents on the receiving end SshJob( node=node2, commands = [ Run("ls -l RANDOM"), Run("sha1sum RANDOM"), ], ), ### these two apply to the Sequence # required applies to the first job in the sequence required = turn_on_datas, # scheduler applies to all jobs in the sequence scheduler = scheduler, )
def one_run(tx_power, phy_rate, antenna_mask, channel, interference, protocol, *, run_name=default_run_name, slicename=default_slicename, load_images=False, node_ids=None, verbose_ssh=False, verbose_jobs=False, dry_run=False, tshark=False, map=False, warmup=False, exp=default_exp, dest=default_node_ids, ping_number=default_ping_number, route_sampling=False): """ Performs data acquisition on all nodes with the following settings Arguments: tx_power: in dBm, a string like 5, 10 or 14. Correspond to the transmission power. phy_rate: a string among 1, 54. Correspond to the wifi rate. antenna_mask: a string among 1, 3, 7. channel: a string like e.g. 1 or 40. Correspond to the channel. protocol: a string among batman , olsr. Correspond to the protocol interference : in dBm, a string like 60 or 50. Correspond to the power of the noise generated in the root. run_name: the name for a subdirectory where all data will be kept successive runs should use the same name for further visualization slicename: the Unix login name (slice name) to enter the gateway load_images: a boolean specifying whether nodes should be re-imaged first node_ids: a list of node ids to run the scenario against; strings or ints are OK; defaults to the nodes [1, 4, 5, 12, 19, 22,27 ,31, 33, 37] tshark: a boolean specifying wether we should format/parse the .pcap. map: a boolean specifying wether we should fetch/parse the route tables of the nodes. warmup: a boolean specifying wether we should run a ping before the experiment to be certain of the stabilisation on the network. exp: a list of nodes from which we will launch the ping from. strings or ints are OK. default to the node [1] ping_number : The number of pings that will be generated """ # set default for the nodes parameter node_ids = [int(id) for id in node_ids ] if node_ids is not None else default_node_ids exp_ids = [int(id) for id in exp] if exp is not None else default_exp dest_ids = [int(id) for id in dest] if dest is not None else default_node_ids # # dry-run mode # just display a one-liner with parameters # if dry_run: print("************************************") print("\n") run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=False) load_msg = "" if not load_images else " LOAD" nodes = " ".join(str(n) for n in node_ids) exps = " ".join(str(n) for n in exp) pingst = [ "PING{}-->{}".format(e, j) for e in exp_ids # and on the destination for j in node_ids if e != j #and not #(j in exp_ids and j < e) ] print( "dry-run:{protocol} {run_name}{load_msg} -" " t{tx_power} r{phy_rate} a{antenna_mask} ch{channel} I{interference}-" "nodes {nodes}" " exp {exps}".format(**locals())) print( "\nNodes from which the experiment will be launched : \n{}\nList of pings generated:\n" .format(exps)) print(pingst) print("\n") if warmup: print("Will do warmup pings\n") if tshark: print( "Will format data using tshark and will agregate the RSSI into one RSSI.txt file" ) if map: print( "Will fetch the routing tables of the node (when stabilited) and will agregate the results\n" ) if route_sampling: print("Will launch route sampling services on nodes") #print("Test creation of ROUTES files") #post_processor= ProcessRoutes(run_root, exp_ids, node_ids) #post_processor.run() #print("\nList of tracepaths generated:\n{}".format(tracepathst)) # in dry-run mode we are done ### # create the logs directory based on input parameters run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=False) if (run_root.is_dir()): purgedir(run_root) run_root = naming_scheme(protocol, run_name, tx_power, phy_rate, antenna_mask, channel, interference, autocreate=True) exp_info_file_name = run_root / "info.txt" with exp_info_file_name.open("w") as info_file: info_file.write("Selected nodes : \n") for node in node_ids[:-1]: info_file.write(f"{node} ") info_file.write(f"{node_ids[-1]}") info_file.write("\nSources : \n") for src in exp_ids[:-1]: info_file.write(f"{src} ") info_file.write(f"{exp_ids[-1]}") info_file.write("\nDestinations : \n") for dest in dest_ids[:-1]: info_file.write(f"{dest} ") info_file.write(f"{dest_ids[-1]}" + "\n") # the nodes involved faraday = SshNode(hostname=default_gateway, username=slicename, formatter=TimeColonFormatter(), verbose=verbose_ssh) # this is a python dictionary that allows to retrieve a node object # from an id node_index = { id: SshNode(gateway=faraday, hostname=fitname(id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) for id in node_ids } if interference != "None": node_scrambler = SshNode(gateway=faraday, hostname=fitname(scrambler_id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) # the global scheduler scheduler = Scheduler(verbose=verbose_jobs) # if tshark: #scheduler_monitoring = Scheduler(verbose=verbose_jobs) #if interference != "None": #scheduler_interferences = Scheduler(verbose=verbose_jobs) ########## check_lease = SshJob( scheduler=scheduler, node=faraday, verbose=verbose_jobs, critical=True, label="rhubarbe check lease", command=Run("rhubarbe leases --check", label="rlease"), #keep_connection = True ) # load images if requested green_light = check_lease if load_images: # the nodes that we **do not** use should be turned off # so if we have selected e.g. nodes 10 12 and 15, we will do # rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15 negated_node_ids = ["~{}".format(id) for id in node_ids] #Add the id of the scrambler in the list and load the gnuradio image negated_node_ids.append("~{}".format(scrambler_id)) load_ids = [int(id) for id in node_ids ] if node_ids is not None else default_node_ids load_ids.append(scrambler_id) # replace green_light in this case #We use a modified image of gnuradio where uhd_siggen handle the signal SIGTERM in order to finish properly green_light = SshJob( node=faraday, required=check_lease, #critical=True, scheduler=scheduler, verbose=verbose_jobs, label="rhubarbe load/wait on nodes {}".format(load_ids), commands=[ Run("rhubarbe", "off", "-a", *negated_node_ids, label="roff {}".format(negated_node_ids)), Run("rhubarbe", "load", *node_ids, label="rload {}".format(node_ids)), Run("rhubarbe", "load", "-i", "gnuradio_batman", scrambler_id, label="load gnuradio batman on {}".format(scrambler_id)), Run("rhubarbe", "wait", *load_ids, label="rwait") ], #keep_connection = True ) ########## # setting up the wireless interface on all nodes # # this is a python feature known as a list comprehension # we just create as many SshJob instances as we have # (id, SshNode) couples in node_index # and gather them all in init_wireless_jobs # they all depend on green_light # # provide node-utilities with the ranges/units it expects frequency = channel_frequency[int(channel)] # tx_power_in_mBm not in dBm tx_power_driver = tx_power * 100 init_wireless_sshjobs = [ SshJob( #scheduler=scheduler, #required=green_light, node=node, verbose=verbose_jobs, label="init {}".format(id), command=RunScript("node-utilities.sh", "init-ad-hoc-network-{}".format(wireless_driver), wireless_driver, "foobar", frequency, phy_rate, antenna_mask, tx_power_driver, label="init add-hoc network"), #keep_connection = True ) for id, node in node_index.items() ] init_wireless_jobs = Scheduler( *init_wireless_sshjobs, scheduler=scheduler, required=green_light, #critical = True, verbose=verbose_jobs, label="Initialisation of wireless chips") green_light_prot = init_wireless_jobs if interference != "None": #Run uhd_siggen with the chosen power frequency_str = frequency / 1000 frequency_str = str(frequency_str) + "G" init_scrambler_job = [ SshJob( forever=True, node=node_scrambler, verbose=verbose_jobs, label="init scrambler on node {}".format(scrambler_id), command=RunScript("node-utilities.sh", "init-scrambler", interference, frequency_str, label="init scambler"), #keep_connection = True ) ] init_scrambler = Scheduler( *init_scrambler_job, scheduler=scheduler, required=green_light, #forever = True, #critical = True, verbose=verbose_jobs, label="Running interference") # then install and run batman on fit nodes run_protocol_job = [ SshJob( #scheduler=scheduler, node=node, #required=green_light_prot, label="init and run {} on fit node {}".format(protocol, i), verbose=verbose_jobs, command=RunScript("node-utilities.sh", "run-{}".format(protocol), label="run {}".format(protocol)), #keep_connection = True ) for i, node in node_index.items() ] run_protocol = Scheduler( *run_protocol_job, scheduler=scheduler, required=green_light_prot, #critical = True, verbose=verbose_jobs, label="init and run routing protocols") # after that, run tcpdump on fit nodes, this job never ends... if tshark: run_tcpdump_job = [ SshJob( #scheduler=scheduler_monitoring, node=node, forever=True, label="run tcpdump on fit node".format(i), verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "run-tcpdump", wireless_driver, i, label="run tcpdump") ], #keep_connection = True ) for i, node in node_index.items() ] run_tcpdump = Scheduler( *run_tcpdump_job, scheduler=scheduler, required=run_protocol, forever=True, #critical = True, verbose=verbose_jobs, label="Monitoring (tcpdum) Jobs") # let the wireless network settle settle_wireless_job = PrintJob( "Let the wireless network settle", sleep=settle_delay, scheduler=scheduler, required=run_protocol, label="settling for {} sec".format(settle_delay)) green_light_experiment = settle_wireless_job if warmup: warmup_pings_job = [ SshJob( node=nodei, #required=green_light_experiment, label="warmup ping {} -> {}".format(i, j), verbose=verbose_jobs, commands=[ Run("echo {} '->' {}".format(i, j), label="ping {} '->' {}".format(i, j)), RunScript("node-utilities.sh", "my-ping", "10.0.0.{}".format(j), ping_timeout, ping_interval, ping_size, ping_number, label="") ], #keep_connection = True ) #for each selected experiment nodes for e in exp_ids # looping on the source (to get the correct sshnodes) for i, nodei in node_index.items() # and on the destination for j, nodej in node_index.items() # and keep only sources that are in the selected experiment nodes and remove destination that are themselves # and remove the couples that have already be done # print("i {index} exp {expe}".format(index = i, expe= exp)) if (i == e) and e != j and not (j in exp_ids and j < e) ] warmup_pings = Scheduler( Sequence(*warmup_pings_job), scheduler=scheduler, required=green_light_experiment, #critical = True, verbose=verbose_jobs, label="Warmup ping") settle_wireless_job2 = PrintJob( "Let the wireless network settle", sleep=settle_delay / 2, scheduler=scheduler, required=warmup_pings, label="settling-warmup for {} sec".format(settle_delay / 2)) green_light_experiment = settle_wireless_job2 ########## # create all the tracepath jobs from the first node in the list # if map: routes_job = [ SshJob( node=nodei, #scheduler=scheduler, #required=green_light_experiment, label="Generating ROUTE file for prot {} on node {}".format( protocol, i), verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "route-{}".format(protocol), ">", "ROUTE-TABLE-{:02d}".format(i), label="get route table"), Pull(remotepaths="ROUTE-TABLE-{:02d}".format(i), localpath=str(run_root), label="") ], #keep_connection = True ) for i, nodei in node_index.items() ] routes = Scheduler( *routes_job, scheduler=scheduler, required=green_light_experiment, #critical = True, verbose=verbose_jobs, label="Snapshoting route files") green_light_experiment = routes if route_sampling: routes_sampling_job2 = [ SshJob( node=nodei, label="Route sampling service for prot {} on node {}".format( protocol, i), verbose=False, #forever = True, commands=[ Push(localpaths=["route_sample_service.sh"], remotepath=".", label=""), Run("source", "route_sample_service.sh;", "route-sample", "ROUTE-TABLE-{:02d}-SAMPLED".format(i), "{}".format(protocol), label="run route sampling service"), ], #keep_connection = True ) for i, nodei in node_index.items() ] routes_sampling_job = [ SshJob( node=nodei, label="Route sampling service for prot {} on node {}".format( protocol, i), verbose=False, forever=True, #critical = True, #required = green_light_experiment, #scheduler = scheduler, commands=[ RunScript("route_sample_service.sh", "route-sample", "ROUTE-TABLE-{:02d}-SAMPLED".format(i), "{}".format(protocol), label="run route sampling service"), ], #keep_connection = True ) for i, nodei in node_index.items() ] routes_sampling = Scheduler( *routes_sampling_job, scheduler=scheduler, verbose=False, forever=True, #critical = True, label="Route Sampling services launch", required=green_light_experiment) ########## # create all the ping jobs, i.e. max*(max-1)/2 # this again is a python list comprehension # see the 2 for instructions at the bottom # # notice that these SshJob instances are not yet added # to the scheduler, we will add them later on # depending on the sequential/parallel strategy pings_job = [ SshJob( node=nodei, #required=green_light_experiment, label="ping {} -> {}".format(i, j), verbose=verbose_jobs, commands=[ Run("echo {} '->' {}".format(i, j), label="ping {}'->' {}".format(i, j)), RunScript("node-utilities.sh", "my-ping", "10.0.0.{}".format(j), ping_timeout, ping_interval, ping_size, ping_number, ">", "PING-{:02d}-{:02d}".format(i, j), label=""), Pull(remotepaths="PING-{:02d}-{:02d}".format(i, j), localpath=str(run_root), label=""), ], #keep_connection = True ) #for each selected experiment nodes for e in exp_ids # looping on the source (to get the correct sshnodes) for i, nodei in node_index.items() # and on the destination for j in dest_ids # and keep only sources that are in the selected experiment nodes and remove destination that are themselves # and remove the couples that have already be done if (i == e) and e != j and not (j in exp_ids and j < e) ] pings = Scheduler( scheduler=scheduler, label="PINGS", #critical = True, verbose=verbose_jobs, required=green_light_experiment) # retrieve all pcap files from fit nodes stop_protocol_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="kill routing protocol on fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ RunScript("node-utilities.sh", "kill-{}".format(protocol), label="kill-{}".format(protocol)), ], #keep_connection = False ) for i, nodei in node_index.items() ] stop_protocol = Scheduler( *stop_protocol_job, scheduler=scheduler, required=pings, #critical = True, label="Stop routing protocols", ) if tshark: retrieve_tcpdump_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="retrieve pcap trace from fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ # RunScript("node-utilities.sh", "kill-{}".format(protocol), label = "kill-{}".format(protocol)), RunScript("node-utilities.sh", "kill-tcpdump", label="kill-tcpdump"), #Run("sleep 1"), Run("echo retrieving pcap trace and result-{i}.txt from fit{i:02d}" .format(i=i), label=""), Pull(remotepaths=["/tmp/fit{}.pcap".format(i)], localpath=str(run_root), label=""), ], #keep_connection = True ) for i, nodei in node_index.items() ] retrieve_tcpdump = Scheduler( *retrieve_tcpdump_job, scheduler=scheduler, required=pings, #critical = True, label="Retrieve tcpdump", ) if route_sampling: retrieve_sampling_job = [ SshJob( #scheduler=scheduler, node=nodei, #required=pings, label="retrieve sampling trace from fit{:02d}".format(i), verbose=verbose_jobs, #critical = True, commands=[ #RunScript("node-utilities.sh", "kill-route-sample", protocol, # label = "kill route sample"), RunScript("route_sample_service.sh", "kill-route-sample", label="kill route sample"), Run("echo retrieving sampling trace from fit{i:02d}". format(i=i), label=""), Pull(remotepaths=["ROUTE-TABLE-{:02d}-SAMPLED".format(i)], localpath=str(run_root), label=""), ], #keep_connection = True ) for i, nodei in node_index.items() ] retrieve_sampling = Scheduler( *retrieve_sampling_job, scheduler=scheduler, required=pings, #critical=True, verbose=verbose_jobs, label="Retrieve & stopping route sampling", ) if tshark: parse_pcaps_job = [ SshJob( #scheduler=scheduler, node=LocalNode(), #required=retrieve_tcpdump, label="parse pcap trace {path}/fit{node}.pcap".format( path=run_root, node=i), verbose=verbose_jobs, #commands = [RunScript("parsepcap.sh", run_root, i)] commands=[ Run("tshark", "-2", "-r", "{path}/fit{node}.pcap".format(path=run_root, node=i), "-R", "'(ip.dst==10.0.0.{node} && icmp) && radiotap.dbm_antsignal'" .format(node=i), "-Tfields", "-e", "'ip.src'", "-e" "'ip.dst'", "-e", "'radiotap.dbm_antsignal'", ">", "{path}/result-{node}.txt".format(path=run_root, node=i), label="parse pcap locally") ], #keep_connection = True ) for i in node_ids ] parse_pcaps = Scheduler( *parse_pcaps_job, scheduler=scheduler, required=retrieve_tcpdump, #critical=True, label="Parse pcap", ) #TODO: TURN OFF USRP if interference != "None": kill_uhd_siggen = SshJob( scheduler=scheduler, node=node_scrambler, required=pings, label="killing uhd_siggen on the scrambler node {}".format( scrambler_id), verbose=verbose_jobs, #critical = True, commands=[Run("pkill", "uhd_siggen")], #keep_connection = True ) kill_2_uhd_siggen = SshJob( scheduler=scheduler, node=faraday, required=kill_uhd_siggen, label="turning off usrp on the scrambler node {}".format( scrambler_id), verbose=verbose_jobs, commands=[ Run("rhubarbe", "usrpoff", "fit{}".format(scrambler_id)) ], #keep_connection = True ) #if map: #scheduler.add(Sequence(*tracepaths, scheduler=scheduler)) #if warmup: # scheduler.add(Sequence(*warmup_pings_job, scheduler=scheduler)) pings.add(Sequence(*pings_job)) # for running sequentially we impose no limit on the scheduler # that will be limitied anyways by the very structure # of the required graph #jobs_window = None if dry_run: scheduler.export_as_pngfile(run_root / "experiment_graph") return True # if not in dry-run mode, let's proceed to the actual experiment ok = scheduler.orchestrate() #jobs_window=jobs_window) scheduler.shutdown() dot_file = run_root / "experiment_graph" if not dot_file.is_file(): scheduler.export_as_dotfile(dot_file) #TODO : Is it necessary? if the user want to see it he can just do it? #call(["dot", "-Tpng", dot_file, "-o", run_root / "experitment_graph.png"]) #ok=True #ok = False # give details if it failed if not ok: scheduler.debrief() scheduler.export_as_dotfile("debug") if ok and map: print("Creation of ROUTES files") post_processor = ProcessRoutes(run_root, exp_ids, node_ids) post_processor.run() if ok and route_sampling: post_processor = ProcessRoutes(run_root, exp_ids, node_ids) post_processor.run_sampled() print("END of creation for ROUTES FILES") # data acquisition is done, let's aggregate results # i.e. compute averages if ok and tshark: post_processor = Aggregator(run_root, node_ids, antenna_mask) post_processor.run() return ok
transfer_job = Sequence( # start the receiver - this of course returns immediately SshJob( node=node2, commands=[ Run("netcat", "-l", "data02", netcat_port, ">", "RANDOM", "&"), ], ), # start the sender SshJob( node=node1, # ignore netcat result critical=False, commands=[ # let the server warm up just in case Run("sleep 1"), Run("netcat", "data02", netcat_port, "<", "RANDOM"), Run("echo SENDER DONE"), ], ), # check contents on the receiving end SshJob( node=node2, commands=[ Run("ls -l RANDOM"), Run("sha1sum RANDOM"), ], ), ### these two apply to the Sequence # required applies to the first job in the sequence required=turn_on_datas, # scheduler applies to all jobs in the sequence scheduler=scheduler, )
def one_run(*, protocol, interference, run_name=default_run_name, slicename=default_slicename, tx_power, phy_rate, antenna_mask, channel, load_images=False, node_ids=DEFAULT_NODE_IDS, src_ids=DEFAULT_SRC_IDS, dest_ids=DEFAULT_DEST_IDS, scrambler_id=DEFAULT_SCRAMBLER_ID, tshark=False, map=False, warmup=False, route_sampling=False, iperf=False, verbose_ssh=False, verbose_jobs=False, dry_run=False, run_number=None): """ Performs data acquisition on all nodes with the following settings Arguments: tx_power: in dBm, a string like 5, 10 or 14. Corresponds to the transmission power. phy_rate: a string among 1, 54. Correspond to the wifi rate. antenna_mask: a string among 1, 3, 7. channel: a string like e.g. 1 or 40. Correspond to the channel. protocol: a string among batman , olsr. Correspond to the protocol interference : in amplitude percentage, a string like 15 or 20. Correspond to the power of the noise generated in the spectrum. Can be either None or "None" to mean no interference. run_name: the name for a subdirectory where all data will be kept successive runs should use the same name for further visualization slicename: the Unix login name (slice name) to enter the gateway load_images: a boolean specifying whether nodes should be re-imaged first node_ids: a list of node ids to run the scenario against; strings or ints are OK; tshark: a boolean specifying wether we should format/parse the .pcap. map: a boolean specifying wether we should fetch/parse the route tables of the nodes. warmup: a boolean specifying whether we should run a ping before the experiment to be certain of the stabilisation on the network. src_ids: a list of nodes from which we will launch the ping from. strings or ints are OK. ping_messages : the number of ping packets that will be generated """ # set default for the nodes parameter node_ids = ([int(id) for id in node_ids] if node_ids is not None else DEFAULT_NODE_IDS) src_ids = ([int(id) for id in src_ids] if src_ids is not None else DEFAULT_SRC_IDS) dest_ids = ([int(id) for id in dest_ids] if dest_ids is not None else DEFAULT_NODE_IDS) # all nodes - i.e. including sources and destinations - # need to run the protocol node_ids = list(set(node_ids).union(set(src_ids).union(set(dest_ids)))) if interference == "None": interference = None # open result dir no matter what run_root = naming_scheme( run_name=run_name, protocol=protocol, interference=interference, autocreate=True) # fix me trace = run_root / f"trace-{%m-%d-%H-%M}" ref_time = apssh_time() trace = run_root / f"trace-{ref_time}" try: with trace.open('w') as feed: def log_line(line): time_line(line, file=feed) load_msg = f"{'WITH' if load_images else 'NO'} image loading" interference_msg = (f"interference={interference} " f"from scrambler={scrambler_id}") nodes = " ".join(str(n) for n in node_ids) srcs = " ".join(str(n) for n in src_ids) dests = " ".join(str(n) for n in dest_ids) ping_labels = [ f"PING {s} ➡︎ {d}" for s in src_ids # and on the destination for d in dest_ids if d != s ] log_line(f"output in {run_root}") log_line(f"trace in {trace}") log_line(f"protocol={protocol}") log_line(f"{load_msg}") log_line(f"{interference_msg}") log_line("----") log_line(f"Selected nodes : {nodes}") log_line(f"Sources : {srcs}") log_line(f"Destinations : {dests}") for label in ping_labels: log_line(f"{label}") log_line("----") for feature in ('warmup', 'tshark', 'map', 'route_sampling', 'iperf'): log_line(f"Feature {feature}: {locals()[feature]}") except Exception as exc: print(f"Cannot write into {trace} - aborting this run") print(f"Found exception {type(exc)} - {exc}") return False # # dry-run mode # just display a one-liner with parameters # prelude = "" if not dry_run else "dry_run:" with trace.open() as feed: print(f"**************** {ref_time} one_run #{run_number}:") for line in feed: print(prelude, line, sep='', end='') if dry_run: return True # the nodes involved faraday = SshNode(hostname=default_gateway, username=slicename, formatter=TimeColonFormatter(), verbose=verbose_ssh) # this is a python dictionary that allows to retrieve a node object # from an id node_index = { id: SshNode(gateway=faraday, hostname=fitname(id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) for id in node_ids } # extracts for sources and destinations src_index = {id:node for (id, node) in node_index.items() if id in src_ids} dest_index = {id:node for (id, node) in node_index.items() if id in dest_ids} if interference: node_scrambler = SshNode( gateway=faraday, hostname=fitname(scrambler_id), username="******", formatter=TimeColonFormatter(), verbose=verbose_ssh) # the global scheduler scheduler = Scheduler(verbose=verbose_jobs) ########## check_lease = SshJob( scheduler=scheduler, node=faraday, verbose=verbose_jobs, label="rhubarbe check lease", command=Run("rhubarbe leases --check", label="rlease"), ) # load images if requested green_light = check_lease # at some point we did not load the scrambler if interference was None # and that was a way to run faster loads with no interference # but now we always load the scrambler node with gnuradio # this is because when we do runs.py -i None 15 30 ... # then the first call to one_run is with interference being None # but it is still important to load the scrambler if load_images: # copy node_ids load_ids = node_ids[:] load_ids.append(scrambler_id) # the nodes that we **do not** use should be turned off # so if we have selected e.g. nodes 10 12 and 15, we will do # rhubarbe off -a ~10 ~12 ~15, meaning all nodes except 10, 12 and 15 negated_node_ids = [f"~{id}" for id in load_ids] # we can do these three things in parallel ready_jobs = [ SshJob(node=faraday, required=green_light, scheduler=scheduler, verbose=verbose_jobs, command=Run("rhubarbe", "off", "-a", *negated_node_ids, label="turn off unused nodes")), SshJob(node=faraday, required=green_light, scheduler=scheduler, verbose=verbose_jobs, label="load batman image", command=Run("rhubarbe", "load", "-i", "batman-olsr", *node_ids, label=f"load ubuntu on {node_ids}")), SshJob( node=faraday, required=green_light, scheduler=scheduler, verbose=verbose_jobs, label="load gnuradio image", command=Run("rhubarbe", "load", "-i", "batman-olsr-gnuradio", scrambler_id, label=f"load gnuradio on {scrambler_id}")), ] # replace green_light in this case green_light = SshJob( node=faraday, required=ready_jobs, scheduler=scheduler, verbose=verbose_jobs, label="wait for nodes to come up", command=Run("rhubarbe", "wait", *load_ids)) ########## # setting up the wireless interface on all nodes # # provide node-utilities with the ranges/units it expects frequency = channel_frequency[int(channel)] # tx_power_in_mBm not in dBm tx_power_driver = tx_power * 100 #just in case somme services failed in the previous experiment reset_failed_services_job = [ SshJob( node=node, verbose=verbose_jobs, label="reset failed services", command=Run("systemctl reset-failed", label="reset-failed services")) for id, node in node_index.items() ] reset_failed_services = Scheduler( *reset_failed_services_job, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="Reset failed services") init_wireless_sshjobs = [ SshJob( node=node, verbose=verbose_jobs, label=f"init {id}", command=RunScript( "node-utilities.sh", f"init-ad-hoc-network-{WIRELESS_DRIVER}", WIRELESS_DRIVER, "foobar", frequency, phy_rate, antenna_mask, tx_power_driver, label="init add-hoc network"), ) for id, node in node_index.items()] init_wireless_jobs = Scheduler( *init_wireless_sshjobs, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="Initialisation of wireless chips") if interference: # Run uhd_siggen with the chosen power init_scrambler_job = SshJob( scheduler=scheduler, required=green_light, forever=True, node=node_scrambler, verbose=verbose_jobs, #TODO : If exit-signal patch is done add exit-signal=["TERM"] # to this run object and call uhd_siggen directly commands=[RunScript("node-utilities.sh", "init-scrambler", label="init scrambler"), Run(f"systemd-run --unit=uhd_siggen -t ", f"uhd_siggen -a usrp -f {frequency}M", f"--sine --amplitude 0.{interference}", label="systemctl start uhd_siggen") ] ) green_light = [init_wireless_jobs, reset_failed_services] # then install and run batman on fit nodes run_protocol_job = [ SshJob( # scheduler=scheduler, node=node, label=f"init and run {protocol} on fit node {id}", verbose=verbose_jobs, # CAREFUL : These ones use sytemd-run # with the ----service-type=forking option! command=RunScript("node-utilities.sh", f"run-{protocol}", label=f"run {protocol}"), ) for id, node in node_index.items()] run_protocol = Scheduler( *run_protocol_job, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="init and run routing protocols") green_light = run_protocol # after that, run tcpdump on fit nodes, this job never ends... if tshark: run_tcpdump_job = [ SshJob( # scheduler=scheduler_monitoring, node=node, forever=True, label=f"run tcpdump on fit node {id}", verbose=verbose_jobs, command=[ Run("systemd-run -t --unit=tcpdump", f"tcpdump -U -i moni-{WIRELESS_DRIVER}", f"-y ieee802_11_radio -w /tmp/fit{id}.pcap", label=f"tcpdump {id}") ] ) for id, node in node_index.items() ] run_tcpdump = Scheduler( *run_tcpdump_job, scheduler=scheduler, required=green_light, forever=True, verbose=verbose_jobs, label="Monitoring - tcpdumps") # let the wireless network settle settle_scheduler = Scheduler( scheduler=scheduler, required=green_light, ) if warmup: # warmup pings don't need to be sequential, so let's # do all the nodes at the same time # on a given node though, we'll ping the other ends sequentially # see the graph for more warmup_jobs = [ SshJob( node=node_s, verbose=verbose_jobs, commands=[ RunScript("node-utilities.sh", "my-ping", f"10.0.0.{d}", warmup_ping_timeout, warmup_ping_interval, warmup_ping_size, warmup_ping_messages, f"warmup {s} ➡︎ {d}", label=f"warmup {s} ➡︎ {d}") for d in dest_index.keys() if s != d ] ) # for each selected experiment nodes for s, node_s in src_index.items() ] warmup_scheduler = Scheduler( *warmup_jobs, scheduler=settle_scheduler, verbose=verbose_jobs, label="Warmup pings") settle_wireless_job2 = PrintJob( "Let the wireless network settle after warmup", sleep=settle_delay_shorter, scheduler=settle_scheduler, required=warmup_scheduler, label=f"settling-warmup for {settle_delay_shorter} sec") # this is a little cheating; could have gone before the bloc above # but produces a nicer graphical output # we might want to help asynciojobs if it offered a means # to specify entry and exit jobs in a scheduler settle_wireless_job = PrintJob( "Let the wireless network settle", sleep=settle_delay_long, scheduler=settle_scheduler, label=f"settling for {settle_delay_long} sec") green_light = settle_scheduler if iperf: iperf_service_jobs = [ SshJob( node=node_d, verbose=verbose_jobs, forever=True, commands=[ Run("systemd-run -t --unit=iperf", "iperf -s -p 1234 -u", label=f"iperf serv on {d}"), ], ) for d, node_d in dest_index.items() ] iperf_serv_sched = Scheduler( *iperf_service_jobs, verbose=verbose_jobs, label="Iperf Servers", # for a nicer graphical output # otherwise the exit arrow # from scheduler 'iperf mode' # to job 'settling for 60s' # gets to start from this box forever=True, ) iperf_cli = [ SshJob( node=node_s, verbose=verbose_jobs, commands=[ Run("sleep 7", label=""), Run(f"iperf", f"-c 10.0.0.{d} -p 1234", f"-u -b {phy_rate}M -t 60", f"-l 1024 > IPERF-{s:02d}-{d:02d}", label=f"run iperf {s} ➡︎ {d}") ] ) for s, node_s in src_index.items() for d, node_d in dest_index.items() if s != d ] iperf_cli_sched = Scheduler( Sequence(*iperf_cli), verbose=verbose_jobs, label="Iperf Clients") iperf_stop = [ SshJob(node=node_d, verbose=verbose_jobs, label=f"Stop iperf on {d}", command=Run("systemctl stop iperf")) for d, node_d in dest_index.items() ] iperf_stop_sched = Scheduler( *iperf_stop, required=iperf_cli_sched, verbose=verbose_jobs, label="Iperf server stop") iperf_fetch = [ SshJob(node=node_s, verbose=verbose_jobs, command=Pull( remotepaths=[f"IPERF-{s:02d}-{d:02d}"], localpath=str(run_root), label="fetch iperf {s} ➡︎ {d}") ) for s, node_s in src_index.items() for d, node_d in dest_index.items() if s != d ] iperf_fetch_sched = Scheduler( *iperf_fetch, required=iperf_stop_sched, verbose=verbose_jobs, label="Iperf fetch report") iperf_jobs = [iperf_serv_sched, iperf_cli_sched, iperf_stop_sched, iperf_fetch_sched] iperf_sched = Scheduler( *iperf_jobs, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="Iperf Module") settle_wireless_job_iperf = PrintJob( "Let the wireless network settle", sleep=settle_delay_shorter, scheduler=scheduler, required=iperf_sched, label=f"settling-iperf for {settle_delay_shorter} sec") green_light = settle_wireless_job_iperf # create all the tracepath jobs from the first node in the list if map: map_jobs = [ SshJob( node=node, label=f"Generating ROUTE file for proto {protocol} on node {id}", verbose=verbose_jobs, commands=[ RunScript(f"node-utilities.sh", f"route-{protocol}", f"> ROUTE-TABLE-{id:02d}", label="get route table"), Pull(remotepaths=[f"ROUTE-TABLE-{id:02d}"], localpath=str(run_root), label="") ], ) for id, node in node_index.items() ] map_scheduler = Scheduler( *map_jobs, scheduler=scheduler, required=green_light, verbose=verbose_jobs, label="Snapshoting route files") green_light = map_scheduler if route_sampling: route_sampling_jobs = [ SshJob( node=node, label=f"Route sampling service for proto {protocol} on node {id}", verbose=False, forever=True, commands=[ Push(localpaths=["route-sample-service.sh"], remotepath=".", label=""), Run("chmod +x route-sample-service.sh", label=""), Run("systemd-run -t --unit=route-sample", "/root/route-sample-service.sh", "route-sample", f"ROUTE-TABLE-{id:02d}-SAMPLED", protocol, label="start route-sampling"), ], ) for id, node in node_index.items() ] route_sampling_scheduler = Scheduler( *route_sampling_jobs, scheduler=scheduler, verbose=False, forever=True, label="Route Sampling services launch", required=green_light) ########## # create all the ping jobs, i.e. max*(max-1)/2 # this again is a python list comprehension # see the 2 for instructions at the bottom # # notice that these SshJob instances are not yet added # to the scheduler, we will add them later on # depending on the sequential/parallel strategy pings_job = [ SshJob( node=node_s, verbose=verbose_jobs, commands=[ Run(f"echo actual ping {s} ➡︎ {d} using {protocol}", label=f"ping {s} ➡︎ {d}"), RunScript("node-utilities.sh", "my-ping", f"10.0.0.{d}", ping_timeout, ping_interval, ping_size, ping_messages, f"actual {s} ➡︎ {d}", ">", f"PING-{s:02d}-{d:02d}", label=""), Pull(remotepaths=[f"PING-{s:02d}-{d:02d}"], localpath=str(run_root), label=""), ], ) # for each selected experiment nodes for s, node_s in src_index.items() for d, node_d in dest_index.items() if s != d ] pings = Scheduler( scheduler=scheduler, label="PINGS", verbose=verbose_jobs, required=green_light) # retrieve all pcap files from fit nodes stop_protocol_job = [ SshJob( # scheduler=scheduler, node=node, # required=pings, label=f"kill routing protocol on {id}", verbose=verbose_jobs, command=RunScript(f"node-utilities.sh", f"kill-{protocol}", label=f"kill-{protocol}"), ) for id, node in node_index.items() ] stop_protocol = Scheduler( *stop_protocol_job, scheduler=scheduler, required=pings, label="Stop routing protocols", ) if tshark: retrieve_tcpdump_job = [ SshJob( # scheduler=scheduler, node=nodei, # required=pings, label=f"retrieve pcap trace from fit{i:02d}", verbose=verbose_jobs, commands=[ Run("systemctl stop tcpdump", label="stop tcpdump"), #Run("systemctl reset-failed tcpdump"), #RunScript("node-utilities.sh", "kill-tcpdump", # label="kill-tcpdump"), Run( f"echo retrieving pcap trace and result-{i}.txt from fit{i:02d}", label=""), Pull(remotepaths=[f"/tmp/fit{i}.pcap"], localpath=str(run_root), label=""), ], ) for i, nodei in node_index.items() ] retrieve_tcpdump = Scheduler( *retrieve_tcpdump_job, scheduler=scheduler, required=pings, label="Retrieve tcpdump", ) if route_sampling: retrieve_sampling_job = [ SshJob( # scheduler=scheduler, node=nodei, # required=pings, label=f"retrieve sampling trace from fit{i:02d}", verbose=verbose_jobs, commands=[ # RunScript("node-utilities.sh", "kill-route-sample", protocol, # label = "kill route sample"), #RunScript("route-sample-service.sh", "kill-route-sample", # label="kill route sample"), Run("systemctl stop route-sample", label="stop route-sample"), Run( f"echo retrieving sampling trace from fit{i:02d}", label=""), Pull(remotepaths=[f"ROUTE-TABLE-{i:02d}-SAMPLED"], localpath=str(run_root), label=""), ], ) for i, nodei in node_index.items() ] retrieve_sampling = Scheduler( *retrieve_sampling_job, scheduler=scheduler, required=pings, verbose=verbose_jobs, label="Stop & retrieve route sampling", ) if tshark: parse_pcaps_job = [ SshJob( # scheduler=scheduler, node=LocalNode(), # required=retrieve_tcpdump, label=f"parse pcap trace {run_root}/fit{i}.pcap", verbose=verbose_jobs, #commands = [RunScript("parsepcap.sh", run_root, i)] command=Run("tshark", "-2", "-r", f"{run_root}/fit{i}.pcap", "-R", f"'(ip.dst==10.0.0.{i} && icmp) && radiotap.dbm_antsignal'", "-Tfields", "-e", "'ip.src'", "-e" "'ip.dst'", "-e", "'radiotap.dbm_antsignal'", ">", f"{run_root}/result-{i}.txt", label=f"parsing pcap from {i}"), ) for i in node_ids ] parse_pcaps = Scheduler( *parse_pcaps_job, scheduler=scheduler, required=retrieve_tcpdump, label="Parse pcap", ) if interference: kill_uhd_siggen = SshJob( scheduler=scheduler, node=node_scrambler, required=pings, label=f"killing uhd_siggen on the scrambler node {scrambler_id}", verbose=verbose_jobs, commands=[Run("systemctl", "stop", "uhd_siggen"), #Run("systemctl reset-failed tcpdump"), ], ) kill_2_uhd_siggen = SshJob( scheduler=scheduler, node=faraday, required=kill_uhd_siggen, label=f"turning off usrp on the scrambler node {scrambler_id}", verbose=verbose_jobs, command=Run("rhubarbe", "usrpoff", scrambler_id), ) pings.add(Sequence(*pings_job)) # for running sequentially we impose no limit on the scheduler # that will be limitied anyways by the very structure # of the required graph # safety check scheduler.export_as_pngfile(run_root / "experiment-graph") if dry_run: scheduler.list() return True # if not in dry-run mode, let's proceed to the actual experiment ok = scheduler.run() # jobs_window=jobs_window) # close all ssh connections close_ssh_in_scheduler(scheduler) # give details if it failed if not ok: scheduler.debrief() scheduler.export_as_pngfile("debug") if ok and map: time_line("Creation of MAP files") post_processor = ProcessRoutes(run_root, src_ids, node_ids) post_processor.run() if ok and route_sampling: time_line("Creation of ROUTE SAMPLING files") post_processor = ProcessRoutes(run_root, src_ids, node_ids) post_processor.run_sampled() # data acquisition is done, let's aggregate results # i.e. compute averages #if ok and tshark: #post_processor = Aggregator(run_root, node_ids, antenna_mask) #post_processor.run() time_line("one_run done") return ok
def main() -> bool: parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument("-U", "--url", default=default_topurl, dest='topurl', help="url to reach nbhosting server") parser.add_argument("-c", "--course-gitdir", default=default_course_gitdir, help="""location of a git repo where to fetch notebooks; needed in order to generate relevant URLs""") parser.add_argument("-i", "--indices", default=[0], action=IntsRanges, help="(cumulative) ranges of indices in the list of known notebooks" " - run nbhtest with -l to see list") parser.add_argument("-u", "--users", default=[1], action=IntsRanges, help="(cumulative) ranges of students indexes; e.g. -u 101-400 -u 501-600") parser.add_argument("-m", "--random", action='store_true', help="if set, a random notebook index is used for each student") parser.add_argument("-b", "--base", default='student', help="basename for students name") parser.add_argument("-p", "--period", default=20, type=float, help="delay between 2 triggers of nbhtest") parser.add_argument("-s", "--sleep", default=default_sleep_internal, type=float, help="delay in seconds to sleep between actions inside nbhtest") parser.add_argument("-w", "--window", default=default_window, type=int, help="window depth for spawning the nbhtest instances") parser.add_argument("-n", "--dry-run", action='store_true') args = parser.parse_args() course_gitdir = args.course_gitdir course, notebooks = list_notebooks(course_gitdir) # in random mode; what are the choices that we randomize on if args.random: if len(args.indices) > 1: choices = args.indices else: choices = list(range(len(notebooks))) local = LocalNode( formatter=TerminalFormatter( format="%H-%M-%S:@line@", verbose=True )) scheduler = Scheduler() jobs = [] for user in args.users: student_name = "{}-{:04d}".format(args.base, user) if args.random: indices = [ random.choice(choices) ] else: indices = args.indices for index in indices: command = "nbhtest.py -U {} -c {} -i {} -u {} -s {} &"\ .format(args.topurl, course_gitdir, index, student_name, args.sleep) if args.dry_run: print("dry-run:", command) else: # schule this command to run job = Sequence( SshJob(scheduler=scheduler, node=local, command = command, ), Job(asyncio.sleep(args.period)) ) jobs.append(job) if args.dry_run: return True overall = scheduler.orchestrate( jobs_window = args.window ) if not overall: scheduler.debrief() print("nbhtests DONE") return overall