def test_deferred_1(self):
"the ability of using a variable"
env = Variables()
env.fetched = 'a simple variable'
# Warning this is a jinja template -- NOT an f-string !!
deferred = Deferred("echo fetched = {{fetched}}", env)
expected = f"fetched = a simple variable\n"
self.check_expansion((deferred, expected))
def test_deferred_2(self):
"the ability of operations"
env = Variables()
env.x1 = 1
env.x2 = 2.0
# Warning this is a jinja template -- NOT an f-string !!
d1 = Deferred("echo x1 = {{x1}}", env)
e1 = f"x1 = 1\n"
d2 = Deferred("echo x2 = {{x2}}", env)
e2 = f"x2 = 2.0\n"
d3 = Deferred("echo sum = {{x1+x2}}", env)
e3 = f"sum = 3.0\n"
self.check_expansion((d1, e1), (d2, e2), (d3, e3))
def test_deferred_service(self):
"""
a service can be defined from a deferred instance
rather than a plain string
"""
s = Scheduler()
env = Variables()
echo_service = Service(Deferred("echo {{run1}}", env),
service_id='echo',
verbose=True)
n = SshNode(localhostname(), username=localuser())
Sequence(SshJob(n,
commands=Run("echo from-first-run",
capture=Capture('run1', env))),
SshJob(n, commands=Run(echo_service.start_command())),
SshJob(n,
commands=Run(echo_service.journal_command(
since="10 second ago"),
capture=Capture('journal', env))),
scheduler=s)
print('STARTING', 20 * '-', echo_service.start_command())
s.run()
print('DONE', 20 * '-', echo_service.start_command())
#print(f"env={env}")
obtained = env.journal
expected = "from-first-run"
found = expected in obtained
self.assertTrue(found)
def test_deferred_chain(self):
"""
one command computes a string that gets passed to another one
this is analogous to
run1=$(ssh localhost echo from-first-run)
final=$(ssh localhost echo ${run1})
the 'final' variable is only needed
for checking everything went well
"""
s = Scheduler()
env = Variables()
n = SshNode(localhostname(), username=localuser())
Sequence(SshJob(n,
commands=Run("echo from-first-run",
capture=Capture('run1', env))),
SshJob(n,
commands=Run(Deferred("echo {{run1}}", env),
capture=Capture('final', env))),
scheduler=s)
s.run()
#print(f"env={env}")
obtained = env.final
expected = "from-first-run"
self.assertEqual(obtained, expected)
def test_environment(self):
needle_foo = 'xxx-foo-xxx'
needle_bar = 'xxx-bar-xxx'
scheduler = Scheduler()
node = SshNode("localhost")
env = Variables()
service = Service("env",
service_id='echo-environ',
environ={
'FOO': needle_foo,
'BAR': needle_bar,
})
SshJob(scheduler=scheduler,
node=node,
commands=[
Run(service.start_command()),
Run(service.journal_command(since='5s ago'),
capture=Capture('journal', env))
])
self.assertEqual(scheduler.run(), True)
self.assertTrue(f"FOO={needle_foo}" in env.journal)
self.assertTrue(f"BAR={needle_bar}" in env.journal)
def test_variables(self):
"""
check how variables are rendered
"""
v1 = Variables()
v2 = Variables()
v2.defined = 'OK-defined'
template = "defined: {{defined}} undefined: {{undefined}}"
scheduler = Scheduler(critical=False)
gateway = SshNode(hostname=localhostname(), username=localuser())
Sequence(SshJob(gateway, command=Deferred(template, v1)),
SshJob(gateway, command=Deferred(template, v2)),
scheduler=scheduler)
produce_png(scheduler, "test_graphics_variables")
def run(*, gateway, slicename,
disaggregated_cn, operator_version, nodes, node_master, node_enb, quectel_nodes, phones, flexran,
drone, verbose, dry_run,
load_images, master_image, worker_image, quectel_image):
"""
Install K8S on R2lab
Arguments:
slicename: the Unix login name (slice name) to enter the gateway
quectel_nodes: list of indices of quectel UE nodes to use
phones: list of indices of phones to use
nodes: a list of node ids to run the scenario on; strings or ints
are OK;
node_master: the master node id, must be part of selected nodes
node_enb: the node id for the enb, which is connected to usrp/duplexer
disaggregated_cn: Boolean; True for the disaggregated CN scenario. False for all-in-one CN.
operator_version: str, either "none" or "v1" or "v2".
"""
if operator_version == "none":
only_kube5g = True
else:
only_kube5g = False
if node_master not in nodes:
print(f"master node {node_master} must be part of selected fit nodes {nodes}")
exit(1)
if node_enb not in nodes:
print(f"eNB worker node {node_enb} must be part of selected fit nodes {nodes}")
exit(1)
# Check if the browser can be automatically run to display the Drone app
if drone:
run_browser = True
if platform == "linux":
cmd_open = "xdg-open"
elif platform == "darwin":
cmd_open = "open"
else:
run_browser = False
if run_browser:
print(f"**** Will run the browser with command {cmd_open}")
else:
print(f"**** Will not be able to run the browser as platform is {platform}")
worker_ids = nodes[:]
worker_ids.remove(node_master)
quectel_ids = quectel_nodes[:]
quectel = len(quectel_ids) > 0
faraday = SshNode(hostname=default_gateway, username=slicename,
verbose=verbose,
formatter=TimeColonFormatter())
master = SshNode(gateway=faraday, hostname=fitname(node_master),
username="******",
verbose=verbose,
formatter=TimeColonFormatter())
node_index = {
id: SshNode(gateway=faraday, hostname=fitname(id),
username="******",formatter=TimeColonFormatter(),
verbose=verbose)
for id in nodes
}
nodes_quectel_index = {
id: SshNode(gateway=faraday, hostname=fitname(id),
username="******",formatter=TimeColonFormatter(),
verbose=verbose)
for id in quectel_nodes
}
worker_index = dict(node_index)
del worker_index[node_master]
fit_master = fitname(node_master)
fit_enb = fitname(node_enb)
# the global scheduler
scheduler = Scheduler(verbose=verbose)
##########
check_lease = SshJob(
scheduler=scheduler,
node = faraday,
critical = True,
verbose=verbose,
command = Run("rhubarbe leases --check"),
)
green_light = check_lease
if load_images:
green_light = [
SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose,
label = f"Load image {master_image} on master {fit_master}",
commands=[
Run(f"rhubarbe load {node_master} -i {master_image}"),
Run(f"rhubarbe wait {node_master}"),
]
),
SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=True,
verbose=verbose,
label = f"Load image {worker_image} on worker nodes",
commands=[
Run(f"rhubarbe usrpoff {node_enb}"), # if usrp is on, load could be problematic...
Run("rhubarbe", "load", *worker_ids, "-i", worker_image),
Run("rhubarbe", "wait", *worker_ids),
Run(f"rhubarbe usrpon {node_enb}"), # ensure a reset of the USRP on the enB node
],
),
SshJob(
scheduler=scheduler,
required=check_lease,
node=faraday,
critical=False,
verbose=verbose,
label="turning off unused nodes",
command=[
Run("rhubarbe bye --all "
+ "".join(f"~{x} " for x in nodes))
]
)
]
if quectel:
prepare_quectel = SshJob(
scheduler=scheduler,
required=green_light,
node=faraday,
critical=True,
verbose=verbose,
label = f"Load image {quectel_image} on quectel UE nodes",
commands=[
Run("rhubarbe", "usrpoff", *quectel_ids),
Run("rhubarbe", "load", *quectel_ids, "-i", quectel_image),
Run("rhubarbe", "wait", *quectel_ids),
Run("rhubarbe", "usrpon", *quectel_ids),
],
),
##########
if quectel:
# wait 30s for Quectel modules show up
wait_quectel_ready = PrintJob(
"Let Quectel modules show up",
scheduler=scheduler,
required=prepare_quectel,
sleep=30,
label="sleep 30s for the Quectel modules to show up"
)
# run the Quectel Connection Manager as a service on each Quectel UE node
quectelCM_service = Service(
command="quectel-CM -s oai.ipv4 -4",
service_id="QuectelCM",
verbose=verbose,
)
init_quectel_nodes = [
SshJob(
scheduler=scheduler,
required=wait_quectel_ready,
node=node,
critical=True,
verbose=verbose,
label=f"Init Quectel UE on fit node {id}",
commands = [
RunScript(find_local_embedded_script("nodes.sh"), "check-quectel-on", includes=INCLUDES),
quectelCM_service.start_command(),
],
) for id, node in nodes_quectel_index.items()
]
# wait 20s for Quectel Connection Manager to start up
wait_quectelCM_ready = PrintJob(
"Let QuectelCM start up",
scheduler=scheduler,
required=init_quectel_nodes,
sleep=20,
label="Sleep 20s for the Quectel Connection Manager(s) to start up"
)
detach_quectel_nodes = [
SshJob(
scheduler=scheduler,
required=wait_quectelCM_ready,
node=node,
critical=True,
verbose=verbose,
label=f"Detach Quectel UE on fit node {id}",
command = RunScript(find_local_embedded_script("nodes.sh"), "quectel-detach", includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
##########
# Initialize k8s on the master node
init_master = SshJob(
scheduler=scheduler,
required=green_light,
node=master,
critical=True,
verbose=verbose,
label = f"Install and launch k8s on the master {node_master}",
commands = [
Run("swapoff -a"),
Run("hostnamectl set-hostname master-node"),
Run("kubeadm init --pod-network-cidr=10.244.0.0/16 > /tmp/join_msg.txt"),
Run("tail -2 /tmp/join_msg.txt > /tmp/join_msg"),
Run("mkdir -p $HOME/.kube"),
Run("cp -i /etc/kubernetes/admin.conf $HOME/.kube/config"),
Run("chown $(id -u):$(id -g) $HOME/.kube/config"),
Run("kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml"),
Run("kubectl get pods --all-namespaces"),
],
)
init_workers = [
SshJob(
scheduler=scheduler,
required=init_master,
node=node,
critical=True,
verbose=verbose,
label=f"Init k8s on fit node {id} and join the cluster",
commands = [
Run("swapoff -a"),
Run("increase-control-mtu"),
Run(f"scp -o 'StrictHostKeyChecking no' {fit_master}:/tmp/join_msg /tmp/join_msg"),
Run("chmod a+x /tmp/join_msg"),
Run("/tmp/join_msg"),
],
) for id, node in worker_index.items()
]
# wait 10s for K8S nodes setup
wait_k8nodes_ready = PrintJob(
"Let k8s set up",
scheduler=scheduler,
required=init_workers,
sleep=10,
label="sleep 10s for the k8s nodes to settle"
)
init_kube5g = SshJob(
scheduler=scheduler,
required = wait_k8nodes_ready,
node = master,
verbose=verbose,
label = f"Add oai:ran label to oai-ran pod on {node_enb} and start 5GOperator pod",
commands = [
Run("kubectl get nodes"),
# add label to the eNB node to help k8s scheduler selects the right fit node
Run(f"kubectl label nodes fit{node_enb} oai=ran"),
Run("kubectl get nodes -Loai"),
## retrieve the kube5g operator
#Run("git clone -b develop [email protected]:mosaic5g/kube5g.git"),
# install a few dependencies
Run("apt install -y python3-pip"),
Run("pip3 install --upgrade pip"),
Run("pip3 install ruamel.yaml==0.16.12 colorlog==4.6.2"),
Run("sed -i 's/oairan:v1-1.0-1/oairan:v1-1.0-3/g' /root/kube5g/common/config-manager/conf_global_default.yaml"),
# specify the R2lab specific configuration
Run("cd /root/kube5g/common/config-manager; ./conf-manager.py -s conf_short_r2lab.yaml"),
# apply the R2lab CRD
Run("cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh -n"),
# start the kube5g operator pod
Run("cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh container start"),
Run("kubectl get pods"),
],
)
# wait 30s for K8S 5G Operator setup
wait_k8_5GOp_ready = PrintJob(
"Let 5G Operator set up",
scheduler=scheduler,
required=init_kube5g,
sleep=30,
label="wait 30s for the 5G Operator pod to settle"
)
if only_kube5g:
finish = SshJob(
scheduler=scheduler,
required = wait_k8_5GOp_ready,
node = master,
verbose=verbose,
label = f"showing nodes and pods before leaving",
commands = [
Run("kubectl get nodes -Loai"),
Run("kubectl get pods"),
],
)
else:
if disaggregated_cn:
cn_type="disaggregated-cn"
# setup_time = 120
setup_time = 200
else:
cn_type="all-in-one"
# setup_time = 60
setup_time = 140
if flexran:
flexran_opt="flexran"
else:
flexran_opt=""
run_kube5g = SshJob(
scheduler=scheduler,
required = wait_k8_5GOp_ready,
node = master,
verbose=verbose,
label = f"deploy {operator_version} {cn_type} {flexran_opt} pods",
commands = [
Run("kubectl get nodes -Loai"),
Run(f"cd /root/kube5g/openshift/kube5g-operator; ./k5goperator.sh deploy {operator_version} {cn_type} {flexran_opt}"),
Run("kubectl get pods"),
],
)
# Coffee Break -- wait 1 or 2mn for K8S 5G pods setup
wait_k8_5Gpods_ready = PrintJob(
"Let all 5G pods set up",
scheduler=scheduler,
required=run_kube5g,
sleep=setup_time,
label=f"waiting {setup_time}s for all 5G pods to settle"
)
check_kube5g = SshJob(
scheduler=scheduler,
required = wait_k8_5Gpods_ready,
node = master,
verbose=verbose,
label = "Check which pods are deployed",
commands = [
Run("kubectl get nodes -Loai"),
Run("kubectl get pods"),
],
)
if drone:
# the place where runtime variables get stored
env = Variables()
#
# Define and run all the services to launch the Drone app locally on a firefox browser
#
drone_service = Service(
command=f"python /root/mosaic5g/store/sdk/frontend/drone/drone.py --port=8088 --tasks --address=192.168.3.{node_enb}",
service_id="drone_app",
verbose=verbose,
)
k8s_port9999_fwd_service = Service(
command=Deferred("kubectl port-forward {{flexran_pod}} 9999:9999 --address 0.0.0.0", env),
service_id="k8s-port9999-fwd",
verbose=verbose,
# somehow this is required for kubectl to run properly
environ={'KUBECONFIG': '/etc/kubernetes/admin.conf'}
)
# can't use a Service instance on the local box if it's not a Linux
# and we have macs...
local_port_fwd = (f"ssh -f -N -4"
f" -L9999:192.168.3.{node_master}:9999"
f" -L8088:192.168.3.{node_enb}:8088"
f" -o ExitOnForwardFailure=yes"
f" {slicename}@faraday.inria.fr")
browser_service = Service(
command=f"sleep 10; {cmd_open} http://127.0.0.1:8088/",
service_id="drone_browser",
verbose=verbose,
)
run_drone=SshJob(
scheduler=scheduler,
required=check_kube5g,
node=worker_index[node_enb],
verbose=verbose,
label=f"Run the drone app on worker node {node_enb} as a service",
commands=[
drone_service.start_command(),
],
)
get_flexran_podname=SshJob(
scheduler=scheduler,
required=check_kube5g,
node=master,
verbose=verbose,
label=f"Retrieve the name of the FlexRAN pod",
commands=[
# xxx here
Run("kubectl get --no-headers=true pods -l app=flexran -o custom-columns=:metadata.name",
capture=Capture('flexran_pod', env)),
],
)
run_k8s_port9999_fwd=SshJob(
scheduler=scheduler,
required=get_flexran_podname,
node=master,
verbose=verbose,
label=f"Run port forwarding on the master node as a service",
commands=[
k8s_port9999_fwd_service.start_command(),
],
)
# On the local machine, impossible to use Services as the latter uses systemd-run, only available on Linux
run_local_ports_fwd = SshJob(
scheduler=scheduler,
required = check_kube5g,
node = LocalNode(),
verbose=verbose,
label = f"Forward local ports 8088 and 9999",
command=Run(local_port_fwd + "&", ignore_outputs=True),
)
if run_browser:
run_local_browser = SshJob(
scheduler=scheduler,
required = (run_drone, run_k8s_port9999_fwd, run_local_ports_fwd),
node = LocalNode(),
verbose=verbose,
label = f"Run the browser on the local node in background",
command=browser_service.command+"&",
)
phones_requirements=run_local_browser
else:
phones_requirements=run_k8s_port9999_fwd
else:
phones_requirements=check_kube5g
########## Test phone(s) connectivity
sleeps_ran = (20, 25)
phone_msgs = [f"wait for {sleep}s for eNB to start up before waking up phone{id}"
for sleep, id in zip(sleeps_ran, phones)]
wait_commands = [f"echo {msg}; sleep {sleep}"
for msg, sleep in zip(phone_msgs, sleeps_ran)]
sleeps_phone = (15, 20)
phone2_msgs = [f"wait for {sleep}s for phone{id} before starting tests"
for sleep, id in zip(sleeps_phone, phones)]
wait2_commands = [f"echo {msg}; sleep {sleep}"
for msg, sleep in zip(phone2_msgs, sleeps_phone)]
job_start_phones = [
SshJob(
node=faraday,
commands=[
Run(wait_command),
RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}",
"r2lab-embedded/shell/macphone.sh", "phone-on",
includes=INCLUDES),
Run(wait2_command),
RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}",
"r2lab-embedded/shell/macphone.sh", "phone-check-cx",
includes=INCLUDES),
RunScript(find_local_embedded_script("faraday.sh"), f"macphone{id}",
"r2lab-embedded/shell/macphone.sh", "phone-start-app",
includes=INCLUDES),
],
label=f"turn off airplane mode on phone {id}",
required=phones_requirements,
scheduler=scheduler)
for id, wait_command, wait2_command in zip(phones, wait_commands, wait2_commands)
]
if quectel:
# wait 60s for Quectel connection(s) to set up
wait_before_attach_quectel = PrintJob(
"Wait again 30s before attaching Quectel device(s)",
scheduler=scheduler,
required=(job_start_phones,check_kube5g,detach_quectel_nodes),
sleep=30,
label="Sleep 30s before attaching Quectel device(s)"
)
job_attach_quectel = [
SshJob(
scheduler=scheduler,
required=wait_before_attach_quectel,
node=node,
critical=True,
verbose=verbose,
label=f"Attach Quectel UE on fit node {id}",
command = RunScript(find_local_embedded_script("nodes.sh"),
"quectel-attach", includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
# wait 30s for Quectel connection(s) to set up
wait_quectel_cx_ready = PrintJob(
"Let the Quectel connection(s) set up",
scheduler=scheduler,
required=job_attach_quectel,
sleep=30,
label="Sleep 30s for the Quectel connection(s) to set up"
)
test_quectel_cx = [
SshJob(
scheduler=scheduler,
required=wait_quectel_cx_ready,
node=node,
critical=False,
verbose=verbose,
label=f"Check the Quectel cx on fit node {id}",
command = RunScript(find_local_embedded_script("nodes.sh"),
"check-quectel-cx", includes=INCLUDES),
) for id, node in nodes_quectel_index.items()
]
##########
# Update the .dot and .png file for illustration purposes
scheduler.check_cycles()
name = "deploy-kube5g"
print(10*'*', 'See main scheduler in',
scheduler.export_as_pngfile(name))
# orchestration scheduler jobs
if verbose:
scheduler.list()
if dry_run:
return True
if not scheduler.orchestrate():
print(f"RUN KO : {scheduler.why()}")
scheduler.debrief()
return False
print(f"RUN OK, you can log now on master node {fit_master} to manually change the scenario")
print(80*'*')