def Run(self):
# Build simulation model
dbg.print_info("Building simulation model for '%s'" % self.name)
flog = open(self.buildsim_log, 'w')
if self.NeedsReceiver():
rv = sim.buildSimWithVPI(self.name, self.tb_dir, self.tb_name_l,
self.GetParams(), self.GetSpecs(),
self.build_dir, flog)
else:
rv = sim.buildSim(self.name, self.tb_dir, self.tb_name_l,
self.GetParams(), self.GetSpecs(),
self.build_dir, flog)
flog.close()
if rv != 0:
dbg.print_error("Failed to build a simulation model for '%s'" %
self.name)
dbg.print_error("Check log: %s" % self.buildsim_log)
return 1
# Run Simulation
dbg.print_info("Running simulation for '%s'" % self.name)
flog = open(self.runsim_log, 'w')
sim.runSim(self.name, self.build_dir, flog)
flog.close()
# Check if test passed
dbg.print_success("Simulation for '%s' finished" % self.name)
return 0
def ExecuteBlockCheck(self, cmd):
(rv, output, error) = self.ExecuteBlock(cmd)
if rv != 0:
dbg.print_error("'%s' failed on '%s'" % (cmd, self.name))
dbg.print_error("%s" % error)
print >> self.flog, cmd
print >> self.flog, "OUTPUT: %s" % output
print >> self.flog, "ERROR: %s" % error
return (rv, output, error)
def WaitForBoot(self):
was_failure = False
for v in self.vm_l:
rv = v.WaitForBoot()
if rv != 0:
dbg.print_error("Failed waiting for %s to boot!" % v.GetName())
was_failure = True
rv = 1 if was_failure else 0
return rv
def WaitForBoot(self):
dbg.print_info("Waiting for '%s' to boot" % self.name)
retry_interval = 2
retry_num = self.max_boot_time / retry_interval
rv = ex.executeCmdBlockWithRetry("hostname", retry_num, retry_interval,
self.flog, self.ssh_name,
self.remote_script_name)
if rv != 0:
dbg.print_error("Waiting for boot failed for '%s'" % self.name)
return rv
def Run(self):
# Generate src/snk files
dbg.print_info("Generating SRC/SNK files for '%s'" % self.name)
rv = self.genSrcSnk()
if rv != 0:
dbg.print_error("Failed to generate SRC/SNK files for '%s'" %
self.name)
dbg.print_error("Check log: %s" % self.gen_ss_log)
return 1
super(UnitTest, self).Run()
def getHostIP(hostname):
if hostname == "hanoi4.princeton.edu":
ip = "172.16.0.10"
elif hostname == "moscow.princeton.edu":
ip = "172.16.0.11"
else:
ip = getVFIP(hostname)
if ip == None:
dbg.print_error("Can't determine IP for %s" % hostname)
return ip
def joinSwarm(h_cfg, join_cmd):
if leaveSwarm(h_cfg) != 0:
return 1
p = ex.executeCmd(join_cmd, subprocess.PIPE, h_cfg.host_name,
"docker_client.sh")
output, error = p.communicate()
if p.returncode != 0:
dbg.print_error(error)
return 1
return 0
def launchSlurmJob(job_name, cmd, slurm_dep_l, flog_path):
slurm_file = job_name + ".slurm"
fp = open(slurm_file, "w")
fp.write("#!/bin/sh\n")
fp.write("#SBATCH -N 1\n")
fp.write("#SBATCH -c 8\n")
fp.write("#SBATCH --mem=65536\n")
fp.write("#SBATCH -t 24:00:00\n")
fp.write("#SBATCH -J %s\n\n" % job_name)
fp.write("%s > %s" % (cmd, flog_path))
fp.close()
os.chmod(slurm_file, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
retry_limit = 128
if len(slurm_dep_l) > 0:
opt_dep = "--dependency=afterok:" + ':'.join(
[str(job_id) for job_id in slurm_dep_l])
else:
opt_dep = ''
slurm_cmd = "sbatch %s %s" % (opt_dep, slurm_file)
dbg.print_info("Slurm command: %s" % slurm_cmd)
proc = subprocess.Popen(shlex.split(slurm_cmd),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
retry_cnt = 1
while proc.returncode != 0:
if retry_cnt > retry_limit:
dbg.print_error("Failed to submit command %s through slurm" % cmd)
return None
else:
dbg.print_info(
"Slurm submission failed. Retrying in 30 seconds...")
time.sleep(30)
dbg.print_info("Submitting slurm job")
proc = subprocess.Popen(shlex.split(slurm_cmd),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
# Get job ID
match = re.match("Submitted batch job (\d+)\n", out)
if match != None:
job_id = int(match.group(1))
dbg.print_info("Submitted batch job %d" % job_id)
return job_id
else:
dbg.print_error("Can't find submitted job id!")
return None
def validateTrace(trace_path):
max_line_w = 200
flog = open(trace_path, 'r')
cnt = 0
for l in flog:
if len(l) > max_line_w:
dbg.print_error("Encountered a line longer than %d characters in %s" % (max_line_w, trace_path))
cnt += 1
dbg.print_info("Number of lines in %s: %d" % (trace_path, cnt))
flog.close()
def AddVMByID(self, id_l):
if self.vm_type == "client":
for i in id_l:
self.vm_l.append(ClientVM(i))
elif self.vm_type == "l2":
for i in id_l:
self.vm_l.append(L2VM(i))
elif self.vm_type == "l1":
self.vm_l.append(L1VM())
elif self.vm_type == "bare":
for i in id_l:
self.vm_l.append(BareVM(i))
else:
dbg.print_error("Uknown VM type: %s" % vm_type)
def tryExecute(cmd, hostname, valid_error_str):
# dbg.print_info("cmd: %s, host: %s, valid error: %s" % \
# (cmd, hostname, valid_error_str))
script_name = "%s_cmd.sh" % hostname
p = ex.executeCmd(cmd, subprocess.PIPE, hostname, script_name)
output, error = p.communicate()
if p.returncode != 0:
if common.isStrInText(valid_error_str, error):
dbg.print_warning(error)
else:
dbg.print_error(error)
return 1
return 0
def strInFile(fpath, str_l):
try:
f = open(fpath, 'r')
fdata = f.read()
f.close()
except IOError:
dbg.print_error("Can't open file: %s" % fpath)
return False
# File was opened and read
for s in str_l:
m = re.search(s, fdata)
if m == None:
return False
return True
def ExecuteAllBlock(self, cmd):
pid_vmname_d = dict()
proc_l = list()
for v in self.vm_l:
proc = v.ExecuteNonBlock(cmd, subp.PIPE)
proc_l.append(proc)
pid_vmname_d[proc.pid] = v.name
err_proc_l = ex.waitProcGroupAndCheckExitCode(proc_l)
if len(err_proc_l) > 0:
for proc in err_proc_l:
dbg.print_error("'%s' failed on %s" %
(cmd, pid_vmname_d[proc.pid]))
output, error = proc.communicate()
dbg.print_error(error)
else:
dbg.print_info("'%s' was executed successfully on %d machines!" %
(cmd, len(self.vm_l)))
def createOverlayNetwork(h_cfg):
if removeNetwork(h_cfg) != 0:
return None
dbg.print_info("%s: creating overlay network '%s'" % \
(h_cfg.host_name, h_cfg.nw_name))
cmd = "docker network create --driver=overlay --attachable %s" % h_cfg.nw_name
nw_id = None
p = ex.executeCmd(cmd, subprocess.PIPE, h_cfg.host_name, "create_nw.sh")
output, error = p.communicate()
if p.returncode != 0:
dbg.print_error("Can not create network '%s' on host '%s'" % \
(h_cfg.nw_name, h_cfg.host_name))
dbg.print_error(error)
return None
else:
nw_id = output
return nw_id
def removeNetworkOld(h_cfg):
no_network_str = "No such network"
# Remove network
cmd = "docker network rm %s" % h_cfg.nw_name
p = ex.executeCmd(cmd, subprocess.PIPE, h_cfg.host_name, "remote_nw.sh")
output, error = p.communicate()
if p.returncode != 0:
if common.isStrInText(no_network_str, error):
dbg.print_warning("%s: '%s' network does not exist!" % \
(h_cfg.host_name, h_cfg.nw_name))
else:
dbg.print_error("%s: can not remove network '%s'!" % \
(h_cfg.nw_name, h_cfg.host_name))
dbg.print_error(error)
return 1
else:
dbg.print_info("Removed network '%s'" % h_cfg.nw_name)
return 0
def Run(self):
# Build simulation model
dbg.print_info("Building simulation model for '%s'" % self.name)
flog = open(self.buildsim_log, 'w')
rv = sim.buildSim(self.name, self.tb_dir, self.tb_name_l,
self.GetParams(), self.GetSpecs(), self.build_dir,
flog)
flog.close()
if rv != 0:
dbg.print_error("Failed to build a simulation model for '%s'" %
self.name)
dbg.print_error("Check log: %s" % self.buildsim_log)
return 1
# Run Simulation
dbg.print_info("Running simulation for '%s'" % self.name)
flog = open(self.runsim_log, 'w')
sim.runSim(self.name, self.build_dir, flog)
flog.close()
# Check if test passed
if sim.isTestPassed(self.runsim_log):
dbg.print_success("Test for '%s' passed" % self.name)
else:
dbg.print_error("Test for '%s' failed" % self.name)
return 0
def Launch(self, flog):
if socket.gethostname() != cfg.L1_VM_NAME:
dbg.print_error(
"L2 VM can be launched only from inside of '%s' machine")
return False
else:
# Unbinding PCI device
cmd = "modprobe vfio-pci"
rv = ex.executeCmdBlockReturnCode(cmd, flog, "localhost",
self.remote_script_name)
if rv != 0:
dbg.print_info("'%s' failed on machine '%s'" %
(cmd, self.name))
return rv
cmd = "echo '\"%s\"' > /sys/bus/pci/devices/%s/driver/unbind" % (
self.bdf, self.bdf)
rv = ex.executeCmdBlockReturnCode(cmd, flog, "localhost",
self.remote_script_name)
if rv != 0:
dbg.print_error("'%s' failed on machine '%s'" %
(cmd, self.name))
return rv
cmd = "echo '\"8086 100e\"' > /sys/bus/pci/drivers/vfio-pci/new_id"
rv = ex.executeCmdBlockReturnCode(cmd, flog, "localhost",
self.remote_script_name)
if rv != 0:
dbg.print_error("'%s' failed on machine '%s'" %
(cmd, self.name))
return rv
super(L2VM, self).Launch(flog)
def initSwarm(h_cfg):
if leaveSwarm(h_cfg) != 0:
return None
# Initialize swarm on a host
cmd = "docker swarm init --advertise-addr %s" % h_cfg.if_name
p = ex.executeCmd(cmd, subprocess.PIPE, h_cfg.host_name,
"docker_server.sh")
output, error = p.communicate()
if p.returncode != 0:
dbg.print_error("Can not initialize swarm on %s" % h_cfg.host_name)
dbg.print_error(error)
return None
# Take only join command from init output
m = re.search("docker swarm join --token .*", output)
if m != None:
join_cmd = m.group(0)
else:
join_cmd = None
return join_cmd
def executeCmdCheckReturnCode(cmd, flog, hostname, script_name):
rv = executeCmdBlockReturnCode(cmd, flog, hostname, script_name)
if rv != 0:
dbg.print_error("'%s' failed on machine '%s'" % (cmd, hostname))
return rv
#!/usr/bin/env python
import dbglib as dbg
dbg.print_info("This is info")
dbg.print_success("This is success")
dbg.print_warning("This is warning")
dbg.print_error("This is an error")
