def downloadTestLogs(testid):
"""
This procedure just send test life cycle log file to scheduler upon test cleanup. This file provide user
information about test execution sequence on agent
"""
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
test_logger.info("Sending test log to daytona host")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[current_test.stathostip]
download_file = current_test.agent_log_file
cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip())
test_logger.info("Test log file transfer complete")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def stopTest(self, *args):
"""
This procedure is invoked by agent when it receives DAYTONA_STOP_TEST message from scheduler. In current test
life cycle stat host agent receive this message from scheduler. In this procedure, agent changes the state of a
test from RUNNING to TESTEND and update running queue
"""
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
testid = int(params)
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
current_test.status = "TESTEND"
save_test(current_test.testid, current_test)
test_logger.info("Test stop")
return "SUCCESS"
else:
raise Exception("Test not running : " + str(current_test.testid))
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def fileDownload(self, *args):
"""
On test completion, agent execute this procedure when it receive DAYTONA_FILE_DOWNLOAD message from scheduler.
We create a TAR file called results.tgz and save it test location, then we send this file to scheduler and save it
in scheduler side file system
"""
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
testid = int(args[2])
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | START")
lctx.debug("Preparing TAR file of system metric folder")
test_logger.info("Preparing TAR file of system metric folder")
common.make_tarfile(current_test.archivedir + "results.tgz", current_test.resultsdir + "/")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[current_test.stathostip]
download_file = current_test.archivedir + "results.tgz"
test_logger.info("Sending TAR file to daytona host")
cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip())
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def cleanup(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
testid = int(params)
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("CLEANUP | " + str(current_test.testid) + " | START")
test_logger.info("Test cleanup")
downloadTestLogs(testid)
LOG.removeLogger(current_test.tobj)
shutil.rmtree(current_test.resultsdir, ignore_errors=True)
delete_test(testid)
lctx.debug("CLEANUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def cleanup(self, *args):
"""
This procedure is called on test completion or timer expiry or if something goes wrong with test. It perform
below tasks:
* Download agent side test execution life cycle logs
* remove the logger object for this particular test
* Delete the test logs file system
* Delete the test from running queue of agent
"""
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
testid = int(params)
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("CLEANUP | " + str(current_test.testid) + " | START")
test_logger.info("Test cleanup")
downloadTestLogs(testid)
LOG.removeLogger(current_test.tobj)
shutil.rmtree(current_test.resultsdir, ignore_errors=True)
delete_test(testid)
lctx.debug("CLEANUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def fileDownload(self, *args):
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
testid = int(args[2])
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) +
" | START")
lctx.debug("Preparing TAR file of system metric folder")
test_logger.info("Preparing TAR file of system metric folder")
common.make_tarfile(current_test.archivedir + "results.tgz",
current_test.resultsdir + "/")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[
current_test.stathostip]
download_file = current_test.archivedir + "results.tgz"
test_logger.info("Sending TAR file to daytona host")
cl.sendFile(current_test.serverip, current_test.serverport,
download_file, dest.strip())
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) +
" | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def updateStatus(self, curStatus, newStatus):
"""
Update test status from curStatus to newStatus in database for a given test.
"""
lctx = LOG.getLogger("dblog", "DH")
lctx.debug("setting status from %s to %s" % (curStatus, newStatus))
if self.testobj.TestInputData.exec_results_path is not None:
test_logger = LOG.gettestlogger(self, "EXEC")
test_logger.info("Setting test status from %s to %s" %
(curStatus, newStatus))
update_res = self.db.query(
"""update TestInputData SET end_status = %s where testid=%s""",
(newStatus, self.testobj.TestInputData.testid), False, True)
self.testobj.TestInputData.end_status = newStatus
update_res = self.db.query(
"""update CommonFrameworkSchedulerQueue SET state = %s, message = %s, state_detail = %s where testid = %s""",
(newStatus, newStatus, newStatus,
self.testobj.TestInputData.testid), False, True)
if newStatus == "finished clean" or newStatus == "failed" or newStatus == "abort" or newStatus == "kill" or newStatus == "timeout clean":
update_res = self.db.query(
"""delete from CommonFrameworkSchedulerQueue where testid=%s""",
(self.testobj.TestInputData.testid, ), False, True)
lctx.debug(
"Deleted entry from CommonFrameworkSchedulerQueue because of failure for : "
+ str(self.testobj.TestInputData.testid))
return
def getStatus(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
testid = int(params)
current_test = get_test(testid)
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug(str(current_test.testid) + ":" + current_test.status)
test_logger.info("Test Status : " + current_test.status)
return current_test.status
else:
return "TESTNA"
def getStatus(self, *args):
"""
Agent execute this procedure whenever scheduler want to check state of a test by sending DAYTONA_GET_STATUS
message to agent. In this procedure we fetch the test from running queue and return saved test state information
"""
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
testid = int(params)
current_test = get_test(testid)
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug(str(current_test.testid) + ":" + current_test.status)
test_logger.info("Test Status : " + current_test.status)
return current_test.status
else:
return "TESTNA"
def stopTest(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
testid = int(params)
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
current_test.status = "TESTEND"
save_test(current_test.testid, current_test)
test_logger.info("Test stop")
return "SUCCESS"
else:
raise Exception("Test not running : " + str(current_test.testid))
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def downloadTestLogs(testid):
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
test_logger.info("Sending test log to daytona host")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[
current_test.stathostip]
download_file = current_test.agent_log_file
cl.sendFile(current_test.serverip, current_test.serverport,
download_file, dest.strip())
test_logger.info("Test log file transfer complete")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def setupTest(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
test_serialized = params.split(",")[0]
host_type = params.split(",")[1]
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test = get_test(t2.testobj.TestInputData.testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START")
test_logger.info("Test setup started")
current_test.tobj = testobj.testDefn()
current_test.tobj = t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid)
shutil.rmtree(dir, ignore_errors=True)
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid) + "/results/"
if host_type == "EXEC":
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.statsdir = prefix + current_test.stathostip + "/sar/"
current_test.resultsdir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/results"
current_test.archivedir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/"
if host_type == "EXEC":
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
test_logger.info("Test directory created")
# todo : check and validate if exec script is provided in expected format and
# the file exists in that location
if host_type == "EXEC":
execscript = current_test.tobj.testobj.TestInputData.execution_script_location
lctx.debug("TEST SETUP : " + str(execscript))
current_test.execscriptfile = current_test.execdir + "/" + execscript
lctx.debug(current_test.execscriptfile)
# check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at
# this location
execscript_location = EXEC_SCRIPT_DIR + execscript
execscript_location = os.path.realpath(execscript_location)
valid_path = os.path.commonprefix(
[execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR
if valid_path:
if os.path.isfile(execscript_location):
ret = shutil.copytree(
os.path.dirname(execscript_location),
os.path.dirname(current_test.execscriptfile))
else:
raise Exception(
"Execution script not found at Daytona Execution Script Location : "
+ EXEC_SCRIPT_DIR)
else:
raise Exception(
"Access Denied : Use Daytona Execution Script Location '"
+ EXEC_SCRIPT_DIR + "' for executing "
"exec scripts")
os.chmod(current_test.execscriptfile, 0744)
test_logger.info("Execution script copied successfully")
save_test(current_test.testid, current_test)
test_logger.info("Test setup complete")
lctx.debug("TEST SETUP | " + str(current_test.testid) +
" | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def startTest(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
testid = int(params.split(",")[0])
host_type = params.split(",")[1]
current_test = get_test(testid)
strace_config = None
perf_config = dict()
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("TESTSTART | " + str(current_test.testid) + " | START")
test_logger.info("Starting test")
current_test.status = "RUNNING"
current_test.actionID = actionID
save_test(current_test.testid, current_test)
if current_test.tobj.testobj.TestInputData.strace:
strace_config = dict()
strace_config["delay"] = str(
current_test.tobj.testobj.TestInputData.strace_delay)
strace_config["duration"] = str(
current_test.tobj.testobj.TestInputData.strace_duration)
strace_config[
"process"] = current_test.tobj.testobj.TestInputData.strace_process
perf_config["delay"] = str(
current_test.tobj.testobj.TestInputData.perf_delay)
perf_config["duration"] = str(
current_test.tobj.testobj.TestInputData.perf_duration)
if current_test.tobj.testobj.TestInputData.perf_process:
perf_config[
"process"] = current_test.tobj.testobj.TestInputData.perf_process
test_logger.info("Configuring perf profiler - " + str(perf_config))
if strace_config is not None:
test_logger.info("Configuring strace profiler - " +
str(strace_config))
system_metrics_gather.perf_strace_gather(current_test.testid,
perf_config,
strace_config)
test_logger.info("Profiler started")
if host_type == "EXEC":
# Copied execscript
execscript = current_test.execscriptfile
args = ""
for a in current_test.tobj.testobj.TestInputData.execScriptArgs:
args = args + " \"" + a[3] + "\""
execline = execscript + args
lctx.debug("Execution line:" + execline)
test_logger.info("Execution script started")
# execute the exec script here
exec_cmd(execline, command, sync, obj, actionID, current_test)
lctx.debug("TESTSTART | " + str(current_test.testid) +
" | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def testmon(self, *mon):
process_results_threads = defaultdict()
while True:
d = "TSMON [R] : |"
remove = False
error = False
for k in self.running_tests:
if self.running_tests[k] is not None:
t = self.running_tests[k]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
test_logger = LOG.gettestlogger(t2, "EXEC")
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
t.updateStatus("running", "failed")
remove = True
break # out of for loop
try:
ret = self.cl.send(
t.testobj.TestInputData.exechostname, self.CPORT,
self.ev.construct(
"DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
status = ret.split(",")[1]
lctx.debug(status)
test_logger.info("Test status : " + status)
except Exception as e:
lctx.debug(e)
t.updateStatus("running", "failed")
error = True
break # out of for loop
if status == "RUNNING":
found = checkTestRunning(
t.testobj.TestInputData.testid)
if not found:
error = True
break
d = d + str(self.running_tests[k].testobj.
TestInputData.testid) + "|"
elif status in ["TESTEND", "TIMEOUT"]:
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
if t.testobj.TestInputData.end_status == "running":
lctx.debug(t.testobj.TestInputData.end_status)
if status == "TIMEOUT":
t.testobj.TestInputData.timeout_flag = True
t.updateStatus("running", "timeout")
else:
t.updateStatus("running", "completed")
pt = common.FuncThread(
self.process_results, True, t,
t.testobj.TestInputData.end_status)
process_results_threads[
t.testobj.TestInputData.testid] = (pt, t)
pt.start()
remove = True
break
elif t.testobj.TestInputData.end_status == "collating" or t.testobj.TestInputData.end_status == "completed" or t.testobj.TestInputData.end_status == "finished clean":
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
else:
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) +
":" + str(status))
break # out of for loop
elif status.strip() in ["FAILED", "TESTNA"]:
if status.strip() == "FAILED":
error = True
elif status.strip() in ["ABORT", "TESTNA"]:
remove = True
t.updateStatus("", "failed")
lctx.error("TEST " + status.strip() +
" : Cleaning test from running queue")
break # out of for loop
else:
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) + ":" +
str(status))
break # out of for loop
lctx.info(d)
d = ""
if error:
retsend = None
test_logger.error("Bad test status " + status +
" - Terminating test")
ip = t.testobj.TestInputData.exechostname
try:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
except:
pass
if retsend and retsend.split(",")[1] == "ALIVE":
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t.testobj.TestInputData.testid)))
test_logger.error("Test Aborted on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
try:
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
except:
pass
if retsend and retsend.split(",")[1] == "ALIVE":
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct(
"DAYTONA_CLEANUP_TEST",
str(t.testobj.TestInputData.testid)))
test_logger.error("Test Aborted on stat host " + s)
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
if remove:
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
time.sleep(2)
def process_results(self, *args):
# set test status completed
# call stop monitors
# send prepare results command to exec
# set test status collating
# copy results files from exec
# copy files from each mon
# set test status finished
# remove test from running Q
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test execution completes, preocessing test results")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
ptop = process_files.ProcessOutputFiles(
LOG.getLogger("processTop", "DH"))
# todo : avoid send client its own ip
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading LOGS from " + ip + " : " +
retsend)
test_logger.error("Error downloading LOGS from " + ip +
" : " + retsend)
else:
test_logger.info(
"Logs download successfull from exec host " + ip)
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
test_logger.error("Error in untar EXEC host results")
lctx.error(e)
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
test_logger.info(
"Exec host logs extracted and processed succesfully")
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
try:
lctx.info("Stopping test on stat host : " + s)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Failed to stop test on stat host " +
s + " : " + retsend)
test_logger.error(
"Failed to stop test on stat host " + s +
" : " + retsend)
else:
test_logger.info("Test stopped on stat host " + s)
lctx.info("Sending results.tgz download to :" +
s.strip() + ":" + str(p))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
test_logger.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
else:
test_logger.info(
"Logs downloaded from stat host " + s)
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(
ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
test_logger.info(
"Stat host " + s +
" logs extracted and processed succesfully")
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on stat host " +
s)
except Exception as e:
lctx.error(e)
test_logger.error(e)
continue
except Exception as e:
lctx.error("Error in processing results")
lctx.error(e)
test_logger.error("Error in processing results")
test_logger.error(e)
t.updateStatus("collating", "failed")
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateEndTime(tstr)
f = None
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} <BR> Framework : {} <BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR><BR> Creation time : {} <BR>Start time : {} <BR>End " \
"time : {} <BR>".format(t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. <BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except Exception as e:
lctx.error("Mail send error")
LOG.removeLogger(t)
return "SUCCESS"
def trigger(self, *args):
"""
trigger starts in a thread, keep track of all triggers and they should complete in a specified time, otherwise
signal a close. triggers are used for test setup, then starting test and then taking test into running
state on agent.
"""
t = args[1]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
time.sleep(6)
# Setting up test logger for capturing test life cycle on scheduler
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test setup started")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
# Sending DAYTONA_SETUP_TEST command on execution host, on receiving this command agent will perform basic
# test setup by creating test directories for saving log files and it will copy execution script in test
# directory for starting execution
ip = t.testobj.TestInputData.exechostname
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on exec host " + ip)
# Triggering test setup on all stat hosts
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
test_logger.info("Starting test setup on stat host " + s)
lctx.debug(s.strip())
p = self.CPORT
# Sending hearbeat message to check whether stat host is up or not
retsend = self.cl.send(
s, p, self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "ALIVE":
raise Exception(
"Remove host not avaliable - No Heartbeat ",
t2.testobj.TestInputData.testid)
else:
test_logger.info("Hearbeat received from stat host " +
s)
# Trigger DAYTONA_HANDSHAKE to verify that both agent and scheduler are able to communicate with
# each other on custom daytona ports
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(t2.testobj.TestInputData.testid) + "," +
s))
lctx.debug(retsend)
if retsend == "SUCCESS":
alive = True
server.serv.registered_hosts[s] = s
addr = socket.gethostbyname(s)
server.serv.registered_hosts[addr] = addr
test_logger.info(
"Handshake successfull with agent on stat host "
+ s)
else:
raise Exception(
"Unable to handshake with agent on stats host:"
+ s, t2.testobj.TestInputData.testid)
else:
raise Exception("Stat host " + s +
" not avaliable - No Heartbeat")
# Trigger test setup on stat hosts, this will create test directory for saving log files
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on stat host " + s)
# Trigger the start of test on exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Failed to start Test : ",
t2.testobj.TestInputData.testid)
test_logger.info("Test started on exec host " + ip)
# Trigger the start of test on STAT hosts (This is for initiating system metric data collection)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
p = self.CPORT
s = s.strip()
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("Failed to start test on STAT host : ",
s)
else:
raise Exception("Failed to start test on STAT host : ", s)
test_logger.info("Test started on stat host " + s)
# Get status from exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
if retsend and len(retsend.split(",")) > 1:
if "RUNNING" == retsend.split(
",")[1] or "MONITOR_ON" == retsend.split(",")[1]:
# update from setup to running
lctx.debug("Updating test status to running in DB")
t.updateStatus("setup", "running")
now = time.time()
tstr = str(
time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateStartTime(tstr)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[
t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
self.lock.acquire()
self.running_tests[t.testobj.TestInputData.frameworkid] = t
self.lock.release()
else:
lctx.error(
"Unable to determine status, testmon will garbage collect this testid"
)
# Garbage collect testid in runnning state that fails to give status
# Killing of threads on client host and remove from list with status=fail is done in testmon
except Exception as e:
# If any trigger fails, then abort test startup with error
lctx.error(e)
test_logger.error(e)
lctx.error("ERROR : Unknown trigger error : " +
str(t.testobj.TestInputData.testid))
t.updateStatus("setup", "failed")
lctx.debug(traceback.print_exc())
lctx.error("Removing Test " + str(t.testobj.TestInputData.testid) +
" from dispatch Q")
self.dispatchQ__lock.acquire()
del self.dispatch_queue[t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
# This will abort test and perform cleanup on exec host with trigger was successful on exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test aborted on exec host " + ip)
# On all other stat hosts we send cleanup in case trigger was successful on any stat host
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_CLEANUP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test abort on stat host " + s)
LOG.removeLogger(t2)
return "FAILED"
return "SUCCESS"
def process_results(self, *args):
"""
This procedure is called by testmon as seperate thread when test execution ends or test timeout occur
on agent.
"""
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
# Setting up test logger for capturing test life cycle on scheduler
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test execution completes, preocessing test results")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
# set test status to collating
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
# Initiate instance of ProcessOutputFiles for docker and top outout file processing
ptop = process_files.ProcessOutputFiles(
LOG.getLogger("processTop", "DH"))
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
# send file download command to exec host (no need to send stop test as this procedure is invoked due
# to test end on exec host)
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading LOGS from " + ip + " : " +
retsend)
test_logger.error("Error downloading LOGS from " + ip +
" : " + retsend)
else:
test_logger.info(
"Logs download successfull from exec host " + ip)
# copy results files from exec to daytona file system and untar results
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
test_logger.error("Error in untar EXEC host results")
lctx.error(e)
# process top and docker stat files downloaded from exec host
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
test_logger.info(
"Exec host logs extracted and processed succesfully")
# send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
try:
# Send DAYTONA_STOP_TEST on all agent hosts to stop SAR data collection after test finish on
# exec host. This message is required to tell stat hosts that test execution is finished on
# exec host. Upon receiving this message on stat host, agent will change test state to TESTEND
# and then other SAR data collection thread will stop writing log files for this test.
lctx.info("Stopping test on stat host : " + s)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Failed to stop test on stat host " +
s + " : " + retsend)
test_logger.error(
"Failed to stop test on stat host " + s +
" : " + retsend)
else:
test_logger.info("Test stopped on stat host " + s)
# send file download command to stat host
lctx.info("Sending results.tgz download to :" +
s.strip() + ":" + str(p))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
test_logger.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
else:
test_logger.info(
"Logs downloaded from stat host " + s)
# copy results files from stat host to daytona file system and untar results
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
# process top and docker stat files downloaded from stat host
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(
ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
test_logger.info(
"Stat host " + s +
" logs extracted and processed succesfully")
# send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on stat host " +
s)
except Exception as e:
# Just continue with other stat hosts if any exception occurs while working on any particular
# host (Continue only when something goes wrong with stat host, because we still want to
# download logs from other stat hosts)
lctx.error(e)
test_logger.error(e)
continue
except Exception as e:
# Throw an error if anything goes wrong with finishing test on exec host and set test state to failed
lctx.error("Error in processing results")
lctx.error(e)
test_logger.error("Error in processing results")
test_logger.error(e)
t.updateStatus("collating", "failed")
# updating test state to timeout clean if test terminated due to timeout else setting it to finished clean
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
# update test end time in database
t.updateEndTime(tstr)
f = None
# Formatting email with results.csv details to send it to CC list if user has mentioned in test details
# (admin need to smtp server for this functionality to work, smtp server details need to be
# mentioned in config.sh)
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} <BR> Framework : {} <BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR><BR> Creation time : {} <BR>Start time : {} <BR>End " \
"time : {} <BR>".format(t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. <BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except Exception as e:
lctx.error("Mail send error")
LOG.removeLogger(t)
return "SUCCESS"
def testmon(self, *mon):
"""
Testmon continuously monitors all the running test. It keeps on checking test status on a exec host where execution
script is running. If anything goes wrong with the test execution, this thread trigger termination actions for
this test. It also trigger graceful test termination and logs collection when test finishes on exec host
"""
process_results_threads = defaultdict()
while True:
d = "TSMON [R] : |"
remove = False
error = False
# Continuously iterate over running test list for checking test status
for k in self.running_tests:
if self.running_tests[k] is not None:
t = self.running_tests[k]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
# Initiating test logger for capturing test life cycle on scheduler, all logs are logged in
# file <testid>.log
test_logger = LOG.gettestlogger(t2, "EXEC")
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
t.updateStatus("running", "failed")
remove = True
break # out of for loop
try:
# Send DAYTONA_GET_STATUS message on exec host mentioned in test for checking test status
ret = self.cl.send(
t.testobj.TestInputData.exechostname, self.CPORT,
self.ev.construct(
"DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
status = ret.split(",")[1]
lctx.debug(status)
test_logger.info("Test status : " + status)
except Exception as e:
lctx.debug(e)
t.updateStatus("running", "failed")
error = True
break # out of for loop
if status == "RUNNING":
# If the test is in running state, then we need to verify that user hasn't terminated this
# test from UI. If user has terminated then testmon will stop test execution on all exec host
# and stat host
found = checkTestRunning(
t.testobj.TestInputData.testid)
if not found:
error = True
break
d = d + str(self.running_tests[k].testobj.
TestInputData.testid) + "|"
elif status in ["TESTEND", "TIMEOUT"]:
# If test ends on exec host or if test timout occurs then trigger graceful shutdown of this test
# Testmon invoke a new thread for this test for logs download and test cleanup from all hosts
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
if t.testobj.TestInputData.end_status == "running":
lctx.debug(t.testobj.TestInputData.end_status)
if status == "TIMEOUT":
t.testobj.TestInputData.timeout_flag = True
t.updateStatus("running", "timeout")
else:
t.updateStatus("running", "completed")
# process_results download log files and perform cleanup on all other hosts
pt = common.FuncThread(
self.process_results, True, t,
t.testobj.TestInputData.end_status)
process_results_threads[
t.testobj.TestInputData.testid] = (pt, t)
pt.start()
remove = True
break
elif t.testobj.TestInputData.end_status == "collating" or t.testobj.TestInputData.end_status == "completed" or t.testobj.TestInputData.end_status == "finished clean":
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
else:
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) +
":" + str(status))
break # out of for loop
elif status.strip() in ["FAILED", "TESTNA"]:
# Test termination if test fails or test is not even running on the host
if status.strip() == "FAILED":
error = True
elif status.strip() in ["ABORT", "TESTNA"]:
remove = True
t.updateStatus("", "failed")
lctx.error("TEST " + status.strip() +
" : Cleaning test from running queue")
break # out of for loop
else:
# Test termination on receiving any unknown test state
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) + ":" +
str(status))
break # out of for loop
lctx.info(d)
d = ""
# Two modes of test termination:
if error:
# If error is set then testmon will perform below steps:
# 1. Send test ABORT on exec host if is alive, this will stop execution script, perform logs cleanup and
# test termination on the host
# 2. Send test cleanup on all other stat host for performing logs cleanup and test termination on the host
# 3. Remove test from the scheduler running queue
retsend = None
test_logger.error("Bad test status " + status +
" - Terminating test")
ip = t.testobj.TestInputData.exechostname
try:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
except:
pass
if retsend and retsend.split(",")[1] == "ALIVE":
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t.testobj.TestInputData.testid)))
test_logger.error("Test Aborted on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
try:
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
except:
pass
if retsend and retsend.split(",")[1] == "ALIVE":
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct(
"DAYTONA_CLEANUP_TEST",
str(t.testobj.TestInputData.testid)))
test_logger.error("Test Aborted on stat host " + s)
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
if remove:
# If remove flag is set, then testmon will only delete this test from the running queue of scheduler
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
time.sleep(2)
def trigger(self, *args):
# trigger starts in a thread, keep track of all triggers and they should complete in a specified time,
# otherwise signal a close
t = args[1]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
time.sleep(6)
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test setup started")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
ip = t.testobj.TestInputData.exechostname
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on exec host " + ip)
# get statistics hosts
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
test_logger.info("Starting test setup on stat host " + s)
lctx.debug(s.strip())
p = self.CPORT
retsend = self.cl.send(
s, p, self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "ALIVE":
raise Exception(
"Remove host not avaliable - No Heartbeat ",
t2.testobj.TestInputData.testid)
else:
test_logger.info("Hearbeat received from stat host " +
s)
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(t2.testobj.TestInputData.testid) + "," +
s))
lctx.debug(retsend)
if retsend == "SUCCESS":
alive = True
server.serv.registered_hosts[s] = s
addr = socket.gethostbyname(s)
server.serv.registered_hosts[addr] = addr
test_logger.info(
"Handshake successfull with agent on stat host "
+ s)
else:
raise Exception(
"Unable to handshake with agent on stats host:"
+ s, t2.testobj.TestInputData.testid)
else:
raise Exception("Stat host " + s +
" not avaliable - No Heartbeat")
# start stats monitors on req hosts
# any host that blocks start monitor blocks the scheduling for the FW
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on stat host " + s)
# Trigger the start of test to test box
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Failed to start Test : ",
t2.testobj.TestInputData.testid)
test_logger.info("Test started on exec host " + ip)
# Trigger the start of test on STAT hosts (This is for initiating system metric data collection)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
p = self.CPORT
s = s.strip()
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("Failed to start test on STAT host : ",
s)
else:
raise Exception("Failed to start test on STAT host : ", s)
test_logger.info("Test started on stat host " + s)
# Get status from tst box
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
if retsend and len(retsend.split(",")) > 1:
if "RUNNING" == retsend.split(
",")[1] or "MONITOR_ON" == retsend.split(",")[1]:
# update from setup to running
lctx.debug("Updating test status to running in DB")
t.updateStatus("setup", "running")
now = time.time()
tstr = str(
time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateStartTime(tstr)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[
t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
self.lock.acquire()
self.running_tests[t.testobj.TestInputData.frameworkid] = t
self.lock.release()
else:
lctx.error(
"Unable to determine status, testmon will garbage collect this testid"
)
# Garbage collect testid in runnning state that fails to give status
# Killing of threads on client host and remove from list with status=fail is done in testmon
except Exception as e:
lctx.error(e)
test_logger.error(e)
lctx.error("ERROR : Unknown trigger error : " +
str(t.testobj.TestInputData.testid))
t.updateStatus("setup", "failed")
lctx.debug(traceback.print_exc())
lctx.error("Removing Test " + str(t.testobj.TestInputData.testid) +
" from dispatch Q")
self.dispatchQ__lock.acquire()
del self.dispatch_queue[t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test aborted on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_CLEANUP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test abort on stat host " + s)
LOG.removeLogger(t2)
return "FAILED"
return "SUCCESS"
def setupTest(self, *args):
"""
Test setup is called when scheduler send "DAYTONA_SETUP_TEST" message to agent. In this procedure agent create
all necessary file system path string and update in test object. After creating file path string it execute command
for making all these file system directories so that agent can later save SAR data. On exec host, it copies execution
script from Execscript folder to test specific directory in order to keep execution script seperate in case of
multiple test execution
:param self:
:param args: tuple of arguments containing obj, command, parameter sent by scheduler to agent for this command,
actionID and sync flag to denote if we need to execute this procedure in sync or async mode
:return: SUCCESS in case everything goes well otherwise it throws ERROR
"""
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
test_serialized = params.split(",")[0]
host_type = params.split(",")[1]
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test = get_test(t2.testobj.TestInputData.testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START")
test_logger.info("Test setup started")
current_test.tobj = testobj.testDefn()
current_test.tobj = t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid)
shutil.rmtree(dir, ignore_errors=True)
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid) + "/results/"
if host_type == "EXEC":
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.statsdir = prefix + current_test.stathostip + "/sar/"
current_test.resultsdir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/results"
current_test.archivedir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/"
if host_type == "EXEC":
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
test_logger.info("Test directory created")
if host_type == "EXEC":
execscript = current_test.tobj.testobj.TestInputData.execution_script_location
lctx.debug("TEST SETUP : " + str(execscript))
current_test.execscriptfile = current_test.execdir + "/" + execscript
lctx.debug(current_test.execscriptfile)
# check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at
# this location
execscript_location = EXEC_SCRIPT_DIR + execscript
execscript_location = os.path.realpath(execscript_location)
valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR
if valid_path:
if os.path.isfile(execscript_location):
ret = shutil.copytree(os.path.dirname(execscript_location),
os.path.dirname(current_test.execscriptfile))
else:
raise Exception(
"Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR)
else:
raise Exception(
"Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing "
"exec scripts")
os.chmod(current_test.execscriptfile, 0744)
test_logger.info("Execution script copied successfully")
save_test(current_test.testid, current_test)
test_logger.info("Test setup complete")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
