def cleanup(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
testid = int(params)
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("CLEANUP | " + str(current_test.testid) + " | START")
test_logger.info("Test cleanup")
downloadTestLogs(testid)
LOG.removeLogger(current_test.tobj)
shutil.rmtree(current_test.resultsdir, ignore_errors=True)
delete_test(testid)
lctx.debug("CLEANUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def cleanup(self, *args):
"""
This procedure is called on test completion or timer expiry or if something goes wrong with test. It perform
below tasks:
* Download agent side test execution life cycle logs
* remove the logger object for this particular test
* Delete the test logs file system
* Delete the test from running queue of agent
"""
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
testid = int(params)
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("CLEANUP | " + str(current_test.testid) + " | START")
test_logger.info("Test cleanup")
downloadTestLogs(testid)
LOG.removeLogger(current_test.tobj)
shutil.rmtree(current_test.resultsdir, ignore_errors=True)
delete_test(testid)
lctx.debug("CLEANUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def dispatch(self, *args):
dispatch_threads = defaultdict()
while True:
for k in self.testmap:
found = False
if k in self.dispatch_queue or k in self.running_tests:
found = True
else:
lctx.debug("Found spot for test")
if found:
continue
try:
tmp_t = self.testmap[k][0]
except Exception as e:
lctx.debug("No test object found in map")
continue
if tmp_t is None:
continue
alive = False
h = tmp_t.testobj.TestInputData.exechostname
test_logger = LOG.init_testlogger(tmp_t, "EXEC")
test_logger.info("Test execution starts")
try:
retsend = self.cl.send(
h, self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 2:
status = retsend.split(",")[1]
else:
raise Exception(
"Execution host not avaliable - No Heartbeat ",
tmp_t.testobj.TestInputData.testid)
if "ALIVE" == status:
test_logger.info(
"HeartBeat received from execution host " + h)
ret = self.cl.send(
h, self.CPORT,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(tmp_t.testobj.TestInputData.testid) + "," +
h))
if ret == "SUCCESS":
alive = True
test_logger.info(
"Handshake successful with execution host " +
h)
lctx.debug(
"Handshake successful in scheduler, adding ip/hostname to reg hosts"
)
server.serv.registered_hosts[h] = h
addr = socket.gethostbyname(h)
lctx.debug(addr)
server.serv.registered_hosts[addr] = addr
else:
raise Exception(
"Unable to handshake with agent on executuion host "
+ h)
except Exception as e:
lctx.error(e)
test_logger.error(e)
# pause the dbmon here as we dont want the same test to be picked again after we pop
self.dbinstance.mon_thread[0].pause()
self.dbinstance.lock.acquire()
t = self.testmap[k].pop(0)
t.updateStatus("waiting", "failed")
self.dbinstance.lock.release()
lctx.debug("Removed test from map : " +
str(t.testobj.TestInputData.testid))
self.dbinstance.mon_thread[0].resume()
LOG.removeLogger(tmp_t)
continue
# todo : add host to reg list if handshake successful
if alive == True and found == False:
# for each framework pick one and move it to running, iff running has an empty slot.
lctx.debug(
"-------Found empty slot in dispatch and running Q-------"
)
# pause the dbmon here as we dont want the same test to be picked again after we pop
self.dbinstance.mon_thread[0].pause()
self.dbinstance.lock.acquire()
t = self.testmap[k].pop(0)
self.dbinstance.lock.release()
lctx.info("< %s" % t.testobj.TestInputData.testid)
self.dispatchQ__lock.acquire()
self.dispatch_queue[k] = t
self.dispatchQ__lock.release()
t.updateStatus("waiting", "setup")
self.dbinstance.mon_thread[0].resume()
try:
trigger_thread = common.FuncThread(
self.trigger, True, t)
dispatch_threads[t.testobj.TestInputData.testid] = (
trigger_thread, t)
trigger_thread.start()
except Exception as e:
lctx.error("Trigger error : " +
str(t.testobj.TestInputData.testid))
test_logger.error("Test setup failed " +
str(t.testobj.TestInputData.testid))
LOG.removeLogger(tmp_t)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[k]
self.dispatchQ__lock.release()
lctx.debug(e)
try:
d = "DISPATCH [S/R] : "
for k in self.dispatch_queue:
d = d + " |" + str(
self.dispatch_queue[k].testobj.TestInputData.testid)
except:
lctx.debug("ERROR : Dispatch Q empty")
lctx.debug(d)
d = ""
time.sleep(2)
def process_results(self, *args):
# set test status completed
# call stop monitors
# send prepare results command to exec
# set test status collating
# copy results files from exec
# copy files from each mon
# set test status finished
# remove test from running Q
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test execution completes, preocessing test results")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
ptop = process_files.ProcessOutputFiles(
LOG.getLogger("processTop", "DH"))
# todo : avoid send client its own ip
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading LOGS from " + ip + " : " +
retsend)
test_logger.error("Error downloading LOGS from " + ip +
" : " + retsend)
else:
test_logger.info(
"Logs download successfull from exec host " + ip)
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
test_logger.error("Error in untar EXEC host results")
lctx.error(e)
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
test_logger.info(
"Exec host logs extracted and processed succesfully")
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
try:
lctx.info("Stopping test on stat host : " + s)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Failed to stop test on stat host " +
s + " : " + retsend)
test_logger.error(
"Failed to stop test on stat host " + s +
" : " + retsend)
else:
test_logger.info("Test stopped on stat host " + s)
lctx.info("Sending results.tgz download to :" +
s.strip() + ":" + str(p))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
test_logger.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
else:
test_logger.info(
"Logs downloaded from stat host " + s)
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(
ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
test_logger.info(
"Stat host " + s +
" logs extracted and processed succesfully")
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on stat host " +
s)
except Exception as e:
lctx.error(e)
test_logger.error(e)
continue
except Exception as e:
lctx.error("Error in processing results")
lctx.error(e)
test_logger.error("Error in processing results")
test_logger.error(e)
t.updateStatus("collating", "failed")
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateEndTime(tstr)
f = None
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} <BR> Framework : {} <BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR><BR> Creation time : {} <BR>Start time : {} <BR>End " \
"time : {} <BR>".format(t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. <BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except Exception as e:
lctx.error("Mail send error")
LOG.removeLogger(t)
return "SUCCESS"
def trigger(self, *args):
# trigger starts in a thread, keep track of all triggers and they should complete in a specified time,
# otherwise signal a close
t = args[1]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
time.sleep(6)
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test setup started")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
ip = t.testobj.TestInputData.exechostname
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on exec host " + ip)
# get statistics hosts
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
test_logger.info("Starting test setup on stat host " + s)
lctx.debug(s.strip())
p = self.CPORT
retsend = self.cl.send(
s, p, self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "ALIVE":
raise Exception(
"Remove host not avaliable - No Heartbeat ",
t2.testobj.TestInputData.testid)
else:
test_logger.info("Hearbeat received from stat host " +
s)
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(t2.testobj.TestInputData.testid) + "," +
s))
lctx.debug(retsend)
if retsend == "SUCCESS":
alive = True
server.serv.registered_hosts[s] = s
addr = socket.gethostbyname(s)
server.serv.registered_hosts[addr] = addr
test_logger.info(
"Handshake successfull with agent on stat host "
+ s)
else:
raise Exception(
"Unable to handshake with agent on stats host:"
+ s, t2.testobj.TestInputData.testid)
else:
raise Exception("Stat host " + s +
" not avaliable - No Heartbeat")
# start stats monitors on req hosts
# any host that blocks start monitor blocks the scheduling for the FW
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on stat host " + s)
# Trigger the start of test to test box
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Failed to start Test : ",
t2.testobj.TestInputData.testid)
test_logger.info("Test started on exec host " + ip)
# Trigger the start of test on STAT hosts (This is for initiating system metric data collection)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
p = self.CPORT
s = s.strip()
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("Failed to start test on STAT host : ",
s)
else:
raise Exception("Failed to start test on STAT host : ", s)
test_logger.info("Test started on stat host " + s)
# Get status from tst box
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
if retsend and len(retsend.split(",")) > 1:
if "RUNNING" == retsend.split(
",")[1] or "MONITOR_ON" == retsend.split(",")[1]:
# update from setup to running
lctx.debug("Updating test status to running in DB")
t.updateStatus("setup", "running")
now = time.time()
tstr = str(
time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateStartTime(tstr)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[
t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
self.lock.acquire()
self.running_tests[t.testobj.TestInputData.frameworkid] = t
self.lock.release()
else:
lctx.error(
"Unable to determine status, testmon will garbage collect this testid"
)
# Garbage collect testid in runnning state that fails to give status
# Killing of threads on client host and remove from list with status=fail is done in testmon
except Exception as e:
lctx.error(e)
test_logger.error(e)
lctx.error("ERROR : Unknown trigger error : " +
str(t.testobj.TestInputData.testid))
t.updateStatus("setup", "failed")
lctx.debug(traceback.print_exc())
lctx.error("Removing Test " + str(t.testobj.TestInputData.testid) +
" from dispatch Q")
self.dispatchQ__lock.acquire()
del self.dispatch_queue[t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test aborted on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_CLEANUP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test abort on stat host " + s)
LOG.removeLogger(t2)
return "FAILED"
return "SUCCESS"
def process_results(self, *args):
"""
This procedure is called by testmon as seperate thread when test execution ends or test timeout occur
on agent.
"""
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
# Setting up test logger for capturing test life cycle on scheduler
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test execution completes, preocessing test results")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
# set test status to collating
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
# Initiate instance of ProcessOutputFiles for docker and top outout file processing
ptop = process_files.ProcessOutputFiles(
LOG.getLogger("processTop", "DH"))
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
# send file download command to exec host (no need to send stop test as this procedure is invoked due
# to test end on exec host)
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading LOGS from " + ip + " : " +
retsend)
test_logger.error("Error downloading LOGS from " + ip +
" : " + retsend)
else:
test_logger.info(
"Logs download successfull from exec host " + ip)
# copy results files from exec to daytona file system and untar results
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
test_logger.error("Error in untar EXEC host results")
lctx.error(e)
# process top and docker stat files downloaded from exec host
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
test_logger.info(
"Exec host logs extracted and processed succesfully")
# send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
try:
# Send DAYTONA_STOP_TEST on all agent hosts to stop SAR data collection after test finish on
# exec host. This message is required to tell stat hosts that test execution is finished on
# exec host. Upon receiving this message on stat host, agent will change test state to TESTEND
# and then other SAR data collection thread will stop writing log files for this test.
lctx.info("Stopping test on stat host : " + s)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Failed to stop test on stat host " +
s + " : " + retsend)
test_logger.error(
"Failed to stop test on stat host " + s +
" : " + retsend)
else:
test_logger.info("Test stopped on stat host " + s)
# send file download command to stat host
lctx.info("Sending results.tgz download to :" +
s.strip() + ":" + str(p))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
test_logger.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
else:
test_logger.info(
"Logs downloaded from stat host " + s)
# copy results files from stat host to daytona file system and untar results
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
# process top and docker stat files downloaded from stat host
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(
ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
test_logger.info(
"Stat host " + s +
" logs extracted and processed succesfully")
# send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on stat host " +
s)
except Exception as e:
# Just continue with other stat hosts if any exception occurs while working on any particular
# host (Continue only when something goes wrong with stat host, because we still want to
# download logs from other stat hosts)
lctx.error(e)
test_logger.error(e)
continue
except Exception as e:
# Throw an error if anything goes wrong with finishing test on exec host and set test state to failed
lctx.error("Error in processing results")
lctx.error(e)
test_logger.error("Error in processing results")
test_logger.error(e)
t.updateStatus("collating", "failed")
# updating test state to timeout clean if test terminated due to timeout else setting it to finished clean
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
# update test end time in database
t.updateEndTime(tstr)
f = None
# Formatting email with results.csv details to send it to CC list if user has mentioned in test details
# (admin need to smtp server for this functionality to work, smtp server details need to be
# mentioned in config.sh)
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} <BR> Framework : {} <BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR><BR> Creation time : {} <BR>Start time : {} <BR>End " \
"time : {} <BR>".format(t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. <BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except Exception as e:
lctx.error("Mail send error")
LOG.removeLogger(t)
return "SUCCESS"
def dispatch(self, *args):
"""
This is dispatch queue of scheduler where test from different framework wait in the waiting queue for scheduler to
bind it with trigger thread. This procedure continuously iterate over testmap populated by DBMon with tests
started by user from UI or CLI. This keep track of all running tests and it allows one test per framework. Once
this procedure find an open test spot for a test from particular framework, this procedure will pop it from testmap,
put it in dispatch queue and assign trigger thread for this test to start test setup and then execution.
"""
dispatch_threads = defaultdict()
while True:
# Continuously iterate on testmap for initiating any test execution
for k in self.testmap:
# iterating for all frameworkid k in testmap which contains list of waiting tests for a particular framework
found = False
# If test for a particular framework is already in running or dispatch queue then this new test need to
# wait until previous test gets finish, hence we do nothing and just continue
if k in self.dispatch_queue or k in self.running_tests:
found = True
else:
lctx.debug("Found spot for test")
if found:
continue
# Proceed if spot is available for executing test for this framework
try:
tmp_t = self.testmap[k][0]
except Exception as e:
lctx.debug("No test object found in map")
continue
if tmp_t is None:
continue
alive = False
h = tmp_t.testobj.TestInputData.exechostname
# Initiating test logger for capturing test life cycle on scheduler, all logs are logged in file <testid>.log
test_logger = LOG.init_testlogger(tmp_t, "EXEC")
test_logger.info("Test execution starts")
try:
# Sending heartbeat on exec host to check if it agent is up on exec host
retsend = self.cl.send(
h, self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 2:
status = retsend.split(",")[1]
else:
raise Exception(
"Execution host not avaliable - No Heartbeat ",
tmp_t.testobj.TestInputData.testid)
if "ALIVE" == status:
test_logger.info(
"HeartBeat received from execution host " + h)
# Sending DAYTONA_HANDSHAKE for verifying connectivity between scheduler and agent on exec host
# using custom daytona ports
ret = self.cl.send(
h, self.CPORT,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(tmp_t.testobj.TestInputData.testid) + "," +
h))
if ret == "SUCCESS":
alive = True
test_logger.info(
"Handshake successful with execution host " +
h)
lctx.debug(
"Handshake successful in scheduler, adding ip/hostname to reg hosts"
)
server.serv.registered_hosts[h] = h
addr = socket.gethostbyname(h)
lctx.debug(addr)
server.serv.registered_hosts[addr] = addr
else:
raise Exception(
"Unable to handshake with agent on executuion host "
+ h)
except Exception as e:
lctx.error(e)
test_logger.error(e)
# pause the dbmon here as we dont want the same test to be picked again after we pop
self.dbinstance.mon_thread[0].pause()
self.dbinstance.lock.acquire()
t = self.testmap[k].pop(0)
t.updateStatus("waiting", "failed")
self.dbinstance.lock.release()
lctx.debug("Removed test from map : " +
str(t.testobj.TestInputData.testid))
self.dbinstance.mon_thread[0].resume()
LOG.removeLogger(tmp_t)
continue
if alive == True and found == False:
# for each framework pick one and move it to running, iff running has an empty slot.
lctx.debug(
"-------Found empty slot in dispatch and running Q-------"
)
# pause the dbmon here as we dont want the same test to be picked again after we pop
self.dbinstance.mon_thread[0].pause()
self.dbinstance.lock.acquire()
t = self.testmap[k].pop(0)
self.dbinstance.lock.release()
lctx.info("< %s" % t.testobj.TestInputData.testid)
# put the test in dispatch queue
self.dispatchQ__lock.acquire()
self.dispatch_queue[k] = t
self.dispatchQ__lock.release()
t.updateStatus("waiting", "setup")
self.dbinstance.mon_thread[0].resume()
try:
# Bind a seperate trigger thread for this test to start test execution
trigger_thread = common.FuncThread(
self.trigger, True, t)
dispatch_threads[t.testobj.TestInputData.testid] = (
trigger_thread, t)
trigger_thread.start()
except Exception as e:
lctx.error("Trigger error : " +
str(t.testobj.TestInputData.testid))
test_logger.error("Test setup failed " +
str(t.testobj.TestInputData.testid))
LOG.removeLogger(tmp_t)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[k]
self.dispatchQ__lock.release()
lctx.debug(e)
try:
# Log list of test currently present in dispatch queue in scheduler debug file
d = "DISPATCH [S/R] : "
for k in self.dispatch_queue:
d = d + " |" + str(
self.dispatch_queue[k].testobj.TestInputData.testid)
except:
lctx.debug("ERROR : Dispatch Q empty")
lctx.debug(d)
d = ""
time.sleep(2)
def trigger(self, *args):
"""
trigger starts in a thread, keep track of all triggers and they should complete in a specified time, otherwise
signal a close. triggers are used for test setup, then starting test and then taking test into running
state on agent.
"""
t = args[1]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
time.sleep(6)
# Setting up test logger for capturing test life cycle on scheduler
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test setup started")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
# Sending DAYTONA_SETUP_TEST command on execution host, on receiving this command agent will perform basic
# test setup by creating test directories for saving log files and it will copy execution script in test
# directory for starting execution
ip = t.testobj.TestInputData.exechostname
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on exec host " + ip)
# Triggering test setup on all stat hosts
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
test_logger.info("Starting test setup on stat host " + s)
lctx.debug(s.strip())
p = self.CPORT
# Sending hearbeat message to check whether stat host is up or not
retsend = self.cl.send(
s, p, self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "ALIVE":
raise Exception(
"Remove host not avaliable - No Heartbeat ",
t2.testobj.TestInputData.testid)
else:
test_logger.info("Hearbeat received from stat host " +
s)
# Trigger DAYTONA_HANDSHAKE to verify that both agent and scheduler are able to communicate with
# each other on custom daytona ports
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(t2.testobj.TestInputData.testid) + "," +
s))
lctx.debug(retsend)
if retsend == "SUCCESS":
alive = True
server.serv.registered_hosts[s] = s
addr = socket.gethostbyname(s)
server.serv.registered_hosts[addr] = addr
test_logger.info(
"Handshake successfull with agent on stat host "
+ s)
else:
raise Exception(
"Unable to handshake with agent on stats host:"
+ s, t2.testobj.TestInputData.testid)
else:
raise Exception("Stat host " + s +
" not avaliable - No Heartbeat")
# Trigger test setup on stat hosts, this will create test directory for saving log files
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on stat host " + s)
# Trigger the start of test on exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Failed to start Test : ",
t2.testobj.TestInputData.testid)
test_logger.info("Test started on exec host " + ip)
# Trigger the start of test on STAT hosts (This is for initiating system metric data collection)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
p = self.CPORT
s = s.strip()
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("Failed to start test on STAT host : ",
s)
else:
raise Exception("Failed to start test on STAT host : ", s)
test_logger.info("Test started on stat host " + s)
# Get status from exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
if retsend and len(retsend.split(",")) > 1:
if "RUNNING" == retsend.split(
",")[1] or "MONITOR_ON" == retsend.split(",")[1]:
# update from setup to running
lctx.debug("Updating test status to running in DB")
t.updateStatus("setup", "running")
now = time.time()
tstr = str(
time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateStartTime(tstr)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[
t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
self.lock.acquire()
self.running_tests[t.testobj.TestInputData.frameworkid] = t
self.lock.release()
else:
lctx.error(
"Unable to determine status, testmon will garbage collect this testid"
)
# Garbage collect testid in runnning state that fails to give status
# Killing of threads on client host and remove from list with status=fail is done in testmon
except Exception as e:
# If any trigger fails, then abort test startup with error
lctx.error(e)
test_logger.error(e)
lctx.error("ERROR : Unknown trigger error : " +
str(t.testobj.TestInputData.testid))
t.updateStatus("setup", "failed")
lctx.debug(traceback.print_exc())
lctx.error("Removing Test " + str(t.testobj.TestInputData.testid) +
" from dispatch Q")
self.dispatchQ__lock.acquire()
del self.dispatch_queue[t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
# This will abort test and perform cleanup on exec host with trigger was successful on exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test aborted on exec host " + ip)
# On all other stat hosts we send cleanup in case trigger was successful on any stat host
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_CLEANUP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test abort on stat host " + s)
LOG.removeLogger(t2)
return "FAILED"
return "SUCCESS"
