import time
from collections import defaultdict
import server
import config
import common
import testobj
from logger import LOG
if __name__ == "__main__":
# Port 0 means to select an arbitrary unused port
lctx = LOG.getLogger("agentlog", "Agent")
cfg = config.CFG("Agent", lctx)
cfg.readCFG("config.ini")
HOST = common.get_local_ip()
PORT = cfg.CPORT
common.createdir(cfg.daytona_agent_root, lctx)
server.serv.role = "Agent"
base_serv = server.serv()
#server.serv.lctx = LOG.getLogger("listenerlog","Agent")
#ser = server.serv.ThreadedTCPServer((HOST, PORT), server.serv.ThreadedTCPRequestHandler)
ser = base_serv.ThreadedTCPServer((HOST, PORT),
server.serv.ThreadedTCPRequestHandler)
server.serv.serverInstance = ser
ip, port = ser.server_address
if __name__ == "__main__":
# Port 0 means to select an arbitrary unused port
lctx = LOG.getLogger("schedulerlog", "DH")
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
common.logger.ROLE = "DHOST"
db = dbaccess.DaytonaDBmon(cfg, LOG.getLogger("dblog", "DH"))
sch = Scheduler(db, cfg, LOG.getLogger("schedulerlog", "DH"))
server.serv.role = "DH"
ase_serv = server.serv()
server.serv.lctx = LOG.getLogger("listenerlog", "DH")
ser = server.serv.ThreadedTCPServer((common.get_local_ip(), sch.PORT),
server.serv.ThreadedTCPRequestHandler)
server.serv.serverInstance = ser
ip, port = ser.server_address
server.lctx = LOG.getLogger("listenerlog", "DH")
# Start a thread with the server -- that thread will then start one
# more thread for each request
server_thread = threading.Thread(target=ser.serve_forever)
# Exit the server thread when the main thread terminates
server_thread.daemon = True
server_thread.start()
lctx.info("Server loop running in thread:" + server_thread.name)
lctx.info("Server started @ %s:%s" % (ip, port))
def process_results(self, *args):
# set test status completed
# call stop monitors
# send prepare results command to exec
# set test status collating
# copy results files from exec
# copy files from each mon
# set test status finished
# remove test from running Q
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_STOP_MONITOR",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
ptop = process_top.ProcessTop(LOG.getLogger(
"processTop", "DH"))
# todo : avoid send client its own ip
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
lctx.error(e)
ptop_ret = ptop.process_top_output(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
try:
lctx.info("Stop Monitor on stat host : " + s)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_MONITOR",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
lctx.info("Sending results.tgz download to :" +
s.strip() + ":" + str(p))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
ptop_ret = ptop.process_top_output(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(
ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
except Exception as e:
lctx.error(e)
continue
except Exception as e:
lctx.error("Error in processing results")
lctx.error(e)
t.updateStatus("collating", "failed")
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateEndTime(tstr)
f = None
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} \
<BR> Framework : {} \
<BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR> \
<BR> Creation time : {} \
<BR>Start time : {} \
<BR>End time : {} <BR>".format(
t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. \
<BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except:
lctx.error("Mail send error")
return "SUCCESS"
def process_results(self, *args):
"""
This procedure is called by testmon as seperate thread when test execution ends or test timeout occur
on agent.
"""
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
# Setting up test logger for capturing test life cycle on scheduler
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test execution completes, preocessing test results")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
# set test status to collating
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
# Initiate instance of ProcessOutputFiles for docker and top outout file processing
ptop = process_files.ProcessOutputFiles(
LOG.getLogger("processTop", "DH"))
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
# send file download command to exec host (no need to send stop test as this procedure is invoked due
# to test end on exec host)
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading LOGS from " + ip + " : " +
retsend)
test_logger.error("Error downloading LOGS from " + ip +
" : " + retsend)
else:
test_logger.info(
"Logs download successfull from exec host " + ip)
# copy results files from exec to daytona file system and untar results
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
test_logger.error("Error in untar EXEC host results")
lctx.error(e)
# process top and docker stat files downloaded from exec host
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
test_logger.info(
"Exec host logs extracted and processed succesfully")
# send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
try:
# Send DAYTONA_STOP_TEST on all agent hosts to stop SAR data collection after test finish on
# exec host. This message is required to tell stat hosts that test execution is finished on
# exec host. Upon receiving this message on stat host, agent will change test state to TESTEND
# and then other SAR data collection thread will stop writing log files for this test.
lctx.info("Stopping test on stat host : " + s)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Failed to stop test on stat host " +
s + " : " + retsend)
test_logger.error(
"Failed to stop test on stat host " + s +
" : " + retsend)
else:
test_logger.info("Test stopped on stat host " + s)
# send file download command to stat host
lctx.info("Sending results.tgz download to :" +
s.strip() + ":" + str(p))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
test_logger.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
else:
test_logger.info(
"Logs downloaded from stat host " + s)
# copy results files from stat host to daytona file system and untar results
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
# process top and docker stat files downloaded from stat host
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(
ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
test_logger.info(
"Stat host " + s +
" logs extracted and processed succesfully")
# send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on stat host " +
s)
except Exception as e:
# Just continue with other stat hosts if any exception occurs while working on any particular
# host (Continue only when something goes wrong with stat host, because we still want to
# download logs from other stat hosts)
lctx.error(e)
test_logger.error(e)
continue
except Exception as e:
# Throw an error if anything goes wrong with finishing test on exec host and set test state to failed
lctx.error("Error in processing results")
lctx.error(e)
test_logger.error("Error in processing results")
test_logger.error(e)
t.updateStatus("collating", "failed")
# updating test state to timeout clean if test terminated due to timeout else setting it to finished clean
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
# update test end time in database
t.updateEndTime(tstr)
f = None
# Formatting email with results.csv details to send it to CC list if user has mentioned in test details
# (admin need to smtp server for this functionality to work, smtp server details need to be
# mentioned in config.sh)
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} <BR> Framework : {} <BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR><BR> Creation time : {} <BR>Start time : {} <BR>End " \
"time : {} <BR>".format(t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. <BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except Exception as e:
lctx.error("Mail send error")
LOG.removeLogger(t)
return "SUCCESS"