def endTest(self, *args):
(obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
lctx.debug("Ending test")
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
(t, aid, tst, ts)=(None, None, None, None)
lctx.debug(args)
lctx.debug(obj)
lctx.debug(obj.async_actions)
for tp in obj.async_actions:
(t, aid, tst, ts)=tp
if tst.testobj.TestInputData.testid == t2.testobj.TestInputData.testid:
lctx.debug("Found ASYNC action pending for this this test")
(t, aid, tst, ts) = obj.async_actions[0]
break
current_test.status = "TESTABORT"
if t is not None:
t.stop()
t.join()
lctx.debug("Stopped ASYNC action pending for this this test")
cleanup()
lctx.debug(command + "[" + str(actionID) + "]")
return "SUCCESS"
def setFinish( self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) t2 = testobj.testDefn() t2.deserialize(test_serialized) lctx.debug(str(current_test.testid) + ":" + current_test.status) current_test.status = "TESTFINISHED" return "SUCCESS"
def startTest( self, *args):
(obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
lctx.debug("starttest : " + str(t2.testobj.TestInputData.testid))
if current_test.tobj.testobj.TestInputData.testid == t2.testobj.TestInputData.testid:
lctx.debug("TestID match")
current_test.status = "TESTRUNNING"
current_test.actionID = actionID
else:
lctx.error("cur test not same as test obj passed in starttest : " + str(t2.testobj.TestInputData.testid))
return "ERROR"
#Original execscript
#execscript = current_test.tobj.testobj.TestInputData.execution_script_location
#Copied execscript
execscript = current_test.execscriptfile
args = ""
for a in current_test.tobj.testobj.TestInputData.execScriptArgs:
args = args + " \"" + a[3] + "\""
execline = execscript + " " + args
lctx.debug("Execution line:" + execline)
#execute the exec script here
exec_cmd(execline, command, sync, obj, actionID)
lctx.debug("Completed start test")
return "SUCCESS"
def stopMonitor( self, *args):
(obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
lctx.debug("stop monitor for test : " + str(t2.testobj.TestInputData.testid))
if current_test.testid != t2.testobj.TestInputData.testid:
lctx.debug("start mon : TestID dont match")
return "ERROR"
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
#stop the sar processes
execline = cfg.daytona_mon_path + "/sar_gather_agent.pl --shutdown --root-dir=" + current_test.statsdir
lctx.info(execline)
exec_cmd(execline, command, sync, obj, actionID)
#prepare mon results tarball here
lctx.debug(current_test.statsdir)
#os.remove(current_test.statsdir + "/sar.dat")
os.remove(current_test.statsdir + "/sar_gather_agent_debug.out")
#os.remove(current_test.statsdir + "/*.pid")
lctx.debug("removed monitor temp files from : " + current_test.archivedir)
common.make_tarfile(current_test.archivedir + "results_stats.tgz", current_test.archivedir)
lctx.debug("Completed stop monitor")
return "SUCCESS"
def fileDownload(self, *args):
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
prm = args[2].split(',')
lctx.debug(prm)
test_serialized = prm[3].strip()
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
dest =""
if prm[4] == "RESULTS" :
dest = t2.testobj.TestInputData.exec_results_path
elif prm[4] == "STATS" :
s = prm[5]
lctx.debug(s)
dest = t2.testobj.TestInputData.stats_results_path[s]
else:
dest = ""
lctx.debug(prm[0])
lctx.debug(prm[1])
lctx.debug(prm[2])
lctx.debug(dest)
try:
cl.sendFile(prm[0].strip(), int(prm[1].strip()), prm[2].strip(), dest.strip())
except CommunicationError as e:
lctx.error(e.value)
return e.value
return "SUCCESS"
def execute(self, command, paramcsv, actionID):
#based on SYNCFLAG release from here
#send actionID for currently being executed action based on this we can stream resp
#keep exec details over time in a buffer with actionID mapped
#send actionID NULL and hold return till exec is complete
module = self.conf.actionMap[command.strip()].split(".")[0]
function = self.conf.actionMap[command.strip()].split(".")[1]
sync = self.conf.actionMap[command.strip()].split(".")[2]
t2 = testobj.testDefn()
try:
param = int(paramcsv)
action.action_lock.acquire()
t2 = action.running_tests[param].tobj
action.action_lock.release()
except Exception as e:
pass
m = __import__(module)
f = getattr(m, function)
if sync == "T": #wait for func to complete and return the ret
self.lctx.debug("Executing SYNC ACTION for " + command.strip() +
" : " + self.conf.actionMap[command.strip()] +
":" + str(actionID))
ret = f(self, self, command, paramcsv, actionID, sync)
self.lctx.debug("ACTION completed for " + command.strip() + " : " +
self.conf.actionMap[command.strip()] + ":" +
str(actionID))
if command == "DAYTONA_CLI":
return "actionID=" + str(
actionID) + "%" + ret + "%" + "SYNC EXEC"
else:
return "actionID=" + str(
actionID) + "," + ret + "," + "SYNC EXEC"
else:
#callback will be called after completion
#actionID = uuid.uuid4()
self.lctx.debug("Executing ASYNC ACTION for " + command.strip() +
" : " + self.conf.actionMap[command.strip()] +
":" + str(actionID))
t1 = common.FuncThread(f, True, self, command, paramcsv, actionID,
sync)
x = (t1, actionID, t2, time.time())
self.lock.acquire()
self.async_actions.append(x)
self.lctx.debug("async_actions size :" +
str(len(self.async_actions)))
self.lock.release()
t1.start()
self.lctx.debug("Executing ACTION for " + command.strip() + " : " +
self.conf.actionMap[command.strip()] + ":" +
str(actionID))
return "actionID=" + str(
actionID) + "," + "SUCCESS," + "ASYNC EXEC"
def startMonitor( self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
lctx.debug("startMonitor")
p = params.split(",")
test_serialized = p[0]
s = p[1].strip()
try :
current_test.tobj = testobj.testDefn()
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test.tobj= t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
lctx.debug("TEST SETUP Monitor : " + str(current_test.tobj.testobj.TestInputData.testid))
#todo : indicate a status that says this host is monitor-active
#current_test.status = "TESTSETUP"
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(current_test.tobj.testobj.TestInputData.testid) + "/results/"
current_test.statsdir = prefix + s + "/sar/"
current_test.archivedir = prefix
common.createdir(cfg.daytona_agent_root, lctx)
common.createdir(current_test.statsdir, lctx)
execline = cfg.daytona_mon_path + "/sar_gather_agent.pl --daemonize --root-dir=" + current_test.statsdir
lctx.info(execline)
exec_cmd(execline, command, sync, obj, actionID)
except Exception as e:
lctx.error(e)
lctx.error(traceback.print_exc())
return "ERROR"
lctx.debug("Completed start monitor")
return "SUCCESS"
def prepareResults(self, *args):
(obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
lctx.debug("preparing results for test : " + str(t2.testobj.TestInputData.testid))
if current_test.testid != t2.testobj.TestInputData.testid:
lctx.debug("start mon : TestID dont match")
return "ERROR"
#stop the sar processes
#prepare results tarball here
common.make_tarfile(current_test.archivedir + "results.tgz", current_test.archivedir)
lctx.debug(command + "[" + str(actionID) + "]")
return "SUCCESS"
def execute(self, command, paramcsv, actionID):
#based on SYNCFLAG release from here
#send actionID for currently being executed action based on this we can stream resp
#keep exec details over time in a buffer with actionID mapped
#send actionID NULL and hold return till exec is complete
module = self.conf.actionMap[command.strip()].split(".")[0]
function = self.conf.actionMap[command.strip()].split(".")[1]
sync = self.conf.actionMap[command.strip()].split(".")[2]
self.lctx.debug(command)
self.lctx.debug(paramcsv)
self.lctx.debug(actionID)
t2 = testobj.testDefn()
tst = ""
if paramcsv != "":
p = paramcsv.split(",")
tst = p[0]
if command == "DAYTONA_FILE_DOWNLOAD":
tst = p[3]
if tst != "":
t2.deserialize(tst)
m = __import__ (module)
f = getattr(m,function)
if sync == "T" : #wait for func to complete and return the ret
self.lctx.debug("Executing SYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID))
ret = f(self, self, command, paramcsv, actionID, sync)
self.lctx.debug("ACTION completed for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID))
return "actionID=" + str(actionID) + "," + ret + "," + "SYNC EXEC"
else :
#callback will be called after completion
#actionID = uuid.uuid4()
self.lctx.debug("Executing ASYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID))
t1 = common.FuncThread(f, True, self, command, paramcsv, actionID, sync)
x = (t1, actionID, t2, time.time())
self.lock.acquire()
self.async_actions.append(x)
self.lctx.debug( "async_actions size :" + str(len(self.async_actions)))
self.lock.release()
t1.start()
self.lctx.debug( "Executing ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID))
return "actionID=" + str(actionID) + "," + "SUCCESS," + "ASYNC EXEC"
def execute(self, command, paramcsv, actionID):
# based on SYNCFLAG release from here
# send actionID for currently being executed action based on this we can stream resp
# keep exec details over time in a buffer with actionID mapped
# send actionID NULL and hold return till exec is complete
module = self.conf.actionMap[command.strip()].split(".")[0]
function = self.conf.actionMap[command.strip()].split(".")[1]
sync = self.conf.actionMap[command.strip()].split(".")[2]
t2 = testobj.testDefn()
if command == "DAYTONA_START_TEST":
testid = int(paramcsv.split(",")[0])
hosttype = paramcsv.split(",")[1]
current_test = action.get_test(testid)
if current_test:
t2 = current_test.tobj
m = __import__ (module)
f = getattr(m,function)
if sync == "T": # wait for func to complete and return the ret
self.lctx.debug("Executing SYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID))
ret = f(self, self, command, paramcsv, actionID, sync)
self.lctx.debug("ACTION completed for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID))
if command == "DAYTONA_CLI":
return "actionID=" + str(actionID) + "%" + ret + "%" + "SYNC EXEC"
else:
return "actionID=" + str(actionID) + "," + ret + "," + "SYNC EXEC"
else:
self.lctx.debug("Executing ASYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID))
t1 = common.FuncThread(f, True, self, command, paramcsv, actionID, sync)
if hosttype == "EXEC":
x = (t1, actionID, t2, time.time())
self.lock.acquire()
self.async_actions.append(x)
self.lctx.debug( "async_actions size :" + str(len(self.async_actions)))
self.lock.release()
t1.start()
self.lctx.debug( "Executing ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID))
return "actionID=" + str(actionID) + "," + "SUCCESS," + "ASYNC EXEC"
def setupTest(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
test_serialized = params.split(",")[0]
host_type = params.split(",")[1]
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test = get_test(t2.testobj.TestInputData.testid)
if current_test:
if current_test.status is not "INIT":
lctx.error("Invalid state for TEST SETUP action : " + current_test.status)
current_test.status = "ABORT"
save_test(current_test.testid, current_test)
return "ERROR"
try:
if current_test:
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START")
current_test.tobj = testobj.testDefn()
current_test.tobj = t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
lctx.debug("TEST SETUP : " + str(current_test.tobj.testobj.TestInputData.testid))
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid)
shutil.rmtree(dir, ignore_errors=True)
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid) + "/results/"
if host_type == "EXEC":
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.statsdir = prefix + current_test.stathostip + "/sar/"
current_test.resultsdir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/results"
current_test.archivedir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/"
common.createdir(cfg.daytona_agent_root, self.lctx)
if host_type == "EXEC":
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
# todo : check and validate if exec script is provided in expected format and
# the file exists in that location
if host_type == "EXEC":
execscript = current_test.tobj.testobj.TestInputData.execution_script_location
lctx.debug("TEST SETUP : " + str(execscript))
current_test.execscriptfile = current_test.execdir + "/" + execscript
lctx.debug(current_test.execscriptfile)
# check if execution script is present in '/tmp/ExecScripts/' - execute script only if it present at
# this location
execscript_location = EXEC_SCRIPT_DIR + execscript
execscript_location = os.path.realpath(execscript_location)
valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR
if valid_path:
if os.path.isfile(execscript_location):
ret = copyfile(execscript_location, current_test.execscriptfile)
else:
raise Exception(
"Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR)
else:
raise Exception(
"Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing "
"exec scripts")
os.chmod(current_test.execscriptfile, 0744)
current_test.status = "SETUP"
save_test(current_test.testid, current_test)
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except shutil.Error as err:
if current_test:
lctx.error("error copying file : " + str(execscript) + " to " + str(current_test.execscriptfile))
current_test.status = "ABORT"
save_test(current_test.testid, current_test)
lctx.error(err)
return "ERROR"
except Exception as e:
if current_test:
current_test.status = "ABORT"
save_test(current_test.testid, current_test)
lctx.error(e)
return "ERROR"
def testmon(self, *mon):
process_results_threads = defaultdict()
while True:
d = "TSMON [R] : |"
remove = False
for k in self.running_tests:
if (self.running_tests[k] != None):
t = self.running_tests[k]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
if (t.testobj.TestInputData.testid !=
t2.testobj.TestInputData.testid):
lctx.error("testobj not same")
t.updateStatus("running", "failed")
remove = True
break #out of for loop
try:
ret = self.cl.send(
t.testobj.TestInputData.exechostname, self.CPORT,
self.ev.construct("DAYTONA_GET_STATUS",
serialize_str))
status = ret.split(",")[1]
lctx.debug(status)
except Exception as e:
lctx.debug(e)
t.updateStatus("running", "failed")
remove = True
break #out of for loop
if "TESTRUNNING" == status or "TESTSETUP" == status:
d = d + str(self.running_tests[k].testobj.
TestInputData.testid) + "|"
elif "TESTEND" == status.strip():
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
if t.testobj.TestInputData.end_status == "running":
lctx.debug(t.testobj.TestInputData.end_status)
if t.testobj.TestInputData.end_status == "running":
lctx.debug(
"Updating status to completed in DB")
t.updateStatus("running", "completed")
pt = common.FuncThread(
self.process_results, True, t,
t.testobj.TestInputData.end_status)
process_results_threads[
t.testobj.TestInputData.testid] = (pt, t)
pt.start()
elif t.testobj.TestInputData.end_status == "collating" or t.testobj.TestInputData.end_status == "completed" or t.testobj.TestInputData.end_status == "finished clean":
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
else:
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) +
":" + str(status))
break #out of for loop
elif "TESTFINISHED" == status.strip():
d = "TSMON [F] : |*" + str(
self.running_tests[k].testobj.TestInputData.testid
) + "*|"
remove = True
break
else:
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) + ":" +
str(status))
break #out of for loop
lctx.info(d)
d = ""
if (remove == True):
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
time.sleep(2)
def process_results(self, *args):
#set test status completed
#call stop monitors
#send prepare results command to exec
#set test status collating
#copy results files from exec
#copy files from each mon
#set test status finished
#remove test from running Q
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
try:
if (t.testobj.TestInputData.testid !=
t2.testobj.TestInputData.testid):
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status == "completed":
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_PREPARE_RESULTS",
serialize_str))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_PREPARE_RESULTS failed : ",
t2.testobj.TestInputData.testid)
#get statistics hosts
for s in t.testobj.TestInputData.stathostname.split(','):
#stop stats monitors on req hosts
#any host that blocks stop monitor blocks the scheduling for the FW
p = 0
if s.strip() == self.HOST:
p = self.PORT
else:
p = self.CPORT
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_STOP_MONITOR",
serialize_str))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_STOP_MONITOR failed : ",
t2.testobj.TestInputData.testid)
t.updateStatus("completed", "collating")
#todo : avoid send client its own ip
lctx.debug("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
results_file = cfg.daytona_agent_root + "/" + t.testobj.TestInputData.frameworkname + "/" + str(
t.testobj.TestInputData.testid
) + "/results/" + "results.tgz"
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(self.HOST) + "," + str(self.PORT) + "," +
results_file + "," + serialize_str + "," + "RESULTS" +
"," + ip))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_FILE_DOWNLOAD failed :",
t2.testobj.TestInputData.testid)
results_file = cfg.daytona_agent_root + "/" + t.testobj.TestInputData.frameworkname + "/" + str(
t.testobj.TestInputData.testid
) + "/results/" + "results_stats.tgz"
for s in t.testobj.TestInputData.stathostname.split(','):
lctx.info("Downloading stats from STATS self.HOSTS : " + s)
lctx.info(s)
#stop stats monitors on req hosts
#any host that blocks stop monitor blocks the scheduling for the FW
p = 0
if s.strip() == self.HOST:
p = self.PORT
else:
p = self.CPORT
lctx.info("Sending DOWNLOAD file to :" + s.strip() + ":" +
str(p) + "File :" + results_file +
"(upload to this host port:)" + str(self.HOST) +
"," + str(self.PORT))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(self.HOST) + "," + str(self.PORT) + "," +
results_file + "," + serialize_str + "," +
"STATS" + "," + s.strip()))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading " + results_file +
" From " + s.strip() + ":" + retsend)
raise Exception(
"Daytona command DAYTONA_FILE_DOWNLOAD failed :",
t2.testobj.TestInputData.testid)
try:
lctx.debug(t2.testobj.TestInputData.exec_results_path +
"results.tgz")
lctx.debug(t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
for s in t2.testobj.TestInputData.stats_results_path:
lctx.debug(
t2.testobj.TestInputData.stats_results_path[s] +
"results_stats.tgz")
lctx.debug(
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results_stats.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
except Exception as e:
lctx.error("Error in untar results")
lctx.error(e)
raise Exception("test result processing error",
t2.testobj.TestInputData.testid)
#todo : invoke other scripts to transform results and update DB
except Exception as e:
lctx.error("Error in processing results")
lctx.error(e)
t.updateStatus("collating", "failed")
try:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_CLEANUP_TEST", serialize_str))
lctx.debug("DAYTONA_CLEANUP_TEST:" + str(retsend))
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST", serialize_str))
lctx.debug(retsend)
for s in t.testobj.TestInputData.stathostname.split(','):
p = 0
if s.strip() == self.HOST:
p = self.PORT
else:
p = self.CPORT
lctx.debug("self.HOST : " + s.strip())
lctx.debug("PORT to send CLEANUP & FINISH : " + str(p))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_CLEANUP_TEST", serialize_str))
lctx.debug(retsend)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_FINISH_TEST", serialize_str))
lctx.debug(retsend)
except Exception as e:
lctx.error("Error in processing results")
t.updateStatus("collating", "failed")
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateEndTime(tstr)
f = open(t2.testobj.TestInputData.exec_results_path + "/results.csv")
to = t.testobj.TestInputData.email
reader = csv.reader(f)
htmlfile = ""
rownum = 0
htmlfile = '<table cellpadding="10">'
for row in reader:
if rownum == 0:
htmlfile = htmlfile + '<tr>'
for column in row:
htmlfile = htmlfile + '<th width="70%">' + column + '</th>'
htmlfile = htmlfile + '</tr>'
else:
htmlfile = htmlfile + '<tr>'
for column in row:
htmlfile = htmlfile + '<td width="70%">' + column + '</td>'
htmlfile = htmlfile + '</tr>'
rownum += 1
htmlfile = htmlfile + '</table>'
f.close()
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} \
<BR> Framework : {} \
<BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR> \
<BR> Creation time : {} \
<BR>Start time : {} \
<BR>End time : {} <BR>".format(
t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. \
<BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except:
lctx.error("Mail send error")
return "SUCCESS"
def mon(self, *args):
#query all waiting state tests and load into to_schedule, this is for restart case (similarly for running)
#todo : reconcile running and scheduled testids, if db altered externally
restarted = True
print "CFG in mon :"
print self.cfg.mysql_host
while True:
self.db = DBAccess(self.cfg, self.lctx)
d = "DBMON [W] : |"
for k in self.tests_to_run:
l = self.tests_to_run[k]
for t in l :
d = d + str(t.testobj.TestInputData.testid) + "|"
self.lctx.info(d)
d = ""
query_result = None
if restarted == True :
query_result = self.db.query("""select testid from CommonFrameworkSchedulerQueue where state = %s or state = %s or state = %s or state = %s or state = %s or state = %s""", ("scheduled","waiting","setup","running","completed","collating"), True, False);
#query_result = self.db.query("""select testid from TestInputData where end_status = %s or end_status = %s or end_status = %s or end_status = %s or end_status = %s or end_status = %s""", ("scheduled","waiting","setup","running","completed","collating"), True, False);
restarted = False
else:
status = "scheduled"
query_result = self.db.query("""select testid from CommonFrameworkSchedulerQueue where state = %s""", (status,), True, False);
#query_result = self.db.query("""select testid from TestInputData where end_status = %s""", (status,), True, False);
#reset all states to scheduled, mostly required in a restart case
#all items that make to the DBMON [Q] will be in scheduled state
for testid in query_result:
to = testobj.testDefn()
to.testobj.TestInputData.testid = testid[0]
res = to.construct(testid[0])
res = to.updateStatus("*", "scheduled")
d = "DBMON [Q] : "
d = d + str(query_result)
self.lctx.info(d)
for testid in query_result:
found = False
for k in self.tests_to_run: #search across all FW
l = self.tests_to_run[k] #list of tests ready to be run
for t in l:
self.lctx.debug(t.testobj.TestInputData.testid)
self.lctx.debug(testid[0])
self.lctx.debug(k)
if t.testobj.TestInputData.testid == testid[0]:
self.lctx.debug("Test already present in runQ")
found = True
if(found == False):
to = testobj.testDefn()
to.testobj.TestInputData.testid = testid[0]
#todo handle return status
res = to.construct(testid[0])
#create required dirs and setup server side env
prefix=self.cfg.daytona_dh_root +"/"+ to.testobj.TestInputData.frameworkname + "/" + str(to.testobj.TestInputData.testid) + "/" + "results" + "/"
to.testobj.TestInputData.exec_results_path = prefix+to.testobj.TestInputData.exechostname + "/"
to.testobj.TestInputData.exec_path = prefix+to.testobj.TestInputData.exechostname
to.testobj.TestInputData.exec_log_path = prefix+to.testobj.TestInputData.exechostname+"/application"
to.testobj.TestInputData.stats_results_path = defaultdict()
for s in to.testobj.TestInputData.stathostname.split(','):
to.testobj.TestInputData.stats_results_path[s.strip()] = prefix+s.strip()+"/"
to.testobj.TestInputData.stats_results_path[to.testobj.TestInputData.exechostname] = prefix+to.testobj.TestInputData.exechostname+"/"
common.createdir(to.testobj.TestInputData.exec_results_path, self.lctx)
common.createdir(to.testobj.TestInputData.exec_path, self.lctx)
common.createdir(to.testobj.TestInputData.exec_log_path, self.lctx)
for s in to.testobj.TestInputData.stats_results_path:
common.createdir(to.testobj.TestInputData.stats_results_path[s], self.lctx)
res = to.updateStatus("scheduled", "waiting")
sz = to.serialize()
t2 = testobj.testDefn()
t2.deserialize(sz)
if t2.testobj.TestInputData.testid != to.testobj.TestInputData.testid :
self.lctx.error("error in ser / dser")
break
#use a lock here
self.lock.acquire()
self.tests_to_run[to.testobj.TestInputData.frameworkid].append(to);
self.lctx.debug("Adding : " + str(to.testobj.TestInputData.testid))
self.lock.release()
self.db.close()
if self.mon_thread[0].check() == False :
return
time.sleep(5) #todo : make this config item
def setupTest( self, *args):
(obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
lctx.debug("setuptest")
try :
current_test.tobj = testobj.testDefn()
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test.tobj= t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
lctx.debug("TEST SETUP : " + str(current_test.tobj.testobj.TestInputData.testid))
current_test.status = "TESTSETUP"
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(current_test.tobj.testobj.TestInputData.testid) + "/results/"
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.resultsdir = prefix
current_test.statsdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/sar/"
current_test.archivedir = prefix
common.createdir(cfg.daytona_agent_root, self.lctx)
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
#todo : check and validate if exec script is provided in expected format and
# the file exists in that location
execscript = current_test.tobj.testobj.TestInputData.execution_script_location.split(":")[1]
lctx.debug("TEST SETUP : " + str(execscript))
tmp = str(execscript).split("/")
tmp.reverse()
lctx.debug(tmp)
filename = tmp[0]
current_test.execscriptfile = current_test.execdir+"/"+filename
lctx.debug(current_test.execscriptfile)
except Exception as e:
lctx.error(e)
lctx.error(traceback.print_exc())
return "ERROR"
try:
ret = copyfile(execscript, current_test.execscriptfile)
except shutil.Error as err:
lctx.error("error copying file : " + str(execscript) + " to " + str(current_test.execscriptfile))
return "ERROR"
try:
os.chmod(current_test.execscriptfile, 0744)
except:
lctx.error("error setting perm file : " + str(current_test.execdir))
return "ERROR"
#create dirs
#get exec script name
#cp the exec script
#set exec perm
#update cur test obj with exec script
#exec any custom setup script
lctx.debug("Completed setuptest")
return "SUCCESS"
def testmon(self, *mon):
process_results_threads = defaultdict()
while True:
d = "TSMON [R] : |"
remove = False
error = False
for k in self.running_tests:
if (self.running_tests[k] != None):
t = self.running_tests[k]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
t.updateStatus("running", "failed")
remove = True
break # out of for loop
try:
ret = self.cl.send(
t.testobj.TestInputData.exechostname, self.CPORT,
self.ev.construct(
"DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
status = ret.split(",")[1]
lctx.debug(status)
except Exception as e:
lctx.debug(e)
t.updateStatus("running", "failed")
error = True
break # out of for loop
if status in [
"RUNNING", "INIT", "SETUP", "MONITOR_ON",
"MONITOR_OFF"
]:
found = checkTestRunning(
t.testobj.TestInputData.testid)
if not found:
error = True
break
d = d + str(self.running_tests[k].testobj.
TestInputData.testid) + "|"
elif status in ["TESTEND", "TIMEOUT"]:
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
if t.testobj.TestInputData.end_status == "running":
lctx.debug(t.testobj.TestInputData.end_status)
if t.testobj.TestInputData.end_status == "running":
if status == "TIMEOUT":
t.testobj.TestInputData.timeout_flag = True
t.updateStatus("running", "timeout")
else:
t.updateStatus("running", "completed")
pt = common.FuncThread(
self.process_results, True, t,
t.testobj.TestInputData.end_status)
process_results_threads[
t.testobj.TestInputData.testid] = (pt, t)
pt.start()
elif t.testobj.TestInputData.end_status == "collating" or t.testobj.TestInputData.end_status == "completed" or t.testobj.TestInputData.end_status == "finished clean":
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
else:
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) +
":" + str(status))
break # out of for loop
elif status.strip() == "FINISHED":
d = "TSMON [F] : |*" + str(
self.running_tests[k].testobj.TestInputData.testid
) + "*|"
remove = True
break
elif status.strip() in ["FAILED", "ABORT", "TESTNA"]:
if status.strip() == "FAILED":
error = True
elif status.strip() in ["ABORT", "TESTNA"]:
remove = True
t.updateStatus("", "failed")
lctx.error("TEST " + status.strip() +
" : Cleaning test from running queue")
break # out of for loop
else:
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) + ":" +
str(status))
break # out of for loop
lctx.info(d)
d = ""
if error:
retsend = None
ip = t.testobj.TestInputData.exechostname
try:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
except:
pass
if retsend and retsend.split(",")[1] == "ALIVE":
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_STOP_MONITOR",
str(t.testobj.TestInputData.testid)))
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t.testobj.TestInputData.testid)))
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
try:
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
except:
pass
if retsend and retsend.split(",")[1] == "ALIVE":
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct(
"DAYTONA_STOP_MONITOR",
str(t.testobj.TestInputData.testid)))
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct(
"DAYTONA_ABORT_TEST",
str(t.testobj.TestInputData.testid)))
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
if remove:
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
time.sleep(2)
def trigger(self, *args):
# trigger starts in a thread, keep track of all triggers and they should complete in a specified time,
# otherwise signal a close
t = args[1]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
time.sleep(6)
track_monitor = []
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
ip = t.testobj.TestInputData.exechostname
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",EXEC"))
lctx.debug(retsend)
st = retsend.split(",")
if len(st) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_START_MONITOR",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
st = retsend.split(",")
if len(st) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test monitor start failure : Test - ",
t2.testobj.TestInputData.testid)
track_monitor.append(ip)
# get statistics hosts
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
lctx.debug("Start monitor")
lctx.debug(s.strip())
p = self.CPORT
retsend = self.cl.send(
s, p, self.ev.construct("DAYTONA_HEARTBEAT", ""))
st = retsend.split(",")
if len(st) > 1:
if retsend.split(",")[1] != "ALIVE":
raise Exception(
"Remove host not avaliable - No Heartbeat ",
t2.testobj.TestInputData.testid)
else:
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_HANDSHAKE",
self.HOST + "," + str(self.PORT) + "," +
str(t2.testobj.TestInputData.testid) + "," +
s))
lctx.debug(retsend)
if retsend == "SUCCESS":
alive = True
server.serv.registered_hosts[s] = s
addr = socket.gethostbyname(s)
server.serv.registered_hosts[addr] = addr
else:
raise Exception(
"Unable to handshake with agent on stats host:"
+ s, t2.testobj.TestInputData.testid)
else:
raise Exception(
"Remove host not avaliable - No Heartbeat ",
t2.testobj.TestInputData.testid)
# start stats monitors on req hosts
# any host that blocks start monitor blocks the scheduling for the FW
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",STAT"))
lctx.debug(retsend)
st = retsend.split(",")
if len(st) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_START_MONITOR",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
st = retsend.split(",")
if len(st) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test monitor start failure : Test - ",
t2.testobj.TestInputData.testid)
track_monitor.append(s.strip())
# Trigger the start of test to test box
retsend = self.cl.send(
t.testobj.TestInputData.exechostname, self.CPORT,
self.ev.construct("DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
st = retsend.split(",")
if len(st) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Failed to start Test : ",
t2.testobj.TestInputData.testid)
# Get status from tst box
retsend = self.cl.send(
t.testobj.TestInputData.exechostname, self.CPORT,
self.ev.construct("DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
st = retsend.split(",")
if len(st) > 1:
if "RUNNING" == st[1] or "MONITOR_ON" == st[1]:
# update from setup to running
lctx.debug("Updating test status to running in DB")
t.updateStatus("setup", "running")
now = time.time()
tstr = str(
time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateStartTime(tstr)
else:
lctx.error(
"Unable to determine status, testmon will garbage collect this testid"
)
# Garbage collect testid in runnning state that fails to give status
# Killing of threads on client host and remove from list with status=fail is done in testmon
except Exception as e:
lctx.error(e)
lctx.error("ERROR : Unknown trigger error : " +
str(t.testobj.TestInputData.testid))
t.updateStatus("setup", "failed")
lctx.debug(traceback.print_exc())
if ip in track_monitor:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_STOP_MONITOR",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
if s.strip() in track_monitor:
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct(
"DAYTONA_STOP_MONITOR",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct(
"DAYTONA_ABORT_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
return "FAILED"
return "SUCCESS"
def getStatus( self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) t2 = testobj.testDefn() t2.deserialize(test_serialized) lctx.debug(str(current_test.testid) + ":" + current_test.status) return current_test.status
def trigger(self, *args):
# trigger starts in a thread, keep track of all triggers and they should complete in a specified time,
# otherwise signal a close
t = args[1]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
time.sleep(6)
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test setup started")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
ip = t.testobj.TestInputData.exechostname
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on exec host " + ip)
# get statistics hosts
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
test_logger.info("Starting test setup on stat host " + s)
lctx.debug(s.strip())
p = self.CPORT
retsend = self.cl.send(
s, p, self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "ALIVE":
raise Exception(
"Remove host not avaliable - No Heartbeat ",
t2.testobj.TestInputData.testid)
else:
test_logger.info("Hearbeat received from stat host " +
s)
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(t2.testobj.TestInputData.testid) + "," +
s))
lctx.debug(retsend)
if retsend == "SUCCESS":
alive = True
server.serv.registered_hosts[s] = s
addr = socket.gethostbyname(s)
server.serv.registered_hosts[addr] = addr
test_logger.info(
"Handshake successfull with agent on stat host "
+ s)
else:
raise Exception(
"Unable to handshake with agent on stats host:"
+ s, t2.testobj.TestInputData.testid)
else:
raise Exception("Stat host " + s +
" not avaliable - No Heartbeat")
# start stats monitors on req hosts
# any host that blocks start monitor blocks the scheduling for the FW
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on stat host " + s)
# Trigger the start of test to test box
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Failed to start Test : ",
t2.testobj.TestInputData.testid)
test_logger.info("Test started on exec host " + ip)
# Trigger the start of test on STAT hosts (This is for initiating system metric data collection)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
p = self.CPORT
s = s.strip()
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("Failed to start test on STAT host : ",
s)
else:
raise Exception("Failed to start test on STAT host : ", s)
test_logger.info("Test started on stat host " + s)
# Get status from tst box
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
if retsend and len(retsend.split(",")) > 1:
if "RUNNING" == retsend.split(
",")[1] or "MONITOR_ON" == retsend.split(",")[1]:
# update from setup to running
lctx.debug("Updating test status to running in DB")
t.updateStatus("setup", "running")
now = time.time()
tstr = str(
time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateStartTime(tstr)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[
t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
self.lock.acquire()
self.running_tests[t.testobj.TestInputData.frameworkid] = t
self.lock.release()
else:
lctx.error(
"Unable to determine status, testmon will garbage collect this testid"
)
# Garbage collect testid in runnning state that fails to give status
# Killing of threads on client host and remove from list with status=fail is done in testmon
except Exception as e:
lctx.error(e)
test_logger.error(e)
lctx.error("ERROR : Unknown trigger error : " +
str(t.testobj.TestInputData.testid))
t.updateStatus("setup", "failed")
lctx.debug(traceback.print_exc())
lctx.error("Removing Test " + str(t.testobj.TestInputData.testid) +
" from dispatch Q")
self.dispatchQ__lock.acquire()
del self.dispatch_queue[t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test aborted on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_CLEANUP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test abort on stat host " + s)
LOG.removeLogger(t2)
return "FAILED"
return "SUCCESS"
def setupTest(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
test_serialized = params.split(",")[0]
host_type = params.split(",")[1]
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test = get_test(t2.testobj.TestInputData.testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START")
test_logger.info("Test setup started")
current_test.tobj = testobj.testDefn()
current_test.tobj = t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid)
shutil.rmtree(dir, ignore_errors=True)
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid) + "/results/"
if host_type == "EXEC":
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.statsdir = prefix + current_test.stathostip + "/sar/"
current_test.resultsdir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/results"
current_test.archivedir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/"
if host_type == "EXEC":
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
test_logger.info("Test directory created")
# todo : check and validate if exec script is provided in expected format and
# the file exists in that location
if host_type == "EXEC":
execscript = current_test.tobj.testobj.TestInputData.execution_script_location
lctx.debug("TEST SETUP : " + str(execscript))
current_test.execscriptfile = current_test.execdir + "/" + execscript
lctx.debug(current_test.execscriptfile)
# check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at
# this location
execscript_location = EXEC_SCRIPT_DIR + execscript
execscript_location = os.path.realpath(execscript_location)
valid_path = os.path.commonprefix(
[execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR
if valid_path:
if os.path.isfile(execscript_location):
ret = shutil.copytree(
os.path.dirname(execscript_location),
os.path.dirname(current_test.execscriptfile))
else:
raise Exception(
"Execution script not found at Daytona Execution Script Location : "
+ EXEC_SCRIPT_DIR)
else:
raise Exception(
"Access Denied : Use Daytona Execution Script Location '"
+ EXEC_SCRIPT_DIR + "' for executing "
"exec scripts")
os.chmod(current_test.execscriptfile, 0744)
test_logger.info("Execution script copied successfully")
save_test(current_test.testid, current_test)
test_logger.info("Test setup complete")
lctx.debug("TEST SETUP | " + str(current_test.testid) +
" | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def trigger(self, *args):
"""
trigger starts in a thread, keep track of all triggers and they should complete in a specified time, otherwise
signal a close. triggers are used for test setup, then starting test and then taking test into running
state on agent.
"""
t = args[1]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
time.sleep(6)
# Setting up test logger for capturing test life cycle on scheduler
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test setup started")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
# Sending DAYTONA_SETUP_TEST command on execution host, on receiving this command agent will perform basic
# test setup by creating test directories for saving log files and it will copy execution script in test
# directory for starting execution
ip = t.testobj.TestInputData.exechostname
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on exec host " + ip)
# Triggering test setup on all stat hosts
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
test_logger.info("Starting test setup on stat host " + s)
lctx.debug(s.strip())
p = self.CPORT
# Sending hearbeat message to check whether stat host is up or not
retsend = self.cl.send(
s, p, self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "ALIVE":
raise Exception(
"Remove host not avaliable - No Heartbeat ",
t2.testobj.TestInputData.testid)
else:
test_logger.info("Hearbeat received from stat host " +
s)
# Trigger DAYTONA_HANDSHAKE to verify that both agent and scheduler are able to communicate with
# each other on custom daytona ports
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(t2.testobj.TestInputData.testid) + "," +
s))
lctx.debug(retsend)
if retsend == "SUCCESS":
alive = True
server.serv.registered_hosts[s] = s
addr = socket.gethostbyname(s)
server.serv.registered_hosts[addr] = addr
test_logger.info(
"Handshake successfull with agent on stat host "
+ s)
else:
raise Exception(
"Unable to handshake with agent on stats host:"
+ s, t2.testobj.TestInputData.testid)
else:
raise Exception("Stat host " + s +
" not avaliable - No Heartbeat")
# Trigger test setup on stat hosts, this will create test directory for saving log files
retsend = self.cl.send(
s.strip(), p,
self.ev.construct("DAYTONA_SETUP_TEST",
serialize_str + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Test Setup Failure : Test - ",
t2.testobj.TestInputData.testid)
test_logger.info("Test setup complete on stat host " + s)
# Trigger the start of test on exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",EXEC"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("test trigger error",
t2.testobj.TestInputData.testid)
else:
raise Exception("Failed to start Test : ",
t2.testobj.TestInputData.testid)
test_logger.info("Test started on exec host " + ip)
# Trigger the start of test on STAT hosts (This is for initiating system metric data collection)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
p = self.CPORT
s = s.strip()
retsend = self.cl.send(
s, p,
self.ev.construct(
"DAYTONA_START_TEST",
str(t2.testobj.TestInputData.testid) + ",STAT"))
lctx.debug(retsend)
if retsend and len(retsend.split(",")) > 1:
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception("Failed to start test on STAT host : ",
s)
else:
raise Exception("Failed to start test on STAT host : ", s)
test_logger.info("Test started on stat host " + s)
# Get status from exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
if retsend and len(retsend.split(",")) > 1:
if "RUNNING" == retsend.split(
",")[1] or "MONITOR_ON" == retsend.split(",")[1]:
# update from setup to running
lctx.debug("Updating test status to running in DB")
t.updateStatus("setup", "running")
now = time.time()
tstr = str(
time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateStartTime(tstr)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[
t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
self.lock.acquire()
self.running_tests[t.testobj.TestInputData.frameworkid] = t
self.lock.release()
else:
lctx.error(
"Unable to determine status, testmon will garbage collect this testid"
)
# Garbage collect testid in runnning state that fails to give status
# Killing of threads on client host and remove from list with status=fail is done in testmon
except Exception as e:
# If any trigger fails, then abort test startup with error
lctx.error(e)
test_logger.error(e)
lctx.error("ERROR : Unknown trigger error : " +
str(t.testobj.TestInputData.testid))
t.updateStatus("setup", "failed")
lctx.debug(traceback.print_exc())
lctx.error("Removing Test " + str(t.testobj.TestInputData.testid) +
" from dispatch Q")
self.dispatchQ__lock.acquire()
del self.dispatch_queue[t.testobj.TestInputData.frameworkid]
self.dispatchQ__lock.release()
# This will abort test and perform cleanup on exec host with trigger was successful on exec host
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test aborted on exec host " + ip)
# On all other stat hosts we send cleanup in case trigger was successful on any stat host
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_CLEANUP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test abort on stat host " + s)
LOG.removeLogger(t2)
return "FAILED"
return "SUCCESS"
#!/usr/bin/env python # -*- coding:cp949 -*- import testobj tobj = testobj.testDefn() tobj.construct(1112)
def process_results(self, *args):
"""
This procedure is called by testmon as seperate thread when test execution ends or test timeout occur
on agent.
"""
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
# Setting up test logger for capturing test life cycle on scheduler
test_logger = LOG.gettestlogger(t2, "EXEC")
test_logger.info("Test execution completes, preocessing test results")
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
# set test status to collating
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
# Initiate instance of ProcessOutputFiles for docker and top outout file processing
ptop = process_files.ProcessOutputFiles(
LOG.getLogger("processTop", "DH"))
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
# send file download command to exec host (no need to send stop test as this procedure is invoked due
# to test end on exec host)
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading LOGS from " + ip + " : " +
retsend)
test_logger.error("Error downloading LOGS from " + ip +
" : " + retsend)
else:
test_logger.info(
"Logs download successfull from exec host " + ip)
# copy results files from exec to daytona file system and untar results
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
test_logger.error("Error in untar EXEC host results")
lctx.error(e)
# process top and docker stat files downloaded from exec host
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
test_logger.info(
"Exec host logs extracted and processed succesfully")
# send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
try:
# Send DAYTONA_STOP_TEST on all agent hosts to stop SAR data collection after test finish on
# exec host. This message is required to tell stat hosts that test execution is finished on
# exec host. Upon receiving this message on stat host, agent will change test state to TESTEND
# and then other SAR data collection thread will stop writing log files for this test.
lctx.info("Stopping test on stat host : " + s)
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Failed to stop test on stat host " +
s + " : " + retsend)
test_logger.error(
"Failed to stop test on stat host " + s +
" : " + retsend)
else:
test_logger.info("Test stopped on stat host " + s)
# send file download command to stat host
lctx.info("Sending results.tgz download to :" +
s.strip() + ":" + str(p))
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
test_logger.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
else:
test_logger.info(
"Logs downloaded from stat host " + s)
# copy results files from stat host to daytona file system and untar results
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
# process top and docker stat files downloaded from stat host
ptop_ret = ptop.process_output_files(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(
ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
test_logger.info(
"Stat host " + s +
" logs extracted and processed succesfully")
# send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
test_logger.info("Test END successfull on stat host " +
s)
except Exception as e:
# Just continue with other stat hosts if any exception occurs while working on any particular
# host (Continue only when something goes wrong with stat host, because we still want to
# download logs from other stat hosts)
lctx.error(e)
test_logger.error(e)
continue
except Exception as e:
# Throw an error if anything goes wrong with finishing test on exec host and set test state to failed
lctx.error("Error in processing results")
lctx.error(e)
test_logger.error("Error in processing results")
test_logger.error(e)
t.updateStatus("collating", "failed")
# updating test state to timeout clean if test terminated due to timeout else setting it to finished clean
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
# update test end time in database
t.updateEndTime(tstr)
f = None
# Formatting email with results.csv details to send it to CC list if user has mentioned in test details
# (admin need to smtp server for this functionality to work, smtp server details need to be
# mentioned in config.sh)
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} <BR> Framework : {} <BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR><BR> Creation time : {} <BR>Start time : {} <BR>End " \
"time : {} <BR>".format(t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. <BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except Exception as e:
lctx.error("Mail send error")
LOG.removeLogger(t)
return "SUCCESS"
def process_results(self, *args):
# set test status completed
# call stop monitors
# send prepare results command to exec
# set test status collating
# copy results files from exec
# copy files from each mon
# set test status finished
# remove test from running Q
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_STOP_MONITOR",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_STOP_MONITOR failed : ",
t2.testobj.TestInputData.testid)
# get statistics hosts
for s in t.testobj.TestInputData.stathostname.split(','):
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
if len(s.strip()) == 0:
break
p = self.CPORT
try:
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_MONITOR",
str(t2.testobj.TestInputData.testid)))
except:
continue
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_STOP_MONITOR failed : ",
t2.testobj.TestInputData.testid)
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ptop = process_top.ProcessTop(LOG.getLogger(
"processTop", "DH"))
# todo : avoid send client its own ip
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_FILE_DOWNLOAD failed :",
t2.testobj.TestInputData.testid)
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
lctx.error(e)
raise Exception("test result processing error",
t2.testobj.TestInputData.testid)
ptop_ret = ptop.process_top_output(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
lctx.info("Downloading stats from STATS self.HOSTS : " + s)
lctx.info(s)
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
lctx.info("Sending results.tgz download to :" + s.strip() +
":" + str(p))
try:
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
except:
continue
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
raise Exception(
"Daytona command DAYTONA_FILE_DOWNLOAD failed :",
t2.testobj.TestInputData.testid)
try:
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
except Exception as e:
lctx.error("Error in untar STAT host " + s +
" results")
lctx.error(e)
raise Exception("test result processing error",
t2.testobj.TestInputData.testid)
ptop_ret = ptop.process_top_output(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
# todo : invoke other scripts to transform results and update DB
except Exception as e:
lctx.error("Error in processing results")
lctx.error(e)
t.updateStatus("collating", "failed")
try:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
p = self.CPORT
lctx.debug("self.HOST : " + s.strip())
lctx.debug("PORT to send CLEANUP & FINISH : " + str(p))
try:
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
except:
pass
lctx.debug(retsend)
except Exception as e:
lctx.error("Error in processing results")
t.updateStatus("collating", "failed")
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateEndTime(tstr)
f = None
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} \
<BR> Framework : {} \
<BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR> \
<BR> Creation time : {} \
<BR>Start time : {} \
<BR>End time : {} <BR>".format(
t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. \
<BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except:
lctx.error("Mail send error")
return "SUCCESS"
def testmon(self, *mon):
"""
Testmon continuously monitors all the running test. It keeps on checking test status on a exec host where execution
script is running. If anything goes wrong with the test execution, this thread trigger termination actions for
this test. It also trigger graceful test termination and logs collection when test finishes on exec host
"""
process_results_threads = defaultdict()
while True:
d = "TSMON [R] : |"
remove = False
error = False
# Continuously iterate over running test list for checking test status
for k in self.running_tests:
if self.running_tests[k] is not None:
t = self.running_tests[k]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
# Initiating test logger for capturing test life cycle on scheduler, all logs are logged in
# file <testid>.log
test_logger = LOG.gettestlogger(t2, "EXEC")
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
t.updateStatus("running", "failed")
remove = True
break # out of for loop
try:
# Send DAYTONA_GET_STATUS message on exec host mentioned in test for checking test status
ret = self.cl.send(
t.testobj.TestInputData.exechostname, self.CPORT,
self.ev.construct(
"DAYTONA_GET_STATUS",
str(t2.testobj.TestInputData.testid)))
status = ret.split(",")[1]
lctx.debug(status)
test_logger.info("Test status : " + status)
except Exception as e:
lctx.debug(e)
t.updateStatus("running", "failed")
error = True
break # out of for loop
if status == "RUNNING":
# If the test is in running state, then we need to verify that user hasn't terminated this
# test from UI. If user has terminated then testmon will stop test execution on all exec host
# and stat host
found = checkTestRunning(
t.testobj.TestInputData.testid)
if not found:
error = True
break
d = d + str(self.running_tests[k].testobj.
TestInputData.testid) + "|"
elif status in ["TESTEND", "TIMEOUT"]:
# If test ends on exec host or if test timout occurs then trigger graceful shutdown of this test
# Testmon invoke a new thread for this test for logs download and test cleanup from all hosts
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
if t.testobj.TestInputData.end_status == "running":
lctx.debug(t.testobj.TestInputData.end_status)
if status == "TIMEOUT":
t.testobj.TestInputData.timeout_flag = True
t.updateStatus("running", "timeout")
else:
t.updateStatus("running", "completed")
# process_results download log files and perform cleanup on all other hosts
pt = common.FuncThread(
self.process_results, True, t,
t.testobj.TestInputData.end_status)
process_results_threads[
t.testobj.TestInputData.testid] = (pt, t)
pt.start()
remove = True
break
elif t.testobj.TestInputData.end_status == "collating" or t.testobj.TestInputData.end_status == "completed" or t.testobj.TestInputData.end_status == "finished clean":
d = d + "*" + str(self.running_tests[k].testobj.
TestInputData.testid) + "*|"
else:
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) +
":" + str(status))
break # out of for loop
elif status.strip() in ["FAILED", "TESTNA"]:
# Test termination if test fails or test is not even running on the host
if status.strip() == "FAILED":
error = True
elif status.strip() in ["ABORT", "TESTNA"]:
remove = True
t.updateStatus("", "failed")
lctx.error("TEST " + status.strip() +
" : Cleaning test from running queue")
break # out of for loop
else:
# Test termination on receiving any unknown test state
remove = True
t.updateStatus("running", "failed")
lctx.error("ERROR : Unknown test status for : " +
str(t.testobj.TestInputData.testid) + ":" +
str(status))
break # out of for loop
lctx.info(d)
d = ""
# Two modes of test termination:
if error:
# If error is set then testmon will perform below steps:
# 1. Send test ABORT on exec host if is alive, this will stop execution script, perform logs cleanup and
# test termination on the host
# 2. Send test cleanup on all other stat host for performing logs cleanup and test termination on the host
# 3. Remove test from the scheduler running queue
retsend = None
test_logger.error("Bad test status " + status +
" - Terminating test")
ip = t.testobj.TestInputData.exechostname
try:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
except:
pass
if retsend and retsend.split(",")[1] == "ALIVE":
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_ABORT_TEST",
str(t.testobj.TestInputData.testid)))
test_logger.error("Test Aborted on exec host " + ip)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
try:
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
except:
pass
if retsend and retsend.split(",")[1] == "ALIVE":
retsend = self.cl.send(
s.strip(), self.CPORT,
self.ev.construct(
"DAYTONA_CLEANUP_TEST",
str(t.testobj.TestInputData.testid)))
test_logger.error("Test Aborted on stat host " + s)
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
if remove:
# If remove flag is set, then testmon will only delete this test from the running queue of scheduler
self.lock.acquire()
for k in self.running_tests:
if self.running_tests[
k].testobj.TestInputData.testid == t.testobj.TestInputData.testid:
lctx.debug("removing entry for this test")
rt = self.running_tests.pop(k)
break
if k in self.running_tests:
del self.running_tests[k]
self.lock.release()
time.sleep(2)
def execute(self, command, paramcsv, actionID):
"""
This function maps daytona command with actual procedure which need to executed upon receiving a
particular message. This mapping is saved in action.map file and procedures are implemented in action.py
Upon mapping with actual procedure this routine spawns a new thread for executing that procedure seperately
Below are some other action performed in this procedure :
# based on SYNCFLAG release from here
# send actionID for currently being executed action based on this we can stream resp
# keep exec details over time in a buffer with actionID mapped
# send actionID NULL and hold return till exec is complete
"""
module = self.conf.actionMap[command.strip()].split(".")[0]
function = self.conf.actionMap[command.strip()].split(".")[1]
sync = self.conf.actionMap[command.strip()].split(".")[2]
t2 = testobj.testDefn()
hosttype = None
if command == "DAYTONA_START_TEST":
testid = int(paramcsv.split(",")[0])
hosttype = paramcsv.split(",")[1]
current_test = action.get_test(testid)
if current_test:
t2 = current_test.tobj
m = __import__(module)
f = getattr(m, function)
if sync == "T": # wait for func to complete and return the ret
self.lctx.debug("Executing SYNC ACTION for " + command.strip() +
" : " + self.conf.actionMap[command.strip()] +
":" + str(actionID))
ret = f(self, self, command, paramcsv, actionID, sync)
self.lctx.debug("ACTION completed for " + command.strip() + " : " +
self.conf.actionMap[command.strip()] + ":" +
str(actionID))
if command == "DAYTONA_CLI":
return "actionID=" + str(
actionID) + "%" + ret + "%" + "SYNC EXEC"
else:
return "actionID=" + str(
actionID) + "," + ret + "," + "SYNC EXEC"
else:
self.lctx.debug("Executing ASYNC ACTION for " + command.strip() +
" : " + self.conf.actionMap[command.strip()] +
":" + str(actionID))
t1 = common.FuncThread(f, True, self, command, paramcsv, actionID,
sync)
if hosttype == "EXEC":
x = (t1, actionID, t2, time.time())
self.lock.acquire()
self.async_actions.append(x)
self.lctx.debug("async_actions size :" +
str(len(self.async_actions)))
self.lock.release()
t1.start()
self.lctx.debug("Executing ACTION for " + command.strip() + " : " +
self.conf.actionMap[command.strip()] + ":" +
str(actionID))
return "actionID=" + str(
actionID) + "," + "SUCCESS," + "ASYNC EXEC"
def setupTest(self, *args):
"""
Test setup is called when scheduler send "DAYTONA_SETUP_TEST" message to agent. In this procedure agent create
all necessary file system path string and update in test object. After creating file path string it execute command
for making all these file system directories so that agent can later save SAR data. On exec host, it copies execution
script from Execscript folder to test specific directory in order to keep execution script seperate in case of
multiple test execution
:param self:
:param args: tuple of arguments containing obj, command, parameter sent by scheduler to agent for this command,
actionID and sync flag to denote if we need to execute this procedure in sync or async mode
:return: SUCCESS in case everything goes well otherwise it throws ERROR
"""
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
test_serialized = params.split(",")[0]
host_type = params.split(",")[1]
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test = get_test(t2.testobj.TestInputData.testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START")
test_logger.info("Test setup started")
current_test.tobj = testobj.testDefn()
current_test.tobj = t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid)
shutil.rmtree(dir, ignore_errors=True)
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid) + "/results/"
if host_type == "EXEC":
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.statsdir = prefix + current_test.stathostip + "/sar/"
current_test.resultsdir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/results"
current_test.archivedir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/"
if host_type == "EXEC":
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
test_logger.info("Test directory created")
if host_type == "EXEC":
execscript = current_test.tobj.testobj.TestInputData.execution_script_location
lctx.debug("TEST SETUP : " + str(execscript))
current_test.execscriptfile = current_test.execdir + "/" + execscript
lctx.debug(current_test.execscriptfile)
# check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at
# this location
execscript_location = EXEC_SCRIPT_DIR + execscript
execscript_location = os.path.realpath(execscript_location)
valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR
if valid_path:
if os.path.isfile(execscript_location):
ret = shutil.copytree(os.path.dirname(execscript_location),
os.path.dirname(current_test.execscriptfile))
else:
raise Exception(
"Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR)
else:
raise Exception(
"Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing "
"exec scripts")
os.chmod(current_test.execscriptfile, 0744)
test_logger.info("Execution script copied successfully")
save_test(current_test.testid, current_test)
test_logger.info("Test setup complete")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
