def endTest(self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) lctx.debug("Ending test") t2 = testobj.testDefn() t2.deserialize(test_serialized) (t, aid, tst, ts)=(None, None, None, None) lctx.debug(args) lctx.debug(obj) lctx.debug(obj.async_actions) for tp in obj.async_actions: (t, aid, tst, ts)=tp if tst.testobj.TestInputData.testid == t2.testobj.TestInputData.testid: lctx.debug("Found ASYNC action pending for this this test") (t, aid, tst, ts) = obj.async_actions[0] break current_test.status = "TESTABORT" if t is not None: t.stop() t.join() lctx.debug("Stopped ASYNC action pending for this this test") cleanup() lctx.debug(command + "[" + str(actionID) + "]") return "SUCCESS"
def setFinish( self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) t2 = testobj.testDefn() t2.deserialize(test_serialized) lctx.debug(str(current_test.testid) + ":" + current_test.status) current_test.status = "TESTFINISHED" return "SUCCESS"
def startTest( self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) t2 = testobj.testDefn() t2.deserialize(test_serialized) lctx.debug("starttest : " + str(t2.testobj.TestInputData.testid)) if current_test.tobj.testobj.TestInputData.testid == t2.testobj.TestInputData.testid: lctx.debug("TestID match") current_test.status = "TESTRUNNING" current_test.actionID = actionID else: lctx.error("cur test not same as test obj passed in starttest : " + str(t2.testobj.TestInputData.testid)) return "ERROR" #Original execscript #execscript = current_test.tobj.testobj.TestInputData.execution_script_location #Copied execscript execscript = current_test.execscriptfile args = "" for a in current_test.tobj.testobj.TestInputData.execScriptArgs: args = args + " \"" + a[3] + "\"" execline = execscript + " " + args lctx.debug("Execution line:" + execline) #execute the exec script here exec_cmd(execline, command, sync, obj, actionID) lctx.debug("Completed start test") return "SUCCESS"
def stopMonitor( self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) t2 = testobj.testDefn() t2.deserialize(test_serialized) lctx.debug("stop monitor for test : " + str(t2.testobj.TestInputData.testid)) if current_test.testid != t2.testobj.TestInputData.testid: lctx.debug("start mon : TestID dont match") return "ERROR" cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") #stop the sar processes execline = cfg.daytona_mon_path + "/sar_gather_agent.pl --shutdown --root-dir=" + current_test.statsdir lctx.info(execline) exec_cmd(execline, command, sync, obj, actionID) #prepare mon results tarball here lctx.debug(current_test.statsdir) #os.remove(current_test.statsdir + "/sar.dat") os.remove(current_test.statsdir + "/sar_gather_agent_debug.out") #os.remove(current_test.statsdir + "/*.pid") lctx.debug("removed monitor temp files from : " + current_test.archivedir) common.make_tarfile(current_test.archivedir + "results_stats.tgz", current_test.archivedir) lctx.debug("Completed stop monitor") return "SUCCESS"
def fileDownload(self, *args): cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) prm = args[2].split(',') lctx.debug(prm) test_serialized = prm[3].strip() t2 = testobj.testDefn() t2.deserialize(test_serialized) dest ="" if prm[4] == "RESULTS" : dest = t2.testobj.TestInputData.exec_results_path elif prm[4] == "STATS" : s = prm[5] lctx.debug(s) dest = t2.testobj.TestInputData.stats_results_path[s] else: dest = "" lctx.debug(prm[0]) lctx.debug(prm[1]) lctx.debug(prm[2]) lctx.debug(dest) try: cl.sendFile(prm[0].strip(), int(prm[1].strip()), prm[2].strip(), dest.strip()) except CommunicationError as e: lctx.error(e.value) return e.value return "SUCCESS"
def execute(self, command, paramcsv, actionID): #based on SYNCFLAG release from here #send actionID for currently being executed action based on this we can stream resp #keep exec details over time in a buffer with actionID mapped #send actionID NULL and hold return till exec is complete module = self.conf.actionMap[command.strip()].split(".")[0] function = self.conf.actionMap[command.strip()].split(".")[1] sync = self.conf.actionMap[command.strip()].split(".")[2] t2 = testobj.testDefn() try: param = int(paramcsv) action.action_lock.acquire() t2 = action.running_tests[param].tobj action.action_lock.release() except Exception as e: pass m = __import__(module) f = getattr(m, function) if sync == "T": #wait for func to complete and return the ret self.lctx.debug("Executing SYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) ret = f(self, self, command, paramcsv, actionID, sync) self.lctx.debug("ACTION completed for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) if command == "DAYTONA_CLI": return "actionID=" + str( actionID) + "%" + ret + "%" + "SYNC EXEC" else: return "actionID=" + str( actionID) + "," + ret + "," + "SYNC EXEC" else: #callback will be called after completion #actionID = uuid.uuid4() self.lctx.debug("Executing ASYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) t1 = common.FuncThread(f, True, self, command, paramcsv, actionID, sync) x = (t1, actionID, t2, time.time()) self.lock.acquire() self.async_actions.append(x) self.lctx.debug("async_actions size :" + str(len(self.async_actions))) self.lock.release() t1.start() self.lctx.debug("Executing ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) return "actionID=" + str( actionID) + "," + "SUCCESS," + "ASYNC EXEC"
def startMonitor( self, *args): (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) lctx.debug("startMonitor") p = params.split(",") test_serialized = p[0] s = p[1].strip() try : current_test.tobj = testobj.testDefn() t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test.tobj= t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid lctx.debug("TEST SETUP Monitor : " + str(current_test.tobj.testobj.TestInputData.testid)) #todo : indicate a status that says this host is monitor-active #current_test.status = "TESTSETUP" cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(current_test.tobj.testobj.TestInputData.testid) + "/results/" current_test.statsdir = prefix + s + "/sar/" current_test.archivedir = prefix common.createdir(cfg.daytona_agent_root, lctx) common.createdir(current_test.statsdir, lctx) execline = cfg.daytona_mon_path + "/sar_gather_agent.pl --daemonize --root-dir=" + current_test.statsdir lctx.info(execline) exec_cmd(execline, command, sync, obj, actionID) except Exception as e: lctx.error(e) lctx.error(traceback.print_exc()) return "ERROR" lctx.debug("Completed start monitor") return "SUCCESS"
def prepareResults(self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) t2 = testobj.testDefn() t2.deserialize(test_serialized) lctx.debug("preparing results for test : " + str(t2.testobj.TestInputData.testid)) if current_test.testid != t2.testobj.TestInputData.testid: lctx.debug("start mon : TestID dont match") return "ERROR" #stop the sar processes #prepare results tarball here common.make_tarfile(current_test.archivedir + "results.tgz", current_test.archivedir) lctx.debug(command + "[" + str(actionID) + "]") return "SUCCESS"
def execute(self, command, paramcsv, actionID): #based on SYNCFLAG release from here #send actionID for currently being executed action based on this we can stream resp #keep exec details over time in a buffer with actionID mapped #send actionID NULL and hold return till exec is complete module = self.conf.actionMap[command.strip()].split(".")[0] function = self.conf.actionMap[command.strip()].split(".")[1] sync = self.conf.actionMap[command.strip()].split(".")[2] self.lctx.debug(command) self.lctx.debug(paramcsv) self.lctx.debug(actionID) t2 = testobj.testDefn() tst = "" if paramcsv != "": p = paramcsv.split(",") tst = p[0] if command == "DAYTONA_FILE_DOWNLOAD": tst = p[3] if tst != "": t2.deserialize(tst) m = __import__ (module) f = getattr(m,function) if sync == "T" : #wait for func to complete and return the ret self.lctx.debug("Executing SYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) ret = f(self, self, command, paramcsv, actionID, sync) self.lctx.debug("ACTION completed for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) return "actionID=" + str(actionID) + "," + ret + "," + "SYNC EXEC" else : #callback will be called after completion #actionID = uuid.uuid4() self.lctx.debug("Executing ASYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) t1 = common.FuncThread(f, True, self, command, paramcsv, actionID, sync) x = (t1, actionID, t2, time.time()) self.lock.acquire() self.async_actions.append(x) self.lctx.debug( "async_actions size :" + str(len(self.async_actions))) self.lock.release() t1.start() self.lctx.debug( "Executing ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) return "actionID=" + str(actionID) + "," + "SUCCESS," + "ASYNC EXEC"
def execute(self, command, paramcsv, actionID): # based on SYNCFLAG release from here # send actionID for currently being executed action based on this we can stream resp # keep exec details over time in a buffer with actionID mapped # send actionID NULL and hold return till exec is complete module = self.conf.actionMap[command.strip()].split(".")[0] function = self.conf.actionMap[command.strip()].split(".")[1] sync = self.conf.actionMap[command.strip()].split(".")[2] t2 = testobj.testDefn() if command == "DAYTONA_START_TEST": testid = int(paramcsv.split(",")[0]) hosttype = paramcsv.split(",")[1] current_test = action.get_test(testid) if current_test: t2 = current_test.tobj m = __import__ (module) f = getattr(m,function) if sync == "T": # wait for func to complete and return the ret self.lctx.debug("Executing SYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) ret = f(self, self, command, paramcsv, actionID, sync) self.lctx.debug("ACTION completed for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) if command == "DAYTONA_CLI": return "actionID=" + str(actionID) + "%" + ret + "%" + "SYNC EXEC" else: return "actionID=" + str(actionID) + "," + ret + "," + "SYNC EXEC" else: self.lctx.debug("Executing ASYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) t1 = common.FuncThread(f, True, self, command, paramcsv, actionID, sync) if hosttype == "EXEC": x = (t1, actionID, t2, time.time()) self.lock.acquire() self.async_actions.append(x) self.lctx.debug( "async_actions size :" + str(len(self.async_actions))) self.lock.release() t1.start() self.lctx.debug( "Executing ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) return "actionID=" + str(actionID) + "," + "SUCCESS," + "ASYNC EXEC"
def setupTest(self, *args): (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) test_serialized = params.split(",")[0] host_type = params.split(",")[1] t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test = get_test(t2.testobj.TestInputData.testid) if current_test: if current_test.status is not "INIT": lctx.error("Invalid state for TEST SETUP action : " + current_test.status) current_test.status = "ABORT" save_test(current_test.testid, current_test) return "ERROR" try: if current_test: lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START") current_test.tobj = testobj.testDefn() current_test.tobj = t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid lctx.debug("TEST SETUP : " + str(current_test.tobj.testobj.TestInputData.testid)) cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) shutil.rmtree(dir, ignore_errors=True) prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) + "/results/" if host_type == "EXEC": current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application" current_test.statsdir = prefix + current_test.stathostip + "/sar/" current_test.resultsdir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/results" current_test.archivedir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/" common.createdir(cfg.daytona_agent_root, self.lctx) if host_type == "EXEC": common.createdir(current_test.execdir, self.lctx) common.createdir(current_test.logdir, self.lctx) common.createdir(current_test.resultsdir, self.lctx) common.createdir(current_test.statsdir, self.lctx) # todo : check and validate if exec script is provided in expected format and # the file exists in that location if host_type == "EXEC": execscript = current_test.tobj.testobj.TestInputData.execution_script_location lctx.debug("TEST SETUP : " + str(execscript)) current_test.execscriptfile = current_test.execdir + "/" + execscript lctx.debug(current_test.execscriptfile) # check if execution script is present in '/tmp/ExecScripts/' - execute script only if it present at # this location execscript_location = EXEC_SCRIPT_DIR + execscript execscript_location = os.path.realpath(execscript_location) valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR if valid_path: if os.path.isfile(execscript_location): ret = copyfile(execscript_location, current_test.execscriptfile) else: raise Exception( "Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR) else: raise Exception( "Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing " "exec scripts") os.chmod(current_test.execscriptfile, 0744) current_test.status = "SETUP" save_test(current_test.testid, current_test) lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except shutil.Error as err: if current_test: lctx.error("error copying file : " + str(execscript) + " to " + str(current_test.execscriptfile)) current_test.status = "ABORT" save_test(current_test.testid, current_test) lctx.error(err) return "ERROR" except Exception as e: if current_test: current_test.status = "ABORT" save_test(current_test.testid, current_test) lctx.error(e) return "ERROR"
def testmon(self, *mon): process_results_threads = defaultdict() while True: d = "TSMON [R] : |" remove = False for k in self.running_tests: if (self.running_tests[k] != None): t = self.running_tests[k] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) if (t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid): lctx.error("testobj not same") t.updateStatus("running", "failed") remove = True break #out of for loop try: ret = self.cl.send( t.testobj.TestInputData.exechostname, self.CPORT, self.ev.construct("DAYTONA_GET_STATUS", serialize_str)) status = ret.split(",")[1] lctx.debug(status) except Exception as e: lctx.debug(e) t.updateStatus("running", "failed") remove = True break #out of for loop if "TESTRUNNING" == status or "TESTSETUP" == status: d = d + str(self.running_tests[k].testobj. TestInputData.testid) + "|" elif "TESTEND" == status.strip(): d = d + "*" + str(self.running_tests[k].testobj. TestInputData.testid) + "*|" if t.testobj.TestInputData.end_status == "running": lctx.debug(t.testobj.TestInputData.end_status) if t.testobj.TestInputData.end_status == "running": lctx.debug( "Updating status to completed in DB") t.updateStatus("running", "completed") pt = common.FuncThread( self.process_results, True, t, t.testobj.TestInputData.end_status) process_results_threads[ t.testobj.TestInputData.testid] = (pt, t) pt.start() elif t.testobj.TestInputData.end_status == "collating" or t.testobj.TestInputData.end_status == "completed" or t.testobj.TestInputData.end_status == "finished clean": d = d + "*" + str(self.running_tests[k].testobj. TestInputData.testid) + "*|" else: remove = True t.updateStatus("running", "failed") lctx.error("ERROR : Unknown test status for : " + str(t.testobj.TestInputData.testid) + ":" + str(status)) break #out of for loop elif "TESTFINISHED" == status.strip(): d = "TSMON [F] : |*" + str( self.running_tests[k].testobj.TestInputData.testid ) + "*|" remove = True break else: remove = True t.updateStatus("running", "failed") lctx.error("ERROR : Unknown test status for : " + str(t.testobj.TestInputData.testid) + ":" + str(status)) break #out of for loop lctx.info(d) d = "" if (remove == True): self.lock.acquire() for k in self.running_tests: if self.running_tests[ k].testobj.TestInputData.testid == t.testobj.TestInputData.testid: lctx.debug("removing entry for this test") rt = self.running_tests.pop(k) break if k in self.running_tests: del self.running_tests[k] self.lock.release() time.sleep(2)
def process_results(self, *args): #set test status completed #call stop monitors #send prepare results command to exec #set test status collating #copy results files from exec #copy files from each mon #set test status finished #remove test from running Q t = args[1] status = args[2] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) try: if (t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid): lctx.error("testobj not same") raise Exception("Test objects do not match : ", t2.testobj.TestInputData.testid) ip = t.testobj.TestInputData.exechostname lctx.debug(status) if status == "completed": retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_PREPARE_RESULTS", serialize_str)) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_PREPARE_RESULTS failed : ", t2.testobj.TestInputData.testid) #get statistics hosts for s in t.testobj.TestInputData.stathostname.split(','): #stop stats monitors on req hosts #any host that blocks stop monitor blocks the scheduling for the FW p = 0 if s.strip() == self.HOST: p = self.PORT else: p = self.CPORT retsend = self.cl.send( s.strip(), p, self.ev.construct("DAYTONA_STOP_MONITOR", serialize_str)) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_STOP_MONITOR failed : ", t2.testobj.TestInputData.testid) t.updateStatus("completed", "collating") #todo : avoid send client its own ip lctx.debug("SENDING results.tgz download to : " + ip + ":" + str(self.CPORT)) results_file = cfg.daytona_agent_root + "/" + t.testobj.TestInputData.frameworkname + "/" + str( t.testobj.TestInputData.testid ) + "/results/" + "results.tgz" retsend = self.cl.send( ip, self.CPORT, self.ev.construct( "DAYTONA_FILE_DOWNLOAD", str(self.HOST) + "," + str(self.PORT) + "," + results_file + "," + serialize_str + "," + "RESULTS" + "," + ip)) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_FILE_DOWNLOAD failed :", t2.testobj.TestInputData.testid) results_file = cfg.daytona_agent_root + "/" + t.testobj.TestInputData.frameworkname + "/" + str( t.testobj.TestInputData.testid ) + "/results/" + "results_stats.tgz" for s in t.testobj.TestInputData.stathostname.split(','): lctx.info("Downloading stats from STATS self.HOSTS : " + s) lctx.info(s) #stop stats monitors on req hosts #any host that blocks stop monitor blocks the scheduling for the FW p = 0 if s.strip() == self.HOST: p = self.PORT else: p = self.CPORT lctx.info("Sending DOWNLOAD file to :" + s.strip() + ":" + str(p) + "File :" + results_file + "(upload to this host port:)" + str(self.HOST) + "," + str(self.PORT)) retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_FILE_DOWNLOAD", str(self.HOST) + "," + str(self.PORT) + "," + results_file + "," + serialize_str + "," + "STATS" + "," + s.strip())) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error("Error downloading " + results_file + " From " + s.strip() + ":" + retsend) raise Exception( "Daytona command DAYTONA_FILE_DOWNLOAD failed :", t2.testobj.TestInputData.testid) try: lctx.debug(t2.testobj.TestInputData.exec_results_path + "results.tgz") lctx.debug(t2.testobj.TestInputData.exec_results_path + "/../") common.untarfile( t2.testobj.TestInputData.exec_results_path + "/results.tgz", t2.testobj.TestInputData.exec_results_path + "/../") for s in t2.testobj.TestInputData.stats_results_path: lctx.debug( t2.testobj.TestInputData.stats_results_path[s] + "results_stats.tgz") lctx.debug( t2.testobj.TestInputData.stats_results_path[s] + "/../") common.untarfile( t2.testobj.TestInputData.stats_results_path[s] + "/results_stats.tgz", t2.testobj.TestInputData.stats_results_path[s] + "/../") except Exception as e: lctx.error("Error in untar results") lctx.error(e) raise Exception("test result processing error", t2.testobj.TestInputData.testid) #todo : invoke other scripts to transform results and update DB except Exception as e: lctx.error("Error in processing results") lctx.error(e) t.updateStatus("collating", "failed") try: retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_CLEANUP_TEST", serialize_str)) lctx.debug("DAYTONA_CLEANUP_TEST:" + str(retsend)) retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_FINISH_TEST", serialize_str)) lctx.debug(retsend) for s in t.testobj.TestInputData.stathostname.split(','): p = 0 if s.strip() == self.HOST: p = self.PORT else: p = self.CPORT lctx.debug("self.HOST : " + s.strip()) lctx.debug("PORT to send CLEANUP & FINISH : " + str(p)) retsend = self.cl.send( s.strip(), p, self.ev.construct("DAYTONA_CLEANUP_TEST", serialize_str)) lctx.debug(retsend) retsend = self.cl.send( s.strip(), p, self.ev.construct("DAYTONA_FINISH_TEST", serialize_str)) lctx.debug(retsend) except Exception as e: lctx.error("Error in processing results") t.updateStatus("collating", "failed") t.updateStatus("collating", "finished clean") now = time.time() tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now))) t.updateEndTime(tstr) f = open(t2.testobj.TestInputData.exec_results_path + "/results.csv") to = t.testobj.TestInputData.email reader = csv.reader(f) htmlfile = "" rownum = 0 htmlfile = '<table cellpadding="10">' for row in reader: if rownum == 0: htmlfile = htmlfile + '<tr>' for column in row: htmlfile = htmlfile + '<th width="70%">' + column + '</th>' htmlfile = htmlfile + '</tr>' else: htmlfile = htmlfile + '<tr>' for column in row: htmlfile = htmlfile + '<td width="70%">' + column + '</td>' htmlfile = htmlfile + '</tr>' rownum += 1 htmlfile = htmlfile + '</table>' f.close() subject = "Test {} completed successfully".format( t.testobj.TestInputData.testid) mail_content = "<BR> Test id : {} \ <BR> Framework : {} \ <BR> Title : {} <BR>".format( t.testobj.TestInputData.testid, t.testobj.TestInputData.frameworkname, t.testobj.TestInputData.title) mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>Purpose : {} <BR> \ <BR> Creation time : {} \ <BR>Start time : {} \ <BR>End time : {} <BR>".format( t.testobj.TestInputData.purpose, t.testobj.TestInputData.creation_time, t.testobj.TestInputData.start_time, t.testobj.TestInputData.end_time) mail_content = mail_content + "<BR>Your test executed successfully. \ <BR>Results (Contents of results.csv)<BR>" mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>" + htmlfile + "<BR>" try: common.send_email(subject, to, mail_content, "", lctx, cfg.email_user, cfg.email_server, cfg.smtp_server, cfg.smtp_port) except: lctx.error("Mail send error") return "SUCCESS"
def mon(self, *args): #query all waiting state tests and load into to_schedule, this is for restart case (similarly for running) #todo : reconcile running and scheduled testids, if db altered externally restarted = True print "CFG in mon :" print self.cfg.mysql_host while True: self.db = DBAccess(self.cfg, self.lctx) d = "DBMON [W] : |" for k in self.tests_to_run: l = self.tests_to_run[k] for t in l : d = d + str(t.testobj.TestInputData.testid) + "|" self.lctx.info(d) d = "" query_result = None if restarted == True : query_result = self.db.query("""select testid from CommonFrameworkSchedulerQueue where state = %s or state = %s or state = %s or state = %s or state = %s or state = %s""", ("scheduled","waiting","setup","running","completed","collating"), True, False); #query_result = self.db.query("""select testid from TestInputData where end_status = %s or end_status = %s or end_status = %s or end_status = %s or end_status = %s or end_status = %s""", ("scheduled","waiting","setup","running","completed","collating"), True, False); restarted = False else: status = "scheduled" query_result = self.db.query("""select testid from CommonFrameworkSchedulerQueue where state = %s""", (status,), True, False); #query_result = self.db.query("""select testid from TestInputData where end_status = %s""", (status,), True, False); #reset all states to scheduled, mostly required in a restart case #all items that make to the DBMON [Q] will be in scheduled state for testid in query_result: to = testobj.testDefn() to.testobj.TestInputData.testid = testid[0] res = to.construct(testid[0]) res = to.updateStatus("*", "scheduled") d = "DBMON [Q] : " d = d + str(query_result) self.lctx.info(d) for testid in query_result: found = False for k in self.tests_to_run: #search across all FW l = self.tests_to_run[k] #list of tests ready to be run for t in l: self.lctx.debug(t.testobj.TestInputData.testid) self.lctx.debug(testid[0]) self.lctx.debug(k) if t.testobj.TestInputData.testid == testid[0]: self.lctx.debug("Test already present in runQ") found = True if(found == False): to = testobj.testDefn() to.testobj.TestInputData.testid = testid[0] #todo handle return status res = to.construct(testid[0]) #create required dirs and setup server side env prefix=self.cfg.daytona_dh_root +"/"+ to.testobj.TestInputData.frameworkname + "/" + str(to.testobj.TestInputData.testid) + "/" + "results" + "/" to.testobj.TestInputData.exec_results_path = prefix+to.testobj.TestInputData.exechostname + "/" to.testobj.TestInputData.exec_path = prefix+to.testobj.TestInputData.exechostname to.testobj.TestInputData.exec_log_path = prefix+to.testobj.TestInputData.exechostname+"/application" to.testobj.TestInputData.stats_results_path = defaultdict() for s in to.testobj.TestInputData.stathostname.split(','): to.testobj.TestInputData.stats_results_path[s.strip()] = prefix+s.strip()+"/" to.testobj.TestInputData.stats_results_path[to.testobj.TestInputData.exechostname] = prefix+to.testobj.TestInputData.exechostname+"/" common.createdir(to.testobj.TestInputData.exec_results_path, self.lctx) common.createdir(to.testobj.TestInputData.exec_path, self.lctx) common.createdir(to.testobj.TestInputData.exec_log_path, self.lctx) for s in to.testobj.TestInputData.stats_results_path: common.createdir(to.testobj.TestInputData.stats_results_path[s], self.lctx) res = to.updateStatus("scheduled", "waiting") sz = to.serialize() t2 = testobj.testDefn() t2.deserialize(sz) if t2.testobj.TestInputData.testid != to.testobj.TestInputData.testid : self.lctx.error("error in ser / dser") break #use a lock here self.lock.acquire() self.tests_to_run[to.testobj.TestInputData.frameworkid].append(to); self.lctx.debug("Adding : " + str(to.testobj.TestInputData.testid)) self.lock.release() self.db.close() if self.mon_thread[0].check() == False : return time.sleep(5) #todo : make this config item
def setupTest( self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) lctx.debug("setuptest") try : current_test.tobj = testobj.testDefn() t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test.tobj= t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid lctx.debug("TEST SETUP : " + str(current_test.tobj.testobj.TestInputData.testid)) current_test.status = "TESTSETUP" cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(current_test.tobj.testobj.TestInputData.testid) + "/results/" current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application" current_test.resultsdir = prefix current_test.statsdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/sar/" current_test.archivedir = prefix common.createdir(cfg.daytona_agent_root, self.lctx) common.createdir(current_test.execdir, self.lctx) common.createdir(current_test.logdir, self.lctx) common.createdir(current_test.resultsdir, self.lctx) common.createdir(current_test.statsdir, self.lctx) #todo : check and validate if exec script is provided in expected format and # the file exists in that location execscript = current_test.tobj.testobj.TestInputData.execution_script_location.split(":")[1] lctx.debug("TEST SETUP : " + str(execscript)) tmp = str(execscript).split("/") tmp.reverse() lctx.debug(tmp) filename = tmp[0] current_test.execscriptfile = current_test.execdir+"/"+filename lctx.debug(current_test.execscriptfile) except Exception as e: lctx.error(e) lctx.error(traceback.print_exc()) return "ERROR" try: ret = copyfile(execscript, current_test.execscriptfile) except shutil.Error as err: lctx.error("error copying file : " + str(execscript) + " to " + str(current_test.execscriptfile)) return "ERROR" try: os.chmod(current_test.execscriptfile, 0744) except: lctx.error("error setting perm file : " + str(current_test.execdir)) return "ERROR" #create dirs #get exec script name #cp the exec script #set exec perm #update cur test obj with exec script #exec any custom setup script lctx.debug("Completed setuptest") return "SUCCESS"
def testmon(self, *mon): process_results_threads = defaultdict() while True: d = "TSMON [R] : |" remove = False error = False for k in self.running_tests: if (self.running_tests[k] != None): t = self.running_tests[k] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid: lctx.error("testobj not same") t.updateStatus("running", "failed") remove = True break # out of for loop try: ret = self.cl.send( t.testobj.TestInputData.exechostname, self.CPORT, self.ev.construct( "DAYTONA_GET_STATUS", str(t2.testobj.TestInputData.testid))) status = ret.split(",")[1] lctx.debug(status) except Exception as e: lctx.debug(e) t.updateStatus("running", "failed") error = True break # out of for loop if status in [ "RUNNING", "INIT", "SETUP", "MONITOR_ON", "MONITOR_OFF" ]: found = checkTestRunning( t.testobj.TestInputData.testid) if not found: error = True break d = d + str(self.running_tests[k].testobj. TestInputData.testid) + "|" elif status in ["TESTEND", "TIMEOUT"]: d = d + "*" + str(self.running_tests[k].testobj. TestInputData.testid) + "*|" if t.testobj.TestInputData.end_status == "running": lctx.debug(t.testobj.TestInputData.end_status) if t.testobj.TestInputData.end_status == "running": if status == "TIMEOUT": t.testobj.TestInputData.timeout_flag = True t.updateStatus("running", "timeout") else: t.updateStatus("running", "completed") pt = common.FuncThread( self.process_results, True, t, t.testobj.TestInputData.end_status) process_results_threads[ t.testobj.TestInputData.testid] = (pt, t) pt.start() elif t.testobj.TestInputData.end_status == "collating" or t.testobj.TestInputData.end_status == "completed" or t.testobj.TestInputData.end_status == "finished clean": d = d + "*" + str(self.running_tests[k].testobj. TestInputData.testid) + "*|" else: remove = True t.updateStatus("running", "failed") lctx.error("ERROR : Unknown test status for : " + str(t.testobj.TestInputData.testid) + ":" + str(status)) break # out of for loop elif status.strip() == "FINISHED": d = "TSMON [F] : |*" + str( self.running_tests[k].testobj.TestInputData.testid ) + "*|" remove = True break elif status.strip() in ["FAILED", "ABORT", "TESTNA"]: if status.strip() == "FAILED": error = True elif status.strip() in ["ABORT", "TESTNA"]: remove = True t.updateStatus("", "failed") lctx.error("TEST " + status.strip() + " : Cleaning test from running queue") break # out of for loop else: remove = True t.updateStatus("running", "failed") lctx.error("ERROR : Unknown test status for : " + str(t.testobj.TestInputData.testid) + ":" + str(status)) break # out of for loop lctx.info(d) d = "" if error: retsend = None ip = t.testobj.TestInputData.exechostname try: retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_HEARTBEAT", "")) except: pass if retsend and retsend.split(",")[1] == "ALIVE": retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_STOP_MONITOR", str(t.testobj.TestInputData.testid))) retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_ABORT_TEST", str(t.testobj.TestInputData.testid))) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break try: retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct("DAYTONA_HEARTBEAT", "")) except: pass if retsend and retsend.split(",")[1] == "ALIVE": retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct( "DAYTONA_STOP_MONITOR", str(t.testobj.TestInputData.testid))) retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct( "DAYTONA_ABORT_TEST", str(t.testobj.TestInputData.testid))) self.lock.acquire() for k in self.running_tests: if self.running_tests[ k].testobj.TestInputData.testid == t.testobj.TestInputData.testid: lctx.debug("removing entry for this test") rt = self.running_tests.pop(k) break if k in self.running_tests: del self.running_tests[k] self.lock.release() if remove: self.lock.acquire() for k in self.running_tests: if self.running_tests[ k].testobj.TestInputData.testid == t.testobj.TestInputData.testid: lctx.debug("removing entry for this test") rt = self.running_tests.pop(k) break if k in self.running_tests: del self.running_tests[k] self.lock.release() time.sleep(2)
def trigger(self, *args): # trigger starts in a thread, keep track of all triggers and they should complete in a specified time, # otherwise signal a close t = args[1] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) time.sleep(6) track_monitor = [] try: if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid: lctx.error("testobj not same") raise Exception("test trigger error", t2.testobj.TestInputData.testid) ip = t.testobj.TestInputData.exechostname retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_SETUP_TEST", serialize_str + ",EXEC")) lctx.debug(retsend) st = retsend.split(",") if len(st) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Test Setup Failure : Test - ", t2.testobj.TestInputData.testid) retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_START_MONITOR", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) st = retsend.split(",") if len(st) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Test monitor start failure : Test - ", t2.testobj.TestInputData.testid) track_monitor.append(ip) # get statistics hosts for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break lctx.debug("Start monitor") lctx.debug(s.strip()) p = self.CPORT retsend = self.cl.send( s, p, self.ev.construct("DAYTONA_HEARTBEAT", "")) st = retsend.split(",") if len(st) > 1: if retsend.split(",")[1] != "ALIVE": raise Exception( "Remove host not avaliable - No Heartbeat ", t2.testobj.TestInputData.testid) else: retsend = self.cl.send( s, p, self.ev.construct( "DAYTONA_HANDSHAKE", self.HOST + "," + str(self.PORT) + "," + str(t2.testobj.TestInputData.testid) + "," + s)) lctx.debug(retsend) if retsend == "SUCCESS": alive = True server.serv.registered_hosts[s] = s addr = socket.gethostbyname(s) server.serv.registered_hosts[addr] = addr else: raise Exception( "Unable to handshake with agent on stats host:" + s, t2.testobj.TestInputData.testid) else: raise Exception( "Remove host not avaliable - No Heartbeat ", t2.testobj.TestInputData.testid) # start stats monitors on req hosts # any host that blocks start monitor blocks the scheduling for the FW retsend = self.cl.send( s.strip(), p, self.ev.construct("DAYTONA_SETUP_TEST", serialize_str + ",STAT")) lctx.debug(retsend) st = retsend.split(",") if len(st) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Test Setup Failure : Test - ", t2.testobj.TestInputData.testid) retsend = self.cl.send( s.strip(), p, self.ev.construct("DAYTONA_START_MONITOR", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) st = retsend.split(",") if len(st) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Test monitor start failure : Test - ", t2.testobj.TestInputData.testid) track_monitor.append(s.strip()) # Trigger the start of test to test box retsend = self.cl.send( t.testobj.TestInputData.exechostname, self.CPORT, self.ev.construct("DAYTONA_START_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) st = retsend.split(",") if len(st) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Failed to start Test : ", t2.testobj.TestInputData.testid) # Get status from tst box retsend = self.cl.send( t.testobj.TestInputData.exechostname, self.CPORT, self.ev.construct("DAYTONA_GET_STATUS", str(t2.testobj.TestInputData.testid))) st = retsend.split(",") if len(st) > 1: if "RUNNING" == st[1] or "MONITOR_ON" == st[1]: # update from setup to running lctx.debug("Updating test status to running in DB") t.updateStatus("setup", "running") now = time.time() tstr = str( time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now))) t.updateStartTime(tstr) else: lctx.error( "Unable to determine status, testmon will garbage collect this testid" ) # Garbage collect testid in runnning state that fails to give status # Killing of threads on client host and remove from list with status=fail is done in testmon except Exception as e: lctx.error(e) lctx.error("ERROR : Unknown trigger error : " + str(t.testobj.TestInputData.testid)) t.updateStatus("setup", "failed") lctx.debug(traceback.print_exc()) if ip in track_monitor: retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_STOP_MONITOR", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_ABORT_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break if s.strip() in track_monitor: retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct( "DAYTONA_STOP_MONITOR", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct( "DAYTONA_ABORT_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) return "FAILED" return "SUCCESS"
def getStatus( self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) t2 = testobj.testDefn() t2.deserialize(test_serialized) lctx.debug(str(current_test.testid) + ":" + current_test.status) return current_test.status
def trigger(self, *args): # trigger starts in a thread, keep track of all triggers and they should complete in a specified time, # otherwise signal a close t = args[1] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) time.sleep(6) test_logger = LOG.gettestlogger(t2, "EXEC") test_logger.info("Test setup started") try: if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid: lctx.error("testobj not same") raise Exception("test trigger error", t2.testobj.TestInputData.testid) ip = t.testobj.TestInputData.exechostname retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_SETUP_TEST", serialize_str + ",EXEC")) lctx.debug(retsend) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Test Setup Failure : Test - ", t2.testobj.TestInputData.testid) test_logger.info("Test setup complete on exec host " + ip) # get statistics hosts for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break test_logger.info("Starting test setup on stat host " + s) lctx.debug(s.strip()) p = self.CPORT retsend = self.cl.send( s, p, self.ev.construct("DAYTONA_HEARTBEAT", "")) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "ALIVE": raise Exception( "Remove host not avaliable - No Heartbeat ", t2.testobj.TestInputData.testid) else: test_logger.info("Hearbeat received from stat host " + s) retsend = self.cl.send( s, p, self.ev.construct( "DAYTONA_HANDSHAKE", "handshake1," + self.HOST + "," + str(self.PORT) + "," + str(t2.testobj.TestInputData.testid) + "," + s)) lctx.debug(retsend) if retsend == "SUCCESS": alive = True server.serv.registered_hosts[s] = s addr = socket.gethostbyname(s) server.serv.registered_hosts[addr] = addr test_logger.info( "Handshake successfull with agent on stat host " + s) else: raise Exception( "Unable to handshake with agent on stats host:" + s, t2.testobj.TestInputData.testid) else: raise Exception("Stat host " + s + " not avaliable - No Heartbeat") # start stats monitors on req hosts # any host that blocks start monitor blocks the scheduling for the FW retsend = self.cl.send( s.strip(), p, self.ev.construct("DAYTONA_SETUP_TEST", serialize_str + ",STAT")) lctx.debug(retsend) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Test Setup Failure : Test - ", t2.testobj.TestInputData.testid) test_logger.info("Test setup complete on stat host " + s) # Trigger the start of test to test box retsend = self.cl.send( ip, self.CPORT, self.ev.construct( "DAYTONA_START_TEST", str(t2.testobj.TestInputData.testid) + ",EXEC")) lctx.debug(retsend) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Failed to start Test : ", t2.testobj.TestInputData.testid) test_logger.info("Test started on exec host " + ip) # Trigger the start of test on STAT hosts (This is for initiating system metric data collection) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break p = self.CPORT s = s.strip() retsend = self.cl.send( s, p, self.ev.construct( "DAYTONA_START_TEST", str(t2.testobj.TestInputData.testid) + ",STAT")) lctx.debug(retsend) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("Failed to start test on STAT host : ", s) else: raise Exception("Failed to start test on STAT host : ", s) test_logger.info("Test started on stat host " + s) # Get status from tst box retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_GET_STATUS", str(t2.testobj.TestInputData.testid))) if retsend and len(retsend.split(",")) > 1: if "RUNNING" == retsend.split( ",")[1] or "MONITOR_ON" == retsend.split(",")[1]: # update from setup to running lctx.debug("Updating test status to running in DB") t.updateStatus("setup", "running") now = time.time() tstr = str( time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now))) t.updateStartTime(tstr) self.dispatchQ__lock.acquire() del self.dispatch_queue[ t.testobj.TestInputData.frameworkid] self.dispatchQ__lock.release() self.lock.acquire() self.running_tests[t.testobj.TestInputData.frameworkid] = t self.lock.release() else: lctx.error( "Unable to determine status, testmon will garbage collect this testid" ) # Garbage collect testid in runnning state that fails to give status # Killing of threads on client host and remove from list with status=fail is done in testmon except Exception as e: lctx.error(e) test_logger.error(e) lctx.error("ERROR : Unknown trigger error : " + str(t.testobj.TestInputData.testid)) t.updateStatus("setup", "failed") lctx.debug(traceback.print_exc()) lctx.error("Removing Test " + str(t.testobj.TestInputData.testid) + " from dispatch Q") self.dispatchQ__lock.acquire() del self.dispatch_queue[t.testobj.TestInputData.frameworkid] self.dispatchQ__lock.release() retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_ABORT_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) test_logger.info("Test aborted on exec host " + ip) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct("DAYTONA_CLEANUP_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) test_logger.info("Test abort on stat host " + s) LOG.removeLogger(t2) return "FAILED" return "SUCCESS"
def setupTest(self, *args): (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) test_serialized = params.split(",")[0] host_type = params.split(",")[1] t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test = get_test(t2.testobj.TestInputData.testid) test_logger = None try: if current_test: test_logger = LOG.gettestlogger(current_test, "STAT") lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START") test_logger.info("Test setup started") current_test.tobj = testobj.testDefn() current_test.tobj = t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) shutil.rmtree(dir, ignore_errors=True) prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) + "/results/" if host_type == "EXEC": current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application" current_test.statsdir = prefix + current_test.stathostip + "/sar/" current_test.resultsdir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/results" current_test.archivedir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/" if host_type == "EXEC": common.createdir(current_test.execdir, self.lctx) common.createdir(current_test.logdir, self.lctx) common.createdir(current_test.resultsdir, self.lctx) common.createdir(current_test.statsdir, self.lctx) test_logger.info("Test directory created") # todo : check and validate if exec script is provided in expected format and # the file exists in that location if host_type == "EXEC": execscript = current_test.tobj.testobj.TestInputData.execution_script_location lctx.debug("TEST SETUP : " + str(execscript)) current_test.execscriptfile = current_test.execdir + "/" + execscript lctx.debug(current_test.execscriptfile) # check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at # this location execscript_location = EXEC_SCRIPT_DIR + execscript execscript_location = os.path.realpath(execscript_location) valid_path = os.path.commonprefix( [execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR if valid_path: if os.path.isfile(execscript_location): ret = shutil.copytree( os.path.dirname(execscript_location), os.path.dirname(current_test.execscriptfile)) else: raise Exception( "Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR) else: raise Exception( "Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing " "exec scripts") os.chmod(current_test.execscriptfile, 0744) test_logger.info("Execution script copied successfully") save_test(current_test.testid, current_test) test_logger.info("Test setup complete") lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: lctx.error(e) if test_logger: test_logger.error(e) return "ERROR"
def trigger(self, *args): """ trigger starts in a thread, keep track of all triggers and they should complete in a specified time, otherwise signal a close. triggers are used for test setup, then starting test and then taking test into running state on agent. """ t = args[1] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) time.sleep(6) # Setting up test logger for capturing test life cycle on scheduler test_logger = LOG.gettestlogger(t2, "EXEC") test_logger.info("Test setup started") try: if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid: lctx.error("testobj not same") raise Exception("test trigger error", t2.testobj.TestInputData.testid) # Sending DAYTONA_SETUP_TEST command on execution host, on receiving this command agent will perform basic # test setup by creating test directories for saving log files and it will copy execution script in test # directory for starting execution ip = t.testobj.TestInputData.exechostname retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_SETUP_TEST", serialize_str + ",EXEC")) lctx.debug(retsend) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Test Setup Failure : Test - ", t2.testobj.TestInputData.testid) test_logger.info("Test setup complete on exec host " + ip) # Triggering test setup on all stat hosts for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break test_logger.info("Starting test setup on stat host " + s) lctx.debug(s.strip()) p = self.CPORT # Sending hearbeat message to check whether stat host is up or not retsend = self.cl.send( s, p, self.ev.construct("DAYTONA_HEARTBEAT", "")) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "ALIVE": raise Exception( "Remove host not avaliable - No Heartbeat ", t2.testobj.TestInputData.testid) else: test_logger.info("Hearbeat received from stat host " + s) # Trigger DAYTONA_HANDSHAKE to verify that both agent and scheduler are able to communicate with # each other on custom daytona ports retsend = self.cl.send( s, p, self.ev.construct( "DAYTONA_HANDSHAKE", "handshake1," + self.HOST + "," + str(self.PORT) + "," + str(t2.testobj.TestInputData.testid) + "," + s)) lctx.debug(retsend) if retsend == "SUCCESS": alive = True server.serv.registered_hosts[s] = s addr = socket.gethostbyname(s) server.serv.registered_hosts[addr] = addr test_logger.info( "Handshake successfull with agent on stat host " + s) else: raise Exception( "Unable to handshake with agent on stats host:" + s, t2.testobj.TestInputData.testid) else: raise Exception("Stat host " + s + " not avaliable - No Heartbeat") # Trigger test setup on stat hosts, this will create test directory for saving log files retsend = self.cl.send( s.strip(), p, self.ev.construct("DAYTONA_SETUP_TEST", serialize_str + ",STAT")) lctx.debug(retsend) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Test Setup Failure : Test - ", t2.testobj.TestInputData.testid) test_logger.info("Test setup complete on stat host " + s) # Trigger the start of test on exec host retsend = self.cl.send( ip, self.CPORT, self.ev.construct( "DAYTONA_START_TEST", str(t2.testobj.TestInputData.testid) + ",EXEC")) lctx.debug(retsend) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("test trigger error", t2.testobj.TestInputData.testid) else: raise Exception("Failed to start Test : ", t2.testobj.TestInputData.testid) test_logger.info("Test started on exec host " + ip) # Trigger the start of test on STAT hosts (This is for initiating system metric data collection) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break p = self.CPORT s = s.strip() retsend = self.cl.send( s, p, self.ev.construct( "DAYTONA_START_TEST", str(t2.testobj.TestInputData.testid) + ",STAT")) lctx.debug(retsend) if retsend and len(retsend.split(",")) > 1: if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception("Failed to start test on STAT host : ", s) else: raise Exception("Failed to start test on STAT host : ", s) test_logger.info("Test started on stat host " + s) # Get status from exec host retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_GET_STATUS", str(t2.testobj.TestInputData.testid))) if retsend and len(retsend.split(",")) > 1: if "RUNNING" == retsend.split( ",")[1] or "MONITOR_ON" == retsend.split(",")[1]: # update from setup to running lctx.debug("Updating test status to running in DB") t.updateStatus("setup", "running") now = time.time() tstr = str( time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now))) t.updateStartTime(tstr) self.dispatchQ__lock.acquire() del self.dispatch_queue[ t.testobj.TestInputData.frameworkid] self.dispatchQ__lock.release() self.lock.acquire() self.running_tests[t.testobj.TestInputData.frameworkid] = t self.lock.release() else: lctx.error( "Unable to determine status, testmon will garbage collect this testid" ) # Garbage collect testid in runnning state that fails to give status # Killing of threads on client host and remove from list with status=fail is done in testmon except Exception as e: # If any trigger fails, then abort test startup with error lctx.error(e) test_logger.error(e) lctx.error("ERROR : Unknown trigger error : " + str(t.testobj.TestInputData.testid)) t.updateStatus("setup", "failed") lctx.debug(traceback.print_exc()) lctx.error("Removing Test " + str(t.testobj.TestInputData.testid) + " from dispatch Q") self.dispatchQ__lock.acquire() del self.dispatch_queue[t.testobj.TestInputData.frameworkid] self.dispatchQ__lock.release() # This will abort test and perform cleanup on exec host with trigger was successful on exec host retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_ABORT_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) test_logger.info("Test aborted on exec host " + ip) # On all other stat hosts we send cleanup in case trigger was successful on any stat host for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct("DAYTONA_CLEANUP_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) test_logger.info("Test abort on stat host " + s) LOG.removeLogger(t2) return "FAILED" return "SUCCESS"
#!/usr/bin/env python # -*- coding:cp949 -*- import testobj tobj = testobj.testDefn() tobj.construct(1112)
def process_results(self, *args): """ This procedure is called by testmon as seperate thread when test execution ends or test timeout occur on agent. """ t = args[1] status = args[2] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) # Setting up test logger for capturing test life cycle on scheduler test_logger = LOG.gettestlogger(t2, "EXEC") test_logger.info("Test execution completes, preocessing test results") try: if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid: lctx.error("testobj not same") raise Exception("Test objects do not match : ", t2.testobj.TestInputData.testid) # set test status to collating if t.testobj.TestInputData.timeout_flag: t.updateStatus("timeout", "collating") else: t.updateStatus("completed", "collating") ip = t.testobj.TestInputData.exechostname lctx.debug(status) if status in ["completed", "timeout"]: # Initiate instance of ProcessOutputFiles for docker and top outout file processing ptop = process_files.ProcessOutputFiles( LOG.getLogger("processTop", "DH")) lctx.info("SENDING results.tgz download to : " + ip + ":" + str(self.CPORT)) # send file download command to exec host (no need to send stop test as this procedure is invoked due # to test end on exec host) retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_FILE_DOWNLOAD", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error("Error downloading LOGS from " + ip + " : " + retsend) test_logger.error("Error downloading LOGS from " + ip + " : " + retsend) else: test_logger.info( "Logs download successfull from exec host " + ip) # copy results files from exec to daytona file system and untar results try: lctx.debug("Untar file : " + t2.testobj.TestInputData.exec_results_path + "results.tgz to location : " + t2.testobj.TestInputData.exec_results_path + "/../") common.untarfile( t2.testobj.TestInputData.exec_results_path + "/results.tgz", t2.testobj.TestInputData.exec_results_path + "/../") except Exception as e: lctx.error("Error in untar EXEC host results") test_logger.error("Error in untar EXEC host results") lctx.error(e) # process top and docker stat files downloaded from exec host ptop_ret = ptop.process_output_files( t2.testobj.TestInputData.stats_results_path[ip] + "sar/") lctx.debug(ptop_ret + " : " + t2.testobj.TestInputData.stats_results_path[ip]) test_logger.info( "Exec host logs extracted and processed succesfully") # send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_FINISH_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) test_logger.info("Test END successfull on exec host " + ip) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break # stop stats monitors on req hosts # any host that blocks stop monitor blocks the scheduling for the FW p = self.CPORT try: # Send DAYTONA_STOP_TEST on all agent hosts to stop SAR data collection after test finish on # exec host. This message is required to tell stat hosts that test execution is finished on # exec host. Upon receiving this message on stat host, agent will change test state to TESTEND # and then other SAR data collection thread will stop writing log files for this test. lctx.info("Stopping test on stat host : " + s) retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_STOP_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error("Failed to stop test on stat host " + s + " : " + retsend) test_logger.error( "Failed to stop test on stat host " + s + " : " + retsend) else: test_logger.info("Test stopped on stat host " + s) # send file download command to stat host lctx.info("Sending results.tgz download to :" + s.strip() + ":" + str(p)) retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_FILE_DOWNLOAD", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error("Error downloading STATS from " + s.strip() + ":" + retsend) test_logger.error("Error downloading STATS from " + s.strip() + ":" + retsend) else: test_logger.info( "Logs downloaded from stat host " + s) # copy results files from stat host to daytona file system and untar results lctx.debug( "Untar file : " + t2.testobj.TestInputData.stats_results_path[s] + "results.tgz to location : " + t2.testobj.TestInputData.stats_results_path[s] + "/../") common.untarfile( t2.testobj.TestInputData.stats_results_path[s] + "/results.tgz", t2.testobj.TestInputData.stats_results_path[s] + "/../") # process top and docker stat files downloaded from stat host ptop_ret = ptop.process_output_files( t2.testobj.TestInputData.stats_results_path[s] + "sar/") lctx.debug( ptop_ret + " : " + t2.testobj.TestInputData.stats_results_path[s]) test_logger.info( "Stat host " + s + " logs extracted and processed succesfully") # send DAYTONA_FINISH_TEST to exec host for finishing and test cleanup retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_FINISH_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) test_logger.info("Test END successfull on stat host " + s) except Exception as e: # Just continue with other stat hosts if any exception occurs while working on any particular # host (Continue only when something goes wrong with stat host, because we still want to # download logs from other stat hosts) lctx.error(e) test_logger.error(e) continue except Exception as e: # Throw an error if anything goes wrong with finishing test on exec host and set test state to failed lctx.error("Error in processing results") lctx.error(e) test_logger.error("Error in processing results") test_logger.error(e) t.updateStatus("collating", "failed") # updating test state to timeout clean if test terminated due to timeout else setting it to finished clean if t.testobj.TestInputData.timeout_flag: t.updateStatus("collating", "timeout clean") else: t.updateStatus("collating", "finished clean") now = time.time() tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now))) # update test end time in database t.updateEndTime(tstr) f = None # Formatting email with results.csv details to send it to CC list if user has mentioned in test details # (admin need to smtp server for this functionality to work, smtp server details need to be # mentioned in config.sh) try: f = open(t2.testobj.TestInputData.exec_results_path + "/results.csv") except IOError as e: lctx.debug("File results.csv not found") pass to = t.testobj.TestInputData.email htmlfile = '<table>' if f: reader = csv.reader(f) rownum = 0 for row in reader: if rownum == 0: htmlfile += '<tr>' for column in row: htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>' htmlfile += '</tr>' else: htmlfile += '<tr>' for column in row: htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>' htmlfile += '</tr>' rownum += 1 f.close() htmlfile += '</table>' host_ip = "http://" + common.get_local_ip( ) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid) subject = "Test {} completed successfully".format( t.testobj.TestInputData.testid) mail_content = "<BR> Test id : {} <BR> Framework : {} <BR> Title : {} <BR>".format( t.testobj.TestInputData.testid, t.testobj.TestInputData.frameworkname, t.testobj.TestInputData.title) mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>Purpose : {} <BR><BR> Creation time : {} <BR>Start time : {} <BR>End " \ "time : {} <BR>".format(t.testobj.TestInputData.purpose, t.testobj.TestInputData.creation_time, t.testobj.TestInputData.start_time, t.testobj.TestInputData.end_time) mail_content = mail_content + "<BR>Your test executed successfully. <BR>Results (Contents of results.csv)<BR>" mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>" + htmlfile + "<BR>" mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "Link:" mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>' try: common.send_email(subject, to, mail_content, "", lctx, cfg.email_user, cfg.email_server, cfg.smtp_server, cfg.smtp_port) except Exception as e: lctx.error("Mail send error") LOG.removeLogger(t) return "SUCCESS"
def process_results(self, *args): # set test status completed # call stop monitors # send prepare results command to exec # set test status collating # copy results files from exec # copy files from each mon # set test status finished # remove test from running Q t = args[1] status = args[2] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) try: if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid: lctx.error("testobj not same") raise Exception("Test objects do not match : ", t2.testobj.TestInputData.testid) ip = t.testobj.TestInputData.exechostname lctx.debug(status) if status in ["completed", "timeout"]: retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_STOP_MONITOR", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_STOP_MONITOR failed : ", t2.testobj.TestInputData.testid) # get statistics hosts for s in t.testobj.TestInputData.stathostname.split(','): # stop stats monitors on req hosts # any host that blocks stop monitor blocks the scheduling for the FW if len(s.strip()) == 0: break p = self.CPORT try: retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_STOP_MONITOR", str(t2.testobj.TestInputData.testid))) except: continue lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_STOP_MONITOR failed : ", t2.testobj.TestInputData.testid) if t.testobj.TestInputData.timeout_flag: t.updateStatus("timeout", "collating") else: t.updateStatus("completed", "collating") ptop = process_top.ProcessTop(LOG.getLogger( "processTop", "DH")) # todo : avoid send client its own ip lctx.info("SENDING results.tgz download to : " + ip + ":" + str(self.CPORT)) retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_FILE_DOWNLOAD", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_FILE_DOWNLOAD failed :", t2.testobj.TestInputData.testid) try: lctx.debug("Untar file : " + t2.testobj.TestInputData.exec_results_path + "results.tgz to location : " + t2.testobj.TestInputData.exec_results_path + "/../") common.untarfile( t2.testobj.TestInputData.exec_results_path + "/results.tgz", t2.testobj.TestInputData.exec_results_path + "/../") except Exception as e: lctx.error("Error in untar EXEC host results") lctx.error(e) raise Exception("test result processing error", t2.testobj.TestInputData.testid) ptop_ret = ptop.process_top_output( t2.testobj.TestInputData.stats_results_path[ip] + "sar/") lctx.debug(ptop_ret + " : " + t2.testobj.TestInputData.stats_results_path[ip]) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break lctx.info("Downloading stats from STATS self.HOSTS : " + s) lctx.info(s) # stop stats monitors on req hosts # any host that blocks stop monitor blocks the scheduling for the FW p = self.CPORT lctx.info("Sending results.tgz download to :" + s.strip() + ":" + str(p)) try: retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_FILE_DOWNLOAD", str(t2.testobj.TestInputData.testid))) except: continue lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error("Error downloading STATS from " + s.strip() + ":" + retsend) raise Exception( "Daytona command DAYTONA_FILE_DOWNLOAD failed :", t2.testobj.TestInputData.testid) try: lctx.debug( "Untar file : " + t2.testobj.TestInputData.stats_results_path[s] + "results.tgz to location : " + t2.testobj.TestInputData.stats_results_path[s] + "/../") common.untarfile( t2.testobj.TestInputData.stats_results_path[s] + "/results.tgz", t2.testobj.TestInputData.stats_results_path[s] + "/../") except Exception as e: lctx.error("Error in untar STAT host " + s + " results") lctx.error(e) raise Exception("test result processing error", t2.testobj.TestInputData.testid) ptop_ret = ptop.process_top_output( t2.testobj.TestInputData.stats_results_path[s] + "sar/") lctx.debug(ptop_ret + " : " + t2.testobj.TestInputData.stats_results_path[s]) # todo : invoke other scripts to transform results and update DB except Exception as e: lctx.error("Error in processing results") lctx.error(e) t.updateStatus("collating", "failed") try: retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_FINISH_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break p = self.CPORT lctx.debug("self.HOST : " + s.strip()) lctx.debug("PORT to send CLEANUP & FINISH : " + str(p)) try: retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_FINISH_TEST", str(t2.testobj.TestInputData.testid))) except: pass lctx.debug(retsend) except Exception as e: lctx.error("Error in processing results") t.updateStatus("collating", "failed") if t.testobj.TestInputData.timeout_flag: t.updateStatus("collating", "timeout clean") else: t.updateStatus("collating", "finished clean") now = time.time() tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now))) t.updateEndTime(tstr) f = None try: f = open(t2.testobj.TestInputData.exec_results_path + "/results.csv") except IOError as e: lctx.debug("File results.csv not found") pass to = t.testobj.TestInputData.email htmlfile = '<table>' if f: reader = csv.reader(f) rownum = 0 for row in reader: if rownum == 0: htmlfile += '<tr>' for column in row: htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>' htmlfile += '</tr>' else: htmlfile += '<tr>' for column in row: htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>' htmlfile += '</tr>' rownum += 1 f.close() htmlfile += '</table>' host_ip = "http://" + common.get_local_ip( ) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid) subject = "Test {} completed successfully".format( t.testobj.TestInputData.testid) mail_content = "<BR> Test id : {} \ <BR> Framework : {} \ <BR> Title : {} <BR>".format( t.testobj.TestInputData.testid, t.testobj.TestInputData.frameworkname, t.testobj.TestInputData.title) mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>Purpose : {} <BR> \ <BR> Creation time : {} \ <BR>Start time : {} \ <BR>End time : {} <BR>".format( t.testobj.TestInputData.purpose, t.testobj.TestInputData.creation_time, t.testobj.TestInputData.start_time, t.testobj.TestInputData.end_time) mail_content = mail_content + "<BR>Your test executed successfully. \ <BR>Results (Contents of results.csv)<BR>" mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>" + htmlfile + "<BR>" mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "Link:" mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>' try: common.send_email(subject, to, mail_content, "", lctx, cfg.email_user, cfg.email_server, cfg.smtp_server, cfg.smtp_port) except: lctx.error("Mail send error") return "SUCCESS"
def testmon(self, *mon): """ Testmon continuously monitors all the running test. It keeps on checking test status on a exec host where execution script is running. If anything goes wrong with the test execution, this thread trigger termination actions for this test. It also trigger graceful test termination and logs collection when test finishes on exec host """ process_results_threads = defaultdict() while True: d = "TSMON [R] : |" remove = False error = False # Continuously iterate over running test list for checking test status for k in self.running_tests: if self.running_tests[k] is not None: t = self.running_tests[k] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) # Initiating test logger for capturing test life cycle on scheduler, all logs are logged in # file <testid>.log test_logger = LOG.gettestlogger(t2, "EXEC") if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid: lctx.error("testobj not same") t.updateStatus("running", "failed") remove = True break # out of for loop try: # Send DAYTONA_GET_STATUS message on exec host mentioned in test for checking test status ret = self.cl.send( t.testobj.TestInputData.exechostname, self.CPORT, self.ev.construct( "DAYTONA_GET_STATUS", str(t2.testobj.TestInputData.testid))) status = ret.split(",")[1] lctx.debug(status) test_logger.info("Test status : " + status) except Exception as e: lctx.debug(e) t.updateStatus("running", "failed") error = True break # out of for loop if status == "RUNNING": # If the test is in running state, then we need to verify that user hasn't terminated this # test from UI. If user has terminated then testmon will stop test execution on all exec host # and stat host found = checkTestRunning( t.testobj.TestInputData.testid) if not found: error = True break d = d + str(self.running_tests[k].testobj. TestInputData.testid) + "|" elif status in ["TESTEND", "TIMEOUT"]: # If test ends on exec host or if test timout occurs then trigger graceful shutdown of this test # Testmon invoke a new thread for this test for logs download and test cleanup from all hosts d = d + "*" + str(self.running_tests[k].testobj. TestInputData.testid) + "*|" if t.testobj.TestInputData.end_status == "running": lctx.debug(t.testobj.TestInputData.end_status) if status == "TIMEOUT": t.testobj.TestInputData.timeout_flag = True t.updateStatus("running", "timeout") else: t.updateStatus("running", "completed") # process_results download log files and perform cleanup on all other hosts pt = common.FuncThread( self.process_results, True, t, t.testobj.TestInputData.end_status) process_results_threads[ t.testobj.TestInputData.testid] = (pt, t) pt.start() remove = True break elif t.testobj.TestInputData.end_status == "collating" or t.testobj.TestInputData.end_status == "completed" or t.testobj.TestInputData.end_status == "finished clean": d = d + "*" + str(self.running_tests[k].testobj. TestInputData.testid) + "*|" else: remove = True t.updateStatus("running", "failed") lctx.error("ERROR : Unknown test status for : " + str(t.testobj.TestInputData.testid) + ":" + str(status)) break # out of for loop elif status.strip() in ["FAILED", "TESTNA"]: # Test termination if test fails or test is not even running on the host if status.strip() == "FAILED": error = True elif status.strip() in ["ABORT", "TESTNA"]: remove = True t.updateStatus("", "failed") lctx.error("TEST " + status.strip() + " : Cleaning test from running queue") break # out of for loop else: # Test termination on receiving any unknown test state remove = True t.updateStatus("running", "failed") lctx.error("ERROR : Unknown test status for : " + str(t.testobj.TestInputData.testid) + ":" + str(status)) break # out of for loop lctx.info(d) d = "" # Two modes of test termination: if error: # If error is set then testmon will perform below steps: # 1. Send test ABORT on exec host if is alive, this will stop execution script, perform logs cleanup and # test termination on the host # 2. Send test cleanup on all other stat host for performing logs cleanup and test termination on the host # 3. Remove test from the scheduler running queue retsend = None test_logger.error("Bad test status " + status + " - Terminating test") ip = t.testobj.TestInputData.exechostname try: retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_HEARTBEAT", "")) except: pass if retsend and retsend.split(",")[1] == "ALIVE": retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_ABORT_TEST", str(t.testobj.TestInputData.testid))) test_logger.error("Test Aborted on exec host " + ip) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break try: retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct("DAYTONA_HEARTBEAT", "")) except: pass if retsend and retsend.split(",")[1] == "ALIVE": retsend = self.cl.send( s.strip(), self.CPORT, self.ev.construct( "DAYTONA_CLEANUP_TEST", str(t.testobj.TestInputData.testid))) test_logger.error("Test Aborted on stat host " + s) self.lock.acquire() for k in self.running_tests: if self.running_tests[ k].testobj.TestInputData.testid == t.testobj.TestInputData.testid: lctx.debug("removing entry for this test") rt = self.running_tests.pop(k) break if k in self.running_tests: del self.running_tests[k] self.lock.release() if remove: # If remove flag is set, then testmon will only delete this test from the running queue of scheduler self.lock.acquire() for k in self.running_tests: if self.running_tests[ k].testobj.TestInputData.testid == t.testobj.TestInputData.testid: lctx.debug("removing entry for this test") rt = self.running_tests.pop(k) break if k in self.running_tests: del self.running_tests[k] self.lock.release() time.sleep(2)
def execute(self, command, paramcsv, actionID): """ This function maps daytona command with actual procedure which need to executed upon receiving a particular message. This mapping is saved in action.map file and procedures are implemented in action.py Upon mapping with actual procedure this routine spawns a new thread for executing that procedure seperately Below are some other action performed in this procedure : # based on SYNCFLAG release from here # send actionID for currently being executed action based on this we can stream resp # keep exec details over time in a buffer with actionID mapped # send actionID NULL and hold return till exec is complete """ module = self.conf.actionMap[command.strip()].split(".")[0] function = self.conf.actionMap[command.strip()].split(".")[1] sync = self.conf.actionMap[command.strip()].split(".")[2] t2 = testobj.testDefn() hosttype = None if command == "DAYTONA_START_TEST": testid = int(paramcsv.split(",")[0]) hosttype = paramcsv.split(",")[1] current_test = action.get_test(testid) if current_test: t2 = current_test.tobj m = __import__(module) f = getattr(m, function) if sync == "T": # wait for func to complete and return the ret self.lctx.debug("Executing SYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) ret = f(self, self, command, paramcsv, actionID, sync) self.lctx.debug("ACTION completed for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) if command == "DAYTONA_CLI": return "actionID=" + str( actionID) + "%" + ret + "%" + "SYNC EXEC" else: return "actionID=" + str( actionID) + "," + ret + "," + "SYNC EXEC" else: self.lctx.debug("Executing ASYNC ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) t1 = common.FuncThread(f, True, self, command, paramcsv, actionID, sync) if hosttype == "EXEC": x = (t1, actionID, t2, time.time()) self.lock.acquire() self.async_actions.append(x) self.lctx.debug("async_actions size :" + str(len(self.async_actions))) self.lock.release() t1.start() self.lctx.debug("Executing ACTION for " + command.strip() + " : " + self.conf.actionMap[command.strip()] + ":" + str(actionID)) return "actionID=" + str( actionID) + "," + "SUCCESS," + "ASYNC EXEC"
def setupTest(self, *args): """ Test setup is called when scheduler send "DAYTONA_SETUP_TEST" message to agent. In this procedure agent create all necessary file system path string and update in test object. After creating file path string it execute command for making all these file system directories so that agent can later save SAR data. On exec host, it copies execution script from Execscript folder to test specific directory in order to keep execution script seperate in case of multiple test execution :param self: :param args: tuple of arguments containing obj, command, parameter sent by scheduler to agent for this command, actionID and sync flag to denote if we need to execute this procedure in sync or async mode :return: SUCCESS in case everything goes well otherwise it throws ERROR """ (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) test_serialized = params.split(",")[0] host_type = params.split(",")[1] t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test = get_test(t2.testobj.TestInputData.testid) test_logger = None try: if current_test: test_logger = LOG.gettestlogger(current_test, "STAT") lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START") test_logger.info("Test setup started") current_test.tobj = testobj.testDefn() current_test.tobj = t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) shutil.rmtree(dir, ignore_errors=True) prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) + "/results/" if host_type == "EXEC": current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application" current_test.statsdir = prefix + current_test.stathostip + "/sar/" current_test.resultsdir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/results" current_test.archivedir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/" if host_type == "EXEC": common.createdir(current_test.execdir, self.lctx) common.createdir(current_test.logdir, self.lctx) common.createdir(current_test.resultsdir, self.lctx) common.createdir(current_test.statsdir, self.lctx) test_logger.info("Test directory created") if host_type == "EXEC": execscript = current_test.tobj.testobj.TestInputData.execution_script_location lctx.debug("TEST SETUP : " + str(execscript)) current_test.execscriptfile = current_test.execdir + "/" + execscript lctx.debug(current_test.execscriptfile) # check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at # this location execscript_location = EXEC_SCRIPT_DIR + execscript execscript_location = os.path.realpath(execscript_location) valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR if valid_path: if os.path.isfile(execscript_location): ret = shutil.copytree(os.path.dirname(execscript_location), os.path.dirname(current_test.execscriptfile)) else: raise Exception( "Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR) else: raise Exception( "Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing " "exec scripts") os.chmod(current_test.execscriptfile, 0744) test_logger.info("Execution script copied successfully") save_test(current_test.testid, current_test) test_logger.info("Test setup complete") lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: lctx.error(e) if test_logger: test_logger.error(e) return "ERROR"