def checkTestRunning(testid): lctx = LOG.getLogger("dblog", "DH") cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") db = dbaccess.DBAccess(cfg, LOG.getLogger("dblog", "DH")) check = db.query( """SELECT COUNT(*) FROM CommonFrameworkSchedulerQueue where testid=%s""", (testid, ), False, False) db.close() if check[0] == 0: return False else: return True
def updateStatus(self, curStatus, newStatus): """ Update test status from curStatus to newStatus in database for a given test. """ lctx = LOG.getLogger("dblog", "DH") lctx.debug("setting status from %s to %s" % (curStatus, newStatus)) if self.testobj.TestInputData.exec_results_path is not None: test_logger = LOG.gettestlogger(self, "EXEC") test_logger.info("Setting test status from %s to %s" % (curStatus, newStatus)) update_res = self.db.query( """update TestInputData SET end_status = %s where testid=%s""", (newStatus, self.testobj.TestInputData.testid), False, True) self.testobj.TestInputData.end_status = newStatus update_res = self.db.query( """update CommonFrameworkSchedulerQueue SET state = %s, message = %s, state_detail = %s where testid = %s""", (newStatus, newStatus, newStatus, self.testobj.TestInputData.testid), False, True) if newStatus == "finished clean" or newStatus == "failed" or newStatus == "abort" or newStatus == "kill" or newStatus == "timeout clean": update_res = self.db.query( """delete from CommonFrameworkSchedulerQueue where testid=%s""", (self.testobj.TestInputData.testid, ), False, True) lctx.debug( "Deleted entry from CommonFrameworkSchedulerQueue because of failure for : " + str(self.testobj.TestInputData.testid)) return
def fileDownload(self, *args): """ On test completion, agent execute this procedure when it receive DAYTONA_FILE_DOWNLOAD message from scheduler. We create a TAR file called results.tgz and save it test location, then we send this file to scheduler and save it in scheduler side file system """ cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) testid = int(args[2]) current_test = get_test(testid) test_logger = None try: if current_test: test_logger = LOG.gettestlogger(current_test, "STAT") lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | START") lctx.debug("Preparing TAR file of system metric folder") test_logger.info("Preparing TAR file of system metric folder") common.make_tarfile(current_test.archivedir + "results.tgz", current_test.resultsdir + "/") dest = current_test.tobj.testobj.TestInputData.stats_results_path[current_test.stathostip] download_file = current_test.archivedir + "results.tgz" test_logger.info("Sending TAR file to daytona host") cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip()) lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: lctx.error(e) if test_logger: test_logger.error(e) return "ERROR"
def fileDownload(self, *args): cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) testid = int(args[2]) current_test = get_test(testid) test_logger = None try: if current_test: test_logger = LOG.gettestlogger(current_test, "STAT") lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | START") lctx.debug("Preparing TAR file of system metric folder") test_logger.info("Preparing TAR file of system metric folder") common.make_tarfile(current_test.archivedir + "results.tgz", current_test.resultsdir + "/") dest = current_test.tobj.testobj.TestInputData.stats_results_path[ current_test.stathostip] download_file = current_test.archivedir + "results.tgz" test_logger.info("Sending TAR file to daytona host") cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip()) lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: lctx.error(e) if test_logger: test_logger.error(e) return "ERROR"
def fileDownload(self, *args): cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) prm = args[2].split(',') lctx.debug(prm) test_serialized = prm[3].strip() t2 = testobj.testDefn() t2.deserialize(test_serialized) dest ="" if prm[4] == "RESULTS" : dest = t2.testobj.TestInputData.exec_results_path elif prm[4] == "STATS" : s = prm[5] lctx.debug(s) dest = t2.testobj.TestInputData.stats_results_path[s] else: dest = "" lctx.debug(prm[0]) lctx.debug(prm[1]) lctx.debug(prm[2]) lctx.debug(dest) try: cl.sendFile(prm[0].strip(), int(prm[1].strip()), prm[2].strip(), dest.strip()) except CommunicationError as e: lctx.error(e.value) return e.value return "SUCCESS"
def downloadTestLogs(testid): """ This procedure just send test life cycle log file to scheduler upon test cleanup. This file provide user information about test execution sequence on agent """ cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) current_test = get_test(testid) test_logger = None try: if current_test: test_logger = LOG.gettestlogger(current_test, "STAT") test_logger.info("Sending test log to daytona host") dest = current_test.tobj.testobj.TestInputData.stats_results_path[current_test.stathostip] download_file = current_test.agent_log_file cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip()) test_logger.info("Test log file transfer complete") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: lctx.error(e) if test_logger: test_logger.error(e) return "ERROR"
def fileDownload(self, *args): cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) testid = int(args[2]) current_test = get_test(testid) if current_test: if current_test.status is not "MONITOR_OFF": lctx.error("Invalid state for START TEST action : " + current_test.status) current_test.status = "FAILED" save_test(current_test.testid, current_test) return "ERROR" try: if current_test: lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | START") dest = current_test.tobj.testobj.TestInputData.stats_results_path[current_test.stathostip] download_file = current_test.archivedir + "results.tgz" cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip()) lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: if current_test: current_test.status = "FAILED" save_test(current_test.testid, current_test) lctx.error(e) return "ERROR"
def updateEndTime(self, timestr): lctx = LOG.getLogger("dblog", "DH") lctx.debug("setting end time to %s" % (timestr)) update_res = self.db.query( """update TestInputData SET end_time = %s where testid=%s""", (timestr, self.testobj.TestInputData.testid), False, True) self.testobj.TestInputData.start_time = timestr return
def __init__(self): """ Constructor initializes logger, config file reader and db handle """ self.lctx = LOG.getLogger("dblog", "DH") self.cfg = config.CFG("DaytonaHost", self.lctx) self.cfg.readCFG("config.ini") self.db = dbaccess.DBAccess(self.cfg, self.lctx)
def scheduler_handshake(current_test): cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) env = envelope.DaytonaEnvelope() ret = cl.send(current_test.serverip, current_test.serverport, env.construct("DAYTONA_HANDSHAKE", "handshake2")) if ret == "SUCCESS": return True else: return False
def checkTestRunning(testid): """ This function checks whether user has not initiated test termination from UI. CommonFrameworkSchedulerQueue table keep the list of all running test initiated by user from UI or CLI. When user terminate any running test from UI, daytona removes the entry of this test from CommonFrameworkSchedulerQueue table. This functions polls database to check if test is still present in the CommonFrameworkSchedulerQueue table """ lctx = LOG.getLogger("dblog", "DH") cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") db = dbaccess.DBAccess(cfg, LOG.getLogger("dblog", "DH")) check = db.query( """SELECT COUNT(*) FROM CommonFrameworkSchedulerQueue where testid=%s""", (testid, ), False, False) db.close() if check[0] == 0: return False else: return True
def updateStartTime(self, timestr): """ Update test execution start time in database """ lctx = LOG.getLogger("dblog", "DH") lctx.debug("setting start time to %s" % (timestr)) update_res = self.db.query( """update TestInputData SET start_time = %s where testid=%s""", (timestr, self.testobj.TestInputData.testid), False, True) self.testobj.TestInputData.start_time = timestr return
def __init__(self, db, cfg, lctx): self.dbinstance = db self.cfg = cfg self.testmap = db.tests_to_run self.cl = client.TCPClient(LOG.getLogger("clientlog", "DH")) self.ev = envelope.DaytonaEnvelope() self.HOST = common.get_local_ip() self.PORT = cfg.DHPORT self.CPORT = cfg.CPORT self.scheduler_thread = common.FuncThread(self.dispatch, True) self.testmon_thread = common.FuncThread(self.testmon, True) self.lctx = lctx
def scheduler_handshake(current_test): """ This procedure is a part of 2-way handshake between scheduler and agent. If agent receive handshake message from scheduler, then agent also send a handshake message to scheduler to check if agent can communicate with scheduler on scheduler port. This part is important as later we need to transfer log files to scheduler using scheduler port :param current_test: Test object :return: true if scheduler respond otherwise false """ cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) env = envelope.DaytonaEnvelope() ret = cl.send(current_test.serverip, current_test.serverport, env.construct("DAYTONA_HANDSHAKE", "handshake2")) if ret == "SUCCESS": return True else: return False
def __init__(self, db, cfg, lctx): """ Scheduler class constructor which initialize class variables and other threads """ self.dbinstance = db self.cfg = cfg self.testmap = db.tests_to_run self.cl = client.TCPClient(LOG.getLogger("clientlog", "DH")) self.ev = envelope.DaytonaEnvelope() self.HOST = common.get_local_ip() self.PORT = cfg.DHPORT self.CPORT = cfg.CPORT self.scheduler_thread = common.FuncThread(self.dispatch, True) self.testmon_thread = common.FuncThread(self.testmon, True) self.lctx = lctx
def updateStatus(self, curStatus, newStatus): lctx = LOG.getLogger("dblog", "DH") lctx.debug("setting status from %s to %s" % (curStatus, newStatus)) update_res = self.db.query( """update TestInputData SET end_status = %s where testid=%s""", (newStatus, self.testobj.TestInputData.testid), False, True) self.testobj.TestInputData.end_status = newStatus update_res = self.db.query( """update CommonFrameworkSchedulerQueue SET state = %s, message = %s, state_detail = %s where testid = %s""", (newStatus, newStatus, newStatus, self.testobj.TestInputData.testid), False, True) if newStatus == "finished clean" or newStatus == "failed" or newStatus == "abort" or newStatus == "kill" or newStatus == "timeout clean": update_res = self.db.query( """delete from CommonFrameworkSchedulerQueue where testid=%s""", (self.testobj.TestInputData.testid, ), False, True) lctx.debug( "Deleted entry from CommonFrameworkSchedulerQueue because of failure for : " + str(self.testobj.TestInputData.testid)) return
def downloadTestLogs(testid): cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) current_test = get_test(testid) test_logger = None try: if current_test: test_logger = LOG.gettestlogger(current_test, "STAT") test_logger.info("Sending test log to daytona host") dest = current_test.tobj.testobj.TestInputData.stats_results_path[ current_test.stathostip] download_file = current_test.agent_log_file cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip()) test_logger.info("Test log file transfer complete") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: lctx.error(e) if test_logger: test_logger.error(e) return "ERROR"
def handle(self): """ This handler is called everytime TCP server running on scheduler and agent receive any message. This handler perform different operation based on action received """ host = self.client_address[0] data = self.request.recv(8192) cur_thread = threading.current_thread() ev = data.split(":") serv.lctx.debug("Envelope contents : " + str(ev)) cmd = ev[1] msgid = ev[2] params = ev[3] serv.lctx.info(cmd) serv.lctx.debug(msgid) serv.lctx.debug(params) if cmd == "DAYTONA_HANDSHAKE": # Message received is a handshake message p = params.split(",") if p[0] == "handshake1": # if payload contains handshake1 then this message came from scheduler to agent. Agent does basic # env setup and send handshake2 message back to scheduler to verify connectivity from both ends serv.registered_hosts[host] = host addr = socket.gethostbyname(host) serv.registered_hosts[addr] = addr # Initialize current_test object with scheduler information and other env setup current_test = activeTest(0, None, None, None) current_test.stathostip = p[4] current_test.stathostport = self.server.server_address[1] current_test.serverip = p[1] current_test.testid = int(p[3]) current_test.serverport = int(p[2]) current_test.status = "SETUP" test_logger = LOG.init_testlogger(current_test, "STAT") if test_logger: current_test.agent_log_file = test_logger.handlers[ 0].baseFilename con = action.scheduler_handshake(current_test) if con: # if response received from scheduler then handshake is successfull action.action_lock.acquire() action.running_tests[int(p[3])] = current_test action.action_lock.release() response = "{}".format("SUCCESS") test_logger.info( "Handshake successfull with daytona host : " + current_test.serverip) else: # else agent is not able to talk to scheduler using scheduler port, return error response = "{}".format("ERROR") test_logger.error( "Handshake failed with daytona host : " + current_test.serverip) self.request.sendall(response) return else: # Payload contains handshake2, means agent sent it on scheduler and scheduler just send OK response # back to agent response = "{}".format("SUCCESS") self.request.sendall(response) return if host in serv.registered_hosts.keys() or cmd in ( "DAYTONA_HEARTBEAT", "DAYTONA_CLI"): # Only process daytona commands from registered daytona host which was registered after handshake if cmd == "DAYTONA_STREAM_END": serv.lctx.debug("End stream...") return if cmd == "DAYTONA_STREAM_START": # Handler for DAYTONA_STREAM_START message received on scheduler. Scheduler will setup environment # for writing logs in execution.log file for this particular test filepath = params + "/execution.log" d = os.path.dirname(filepath) if not os.path.exists(d): os.makedirs(d) f = open(filepath, 'wb') serv.lctx.debug(filepath) serv.lctx.debug("Receiving stream..." + filepath) response = "{}".format("STREAMFILEREADY") self.request.send(response) l = self.request.recv(8192) serv.lctx.debug(len(l)) while l: serv.lctx.debug("Receiving stream...") f.write(l) print l serv.lctx.debug(l) f.flush() l = self.request.recv(8192) if l == "DAYTONA_STREAM_END": serv.lctx.debug("receiving term string : ") break f.close() serv.lctx.debug("Done receiving stream into : " + filepath) return if cmd == "DAYTONA_FILE_UPLOAD": # Handler for uploading SAR data TAR file on scheduler p = params.split(",") serv.lctx.debug("SER SERVER : " + params) fn = p[0].split("/") fn.reverse() loc = p[1].strip() serv.lctx.debug("SER SERVER : " + loc) filepath = loc + fn[0] d = os.path.dirname(filepath) if not os.path.exists(d): os.makedirs(d) serv.lctx.debug("Receiving..." + filepath) response = "{}".format("FILEREADY") self.request.send(response) f = open(filepath, 'wb') l = self.request.recv(8192) serv.lctx.debug(len(l)) while l: serv.lctx.debug("Receiving...") f.write(l) f.flush() l = self.request.recv(8192) serv.lctx.debug(len(l)) f.close() serv.lctx.debug("Done receiving results : " + filepath) return if cmd == "DAYTONA_STOP_SERVER": # Currently not in use serverInstance.shutdown() serverInstance.server_close() response = "{}: {}".format(cur_thread.name, "Shutting down server") self.request.sendall(response) if len(self.act.async_actions) > 0: for pending in self.act.async_actions: (t1, actionID, tst, ts) = pending t1.stop() serv.lctx.debug( "DAYTONA_STOP_SERVER handler, Async action thread ended after stop : " + cur_thread.name) return # todo : Set server to shutdown state, reject all incomming reqs if this flag set wait for all threads # to shutdown (with timeout) gracefully shutdown before timeout, or force close beyond timeout # exResp = self.act.execute(cmd, params, msgid) if serv.actc is None: serv.actc = ActionCaller( LOG.getLogger("listenerlog", serv.role)) # Call execute routine for mapping this daytona command with actual procedure. exResp = serv.actc.execute(cmd, params, msgid) response = "{}: {}".format(cur_thread.name, exResp) self.request.sendall(response) if len(serv.actc.async_actions) > 0: serv.lctx.debug("Async action list : " + str(len(serv.actc.async_actions))) for pending in self.act.async_actions: (t1, actionID, tst, ts) = pending t1.join() serv.lctx.debug( "Async action thread ended after join : " + t1.name) else: serv.lctx.error( "Command recieved from unknown host before handshake") serv.lctx.error(host)
def daytonaCli(self, *args): (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) lctx = LOG.getLogger("scheduler-clilog", "DH") cli_param_map = pickle.loads(params) if len(cli_param_map) != 3: return "Error|Not enough arguments" user = cli_param_map['user'] password = cli_param_map['password'] cli_command = cli_param_map['param'].split("|")[0] cli_param = cli_param_map['param'].split("|")[1] lctx.debug("Host received CLI command : " + cli_command) db = dbCliHandle() auth = db.authenticate_user(user, password) if auth != "SUCCESS": return auth if cli_command == "get_frameworkid_arg": arglist = db.getFrameworkIdArgs(cli_param) return arglist elif cli_command == "get_framework_arg": arglist = db.getFrameworkArgs(cli_param) return arglist elif cli_command == "add_test": res = db.addTest(cli_param, user) return res elif cli_command == "add_run_test": res = db.addTest(cli_param, user, 'scheduled') if res: if res.split("|")[0] == 'Error': return res else: testid = res.split("|")[1] else: return "Error|Test add failed" res = db.runTest(testid, user) if res.split("|")[0] == 'SUCCESS': return "SUCCESS|" + testid else: return res elif cli_command == "run_test": res = db.runTest(cli_param, user) return res elif cli_command == "get_result": res = db.getResult(cli_param, user) return res elif cli_command == "get_test_by_id": res = db.getTestByID(cli_param) return res elif cli_command == "update_test": res = db.updateTest(cli_param, user) return res elif cli_command == "update_run_test": res = db.updateTest(cli_param, user, 'scheduled') if res: if res.split("|")[0] == 'Error': return res else: testid = res.split("|")[1] else: return "Error|Test update failed" res = db.runTest(testid, user) if res.split("|")[0] == 'SUCCESS': return "SUCCESS|" + testid else: return res else: return "Error|Unknown command received on server"
def process_results(self, *args): # set test status completed # call stop monitors # send prepare results command to exec # set test status collating # copy results files from exec # copy files from each mon # set test status finished # remove test from running Q t = args[1] status = args[2] serialize_str = t.serialize() t2 = testobj.testDefn() t2.deserialize(serialize_str) try: if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid: lctx.error("testobj not same") raise Exception("Test objects do not match : ", t2.testobj.TestInputData.testid) ip = t.testobj.TestInputData.exechostname lctx.debug(status) if status in ["completed", "timeout"]: retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_STOP_MONITOR", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_STOP_MONITOR failed : ", t2.testobj.TestInputData.testid) # get statistics hosts for s in t.testobj.TestInputData.stathostname.split(','): # stop stats monitors on req hosts # any host that blocks stop monitor blocks the scheduling for the FW if len(s.strip()) == 0: break p = self.CPORT try: retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_STOP_MONITOR", str(t2.testobj.TestInputData.testid))) except: continue lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_STOP_MONITOR failed : ", t2.testobj.TestInputData.testid) if t.testobj.TestInputData.timeout_flag: t.updateStatus("timeout", "collating") else: t.updateStatus("completed", "collating") ptop = process_top.ProcessTop(LOG.getLogger( "processTop", "DH")) # todo : avoid send client its own ip lctx.info("SENDING results.tgz download to : " + ip + ":" + str(self.CPORT)) retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_FILE_DOWNLOAD", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error(retsend) raise Exception( "Daytona command DAYTONA_FILE_DOWNLOAD failed :", t2.testobj.TestInputData.testid) try: lctx.debug("Untar file : " + t2.testobj.TestInputData.exec_results_path + "results.tgz to location : " + t2.testobj.TestInputData.exec_results_path + "/../") common.untarfile( t2.testobj.TestInputData.exec_results_path + "/results.tgz", t2.testobj.TestInputData.exec_results_path + "/../") except Exception as e: lctx.error("Error in untar EXEC host results") lctx.error(e) raise Exception("test result processing error", t2.testobj.TestInputData.testid) ptop_ret = ptop.process_top_output( t2.testobj.TestInputData.stats_results_path[ip] + "sar/") lctx.debug(ptop_ret + " : " + t2.testobj.TestInputData.stats_results_path[ip]) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break lctx.info("Downloading stats from STATS self.HOSTS : " + s) lctx.info(s) # stop stats monitors on req hosts # any host that blocks stop monitor blocks the scheduling for the FW p = self.CPORT lctx.info("Sending results.tgz download to :" + s.strip() + ":" + str(p)) try: retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_FILE_DOWNLOAD", str(t2.testobj.TestInputData.testid))) except: continue lctx.debug(retsend) if retsend.split(",")[1] != "SUCCESS": lctx.error("Error downloading STATS from " + s.strip() + ":" + retsend) raise Exception( "Daytona command DAYTONA_FILE_DOWNLOAD failed :", t2.testobj.TestInputData.testid) try: lctx.debug( "Untar file : " + t2.testobj.TestInputData.stats_results_path[s] + "results.tgz to location : " + t2.testobj.TestInputData.stats_results_path[s] + "/../") common.untarfile( t2.testobj.TestInputData.stats_results_path[s] + "/results.tgz", t2.testobj.TestInputData.stats_results_path[s] + "/../") except Exception as e: lctx.error("Error in untar STAT host " + s + " results") lctx.error(e) raise Exception("test result processing error", t2.testobj.TestInputData.testid) ptop_ret = ptop.process_top_output( t2.testobj.TestInputData.stats_results_path[s] + "sar/") lctx.debug(ptop_ret + " : " + t2.testobj.TestInputData.stats_results_path[s]) # todo : invoke other scripts to transform results and update DB except Exception as e: lctx.error("Error in processing results") lctx.error(e) t.updateStatus("collating", "failed") try: retsend = self.cl.send( ip, self.CPORT, self.ev.construct("DAYTONA_FINISH_TEST", str(t2.testobj.TestInputData.testid))) lctx.debug(retsend) for s in t.testobj.TestInputData.stathostname.split(','): if len(s.strip()) == 0: break p = self.CPORT lctx.debug("self.HOST : " + s.strip()) lctx.debug("PORT to send CLEANUP & FINISH : " + str(p)) try: retsend = self.cl.send( s.strip(), p, self.ev.construct( "DAYTONA_FINISH_TEST", str(t2.testobj.TestInputData.testid))) except: pass lctx.debug(retsend) except Exception as e: lctx.error("Error in processing results") t.updateStatus("collating", "failed") if t.testobj.TestInputData.timeout_flag: t.updateStatus("collating", "timeout clean") else: t.updateStatus("collating", "finished clean") now = time.time() tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now))) t.updateEndTime(tstr) f = None try: f = open(t2.testobj.TestInputData.exec_results_path + "/results.csv") except IOError as e: lctx.debug("File results.csv not found") pass to = t.testobj.TestInputData.email htmlfile = '<table>' if f: reader = csv.reader(f) rownum = 0 for row in reader: if rownum == 0: htmlfile += '<tr>' for column in row: htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>' htmlfile += '</tr>' else: htmlfile += '<tr>' for column in row: htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>' htmlfile += '</tr>' rownum += 1 f.close() htmlfile += '</table>' host_ip = "http://" + common.get_local_ip( ) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid) subject = "Test {} completed successfully".format( t.testobj.TestInputData.testid) mail_content = "<BR> Test id : {} \ <BR> Framework : {} \ <BR> Title : {} <BR>".format( t.testobj.TestInputData.testid, t.testobj.TestInputData.frameworkname, t.testobj.TestInputData.title) mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>Purpose : {} <BR> \ <BR> Creation time : {} \ <BR>Start time : {} \ <BR>End time : {} <BR>".format( t.testobj.TestInputData.purpose, t.testobj.TestInputData.creation_time, t.testobj.TestInputData.start_time, t.testobj.TestInputData.end_time) mail_content = mail_content + "<BR>Your test executed successfully. \ <BR>Results (Contents of results.csv)<BR>" mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>" + htmlfile + "<BR>" mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "Link:" mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>' try: common.send_email(subject, to, mail_content, "", lctx, cfg.email_user, cfg.email_server, cfg.smtp_server, cfg.smtp_port) except: lctx.error("Mail send error") return "SUCCESS"
else: print "Unknown response received from host" print response sys.exit() else: print "Error - Unknown response received from host" print retsend sys.exit() else: print "Error - No response received from host" sys.exit() if __name__ == "__main__": read_arguments() tcp_client = client.TCPClient(LOG.getLogger("tcpclientlog", "daytona-cli")) env = envelope.DaytonaEnvelope() check_host_hearbeat(tcp_client, env) if action == 0: define_test(tcp_client, env) elif action == 1: add_test(tcp_client, env) elif action == 2: update_test(tcp_client, env) elif action == 3: run_test(tcp_client, env) elif action == 5: get_result(tcp_client, env) else: print "Error - Invalid action" display_usage()
cfg.readCFG("config.ini") db = dbaccess.DBAccess(cfg, LOG.getLogger("dblog", "DH")) check = db.query( """SELECT COUNT(*) FROM CommonFrameworkSchedulerQueue where testid=%s""", (testid, ), False, False) db.close() if check[0] == 0: return False else: return True if __name__ == "__main__": # Port 0 means to select an arbitrary unused port lctx = LOG.getLogger("schedulerlog", "DH") cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") common.logger.ROLE = "DHOST" db = dbaccess.DaytonaDBmon(cfg, LOG.getLogger("dblog", "DH")) sch = Scheduler(db, cfg, LOG.getLogger("schedulerlog", "DH")) server.serv.role = "DH" ase_serv = server.serv() server.serv.lctx = LOG.getLogger("listenerlog", "DH") ser = server.serv.ThreadedTCPServer((common.get_local_ip(), sch.PORT), server.serv.ThreadedTCPRequestHandler) server.serv.serverInstance = ser ip, port = ser.server_address server.lctx = LOG.getLogger("listenerlog", "DH")
# -*- coding:cp949 -*- import threading import SocketServer import time from collections import defaultdict import server import config import common import testobj from logger import LOG if __name__ == "__main__": # Port 0 means to select an arbitrary unused port lctx = LOG.getLogger("agentlog", "Agent") cfg = config.CFG("Agent", lctx) cfg.readCFG("config.ini") HOST = common.get_local_ip() PORT = cfg.CPORT common.createdir(cfg.daytona_agent_root, lctx) server.serv.role = "Agent" base_serv = server.serv() #server.serv.lctx = LOG.getLogger("listenerlog","Agent") #ser = server.serv.ThreadedTCPServer((HOST, PORT), server.serv.ThreadedTCPRequestHandler) ser = base_serv.ThreadedTCPServer((HOST, PORT),
def __init__(self): self.lctx = LOG.getLogger("dblog", "DH") self.cfg = config.CFG("DaytonaHost", self.lctx) self.cfg.readCFG("config.ini") self.db = dbaccess.DBAccess(self.cfg, self.lctx)
def exec_cmd(cmd, daytona_cmd, sync, obj, actionid, current_test): lctx.debug("Execute cmd : " + cmd) sfile = None cl = None ######## if daytona_cmd == "DAYTONA_START_TEST": cl = client.TCPClient(LOG.getLogger("clientlog", "Agent")) (current_test.stream, sfile) = cl.stream_start( current_test.serverip, current_test.serverport, str(current_test.tobj.testobj.TestInputData.exec_log_path)) ######## if sfile is not None: sfile.flush() cthread = commandThread(cmd, daytona_cmd, sfile, current_test.execdir, current_test.testid) current_test.exec_thread = cthread cthread.start() (t, aid, tst, ts) = (None, None, None, None) if sync == "T": lctx.debug("Execute cmd in Sync ctx") cthread.join() if sfile is not None: sfile.flush() else: # async action entry in the server table (need this to check self alive below) for tp in obj.async_actions: if tp[1] == actionid: (t, aid, tst, ts) = tp lctx.debug("Execute cmd in asSync ctx : " + str(actionid)) timer_expire = False while True: lctx.debug("waiting for async action to complete : " + str(actionid)) if cthread.stdout is not None: lctx.debug("printting output of stream ") if sfile is not None: sfile.flush() if tst.testobj.TestInputData.timeout > 0: if time.time() - ts > tst.testobj.TestInputData.timeout: lctx.error( "Timer expired for this test, need to end this async action" ) timer_expire = True # todo : breakout of while after a timer event and exit after terminating thread # timeout is monitored outside in the scheduler server , for all async ops , the thread is paused from # there via a CMD (ENDTEST) if t.check() == False or cthread.is_alive( ) == False or timer_expire: if daytona_cmd == "DAYTONA_START_TEST": if cthread.is_alive(): exec_script_lock.acquire() if current_test.testid in exec_script_pid: p = exec_script_pid[current_test.testid] del exec_script_pid[current_test.testid] exec_script_lock.release() if p: os.killpg(p.pid, signal.SIGTERM) lctx.debug("end stream") cl.stream_end( current_test.serverip, current_test.serverport, str(current_test.tobj.testobj.TestInputData. exec_log_path), current_test.stream, sfile) # callback # removeactionid lctx.debug("Callback here removing item") obj.removeActionItem(actionid) break time.sleep(3) if daytona_cmd == "DAYTONA_START_TEST": # todo : verify the TEST END on filesystem OR Failure if timer_expire: current_test.status = "TIMEOUT" else: lctx.debug("Setting current test status to TESTEND") current_test.status = "TESTEND" lctx.debug(daytona_cmd + " END [" + str(actionid) + "]") if save_test(current_test.testid, current_test): return "SUCCESS" else: return "ERROR"
htmlfile = htmlfile + '<td width="70%">' + column + '</td>' htmlfile = htmlfile + '</tr>' rownum += 1 htmlfile = htmlfile + '</table>' subject = "Test SAMPLE completed successfully" mail_content = "<BR>Test id : {sample id} \ <BR> Framework : {sample framework} \ <BR> Title : {sample title}<BR><BR>" mail_content = mail_content + "<BR>==========================================================<BR><BR>" mail_content = mail_content + "<BR>Purpose : {sample}<BR> \ <BR>Creation time : {TIME} \ <BR>Start time : {TIME} \ <BR>End time : {TIME}<BR>" mail_content = mail_content + "<BR>Your test executed successfully.\ <BR>Results (Contents of results.csv)<BR>" mail_content = mail_content + "<BR>==========================================================<BR>" mail_content = mail_content + "<BR>" + htmlfile + "<BR>" lctx = LOG.getLogger("schedulerlog", "DH") curhost = "ip-172-31-19-107.us-west-2.compute.internal" smtp_server = "localhost" smtp_port = 25 common.send_email("test email with results", "*****@*****.**", mail_content, "", lctx, "ubuntu", curhost, smtp_server, smtp_port)
def daytonaCli(self, *args): """ This is daytona CLI command handler when scheduler receives any request from Daytona cli script to perform any task. Daytona cli provide ability to get framework definition, add test for particular framework, run test, update test details and fetch test results. User need to provide daytona credentials in order to execute commands using daytona CLI scripts """ (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) lctx = LOG.getLogger("scheduler-clilog", "DH") cli_param_map = pickle.loads(params) if len(cli_param_map) != 3: return "Error|Not enough arguments" # Retriveing username and password from CLI command for authentication user = cli_param_map['user'] password = cli_param_map['password'] # Retriveing actual daytona command and associated parameter from CLI command cli_command = cli_param_map['param'].split("|")[0] cli_param = cli_param_map['param'].split("|")[1] lctx.debug("Host received CLI command : " + cli_command) db = dbCliHandle() # Calling authenticate_user webservice for verifying username and password combination auth = db.authenticate_user(user, password) if auth != "SUCCESS": return auth # Invoke function based on CLI action mentioned on CLI command if cli_command == "get_frameworkid_arg": arglist = db.getFrameworkIdArgs(cli_param) return arglist elif cli_command == "get_framework_arg": arglist = db.getFrameworkArgs(cli_param) return arglist elif cli_command == "add_test": res = db.addTest(cli_param, user) return res elif cli_command == "add_run_test": res = db.addTest(cli_param, user, 'scheduled') if res: if res.split("|")[0] == 'Error': return res else: testid = res.split("|")[1] else: return "Error|Test add failed" res = db.runTest(testid, user) if res.split("|")[0] == 'SUCCESS': return "SUCCESS|" + testid else: return res elif cli_command == "run_test": res = db.runTest(cli_param, user) return res elif cli_command == "get_result": res = db.getResult(cli_param, user) return res elif cli_command == "get_test_by_id": res = db.getTestByID(cli_param) return res elif cli_command == "update_test": res = db.updateTest(cli_param, user) return res elif cli_command == "update_run_test": res = db.updateTest(cli_param, user, 'scheduled') if res: if res.split("|")[0] == 'Error': return res else: testid = res.split("|")[1] else: return "Error|Test update failed" res = db.runTest(testid, user) if res.split("|")[0] == 'SUCCESS': return "SUCCESS|" + testid else: return res else: return "Error|Unknown command received on server"
def handle(self): host = self.client_address[0] data ="" data = self.request.recv(8192) cur_thread = threading.current_thread() ev = data.split(":") serv.lctx.debug("Envelope contents : " + str(ev)) cmd = ev[1] msgid = ev[2] params = ev[3] serv.lctx.info(cmd) serv.lctx.debug(cmd) serv.lctx.debug(msgid) serv.lctx.debug(params) if cmd == "DAYTONA_HANDSHAKE": #todo : maintain a list of daytona host that this server talks to #only respond to the ones in the list serv.registered_hosts[host] = host addr = socket.gethostbyname(host) serv.registered_hosts[addr] = addr p = params.split(",") action.current_test.serverip = p[0] action.current_test.serverport = int(p[1]) action.current_test.status = "TESTSETUP" response = "{}".format("SUCCESS") self.request.sendall(response) return if host in serv.registered_hosts.keys() or cmd == "DAYTONA_HEARTBEAT": if cmd == "DAYTONA_STREAM_END": serv.lctx.debug("End stream...") return if cmd == "DAYTONA_STREAM_START": filepath = params+"/execution.log" d = os.path.dirname(filepath) if not os.path.exists(d): os.makedirs(d) f = open(filepath,'wb') serv.lctx.debug(filepath) serv.lctx.debug("Receiving stream..." + filepath) response = "{}".format("STREAMFILEREADY") self.request.send(response) l = self.request.recv(8192) serv.lctx.debug(len(l)) while (l): serv.lctx.debug("Receiving stream...") f.write(l) print l serv.lctx.debug(l) f.flush() l = self.request.recv(8192) if l == "DAYTONA_STREAM_END": serv.lctx.debug("receiving term string : ") break f.close() #response = "{}".format("SUCCESS") #self.request.sendall(response) serv.lctx.debug("Done receiving stream into : " + filepath) return if cmd == "DAYTONA_FILE_UPLOAD": p = params.split(",") serv.lctx.debug("SER SERVER : " + params) fn = p[0].split("/") fn.reverse() loc = p[1].strip() serv.lctx.debug("SER SERVER : " + loc) filepath = loc+fn[0] d = os.path.dirname(filepath) if not os.path.exists(d): os.makedirs(d) serv.lctx.debug("Receiving..." + filepath) response = "{}".format("FILEREADY") self.request.send(response) f = open(filepath,'wb') l = self.request.recv(8192) serv.lctx.debug(len(l)) while (l): serv.lctx.debug("Receiving...") f.write(l) f.flush() l = self.request.recv(8192) serv.lctx.debug(len(l)) f.close() serv.lctx.debug("Done receiving results : " + filepath) return if cmd == "DAYTONA_STOP_SERVER": serverInstance.shutdown() serverInstance.server_close() response = "{}: {}".format(cur_thread.name, "Shutting down server") self.request.sendall(response) if len(self.act.async_actions) > 0 : for pending in self.act.async_actions: (t1, actionID, tst, ts) = pending t1.stop() serv.lctx.debug("DAYTONA_STOP_SERVER handler, Async action thread ended after stop : " + cur_thread.name) return #todo : Set server to shutdown state, reject all incomming reqs if this flag set # wait for all threads to shutdown (with timeout) # gracefully shutdown before timeout, or force close beyond timeout #exResp = self.act.execute(cmd, params, msgid) if serv.actc == None: serv.actc = ActionCaller(LOG.getLogger("listenerlog", serv.role)) exResp = serv.actc.execute(cmd, params, msgid) response = "{}: {}".format(cur_thread.name, exResp) self.request.sendall(response) if len(serv.actc.async_actions) > 0 : serv.lctx.debug("Async action list : " + str(len(serv.actc.async_actions))) for pending in self.act.async_actions: (t1, actionID, tst, ts) = pending t1.join() serv.lctx.debug("Async action thread ended after join : " + t1.name) else: serv.lctx.error("Command recieved from unknown host before handshake") serv.lctx.error(host)
def __init__(self): serv.lctx = LOG.getLogger("listenerlog",serv.role) action.lctx = LOG.getLogger("actionlog", serv.role) #todo this integration has to be reviewed actc = ActionCaller(LOG.getLogger("actionlog", serv.role)) serv.registered_hosts={}
def construct(self, tid): import dbaccess import config lctx = LOG.getLogger("dblog", "DH") cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") self.db = dbaccess.DBAccess(cfg, LOG.getLogger("dblog", "DH")) self.testobj.TestInputData.testid = tid query_result = self.db.query( """select testid, frameworkid, start_time, end_time, end_status, end_detail, exechostname, stathostname, timeout, cc_list, title, purpose, creation_time from TestInputData where testid = %s""", (self.testobj.TestInputData.testid, ), False, False) (self.testobj.TestInputData.testid, self.testobj.TestInputData.frameworkid, self.testobj.TestInputData.start_time, self.testobj.TestInputData.end_time, self.testobj.TestInputData.end_status, self.testobj.TestInputData.end_detail, self.testobj.TestInputData.exechostname, self.testobj.TestInputData.stathostname, self.testobj.TestInputData.timeout, self.testobj.TestInputData.email, self.testobj.TestInputData.title, self.testobj.TestInputData.purpose, self.testobj.TestInputData.creation_time) = query_result lctx.debug(query_result) query_result = self.db.query( """select ha.hostname, hat.name, hat.shared, hat.execution, hat.statistics from HostAssociation ha join HostAssociationType hat on ha.hostassociationtypeid = hat.hostassociationtypeid where testid = %s and hat.frameworkid = %s""", (self.testobj.TestInputData.testid, self.testobj.TestInputData.frameworkid), True, False) for r in query_result: lctx.debug(r) if r[1] == 'statistics' and r[4] == 1: self.testobj.TestInputData.stathostname = r[ 0] + "," + self.testobj.TestInputData.stathostname elif r[1] == 'execution' and r[3] == 1: self.testobj.TestInputData.exechostname = r[ 0] + "," + self.testobj.TestInputData.exechostname self.testobj.TestInputData.stathostname = self.testobj.TestInputData.stathostname[: -1] self.testobj.TestInputData.exechostname = self.testobj.TestInputData.exechostname[: -1] lctx.debug(self.testobj.TestInputData.exechostname) lctx.debug(self.testobj.TestInputData.stathostname) query_result = self.db.query( """select * from TestArgs where testid = %s""", (self.testobj.TestInputData.testid, ), True, False) self.testobj.TestInputData.execScriptArgs = query_result lctx.debug(query_result) query_result = self.db.query( """select file_root, execution_script_location, frameworkname from ApplicationFrameworkMetadata where frameworkid = %s""", (self.testobj.TestInputData.frameworkid, ), False, False) (self.testobj.TestInputData.file_root, self.testobj.TestInputData.execution_script_location, self.testobj.TestInputData.frameworkname) = query_result lctx.debug(query_result) query_result = self.db.query( """select processname, delay, duration from ProfilerFramework where testid = %s and profiler = %s""", (self.testobj.TestInputData.testid, 'STRACE'), False, False) if query_result: self.testobj.TestInputData.strace = True (self.testobj.TestInputData.strace_process, self.testobj.TestInputData.strace_delay, self.testobj.TestInputData.strace_duration) = query_result query_result = self.db.query( """select processname, delay, duration from ProfilerFramework where testid = %s and profiler = %s""", (self.testobj.TestInputData.testid, 'PERF'), False, False) if query_result: (self.testobj.TestInputData.perf_process, self.testobj.TestInputData.perf_delay, self.testobj.TestInputData.perf_duration) = query_result