def __init__(self, ctrl, path="logs"): self.ctrl = ctrl self.path = path self.buffer = "" self.file = "" self.load_config() createdir(self.path)
def __init__(self, cfg, lctx): self.lctx = lctx self.db = DBAccess(cfg,lctx) self.cfg = cfg self.startMon() print "CFG in init :" print self.cfg.mysql_host common.createdir(cfg.daytona_dh_root, self.lctx) time.sleep(5) # wait for 5 secs for all recs to be loaded into map
def startMonitor( self, *args): (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) lctx.debug("startMonitor") p = params.split(",") test_serialized = p[0] s = p[1].strip() try : current_test.tobj = testobj.testDefn() t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test.tobj= t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid lctx.debug("TEST SETUP Monitor : " + str(current_test.tobj.testobj.TestInputData.testid)) #todo : indicate a status that says this host is monitor-active #current_test.status = "TESTSETUP" cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(current_test.tobj.testobj.TestInputData.testid) + "/results/" current_test.statsdir = prefix + s + "/sar/" current_test.archivedir = prefix common.createdir(cfg.daytona_agent_root, lctx) common.createdir(current_test.statsdir, lctx) execline = cfg.daytona_mon_path + "/sar_gather_agent.pl --daemonize --root-dir=" + current_test.statsdir lctx.info(execline) exec_cmd(execline, command, sync, obj, actionID) except Exception as e: lctx.error(e) lctx.error(traceback.print_exc()) return "ERROR" lctx.debug("Completed start monitor") return "SUCCESS"
import server import config import common import testobj from logger import LOG if __name__ == "__main__": # Port 0 means to select an arbitrary unused port lctx = LOG.getLogger("agentlog", "Agent") cfg = config.CFG("Agent", lctx) cfg.readCFG("config.ini") HOST = common.get_local_ip() PORT = cfg.CPORT common.createdir(cfg.daytona_agent_root, lctx) server.serv.role = "Agent" base_serv = server.serv() #server.serv.lctx = LOG.getLogger("listenerlog","Agent") #ser = server.serv.ThreadedTCPServer((HOST, PORT), server.serv.ThreadedTCPRequestHandler) ser = base_serv.ThreadedTCPServer((HOST, PORT), server.serv.ThreadedTCPRequestHandler) server.serv.serverInstance = ser ip, port = ser.server_address #server.lctx = LOG.getLogger("listenerlog", "Agent") # Start a thread with the server -- that thread will then start one
def setupTest(self, *args): (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) test_serialized = params.split(",")[0] host_type = params.split(",")[1] t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test = get_test(t2.testobj.TestInputData.testid) test_logger = None try: if current_test: test_logger = LOG.gettestlogger(current_test, "STAT") lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START") test_logger.info("Test setup started") current_test.tobj = testobj.testDefn() current_test.tobj = t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) shutil.rmtree(dir, ignore_errors=True) prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) + "/results/" if host_type == "EXEC": current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application" current_test.statsdir = prefix + current_test.stathostip + "/sar/" current_test.resultsdir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/results" current_test.archivedir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/" if host_type == "EXEC": common.createdir(current_test.execdir, self.lctx) common.createdir(current_test.logdir, self.lctx) common.createdir(current_test.resultsdir, self.lctx) common.createdir(current_test.statsdir, self.lctx) test_logger.info("Test directory created") # todo : check and validate if exec script is provided in expected format and # the file exists in that location if host_type == "EXEC": execscript = current_test.tobj.testobj.TestInputData.execution_script_location lctx.debug("TEST SETUP : " + str(execscript)) current_test.execscriptfile = current_test.execdir + "/" + execscript lctx.debug(current_test.execscriptfile) # check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at # this location execscript_location = EXEC_SCRIPT_DIR + execscript execscript_location = os.path.realpath(execscript_location) valid_path = os.path.commonprefix( [execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR if valid_path: if os.path.isfile(execscript_location): ret = shutil.copytree( os.path.dirname(execscript_location), os.path.dirname(current_test.execscriptfile)) else: raise Exception( "Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR) else: raise Exception( "Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing " "exec scripts") os.chmod(current_test.execscriptfile, 0744) test_logger.info("Execution script copied successfully") save_test(current_test.testid, current_test) test_logger.info("Test setup complete") lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: lctx.error(e) if test_logger: test_logger.error(e) return "ERROR"
def setupTest(self, *args): (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) test_serialized = params.split(",")[0] host_type = params.split(",")[1] t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test = get_test(t2.testobj.TestInputData.testid) if current_test: if current_test.status is not "INIT": lctx.error("Invalid state for TEST SETUP action : " + current_test.status) current_test.status = "ABORT" save_test(current_test.testid, current_test) return "ERROR" try: if current_test: lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START") current_test.tobj = testobj.testDefn() current_test.tobj = t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid lctx.debug("TEST SETUP : " + str(current_test.tobj.testobj.TestInputData.testid)) cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) shutil.rmtree(dir, ignore_errors=True) prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) + "/results/" if host_type == "EXEC": current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application" current_test.statsdir = prefix + current_test.stathostip + "/sar/" current_test.resultsdir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/results" current_test.archivedir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/" common.createdir(cfg.daytona_agent_root, self.lctx) if host_type == "EXEC": common.createdir(current_test.execdir, self.lctx) common.createdir(current_test.logdir, self.lctx) common.createdir(current_test.resultsdir, self.lctx) common.createdir(current_test.statsdir, self.lctx) # todo : check and validate if exec script is provided in expected format and # the file exists in that location if host_type == "EXEC": execscript = current_test.tobj.testobj.TestInputData.execution_script_location lctx.debug("TEST SETUP : " + str(execscript)) current_test.execscriptfile = current_test.execdir + "/" + execscript lctx.debug(current_test.execscriptfile) # check if execution script is present in '/tmp/ExecScripts/' - execute script only if it present at # this location execscript_location = EXEC_SCRIPT_DIR + execscript execscript_location = os.path.realpath(execscript_location) valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR if valid_path: if os.path.isfile(execscript_location): ret = copyfile(execscript_location, current_test.execscriptfile) else: raise Exception( "Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR) else: raise Exception( "Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing " "exec scripts") os.chmod(current_test.execscriptfile, 0744) current_test.status = "SETUP" save_test(current_test.testid, current_test) lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except shutil.Error as err: if current_test: lctx.error("error copying file : " + str(execscript) + " to " + str(current_test.execscriptfile)) current_test.status = "ABORT" save_test(current_test.testid, current_test) lctx.error(err) return "ERROR" except Exception as e: if current_test: current_test.status = "ABORT" save_test(current_test.testid, current_test) lctx.error(e) return "ERROR"
def mon(self, *args): #query all waiting state tests and load into to_schedule, this is for restart case (similarly for running) #todo : reconcile running and scheduled testids, if db altered externally restarted = True print "CFG in mon :" print self.cfg.mysql_host while True: self.db = DBAccess(self.cfg, self.lctx) d = "DBMON [W] : |" for k in self.tests_to_run: l = self.tests_to_run[k] for t in l : d = d + str(t.testobj.TestInputData.testid) + "|" self.lctx.info(d) d = "" query_result = None if restarted == True : query_result = self.db.query("""select testid from CommonFrameworkSchedulerQueue where state = %s or state = %s or state = %s or state = %s or state = %s or state = %s""", ("scheduled","waiting","setup","running","completed","collating"), True, False); #query_result = self.db.query("""select testid from TestInputData where end_status = %s or end_status = %s or end_status = %s or end_status = %s or end_status = %s or end_status = %s""", ("scheduled","waiting","setup","running","completed","collating"), True, False); restarted = False else: status = "scheduled" query_result = self.db.query("""select testid from CommonFrameworkSchedulerQueue where state = %s""", (status,), True, False); #query_result = self.db.query("""select testid from TestInputData where end_status = %s""", (status,), True, False); #reset all states to scheduled, mostly required in a restart case #all items that make to the DBMON [Q] will be in scheduled state for testid in query_result: to = testobj.testDefn() to.testobj.TestInputData.testid = testid[0] res = to.construct(testid[0]) res = to.updateStatus("*", "scheduled") d = "DBMON [Q] : " d = d + str(query_result) self.lctx.info(d) for testid in query_result: found = False for k in self.tests_to_run: #search across all FW l = self.tests_to_run[k] #list of tests ready to be run for t in l: self.lctx.debug(t.testobj.TestInputData.testid) self.lctx.debug(testid[0]) self.lctx.debug(k) if t.testobj.TestInputData.testid == testid[0]: self.lctx.debug("Test already present in runQ") found = True if(found == False): to = testobj.testDefn() to.testobj.TestInputData.testid = testid[0] #todo handle return status res = to.construct(testid[0]) #create required dirs and setup server side env prefix=self.cfg.daytona_dh_root +"/"+ to.testobj.TestInputData.frameworkname + "/" + str(to.testobj.TestInputData.testid) + "/" + "results" + "/" to.testobj.TestInputData.exec_results_path = prefix+to.testobj.TestInputData.exechostname + "/" to.testobj.TestInputData.exec_path = prefix+to.testobj.TestInputData.exechostname to.testobj.TestInputData.exec_log_path = prefix+to.testobj.TestInputData.exechostname+"/application" to.testobj.TestInputData.stats_results_path = defaultdict() for s in to.testobj.TestInputData.stathostname.split(','): to.testobj.TestInputData.stats_results_path[s.strip()] = prefix+s.strip()+"/" to.testobj.TestInputData.stats_results_path[to.testobj.TestInputData.exechostname] = prefix+to.testobj.TestInputData.exechostname+"/" common.createdir(to.testobj.TestInputData.exec_results_path, self.lctx) common.createdir(to.testobj.TestInputData.exec_path, self.lctx) common.createdir(to.testobj.TestInputData.exec_log_path, self.lctx) for s in to.testobj.TestInputData.stats_results_path: common.createdir(to.testobj.TestInputData.stats_results_path[s], self.lctx) res = to.updateStatus("scheduled", "waiting") sz = to.serialize() t2 = testobj.testDefn() t2.deserialize(sz) if t2.testobj.TestInputData.testid != to.testobj.TestInputData.testid : self.lctx.error("error in ser / dser") break #use a lock here self.lock.acquire() self.tests_to_run[to.testobj.TestInputData.frameworkid].append(to); self.lctx.debug("Adding : " + str(to.testobj.TestInputData.testid)) self.lock.release() self.db.close() if self.mon_thread[0].check() == False : return time.sleep(5) #todo : make this config item
def setupTest( self, *args): (obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) lctx.debug("setuptest") try : current_test.tobj = testobj.testDefn() t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test.tobj= t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid lctx.debug("TEST SETUP : " + str(current_test.tobj.testobj.TestInputData.testid)) current_test.status = "TESTSETUP" cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(current_test.tobj.testobj.TestInputData.testid) + "/results/" current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application" current_test.resultsdir = prefix current_test.statsdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/sar/" current_test.archivedir = prefix common.createdir(cfg.daytona_agent_root, self.lctx) common.createdir(current_test.execdir, self.lctx) common.createdir(current_test.logdir, self.lctx) common.createdir(current_test.resultsdir, self.lctx) common.createdir(current_test.statsdir, self.lctx) #todo : check and validate if exec script is provided in expected format and # the file exists in that location execscript = current_test.tobj.testobj.TestInputData.execution_script_location.split(":")[1] lctx.debug("TEST SETUP : " + str(execscript)) tmp = str(execscript).split("/") tmp.reverse() lctx.debug(tmp) filename = tmp[0] current_test.execscriptfile = current_test.execdir+"/"+filename lctx.debug(current_test.execscriptfile) except Exception as e: lctx.error(e) lctx.error(traceback.print_exc()) return "ERROR" try: ret = copyfile(execscript, current_test.execscriptfile) except shutil.Error as err: lctx.error("error copying file : " + str(execscript) + " to " + str(current_test.execscriptfile)) return "ERROR" try: os.chmod(current_test.execscriptfile, 0744) except: lctx.error("error setting perm file : " + str(current_test.execdir)) return "ERROR" #create dirs #get exec script name #cp the exec script #set exec perm #update cur test obj with exec script #exec any custom setup script lctx.debug("Completed setuptest") return "SUCCESS"
def setupTest(self, *args): """ Test setup is called when scheduler send "DAYTONA_SETUP_TEST" message to agent. In this procedure agent create all necessary file system path string and update in test object. After creating file path string it execute command for making all these file system directories so that agent can later save SAR data. On exec host, it copies execution script from Execscript folder to test specific directory in order to keep execution script seperate in case of multiple test execution :param self: :param args: tuple of arguments containing obj, command, parameter sent by scheduler to agent for this command, actionID and sync flag to denote if we need to execute this procedure in sync or async mode :return: SUCCESS in case everything goes well otherwise it throws ERROR """ (obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4]) test_serialized = params.split(",")[0] host_type = params.split(",")[1] t2 = testobj.testDefn() t2.deserialize(test_serialized) current_test = get_test(t2.testobj.TestInputData.testid) test_logger = None try: if current_test: test_logger = LOG.gettestlogger(current_test, "STAT") lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START") test_logger.info("Test setup started") current_test.tobj = testobj.testDefn() current_test.tobj = t2 current_test.testid = current_test.tobj.testobj.TestInputData.testid cfg = config.CFG("DaytonaHost", lctx) cfg.readCFG("config.ini") dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) shutil.rmtree(dir, ignore_errors=True) prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str( current_test.tobj.testobj.TestInputData.testid) + "/results/" if host_type == "EXEC": current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application" current_test.statsdir = prefix + current_test.stathostip + "/sar/" current_test.resultsdir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/results" current_test.archivedir = cfg.daytona_agent_root + "/" + \ current_test.tobj.testobj.TestInputData.frameworkname + "/" + \ str(current_test.tobj.testobj.TestInputData.testid) + "/" if host_type == "EXEC": common.createdir(current_test.execdir, self.lctx) common.createdir(current_test.logdir, self.lctx) common.createdir(current_test.resultsdir, self.lctx) common.createdir(current_test.statsdir, self.lctx) test_logger.info("Test directory created") if host_type == "EXEC": execscript = current_test.tobj.testobj.TestInputData.execution_script_location lctx.debug("TEST SETUP : " + str(execscript)) current_test.execscriptfile = current_test.execdir + "/" + execscript lctx.debug(current_test.execscriptfile) # check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at # this location execscript_location = EXEC_SCRIPT_DIR + execscript execscript_location = os.path.realpath(execscript_location) valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR if valid_path: if os.path.isfile(execscript_location): ret = shutil.copytree(os.path.dirname(execscript_location), os.path.dirname(current_test.execscriptfile)) else: raise Exception( "Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR) else: raise Exception( "Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing " "exec scripts") os.chmod(current_test.execscriptfile, 0744) test_logger.info("Execution script copied successfully") save_test(current_test.testid, current_test) test_logger.info("Test setup complete") lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE") return "SUCCESS" else: raise Exception("Invalid Test ID") except Exception as e: lctx.error(e) if test_logger: test_logger.error(e) return "ERROR"