def __init__(self, ctrl, path="logs"): self.ctrl = ctrl self.path = path self.buffer = "" self.file = "" self.load_config() createdir(self.path)
def __init__(self, cfg, lctx):
self.lctx = lctx
self.db = DBAccess(cfg,lctx)
self.cfg = cfg
self.startMon()
print "CFG in init :"
print self.cfg.mysql_host
common.createdir(cfg.daytona_dh_root, self.lctx)
time.sleep(5) # wait for 5 secs for all recs to be loaded into map
def startMonitor( self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
lctx.debug("startMonitor")
p = params.split(",")
test_serialized = p[0]
s = p[1].strip()
try :
current_test.tobj = testobj.testDefn()
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test.tobj= t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
lctx.debug("TEST SETUP Monitor : " + str(current_test.tobj.testobj.TestInputData.testid))
#todo : indicate a status that says this host is monitor-active
#current_test.status = "TESTSETUP"
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(current_test.tobj.testobj.TestInputData.testid) + "/results/"
current_test.statsdir = prefix + s + "/sar/"
current_test.archivedir = prefix
common.createdir(cfg.daytona_agent_root, lctx)
common.createdir(current_test.statsdir, lctx)
execline = cfg.daytona_mon_path + "/sar_gather_agent.pl --daemonize --root-dir=" + current_test.statsdir
lctx.info(execline)
exec_cmd(execline, command, sync, obj, actionID)
except Exception as e:
lctx.error(e)
lctx.error(traceback.print_exc())
return "ERROR"
lctx.debug("Completed start monitor")
return "SUCCESS"
import server
import config
import common
import testobj
from logger import LOG
if __name__ == "__main__":
# Port 0 means to select an arbitrary unused port
lctx = LOG.getLogger("agentlog", "Agent")
cfg = config.CFG("Agent", lctx)
cfg.readCFG("config.ini")
HOST = common.get_local_ip()
PORT = cfg.CPORT
common.createdir(cfg.daytona_agent_root, lctx)
server.serv.role = "Agent"
base_serv = server.serv()
#server.serv.lctx = LOG.getLogger("listenerlog","Agent")
#ser = server.serv.ThreadedTCPServer((HOST, PORT), server.serv.ThreadedTCPRequestHandler)
ser = base_serv.ThreadedTCPServer((HOST, PORT),
server.serv.ThreadedTCPRequestHandler)
server.serv.serverInstance = ser
ip, port = ser.server_address
#server.lctx = LOG.getLogger("listenerlog", "Agent")
# Start a thread with the server -- that thread will then start one
def setupTest(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
test_serialized = params.split(",")[0]
host_type = params.split(",")[1]
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test = get_test(t2.testobj.TestInputData.testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START")
test_logger.info("Test setup started")
current_test.tobj = testobj.testDefn()
current_test.tobj = t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid)
shutil.rmtree(dir, ignore_errors=True)
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid) + "/results/"
if host_type == "EXEC":
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.statsdir = prefix + current_test.stathostip + "/sar/"
current_test.resultsdir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/results"
current_test.archivedir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/"
if host_type == "EXEC":
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
test_logger.info("Test directory created")
# todo : check and validate if exec script is provided in expected format and
# the file exists in that location
if host_type == "EXEC":
execscript = current_test.tobj.testobj.TestInputData.execution_script_location
lctx.debug("TEST SETUP : " + str(execscript))
current_test.execscriptfile = current_test.execdir + "/" + execscript
lctx.debug(current_test.execscriptfile)
# check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at
# this location
execscript_location = EXEC_SCRIPT_DIR + execscript
execscript_location = os.path.realpath(execscript_location)
valid_path = os.path.commonprefix(
[execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR
if valid_path:
if os.path.isfile(execscript_location):
ret = shutil.copytree(
os.path.dirname(execscript_location),
os.path.dirname(current_test.execscriptfile))
else:
raise Exception(
"Execution script not found at Daytona Execution Script Location : "
+ EXEC_SCRIPT_DIR)
else:
raise Exception(
"Access Denied : Use Daytona Execution Script Location '"
+ EXEC_SCRIPT_DIR + "' for executing "
"exec scripts")
os.chmod(current_test.execscriptfile, 0744)
test_logger.info("Execution script copied successfully")
save_test(current_test.testid, current_test)
test_logger.info("Test setup complete")
lctx.debug("TEST SETUP | " + str(current_test.testid) +
" | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def setupTest(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
test_serialized = params.split(",")[0]
host_type = params.split(",")[1]
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test = get_test(t2.testobj.TestInputData.testid)
if current_test:
if current_test.status is not "INIT":
lctx.error("Invalid state for TEST SETUP action : " + current_test.status)
current_test.status = "ABORT"
save_test(current_test.testid, current_test)
return "ERROR"
try:
if current_test:
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START")
current_test.tobj = testobj.testDefn()
current_test.tobj = t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
lctx.debug("TEST SETUP : " + str(current_test.tobj.testobj.TestInputData.testid))
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid)
shutil.rmtree(dir, ignore_errors=True)
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid) + "/results/"
if host_type == "EXEC":
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.statsdir = prefix + current_test.stathostip + "/sar/"
current_test.resultsdir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/results"
current_test.archivedir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/"
common.createdir(cfg.daytona_agent_root, self.lctx)
if host_type == "EXEC":
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
# todo : check and validate if exec script is provided in expected format and
# the file exists in that location
if host_type == "EXEC":
execscript = current_test.tobj.testobj.TestInputData.execution_script_location
lctx.debug("TEST SETUP : " + str(execscript))
current_test.execscriptfile = current_test.execdir + "/" + execscript
lctx.debug(current_test.execscriptfile)
# check if execution script is present in '/tmp/ExecScripts/' - execute script only if it present at
# this location
execscript_location = EXEC_SCRIPT_DIR + execscript
execscript_location = os.path.realpath(execscript_location)
valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR
if valid_path:
if os.path.isfile(execscript_location):
ret = copyfile(execscript_location, current_test.execscriptfile)
else:
raise Exception(
"Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR)
else:
raise Exception(
"Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing "
"exec scripts")
os.chmod(current_test.execscriptfile, 0744)
current_test.status = "SETUP"
save_test(current_test.testid, current_test)
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except shutil.Error as err:
if current_test:
lctx.error("error copying file : " + str(execscript) + " to " + str(current_test.execscriptfile))
current_test.status = "ABORT"
save_test(current_test.testid, current_test)
lctx.error(err)
return "ERROR"
except Exception as e:
if current_test:
current_test.status = "ABORT"
save_test(current_test.testid, current_test)
lctx.error(e)
return "ERROR"
def mon(self, *args):
#query all waiting state tests and load into to_schedule, this is for restart case (similarly for running)
#todo : reconcile running and scheduled testids, if db altered externally
restarted = True
print "CFG in mon :"
print self.cfg.mysql_host
while True:
self.db = DBAccess(self.cfg, self.lctx)
d = "DBMON [W] : |"
for k in self.tests_to_run:
l = self.tests_to_run[k]
for t in l :
d = d + str(t.testobj.TestInputData.testid) + "|"
self.lctx.info(d)
d = ""
query_result = None
if restarted == True :
query_result = self.db.query("""select testid from CommonFrameworkSchedulerQueue where state = %s or state = %s or state = %s or state = %s or state = %s or state = %s""", ("scheduled","waiting","setup","running","completed","collating"), True, False);
#query_result = self.db.query("""select testid from TestInputData where end_status = %s or end_status = %s or end_status = %s or end_status = %s or end_status = %s or end_status = %s""", ("scheduled","waiting","setup","running","completed","collating"), True, False);
restarted = False
else:
status = "scheduled"
query_result = self.db.query("""select testid from CommonFrameworkSchedulerQueue where state = %s""", (status,), True, False);
#query_result = self.db.query("""select testid from TestInputData where end_status = %s""", (status,), True, False);
#reset all states to scheduled, mostly required in a restart case
#all items that make to the DBMON [Q] will be in scheduled state
for testid in query_result:
to = testobj.testDefn()
to.testobj.TestInputData.testid = testid[0]
res = to.construct(testid[0])
res = to.updateStatus("*", "scheduled")
d = "DBMON [Q] : "
d = d + str(query_result)
self.lctx.info(d)
for testid in query_result:
found = False
for k in self.tests_to_run: #search across all FW
l = self.tests_to_run[k] #list of tests ready to be run
for t in l:
self.lctx.debug(t.testobj.TestInputData.testid)
self.lctx.debug(testid[0])
self.lctx.debug(k)
if t.testobj.TestInputData.testid == testid[0]:
self.lctx.debug("Test already present in runQ")
found = True
if(found == False):
to = testobj.testDefn()
to.testobj.TestInputData.testid = testid[0]
#todo handle return status
res = to.construct(testid[0])
#create required dirs and setup server side env
prefix=self.cfg.daytona_dh_root +"/"+ to.testobj.TestInputData.frameworkname + "/" + str(to.testobj.TestInputData.testid) + "/" + "results" + "/"
to.testobj.TestInputData.exec_results_path = prefix+to.testobj.TestInputData.exechostname + "/"
to.testobj.TestInputData.exec_path = prefix+to.testobj.TestInputData.exechostname
to.testobj.TestInputData.exec_log_path = prefix+to.testobj.TestInputData.exechostname+"/application"
to.testobj.TestInputData.stats_results_path = defaultdict()
for s in to.testobj.TestInputData.stathostname.split(','):
to.testobj.TestInputData.stats_results_path[s.strip()] = prefix+s.strip()+"/"
to.testobj.TestInputData.stats_results_path[to.testobj.TestInputData.exechostname] = prefix+to.testobj.TestInputData.exechostname+"/"
common.createdir(to.testobj.TestInputData.exec_results_path, self.lctx)
common.createdir(to.testobj.TestInputData.exec_path, self.lctx)
common.createdir(to.testobj.TestInputData.exec_log_path, self.lctx)
for s in to.testobj.TestInputData.stats_results_path:
common.createdir(to.testobj.TestInputData.stats_results_path[s], self.lctx)
res = to.updateStatus("scheduled", "waiting")
sz = to.serialize()
t2 = testobj.testDefn()
t2.deserialize(sz)
if t2.testobj.TestInputData.testid != to.testobj.TestInputData.testid :
self.lctx.error("error in ser / dser")
break
#use a lock here
self.lock.acquire()
self.tests_to_run[to.testobj.TestInputData.frameworkid].append(to);
self.lctx.debug("Adding : " + str(to.testobj.TestInputData.testid))
self.lock.release()
self.db.close()
if self.mon_thread[0].check() == False :
return
time.sleep(5) #todo : make this config item
def setupTest( self, *args):
(obj, command, test_serialized, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
lctx.debug("setuptest")
try :
current_test.tobj = testobj.testDefn()
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test.tobj= t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
lctx.debug("TEST SETUP : " + str(current_test.tobj.testobj.TestInputData.testid))
current_test.status = "TESTSETUP"
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(current_test.tobj.testobj.TestInputData.testid) + "/results/"
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.resultsdir = prefix
current_test.statsdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/sar/"
current_test.archivedir = prefix
common.createdir(cfg.daytona_agent_root, self.lctx)
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
#todo : check and validate if exec script is provided in expected format and
# the file exists in that location
execscript = current_test.tobj.testobj.TestInputData.execution_script_location.split(":")[1]
lctx.debug("TEST SETUP : " + str(execscript))
tmp = str(execscript).split("/")
tmp.reverse()
lctx.debug(tmp)
filename = tmp[0]
current_test.execscriptfile = current_test.execdir+"/"+filename
lctx.debug(current_test.execscriptfile)
except Exception as e:
lctx.error(e)
lctx.error(traceback.print_exc())
return "ERROR"
try:
ret = copyfile(execscript, current_test.execscriptfile)
except shutil.Error as err:
lctx.error("error copying file : " + str(execscript) + " to " + str(current_test.execscriptfile))
return "ERROR"
try:
os.chmod(current_test.execscriptfile, 0744)
except:
lctx.error("error setting perm file : " + str(current_test.execdir))
return "ERROR"
#create dirs
#get exec script name
#cp the exec script
#set exec perm
#update cur test obj with exec script
#exec any custom setup script
lctx.debug("Completed setuptest")
return "SUCCESS"
def setupTest(self, *args):
"""
Test setup is called when scheduler send "DAYTONA_SETUP_TEST" message to agent. In this procedure agent create
all necessary file system path string and update in test object. After creating file path string it execute command
for making all these file system directories so that agent can later save SAR data. On exec host, it copies execution
script from Execscript folder to test specific directory in order to keep execution script seperate in case of
multiple test execution
:param self:
:param args: tuple of arguments containing obj, command, parameter sent by scheduler to agent for this command,
actionID and sync flag to denote if we need to execute this procedure in sync or async mode
:return: SUCCESS in case everything goes well otherwise it throws ERROR
"""
(obj, command, params, actionID, sync) = (args[0], args[1], args[2], args[3], args[4])
test_serialized = params.split(",")[0]
host_type = params.split(",")[1]
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
current_test = get_test(t2.testobj.TestInputData.testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | START")
test_logger.info("Test setup started")
current_test.tobj = testobj.testDefn()
current_test.tobj = t2
current_test.testid = current_test.tobj.testobj.TestInputData.testid
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
dir = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid)
shutil.rmtree(dir, ignore_errors=True)
prefix = cfg.daytona_agent_root + "/" + current_test.tobj.testobj.TestInputData.frameworkname + "/" + str(
current_test.tobj.testobj.TestInputData.testid) + "/results/"
if host_type == "EXEC":
current_test.execdir = prefix + current_test.tobj.testobj.TestInputData.exechostname
current_test.logdir = prefix + current_test.tobj.testobj.TestInputData.exechostname + "/application"
current_test.statsdir = prefix + current_test.stathostip + "/sar/"
current_test.resultsdir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/results"
current_test.archivedir = cfg.daytona_agent_root + "/" + \
current_test.tobj.testobj.TestInputData.frameworkname + "/" + \
str(current_test.tobj.testobj.TestInputData.testid) + "/"
if host_type == "EXEC":
common.createdir(current_test.execdir, self.lctx)
common.createdir(current_test.logdir, self.lctx)
common.createdir(current_test.resultsdir, self.lctx)
common.createdir(current_test.statsdir, self.lctx)
test_logger.info("Test directory created")
if host_type == "EXEC":
execscript = current_test.tobj.testobj.TestInputData.execution_script_location
lctx.debug("TEST SETUP : " + str(execscript))
current_test.execscriptfile = current_test.execdir + "/" + execscript
lctx.debug(current_test.execscriptfile)
# check if execution script is present in EXEC_SCRIPT_DIR - execute script only if it present at
# this location
execscript_location = EXEC_SCRIPT_DIR + execscript
execscript_location = os.path.realpath(execscript_location)
valid_path = os.path.commonprefix([execscript_location, EXEC_SCRIPT_DIR]) == EXEC_SCRIPT_DIR
if valid_path:
if os.path.isfile(execscript_location):
ret = shutil.copytree(os.path.dirname(execscript_location),
os.path.dirname(current_test.execscriptfile))
else:
raise Exception(
"Execution script not found at Daytona Execution Script Location : " + EXEC_SCRIPT_DIR)
else:
raise Exception(
"Access Denied : Use Daytona Execution Script Location '" + EXEC_SCRIPT_DIR + "' for executing "
"exec scripts")
os.chmod(current_test.execscriptfile, 0744)
test_logger.info("Execution script copied successfully")
save_test(current_test.testid, current_test)
test_logger.info("Test setup complete")
lctx.debug("TEST SETUP | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
