def checkTestRunning(testid):
lctx = LOG.getLogger("dblog", "DH")
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
db = dbaccess.DBAccess(cfg, LOG.getLogger("dblog", "DH"))
check = db.query(
"""SELECT COUNT(*) FROM CommonFrameworkSchedulerQueue where testid=%s""",
(testid, ), False, False)
db.close()
if check[0] == 0:
return False
else:
return True
def updateStatus(self, curStatus, newStatus):
"""
Update test status from curStatus to newStatus in database for a given test.
"""
lctx = LOG.getLogger("dblog", "DH")
lctx.debug("setting status from %s to %s" % (curStatus, newStatus))
if self.testobj.TestInputData.exec_results_path is not None:
test_logger = LOG.gettestlogger(self, "EXEC")
test_logger.info("Setting test status from %s to %s" %
(curStatus, newStatus))
update_res = self.db.query(
"""update TestInputData SET end_status = %s where testid=%s""",
(newStatus, self.testobj.TestInputData.testid), False, True)
self.testobj.TestInputData.end_status = newStatus
update_res = self.db.query(
"""update CommonFrameworkSchedulerQueue SET state = %s, message = %s, state_detail = %s where testid = %s""",
(newStatus, newStatus, newStatus,
self.testobj.TestInputData.testid), False, True)
if newStatus == "finished clean" or newStatus == "failed" or newStatus == "abort" or newStatus == "kill" or newStatus == "timeout clean":
update_res = self.db.query(
"""delete from CommonFrameworkSchedulerQueue where testid=%s""",
(self.testobj.TestInputData.testid, ), False, True)
lctx.debug(
"Deleted entry from CommonFrameworkSchedulerQueue because of failure for : "
+ str(self.testobj.TestInputData.testid))
return
def fileDownload(self, *args):
"""
On test completion, agent execute this procedure when it receive DAYTONA_FILE_DOWNLOAD message from scheduler.
We create a TAR file called results.tgz and save it test location, then we send this file to scheduler and save it
in scheduler side file system
"""
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
testid = int(args[2])
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | START")
lctx.debug("Preparing TAR file of system metric folder")
test_logger.info("Preparing TAR file of system metric folder")
common.make_tarfile(current_test.archivedir + "results.tgz", current_test.resultsdir + "/")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[current_test.stathostip]
download_file = current_test.archivedir + "results.tgz"
test_logger.info("Sending TAR file to daytona host")
cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip())
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def fileDownload(self, *args):
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
testid = int(args[2])
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) +
" | START")
lctx.debug("Preparing TAR file of system metric folder")
test_logger.info("Preparing TAR file of system metric folder")
common.make_tarfile(current_test.archivedir + "results.tgz",
current_test.resultsdir + "/")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[
current_test.stathostip]
download_file = current_test.archivedir + "results.tgz"
test_logger.info("Sending TAR file to daytona host")
cl.sendFile(current_test.serverip, current_test.serverport,
download_file, dest.strip())
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) +
" | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def fileDownload(self, *args):
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
prm = args[2].split(',')
lctx.debug(prm)
test_serialized = prm[3].strip()
t2 = testobj.testDefn()
t2.deserialize(test_serialized)
dest =""
if prm[4] == "RESULTS" :
dest = t2.testobj.TestInputData.exec_results_path
elif prm[4] == "STATS" :
s = prm[5]
lctx.debug(s)
dest = t2.testobj.TestInputData.stats_results_path[s]
else:
dest = ""
lctx.debug(prm[0])
lctx.debug(prm[1])
lctx.debug(prm[2])
lctx.debug(dest)
try:
cl.sendFile(prm[0].strip(), int(prm[1].strip()), prm[2].strip(), dest.strip())
except CommunicationError as e:
lctx.error(e.value)
return e.value
return "SUCCESS"
def downloadTestLogs(testid):
"""
This procedure just send test life cycle log file to scheduler upon test cleanup. This file provide user
information about test execution sequence on agent
"""
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
test_logger.info("Sending test log to daytona host")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[current_test.stathostip]
download_file = current_test.agent_log_file
cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip())
test_logger.info("Test log file transfer complete")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def fileDownload(self, *args):
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
testid = int(args[2])
current_test = get_test(testid)
if current_test:
if current_test.status is not "MONITOR_OFF":
lctx.error("Invalid state for START TEST action : " + current_test.status)
current_test.status = "FAILED"
save_test(current_test.testid, current_test)
return "ERROR"
try:
if current_test:
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | START")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[current_test.stathostip]
download_file = current_test.archivedir + "results.tgz"
cl.sendFile(current_test.serverip, current_test.serverport, download_file, dest.strip())
lctx.debug("FILE DOWNLOAD | " + str(current_test.testid) + " | COMPLETE")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
if current_test:
current_test.status = "FAILED"
save_test(current_test.testid, current_test)
lctx.error(e)
return "ERROR"
def updateEndTime(self, timestr):
lctx = LOG.getLogger("dblog", "DH")
lctx.debug("setting end time to %s" % (timestr))
update_res = self.db.query(
"""update TestInputData SET end_time = %s where testid=%s""",
(timestr, self.testobj.TestInputData.testid), False, True)
self.testobj.TestInputData.start_time = timestr
return
def __init__(self):
"""
Constructor initializes logger, config file reader and db handle
"""
self.lctx = LOG.getLogger("dblog", "DH")
self.cfg = config.CFG("DaytonaHost", self.lctx)
self.cfg.readCFG("config.ini")
self.db = dbaccess.DBAccess(self.cfg, self.lctx)
def scheduler_handshake(current_test):
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
env = envelope.DaytonaEnvelope()
ret = cl.send(current_test.serverip, current_test.serverport,
env.construct("DAYTONA_HANDSHAKE", "handshake2"))
if ret == "SUCCESS":
return True
else:
return False
def checkTestRunning(testid):
"""
This function checks whether user has not initiated test termination from UI. CommonFrameworkSchedulerQueue table
keep the list of all running test initiated by user from UI or CLI. When user terminate any running test from UI,
daytona removes the entry of this test from CommonFrameworkSchedulerQueue table. This functions polls database to
check if test is still present in the CommonFrameworkSchedulerQueue table
"""
lctx = LOG.getLogger("dblog", "DH")
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
db = dbaccess.DBAccess(cfg, LOG.getLogger("dblog", "DH"))
check = db.query(
"""SELECT COUNT(*) FROM CommonFrameworkSchedulerQueue where testid=%s""",
(testid, ), False, False)
db.close()
if check[0] == 0:
return False
else:
return True
def updateStartTime(self, timestr):
"""
Update test execution start time in database
"""
lctx = LOG.getLogger("dblog", "DH")
lctx.debug("setting start time to %s" % (timestr))
update_res = self.db.query(
"""update TestInputData SET start_time = %s where testid=%s""",
(timestr, self.testobj.TestInputData.testid), False, True)
self.testobj.TestInputData.start_time = timestr
return
def __init__(self, db, cfg, lctx):
self.dbinstance = db
self.cfg = cfg
self.testmap = db.tests_to_run
self.cl = client.TCPClient(LOG.getLogger("clientlog", "DH"))
self.ev = envelope.DaytonaEnvelope()
self.HOST = common.get_local_ip()
self.PORT = cfg.DHPORT
self.CPORT = cfg.CPORT
self.scheduler_thread = common.FuncThread(self.dispatch, True)
self.testmon_thread = common.FuncThread(self.testmon, True)
self.lctx = lctx
def scheduler_handshake(current_test):
"""
This procedure is a part of 2-way handshake between scheduler and agent. If agent receive handshake message from
scheduler, then agent also send a handshake message to scheduler to check if agent can communicate with scheduler
on scheduler port. This part is important as later we need to transfer log files to scheduler using scheduler port
:param current_test: Test object
:return: true if scheduler respond otherwise false
"""
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
env = envelope.DaytonaEnvelope()
ret = cl.send(current_test.serverip, current_test.serverport, env.construct("DAYTONA_HANDSHAKE", "handshake2"))
if ret == "SUCCESS":
return True
else:
return False
def __init__(self, db, cfg, lctx):
"""
Scheduler class constructor which initialize class variables and other threads
"""
self.dbinstance = db
self.cfg = cfg
self.testmap = db.tests_to_run
self.cl = client.TCPClient(LOG.getLogger("clientlog", "DH"))
self.ev = envelope.DaytonaEnvelope()
self.HOST = common.get_local_ip()
self.PORT = cfg.DHPORT
self.CPORT = cfg.CPORT
self.scheduler_thread = common.FuncThread(self.dispatch, True)
self.testmon_thread = common.FuncThread(self.testmon, True)
self.lctx = lctx
def updateStatus(self, curStatus, newStatus):
lctx = LOG.getLogger("dblog", "DH")
lctx.debug("setting status from %s to %s" % (curStatus, newStatus))
update_res = self.db.query(
"""update TestInputData SET end_status = %s where testid=%s""",
(newStatus, self.testobj.TestInputData.testid), False, True)
self.testobj.TestInputData.end_status = newStatus
update_res = self.db.query(
"""update CommonFrameworkSchedulerQueue SET state = %s, message = %s, state_detail = %s where testid = %s""",
(newStatus, newStatus, newStatus,
self.testobj.TestInputData.testid), False, True)
if newStatus == "finished clean" or newStatus == "failed" or newStatus == "abort" or newStatus == "kill" or newStatus == "timeout clean":
update_res = self.db.query(
"""delete from CommonFrameworkSchedulerQueue where testid=%s""",
(self.testobj.TestInputData.testid, ), False, True)
lctx.debug(
"Deleted entry from CommonFrameworkSchedulerQueue because of failure for : "
+ str(self.testobj.TestInputData.testid))
return
def downloadTestLogs(testid):
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
current_test = get_test(testid)
test_logger = None
try:
if current_test:
test_logger = LOG.gettestlogger(current_test, "STAT")
test_logger.info("Sending test log to daytona host")
dest = current_test.tobj.testobj.TestInputData.stats_results_path[
current_test.stathostip]
download_file = current_test.agent_log_file
cl.sendFile(current_test.serverip, current_test.serverport,
download_file, dest.strip())
test_logger.info("Test log file transfer complete")
return "SUCCESS"
else:
raise Exception("Invalid Test ID")
except Exception as e:
lctx.error(e)
if test_logger:
test_logger.error(e)
return "ERROR"
def handle(self):
"""
This handler is called everytime TCP server running on scheduler and agent receive any message. This
handler perform different operation based on action received
"""
host = self.client_address[0]
data = self.request.recv(8192)
cur_thread = threading.current_thread()
ev = data.split(":")
serv.lctx.debug("Envelope contents : " + str(ev))
cmd = ev[1]
msgid = ev[2]
params = ev[3]
serv.lctx.info(cmd)
serv.lctx.debug(msgid)
serv.lctx.debug(params)
if cmd == "DAYTONA_HANDSHAKE":
# Message received is a handshake message
p = params.split(",")
if p[0] == "handshake1":
# if payload contains handshake1 then this message came from scheduler to agent. Agent does basic
# env setup and send handshake2 message back to scheduler to verify connectivity from both ends
serv.registered_hosts[host] = host
addr = socket.gethostbyname(host)
serv.registered_hosts[addr] = addr
# Initialize current_test object with scheduler information and other env setup
current_test = activeTest(0, None, None, None)
current_test.stathostip = p[4]
current_test.stathostport = self.server.server_address[1]
current_test.serverip = p[1]
current_test.testid = int(p[3])
current_test.serverport = int(p[2])
current_test.status = "SETUP"
test_logger = LOG.init_testlogger(current_test, "STAT")
if test_logger:
current_test.agent_log_file = test_logger.handlers[
0].baseFilename
con = action.scheduler_handshake(current_test)
if con:
# if response received from scheduler then handshake is successfull
action.action_lock.acquire()
action.running_tests[int(p[3])] = current_test
action.action_lock.release()
response = "{}".format("SUCCESS")
test_logger.info(
"Handshake successfull with daytona host : " +
current_test.serverip)
else:
# else agent is not able to talk to scheduler using scheduler port, return error
response = "{}".format("ERROR")
test_logger.error(
"Handshake failed with daytona host : " +
current_test.serverip)
self.request.sendall(response)
return
else:
# Payload contains handshake2, means agent sent it on scheduler and scheduler just send OK response
# back to agent
response = "{}".format("SUCCESS")
self.request.sendall(response)
return
if host in serv.registered_hosts.keys() or cmd in (
"DAYTONA_HEARTBEAT", "DAYTONA_CLI"):
# Only process daytona commands from registered daytona host which was registered after handshake
if cmd == "DAYTONA_STREAM_END":
serv.lctx.debug("End stream...")
return
if cmd == "DAYTONA_STREAM_START":
# Handler for DAYTONA_STREAM_START message received on scheduler. Scheduler will setup environment
# for writing logs in execution.log file for this particular test
filepath = params + "/execution.log"
d = os.path.dirname(filepath)
if not os.path.exists(d):
os.makedirs(d)
f = open(filepath, 'wb')
serv.lctx.debug(filepath)
serv.lctx.debug("Receiving stream..." + filepath)
response = "{}".format("STREAMFILEREADY")
self.request.send(response)
l = self.request.recv(8192)
serv.lctx.debug(len(l))
while l:
serv.lctx.debug("Receiving stream...")
f.write(l)
print l
serv.lctx.debug(l)
f.flush()
l = self.request.recv(8192)
if l == "DAYTONA_STREAM_END":
serv.lctx.debug("receiving term string : ")
break
f.close()
serv.lctx.debug("Done receiving stream into : " + filepath)
return
if cmd == "DAYTONA_FILE_UPLOAD":
# Handler for uploading SAR data TAR file on scheduler
p = params.split(",")
serv.lctx.debug("SER SERVER : " + params)
fn = p[0].split("/")
fn.reverse()
loc = p[1].strip()
serv.lctx.debug("SER SERVER : " + loc)
filepath = loc + fn[0]
d = os.path.dirname(filepath)
if not os.path.exists(d):
os.makedirs(d)
serv.lctx.debug("Receiving..." + filepath)
response = "{}".format("FILEREADY")
self.request.send(response)
f = open(filepath, 'wb')
l = self.request.recv(8192)
serv.lctx.debug(len(l))
while l:
serv.lctx.debug("Receiving...")
f.write(l)
f.flush()
l = self.request.recv(8192)
serv.lctx.debug(len(l))
f.close()
serv.lctx.debug("Done receiving results : " + filepath)
return
if cmd == "DAYTONA_STOP_SERVER":
# Currently not in use
serverInstance.shutdown()
serverInstance.server_close()
response = "{}: {}".format(cur_thread.name,
"Shutting down server")
self.request.sendall(response)
if len(self.act.async_actions) > 0:
for pending in self.act.async_actions:
(t1, actionID, tst, ts) = pending
t1.stop()
serv.lctx.debug(
"DAYTONA_STOP_SERVER handler, Async action thread ended after stop : "
+ cur_thread.name)
return
# todo : Set server to shutdown state, reject all incomming reqs if this flag set wait for all threads
# to shutdown (with timeout) gracefully shutdown before timeout, or force close beyond timeout
# exResp = self.act.execute(cmd, params, msgid)
if serv.actc is None:
serv.actc = ActionCaller(
LOG.getLogger("listenerlog", serv.role))
# Call execute routine for mapping this daytona command with actual procedure.
exResp = serv.actc.execute(cmd, params, msgid)
response = "{}: {}".format(cur_thread.name, exResp)
self.request.sendall(response)
if len(serv.actc.async_actions) > 0:
serv.lctx.debug("Async action list : " +
str(len(serv.actc.async_actions)))
for pending in self.act.async_actions:
(t1, actionID, tst, ts) = pending
t1.join()
serv.lctx.debug(
"Async action thread ended after join : " +
t1.name)
else:
serv.lctx.error(
"Command recieved from unknown host before handshake")
serv.lctx.error(host)
def daytonaCli(self, *args):
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
lctx = LOG.getLogger("scheduler-clilog", "DH")
cli_param_map = pickle.loads(params)
if len(cli_param_map) != 3:
return "Error|Not enough arguments"
user = cli_param_map['user']
password = cli_param_map['password']
cli_command = cli_param_map['param'].split("|")[0]
cli_param = cli_param_map['param'].split("|")[1]
lctx.debug("Host received CLI command : " + cli_command)
db = dbCliHandle()
auth = db.authenticate_user(user, password)
if auth != "SUCCESS":
return auth
if cli_command == "get_frameworkid_arg":
arglist = db.getFrameworkIdArgs(cli_param)
return arglist
elif cli_command == "get_framework_arg":
arglist = db.getFrameworkArgs(cli_param)
return arglist
elif cli_command == "add_test":
res = db.addTest(cli_param, user)
return res
elif cli_command == "add_run_test":
res = db.addTest(cli_param, user, 'scheduled')
if res:
if res.split("|")[0] == 'Error':
return res
else:
testid = res.split("|")[1]
else:
return "Error|Test add failed"
res = db.runTest(testid, user)
if res.split("|")[0] == 'SUCCESS':
return "SUCCESS|" + testid
else:
return res
elif cli_command == "run_test":
res = db.runTest(cli_param, user)
return res
elif cli_command == "get_result":
res = db.getResult(cli_param, user)
return res
elif cli_command == "get_test_by_id":
res = db.getTestByID(cli_param)
return res
elif cli_command == "update_test":
res = db.updateTest(cli_param, user)
return res
elif cli_command == "update_run_test":
res = db.updateTest(cli_param, user, 'scheduled')
if res:
if res.split("|")[0] == 'Error':
return res
else:
testid = res.split("|")[1]
else:
return "Error|Test update failed"
res = db.runTest(testid, user)
if res.split("|")[0] == 'SUCCESS':
return "SUCCESS|" + testid
else:
return res
else:
return "Error|Unknown command received on server"
def process_results(self, *args):
# set test status completed
# call stop monitors
# send prepare results command to exec
# set test status collating
# copy results files from exec
# copy files from each mon
# set test status finished
# remove test from running Q
t = args[1]
status = args[2]
serialize_str = t.serialize()
t2 = testobj.testDefn()
t2.deserialize(serialize_str)
try:
if t.testobj.TestInputData.testid != t2.testobj.TestInputData.testid:
lctx.error("testobj not same")
raise Exception("Test objects do not match : ",
t2.testobj.TestInputData.testid)
ip = t.testobj.TestInputData.exechostname
lctx.debug(status)
if status in ["completed", "timeout"]:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_STOP_MONITOR",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_STOP_MONITOR failed : ",
t2.testobj.TestInputData.testid)
# get statistics hosts
for s in t.testobj.TestInputData.stathostname.split(','):
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
if len(s.strip()) == 0:
break
p = self.CPORT
try:
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_STOP_MONITOR",
str(t2.testobj.TestInputData.testid)))
except:
continue
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_STOP_MONITOR failed : ",
t2.testobj.TestInputData.testid)
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("timeout", "collating")
else:
t.updateStatus("completed", "collating")
ptop = process_top.ProcessTop(LOG.getLogger(
"processTop", "DH"))
# todo : avoid send client its own ip
lctx.info("SENDING results.tgz download to : " + ip + ":" +
str(self.CPORT))
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error(retsend)
raise Exception(
"Daytona command DAYTONA_FILE_DOWNLOAD failed :",
t2.testobj.TestInputData.testid)
try:
lctx.debug("Untar file : " +
t2.testobj.TestInputData.exec_results_path +
"results.tgz to location : " +
t2.testobj.TestInputData.exec_results_path +
"/../")
common.untarfile(
t2.testobj.TestInputData.exec_results_path +
"/results.tgz",
t2.testobj.TestInputData.exec_results_path + "/../")
except Exception as e:
lctx.error("Error in untar EXEC host results")
lctx.error(e)
raise Exception("test result processing error",
t2.testobj.TestInputData.testid)
ptop_ret = ptop.process_top_output(
t2.testobj.TestInputData.stats_results_path[ip] + "sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[ip])
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
lctx.info("Downloading stats from STATS self.HOSTS : " + s)
lctx.info(s)
# stop stats monitors on req hosts
# any host that blocks stop monitor blocks the scheduling for the FW
p = self.CPORT
lctx.info("Sending results.tgz download to :" + s.strip() +
":" + str(p))
try:
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FILE_DOWNLOAD",
str(t2.testobj.TestInputData.testid)))
except:
continue
lctx.debug(retsend)
if retsend.split(",")[1] != "SUCCESS":
lctx.error("Error downloading STATS from " +
s.strip() + ":" + retsend)
raise Exception(
"Daytona command DAYTONA_FILE_DOWNLOAD failed :",
t2.testobj.TestInputData.testid)
try:
lctx.debug(
"Untar file : " +
t2.testobj.TestInputData.stats_results_path[s] +
"results.tgz to location : " +
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
common.untarfile(
t2.testobj.TestInputData.stats_results_path[s] +
"/results.tgz",
t2.testobj.TestInputData.stats_results_path[s] +
"/../")
except Exception as e:
lctx.error("Error in untar STAT host " + s +
" results")
lctx.error(e)
raise Exception("test result processing error",
t2.testobj.TestInputData.testid)
ptop_ret = ptop.process_top_output(
t2.testobj.TestInputData.stats_results_path[s] +
"sar/")
lctx.debug(ptop_ret + " : " +
t2.testobj.TestInputData.stats_results_path[s])
# todo : invoke other scripts to transform results and update DB
except Exception as e:
lctx.error("Error in processing results")
lctx.error(e)
t.updateStatus("collating", "failed")
try:
retsend = self.cl.send(
ip, self.CPORT,
self.ev.construct("DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
lctx.debug(retsend)
for s in t.testobj.TestInputData.stathostname.split(','):
if len(s.strip()) == 0:
break
p = self.CPORT
lctx.debug("self.HOST : " + s.strip())
lctx.debug("PORT to send CLEANUP & FINISH : " + str(p))
try:
retsend = self.cl.send(
s.strip(), p,
self.ev.construct(
"DAYTONA_FINISH_TEST",
str(t2.testobj.TestInputData.testid)))
except:
pass
lctx.debug(retsend)
except Exception as e:
lctx.error("Error in processing results")
t.updateStatus("collating", "failed")
if t.testobj.TestInputData.timeout_flag:
t.updateStatus("collating", "timeout clean")
else:
t.updateStatus("collating", "finished clean")
now = time.time()
tstr = str(time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(now)))
t.updateEndTime(tstr)
f = None
try:
f = open(t2.testobj.TestInputData.exec_results_path +
"/results.csv")
except IOError as e:
lctx.debug("File results.csv not found")
pass
to = t.testobj.TestInputData.email
htmlfile = '<table>'
if f:
reader = csv.reader(f)
rownum = 0
for row in reader:
if rownum == 0:
htmlfile += '<tr>'
for column in row:
htmlfile += '<th style="text-align: left;" width="70%">' + column + '</th>'
htmlfile += '</tr>'
else:
htmlfile += '<tr>'
for column in row:
htmlfile += '<td style="text-align: left;" width="70%">' + column + '</td>'
htmlfile += '</tr>'
rownum += 1
f.close()
htmlfile += '</table>'
host_ip = "http://" + common.get_local_ip(
) + "/test_info.php?testid=" + str(t.testobj.TestInputData.testid)
subject = "Test {} completed successfully".format(
t.testobj.TestInputData.testid)
mail_content = "<BR> Test id : {} \
<BR> Framework : {} \
<BR> Title : {} <BR>".format(
t.testobj.TestInputData.testid,
t.testobj.TestInputData.frameworkname,
t.testobj.TestInputData.title)
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>Purpose : {} <BR> \
<BR> Creation time : {} \
<BR>Start time : {} \
<BR>End time : {} <BR>".format(
t.testobj.TestInputData.purpose,
t.testobj.TestInputData.creation_time,
t.testobj.TestInputData.start_time,
t.testobj.TestInputData.end_time)
mail_content = mail_content + "<BR>Your test executed successfully. \
<BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "Link:"
mail_content = mail_content + '<BR><a href="' + host_ip + '">' + host_ip + '</a>'
try:
common.send_email(subject, to, mail_content, "", lctx,
cfg.email_user, cfg.email_server,
cfg.smtp_server, cfg.smtp_port)
except:
lctx.error("Mail send error")
return "SUCCESS"
else:
print "Unknown response received from host"
print response
sys.exit()
else:
print "Error - Unknown response received from host"
print retsend
sys.exit()
else:
print "Error - No response received from host"
sys.exit()
if __name__ == "__main__":
read_arguments()
tcp_client = client.TCPClient(LOG.getLogger("tcpclientlog", "daytona-cli"))
env = envelope.DaytonaEnvelope()
check_host_hearbeat(tcp_client, env)
if action == 0:
define_test(tcp_client, env)
elif action == 1:
add_test(tcp_client, env)
elif action == 2:
update_test(tcp_client, env)
elif action == 3:
run_test(tcp_client, env)
elif action == 5:
get_result(tcp_client, env)
else:
print "Error - Invalid action"
display_usage()
cfg.readCFG("config.ini")
db = dbaccess.DBAccess(cfg, LOG.getLogger("dblog", "DH"))
check = db.query(
"""SELECT COUNT(*) FROM CommonFrameworkSchedulerQueue where testid=%s""",
(testid, ), False, False)
db.close()
if check[0] == 0:
return False
else:
return True
if __name__ == "__main__":
# Port 0 means to select an arbitrary unused port
lctx = LOG.getLogger("schedulerlog", "DH")
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
common.logger.ROLE = "DHOST"
db = dbaccess.DaytonaDBmon(cfg, LOG.getLogger("dblog", "DH"))
sch = Scheduler(db, cfg, LOG.getLogger("schedulerlog", "DH"))
server.serv.role = "DH"
ase_serv = server.serv()
server.serv.lctx = LOG.getLogger("listenerlog", "DH")
ser = server.serv.ThreadedTCPServer((common.get_local_ip(), sch.PORT),
server.serv.ThreadedTCPRequestHandler)
server.serv.serverInstance = ser
ip, port = ser.server_address
server.lctx = LOG.getLogger("listenerlog", "DH")
# -*- coding:cp949 -*-
import threading
import SocketServer
import time
from collections import defaultdict
import server
import config
import common
import testobj
from logger import LOG
if __name__ == "__main__":
# Port 0 means to select an arbitrary unused port
lctx = LOG.getLogger("agentlog", "Agent")
cfg = config.CFG("Agent", lctx)
cfg.readCFG("config.ini")
HOST = common.get_local_ip()
PORT = cfg.CPORT
common.createdir(cfg.daytona_agent_root, lctx)
server.serv.role = "Agent"
base_serv = server.serv()
#server.serv.lctx = LOG.getLogger("listenerlog","Agent")
#ser = server.serv.ThreadedTCPServer((HOST, PORT), server.serv.ThreadedTCPRequestHandler)
ser = base_serv.ThreadedTCPServer((HOST, PORT),
def __init__(self):
self.lctx = LOG.getLogger("dblog", "DH")
self.cfg = config.CFG("DaytonaHost", self.lctx)
self.cfg.readCFG("config.ini")
self.db = dbaccess.DBAccess(self.cfg, self.lctx)
def exec_cmd(cmd, daytona_cmd, sync, obj, actionid, current_test):
lctx.debug("Execute cmd : " + cmd)
sfile = None
cl = None
########
if daytona_cmd == "DAYTONA_START_TEST":
cl = client.TCPClient(LOG.getLogger("clientlog", "Agent"))
(current_test.stream, sfile) = cl.stream_start(
current_test.serverip, current_test.serverport,
str(current_test.tobj.testobj.TestInputData.exec_log_path))
########
if sfile is not None:
sfile.flush()
cthread = commandThread(cmd, daytona_cmd, sfile, current_test.execdir,
current_test.testid)
current_test.exec_thread = cthread
cthread.start()
(t, aid, tst, ts) = (None, None, None, None)
if sync == "T":
lctx.debug("Execute cmd in Sync ctx")
cthread.join()
if sfile is not None:
sfile.flush()
else:
# async action entry in the server table (need this to check self alive below)
for tp in obj.async_actions:
if tp[1] == actionid:
(t, aid, tst, ts) = tp
lctx.debug("Execute cmd in asSync ctx : " + str(actionid))
timer_expire = False
while True:
lctx.debug("waiting for async action to complete : " +
str(actionid))
if cthread.stdout is not None:
lctx.debug("printting output of stream ")
if sfile is not None:
sfile.flush()
if tst.testobj.TestInputData.timeout > 0:
if time.time() - ts > tst.testobj.TestInputData.timeout:
lctx.error(
"Timer expired for this test, need to end this async action"
)
timer_expire = True
# todo : breakout of while after a timer event and exit after terminating thread
# timeout is monitored outside in the scheduler server , for all async ops , the thread is paused from
# there via a CMD (ENDTEST)
if t.check() == False or cthread.is_alive(
) == False or timer_expire:
if daytona_cmd == "DAYTONA_START_TEST":
if cthread.is_alive():
exec_script_lock.acquire()
if current_test.testid in exec_script_pid:
p = exec_script_pid[current_test.testid]
del exec_script_pid[current_test.testid]
exec_script_lock.release()
if p:
os.killpg(p.pid, signal.SIGTERM)
lctx.debug("end stream")
cl.stream_end(
current_test.serverip, current_test.serverport,
str(current_test.tobj.testobj.TestInputData.
exec_log_path), current_test.stream, sfile)
# callback
# removeactionid
lctx.debug("Callback here removing item")
obj.removeActionItem(actionid)
break
time.sleep(3)
if daytona_cmd == "DAYTONA_START_TEST":
# todo : verify the TEST END on filesystem OR Failure
if timer_expire:
current_test.status = "TIMEOUT"
else:
lctx.debug("Setting current test status to TESTEND")
current_test.status = "TESTEND"
lctx.debug(daytona_cmd + " END [" + str(actionid) + "]")
if save_test(current_test.testid, current_test):
return "SUCCESS"
else:
return "ERROR"
htmlfile = htmlfile + '<td width="70%">' + column + '</td>'
htmlfile = htmlfile + '</tr>'
rownum += 1
htmlfile = htmlfile + '</table>'
subject = "Test SAMPLE completed successfully"
mail_content = "<BR>Test id : {sample id} \
<BR> Framework : {sample framework} \
<BR> Title : {sample title}<BR><BR>"
mail_content = mail_content + "<BR>==========================================================<BR><BR>"
mail_content = mail_content + "<BR>Purpose : {sample}<BR> \
<BR>Creation time : {TIME} \
<BR>Start time : {TIME} \
<BR>End time : {TIME}<BR>"
mail_content = mail_content + "<BR>Your test executed successfully.\
<BR>Results (Contents of results.csv)<BR>"
mail_content = mail_content + "<BR>==========================================================<BR>"
mail_content = mail_content + "<BR>" + htmlfile + "<BR>"
lctx = LOG.getLogger("schedulerlog", "DH")
curhost = "ip-172-31-19-107.us-west-2.compute.internal"
smtp_server = "localhost"
smtp_port = 25
common.send_email("test email with results", "*****@*****.**",
mail_content, "", lctx, "ubuntu", curhost, smtp_server,
smtp_port)
def daytonaCli(self, *args):
"""
This is daytona CLI command handler when scheduler receives any request from Daytona cli script to perform
any task. Daytona cli provide ability to get framework definition, add test for particular framework, run test,
update test details and fetch test results. User need to provide daytona credentials in order to execute commands
using daytona CLI scripts
"""
(obj, command, params, actionID, sync) = (args[0], args[1], args[2],
args[3], args[4])
lctx = LOG.getLogger("scheduler-clilog", "DH")
cli_param_map = pickle.loads(params)
if len(cli_param_map) != 3:
return "Error|Not enough arguments"
# Retriveing username and password from CLI command for authentication
user = cli_param_map['user']
password = cli_param_map['password']
# Retriveing actual daytona command and associated parameter from CLI command
cli_command = cli_param_map['param'].split("|")[0]
cli_param = cli_param_map['param'].split("|")[1]
lctx.debug("Host received CLI command : " + cli_command)
db = dbCliHandle()
# Calling authenticate_user webservice for verifying username and password combination
auth = db.authenticate_user(user, password)
if auth != "SUCCESS":
return auth
# Invoke function based on CLI action mentioned on CLI command
if cli_command == "get_frameworkid_arg":
arglist = db.getFrameworkIdArgs(cli_param)
return arglist
elif cli_command == "get_framework_arg":
arglist = db.getFrameworkArgs(cli_param)
return arglist
elif cli_command == "add_test":
res = db.addTest(cli_param, user)
return res
elif cli_command == "add_run_test":
res = db.addTest(cli_param, user, 'scheduled')
if res:
if res.split("|")[0] == 'Error':
return res
else:
testid = res.split("|")[1]
else:
return "Error|Test add failed"
res = db.runTest(testid, user)
if res.split("|")[0] == 'SUCCESS':
return "SUCCESS|" + testid
else:
return res
elif cli_command == "run_test":
res = db.runTest(cli_param, user)
return res
elif cli_command == "get_result":
res = db.getResult(cli_param, user)
return res
elif cli_command == "get_test_by_id":
res = db.getTestByID(cli_param)
return res
elif cli_command == "update_test":
res = db.updateTest(cli_param, user)
return res
elif cli_command == "update_run_test":
res = db.updateTest(cli_param, user, 'scheduled')
if res:
if res.split("|")[0] == 'Error':
return res
else:
testid = res.split("|")[1]
else:
return "Error|Test update failed"
res = db.runTest(testid, user)
if res.split("|")[0] == 'SUCCESS':
return "SUCCESS|" + testid
else:
return res
else:
return "Error|Unknown command received on server"
def handle(self):
host = self.client_address[0]
data =""
data = self.request.recv(8192)
cur_thread = threading.current_thread()
ev = data.split(":")
serv.lctx.debug("Envelope contents : " + str(ev))
cmd = ev[1]
msgid = ev[2]
params = ev[3]
serv.lctx.info(cmd)
serv.lctx.debug(cmd)
serv.lctx.debug(msgid)
serv.lctx.debug(params)
if cmd == "DAYTONA_HANDSHAKE":
#todo : maintain a list of daytona host that this server talks to
#only respond to the ones in the list
serv.registered_hosts[host] = host
addr = socket.gethostbyname(host)
serv.registered_hosts[addr] = addr
p = params.split(",")
action.current_test.serverip = p[0]
action.current_test.serverport = int(p[1])
action.current_test.status = "TESTSETUP"
response = "{}".format("SUCCESS")
self.request.sendall(response)
return
if host in serv.registered_hosts.keys() or cmd == "DAYTONA_HEARTBEAT":
if cmd == "DAYTONA_STREAM_END":
serv.lctx.debug("End stream...")
return
if cmd == "DAYTONA_STREAM_START":
filepath = params+"/execution.log"
d = os.path.dirname(filepath)
if not os.path.exists(d):
os.makedirs(d)
f = open(filepath,'wb')
serv.lctx.debug(filepath)
serv.lctx.debug("Receiving stream..." + filepath)
response = "{}".format("STREAMFILEREADY")
self.request.send(response)
l = self.request.recv(8192)
serv.lctx.debug(len(l))
while (l):
serv.lctx.debug("Receiving stream...")
f.write(l)
print l
serv.lctx.debug(l)
f.flush()
l = self.request.recv(8192)
if l == "DAYTONA_STREAM_END":
serv.lctx.debug("receiving term string : ")
break
f.close()
#response = "{}".format("SUCCESS")
#self.request.sendall(response)
serv.lctx.debug("Done receiving stream into : " + filepath)
return
if cmd == "DAYTONA_FILE_UPLOAD":
p = params.split(",")
serv.lctx.debug("SER SERVER : " + params)
fn = p[0].split("/")
fn.reverse()
loc = p[1].strip()
serv.lctx.debug("SER SERVER : " + loc)
filepath = loc+fn[0]
d = os.path.dirname(filepath)
if not os.path.exists(d):
os.makedirs(d)
serv.lctx.debug("Receiving..." + filepath)
response = "{}".format("FILEREADY")
self.request.send(response)
f = open(filepath,'wb')
l = self.request.recv(8192)
serv.lctx.debug(len(l))
while (l):
serv.lctx.debug("Receiving...")
f.write(l)
f.flush()
l = self.request.recv(8192)
serv.lctx.debug(len(l))
f.close()
serv.lctx.debug("Done receiving results : " + filepath)
return
if cmd == "DAYTONA_STOP_SERVER":
serverInstance.shutdown()
serverInstance.server_close()
response = "{}: {}".format(cur_thread.name, "Shutting down server")
self.request.sendall(response)
if len(self.act.async_actions) > 0 :
for pending in self.act.async_actions:
(t1, actionID, tst, ts) = pending
t1.stop()
serv.lctx.debug("DAYTONA_STOP_SERVER handler, Async action thread ended after stop : " + cur_thread.name)
return
#todo : Set server to shutdown state, reject all incomming reqs if this flag set
# wait for all threads to shutdown (with timeout)
# gracefully shutdown before timeout, or force close beyond timeout
#exResp = self.act.execute(cmd, params, msgid)
if serv.actc == None:
serv.actc = ActionCaller(LOG.getLogger("listenerlog", serv.role))
exResp = serv.actc.execute(cmd, params, msgid)
response = "{}: {}".format(cur_thread.name, exResp)
self.request.sendall(response)
if len(serv.actc.async_actions) > 0 :
serv.lctx.debug("Async action list : " + str(len(serv.actc.async_actions)))
for pending in self.act.async_actions:
(t1, actionID, tst, ts) = pending
t1.join()
serv.lctx.debug("Async action thread ended after join : " + t1.name)
else:
serv.lctx.error("Command recieved from unknown host before handshake")
serv.lctx.error(host)
def __init__(self):
serv.lctx = LOG.getLogger("listenerlog",serv.role)
action.lctx = LOG.getLogger("actionlog", serv.role)
#todo this integration has to be reviewed
actc = ActionCaller(LOG.getLogger("actionlog", serv.role))
serv.registered_hosts={}
def construct(self, tid):
import dbaccess
import config
lctx = LOG.getLogger("dblog", "DH")
cfg = config.CFG("DaytonaHost", lctx)
cfg.readCFG("config.ini")
self.db = dbaccess.DBAccess(cfg, LOG.getLogger("dblog", "DH"))
self.testobj.TestInputData.testid = tid
query_result = self.db.query(
"""select testid, frameworkid, start_time,
end_time, end_status, end_detail,
exechostname, stathostname, timeout, cc_list, title, purpose, creation_time
from TestInputData where testid = %s""",
(self.testobj.TestInputData.testid, ), False, False)
(self.testobj.TestInputData.testid,
self.testobj.TestInputData.frameworkid,
self.testobj.TestInputData.start_time,
self.testobj.TestInputData.end_time,
self.testobj.TestInputData.end_status,
self.testobj.TestInputData.end_detail,
self.testobj.TestInputData.exechostname,
self.testobj.TestInputData.stathostname,
self.testobj.TestInputData.timeout, self.testobj.TestInputData.email,
self.testobj.TestInputData.title, self.testobj.TestInputData.purpose,
self.testobj.TestInputData.creation_time) = query_result
lctx.debug(query_result)
query_result = self.db.query(
"""select ha.hostname, hat.name, hat.shared, hat.execution, hat.statistics
from HostAssociation ha
join HostAssociationType hat
on ha.hostassociationtypeid = hat.hostassociationtypeid
where testid = %s and hat.frameworkid = %s""",
(self.testobj.TestInputData.testid,
self.testobj.TestInputData.frameworkid), True, False)
for r in query_result:
lctx.debug(r)
if r[1] == 'statistics' and r[4] == 1:
self.testobj.TestInputData.stathostname = r[
0] + "," + self.testobj.TestInputData.stathostname
elif r[1] == 'execution' and r[3] == 1:
self.testobj.TestInputData.exechostname = r[
0] + "," + self.testobj.TestInputData.exechostname
self.testobj.TestInputData.stathostname = self.testobj.TestInputData.stathostname[:
-1]
self.testobj.TestInputData.exechostname = self.testobj.TestInputData.exechostname[:
-1]
lctx.debug(self.testobj.TestInputData.exechostname)
lctx.debug(self.testobj.TestInputData.stathostname)
query_result = self.db.query(
"""select * from TestArgs where testid = %s""",
(self.testobj.TestInputData.testid, ), True, False)
self.testobj.TestInputData.execScriptArgs = query_result
lctx.debug(query_result)
query_result = self.db.query(
"""select file_root, execution_script_location, frameworkname from ApplicationFrameworkMetadata where frameworkid = %s""",
(self.testobj.TestInputData.frameworkid, ), False, False)
(self.testobj.TestInputData.file_root,
self.testobj.TestInputData.execution_script_location,
self.testobj.TestInputData.frameworkname) = query_result
lctx.debug(query_result)
query_result = self.db.query(
"""select processname, delay, duration from ProfilerFramework where testid = %s and profiler = %s""",
(self.testobj.TestInputData.testid, 'STRACE'), False, False)
if query_result:
self.testobj.TestInputData.strace = True
(self.testobj.TestInputData.strace_process,
self.testobj.TestInputData.strace_delay,
self.testobj.TestInputData.strace_duration) = query_result
query_result = self.db.query(
"""select processname, delay, duration from ProfilerFramework where testid = %s and profiler = %s""",
(self.testobj.TestInputData.testid, 'PERF'), False, False)
if query_result:
(self.testobj.TestInputData.perf_process,
self.testobj.TestInputData.perf_delay,
self.testobj.TestInputData.perf_duration) = query_result
