def dispatch(self, *args):
dispatch_threads = defaultdict()
while True:
for k in self.testmap:
found = False
if k in self.dispatch_queue or k in self.running_tests:
found = True
else:
lctx.debug("Found spot for test")
if found:
continue
try:
tmp_t = self.testmap[k][0]
except Exception as e:
lctx.debug("No test object found in map")
continue
if tmp_t is None:
continue
alive = False
h = tmp_t.testobj.TestInputData.exechostname
test_logger = LOG.init_testlogger(tmp_t, "EXEC")
test_logger.info("Test execution starts")
try:
retsend = self.cl.send(
h, self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 2:
status = retsend.split(",")[1]
else:
raise Exception(
"Execution host not avaliable - No Heartbeat ",
tmp_t.testobj.TestInputData.testid)
if "ALIVE" == status:
test_logger.info(
"HeartBeat received from execution host " + h)
ret = self.cl.send(
h, self.CPORT,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(tmp_t.testobj.TestInputData.testid) + "," +
h))
if ret == "SUCCESS":
alive = True
test_logger.info(
"Handshake successful with execution host " +
h)
lctx.debug(
"Handshake successful in scheduler, adding ip/hostname to reg hosts"
)
server.serv.registered_hosts[h] = h
addr = socket.gethostbyname(h)
lctx.debug(addr)
server.serv.registered_hosts[addr] = addr
else:
raise Exception(
"Unable to handshake with agent on executuion host "
+ h)
except Exception as e:
lctx.error(e)
test_logger.error(e)
# pause the dbmon here as we dont want the same test to be picked again after we pop
self.dbinstance.mon_thread[0].pause()
self.dbinstance.lock.acquire()
t = self.testmap[k].pop(0)
t.updateStatus("waiting", "failed")
self.dbinstance.lock.release()
lctx.debug("Removed test from map : " +
str(t.testobj.TestInputData.testid))
self.dbinstance.mon_thread[0].resume()
LOG.removeLogger(tmp_t)
continue
# todo : add host to reg list if handshake successful
if alive == True and found == False:
# for each framework pick one and move it to running, iff running has an empty slot.
lctx.debug(
"-------Found empty slot in dispatch and running Q-------"
)
# pause the dbmon here as we dont want the same test to be picked again after we pop
self.dbinstance.mon_thread[0].pause()
self.dbinstance.lock.acquire()
t = self.testmap[k].pop(0)
self.dbinstance.lock.release()
lctx.info("< %s" % t.testobj.TestInputData.testid)
self.dispatchQ__lock.acquire()
self.dispatch_queue[k] = t
self.dispatchQ__lock.release()
t.updateStatus("waiting", "setup")
self.dbinstance.mon_thread[0].resume()
try:
trigger_thread = common.FuncThread(
self.trigger, True, t)
dispatch_threads[t.testobj.TestInputData.testid] = (
trigger_thread, t)
trigger_thread.start()
except Exception as e:
lctx.error("Trigger error : " +
str(t.testobj.TestInputData.testid))
test_logger.error("Test setup failed " +
str(t.testobj.TestInputData.testid))
LOG.removeLogger(tmp_t)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[k]
self.dispatchQ__lock.release()
lctx.debug(e)
try:
d = "DISPATCH [S/R] : "
for k in self.dispatch_queue:
d = d + " |" + str(
self.dispatch_queue[k].testobj.TestInputData.testid)
except:
lctx.debug("ERROR : Dispatch Q empty")
lctx.debug(d)
d = ""
time.sleep(2)
def handle(self):
"""
This handler is called everytime TCP server running on scheduler and agent receive any message. This
handler perform different operation based on action received
"""
host = self.client_address[0]
data = self.request.recv(8192)
cur_thread = threading.current_thread()
ev = data.split(":")
serv.lctx.debug("Envelope contents : " + str(ev))
cmd = ev[1]
msgid = ev[2]
params = ev[3]
serv.lctx.info(cmd)
serv.lctx.debug(msgid)
serv.lctx.debug(params)
if cmd == "DAYTONA_HANDSHAKE":
# Message received is a handshake message
p = params.split(",")
if p[0] == "handshake1":
# if payload contains handshake1 then this message came from scheduler to agent. Agent does basic
# env setup and send handshake2 message back to scheduler to verify connectivity from both ends
serv.registered_hosts[host] = host
addr = socket.gethostbyname(host)
serv.registered_hosts[addr] = addr
# Initialize current_test object with scheduler information and other env setup
current_test = activeTest(0, None, None, None)
current_test.stathostip = p[4]
current_test.stathostport = self.server.server_address[1]
current_test.serverip = p[1]
current_test.testid = int(p[3])
current_test.serverport = int(p[2])
current_test.status = "SETUP"
test_logger = LOG.init_testlogger(current_test, "STAT")
if test_logger:
current_test.agent_log_file = test_logger.handlers[
0].baseFilename
con = action.scheduler_handshake(current_test)
if con:
# if response received from scheduler then handshake is successfull
action.action_lock.acquire()
action.running_tests[int(p[3])] = current_test
action.action_lock.release()
response = "{}".format("SUCCESS")
test_logger.info(
"Handshake successfull with daytona host : " +
current_test.serverip)
else:
# else agent is not able to talk to scheduler using scheduler port, return error
response = "{}".format("ERROR")
test_logger.error(
"Handshake failed with daytona host : " +
current_test.serverip)
self.request.sendall(response)
return
else:
# Payload contains handshake2, means agent sent it on scheduler and scheduler just send OK response
# back to agent
response = "{}".format("SUCCESS")
self.request.sendall(response)
return
if host in serv.registered_hosts.keys() or cmd in (
"DAYTONA_HEARTBEAT", "DAYTONA_CLI"):
# Only process daytona commands from registered daytona host which was registered after handshake
if cmd == "DAYTONA_STREAM_END":
serv.lctx.debug("End stream...")
return
if cmd == "DAYTONA_STREAM_START":
# Handler for DAYTONA_STREAM_START message received on scheduler. Scheduler will setup environment
# for writing logs in execution.log file for this particular test
filepath = params + "/execution.log"
d = os.path.dirname(filepath)
if not os.path.exists(d):
os.makedirs(d)
f = open(filepath, 'wb')
serv.lctx.debug(filepath)
serv.lctx.debug("Receiving stream..." + filepath)
response = "{}".format("STREAMFILEREADY")
self.request.send(response)
l = self.request.recv(8192)
serv.lctx.debug(len(l))
while l:
serv.lctx.debug("Receiving stream...")
f.write(l)
print l
serv.lctx.debug(l)
f.flush()
l = self.request.recv(8192)
if l == "DAYTONA_STREAM_END":
serv.lctx.debug("receiving term string : ")
break
f.close()
serv.lctx.debug("Done receiving stream into : " + filepath)
return
if cmd == "DAYTONA_FILE_UPLOAD":
# Handler for uploading SAR data TAR file on scheduler
p = params.split(",")
serv.lctx.debug("SER SERVER : " + params)
fn = p[0].split("/")
fn.reverse()
loc = p[1].strip()
serv.lctx.debug("SER SERVER : " + loc)
filepath = loc + fn[0]
d = os.path.dirname(filepath)
if not os.path.exists(d):
os.makedirs(d)
serv.lctx.debug("Receiving..." + filepath)
response = "{}".format("FILEREADY")
self.request.send(response)
f = open(filepath, 'wb')
l = self.request.recv(8192)
serv.lctx.debug(len(l))
while l:
serv.lctx.debug("Receiving...")
f.write(l)
f.flush()
l = self.request.recv(8192)
serv.lctx.debug(len(l))
f.close()
serv.lctx.debug("Done receiving results : " + filepath)
return
if cmd == "DAYTONA_STOP_SERVER":
# Currently not in use
serverInstance.shutdown()
serverInstance.server_close()
response = "{}: {}".format(cur_thread.name,
"Shutting down server")
self.request.sendall(response)
if len(self.act.async_actions) > 0:
for pending in self.act.async_actions:
(t1, actionID, tst, ts) = pending
t1.stop()
serv.lctx.debug(
"DAYTONA_STOP_SERVER handler, Async action thread ended after stop : "
+ cur_thread.name)
return
# todo : Set server to shutdown state, reject all incomming reqs if this flag set wait for all threads
# to shutdown (with timeout) gracefully shutdown before timeout, or force close beyond timeout
# exResp = self.act.execute(cmd, params, msgid)
if serv.actc is None:
serv.actc = ActionCaller(
LOG.getLogger("listenerlog", serv.role))
# Call execute routine for mapping this daytona command with actual procedure.
exResp = serv.actc.execute(cmd, params, msgid)
response = "{}: {}".format(cur_thread.name, exResp)
self.request.sendall(response)
if len(serv.actc.async_actions) > 0:
serv.lctx.debug("Async action list : " +
str(len(serv.actc.async_actions)))
for pending in self.act.async_actions:
(t1, actionID, tst, ts) = pending
t1.join()
serv.lctx.debug(
"Async action thread ended after join : " +
t1.name)
else:
serv.lctx.error(
"Command recieved from unknown host before handshake")
serv.lctx.error(host)
def handle(self):
host = self.client_address[0]
data = self.request.recv(8192)
cur_thread = threading.current_thread()
ev = data.split(":")
serv.lctx.debug("Envelope contents : " + str(ev))
cmd = ev[1]
msgid = ev[2]
params = ev[3]
serv.lctx.info(cmd)
serv.lctx.debug(msgid)
serv.lctx.debug(params)
if cmd == "DAYTONA_HANDSHAKE":
# todo : maintain a list of daytona host that this server talks to only respond to the ones in the list
p = params.split(",")
if p[0] == "handshake1":
serv.registered_hosts[host] = host
addr = socket.gethostbyname(host)
serv.registered_hosts[addr] = addr
current_test = activeTest(0, None, None, None)
current_test.stathostip = p[4]
current_test.stathostport = self.server.server_address[1]
current_test.serverip = p[1]
current_test.testid = int(p[3])
current_test.serverport = int(p[2])
current_test.status = "SETUP"
test_logger = LOG.init_testlogger(current_test,"STAT")
if test_logger:
current_test.agent_log_file = test_logger.handlers[0].baseFilename
con = action.scheduler_handshake(current_test)
if con:
action.action_lock.acquire()
action.running_tests[int(p[3])] = current_test
action.action_lock.release()
response = "{}".format("SUCCESS")
test_logger.info("Handshake successfull with daytona host : " + current_test.serverip)
else:
response = "{}".format("ERROR")
test_logger.error("Handshake failed with daytona host : " + current_test.serverip)
self.request.sendall(response)
return
else:
response = "{}".format("SUCCESS")
self.request.sendall(response)
return
if host in serv.registered_hosts.keys() or cmd in ("DAYTONA_HEARTBEAT", "DAYTONA_CLI"):
if cmd == "DAYTONA_STREAM_END":
serv.lctx.debug("End stream...")
return
if cmd == "DAYTONA_STREAM_START":
filepath = params+"/execution.log"
d = os.path.dirname(filepath)
if not os.path.exists(d):
os.makedirs(d)
f = open(filepath,'wb')
serv.lctx.debug(filepath)
serv.lctx.debug("Receiving stream..." + filepath)
response = "{}".format("STREAMFILEREADY")
self.request.send(response)
l = self.request.recv(8192)
serv.lctx.debug(len(l))
while (l):
serv.lctx.debug("Receiving stream...")
f.write(l)
print l
serv.lctx.debug(l)
f.flush()
l = self.request.recv(8192)
if l == "DAYTONA_STREAM_END":
serv.lctx.debug("receiving term string : ")
break
f.close()
# response = "{}".format("SUCCESS")
# self.request.sendall(response)
serv.lctx.debug("Done receiving stream into : " + filepath)
return
if cmd == "DAYTONA_FILE_UPLOAD":
p = params.split(",")
serv.lctx.debug("SER SERVER : " + params)
fn = p[0].split("/")
fn.reverse()
loc = p[1].strip()
serv.lctx.debug("SER SERVER : " + loc)
filepath = loc+fn[0]
d = os.path.dirname(filepath)
if not os.path.exists(d):
os.makedirs(d)
serv.lctx.debug("Receiving..." + filepath)
response = "{}".format("FILEREADY")
self.request.send(response)
f = open(filepath, 'wb')
l = self.request.recv(8192)
serv.lctx.debug(len(l))
while l:
serv.lctx.debug("Receiving...")
f.write(l)
f.flush()
l = self.request.recv(8192)
serv.lctx.debug(len(l))
f.close()
serv.lctx.debug("Done receiving results : " + filepath)
return
if cmd == "DAYTONA_STOP_SERVER":
serverInstance.shutdown()
serverInstance.server_close()
response = "{}: {}".format(cur_thread.name, "Shutting down server")
self.request.sendall(response)
if len(self.act.async_actions) > 0 :
for pending in self.act.async_actions:
(t1, actionID, tst, ts) = pending
t1.stop()
serv.lctx.debug("DAYTONA_STOP_SERVER handler, Async action thread ended after stop : " + cur_thread.name)
return
# todo : Set server to shutdown state, reject all incomming reqs if this flag set wait for all threads
# to shutdown (with timeout) gracefully shutdown before timeout, or force close beyond timeout
# exResp = self.act.execute(cmd, params, msgid)
if serv.actc is None:
serv.actc = ActionCaller(LOG.getLogger("listenerlog", serv.role))
exResp = serv.actc.execute(cmd, params, msgid)
response = "{}: {}".format(cur_thread.name, exResp)
self.request.sendall(response)
if len(serv.actc.async_actions) > 0 :
serv.lctx.debug("Async action list : " + str(len(serv.actc.async_actions)))
for pending in self.act.async_actions:
(t1, actionID, tst, ts) = pending
t1.join()
serv.lctx.debug("Async action thread ended after join : " + t1.name)
else:
serv.lctx.error("Command recieved from unknown host before handshake")
serv.lctx.error(host)
def dispatch(self, *args):
"""
This is dispatch queue of scheduler where test from different framework wait in the waiting queue for scheduler to
bind it with trigger thread. This procedure continuously iterate over testmap populated by DBMon with tests
started by user from UI or CLI. This keep track of all running tests and it allows one test per framework. Once
this procedure find an open test spot for a test from particular framework, this procedure will pop it from testmap,
put it in dispatch queue and assign trigger thread for this test to start test setup and then execution.
"""
dispatch_threads = defaultdict()
while True:
# Continuously iterate on testmap for initiating any test execution
for k in self.testmap:
# iterating for all frameworkid k in testmap which contains list of waiting tests for a particular framework
found = False
# If test for a particular framework is already in running or dispatch queue then this new test need to
# wait until previous test gets finish, hence we do nothing and just continue
if k in self.dispatch_queue or k in self.running_tests:
found = True
else:
lctx.debug("Found spot for test")
if found:
continue
# Proceed if spot is available for executing test for this framework
try:
tmp_t = self.testmap[k][0]
except Exception as e:
lctx.debug("No test object found in map")
continue
if tmp_t is None:
continue
alive = False
h = tmp_t.testobj.TestInputData.exechostname
# Initiating test logger for capturing test life cycle on scheduler, all logs are logged in file <testid>.log
test_logger = LOG.init_testlogger(tmp_t, "EXEC")
test_logger.info("Test execution starts")
try:
# Sending heartbeat on exec host to check if it agent is up on exec host
retsend = self.cl.send(
h, self.CPORT,
self.ev.construct("DAYTONA_HEARTBEAT", ""))
if retsend and len(retsend.split(",")) > 2:
status = retsend.split(",")[1]
else:
raise Exception(
"Execution host not avaliable - No Heartbeat ",
tmp_t.testobj.TestInputData.testid)
if "ALIVE" == status:
test_logger.info(
"HeartBeat received from execution host " + h)
# Sending DAYTONA_HANDSHAKE for verifying connectivity between scheduler and agent on exec host
# using custom daytona ports
ret = self.cl.send(
h, self.CPORT,
self.ev.construct(
"DAYTONA_HANDSHAKE", "handshake1," +
self.HOST + "," + str(self.PORT) + "," +
str(tmp_t.testobj.TestInputData.testid) + "," +
h))
if ret == "SUCCESS":
alive = True
test_logger.info(
"Handshake successful with execution host " +
h)
lctx.debug(
"Handshake successful in scheduler, adding ip/hostname to reg hosts"
)
server.serv.registered_hosts[h] = h
addr = socket.gethostbyname(h)
lctx.debug(addr)
server.serv.registered_hosts[addr] = addr
else:
raise Exception(
"Unable to handshake with agent on executuion host "
+ h)
except Exception as e:
lctx.error(e)
test_logger.error(e)
# pause the dbmon here as we dont want the same test to be picked again after we pop
self.dbinstance.mon_thread[0].pause()
self.dbinstance.lock.acquire()
t = self.testmap[k].pop(0)
t.updateStatus("waiting", "failed")
self.dbinstance.lock.release()
lctx.debug("Removed test from map : " +
str(t.testobj.TestInputData.testid))
self.dbinstance.mon_thread[0].resume()
LOG.removeLogger(tmp_t)
continue
if alive == True and found == False:
# for each framework pick one and move it to running, iff running has an empty slot.
lctx.debug(
"-------Found empty slot in dispatch and running Q-------"
)
# pause the dbmon here as we dont want the same test to be picked again after we pop
self.dbinstance.mon_thread[0].pause()
self.dbinstance.lock.acquire()
t = self.testmap[k].pop(0)
self.dbinstance.lock.release()
lctx.info("< %s" % t.testobj.TestInputData.testid)
# put the test in dispatch queue
self.dispatchQ__lock.acquire()
self.dispatch_queue[k] = t
self.dispatchQ__lock.release()
t.updateStatus("waiting", "setup")
self.dbinstance.mon_thread[0].resume()
try:
# Bind a seperate trigger thread for this test to start test execution
trigger_thread = common.FuncThread(
self.trigger, True, t)
dispatch_threads[t.testobj.TestInputData.testid] = (
trigger_thread, t)
trigger_thread.start()
except Exception as e:
lctx.error("Trigger error : " +
str(t.testobj.TestInputData.testid))
test_logger.error("Test setup failed " +
str(t.testobj.TestInputData.testid))
LOG.removeLogger(tmp_t)
self.dispatchQ__lock.acquire()
del self.dispatch_queue[k]
self.dispatchQ__lock.release()
lctx.debug(e)
try:
# Log list of test currently present in dispatch queue in scheduler debug file
d = "DISPATCH [S/R] : "
for k in self.dispatch_queue:
d = d + " |" + str(
self.dispatch_queue[k].testobj.TestInputData.testid)
except:
lctx.debug("ERROR : Dispatch Q empty")
lctx.debug(d)
d = ""
time.sleep(2)
