def dispatch_tests(server, commit_id, branch):
"""
Dispatches tests to test runners
:param server: dispatcher server
:param commit_id: commit id to dispatch to test runners
:param branch: branch to run tests on
"""
while True:
logger.info(
f"Attempting to dispatch commit {commit_id} on branch {branch} to runners..."
)
for runner in server.runners:
response = communicate(
runner["host"], int(runner["port"]), f"runtest:{commit_id}:{branch}"
)
if response == b"OK":
logger.info(
f"Adding ID: {commit_id} for branch {branch} to Runner: {runner['host']}:{runner['port']}"
)
server.dispatched_commits[commit_id] = runner
if commit_id in server.pending_commits:
server.pending_commits.pop(commit_id, None)
return
time.sleep(2)
def register(self):
"""
registers new test runners to the runners pool
"""
address = self.command_groups.group(3)
host, port = re.findall(r":(\w*)", address)
runner = {"host": host, "port": port}
logger.info(f"Registering new test runner {host}:{port}")
self.server.runners.append(runner)
self.request.sendall(b"OK")
def redistribute(server):
"""
This is used to `redistribute` the commit_ids that are in the pending_commits `queue`(pending_commits list)
It then calls dispatch_tests if there are pending commits
"""
while not server.dead:
for commit in server.pending_commits:
logger.info(
f"Redistributing pending commits {server.pending_commits} ...")
dispatch_tests(server, commit)
time.sleep(5)
def observer(dispatcher_host, dispatcher_port, repo, poll, branch):
"""
Repo Observer that communicates with the dispatcher server to send tests that are found
on the repository on new changes commited to the repo. This will watch the repo every 5 seconds for any
new commit that is made & make a dispatch to the dispatch server to initiate running of new tests.
"""
logger.info(f"Running Repo Observer")
while True:
try:
# call the bash script that will update the repo and check
# for changes. If there's a change, it will drop a .commit_id file
# with the latest commit in the current working directory
logger.info(f"cloning repo {repo}")
subprocess.check_output(
[f"{basedir}/update_repo.sh", repo, branch])
except subprocess.CalledProcessError as e:
logger.error(
f"Failed to update & check repo {repo}, err: {e.output}")
raise RepoObserverError(
f"Could not update & check repository. Err: {e.output}")
commit_id_file = f"{basedir}/.commit_id"
if os.path.isfile(commit_id_file):
# great, we have a change! let's execute the tests
# First, check the status of the dispatcher server to see
# if we can send the tests
try:
logger.info(
f"Checking dispatcher server status {dispatcher_host}:{dispatcher_port}"
)
response = communicate(dispatcher_host, int(dispatcher_port),
"status")
except socket.error as e:
logger.error(
f"Dispather Server Contact error. Is Dispatcher running? err: {e}"
)
raise RepoObserverError(f"Could not contact dispatcher {e}")
if response == b"OK":
# Dispatcher is available
commit = ""
with open(commit_id_file) as commit_file:
commit = commit_file.readline()
response = communicate(dispatcher_host, int(dispatcher_port),
f"dispatch:{commit}:{branch}")
if response != b"OK":
logger.error(
f"Failed to dispatch test to dispatcher. Is Dispatcher OK? err: {response}"
)
raise RepoObserverError(
f"Could not dispatch test: {response}")
logger.info(f"Dispatched tests for {repo}!")
else:
# Dispatcher has an issue
raise RepoObserverError(f"Could not dispatch test {response}")
time.sleep(poll)
def reporter_service(host, port):
"""
Entry point to Reporter Service
"""
server = ThreadingTCPServer((host, int(port)), ReporterHandler)
logger.info(f"Reporter Service running on address {host}:{port}")
try:
# Run forever unless stopped
server.serve_forever()
except (KeyboardInterrupt, Exception):
# in case it is stopped or encounters any error
server.dead = True
def run_tests(self, commit_id, branch, repo_folder):
"""
Runs tests as found in the repository
"""
output = subprocess.check_output([
f"{basedir}/test_runner_script.sh", repo_folder, commit_id, branch
])
test_folder = os.path.join(repo_folder, "tests")
suite = unittest.TestLoader().discover(test_folder)
result_file = open("results", "w")
unittest.TextTestRunner(result_file).run(suite)
result_file.close()
result_file = open("results", "r")
# send results to reporter service
output = result_file.read()
# check that reporter service is alive & running before sending output
# TODO: cache/retry logic to send test results to reporter service in case it is unreachable
try:
response = communicate(
self.server.reporter_service["host"],
int(self.server.reporter_service["port"]),
"status",
)
if response != b"OK":
logger.warning(
"Reporter Service does not seem ok, Can't send test results..."
)
return
elif response == b"OK":
logger.info("Sending test results to Reporter Service...")
communicate(
self.server.reporter_service["host"],
int(self.server.reporter_service["port"]),
f"results:{commit_id}:{len(output)}:{output}",
)
except socket.error as e:
logger.error("Cannot communicate with Reporter Service...")
return
def dispatch(self):
"""
dispatch command is used to dispatch a commit against a test runner. When the repo
observer sends this command as 'dispatch:<commit_id>'. The dispatcher parses the commit_id
& sends it to a test runner
"""
logger.info("Dispatching to test runner")
commit_id_and_branch = self.command_groups.group(3)[1:]
c_and_b = commit_id_and_branch.split(":")
commit_id = c_and_b[0]
branch = c_and_b[1]
logger.debug(f"Received commit_id {commit_id}")
if not self.server.runners:
self.request.sendall(b"No runners are registered")
else:
# we can dispatch tests, we have at least 1 test runner available
self.request.sendall(b"OK")
dispatch_tests(self.server, commit_id, branch)
def results(self):
"""
This command is used by the test runners to post back results to the dispatcher server
it is used in the format:
`results:<commit ID>:<length of results in bytes>:<results>`
<commit ID> is used to identify which commit ID the tests were run against
<length of results in bytes> is used to figure out how big a buffer is needed for the results data
<results> holds actual result output
"""
logger.info("Received test results from Test Runner")
results = self.command_groups.group(3)[1:]
results = results.split(":")
commit_id = results[0]
length_msg = int(results[1])
# 3 is the number of ":" in the sent command
remaining_buffer = self.BUF_SIZE - (len("results") + len(commit_id) +
len(results[1]) + 3)
if length_msg > remaining_buffer:
self.data += self.request.recv(length_msg -
remaining_buffer).strip()
test_results_path = f"{basedir}/test_results"
if not os.path.exists(test_results_path):
os.makedirs(test_results_path)
with open(f"{test_results_path}/{commit_id}", "w") as f:
data = f"{self.data}".split(":")[3:]
data = "\n".join(data)
f.write(data)
self.request.sendall(b"OK")
def runtest(self):
"""
This accepts messages of form runtest:<commit_id> & kicks of tests for the given commit id.
If the server is busy, it will respond to the dispatcher server with a BUSY response. If the
server is not busy, it will respond with OK status & will then kick of running tests & set its
status to BUSY
"""
if self.server.busy:
logger.warning("I am currently busy...")
self.request.sendall(b"BUSY")
else:
logger.info("Not busy at the moment :)")
self.request.sendall(b"OK")
commit_id_and_branch = self.command_groups.group(3)[1:]
c_and_b = commit_id_and_branch.split(":")
commit_id = c_and_b[0]
branch = c_and_b[1]
self.server.busy = True
self.run_tests(commit_id, branch, self.server.repo_folder)
self.server.busy = False
def dispatcher_server(host, port):
"""
Entry point to dispatch server
"""
server = ThreadingTCPServer((host, int(port)), DispatcherHandler)
logger.info(f"Dispatcher Server running on address {host}:{port}")
runner_heartbeat = Thread(target=runner_checker, args=(server, ))
redistributor = Thread(target=redistribute, args=(server, ))
try:
runner_heartbeat.start()
redistributor.start()
# Run forever unless stopped
server.serve_forever()
except (KeyboardInterrupt, Exception):
# in case it is stopped or encounters any error
server.dead = True
runner_heartbeat.join()
redistributor.join()
def check_status(self):
"""
Checks the status of the dispatcher server
"""
logger.info("Checking Dispatch Server Status")
self.request.sendall(b"OK")
def test_runner_server(host, port, repo, dispatcher_host, dispatcher_port,
reporter_host, reporter_port):
"""
This invokes the Test Runner server.
:param host: host to use
:param port: port to use
:param repo: repository to watch
:param dispatcher_host: Dispatcher host
:param dispatcher_port: Dispatcher Port
:param reporter_host: Reporter Host
:param reporter_port: Reporter Port
"""
range_start = 8900
runner_host = host
runner_port = None
tries = 0
if not port:
runner_port = range_start
while tries < 100:
try:
logger.info(
f"TestRunner Server running on address -> {runner_host}:{runner_port}"
)
server = ThreadingTCPServer((runner_host, runner_port),
TestRunnerHandler)
break
except socket.error as e:
logger.error(f"Error starting server, {e}")
if e.errno == errno.EADDRINUSE:
tries += 1
runner_port = runner_port + tries
continue
else:
raise e
else:
raise TestRunnerError(
f"Could not bind to ports in range {range_start}-{range_start+tries}"
)
else:
runner_port = int(port)
logger.info(
f"TestRunner Server running on address -> {runner_host}:{runner_port}"
)
server = ThreadingTCPServer((runner_host, runner_port),
TestRunnerHandler)
server.repo_folder = repo
server.dispatcher_server = {
"host": dispatcher_host,
"port": dispatcher_port
}
server.reporter_service = {"host": reporter_host, "port": reporter_port}
response = communicate(
server.dispatcher_server["host"],
int(server.dispatcher_server["port"]),
f"register:{runner_host}:{runner_port}",
)
if response != b"OK":
logger.error(f"Cannot register with dispatcher: {response}")
raise TestRunnerError("Can't register with dispatcher!")
thread = Thread(target=dispatcher_checker, args=(server, ))
try:
thread.start()
# this will run unless stopped by keyboard interrupt or by an exception
server.serve_forever()
except (KeyboardInterrupt, Exception):
server.dead = True
thread.join()