def dispatch_tests(server, commit_id, branch): """ Dispatches tests to test runners :param server: dispatcher server :param commit_id: commit id to dispatch to test runners :param branch: branch to run tests on """ while True: logger.info( f"Attempting to dispatch commit {commit_id} on branch {branch} to runners..." ) for runner in server.runners: response = communicate( runner["host"], int(runner["port"]), f"runtest:{commit_id}:{branch}" ) if response == b"OK": logger.info( f"Adding ID: {commit_id} for branch {branch} to Runner: {runner['host']}:{runner['port']}" ) server.dispatched_commits[commit_id] = runner if commit_id in server.pending_commits: server.pending_commits.pop(commit_id, None) return time.sleep(2)
def register(self): """ registers new test runners to the runners pool """ address = self.command_groups.group(3) host, port = re.findall(r":(\w*)", address) runner = {"host": host, "port": port} logger.info(f"Registering new test runner {host}:{port}") self.server.runners.append(runner) self.request.sendall(b"OK")
def redistribute(server): """ This is used to `redistribute` the commit_ids that are in the pending_commits `queue`(pending_commits list) It then calls dispatch_tests if there are pending commits """ while not server.dead: for commit in server.pending_commits: logger.info( f"Redistributing pending commits {server.pending_commits} ...") dispatch_tests(server, commit) time.sleep(5)
def observer(dispatcher_host, dispatcher_port, repo, poll, branch): """ Repo Observer that communicates with the dispatcher server to send tests that are found on the repository on new changes commited to the repo. This will watch the repo every 5 seconds for any new commit that is made & make a dispatch to the dispatch server to initiate running of new tests. """ logger.info(f"Running Repo Observer") while True: try: # call the bash script that will update the repo and check # for changes. If there's a change, it will drop a .commit_id file # with the latest commit in the current working directory logger.info(f"cloning repo {repo}") subprocess.check_output( [f"{basedir}/update_repo.sh", repo, branch]) except subprocess.CalledProcessError as e: logger.error( f"Failed to update & check repo {repo}, err: {e.output}") raise RepoObserverError( f"Could not update & check repository. Err: {e.output}") commit_id_file = f"{basedir}/.commit_id" if os.path.isfile(commit_id_file): # great, we have a change! let's execute the tests # First, check the status of the dispatcher server to see # if we can send the tests try: logger.info( f"Checking dispatcher server status {dispatcher_host}:{dispatcher_port}" ) response = communicate(dispatcher_host, int(dispatcher_port), "status") except socket.error as e: logger.error( f"Dispather Server Contact error. Is Dispatcher running? err: {e}" ) raise RepoObserverError(f"Could not contact dispatcher {e}") if response == b"OK": # Dispatcher is available commit = "" with open(commit_id_file) as commit_file: commit = commit_file.readline() response = communicate(dispatcher_host, int(dispatcher_port), f"dispatch:{commit}:{branch}") if response != b"OK": logger.error( f"Failed to dispatch test to dispatcher. Is Dispatcher OK? err: {response}" ) raise RepoObserverError( f"Could not dispatch test: {response}") logger.info(f"Dispatched tests for {repo}!") else: # Dispatcher has an issue raise RepoObserverError(f"Could not dispatch test {response}") time.sleep(poll)
def reporter_service(host, port): """ Entry point to Reporter Service """ server = ThreadingTCPServer((host, int(port)), ReporterHandler) logger.info(f"Reporter Service running on address {host}:{port}") try: # Run forever unless stopped server.serve_forever() except (KeyboardInterrupt, Exception): # in case it is stopped or encounters any error server.dead = True
def run_tests(self, commit_id, branch, repo_folder): """ Runs tests as found in the repository """ output = subprocess.check_output([ f"{basedir}/test_runner_script.sh", repo_folder, commit_id, branch ]) test_folder = os.path.join(repo_folder, "tests") suite = unittest.TestLoader().discover(test_folder) result_file = open("results", "w") unittest.TextTestRunner(result_file).run(suite) result_file.close() result_file = open("results", "r") # send results to reporter service output = result_file.read() # check that reporter service is alive & running before sending output # TODO: cache/retry logic to send test results to reporter service in case it is unreachable try: response = communicate( self.server.reporter_service["host"], int(self.server.reporter_service["port"]), "status", ) if response != b"OK": logger.warning( "Reporter Service does not seem ok, Can't send test results..." ) return elif response == b"OK": logger.info("Sending test results to Reporter Service...") communicate( self.server.reporter_service["host"], int(self.server.reporter_service["port"]), f"results:{commit_id}:{len(output)}:{output}", ) except socket.error as e: logger.error("Cannot communicate with Reporter Service...") return
def dispatch(self): """ dispatch command is used to dispatch a commit against a test runner. When the repo observer sends this command as 'dispatch:<commit_id>'. The dispatcher parses the commit_id & sends it to a test runner """ logger.info("Dispatching to test runner") commit_id_and_branch = self.command_groups.group(3)[1:] c_and_b = commit_id_and_branch.split(":") commit_id = c_and_b[0] branch = c_and_b[1] logger.debug(f"Received commit_id {commit_id}") if not self.server.runners: self.request.sendall(b"No runners are registered") else: # we can dispatch tests, we have at least 1 test runner available self.request.sendall(b"OK") dispatch_tests(self.server, commit_id, branch)
def results(self): """ This command is used by the test runners to post back results to the dispatcher server it is used in the format: `results:<commit ID>:<length of results in bytes>:<results>` <commit ID> is used to identify which commit ID the tests were run against <length of results in bytes> is used to figure out how big a buffer is needed for the results data <results> holds actual result output """ logger.info("Received test results from Test Runner") results = self.command_groups.group(3)[1:] results = results.split(":") commit_id = results[0] length_msg = int(results[1]) # 3 is the number of ":" in the sent command remaining_buffer = self.BUF_SIZE - (len("results") + len(commit_id) + len(results[1]) + 3) if length_msg > remaining_buffer: self.data += self.request.recv(length_msg - remaining_buffer).strip() test_results_path = f"{basedir}/test_results" if not os.path.exists(test_results_path): os.makedirs(test_results_path) with open(f"{test_results_path}/{commit_id}", "w") as f: data = f"{self.data}".split(":")[3:] data = "\n".join(data) f.write(data) self.request.sendall(b"OK")
def runtest(self): """ This accepts messages of form runtest:<commit_id> & kicks of tests for the given commit id. If the server is busy, it will respond to the dispatcher server with a BUSY response. If the server is not busy, it will respond with OK status & will then kick of running tests & set its status to BUSY """ if self.server.busy: logger.warning("I am currently busy...") self.request.sendall(b"BUSY") else: logger.info("Not busy at the moment :)") self.request.sendall(b"OK") commit_id_and_branch = self.command_groups.group(3)[1:] c_and_b = commit_id_and_branch.split(":") commit_id = c_and_b[0] branch = c_and_b[1] self.server.busy = True self.run_tests(commit_id, branch, self.server.repo_folder) self.server.busy = False
def dispatcher_server(host, port): """ Entry point to dispatch server """ server = ThreadingTCPServer((host, int(port)), DispatcherHandler) logger.info(f"Dispatcher Server running on address {host}:{port}") runner_heartbeat = Thread(target=runner_checker, args=(server, )) redistributor = Thread(target=redistribute, args=(server, )) try: runner_heartbeat.start() redistributor.start() # Run forever unless stopped server.serve_forever() except (KeyboardInterrupt, Exception): # in case it is stopped or encounters any error server.dead = True runner_heartbeat.join() redistributor.join()
def check_status(self): """ Checks the status of the dispatcher server """ logger.info("Checking Dispatch Server Status") self.request.sendall(b"OK")
def test_runner_server(host, port, repo, dispatcher_host, dispatcher_port, reporter_host, reporter_port): """ This invokes the Test Runner server. :param host: host to use :param port: port to use :param repo: repository to watch :param dispatcher_host: Dispatcher host :param dispatcher_port: Dispatcher Port :param reporter_host: Reporter Host :param reporter_port: Reporter Port """ range_start = 8900 runner_host = host runner_port = None tries = 0 if not port: runner_port = range_start while tries < 100: try: logger.info( f"TestRunner Server running on address -> {runner_host}:{runner_port}" ) server = ThreadingTCPServer((runner_host, runner_port), TestRunnerHandler) break except socket.error as e: logger.error(f"Error starting server, {e}") if e.errno == errno.EADDRINUSE: tries += 1 runner_port = runner_port + tries continue else: raise e else: raise TestRunnerError( f"Could not bind to ports in range {range_start}-{range_start+tries}" ) else: runner_port = int(port) logger.info( f"TestRunner Server running on address -> {runner_host}:{runner_port}" ) server = ThreadingTCPServer((runner_host, runner_port), TestRunnerHandler) server.repo_folder = repo server.dispatcher_server = { "host": dispatcher_host, "port": dispatcher_port } server.reporter_service = {"host": reporter_host, "port": reporter_port} response = communicate( server.dispatcher_server["host"], int(server.dispatcher_server["port"]), f"register:{runner_host}:{runner_port}", ) if response != b"OK": logger.error(f"Cannot register with dispatcher: {response}") raise TestRunnerError("Can't register with dispatcher!") thread = Thread(target=dispatcher_checker, args=(server, )) try: thread.start() # this will run unless stopped by keyboard interrupt or by an exception server.serve_forever() except (KeyboardInterrupt, Exception): server.dead = True thread.join()