def handle_request(): request_data = request.form if 'ip' not in request_data or 'group_id' not in request_data or 'port' not in request_data: logger.log_error('malformed post request data.') return 'malformed post request data.', 400 group_id = request_data['group_id'] if test_and_set_active(group_id): logger.log_info('lock acquired for team "{}" with group_id {}'.format( team_names[int(group_id)], group_id)) ip = 'http://{}:{}'.format(request_data['ip'], request_data['port']) test_order = None if 'test_order' in request_data: test_order = literal_eval(request_data['test_order']) logger.log_info( 'custom test order {} was given for team "{}" with group_id {}' .format(test_order, team_names[int(group_id)], group_id)) if type(test_order) == int: test_order = [test_order] process_request(ip, group_id, test_order) logger.log_success( 'test for team "{}" with group_id {} initiated successfully'. format(team_names[int(group_id)], group_id)) return "success - test initiated" else: logger.log_error( 'another test for team "{}" with group_id {} is in progress'. format(team_names[int(group_id)], group_id)) return "error - existing test in progress", 406
def handle_request(): request_data = request.form if 'test_id' not in request_data or 'group_id' not in request_data or 'git_url' not in request_data: logger.log_error('malformed post request data.') return 'malformed post request data.', 400 if 'secret' not in request_data or request_data[ 'secret'] != config.REPORT_SECRET_KEY: logger.log_error("unauthorized data") return 'unauthorized request', 403 group_id = request_data['group_id'] if test_and_set_active(group_id): logger.log_info( 'lock acquired for team with group_id {}'.format(group_id)) test_id = int(request_data['test_id']) git_url = request_data['git_url'] logger.log_info( 'test id {} was given for team with group_id {}'.format( test_id, group_id)) process_request(git_url=git_url, group_id=group_id, test_id=test_id) logger.log_success( 'test for team with group_id {} initiated successfully'.format( group_id)) return "success - test initiated" else: logger.log_error( 'another test for team with group_id {} is in progress'.format( group_id)) return "error - existing test in progress", 406
def setup(self, repo_dir: str, group_id: str, git_url: str): project_dir = update_git(root_dir=repo_dir, group_id=group_id, git_url=git_url) # try: # copyfile( # ContestantProjectHandler.CONTESTANT_SETTINGS_PATH, # os.path.join(project_dir, ContestantProjectHandler.CONTESTANT_SETTINGS_NAME) # ) # except Exception: # logger.log_error("Could not copy django settings for group {}".format(group_id)) # raise Exception("Could not copy django settings") out, error = run_cmd(cmd="./scripts/remove_extra_files.sh " + project_dir, directory=".") # TODO handle logs if len(error) != 0: logger.log_info("error in removing extras: {}".format(str(error))) out, error = run_cmd(cmd="./scripts/build_image.sh " + project_dir, directory=".") logger.log_log("out: " + str(out) + " err: " + str(error)) build_msg = out.decode("utf-8") logger.log_info( "Project for group {} build successfully with message: {}".format( group_id, build_msg)) try: image_id = re.search(r"Successfully built ((\w|\d)+)\n", build_msg).group(1) except Exception: if re.findall(COPY_REQUIREMENTS_REGEX, build_msg) is not None and re.findall( INSTALL_REQUIREMENTS_REGEX, build_msg) is None: logger.log_warn( "Could not find requirements.txt for group {}".format( group_id)) raise Exception("Could not find requirements.txt file") # if re.findall(INSTALL_REQUIREMENTS_REGEX + SUCCESSFUL_STEP_REGEX + ERROR_REGEX, build_msg) is not None: # logger.log_warn("Could not install requirements for group {}".format(group_id)) # raise Exception("Could not install requirements") logger.log_warn( "Failed to build docker image for group {}.".format(group_id)) raise Exception( "Build error - Build Message Follows\n\n{}".format(build_msg)) # try: # os.remove(os.path.join(project_dir, ContestantProjectHandler.CONTESTANT_SETTINGS_NAME)) # except Exception: # logger.log_error("Could not remove django settings for group {}".format(group_id)) # raise Exception("Could not remove django settings") logger.log_success( "Image built for team {} successfully".format(group_id)) return image_id
def worker_function(team_id, phase_id, trial_id, dataset_number, submissions): logger.log_info( 'scoring submission for team with team_id {}.'.format(team_id)) question_scores = worker_score_questions(team_id, phase_id, dataset_number, submissions) logger.log_info( 'reporting test results for team with team_id {}, to competition server' .format(team_id)) report_test_results(team_id, phase_id, trial_id, dataset_number, question_scores) logger.log_success( 'test for team with team_id {} finished successfully'.format(team_id))
def worker_function(ip, group_id, test_order): logger.log_info( 'running tests for team "{}" with group_id {} on ip address {}'.format( team_names[int(group_id)], group_id, ip)) test_results = worker_run_tests(ip, test_order, group_id) logger.log_info('releasing lock for team "{}" with group_id {}'.format( team_names[int(group_id)], group_id)) deactivate_status(group_id) logger.log_info( 'reporting test results for team "{}" with group_id {} on ip address {} to competition server' .format(team_names[int(group_id)], group_id, ip)) report_test_results(group_id, test_results) logger.log_success( 'test for team "{}" with group_id {} finished successfully'.format( team_names[int(group_id)], group_id))
def worker_function(git_url, group_id, test_id): logger.log_info( 'running tests for team with group_id {} on git url {}'.format( group_id, git_url)) test_results = worker_run_tests(git_url=git_url, test_id=test_id, group_id=group_id) logger.log_info( 'releasing lock for team with group_id {}'.format(group_id)) deactivate_status(group_id) logger.log_info( 'reporting test results for team with group_id {} on git url {} to competition server' .format(group_id, git_url)) report_test_results(group_id=group_id, test_id=test_id, test_results=test_results) logger.log_success( 'test for team with group_id {} finished successfully'.format( group_id))
def run(self, image_id: str, port: int): container_name = "webelopers_" + str(port) + "_" + str(uuid.uuid1()) try: result = self.client.containers.run( image=image_id, detach=True, auto_remove=True, mem_limit=str(MAX_MEMORY_CONTESTANT) + "m", mem_reservation=str(MAX_MEMORY_CONTESTANT // 2) + "m", name=container_name, oom_kill_disable=False, ports={"8000": port}, ) logger.log_success( "Project is running on container {} with id {} for image {}". format(container_name, result, image_id)) return container_name except Exception as e: logger.log_warn("Could not run container {} with error {}".format( container_name, str(e))) raise Exception("Could not run docker")
def report_test_results(group_id, test_id, test_results): logger.log_log( 'log report for team with group_id {} for test_id {}'.format( group_id, test_id)) logger.log_log(test_results) test_results['group_id'] = group_id test_results['test_id'] = test_id test_results['secret'] = config.REPORT_SECRET_KEY try: headers = {'Content-type': 'application/json', 'Accept': 'text/plain'} requests.post('http://{}:{}/{}'.format(config.REPORT_SERVER_HOST, config.REPORT_SERVER_PORT, config.REPORT_SERVER_PATH), data=json.dumps({'data': test_results}), headers=headers) logger.log_success( "Report test resutls for group {} successfull".format(group_id)) except: logger.log_error( "failed to report test {} results for team id {}".format( test_id, group_id))
def handle_request(): request_data = request.get_json() logger.log_info('received_request', request_data) for field in config.JUDGE_MANDATORY_REQUEST_FIELDS: if field not in request_data: error_message = 'malformed post request data, excepting field {}'.format( field) logger.log_error(error_message) return error_message, 400 submissions = request_data['submissions'] for submission in submissions: for field in config.SUBMISSION_MANDATORY_FIELDS: if field not in submission: error_message = 'malformed post request data, excepting submission field {}'.format( field) logger.log_error(error_message) return error_message, 400 if submission['question_type'] not in config.QuestionType.list_types(): error_message = 'malformed post request data, invalid question type: {}'.format( submission['question_type']) logger.log_error(error_message) return error_message, 400 team_id = request_data['team_id'] logger.log_info('lock acquired for team with team_id {}'.format(team_id)) phase_id = request_data['phase_id'] trial_id = request_data['trial_id'] dataset_number = request_data['dataset_number'] process_request(team_id, phase_id, trial_id, dataset_number, submissions) logger.log_success( 'test for team with team_id {} initiated successfully'.format(team_id)) return "success - test initiated", 200
# STEP 3 read files FAILED_TO_READ_FILES = [p for p, f in FILE_PATH_STR_PAIRS if f is None] FILE_PATH_STR_PAIRS = [x for x in FILE_PATH_STR_PAIRS if not x[1] is None] NUMBER_OF_READ_FILES = len(FILE_PATH_STR_PAIRS) if NUMBER_OF_FILES > NUMBER_OF_READ_FILES: logger.log_failed_reads( NUMBER_OF_FILES, NUMBER_OF_READ_FILES, FAILED_TO_READ_FILES ) # STEP 4 remove failed to read files print(f"\nextracting data from {NUMBER_OF_READ_FILES} files...") DATA_FRAME, ERRORS = make_table( proc.SELECTORS, FILE_PATH_STR_PAIRS ) proc.WRITER(EXCEL_PATH, DATA_FRAME) # STEP 5 generate excel file if not ERRORS.empty: logger.log_errors(ERRORS) logger.log_success(NUMBER_OF_READ_FILES, NUMBER_OF_FILES, EXCEL_PATH) # STEP 6 print results