def add_error_to_response(resp, explanation, tb, falcon_http_error_code, code_filename): """ Modify the falcon HTTP response object with an error to be shown to the user. Also deletes the user's code as the engine cannot run it. :param resp: The falcon HTTP response object to be modified. :param explanation: A human-friendly explanation of the error. :param tb: Traceback string. :param falcon_http_error_code: Falcon HTTP error code to return. :param code_filename: Filepath to user code to be deleted. :return: nothing """ logger.error(explanation) logger.error(tb) util.delete_file(code_filename) # URL friendly traceback we can embed into a mailto: link. url_friendly_tb = urllib.parse.quote(tb) DISCOURSE_LINK = '<a href="https://discourse.projectlovelace.net/">https://discourse.projectlovelace.net/</a>' EMAIL_LINK = ( '<a href="mailto:[email protected]?&subject=Project Lovelace error report' + "&body={:}%0A%0A{:}".format(explanation, url_friendly_tb) + '">[email protected]</a>') NOTICE = ( "A stacktrace should appear below with more information about this error which might help\n" "you debug your code. But if it's not your code then it might be our fault :( If this is a\n" "website error and you have the time, we'd really appreciate it if you could report this\n" "on Discourse (" + DISCOURSE_LINK + ") or via email (" + EMAIL_LINK + ").\n" "All the information is embedded in the email link so all you have to do is press send.\n" "Thanks so much!") error_message = "{:s}\n\n{:s}\n\nError: {:}".format( explanation, NOTICE, tb) resp_dict = {"error": error_message} resp.status = falcon_http_error_code resp.set_header("Access-Control-Allow-Origin", "*") resp.body = json.dumps(resp_dict) return
def on_post(self, req, resp): payload = req.media code = payload["code"] language = payload["language"] if not code: resp_dict = {"error": "No code provided!"} resp.status = falcon.HTTP_400 resp.set_header("Access-Control-Allow-Origin", "*") resp.body = json.dumps(resp_dict) return code_filename = write_code_to_file(code, language) try: # Fetch problem ID and load the correct problem module. problem_name = payload["problem"].replace("-", "_") problem_module = "problems.{:s}".format(problem_name) logger.debug( "Importing problem_name={:s} problem_module={:s}...".format( problem_name, problem_module)) problems = importlib.import_module("problems") problem = importlib.import_module(problem_module) except Exception: explanation = ( "Could not import module {:s}. " "Returning HTTP 400 Bad Request due to possibly invalid JSON.". format(problem_module)) add_error_to_response(resp, explanation, traceback.format_exc(), falcon.HTTP_400, code_filename) return function_name = problem.FUNCTION_NAME problem_dir = problem_name # Copy static resources into engine directory and push them into the Linux container. static_resources = [] for resource_file_name in problem.STATIC_RESOURCES: from_path = os.path.join(cwd, "..", "resources", problem_dir, resource_file_name) to_path = os.path.join(cwd, resource_file_name) logger.debug("Copying static resource from {:s} to {:s}".format( from_path, to_path)) try: shutil.copyfile(from_path, to_path) except Exception: explanation = "Engine failed to copy a static resource. Returning falcon HTTP 500." add_error_to_response(resp, explanation, traceback.format_exc(), falcon.HTTP_500, code_filename) return static_resources.append(to_path) container_path = "/root/{:}".format(resource_file_name) logger.debug("Pushing static resource to container {:}{:}".format( self.container_id, container_path)) _ = docker_file_push(self.container_id, from_path, container_path) if not problem.STATIC_RESOURCES: logger.debug("No static resources to push") logger.info("Generating test cases...") test_cases = [] try: for i, test_type in enumerate(problem.TestCaseType): for j in range(test_type.multiplicity): logger.debug( "Generating test case {:d}: {:s} ({:d}/{:d})...". format( len(test_cases) + 1, str(test_type), j + 1, test_type.multiplicity)) test_cases.append(problem.generate_test_case(test_type)) except Exception: explanation = "Engine failed to generate a test case. Returning falcon HTTP 500." add_error_to_response(resp, explanation, traceback.format_exc(), falcon.HTTP_500, code_filename) return # Copy over all the dynamic resources generated by the test cases. dynamic_resources = [] for i, tc in enumerate(test_cases): if "DYNAMIC_RESOURCES" in tc.input: for dynamic_resource_filename in tc.input["DYNAMIC_RESOURCES"]: resource_path = os.path.join(cwd, "..", "resources", problem_dir, dynamic_resource_filename) destination_path = os.path.join(cwd, dynamic_resource_filename) logger.debug( "Copying test case resource from {:s} to {:s}...". format(resource_path, destination_path)) shutil.copyfile(resource_path, destination_path) dynamic_resources.append(resource_path) dynamic_resources.append(destination_path) container_path = "/root/{:}".format( dynamic_resource_filename) logger.debug( "Pushing dynamic resource to container {:}{:}".format( self.container_id, container_path)) _ = docker_file_push(self.container_id, resource_path, container_path) if not dynamic_resources: logger.debug("No dynamic resources to push") runner = CodeRunner(language) input_tuples = [tc.input_tuple() for tc in test_cases] output_tuples = [tc.output_tuple() for tc in test_cases] try: user_outputs, p_infos = runner.run(self.container_name, code_filename, function_name, input_tuples, output_tuples) except (FilePushError, FilePullError): explanation = "File could not be pushed to or pulled from docker container. Returning falcon HTTP 500." add_error_to_response(resp, explanation, traceback.format_exc(), falcon.HTTP_500, code_filename) return except EngineExecutionError: explanation = ( "Return code from executing user code in docker container is nonzero. " "Returning falcon HTTP 400.") add_error_to_response(resp, explanation, traceback.format_exc(), falcon.HTTP_400, code_filename) return # Pull any user generated files. files_pulled = False for i, tc in enumerate(test_cases): if "USER_GENERATED_FILES" in tc.output: for user_generated_filename in tc.output[ "USER_GENERATED_FILES"]: container_filepath = "/root/{:s}".format( user_generated_filename) logger.debug( "Pulling user generated file from container {:s}{:s}". format(self.container_name, container_filepath)) _ = docker_file_pull(self.container_id, container_filepath, user_generated_filename) files_pulled = True if not files_pulled: logger.debug("No user generated files to pull") n_cases = len(test_cases) n_passes = 0 # Number of test cases passed. test_case_details = ( [] ) # List of dicts each containing the details of a particular test case. # Verify that user outputs are all correct (i.e. check whether each test case passes or fails). for input_tuple, user_output, p_info, tc in zip( input_tuples, user_outputs, p_infos, test_cases): if user_output[0] is None: logger.debug( "Looks like user's function returned None: output={:}". format(user_output)) passed = False expected_output = "Your function returned None. It shouldn't do that." else: try: user_test_case = problem.ProblemTestCase( None, problem.INPUT_VARS, input_tuple, problem.OUTPUT_VARS, user_output) passed, correct_test_case = problems.test_case.test_case_solution_correct( tc, user_test_case, problem.ATOL, problem.RTOL) expected_output = correct_test_case.output_tuple() except Exception: explanation = "Internal engine error during user test case verification. Returning falcon HTTP 500." add_error_to_response(resp, explanation, traceback.format_exc(), falcon.HTTP_500, code_filename) return if passed: n_passes += 1 test_case_details.append({ "testCaseType": tc.test_type.test_name, "input": input_tuple, "output": user_output, "expected": expected_output, "inputString": str(input_tuple), "outputString": str(user_output), "expectedString": str(expected_output), "passed": passed, "processInfo": p_info, }) if "DYNAMIC_RESOURCES" in tc.input: for dynamic_resource_path in dynamic_resources: logger.debug("Deleting dynamic resource: {:s}".format( dynamic_resource_path)) util.delete_file(dynamic_resource_path) logger.info("Passed %d/%d test cases.", n_passes, n_cases) resp_dict = { "success": True if n_passes == n_cases else False, "numTestCases": n_cases, "numTestCasesPassed": n_passes, "testCaseDetails": test_case_details, } resp.status = falcon.HTTP_200 resp.set_header("Access-Control-Allow-Origin", "*") resp.body = json.dumps(resp_dict) util.delete_file(code_filename) logger.debug("User code file deleted: {:s}".format(code_filename)) for file_path in static_resources: logging.debug("Deleting static resource {:s}".format(file_path)) util.delete_file(file_path)
def run(self, container_id, code_filename, function_name, input_tuples, correct_output_tuples): logger.info("Running {:s} with {:d} inputs...".format( code_filename, len(input_tuples))) run_id = code_filename.split(".")[0] # Pickle all the input tuples into one file. if self.file_type == "pickle": input_pickle = "{:s}.input.pickle".format(run_id) with open(input_pickle, mode="wb") as f: logger.debug( "Pickling input tuples in {:s}...".format(input_pickle)) pickle.dump(input_tuples, file=f, protocol=pickle.HIGHEST_PROTOCOL) elif self.file_type == "json": input_pickle = "{:s}.input.json".format(run_id) with open(input_pickle, mode="w") as f: logger.debug( "Pickling input tuples in {:s}...".format(input_pickle)) json.dump(input_tuples, f, cls=NumpyEncoder) # Copy the relevant boilerplate run script into the current working directory. runner_file = "{:s}.run.py".format(run_id) shutil.copy(self.run_script_filename, runner_file) # Replace "$FUNCTION_NAME" in the run script with the actual function name to call # (as defined in the problem module). logger.debug("Replacing $FUNCTION_NAME->{:s} in {:s}...".format( function_name, runner_file)) with fileinput.FileInput(runner_file, inplace=True) as f: for line in f: print(line.replace("$FUNCTION_NAME", function_name), end="") # Push all the files we need into the Linux container. required_files = [code_filename, runner_file, input_pickle] if self.push_correct_output: correct_output_pickle = "{:s}.correct.pickle".format(run_id) with open(correct_output_pickle, mode="wb") as f: logger.debug( "Pickling correct output tuples in {:s}...".format( correct_output_pickle)) pickle.dump(correct_output_tuples, file=f, protocol=pickle.HIGHEST_PROTOCOL) required_files.append(correct_output_pickle) for file_name in required_files + self.util_files: source_path = file_name target_path = "/root/{:s}".format(file_name) try: push_stdout = docker_file_push(container_id, source_path, target_path) except subprocess.CalledProcessError: # If pushing a file fails then declutter remaining files and raise an exception. for fn in required_files: util.delete_file(fn) raise FilePushError() # Tell the Linux container to execute the run script that will run the user's code. runner_path = "/root/{}".format(runner_file) command = ["python3", runner_path] logger.debug("Trying to execute function in docker...") try: exec_retval, exec_stdout = docker_execute(container_id, command) except docker.errors.APIError: # If we fail to connect through docker, clean up the files for fn in required_files: util.delete_file(fn) raise EngineExecutionError(exec_stdout) # Or if the code failed to run properly, clean up the files if exec_retval != 0: for fn in required_files: util.delete_file(fn) # The `timeout` Linux command exits with return code 124 if the command times out. if exec_retval == 124: raise EngineTimeoutError("Your code took too long to run.") else: raise EngineExecutionError(exec_stdout) user_outputs = [] process_infos = [] # Read all the output that the user produced. # Each test case's output will end up it one output pickle file. for i, _ in enumerate(input_tuples): output_pickle = "{:s}.output{:d}.pickle".format(run_id, i) source_path = "/root/{:s}".format(output_pickle) target_path = output_pickle try: pull_stdout = docker_file_pull(container_id, source_path, target_path) except CalledProcessError: for fn in required_files: util.delete_file(fn) raise FilePullError(pull_stdout) with open(output_pickle, mode="rb") as f: output_dict = pickle.load(f) # TODO: exec_retval will always be zero here, so why return it? p_info = { "return_value": exec_retval, "stdout": exec_stdout, "runtime": output_dict["runtime"], "max_mem_usage": output_dict["max_mem_usage"], } user_outputs.append(output_dict["user_output"]) process_infos.append(p_info) logger.debug("runtime: {:g} s, max_mem_usage: {:g} kB".format( p_info["runtime"], p_info["max_mem_usage"])) util.delete_file(output_pickle) logger.info("Finished running user code.") for fn in required_files: util.delete_file(fn) return user_outputs, process_infos