def upload_grade(api: markusapi.Markus, assignment_id: int, criteria: Dict[str, Tuple[str, float]], utorid: str, gbook: gb.GradeBook = None, gf_file: TextIO = None, group_id: int = None, complete=True): """Upload grades for one student. criteria maps test-name-in-gf-file to (criteria-name, out-of) on MarkUs. criteria on MarkUs needs to be set up beforehand. Can't find a way to upload it with API... TODO. if gbook is None, reload from gf_file if group_id is None, download group info from MarkUs. if complete, set MarkUs submission status to "complete" """ if group_id is None: try: group_id = _get_group_id(api, assignment_id, utorid) except NoMarkUsGroupError as error: print(error) return if gbook is None: gbook = gb.GradeBook.load_gf_file(gf_file, 'utorid', True) try: grades = gbook.get_student_grades_by_utorid(utorid) criteria_mark_map = { criteria[test_name][0]: grades.get_grade(test_name) * criteria[test_name][2] / criteria[test_name][1] for test_name in criteria } except Exception as exn: # no grade for this student print('Warning: no grades info for {}. Uploading 0. ({})'.format( utorid, exn)) criteria_mark_map = {name: 0 for (name, _, _) in criteria.values()} # HACK: undo complete state api.update_marking_state(assignment_id, group_id, 'incomplete') response = api.update_marks_single_group(criteria_mark_map, assignment_id, group_id) # 200 is success if response.get('status', 0) == 500 or response.get('code', 0) != '200': print('Could not upload grades for {}: {}.'.format(utorid, response)) return if complete: response = api.update_marking_state(assignment_id, group_id, 'complete') # 200 is success if response.get('status', 0) == 500 or response.get('code', 0) != '200': print('Could not set state to complete for {}: {}.'.format( utorid, response))
def upload_result_file(api: markusapi.Markus, assignment_id: int, local_dir: str, result_file_name: str, utorid: str, group_id: int = None): """Upload local_dir/utorid/result_file_name into the student repo on MarkUs. if group_id is None, download groups from MarkUs and find the group_id. """ if group_id is None: try: group_id = _get_group_id(api, assignment_id, utorid) except NoMarkUsGroupError as error: print(error) return result_file_path = os.path.join(local_dir, utorid, result_file_name) try: with open(result_file_path) as result_file: contents = result_file.read() except FileNotFoundError: print('Warning: no result file for {}.'.format(utorid)) return response = api.upload_file_to_repo(assignment_id, group_id, result_file_name, contents) # 201 is success if response.get('status', 0) == 500 or response.get('code', 0) != '201': print('Could not upload result file for {}: {}.'.format( utorid, response))
def test_init_parse_url(self): api_key = '' url = 'https://markus.com/api/users?id=1' obj = Markus(api_key, url) assert obj.parsed_url.scheme == 'https' assert obj.parsed_url.netloc == 'markus.com' assert obj.parsed_url.path == '/api/users' assert obj.parsed_url.query == 'id=1'
def get_utorid_to_group(api: markusapi.Markus, assignment_id: int) -> Dict[str, dict]: """Return a mapping from utroid to MarkUs group record.""" return { group['group_name']: group for group in api.get_groups(assignment_id) }
def upload_result_files(api: markusapi.Markus, assignment_id: int, local_dir: str, result_file_name: str): """Upload local_dir/utorid/result_file_name into each student repo on MarkUs. """ groups = api.get_groups(assignment_id) for group in groups: group_id, utorid = group['id'], group['group_name'] upload_result_file(api, assignment_id, local_dir, result_file_name, utorid, group_id)
def run_test(markus_address, server_api_key, test_scripts, hooks_script, files_path, assignment_id, group_id, group_repo_name, submission_id, run_id, enqueue_time): """ Run autotesting tests using the tests in test_scripts on the files in files_path. This function should be used by an rq worker. """ results = [] error = None time_to_service = int(round(time.time() - enqueue_time, 3) * 1000) test_script_path = test_script_directory(markus_address, assignment_id) hooks_script_path = os.path.join(test_script_path, hooks_script) if hooks_script else None hooks_module, all_hooks_error = load_hooks(hooks_script_path) if hooks_script_path else (None, '') api = Markus(server_api_key, markus_address) try: job = rq.get_current_job() update_pop_interval_stat(job.origin) with tester_user() as user_data: test_username = user_data.get('username') tests_path = user_data['worker_dir'] hooks_kwargs = {'api': api, 'tests_path': tests_path, 'assignment_id': assignment_id, 'group_id': group_id, 'group_repo_name' : group_repo_name} try: setup_files(files_path, tests_path, test_scripts, hooks_script, markus_address, assignment_id) all_hooks_error += run_hooks(hooks_module, HOOK_NAMES['before_all'], kwargs=hooks_kwargs) cmd = test_run_command(test_username=test_username) results = run_test_scripts(cmd, test_scripts, tests_path, test_username, hooks_module, hook_kwargs=hooks_kwargs) finally: stop_tester_processes(test_username) all_hooks_error += run_hooks(hooks_module, HOOK_NAMES['after_all'], kwargs=hooks_kwargs) clear_working_directory(tests_path, test_username) except Exception as e: error = str(e) finally: results_data = finalize_results_data(results, error, all_hooks_error, time_to_service) store_results(results_data, markus_address, assignment_id, group_id, submission_id) report(results_data, api, assignment_id, group_id, run_id)
def upload_grades(api: markusapi.Markus, assignment_id: int, gf_file: TextIO, criteria: Dict[str, Tuple[str, float]], complete=True): """Upload grades. criteria maps test-name-in-gf-file to (criteria-name, out-of) on MarkUs. criteria on MarkUs needs to be set up beforehand. Can't find a way to upload it with API... TODO. if complete, set MarkUs submission status to "complete" """ gbook = gb.GradeBook.load_gf_file(gf_file, 'utorid', True) groups = api.get_groups(assignment_id) for group in groups: group_id, utorid = group['id'], group['group_name'] upload_grade(api, assignment_id, criteria, utorid, gbook, None, group_id, complete)
def get_submissions(api: markusapi.Markus, assignment_id: int, markus_files: List[str], local_dir: str, local_files: List[str]): """Download all submissions for assignment_id and store them locally. If markus_files is None, then download a zip file of entire submission. The name of the local zip file must be specified in local_files[0]. markus_files: names/paths of the files on MarkUs, for each student, to download. local_dir, local_files: write files locally in local_dir/utorid/local_file, for each student (utorid). """ groups = api.get_groups(assignment_id) for group in groups: group_id, utorid = group['id'], group['group_name'] get_submission(api, assignment_id, markus_files, local_dir, local_files, utorid, group_id)
def test_decode_text_response(self, **kwargs): result = Markus.decode_text_response(**kwargs) assert isinstance(result, str)
def test_get_path(self, kwargs): path = Markus.get_path(**kwargs) for k, v in kwargs.items(): assert k + (f'/{v}' if v is not None else '') in path
def dummy_markus(scheme='http'): return Markus('', f'{scheme}://localhost:8080')
returning a map from criteria title to mark. Criteria titles need to be properly formatted, as they appear in the assignment's rubric (punctuation included). Marks need to be valid numerics, or 'nil'. """ d = {} d['My Criteria 1.'] = 1.0 d['My Criteria 2.'] = 'nil' return d """ --------Ideally, nothing below need be touched-------- """ # Initialize an instance of the API class api = Markus(API_KEY, ROOT_URL) print('Initialized Markus object successfully.') group_names = api.get_groups(ASSIGNMENT_ID).keys() # Upload the test results. for group in group_names: with open(ROOT_DIR + '/' + group + '/' + FILE_NAME) as open_file: try: file_contents = open_file.read() api.upload_test_results(ASSIGNMENT_ID, group, FILE_NAME, file_contents) except: print('Error: uploading results for {} failed.'.format(locals())) print('Done uploading results.') # All test results files are now uploaded.
Parse the contents of a test results file (as a string), returning a map from criteria title to mark. Criteria titles need to be properly formatted, as they appear in the assignment's marking scheme (punctuation included). Marks need to be valid numerics, or 'nil'. If the criterion is a Rubric, the mark just needs to be the rubric level, and will be multiplied by the weight automatically. """ d = {'My Criterion 1.': 1.0, 'My Criterion 2.': 'nil'} return d """ --------Ideally, nothing below need be touched-------- """ # Initialize an instance of the API class api = Markus(API_KEY, ROOT_URL) print('Initialized Markus object successfully.') groups = api.get_groups(ASSIGNMENT_ID) for group in groups: group_name = group['group_name'] group_id = group['id'] try: with open(ROOT_DIR + '/' + group_name + '/' + FILE_NAME) as open_file: file_contents = open_file.read() # Upload the feedback file try: response = api.upload_feedback_file(ASSIGNMENT_ID, group_id, FILE_NAME, file_contents) print('Uploaded feedback file for {}, Markus responded: {}'.format(group_name, response)) except: print('Error: uploading feedback file for {} failed'.format(group_name))
def test_decode_text_response(self, in_dict): res = json.dumps(in_dict).encode() result = Markus.decode_text_response(['', '', res]) assert isinstance(result, str)
Criteria titles need to be properly formatted, as they appear in the assignment's marking scheme (punctuation included). Marks need to be valid numerics, or 'nil'. If the criterion is a Rubric, the mark just needs to be the rubric level, and will be multiplied by the weight automatically. """ d = {'My Criterion 1.': 1.0, 'My Criterion 2.': 'nil'} return d """ --------Ideally, nothing below need be touched-------- """ if __name__ == '__main__': # Initialize an instance of the API class api = Markus(API_KEY, ROOT_URL) print('Initialized Markus object successfully.') groups = api.get_groups(ASSIGNMENT_ID) for group in groups: group_name = group['group_name'] group_id = group['id'] try: with open(os.path.join(ROOT_DIR, group_name, FILE_NAME)) as open_file: file_contents = open_file.read() # Upload the feedback file try: response = api.upload_feedback_file( ASSIGNMENT_ID, group_id, FILE_NAME, file_contents) print(
def run_test( markus_address, server_api_key, test_categories, files_path, assignment_id, group_id, group_repo_name, submission_id, run_id, enqueue_time, ): """ Run autotesting tests using the tests in the test_specs json file on the files in files_path. This function should be used by an rq worker. """ results = [] error = None hooks_error = None time_to_service = int(round(time.time() - enqueue_time, 3) * 1000) test_script_path = test_script_directory(markus_address, assignment_id) hooks_script_path = os.path.join(test_script_path, HOOKS_FILENAME) test_specs_path = os.path.join(test_script_path, SETTINGS_FILENAME) api = Markus(server_api_key, markus_address) with open(test_specs_path) as f: test_specs = json.load(f) try: job = rq.get_current_job() update_pop_interval_stat(job.origin) test_username, tests_path = tester_user() hooks_kwargs = { "api": api, "assignment_id": assignment_id, "group_id": group_id, } testers = { settings["tester_type"] for settings in test_specs["testers"] } hooks = Hooks(hooks_script_path, testers, cwd=tests_path, kwargs=hooks_kwargs) try: setup_files(files_path, tests_path, markus_address, assignment_id) cmd = run_test_command(test_username=test_username) results, hooks_error = run_test_specs(cmd, test_specs, test_categories, tests_path, test_username, hooks) finally: stop_tester_processes(test_username) clear_working_directory(tests_path, test_username) except Exception as e: error = str(e) finally: results_data = finalize_results_data(results, error, hooks_error, time_to_service) store_results(results_data, markus_address, assignment_id, group_id, submission_id) report(results_data, api, assignment_id, group_id, run_id)