def upload_grade(api: markusapi.Markus,
assignment_id: int,
criteria: Dict[str, Tuple[str, float]],
utorid: str,
gbook: gb.GradeBook = None,
gf_file: TextIO = None,
group_id: int = None,
complete=True):
"""Upload grades for one student.
criteria maps test-name-in-gf-file to (criteria-name, out-of) on MarkUs.
criteria on MarkUs needs to be set up beforehand. Can't find a way
to upload it with API... TODO.
if gbook is None, reload from gf_file
if group_id is None, download group info from MarkUs.
if complete, set MarkUs submission status to "complete"
"""
if group_id is None:
try:
group_id = _get_group_id(api, assignment_id, utorid)
except NoMarkUsGroupError as error:
print(error)
return
if gbook is None:
gbook = gb.GradeBook.load_gf_file(gf_file, 'utorid', True)
try:
grades = gbook.get_student_grades_by_utorid(utorid)
criteria_mark_map = {
criteria[test_name][0]: grades.get_grade(test_name) *
criteria[test_name][2] / criteria[test_name][1]
for test_name in criteria
}
except Exception as exn: # no grade for this student
print('Warning: no grades info for {}. Uploading 0. ({})'.format(
utorid, exn))
criteria_mark_map = {name: 0 for (name, _, _) in criteria.values()}
# HACK: undo complete state
api.update_marking_state(assignment_id, group_id, 'incomplete')
response = api.update_marks_single_group(criteria_mark_map, assignment_id,
group_id)
# 200 is success
if response.get('status', 0) == 500 or response.get('code', 0) != '200':
print('Could not upload grades for {}: {}.'.format(utorid, response))
return
if complete:
response = api.update_marking_state(assignment_id, group_id,
'complete')
# 200 is success
if response.get('status', 0) == 500 or response.get('code', 0) != '200':
print('Could not set state to complete for {}: {}.'.format(
utorid, response))
def upload_result_file(api: markusapi.Markus,
assignment_id: int,
local_dir: str,
result_file_name: str,
utorid: str,
group_id: int = None):
"""Upload local_dir/utorid/result_file_name into the student repo on
MarkUs.
if group_id is None, download groups from MarkUs and find the group_id.
"""
if group_id is None:
try:
group_id = _get_group_id(api, assignment_id, utorid)
except NoMarkUsGroupError as error:
print(error)
return
result_file_path = os.path.join(local_dir, utorid, result_file_name)
try:
with open(result_file_path) as result_file:
contents = result_file.read()
except FileNotFoundError:
print('Warning: no result file for {}.'.format(utorid))
return
response = api.upload_file_to_repo(assignment_id, group_id,
result_file_name, contents)
# 201 is success
if response.get('status', 0) == 500 or response.get('code', 0) != '201':
print('Could not upload result file for {}: {}.'.format(
utorid, response))
def test_init_parse_url(self):
api_key = ''
url = 'https://markus.com/api/users?id=1'
obj = Markus(api_key, url)
assert obj.parsed_url.scheme == 'https'
assert obj.parsed_url.netloc == 'markus.com'
assert obj.parsed_url.path == '/api/users'
assert obj.parsed_url.query == 'id=1'
def get_utorid_to_group(api: markusapi.Markus,
assignment_id: int) -> Dict[str, dict]:
"""Return a mapping from utroid to MarkUs group record."""
return {
group['group_name']: group
for group in api.get_groups(assignment_id)
}
def upload_result_files(api: markusapi.Markus, assignment_id: int,
local_dir: str, result_file_name: str):
"""Upload local_dir/utorid/result_file_name into each student repo on
MarkUs.
"""
groups = api.get_groups(assignment_id)
for group in groups:
group_id, utorid = group['id'], group['group_name']
upload_result_file(api, assignment_id, local_dir, result_file_name,
utorid, group_id)
def run_test(markus_address, server_api_key, test_scripts, hooks_script, files_path,
assignment_id, group_id, group_repo_name, submission_id, run_id, enqueue_time):
"""
Run autotesting tests using the tests in test_scripts on the files in files_path.
This function should be used by an rq worker.
"""
results = []
error = None
time_to_service = int(round(time.time() - enqueue_time, 3) * 1000)
test_script_path = test_script_directory(markus_address, assignment_id)
hooks_script_path = os.path.join(test_script_path, hooks_script) if hooks_script else None
hooks_module, all_hooks_error = load_hooks(hooks_script_path) if hooks_script_path else (None, '')
api = Markus(server_api_key, markus_address)
try:
job = rq.get_current_job()
update_pop_interval_stat(job.origin)
with tester_user() as user_data:
test_username = user_data.get('username')
tests_path = user_data['worker_dir']
hooks_kwargs = {'api': api,
'tests_path': tests_path,
'assignment_id': assignment_id,
'group_id': group_id,
'group_repo_name' : group_repo_name}
try:
setup_files(files_path, tests_path, test_scripts, hooks_script,
markus_address, assignment_id)
all_hooks_error += run_hooks(hooks_module, HOOK_NAMES['before_all'], kwargs=hooks_kwargs)
cmd = test_run_command(test_username=test_username)
results = run_test_scripts(cmd,
test_scripts,
tests_path,
test_username,
hooks_module,
hook_kwargs=hooks_kwargs)
finally:
stop_tester_processes(test_username)
all_hooks_error += run_hooks(hooks_module, HOOK_NAMES['after_all'], kwargs=hooks_kwargs)
clear_working_directory(tests_path, test_username)
except Exception as e:
error = str(e)
finally:
results_data = finalize_results_data(results, error, all_hooks_error, time_to_service)
store_results(results_data, markus_address, assignment_id, group_id, submission_id)
report(results_data, api, assignment_id, group_id, run_id)
def upload_grades(api: markusapi.Markus,
assignment_id: int,
gf_file: TextIO,
criteria: Dict[str, Tuple[str, float]],
complete=True):
"""Upload grades.
criteria maps test-name-in-gf-file to (criteria-name, out-of) on MarkUs.
criteria on MarkUs needs to be set up beforehand. Can't find a way
to upload it with API... TODO.
if complete, set MarkUs submission status to "complete"
"""
gbook = gb.GradeBook.load_gf_file(gf_file, 'utorid', True)
groups = api.get_groups(assignment_id)
for group in groups:
group_id, utorid = group['id'], group['group_name']
upload_grade(api, assignment_id, criteria, utorid, gbook, None,
group_id, complete)
def get_submissions(api: markusapi.Markus, assignment_id: int,
markus_files: List[str], local_dir: str,
local_files: List[str]):
"""Download all submissions for assignment_id and store them locally.
If markus_files is None, then download a zip file of entire
submission. The name of the local zip file must be specified in
local_files[0].
markus_files: names/paths of the files on MarkUs, for each
student, to download.
local_dir, local_files: write files locally in local_dir/utorid/local_file,
for each student (utorid).
"""
groups = api.get_groups(assignment_id)
for group in groups:
group_id, utorid = group['id'], group['group_name']
get_submission(api, assignment_id, markus_files, local_dir,
local_files, utorid, group_id)
def test_decode_text_response(self, **kwargs):
result = Markus.decode_text_response(**kwargs)
assert isinstance(result, str)
def test_get_path(self, kwargs):
path = Markus.get_path(**kwargs)
for k, v in kwargs.items():
assert k + (f'/{v}' if v is not None else '') in path
def dummy_markus(scheme='http'):
return Markus('', f'{scheme}://localhost:8080')
returning a map from criteria title to mark.
Criteria titles need to be properly formatted, as they appear
in the assignment's rubric (punctuation included).
Marks need to be valid numerics, or 'nil'.
"""
d = {}
d['My Criteria 1.'] = 1.0
d['My Criteria 2.'] = 'nil'
return d
""" --------Ideally, nothing below need be touched-------- """
# Initialize an instance of the API class
api = Markus(API_KEY, ROOT_URL)
print('Initialized Markus object successfully.')
group_names = api.get_groups(ASSIGNMENT_ID).keys()
# Upload the test results.
for group in group_names:
with open(ROOT_DIR + '/' + group + '/' + FILE_NAME) as open_file:
try:
file_contents = open_file.read()
api.upload_test_results(ASSIGNMENT_ID, group, FILE_NAME,
file_contents)
except:
print('Error: uploading results for {} failed.'.format(locals()))
print('Done uploading results.')
# All test results files are now uploaded.
Parse the contents of a test results file (as a string),
returning a map from criteria title to mark.
Criteria titles need to be properly formatted, as they appear
in the assignment's marking scheme (punctuation included).
Marks need to be valid numerics, or 'nil'.
If the criterion is a Rubric, the mark just needs to be the
rubric level, and will be multiplied by the weight automatically.
"""
d = {'My Criterion 1.': 1.0, 'My Criterion 2.': 'nil'}
return d
""" --------Ideally, nothing below need be touched-------- """
# Initialize an instance of the API class
api = Markus(API_KEY, ROOT_URL)
print('Initialized Markus object successfully.')
groups = api.get_groups(ASSIGNMENT_ID)
for group in groups:
group_name = group['group_name']
group_id = group['id']
try:
with open(ROOT_DIR + '/' + group_name + '/' + FILE_NAME) as open_file:
file_contents = open_file.read()
# Upload the feedback file
try:
response = api.upload_feedback_file(ASSIGNMENT_ID, group_id, FILE_NAME, file_contents)
print('Uploaded feedback file for {}, Markus responded: {}'.format(group_name, response))
except:
print('Error: uploading feedback file for {} failed'.format(group_name))
returning a map from criteria title to mark.
Criteria titles need to be properly formatted, as they appear
in the assignment's rubric (punctuation included).
Marks need to be valid numerics, or 'nil'.
"""
d = {}
d['My Criteria 1.'] = 1.0
d['My Criteria 2.'] = 'nil'
return d
""" --------Ideally, nothing below need be touched-------- """
# Initialize an instance of the API class
api = Markus(API_KEY, ROOT_URL)
print('Initialized Markus object successfully.')
group_names = api.get_groups(ASSIGNMENT_ID).keys()
# Upload the test results.
for group in group_names:
with open(ROOT_DIR + '/' + group + '/' + FILE_NAME) as open_file:
try:
file_contents = open_file.read()
api.upload_test_results(ASSIGNMENT_ID, group,
FILE_NAME, file_contents)
except:
print('Error: uploading results for {} failed.'.format(locals()))
print('Done uploading results.')
# All test results files are now uploaded.
def test_decode_text_response(self, in_dict):
res = json.dumps(in_dict).encode()
result = Markus.decode_text_response(['', '', res])
assert isinstance(result, str)
Criteria titles need to be properly formatted, as they appear
in the assignment's marking scheme (punctuation included).
Marks need to be valid numerics, or 'nil'.
If the criterion is a Rubric, the mark just needs to be the
rubric level, and will be multiplied by the weight automatically.
"""
d = {'My Criterion 1.': 1.0, 'My Criterion 2.': 'nil'}
return d
""" --------Ideally, nothing below need be touched-------- """
if __name__ == '__main__':
# Initialize an instance of the API class
api = Markus(API_KEY, ROOT_URL)
print('Initialized Markus object successfully.')
groups = api.get_groups(ASSIGNMENT_ID)
for group in groups:
group_name = group['group_name']
group_id = group['id']
try:
with open(os.path.join(ROOT_DIR, group_name,
FILE_NAME)) as open_file:
file_contents = open_file.read()
# Upload the feedback file
try:
response = api.upload_feedback_file(
ASSIGNMENT_ID, group_id, FILE_NAME, file_contents)
print(
def run_test(
markus_address,
server_api_key,
test_categories,
files_path,
assignment_id,
group_id,
group_repo_name,
submission_id,
run_id,
enqueue_time,
):
"""
Run autotesting tests using the tests in the test_specs json file on the files in files_path.
This function should be used by an rq worker.
"""
results = []
error = None
hooks_error = None
time_to_service = int(round(time.time() - enqueue_time, 3) * 1000)
test_script_path = test_script_directory(markus_address, assignment_id)
hooks_script_path = os.path.join(test_script_path, HOOKS_FILENAME)
test_specs_path = os.path.join(test_script_path, SETTINGS_FILENAME)
api = Markus(server_api_key, markus_address)
with open(test_specs_path) as f:
test_specs = json.load(f)
try:
job = rq.get_current_job()
update_pop_interval_stat(job.origin)
test_username, tests_path = tester_user()
hooks_kwargs = {
"api": api,
"assignment_id": assignment_id,
"group_id": group_id,
}
testers = {
settings["tester_type"]
for settings in test_specs["testers"]
}
hooks = Hooks(hooks_script_path,
testers,
cwd=tests_path,
kwargs=hooks_kwargs)
try:
setup_files(files_path, tests_path, markus_address, assignment_id)
cmd = run_test_command(test_username=test_username)
results, hooks_error = run_test_specs(cmd, test_specs,
test_categories, tests_path,
test_username, hooks)
finally:
stop_tester_processes(test_username)
clear_working_directory(tests_path, test_username)
except Exception as e:
error = str(e)
finally:
results_data = finalize_results_data(results, error, hooks_error,
time_to_service)
store_results(results_data, markus_address, assignment_id, group_id,
submission_id)
report(results_data, api, assignment_id, group_id, run_id)
