def update_test_specs(files_path, assignment_id, markus_address, test_specs):
"""
Copy new test scripts for a given assignment to from the files_path
to a new location. Indicate that these new test scripts should be used instead of
the old ones. And remove the old ones when it is safe to do so (they are not in the
process of being copied to a working directory).
This function should be used by an rq worker.
"""
# TODO: catch and log errors
test_script_dir_name = "test_scripts_{}".format(int(time.time()))
clean_markus_address = clean_dir_name(markus_address)
new_dir = os.path.join(*stringify(TEST_SCRIPT_DIR, clean_markus_address,
assignment_id, test_script_dir_name))
new_files_dir = os.path.join(new_dir, FILES_DIRNAME)
move_tree(files_path, new_files_dir)
if "hooks_file" in test_specs:
src = os.path.join(new_files_dir, test_specs["hooks_file"])
if os.path.isfile(src):
os.rename(src, os.path.join(new_dir, HOOKS_FILENAME))
test_specs = create_tester_environments(new_files_dir, test_specs)
settings_filename = os.path.join(new_dir, SETTINGS_FILENAME)
with open(settings_filename, "w") as f:
json.dump(test_specs, f)
old_test_script_dir = test_script_directory(markus_address, assignment_id)
test_script_directory(markus_address, assignment_id, set_to=new_dir)
if old_test_script_dir is not None:
with fd_open(old_test_script_dir) as fd:
with fd_lock(fd, exclusive=True):
destroy_tester_environments(old_test_script_dir)
shutil.rmtree(old_test_script_dir,
onerror=ignore_missing_dir_error)
def _check_test_script_files_exist(client: ClientType) -> None:
"""
Raise a TestScriptFilesError if the tests script files for this test cannot be found.
"""
if test_script_directory(client.unique_script_str()) is None:
raise TestScriptFilesError(
"cannot find test script files: please upload some before running tests"
)
def update_test_specs(client_type: str, client_data: Dict) -> None:
"""
Download test script files and test specs for the given client.
Indicate that these new test scripts should be used instead of
the old ones. Remove the old ones when it is safe to do so (they are not in the
process of being copied to a working directory).
"""
# TODO: catch and log errors
client = get_client(client_type, client_data)
test_script_dir_name = "test_scripts_{}".format(int(time.time()))
unique_script_str = client.unique_script_str()
new_dir = os.path.join(TEST_SCRIPT_DIR, clean_dir_name(unique_script_str),
test_script_dir_name)
test_specs = client.get_test_specs()
new_files_dir = os.path.join(new_dir, FILES_DIRNAME)
os.makedirs(new_files_dir, exist_ok=True)
client.write_test_files(new_files_dir)
filenames = [
os.path.relpath(path, new_files_dir)
for fd, path in recursive_iglob(new_files_dir) if fd == "f"
]
try:
validate_against_schema(test_specs, filenames)
except Exception as e:
sys.stderr.write(f"Form Validation Error: {str(e)}")
sys.exit(1)
test_specs = _create_tester_environments(new_files_dir, test_specs)
settings_filename = os.path.join(new_dir, SETTINGS_FILENAME)
with open(settings_filename, "w") as f:
json.dump(test_specs, f)
old_test_script_dir = test_script_directory(unique_script_str)
test_script_directory(unique_script_str, set_to=new_dir)
if old_test_script_dir is not None and os.path.isdir(old_test_script_dir):
_destroy_tester_environments(old_test_script_dir)
shutil.rmtree(old_test_script_dir, onerror=ignore_missing_dir_error)
def copy_test_script_files(markus_address, assignment_id, tests_path):
"""
Copy test script files for a given assignment to the tests_path
directory if they exist. tests_path may already exist and contain
files and subdirectories.
"""
test_script_outer_dir = redis_management.test_script_directory(
markus_address, assignment_id)
test_script_dir = os.path.join(test_script_outer_dir, FILES_DIRNAME)
if os.path.isdir(test_script_dir):
with fd_open(test_script_dir) as fd:
with fd_lock(fd, exclusive=False):
return copy_tree(test_script_dir, tests_path)
return []
def _copy_test_script_files(client: ClientType,
tests_path: str) -> List[Tuple[str, str]]:
"""
Copy test script files for a given assignment to the tests_path
directory if they exist. tests_path may already exist and contain
files and subdirectories.
If the call to copy_tree raises a FileNotFoundError because the test
script directory has changed, retry the call to this function.
"""
test_script_outer_dir = test_script_directory(client.unique_script_str())
test_script_dir = os.path.join(test_script_outer_dir, FILES_DIRNAME)
if os.path.isdir(test_script_dir):
try:
return copy_tree(test_script_dir, tests_path)
except FileNotFoundError:
if test_script_directory(
client.unique_script_str()) != test_script_outer_dir:
_clear_working_directory(tests_path, getpass.getuser())
return _copy_test_script_files(client, tests_path)
else:
raise
return []
def _get_job_timeouts(client: ClientType, multiplier: float = 1.5) -> int:
"""
Return an integer equal to multiplier times the sum of all timeouts in the
test_specs dictionary.
"""
test_files_dir = test_script_directory(client.unique_script_str())
with open(os.path.join(test_files_dir, SETTINGS_FILENAME)) as f:
test_specs = json.load(f)
total_timeout = 0
for settings in test_specs["testers"]:
for test_data in settings["test_data"]:
total_timeout += test_data["timeout"]
if total_timeout:
return int(total_timeout * multiplier)
raise TestParameterError(f"There are no tests to run")
def enqueue_test(user_type, batch_id, **kw):
"""
Enqueue a test run job with keyword arguments specified in **kw
"""
kw["enqueue_time"] = time.time()
queue = _get_queue(user_type=user_type, batch_id=batch_id, **kw)
_check_args(run_test, kwargs=kw)
_check_test_script_files_exist(**kw)
test_files_dir = test_script_directory(kw["markus_address"], kw["assignment_id"])
with open(os.path.join(test_files_dir, SETTINGS_FILENAME)) as f:
test_specs = json.load(f)
_print_queue_info(queue)
timeout = _get_job_timeout(test_specs, kw["test_categories"])
queue.enqueue_call(
run_test, kwargs=kw, job_id=_format_job_id(**kw), timeout=timeout
)
def test_sets_value(self, redis_conn):
""" Gets existing value """
rm.test_script_directory("c", set_to="d")
assert redis_conn.return_value.hget(
config["redis", "_current_test_script_hash"], "c") == b"d"
def test_gets_value(self, redis_conn):
""" Gets existing value """
redis_conn.return_value.hset(
config["redis", "_current_test_script_hash"], "a", "b")
assert rm.test_script_directory("a") == "b"
def run_test(
markus_address,
server_api_key,
test_categories,
files_path,
assignment_id,
group_id,
group_repo_name,
submission_id,
run_id,
enqueue_time,
):
"""
Run autotesting tests using the tests in the test_specs json file on the files in files_path.
This function should be used by an rq worker.
"""
results = []
error = None
hooks_error = None
time_to_service = int(round(time.time() - enqueue_time, 3) * 1000)
test_script_path = test_script_directory(markus_address, assignment_id)
hooks_script_path = os.path.join(test_script_path, HOOKS_FILENAME)
test_specs_path = os.path.join(test_script_path, SETTINGS_FILENAME)
api = Markus(server_api_key, markus_address)
with open(test_specs_path) as f:
test_specs = json.load(f)
try:
job = rq.get_current_job()
update_pop_interval_stat(job.origin)
test_username, tests_path = tester_user()
hooks_kwargs = {
"api": api,
"assignment_id": assignment_id,
"group_id": group_id,
}
testers = {
settings["tester_type"]
for settings in test_specs["testers"]
}
hooks = Hooks(hooks_script_path,
testers,
cwd=tests_path,
kwargs=hooks_kwargs)
try:
setup_files(files_path, tests_path, markus_address, assignment_id)
cmd = run_test_command(test_username=test_username)
results, hooks_error = run_test_specs(cmd, test_specs,
test_categories, tests_path,
test_username, hooks)
finally:
stop_tester_processes(test_username)
clear_working_directory(tests_path, test_username)
except Exception as e:
error = str(e)
finally:
results_data = finalize_results_data(results, error, hooks_error,
time_to_service)
store_results(results_data, markus_address, assignment_id, group_id,
submission_id)
report(results_data, api, assignment_id, group_id, run_id)
def _get_test_specs(client: ClientType) -> Dict:
test_script_path = test_script_directory(client.unique_script_str())
test_specs_path = os.path.join(test_script_path, SETTINGS_FILENAME)
with open(test_specs_path) as f:
return json.load(f)
def _check_test_script_files_exist(markus_address, assignment_id, **_kw):
if test_script_directory(markus_address, assignment_id) is None:
raise TestScriptFilesError(
"cannot find test script files: please upload some before running tests"
)