def _setup_files(client: ClientType, tests_path: str,
test_username: str) -> None:
"""
Copy test script files and student files to the working directory tests_path,
then make it the current working directory.
The following permissions are also set:
- tests_path directory: rwxrwx--T
- test subdirectories: rwxrwx--T
- test files: rw-r-----
- student subdirectories: rwxrwx---
- student files: rw-rw----
"""
os.chmod(tests_path, 0o1770)
client.write_student_files(tests_path)
for fd, file_or_dir in recursive_iglob(tests_path):
if fd == "d":
os.chmod(file_or_dir, 0o770)
else:
os.chmod(file_or_dir, 0o770)
shutil.chown(file_or_dir, group=test_username)
script_files = _copy_test_script_files(client, tests_path)
for fd, file_or_dir in script_files:
if fd == "d":
os.chmod(file_or_dir, 0o1770)
else:
os.chmod(file_or_dir, 0o750)
shutil.chown(file_or_dir, group=test_username)
def _check_test_script_files_exist(client: ClientType) -> None:
"""
Raise a TestScriptFilesError if the tests script files for this test cannot be found.
"""
if test_script_directory(client.unique_script_str()) is None:
raise TestScriptFilesError(
"cannot find test script files: please upload some before running tests"
)
def _store_results(
client: ClientType, results_data: Dict[str, Union[List[ResultData],
str, int]]) -> None:
"""
Write the results of multiple test script runs to an output file as a json string.
"""
destination = os.path.join(
TEST_RESULT_DIR, clean_dir_name(client.unique_run_str())) + ".json"
with open(destination, "w") as f:
json.dump(results_data, f, indent=4)
def _copy_test_script_files(client: ClientType,
tests_path: str) -> List[Tuple[str, str]]:
"""
Copy test script files for a given assignment to the tests_path
directory if they exist. tests_path may already exist and contain
files and subdirectories.
If the call to copy_tree raises a FileNotFoundError because the test
script directory has changed, retry the call to this function.
"""
test_script_outer_dir = test_script_directory(client.unique_script_str())
test_script_dir = os.path.join(test_script_outer_dir, FILES_DIRNAME)
if os.path.isdir(test_script_dir):
try:
return copy_tree(test_script_dir, tests_path)
except FileNotFoundError:
if test_script_directory(
client.unique_script_str()) != test_script_outer_dir:
_clear_working_directory(tests_path, getpass.getuser())
return _copy_test_script_files(client, tests_path)
else:
raise
return []
def _get_job_timeouts(client: ClientType, multiplier: float = 1.5) -> int:
"""
Return an integer equal to multiplier times the sum of all timeouts in the
test_specs dictionary.
"""
test_files_dir = test_script_directory(client.unique_script_str())
with open(os.path.join(test_files_dir, SETTINGS_FILENAME)) as f:
test_specs = json.load(f)
total_timeout = 0
for settings in test_specs["testers"]:
for test_data in settings["test_data"]:
total_timeout += test_data["timeout"]
if total_timeout:
return int(total_timeout * multiplier)
raise TestParameterError(f"There are no tests to run")
def _get_test_specs(client: ClientType) -> Dict:
test_script_path = test_script_directory(client.unique_script_str())
test_specs_path = os.path.join(test_script_path, SETTINGS_FILENAME)
with open(test_specs_path) as f:
return json.load(f)
def _run_test_specs(
cmd: str,
client: ClientType,
test_categories: List[str],
tests_path: str,
test_username: str,
) -> Tuple[List[ResultData], str]:
"""
Run each test script in test_scripts in the tests_path directory using the
command cmd. Return the results.
"""
results = []
hook_errors = ""
test_specs = _get_test_specs(client)
for settings in test_specs["testers"]:
tester_type = settings["tester_type"]
env_dir = settings.get("env_loc", DEFAULT_ENV_DIR)
cmd_str = _create_test_script_command(env_dir, tester_type)
args = cmd.format(cmd_str)
for test_data in settings["test_data"]:
test_category = test_data.get("category", [])
if set(test_category) & set(test_categories):
start = time.time()
out, err = "", ""
timeout_expired = None
timeout = test_data.get("timeout")
try:
env_vars = _get_env_vars(test_username)
proc = subprocess.Popen(
args,
start_new_session=True,
cwd=tests_path,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
preexec_fn=set_rlimits_before_test,
env={
**os.environ,
**env_vars
},
)
try:
settings_json = json.dumps({
**settings, "test_data":
test_data
}).encode("utf-8")
out, err = proc.communicate(input=settings_json,
timeout=timeout)
except subprocess.TimeoutExpired:
if test_username == getpass.getuser():
pgrp = os.getpgid(proc.pid)
os.killpg(pgrp, signal.SIGKILL)
else:
if not _kill_with_reaper(test_username):
_kill_without_reaper(test_username)
out, err = proc.communicate()
timeout_expired = timeout
hook_errors += client.after_test(test_data, tests_path)
except Exception as e:
err += "\n\n{}".format(e)
finally:
out = decode_if_bytes(out)
err = decode_if_bytes(err)
duration = int(round(time.time() - start, 3) * 1000)
extra_info = test_data.get("extra_info", {})
results.append(
_create_test_group_result(out, err, duration,
extra_info, timeout_expired))
return results, hook_errors