def get_rate(tests_results: str, task_index: int) -> float:
tasks = TASK.tasks()
tests_results = unpack_tests_results(tests_results, tasks)
if task_index >= len(tasks) or task_index >= len(tests_results):
log_and_raise_error(
f'Task index {task_index} is more than length of tasks list', log)
return tests_results[task_index]
def run_tests(path: str) -> str:
log.info(f'Start running tests on path {path}')
output_directory = get_output_directory(
path, consts.RUNNING_TESTS_OUTPUT_DIRECTORY)
files = get_all_file_system_items(path, ct_file_condition)
str_len_files = str(len(files))
log.info(f'Found {str_len_files} files to run tests on them')
files = filter_already_tested_files(files, output_directory)
str_len_files = str(len(files))
log.info(
f'Found {str_len_files} files to run tests on them after filtering already tested'
)
tasks = TASK.tasks()
in_and_out_files_dict = create_in_and_out_dict(tasks)
for i, file in enumerate(files):
file_log_info = f'file: {str(i + 1)}/{str_len_files}'
log.info(f'Start running tests on {file_log_info}, {file}')
data = pd.read_csv(file, encoding=consts.ISO_ENCODING)
language, data = __check_tasks_on_correct_fragments(
data, tasks, in_and_out_files_dict, file_log_info)
log.info(f'Finish running tests on {file_log_info}, {file}')
write_based_on_language(output_directory, file, data, language)
return output_directory
def __get_task_by_ct_file(file: str) -> Optional[TASK]:
task_key = get_name_from_path(get_parent_folder(file),
with_extension=False)
try:
return TASK(task_key)
except ValueError:
log.info(f'Unexpected task for the file {file}')
return None
def run_tests(path: str) -> str:
"""
Run tests on all code snapshots in the data for the task.
Note: the enum class TASK (see consts.py file) must have the task key.
It also must match the name of the folder with test files in the resources/tasks_tests.
For example, if your task has key [my_key], you should add a new value into TASK const with value [my_key]
and add a new folder [my_key] with input and output files for tests in the resources/tasks_tests folder.
The test result is an array containing values for all tasks from the TASK enum class.
If the code snapshot is incorrect, then the value -1 is specified.
To deserialize this array of ratings, use the function unpack_tests_results from task_scoring.py.
To get the rate only for the current task use the calculate_current_task_rate function from plots/scoring_solutions_plots.py
For more details see
https://github.com/JetBrains-Research/task-tracker-post-processing/wiki/Data-processing:-find-tests-results-for-the-tasks
"""
log.info(f'Start running tests on path {path}')
output_directory = get_output_directory(
path, consts.RUNNING_TESTS_OUTPUT_DIRECTORY)
files = get_all_file_system_items(path, tt_file_condition)
str_len_files = str(len(files))
log.info(f'Found {str_len_files} files to run tests on them')
files = filter_already_tested_files(files, output_directory)
str_len_files = str(len(files))
log.info(
f'Found {str_len_files} files to run tests on them after filtering already tested'
)
tasks = TASK.tasks()
in_and_out_files_dict = create_in_and_out_dict(tasks)
for i, file in enumerate(files):
file_log_info = f'file: {str(i + 1)}/{str_len_files}'
log.info(f'Start running tests on {file_log_info}, {file}')
current_task = __get_task_by_ct_file(file)
if not current_task:
# We don't need to handle other files with tasks which are not in the TASK enum class
continue
data = pd.read_csv(file, encoding=consts.ISO_ENCODING)
language, data = __check_tasks_on_correct_fragments(
data,
tasks,
in_and_out_files_dict,
file_log_info,
current_task=current_task)
log.info(f'Finish running tests on {file_log_info}, {file}')
output_directory_with_user_folder = os.path.join(
output_directory, __get_user_folder_name_from_path(file))
write_based_on_language(output_directory_with_user_folder, file, data,
language)
return output_directory
def run_test_task(task: TASK, expected_pairs: Dict[SOLUTION, Tuple[int, int]], language: LANGUAGE) -> None:
remove_compiled_files()
in_and_out_files_dict = create_in_and_out_dict(TASK.tasks())
for s in SOLUTION:
code = get_source_code(task, language, s.value)
expected_pair = expected_pairs[s.value]
expected_rate = expected_pair[1] / expected_pair[0]
actual_rate = get_actual_rate(task, language, code, in_and_out_files_dict)
assert expected_rate == actual_rate, \
f'Actual rate for task {task}, language {language}, solution {s} is wrong, code:\n{code}. ' \
f'Expected rate = {expected_rate}. Actual rate = {actual_rate}'
def is_incorrect_fragment(tests_results: str) -> bool:
return TEST_RESULT.INCORRECT_CODE.value in unpack_tests_results(
tests_results, TASK.tasks())
def test_fragments(self, fragment_file: str) -> None:
in_and_out_files_dict = create_in_and_out_dict(TASK.tasks())
language = get_language_by_extension(get_extension_from_file(fragment_file))
check_tasks(TASK.tasks(), get_content_from_file(fragment_file), in_and_out_files_dict, language, False)
def str_to_task(self, task: str) -> TASK:
try:
return TASK(task)
except ValueError:
log_and_raise_error(f'Task value has to be one from the values: {TASK.tasks_values()}', self._log)
def calculate_current_task_rate(df: pd.DataFrame) -> pd.DataFrame:
file_name = df[FILE_NAME].unique()[0]
current_task = TASK(get_name_from_path(file_name, False))
return df[TESTS_RESULTS].apply(lambda x: unpack_tests_results(x, TASK.tasks())[TASK.tasks().index(current_task)])