def execute(self, stdin: str) -> str: if self.is_finished(): from hstest.stage_test import StageTest StageTest.curr_test_run.set_error_in_test( ErrorWithFeedback( f"The program {self} has unexpectedly terminated.\n" + "It finished execution too early, should continue running." )) raise TestedProgramFinishedEarly() if stdin is None: self.stop_input() return "" if not self.is_waiting_input(): raise UnexpectedError( f"Program {self} is not waiting for the input " + f"(state == \"{self._machine.state}\")") if self.__no_more_input: raise UnexpectedError( f"Can't pass input to the program {self} - input was prohibited." ) self._input = stdin if self.__in_background: self._machine.set_state(ProgramState.RUNNING) return "" # suspends thread while the program is executing, # waits for non-RUNNING state to be reached self._machine.set_and_wait(ProgramState.RUNNING) return self.__get_execution_output()
def test(self) -> CheckResult: create_files(self._test_case.files) # startThreads(testCase.getProcesses()) if Settings.do_reset_output: OutputHandler.reset_output() result = None try: result = self._test_runner.test(self) except BaseException as ex: self.set_error_in_test(ex) # stopThreads(testCase.getProcesses(), pool) delete_files(self._test_case.files) if result is None: self._check_errors() if isinstance(self._error_in_test, TestPassed): result = correct() if result is None: raise UnexpectedError("Result is None after testing") return result
def _init_tests(self) -> List[TestRun]: if self.runner is None: self.runner = self._init_runner() test_runs: List[TestRun] = [] test_cases: List[TestCase] = list(self.generate()) test_cases += search_dynamic_tests(self) if len(test_cases) == 0: raise UnexpectedError("No tests found") curr_test: int = 0 test_count = len(test_cases) for test_case in test_cases: test_case.source_name = self.source_name if test_case.check_func is None: test_case.check_func = self.check if test_case.attach is None: test_case.attach = self.attach curr_test += 1 test_runs += [ TestRun(curr_test, test_count, test_case, self.runner) ] return test_runs
def get_url(self, source: str = None): create_url = lambda port: f'http://localhost:{port}/' if len(self.attach.sources) == 1: return create_url(self.attach.sources[0][1]) elif len(self.attach.sources) == 0: raise UnexpectedError(f'Cannot find sources') sources_fits = [i for i in self.attach.sources if i[0] == source] if len(sources_fits) == 0: raise UnexpectedError(f'Bad source: {source}') elif len(sources_fits) > 1: raise UnexpectedError( f'Multiple sources ({len(sources_fits)}) found: {source}') return create_url(sources_fits[0][1])
def set_state(self, new_state: Any): with self.cv: if new_state not in self._transitions[self.state]: raise UnexpectedError("Cannot transit from " + self.state + " to " + new_state) self._state = new_state self.cv.notify_all()
def from_stepik(stepik_tests: List[StepikTest]) -> List['TestCase']: hs_tests = [] for test in stepik_tests: if type(test) in (list, tuple): hs_test = TestCase(stdin=test[0], attach=test[1]) elif type(test) is str: hs_test = TestCase(stdin=test) else: raise UnexpectedError("Bad test: " + str(test)) hs_tests += [hs_test] return hs_tests
def check_errors(self): if self.repeat < 0: raise UnexpectedError( f'Dynamic test "{self.method_name}" ' f'should not be repeated < 0 times, found {self.repeat}') if self.files is not None: if type(self.files) != dict: raise UnexpectedError( f"'Files' parameter in dynamic test should be of type " f"\"dict\", found {type(self.files)}.") for k, v in self.files.items(): if type(k) != str: raise UnexpectedError( f"All keys in 'files' parameter in dynamic test should be " f"of type \"str\", found {type(k)}.") if type(v) != str: raise UnexpectedError( f"All values in 'files' parameter in dynamic test should be " f"of type \"str\", found {type(v)}.")
def launch_django_application(self, test_case: TestCase): if not isinstance(test_case.attach, DjangoSettings): raise UnexpectedError( f'Django tests should have DjangoSettings class as an attach, ' f'found {type(test_case.attach)}') source = test_case.source_name if source is None or not len(source): source = 'manage' full_source = source.replace('.', os.sep) + '.py' full_path = os.path.abspath(full_source) if not os.path.exists(full_path): filename = os.path.basename(full_source) folder, file = PythonRunnableFile.runnable_searcher( file_filter=lambda _, f: f == filename) full_path = os.path.abspath(folder + os.sep + file) self.full_path = full_path self.port = self.__find_free_port(test_case.attach.tryout_ports) if test_case.attach.use_database: self.__prepare_database(test_case.attach.test_database) self.process = ProcessWrapper(sys.executable, self.full_path, 'runserver', self.port, '--noreload') i: int = 100 search_phrase = 'Starting development server at' while i: if search_phrase in self.process.stdout: test_case.attach.port = self.port break i -= 1 if self.process.is_error_happened(): i = 0 else: sleep(0.1) else: stdout = self.process.stdout.strip() stderr = self.process.stderr.strip() error_info = f'Cannot start Django server because cannot find "{search_phrase}" in process\' output' if len(stdout): error_info += '\n\nstdout:\n' + stdout if len(stderr): error_info += '\n\nstderr:\n' + stderr raise ErrorWithFeedback(error_info)
def extract_parametrized_data(self): if self.data is None: self.data = [[]] if type(self.data) not in [list, tuple]: raise UnexpectedError( f"{self.name} should be of type " f"\"list\" or \"tuple\", found {type(self.data)}.") if len(self.data) == 0: raise UnexpectedError(f"{self.name} should not be empty.") found_lists_inside = True for obj in self.data: if type(obj) not in [list, tuple]: found_lists_inside = False break if found_lists_inside: self.args_list = self.data else: self.args_list = [[obj] for obj in self.data]
def start(self, *args: str): if not self._machine.in_state(ProgramState.NOT_STARTED): raise UnexpectedError(f"Cannot start the program {self} twice") self._launch(*args) if self.__in_background: self._machine.wait_not_state(ProgramState.NOT_STARTED) return "" self._machine.wait_not_states(ProgramState.NOT_STARTED, ProgramState.RUNNING) return self.__get_execution_output()
def __init__(self, source: str = None): from hstest import StageTest runner = StageTest.curr_test_run.test_runner from hstest.testing.runner.async_main_file_runner import AsyncMainFileRunner if not isinstance(runner, AsyncMainFileRunner): raise UnexpectedError( 'TestedProgram is supported only while using AsyncMainFileRunner runner, ' 'not ' + str(type(runner))) if source is None: from hstest.stage_test import StageTest source = StageTest.curr_test_run.test_case.source_name self._program_executor: ProgramExecutor = runner.executor(source) self._run_args: Optional[List[str]] = None
def eject_next_input(self, curr_output: str) -> Optional[str]: if len(self.input_funcs) == 0: return None input_function = self.input_funcs[0] trigger_count = input_function.trigger_count if trigger_count > 0: input_function.trigger() next_func = input_function.input_function new_input: Optional[str] try: obj = next_func(curr_output) if isinstance(obj, str) or obj is None: new_input = obj elif isinstance(obj, CheckResult): if obj.is_correct: raise TestPassed() else: raise WrongAnswer(obj.feedback) else: raise UnexpectedError( 'Dynamic input should return ' + f'str or CheckResult objects only. Found: {type(obj)}') except BaseException as ex: from hstest.stage_test import StageTest StageTest.curr_test_run.set_error_in_test(ex) return None if input_function.trigger_count == 0: self.input_funcs.pop(0) if new_input is not None: new_input = clean_text(new_input) return new_input
def check(self, reply: str, attach: Any) -> CheckResult: raise UnexpectedError('Can\'t check result: override "check" method')
def __init__(self, *, stdin: DynamicInput = '', args: List[str] = None, attach: Any = None, feedback: str = '', files: Dict[str, str] = None, time_limit: int = DEFAULT_TIME_LIMIT, check_function: CheckFunction = None, feedback_on_exception: Dict[Type[Exception], str] = None, copy_to_attach: bool = False, dynamic_testing: DynamicTesting = None): self.source_name = None self.input: Optional[str] = None self.args: List[str] = [] if args is None else args self.attach: Any = attach self.feedback = feedback self.files: Dict[str, str] = {} if files is None else files self.time_limit: int = time_limit self.check_func: CheckFunction = check_function self.feedback_on_exception: Dict[Type[Exception], str] = ( {} if feedback_on_exception is None else feedback_on_exception) self.input_funcs = [] self._dynamic_testing: DynamicTesting = dynamic_testing if dynamic_testing is not None: return if copy_to_attach: if attach is not None: raise UnexpectedError('Attach is not None ' 'but copying from stdin is specified') if type(stdin) != str: raise UnexpectedError( 'To copy stdin to attach stdin should be of type str ' f'but found type {type(stdin)}') self.attach = stdin if type(stdin) == str: self.input = stdin self.input_funcs = [DynamicInputFunction(1, lambda x: stdin)] else: if type(stdin) != list: raise UnexpectedError( 'Stdin should be either of type str or list ' f'but found type {type(stdin)}') for elem in stdin: # type: RuntimeEvaluatedInput if type(elem) == DynamicInputFunction: self.input_funcs += [elem] elif type(elem) == str: self.input_funcs += [ DynamicInputFunction(1, lambda x, inp=elem: inp) ] elif str(type(elem)) in [ "<class 'function'>", "<class 'method'>" ]: self.input_funcs += [DynamicInputFunction(1, elem)] elif type(elem) in (tuple, list): if len(elem) == 2: trigger_count: int = elem[0] input_function: InputFunction = elem[1] if type(trigger_count) != int: raise UnexpectedError( f'Stdin element\'s 1st element should be of type int, ' f'found {type(trigger_count)}') if str(type(input_function)) not in [ "<class 'function'>", "<class 'method'>" ]: raise UnexpectedError( f'Stdin element\'s 2nd element should be of type function, ' f'found {type(input_function)}') self.input_funcs += [ DynamicInputFunction(trigger_count, input_function) ] else: raise UnexpectedError( f'Stdin element should have size 2, found {len(elem)}' ) else: raise UnexpectedError( f'Stdin element should have type DynamicInputFunction or ' f'tuple of size 1 or 2, found element of type {type(elem)}' )
def launch_flask_applications(self, test_case: TestCase): if not isinstance(test_case.attach, FlaskSettings): raise UnexpectedError( f'Flask tests should have FlaskSettings class as an attach, ' f'found {type(test_case.attach)}') sources = test_case.attach.sources if len(sources) == 0: raise UnexpectedError( f'Cannot find Flask applications to run, no sources were defined in tests' ) new_sources = [] for source in sources: filename, port = source full_source = filename.replace('.', os.sep) + '.py' full_path = os.path.abspath(full_source) if not os.path.exists(full_path): raise ErrorWithFeedback( f'Cannot find file named "{os.path.basename(full_path)}" ' f'in folder "{os.path.dirname(full_path)}". ' f'Check if you deleted it.') if port is None: port = self.__find_free_port(test_case.attach.tryout_ports) process = PopenWrapper(sys.executable, full_path, f'localhost:{port}') self.processes += [(full_source, process)] i: int = 100 search_phrase = '(Press CTRL+C to quit)' while i: if search_phrase in process.stderr: break i -= 1 if process.is_error_happened(): i = 0 else: sleep(0.1) else: stdout = process.stdout.strip() stderr = process.stderr.strip() error_info = ( f'Cannot start Flask server {full_source} ' f'because cannot find "{search_phrase}" in process\' output' ) if len(stdout): error_info += '\n\nstdout:\n' + stdout if len(stderr): error_info += '\n\nstderr:\n' + stderr raise ErrorWithFeedback(error_info) new_sources += [(filename, port)] test_case.attach.sources = new_sources
def check_errors(self): if self.repeat < 0: raise UnexpectedError( f'Dynamic test "{self.method_name}" ' f'should not be repeated < 0 times, found {self.repeat}')