def add_to_result( self, result, context: 'TestContext', language: str, clip_answer_score: Callable[[Decimal], Decimal]) -> Decimal: key_prefix = ("question", self.question.title, "answer") # format of full result key: # ("question", title of question, "answer", key_1, ..., key_n) for dimension_key, dimension_value in self._get_answer_dimensions( context, language).items(): result.add(Result.key(*key_prefix, dimension_key), dimension_value, self._get_dimension_key_type(dimension_key)) score = clip_answer_score(self.current_score) for key in Result.reached_score_keys(self.question.title): result.add(key, Result.format_score(score)) for key in Result.maximum_score_keys(self.question.title): result.add( key, Result.format_score(self.question.get_maximum_score(context))) return score
def _create_result_with_details(self, driver: selenium.webdriver.Remote, report: Callable[[str], None], e: Exception, trace: str): files = dict() files['error/trace.txt'] = trace.encode('utf8') url, html, alert = get_driver_error_details(driver) if alert: files['error/alert.txt'] = alert.encode('utf8') if html: files['error/page.html'] = html.encode('utf8') try: files['error/screen.png'] = base64.b64decode( driver.get_screenshot_as_base64()) except: traceback.print_exc() filenames = map(lambda s: '%s_%s' % (self.username, s), files.keys()) error = 'test failed on url %s. for details, see %s.' % ( url, ', '.join(filenames)) report(error) return Result.from_error(Origin.recorded, e.get_error_domain(), error, files)
def add_to_result(self, result, context, language, clip_answer_score): key_prefix = ("question", self.question.title, "answer") # format of full result key: # ("question", title of question, "answer", key_1, ..., key_n) for dimension_key, dimension_value in self._get_answer_dimensions( context, language).items(): result.add(Result.key(*key_prefix, dimension_key), dimension_value, self._get_dimension_key_type(dimension_key)) score = clip_answer_score(self.current_score) for key in Result.score_keys(self.question.title): result.add_as_formatted_score(key, score) return score
def run(self, browser, master_report: Callable[[str], None]): driver = browser.driver machine_info = "running test on machine #%s (%s)." % ( self.machine_index, self.machine) master_report(machine_info) try: with run_interaction(): user_driver = UserDriver(driver, self.ilias_url, self.ilias_version, master_report, verify_ssl=self.verify_ssl) with user_driver.login(self.username, self.password): test_driver = user_driver.create_test_driver( PackagedTest(self.test_id)) test_driver.goto(self.test_url) if self.machine_index <= self.n_deterministic_machines: # some machines can operate deterministically as a well-defined baseline regression test context = RegressionContext( self.machine_index * 73939133, self.questions, self.settings, self.workarounds, self.admin_lang, self.ilias_version) else: context = RandomContext(self.questions, self.settings, self.workarounds, self.admin_lang, self.ilias_version) exam_driver = test_driver.start(self.username, context, self.questions, self.exam_configuration) try: exam_driver.add_protocol(machine_info) def report(s): master_report(s) exam_driver.add_protocol(s) robot = ExamRobot(exam_driver, context, report, self.questions, self.settings) robot.run(self.settings.test_passes) except TiltrException as e: traceback.print_exc() r = self._create_result_with_details( driver, master_report, e, traceback.format_exc()) exam_driver.add_protocol_to_result(r) return r exam_driver.close() result = exam_driver.get_expected_result( self.workarounds, self.admin_lang) result.attach_coverage(context.coverage) except TiltrException as e: traceback.print_exc() return self._create_result_with_details(driver, master_report, e, traceback.format_exc()) except WebDriverException as webdriver_error: e = InteractionException(str(webdriver_error)) traceback.print_exc() master_report("test aborted: %s" % traceback.format_exc()) return Result.from_error(Origin.recorded, e.get_error_domain(), traceback.format_exc()) except: traceback.print_exc() master_report("test aborted with an unexpected error: %s" % traceback.format_exc()) return None master_report("done running test.") return result
def run(self): # isolate the selenium driver from our main process through a fork. this fixes severe problems with # chrome zomie processes piling up inside the selenium chrome docker container. pipein, pipeout = os.pipe() if os.fork() == 0: os.close(pipein) def write(*args): os.write(pipeout, (json.dumps(args) + "\n").encode('utf8')) try: try: with self._create_browser() as browser: def report(*args): if time.time() > self.screenshot_valid_time: try: screenshot = browser.driver.get_screenshot_as_base64( ) self.screenshot_valid_time = time.time( ) + self.screenshot_refresh_time write("SCREENSHOT", screenshot) except: pass # screenshot failed write("ECHO", " ".join("%s" % arg for arg in args)) report("machine browser has wait time %d." % self.wait_time) report( 'running on user agent', browser.driver.execute_script( 'return navigator.userAgent')) expected_result = self.command.run(browser, report) except WebDriverException as webdriver_error: # we end up here in case our browser / selenium does not start and fails to close down. e = InteractionException(str(webdriver_error)) traceback.print_exc() expected_result = Result.from_error( Origin.recorded, e.get_error_domain(), traceback.format_exc()) if expected_result is None: write("ERROR", "no result obtained") else: write("DONE", expected_result.to_json()) except: traceback.print_exc() write("ERROR", traceback.format_exc()) sys.exit(0) else: os.close(pipeout) try: with os.fdopen(pipein) as fdpipein: while True: line = fdpipein.readline()[:-1] if not line: break data = json.loads(line) if data[0] == 'SCREENSHOT': self.screenshot = data[1] else: self.messages.append(data) finally: os.close(pipein)
def take_exam(args): asyncio.set_event_loop(asyncio.new_event_loop()) command = args["command"] machine = command.machine batch_id = args["batch_id"] report = args["report"] result_json = None report("master", "passing take_exam to %s." % machine) try: r = requests.post("http://%s:8888/start/%s" % (machine, batch_id), data={"command_json": command.to_json()}) if r.status_code != 200: raise InteractionException("start call failed: %s" % r.status_code) report("master", "test started on %s." % machine) index = 0 while result_json is None: # we don't want too much traffic for updating machine states. only check # one at a time. monitor_mutex.acquire() try: time.sleep(1) r = requests.get("http://%s:8888/monitor/%s/%d" % (machine, batch_id, index)) finally: monitor_mutex.release() if r.status_code != 200: raise InteractionException("monitor call failed: %s" % r.status_code) messages = json.loads(r.text) for command, payload in messages: if command == "ECHO": report(machine, payload) elif command == "DONE": result_json = payload elif command == "ERROR": raise Exception(payload) else: raise InteractionException("unknown command %s" % command) index += len(messages) except TiltrException as e: traceback.print_exc() try: report("error", "machine %s failed." % machine) report("traceback", traceback.format_exc()) except: print("report failed.") return Result.from_error(Origin.recorded, e.get_error_domain(), traceback.format_exc()) except requests.exceptions.ConnectionError: traceback.print_exc() try: report("error", "machine %s failed." % machine) report("traceback", traceback.format_exc()) except: print("report failed.") return Result.from_error(Origin.recorded, ErrorDomain.interaction, traceback.format_exc()) except: traceback.print_exc() try: report("error", "machine %s failed." % machine) report("traceback", traceback.format_exc()) except: print("report failed.") return Result.from_error(Origin.recorded, ErrorDomain.integrity, traceback.format_exc()) report("master", "received take_exam results from %s." % machine) return Result(from_json=result_json)
def _apply_readjustment(self, index, master, test_driver, all_recorded_results, is_reimport): # note that this will destroy the original test's scores. usually we should have copied # this test and this should only run on a temporary copy. context = RandomContext(self.questions, self.settings, self.workarounds, self.language) protocol = self.protocols["readjustments"] def report(s): if isinstance(s, Texttable): for line in s.draw().split('\n'): report(line) else: protocol.append(s) if s: master.report(s) report("") report("## READJUSTMENT ROUND %d%s" % (index + 1, " (AFTER REIMPORT)" if is_reimport else "")) report("") index = 0 retries = 0 modified_questions = set() def close_stats_window(): master.driver.execute_script(""" (function() { var overlay = document.getElementsByClassName("ilOverlay")[0]; if (overlay) { overlay.style.display = "none"; } }()) """) while True: with wait_for_page_load(master.driver): test_driver.goto_scoring_adjustment() links = [] for a in master.driver.find_element_by_name( "questionbrowser").find_elements_by_css_selector( "table a"): question_title = a.text.strip() if question_title in self.questions: links.append((a, self.questions[question_title])) if index >= len(links): break link, question = links[index] with wait_for_page_load(master.driver): link.click() close_stats_window() old_maximum_score = question.get_maximum_score() try: while True: report('') report('### QUESTION "%s"' % question.title.upper()) report('') readjusted, removed_answer_keys = question.readjust_scores( master.driver, context, report) if not readjusted: # this question type does not support readjustments. break modified_questions.add(question.title) master.report("saving.") with wait_for_page_load(master.driver): master.driver.find_element_by_name( "cmd[savescoringfortest]").click() close_stats_window() if not master.driver.find_elements_by_css_selector( ".alert-danger"): # readjustment was applied successfully. # in certain cases (matching questions), reassessment can remove already given answers # from the result sets. for key in removed_answer_keys: for result in all_recorded_results: result.remove( Result.key("question", question.title, "answer", key)) break else: maximum_score = question.get_maximum_score() if maximum_score > 0: report( "ILIAS rejected new scores even though they are valid (%f)." % maximum_score) else: report("ILIAS rejected invalid new scores.") except TimeoutException: retries += 1 if retries >= 1: raise InteractionException( "failed to readjust scores. giving up.") else: master.report("readjustment failed, retrying.") continue report("") report("maximum score went from %s to %s." % (old_maximum_score, question.get_maximum_score())) index += 1 retries = 0 # recompute user score's for all questions. report("") report("## REASSESSING EXPECTED USER SCORES") report("") for user, result in zip(self.users, all_recorded_results): report("### USER %s" % user.get_username()) new_scores_table = Texttable() new_scores_table.set_deco(Texttable.HEADER) new_scores_table.set_cols_dtype(['a', 'a']) answers_table = Texttable() answers_table.set_deco(Texttable.HEADER) answers_table.set_cols_dtype(['a', 'a']) for question_title, question in self.questions.items(): if question_title not in modified_questions: continue score = question.compute_score_from_result(result, context) score = remove_trailing_zeros(str(score)) for key in Result.score_keys(question_title): result.update(key, score) new_scores_table.add_row([question_title, score]) answers_table.add_row(["QUESTION " + question_title, ""]) for key, value in result.properties.items(): if key[0] == "question" and key[ 1] == question_title and key[2] == "answer": dimensions = key[3:] if len(dimensions) == 1: dimension = str(dimensions[0]) else: dimension = str( list(map(lambda x: '"%s"' % str(x), dimensions))) answers_table.add_row([dimension, value]) answers_table.add_row(["", ""]) report("recomputed these scores:") for line in new_scores_table.draw().split("\n"): report(line) report("") report("based on these answers:") for line in answers_table.draw().split("\n"): report(line) report("") maximum_score = Decimal(0) for question in self.questions.values(): maximum_score += question.get_maximum_score() # recompute reached scores and marks. for result in all_recorded_results: result.update(("xls", "score_maximum"), maximum_score) reached_score = Decimal(0) for value in result.scores(): reached_score += Decimal(value) for channel in ("xls", "gui"): result.update((channel, "score_reached"), remove_trailing_zeros(str(reached_score))) mark = Marks(self.exam_configuration.marks).lookup( (100 * reached_score) / maximum_score) for channel in ("xls", "gui"): result.update((channel, "short_mark"), str(mark.short).strip())
def _check_results(self, index, master, test_driver, workbook, all_recorded_results, is_reimport): all_assertions_ok = True gui_stats = test_driver.get_statistics_from_web_gui( [user.get_username() for user in self.users]) pdfs = test_driver.export_pdf() prefix = 'reimport/' if is_reimport else 'original/' for user, recorded_result in zip(self.users, all_recorded_results): master.report("checking results for user %s." % user.get_username()) # fetch and check results. assert self.questions is not None ilias_result = workbook_to_result(workbook, user.get_username(), self.questions, self.workarounds, master.report) # check score via statistics gui as well. ilias_result.add(("gui", "score_reached"), gui_stats[user.get_username()].score) ilias_result.add(("gui", "short_mark"), gui_stats[user.get_username()].short_mark) # add scores from pdf. for question_title, score in pdfs[ user.get_username()].scores.items(): ilias_result.add( ("pdf", "question", Result.normalize_question_title(question_title), "score"), score) # save pdf in tiltr database. self.files[prefix + "%s.pdf" % user.get_username()] = pdfs[user.get_username()].bytes # perform response checks. def report(message): if message: self.protocols[user.get_username()].append(message) self.protocols[user.get_username()].extend([ "", "# VERIFICATION%s%s" % ((" FOR READJUSTMENT ROUND %d" % index) if index > 0 else "", " (FOR REIMPORTED VERSION)" if is_reimport else ""), "" ]) if not recorded_result.check_against(ilias_result, report, self.workarounds): message = "verification failed for user %s." % user.get_username( ) master.report(message) self.protocols["log"].append("[fail] " + message) all_assertions_ok = False if not is_reimport: # add coverage info. for question_title, answers in ilias_result.get_answers( ).items(): question = self.questions[question_title] question.add_export_coverage(self.coverage, answers, self.language) return all_assertions_ok