def add_to_result(
self, result, context: 'TestContext', language: str,
clip_answer_score: Callable[[Decimal], Decimal]) -> Decimal:
key_prefix = ("question", self.question.title, "answer")
# format of full result key:
# ("question", title of question, "answer", key_1, ..., key_n)
for dimension_key, dimension_value in self._get_answer_dimensions(
context, language).items():
result.add(Result.key(*key_prefix, dimension_key), dimension_value,
self._get_dimension_key_type(dimension_key))
score = clip_answer_score(self.current_score)
for key in Result.reached_score_keys(self.question.title):
result.add(key, Result.format_score(score))
for key in Result.maximum_score_keys(self.question.title):
result.add(
key,
Result.format_score(self.question.get_maximum_score(context)))
return score
def _create_result_with_details(self, driver: selenium.webdriver.Remote,
report: Callable[[str], None],
e: Exception, trace: str):
files = dict()
files['error/trace.txt'] = trace.encode('utf8')
url, html, alert = get_driver_error_details(driver)
if alert:
files['error/alert.txt'] = alert.encode('utf8')
if html:
files['error/page.html'] = html.encode('utf8')
try:
files['error/screen.png'] = base64.b64decode(
driver.get_screenshot_as_base64())
except:
traceback.print_exc()
filenames = map(lambda s: '%s_%s' % (self.username, s), files.keys())
error = 'test failed on url %s. for details, see %s.' % (
url, ', '.join(filenames))
report(error)
return Result.from_error(Origin.recorded, e.get_error_domain(), error,
files)
def add_to_result(self, result, context, language, clip_answer_score):
key_prefix = ("question", self.question.title, "answer")
# format of full result key:
# ("question", title of question, "answer", key_1, ..., key_n)
for dimension_key, dimension_value in self._get_answer_dimensions(
context, language).items():
result.add(Result.key(*key_prefix, dimension_key), dimension_value,
self._get_dimension_key_type(dimension_key))
score = clip_answer_score(self.current_score)
for key in Result.score_keys(self.question.title):
result.add_as_formatted_score(key, score)
return score
def run(self, browser, master_report: Callable[[str], None]):
driver = browser.driver
machine_info = "running test on machine #%s (%s)." % (
self.machine_index, self.machine)
master_report(machine_info)
try:
with run_interaction():
user_driver = UserDriver(driver,
self.ilias_url,
self.ilias_version,
master_report,
verify_ssl=self.verify_ssl)
with user_driver.login(self.username, self.password):
test_driver = user_driver.create_test_driver(
PackagedTest(self.test_id))
test_driver.goto(self.test_url)
if self.machine_index <= self.n_deterministic_machines:
# some machines can operate deterministically as a well-defined baseline regression test
context = RegressionContext(
self.machine_index * 73939133, self.questions,
self.settings, self.workarounds, self.admin_lang,
self.ilias_version)
else:
context = RandomContext(self.questions, self.settings,
self.workarounds,
self.admin_lang,
self.ilias_version)
exam_driver = test_driver.start(self.username, context,
self.questions,
self.exam_configuration)
try:
exam_driver.add_protocol(machine_info)
def report(s):
master_report(s)
exam_driver.add_protocol(s)
robot = ExamRobot(exam_driver, context, report,
self.questions, self.settings)
robot.run(self.settings.test_passes)
except TiltrException as e:
traceback.print_exc()
r = self._create_result_with_details(
driver, master_report, e, traceback.format_exc())
exam_driver.add_protocol_to_result(r)
return r
exam_driver.close()
result = exam_driver.get_expected_result(
self.workarounds, self.admin_lang)
result.attach_coverage(context.coverage)
except TiltrException as e:
traceback.print_exc()
return self._create_result_with_details(driver, master_report, e,
traceback.format_exc())
except WebDriverException as webdriver_error:
e = InteractionException(str(webdriver_error))
traceback.print_exc()
master_report("test aborted: %s" % traceback.format_exc())
return Result.from_error(Origin.recorded, e.get_error_domain(),
traceback.format_exc())
except:
traceback.print_exc()
master_report("test aborted with an unexpected error: %s" %
traceback.format_exc())
return None
master_report("done running test.")
return result
def run(self):
# isolate the selenium driver from our main process through a fork. this fixes severe problems with
# chrome zomie processes piling up inside the selenium chrome docker container.
pipein, pipeout = os.pipe()
if os.fork() == 0:
os.close(pipein)
def write(*args):
os.write(pipeout, (json.dumps(args) + "\n").encode('utf8'))
try:
try:
with self._create_browser() as browser:
def report(*args):
if time.time() > self.screenshot_valid_time:
try:
screenshot = browser.driver.get_screenshot_as_base64(
)
self.screenshot_valid_time = time.time(
) + self.screenshot_refresh_time
write("SCREENSHOT", screenshot)
except:
pass # screenshot failed
write("ECHO", " ".join("%s" % arg for arg in args))
report("machine browser has wait time %d." %
self.wait_time)
report(
'running on user agent',
browser.driver.execute_script(
'return navigator.userAgent'))
expected_result = self.command.run(browser, report)
except WebDriverException as webdriver_error:
# we end up here in case our browser / selenium does not start and fails to close down.
e = InteractionException(str(webdriver_error))
traceback.print_exc()
expected_result = Result.from_error(
Origin.recorded, e.get_error_domain(),
traceback.format_exc())
if expected_result is None:
write("ERROR", "no result obtained")
else:
write("DONE", expected_result.to_json())
except:
traceback.print_exc()
write("ERROR", traceback.format_exc())
sys.exit(0)
else:
os.close(pipeout)
try:
with os.fdopen(pipein) as fdpipein:
while True:
line = fdpipein.readline()[:-1]
if not line:
break
data = json.loads(line)
if data[0] == 'SCREENSHOT':
self.screenshot = data[1]
else:
self.messages.append(data)
finally:
os.close(pipein)
def take_exam(args):
asyncio.set_event_loop(asyncio.new_event_loop())
command = args["command"]
machine = command.machine
batch_id = args["batch_id"]
report = args["report"]
result_json = None
report("master", "passing take_exam to %s." % machine)
try:
r = requests.post("http://%s:8888/start/%s" % (machine, batch_id),
data={"command_json": command.to_json()})
if r.status_code != 200:
raise InteractionException("start call failed: %s" % r.status_code)
report("master", "test started on %s." % machine)
index = 0
while result_json is None:
# we don't want too much traffic for updating machine states. only check
# one at a time.
monitor_mutex.acquire()
try:
time.sleep(1)
r = requests.get("http://%s:8888/monitor/%s/%d" %
(machine, batch_id, index))
finally:
monitor_mutex.release()
if r.status_code != 200:
raise InteractionException("monitor call failed: %s" %
r.status_code)
messages = json.loads(r.text)
for command, payload in messages:
if command == "ECHO":
report(machine, payload)
elif command == "DONE":
result_json = payload
elif command == "ERROR":
raise Exception(payload)
else:
raise InteractionException("unknown command %s" % command)
index += len(messages)
except TiltrException as e:
traceback.print_exc()
try:
report("error", "machine %s failed." % machine)
report("traceback", traceback.format_exc())
except:
print("report failed.")
return Result.from_error(Origin.recorded, e.get_error_domain(),
traceback.format_exc())
except requests.exceptions.ConnectionError:
traceback.print_exc()
try:
report("error", "machine %s failed." % machine)
report("traceback", traceback.format_exc())
except:
print("report failed.")
return Result.from_error(Origin.recorded, ErrorDomain.interaction,
traceback.format_exc())
except:
traceback.print_exc()
try:
report("error", "machine %s failed." % machine)
report("traceback", traceback.format_exc())
except:
print("report failed.")
return Result.from_error(Origin.recorded, ErrorDomain.integrity,
traceback.format_exc())
report("master", "received take_exam results from %s." % machine)
return Result(from_json=result_json)
def _apply_readjustment(self, index, master, test_driver,
all_recorded_results, is_reimport):
# note that this will destroy the original test's scores. usually we should have copied
# this test and this should only run on a temporary copy.
context = RandomContext(self.questions, self.settings,
self.workarounds, self.language)
protocol = self.protocols["readjustments"]
def report(s):
if isinstance(s, Texttable):
for line in s.draw().split('\n'):
report(line)
else:
protocol.append(s)
if s:
master.report(s)
report("")
report("## READJUSTMENT ROUND %d%s" %
(index + 1, " (AFTER REIMPORT)" if is_reimport else ""))
report("")
index = 0
retries = 0
modified_questions = set()
def close_stats_window():
master.driver.execute_script("""
(function() {
var overlay = document.getElementsByClassName("ilOverlay")[0];
if (overlay) {
overlay.style.display = "none";
}
}())
""")
while True:
with wait_for_page_load(master.driver):
test_driver.goto_scoring_adjustment()
links = []
for a in master.driver.find_element_by_name(
"questionbrowser").find_elements_by_css_selector(
"table a"):
question_title = a.text.strip()
if question_title in self.questions:
links.append((a, self.questions[question_title]))
if index >= len(links):
break
link, question = links[index]
with wait_for_page_load(master.driver):
link.click()
close_stats_window()
old_maximum_score = question.get_maximum_score()
try:
while True:
report('')
report('### QUESTION "%s"' % question.title.upper())
report('')
readjusted, removed_answer_keys = question.readjust_scores(
master.driver, context, report)
if not readjusted:
# this question type does not support readjustments.
break
modified_questions.add(question.title)
master.report("saving.")
with wait_for_page_load(master.driver):
master.driver.find_element_by_name(
"cmd[savescoringfortest]").click()
close_stats_window()
if not master.driver.find_elements_by_css_selector(
".alert-danger"):
# readjustment was applied successfully.
# in certain cases (matching questions), reassessment can remove already given answers
# from the result sets.
for key in removed_answer_keys:
for result in all_recorded_results:
result.remove(
Result.key("question", question.title,
"answer", key))
break
else:
maximum_score = question.get_maximum_score()
if maximum_score > 0:
report(
"ILIAS rejected new scores even though they are valid (%f)."
% maximum_score)
else:
report("ILIAS rejected invalid new scores.")
except TimeoutException:
retries += 1
if retries >= 1:
raise InteractionException(
"failed to readjust scores. giving up.")
else:
master.report("readjustment failed, retrying.")
continue
report("")
report("maximum score went from %s to %s." %
(old_maximum_score, question.get_maximum_score()))
index += 1
retries = 0
# recompute user score's for all questions.
report("")
report("## REASSESSING EXPECTED USER SCORES")
report("")
for user, result in zip(self.users, all_recorded_results):
report("### USER %s" % user.get_username())
new_scores_table = Texttable()
new_scores_table.set_deco(Texttable.HEADER)
new_scores_table.set_cols_dtype(['a', 'a'])
answers_table = Texttable()
answers_table.set_deco(Texttable.HEADER)
answers_table.set_cols_dtype(['a', 'a'])
for question_title, question in self.questions.items():
if question_title not in modified_questions:
continue
score = question.compute_score_from_result(result, context)
score = remove_trailing_zeros(str(score))
for key in Result.score_keys(question_title):
result.update(key, score)
new_scores_table.add_row([question_title, score])
answers_table.add_row(["QUESTION " + question_title, ""])
for key, value in result.properties.items():
if key[0] == "question" and key[
1] == question_title and key[2] == "answer":
dimensions = key[3:]
if len(dimensions) == 1:
dimension = str(dimensions[0])
else:
dimension = str(
list(map(lambda x: '"%s"' % str(x),
dimensions)))
answers_table.add_row([dimension, value])
answers_table.add_row(["", ""])
report("recomputed these scores:")
for line in new_scores_table.draw().split("\n"):
report(line)
report("")
report("based on these answers:")
for line in answers_table.draw().split("\n"):
report(line)
report("")
maximum_score = Decimal(0)
for question in self.questions.values():
maximum_score += question.get_maximum_score()
# recompute reached scores and marks.
for result in all_recorded_results:
result.update(("xls", "score_maximum"), maximum_score)
reached_score = Decimal(0)
for value in result.scores():
reached_score += Decimal(value)
for channel in ("xls", "gui"):
result.update((channel, "score_reached"),
remove_trailing_zeros(str(reached_score)))
mark = Marks(self.exam_configuration.marks).lookup(
(100 * reached_score) / maximum_score)
for channel in ("xls", "gui"):
result.update((channel, "short_mark"), str(mark.short).strip())
def _check_results(self, index, master, test_driver, workbook,
all_recorded_results, is_reimport):
all_assertions_ok = True
gui_stats = test_driver.get_statistics_from_web_gui(
[user.get_username() for user in self.users])
pdfs = test_driver.export_pdf()
prefix = 'reimport/' if is_reimport else 'original/'
for user, recorded_result in zip(self.users, all_recorded_results):
master.report("checking results for user %s." %
user.get_username())
# fetch and check results.
assert self.questions is not None
ilias_result = workbook_to_result(workbook, user.get_username(),
self.questions, self.workarounds,
master.report)
# check score via statistics gui as well.
ilias_result.add(("gui", "score_reached"),
gui_stats[user.get_username()].score)
ilias_result.add(("gui", "short_mark"),
gui_stats[user.get_username()].short_mark)
# add scores from pdf.
for question_title, score in pdfs[
user.get_username()].scores.items():
ilias_result.add(
("pdf", "question",
Result.normalize_question_title(question_title), "score"),
score)
# save pdf in tiltr database.
self.files[prefix + "%s.pdf" %
user.get_username()] = pdfs[user.get_username()].bytes
# perform response checks.
def report(message):
if message:
self.protocols[user.get_username()].append(message)
self.protocols[user.get_username()].extend([
"",
"# VERIFICATION%s%s" %
((" FOR READJUSTMENT ROUND %d" % index) if index > 0 else "",
" (FOR REIMPORTED VERSION)" if is_reimport else ""), ""
])
if not recorded_result.check_against(ilias_result, report,
self.workarounds):
message = "verification failed for user %s." % user.get_username(
)
master.report(message)
self.protocols["log"].append("[fail] " + message)
all_assertions_ok = False
if not is_reimport:
# add coverage info.
for question_title, answers in ilias_result.get_answers(
).items():
question = self.questions[question_title]
question.add_export_coverage(self.coverage, answers,
self.language)
return all_assertions_ok