def test_rounded_percentages():
tests_output_file_name: str = 'test_rounded_percentages.out'
output_file_folder: str = './tests/.out/'
input_data = PageGenInputData(
PageGenSettings(
ref_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='13a4c1dca0dd58d62acc741866fb945f3fe81592',
),
comp_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='614c0134750071ffe08dc376e9cc8caf210974bf',
),
),
[
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-03-4-rom_c18",
Diff("Exit value", 0, 0, 0, 0),
[Diff("bound", 13, 12, -1, -7.6923),
Diff("time", 0, 0, 0, 0)]),
],
)
generate_page(tests_output_file_name, output_file_folder, input_data)
f = open("./tests/.out/test_rounded_percentages.md", "r", encoding="utf-8")
file_content = f.read()
assert "7.692" not in file_content
assert "7.69" in file_content
def test_table_hidden_if_no_result():
tests_output_file_name: str = 'test_table_hidden_if_no_result.out'
output_file_folder: str = './tests/.out/'
input_data = PageGenInputData(
PageGenSettings(
ref_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='13a4c1dca0dd58d62acc741866fb945f3fe81592',
),
comp_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='614c0134750071ffe08dc376e9cc8caf210974bf',
),
),
[
TestResult(
'/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-03-4-rom_c18',
Diff('Exit value 1', 0, 0, 0, 0), []),
],
)
generate_page(tests_output_file_name, output_file_folder, input_data)
f = open('./tests/.out/test_table_hidden_if_no_result.md',
'r',
encoding="utf-8")
file_content = f.read()
assert '| Measure' not in file_content
assert '*The test generated no result.*' in file_content
def test_run_from_build_and_result(self):
branch = Branch.create_if_possible('some-branch', 'Some Branch')
platform = Platform.create_if_possible('some-platform', 'Some Platform')
builder = Builder.get(Builder.create('some-builder', 'Some Builder'))
test_name = ' some-test'
def create_build(build_number, revision):
timestamp = datetime.now().replace(microsecond=0)
build = Build(branch=branch, platform=platform, builder=builder, buildNumber=build_number,
revision=revision, timestamp=timestamp)
build.put()
return build
build = create_build(1, 101)
result = TestResult(name=test_name, value=123.0, build=build)
result.put()
self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 123.0)
build = create_build(2, 102)
result = TestResult(name=test_name, value=456.0, valueMedian=789.0, build=build)
result.put()
self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0)
result.valueStdev = 7.0
result.put()
self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0)
result.valueStdev = None
result.valueMin = 123.0
result.valueMax = 789.0
result.put()
self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0)
result.valueStdev = 8.0
result.valueMin = 123.0
result.valueMax = 789.0
result.put()
self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0,
statistics={'stdev': 8.0, 'min': 123.0, 'max': 789.0})
result.valueMedian = 345.0 # Median is never used by the frontend.
result.valueStdev = 8.0
result.valueMin = 123.0
result.valueMax = 789.0
result.put()
self._assert_entry(Runs._entry_from_build_and_result(build, result), build, result, 456.0,
statistics={'stdev': 8.0, 'min': 123.0, 'max': 789.0})
def test_with_data():
tests_output_file_name: str = 'test_with_data.out'
output_file_folder: str = './tests/.out/'
input_data = PageGenInputData(
PageGenSettings(
ref_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='13a4c1dca0dd58d62acc741866fb945f3fe81592',
),
comp_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='614c0134750071ffe08dc376e9cc8caf210974bf',
),
),
[
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-03-4-rom_c18",
Diff("Exit value", 0, 0, 0, 0),
[Diff("bound", 13, 12, -1, -7.6923),
Diff("time", 0, 0, 0, 0)]),
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-04-4-rom_c18",
Diff("Exit value", 1, 1, 0, 0), []),
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-07-4-rom_c18",
Diff("Exit value", -1, -1, 0, 0), []),
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP18/NurseRostering/NurseRostering-17_c18",
Diff("Exit value", -1, -1, 0, 0), []),
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP18/NurseRostering/NurseRostering-20_c18",
Diff("Exit value", -1, -1, 0, 0), []),
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP18/Rlfap/Rlfap-opt/Rlfap-scen-03-opt_c18",
Diff("Exit value", -1, -1, 0, 0), []),
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP18/Rlfap/Rlfap-opt/Rlfap-scen-06-opt_c18",
Diff("Exit value", -1, -1, 0, 0), []),
TestResult(
"/home/evaluation/evaluation/pub/bench/XCSP3/Filters-ar_1_2.xml",
Diff("Exit value", -1, -1, 0, 0), []),
],
)
generate_page(tests_output_file_name, output_file_folder, input_data)
def _create_results(self, branch, platform, builder, test_name, values, timestamps=None, starting_revision=100):
builds = []
results = []
for i, value in enumerate(values):
build = Build(branch=branch, platform=platform, builder=builder,
buildNumber=i, revision=starting_revision + i, timestamp=timestamps[i] if timestamps else datetime.now())
build.put()
result = TestResult(name=test_name, build=build, value=value)
result.put()
builds.append(build)
results.append(result)
return builds, results
def test_failure_color():
tests_output_file_name: str = 'test_failure_color.out'
output_file_folder: str = './tests/.out/'
input_data = PageGenInputData(
PageGenSettings(
ref_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='13a4c1dca0dd58d62acc741866fb945f3fe81592',
),
comp_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='614c0134750071ffe08dc376e9cc8caf210974bf',
),
),
[
TestResult(
'/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-03-4-rom_c18',
Diff('Exit value 1', -1, -1, 0, 0), []),
TestResult(
'/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-04-4-rom_c18',
Diff('Exit value 2', 2, -1, -3, -150), []),
TestResult(
'/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-07-4-rom_c18',
Diff('Exit value 3', 0, -1, -1, 0), []),
],
)
generate_page(tests_output_file_name, output_file_folder, input_data)
f = open('./tests/.out/test_failure_color.md', 'r', encoding="utf-8")
file_content = f.read()
assert '**Exit value 1:** `-1` <span style="color: red">⨯ (was `-1`)</span>' in file_content
assert '**Exit value 2:** `-1` <span style="color: red">⨯ (was `2`)</span>' in file_content
assert '**Exit value 3:** `-1` <span style="color: red">⨯ (was `0`)</span>' in file_content
def _create_results(branch, platform, builder, test_name, values):
results = []
for i, value in enumerate(values):
build = Build(branch=branch,
platform=platform,
builder=builder,
buildNumber=i,
revision=100 + i,
timestamp=datetime.now())
build.put()
result = TestResult(name=test_name, build=build, value=value)
result.put()
Test.update_or_insert(test_name, branch, platform)
results.append(result)
return results
def persist_data_to_db(self, payload):
try:
newd = json.loads(self.payload)
student_list = newd['students']
for student in student_list:
test_list = student['tests']
student = Student(name=student['firstName'],
surname=student['lastName'])
db.session.add(student)
for key, value in test_list.items():
student_test = TestResult(test=key,
result=value,
student_id=student.id)
db.session.add(student_test)
db.session.commit()
except Exception as e:
return (str(e))
def compare(self):
comp_results = []
for result_ref in self.dataref[RawDataKeys().results_key]:
try:
result_comp = self.pair(result_ref)
except Exception as e:
print(
f'Could not pair results for {result_ref[RawDataKeys().result_input_file_path_key]}. Skipping.'
)
continue
data_ref = RawData.from_result(result_ref)
data_comp = RawData.from_result(result_comp)
result = TestResult(
data_ref.path,
Comparator.make_exit_diff(data_ref, data_comp),
Comparator.make_diffs(data_ref, data_comp),
)
comp_results.append(result)
return comp_results
def test_metadata_are_used_to_generate_front_matter():
tests_output_file_name: str = 'test_metadata_are_used_to_generate_front_matter.out'
output_file_folder: str = './tests/.out/'
input_data = PageGenInputData(
PageGenSettings(
ref_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='https://github.com/chocoteam/choco-solver',
code_commit='abcdefghij',
),
comp_test_metadata=TestOutputMetadata(
test_folder_path='/home/evaluation/evaluation/pub/bench/',
page_title='Title',
page_description='Description',
repository_url='http://wesite.com/repository/',
code_commit='1234567890',
),
similar_percent_limit=50,
),
[
TestResult(
'/home/evaluation/evaluation/pub/bench/XCSP18/CrosswordDesign/CrosswordDesign-03-4-rom_c18',
Diff('Exit value 1', 50, 40, -10, -20), []),
],
)
generate_page(tests_output_file_name, output_file_folder, input_data)
f = open('./tests/.out/test_metadata_are_used_to_generate_front_matter.md',
'r',
encoding="utf-8")
file_content = f.read()
assert 'title: "Title"' in file_content
assert 'description: >\n Description\n\n Results of' in file_content
assert 'Results of [`1234567`](http://wesite.com/repository/commit/1234567890) are compared with [`abcdefg`](https://github.com/chocoteam/choco-solver/commit/abcdefghij).' in file_content
assert '≈ `-10` (`-20%`)' in file_content
