def callback(test, result, failures, unexpected):
failed_result = Expectations.string_to_state_ids(
result.get('actual', ''))
expected = set(
Expectations.string_to_state_ids(
result.get('expected', '')))
if Expectations.STRING_TO_STATE_ID[
Expectations.FAIL] in expected:
expected.add(
Expectations.STRING_TO_STATE_ID[Expectations.TEXT])
expected.add(Expectations.STRING_TO_STATE_ID[
Expectations.AUDIO])
expected.add(Expectations.STRING_TO_STATE_ID[
Expectations.IMAGE])
unexpected_result = set(failed_result) - expected
if failed_result:
worst = min(failed_result)
if worst < Expectations.STRING_TO_STATE_ID[
Expectations.WARNING]:
failures[test] = min(
worst,
failures.get(
test, Expectations.STRING_TO_STATE_ID[
Expectations.PASS]))
if unexpected_result:
worst = min(unexpected_result)
if worst < Expectations.STRING_TO_STATE_ID[
Expectations.WARNING]:
unexpected[test] = min(
worst,
unexpected.get(
test, Expectations.STRING_TO_STATE_ID[
Expectations.PASS]))
def callback(test, result):
run_stats['tests_run'] += 1
actual_results = Expectations.string_to_state_ids(
result.get('actual', ''))
expected_results = set(
Expectations.string_to_state_ids(result.get(
'expected', '')))
# FAIL is a special case, we want to treat tests with TEXT, AUDIO and IMAGE diffs as failures
if Expectations.STRING_TO_STATE_ID[
Expectations.FAIL] in expected_results:
expected_results.add(
Expectations.STRING_TO_STATE_ID[Expectations.TEXT])
expected_results.add(
Expectations.STRING_TO_STATE_ID[Expectations.AUDIO])
expected_results.add(
Expectations.STRING_TO_STATE_ID[Expectations.IMAGE])
worst_result = min(actual_results)
unexpected_results = set(actual_results) - expected_results
worst_unexpected_result = min(
unexpected_results
or [Expectations.STRING_TO_STATE_ID[Expectations.PASS]])
for key, point in failure_trigger_points.items():
if worst_result <= point:
run_stats[f'tests_{key}'] += 1
if worst_unexpected_result <= point:
run_stats[f'tests_unexpected_{key}'] += 1
def unpack(self):
results = json.loads(self.tests) if self.tests else {}
for test in results.keys():
results[test] = Expectations.state_ids_to_string(
[results[test]])
results['uuid'] = self.uuid
results['start_time'] = calendar.timegm(
self.start_time.timetuple())
return results
def register(self,
configuration,
commits,
suite,
test_results,
timestamp=None):
try:
if not isinstance(suite, str):
raise TypeError(f'Expected type {str}, got {type(suite)}')
timestamp = timestamp or time.time()
if isinstance(timestamp, datetime):
timestamp = calendar.timegm(timestamp.timetuple())
with self:
uuid = self.commit_context.uuid_for_commits(commits)
ttl = int((uuid // Commit.UUID_MULTIPLIER) + self.ttl_seconds -
time.time()) if self.ttl_seconds else None
def callback(test, result, failures, unexpected):
failed_result = Expectations.string_to_state_ids(
result.get('actual', ''))
expected = set(
Expectations.string_to_state_ids(
result.get('expected', '')))
if Expectations.STRING_TO_STATE_ID[
Expectations.FAIL] in expected:
expected.add(
Expectations.STRING_TO_STATE_ID[Expectations.TEXT])
expected.add(Expectations.STRING_TO_STATE_ID[
Expectations.AUDIO])
expected.add(Expectations.STRING_TO_STATE_ID[
Expectations.IMAGE])
unexpected_result = set(failed_result) - expected
if failed_result:
worst = min(failed_result)
if worst < Expectations.STRING_TO_STATE_ID[
Expectations.WARNING]:
failures[test] = min(
worst,
failures.get(
test, Expectations.STRING_TO_STATE_ID[
Expectations.PASS]))
if unexpected_result:
worst = min(unexpected_result)
if worst < Expectations.STRING_TO_STATE_ID[
Expectations.WARNING]:
unexpected[test] = min(
worst,
unexpected.get(
test, Expectations.STRING_TO_STATE_ID[
Expectations.PASS]))
with self.cassandra.batch_query_context():
for branch in self.commit_context.branch_keys_for_commits(
commits):
failures = {}
unexpected = {}
Expectations.iterate_through_nested_results(
test_results.get('results'),
lambda test, result: callback(test,
result,
failures=failures,
unexpected=unexpected
),
)
for table in [
self.TestFailuresByCommit,
self.TestFailuresByStartTime
]:
self.configuration_context.insert_row_with_configuration(
table.__table_name__,
configuration=configuration,
suite=suite,
branch=branch,
uuid=uuid,
ttl=ttl,
sdk=configuration.sdk or '?',
start_time=timestamp,
tests=json.dumps(failures),
)
for table in [
self.UnexpectedTestFailuresByCommit,
self.UnexpectedTestFailuresByStartTime
]:
self.configuration_context.insert_row_with_configuration(
table.__table_name__,
configuration=configuration,
suite=suite,
branch=branch,
uuid=uuid,
ttl=ttl,
sdk=configuration.sdk or '?',
start_time=timestamp,
tests=json.dumps(unexpected),
)
except Exception as e:
return self.partial_status(e)
return self.partial_status()
def register(self, configuration, commits, suite, test_results, timestamp=None):
timestamp = timestamp or time.time()
if not isinstance(timestamp, datetime):
timestamp = datetime.utcfromtimestamp(int(timestamp))
try:
if not isinstance(suite, str):
raise TypeError(f'Expected type {str}, got {type(suite)}')
if isinstance(timestamp, datetime):
timestamp = calendar.timegm(timestamp.timetuple())
run_stats = test_results.get('run_stats', {})
failure_trigger_points = dict(
failed=Expectations.STRING_TO_STATE_ID[Expectations.ERROR],
timedout=Expectations.STRING_TO_STATE_ID[Expectations.TIMEOUT],
crashed=Expectations.STRING_TO_STATE_ID[Expectations.CRASH],
)
run_stats['tests_run'] = 0
for key in failure_trigger_points.keys():
run_stats[f'tests_{key}'] = 0
run_stats[f'tests_unexpected_{key}'] = 0
def callback(test, result):
run_stats['tests_run'] += 1
actual_results = Expectations.string_to_state_ids(result.get('actual', ''))
expected_results = set(Expectations.string_to_state_ids(result.get('expected', '')))
# FAIL is a special case, we want to treat tests with TEXT, AUDIO and IMAGE diffs as failures
if Expectations.STRING_TO_STATE_ID[Expectations.FAIL] in expected_results:
expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.TEXT])
expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.AUDIO])
expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.IMAGE])
worst_result = min(actual_results)
unexpected_results = set(actual_results) - expected_results
worst_unexpected_result = min(unexpected_results or [Expectations.STRING_TO_STATE_ID[Expectations.PASS]])
for key, point in failure_trigger_points.items():
if worst_result <= point:
run_stats[f'tests_{key}'] += 1
if worst_unexpected_result <= point:
run_stats[f'tests_unexpected_{key}'] += 1
Expectations.iterate_through_nested_results(test_results.get('results'), callback)
uuid = self.commit_context.uuid_for_commits(commits)
ttl = int((uuid // Commit.UUID_MULTIPLIER) + self.ttl_seconds - time.time()) if self.ttl_seconds else None
with self, self.cassandra.batch_query_context():
for branch in self.commit_context.branch_keys_for_commits(commits):
for table in [self.SuiteResultsByCommit, self.SuiteResultsByStartTime]:
self.configuration_context.insert_row_with_configuration(
table.__table_name__, configuration=configuration, suite=suite,
branch=branch, uuid=uuid, ttl=ttl,
sdk=configuration.sdk or '?', start_time=timestamp,
details=json.dumps(test_results.get('details', {})),
stats=json.dumps(run_stats),
)
except Exception as e:
return self.partial_status(e)
return self.partial_status()