def callback(test, result, failures, unexpected): failed_result = Expectations.string_to_state_ids( result.get('actual', '')) expected = set( Expectations.string_to_state_ids( result.get('expected', ''))) if Expectations.STRING_TO_STATE_ID[ Expectations.FAIL] in expected: expected.add( Expectations.STRING_TO_STATE_ID[Expectations.TEXT]) expected.add(Expectations.STRING_TO_STATE_ID[ Expectations.AUDIO]) expected.add(Expectations.STRING_TO_STATE_ID[ Expectations.IMAGE]) unexpected_result = set(failed_result) - expected if failed_result: worst = min(failed_result) if worst < Expectations.STRING_TO_STATE_ID[ Expectations.WARNING]: failures[test] = min( worst, failures.get( test, Expectations.STRING_TO_STATE_ID[ Expectations.PASS])) if unexpected_result: worst = min(unexpected_result) if worst < Expectations.STRING_TO_STATE_ID[ Expectations.WARNING]: unexpected[test] = min( worst, unexpected.get( test, Expectations.STRING_TO_STATE_ID[ Expectations.PASS]))
def callback(test, result): run_stats['tests_run'] += 1 actual_results = Expectations.string_to_state_ids( result.get('actual', '')) expected_results = set( Expectations.string_to_state_ids(result.get( 'expected', ''))) # FAIL is a special case, we want to treat tests with TEXT, AUDIO and IMAGE diffs as failures if Expectations.STRING_TO_STATE_ID[ Expectations.FAIL] in expected_results: expected_results.add( Expectations.STRING_TO_STATE_ID[Expectations.TEXT]) expected_results.add( Expectations.STRING_TO_STATE_ID[Expectations.AUDIO]) expected_results.add( Expectations.STRING_TO_STATE_ID[Expectations.IMAGE]) worst_result = min(actual_results) unexpected_results = set(actual_results) - expected_results worst_unexpected_result = min( unexpected_results or [Expectations.STRING_TO_STATE_ID[Expectations.PASS]]) for key, point in failure_trigger_points.items(): if worst_result <= point: run_stats[f'tests_{key}'] += 1 if worst_unexpected_result <= point: run_stats[f'tests_unexpected_{key}'] += 1
def unpack(self): results = json.loads(self.tests) if self.tests else {} for test in results.keys(): results[test] = Expectations.state_ids_to_string( [results[test]]) results['uuid'] = self.uuid results['start_time'] = calendar.timegm( self.start_time.timetuple()) return results
def register(self, configuration, commits, suite, test_results, timestamp=None): try: if not isinstance(suite, str): raise TypeError(f'Expected type {str}, got {type(suite)}') timestamp = timestamp or time.time() if isinstance(timestamp, datetime): timestamp = calendar.timegm(timestamp.timetuple()) with self: uuid = self.commit_context.uuid_for_commits(commits) ttl = int((uuid // Commit.UUID_MULTIPLIER) + self.ttl_seconds - time.time()) if self.ttl_seconds else None def callback(test, result, failures, unexpected): failed_result = Expectations.string_to_state_ids( result.get('actual', '')) expected = set( Expectations.string_to_state_ids( result.get('expected', ''))) if Expectations.STRING_TO_STATE_ID[ Expectations.FAIL] in expected: expected.add( Expectations.STRING_TO_STATE_ID[Expectations.TEXT]) expected.add(Expectations.STRING_TO_STATE_ID[ Expectations.AUDIO]) expected.add(Expectations.STRING_TO_STATE_ID[ Expectations.IMAGE]) unexpected_result = set(failed_result) - expected if failed_result: worst = min(failed_result) if worst < Expectations.STRING_TO_STATE_ID[ Expectations.WARNING]: failures[test] = min( worst, failures.get( test, Expectations.STRING_TO_STATE_ID[ Expectations.PASS])) if unexpected_result: worst = min(unexpected_result) if worst < Expectations.STRING_TO_STATE_ID[ Expectations.WARNING]: unexpected[test] = min( worst, unexpected.get( test, Expectations.STRING_TO_STATE_ID[ Expectations.PASS])) with self.cassandra.batch_query_context(): for branch in self.commit_context.branch_keys_for_commits( commits): failures = {} unexpected = {} Expectations.iterate_through_nested_results( test_results.get('results'), lambda test, result: callback(test, result, failures=failures, unexpected=unexpected ), ) for table in [ self.TestFailuresByCommit, self.TestFailuresByStartTime ]: self.configuration_context.insert_row_with_configuration( table.__table_name__, configuration=configuration, suite=suite, branch=branch, uuid=uuid, ttl=ttl, sdk=configuration.sdk or '?', start_time=timestamp, tests=json.dumps(failures), ) for table in [ self.UnexpectedTestFailuresByCommit, self.UnexpectedTestFailuresByStartTime ]: self.configuration_context.insert_row_with_configuration( table.__table_name__, configuration=configuration, suite=suite, branch=branch, uuid=uuid, ttl=ttl, sdk=configuration.sdk or '?', start_time=timestamp, tests=json.dumps(unexpected), ) except Exception as e: return self.partial_status(e) return self.partial_status()
def register(self, configuration, commits, suite, test_results, timestamp=None): timestamp = timestamp or time.time() if not isinstance(timestamp, datetime): timestamp = datetime.utcfromtimestamp(int(timestamp)) try: if not isinstance(suite, str): raise TypeError(f'Expected type {str}, got {type(suite)}') if isinstance(timestamp, datetime): timestamp = calendar.timegm(timestamp.timetuple()) run_stats = test_results.get('run_stats', {}) failure_trigger_points = dict( failed=Expectations.STRING_TO_STATE_ID[Expectations.ERROR], timedout=Expectations.STRING_TO_STATE_ID[Expectations.TIMEOUT], crashed=Expectations.STRING_TO_STATE_ID[Expectations.CRASH], ) run_stats['tests_run'] = 0 for key in failure_trigger_points.keys(): run_stats[f'tests_{key}'] = 0 run_stats[f'tests_unexpected_{key}'] = 0 def callback(test, result): run_stats['tests_run'] += 1 actual_results = Expectations.string_to_state_ids(result.get('actual', '')) expected_results = set(Expectations.string_to_state_ids(result.get('expected', ''))) # FAIL is a special case, we want to treat tests with TEXT, AUDIO and IMAGE diffs as failures if Expectations.STRING_TO_STATE_ID[Expectations.FAIL] in expected_results: expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.TEXT]) expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.AUDIO]) expected_results.add(Expectations.STRING_TO_STATE_ID[Expectations.IMAGE]) worst_result = min(actual_results) unexpected_results = set(actual_results) - expected_results worst_unexpected_result = min(unexpected_results or [Expectations.STRING_TO_STATE_ID[Expectations.PASS]]) for key, point in failure_trigger_points.items(): if worst_result <= point: run_stats[f'tests_{key}'] += 1 if worst_unexpected_result <= point: run_stats[f'tests_unexpected_{key}'] += 1 Expectations.iterate_through_nested_results(test_results.get('results'), callback) uuid = self.commit_context.uuid_for_commits(commits) ttl = int((uuid // Commit.UUID_MULTIPLIER) + self.ttl_seconds - time.time()) if self.ttl_seconds else None with self, self.cassandra.batch_query_context(): for branch in self.commit_context.branch_keys_for_commits(commits): for table in [self.SuiteResultsByCommit, self.SuiteResultsByStartTime]: self.configuration_context.insert_row_with_configuration( table.__table_name__, configuration=configuration, suite=suite, branch=branch, uuid=uuid, ttl=ttl, sdk=configuration.sdk or '?', start_time=timestamp, details=json.dumps(test_results.get('details', {})), stats=json.dumps(run_stats), ) except Exception as e: return self.partial_status(e) return self.partial_status()