def testMergeFrom(self):
test_signals = [
{
'files': {
'a.cc': [2],
'd.cc': []
},
'keywords': {},
'failed_targets': [{
'target': 'a.o',
'source': 'a.cc'
}, {
'target': 'b.o',
'source': 'b.cc'
}]
},
{
'files': {
'a.cc': [2, 3, 4],
'b.cc': [],
'd.cc': [1]
},
'keywords': {},
'failed_targets': [{
'target': 'a.o',
'source': 'a.cc'
}, {
'target': 'c.exe'
}]
},
]
step_signal = FailureSignal()
for test_signal in test_signals:
step_signal.MergeFrom(test_signal)
expected_step_signal_files = {
'a.cc': [2, 3, 4],
'd.cc': [1],
'b.cc': []
}
expected_step_failed_targets = [{
'target': 'a.o',
'source': 'a.cc'
}, {
'target': 'b.o',
'source': 'b.cc'
}, {
'target': 'c.exe'
}]
self.assertEqual(expected_step_signal_files, step_signal.files)
self.assertEqual(expected_step_failed_targets,
step_signal.failed_targets)
def run(self, failure_info):
"""Extracts failure signals from failed steps.
Args:
failure_info (dict): Output of pipeline DetectFirstFailurePipeline.run().
Returns:
A dict like below:
{
'step_name1': waterfall.failure_signal.FailureSignal.ToDict(),
...
}
"""
signals = {}
if not failure_info['failed'] or not failure_info['chromium_revision']:
# Bail out if no failed step or no chromium revision.
return signals
# Bail out on infra failure
if failure_info.get('failure_type') == failure_type.INFRA:
return signals
master_name = failure_info['master_name']
builder_name = failure_info['builder_name']
build_number = failure_info['build_number']
for step_name in failure_info.get('failed_steps', []):
if not waterfall_config.StepIsSupportedForMaster(
step_name, master_name):
# Bail out if the step is not supported.
continue
step = WfStep.Get(master_name, builder_name, build_number,
step_name)
if step and step.log_data:
failure_log = step.log_data
else:
# TODO: do test-level analysis instead of step-level.
# TODO: Use swarming test result instead of archived gtest results
gtest_result = buildbot.GetGtestResultLog(
master_name, builder_name, build_number, step_name)
if gtest_result:
failure_log = _GetReliableTestFailureLog(gtest_result)
if gtest_result is None or failure_log == 'invalid':
if not lock_util.WaitUntilDownloadAllowed(
master_name): # pragma: no cover
raise pipeline.Retry(
'Failed to pull log of step %s of master %s' %
(step_name, master_name))
try:
failure_log = buildbot.GetStepLog(
master_name, builder_name, build_number, step_name,
self.HTTP_CLIENT)
except ResponseTooLargeError: # pragma: no cover.
logging.exception(
'Log of step "%s" is too large for urlfetch.',
step_name)
# If the stdio log of a step is too large, we don't want to pull it
# again in next run, because that might lead to DDoS to the master.
# TODO: Use archived stdio logs in Google Storage instead.
failure_log = 'Stdio log is too large for urlfetch.'
if not failure_log: # pragma: no cover
raise pipeline.Retry(
'Failed to pull stdio of step %s of master %s' %
(step_name, master_name))
# Save step log in datastore and avoid downloading again during retry.
if not step: # pragma: no cover
step = WfStep.Create(master_name, builder_name,
build_number, step_name)
step.log_data = _ExtractStorablePortionOfLog(failure_log)
try:
step.put()
except Exception as e: # pragma: no cover
# Sometimes, the step log is too large to save in datastore.
logging.exception(e)
# TODO: save result in datastore?
if step.isolated:
try:
json_failure_log = (json.loads(failure_log)
if failure_log != 'flaky' else {})
except ValueError: # pragma: no cover
json_failure_log = {}
logging.warning('failure_log %s is not valid JSON.' %
failure_log)
signals[step_name] = {'tests': {}}
step_signal = FailureSignal()
for test_name, test_failure_log in json_failure_log.iteritems(
):
signals[step_name]['tests'][
test_name] = extractors.ExtractSignal(
master_name, builder_name, step_name, test_name,
base64.b64decode(test_failure_log)).ToDict()
# Save signals in test failure log to step level.
step_signal.MergeFrom(
signals[step_name]['tests'][test_name])
signals[step_name]['files'] = step_signal.files
signals[step_name]['keywords'] = step_signal.keywords
else:
signals[step_name] = extractors.ExtractSignal(
master_name, builder_name, step_name, None,
failure_log).ToDict()
return signals
def testMergeFrom(self):
test_signals = [
{
'files': {
'a.cc': [2],
'd.cc': []
},
'keywords': {},
'failed_targets': [{
'target': 'a.o',
'source': 'a.cc'
}, {
'target': 'b.o',
'source': 'b.cc'
}],
'failed_edges': [{
'rule': 'CXX',
'output_nodes': ['a.o', 'aa.o'],
'dependencies': ['a.h', 'a.c']
}]
},
{
'files': {
'a.cc': [2, 3, 4],
'b.cc': [],
'd.cc': [1]
},
'keywords': {},
'failed_targets': [{
'target': 'a.o',
'source': 'a.cc'
}, {
'target': 'c.exe'
}],
'failed_edges': [{
'rule': 'LINK',
'output_nodes': ['b.o'],
'dependencies': []
}]
},
]
step_signal = FailureSignal()
for test_signal in test_signals:
step_signal.MergeFrom(test_signal)
expected_step_signal_files = {
'a.cc': [2, 3, 4],
'd.cc': [1],
'b.cc': []
}
expected_step_failed_targets = [{
'target': 'a.o',
'source': 'a.cc'
}, {
'target': 'b.o',
'source': 'b.cc'
}, {
'target': 'c.exe'
}]
expected_step_failed_edges = [{
'rule': 'CXX',
'output_nodes': ['a.o', 'aa.o'],
'dependencies': ['a.h', 'a.c']
}, {
'rule': 'LINK',
'output_nodes': ['b.o'],
'dependencies': []
}]
self.assertEqual(expected_step_signal_files, step_signal.files)
self.assertEqual(expected_step_failed_targets,
step_signal.failed_targets)
self.assertEqual(expected_step_failed_edges, step_signal.failed_edges)
def ExtractSignalsForTestFailure(failure_info, http_client):
signals = {}
master_name = failure_info.master_name
builder_name = failure_info.builder_name
build_number = failure_info.build_number
failed_steps = failure_info.failed_steps or {}
for step_name in failed_steps:
failure_log = None
if not failed_steps[step_name].supported:
# Bail out if the step is not supported.
continue
# 1. Tries to get stored failure log from step.
step = (WfStep.Get(master_name, builder_name, build_number, step_name)
or WfStep.Create(master_name, builder_name, build_number,
step_name))
if step.log_data and step.log_data != constants.TOO_LARGE_LOG:
failure_log = step.log_data
else:
json_formatted_log = True
# 2. Gets test results.
list_isolated_data = failed_steps[step_name].list_isolated_data
list_isolated_data = (list_isolated_data.ToSerializable()
if list_isolated_data else [])
merged_test_results = (
swarmed_test_util.RetrieveShardedTestResultsFromIsolatedServer(
list_isolated_data, http_client))
if merged_test_results:
test_results = test_results_util.GetTestResultObject(
merged_test_results)
if test_results:
failure_log, _ = (
test_results_service.
GetFailedTestsInformationFromTestResult(test_results))
failure_log = json.dumps(
failure_log
) if failure_log else constants.FLAKY_FAILURE_LOG
else:
failure_log = constants.WRONG_FORMAT_LOG
if not merged_test_results or failure_log in [
constants.INVALID_FAILURE_LOG, constants.WRONG_FORMAT_LOG
]:
# 3. Gets stdout log.
json_formatted_log = False
failure_log = extract_signal.GetStdoutLog(
master_name, builder_name, build_number, step_name,
http_client)
try:
if not failure_log:
raise extract_signal.FailedToGetFailureLogError(
'Failed to pull failure log (stdio or ninja output) of step %s of'
' %s/%s/%d' %
(step_name, master_name, builder_name, build_number))
except extract_signal.FailedToGetFailureLogError:
return {}
# Save step log in datastore and avoid downloading again during retry.
step.log_data = extract_signal.ExtractStorablePortionOfLog(
failure_log, json_formatted_log
) if step.log_data != constants.TOO_LARGE_LOG else step.log_data
step.isolated = step.isolated or json_formatted_log
try:
step.put()
except Exception as e: # pragma: no cover
# Sometimes, the step log is too large to save in datastore.
logging.exception(e)
if step.isolated:
try:
json_failure_log = (json.loads(failure_log) if
failure_log != constants.FLAKY_FAILURE_LOG
else {})
except ValueError:
json_failure_log = {}
logging.warning('failure_log %s is not valid JSON.' %
failure_log)
signals[step_name] = {'tests': {}}
step_signal = FailureSignal()
for test_name, test_failure_log in json_failure_log.iteritems():
signals[step_name]['tests'][
test_name] = extractors.ExtractSignal(
master_name, builder_name, step_name, test_name,
base64.b64decode(test_failure_log)).ToDict()
# Save signals in test failure log to step level.
step_signal.MergeFrom(signals[step_name]['tests'][test_name])
signals[step_name]['files'] = step_signal.files
else:
signals[step_name] = extractors.ExtractSignal(
master_name, builder_name, step_name, None,
failure_log).ToDict()
return signals
