def GetStdoutLog(master_name, builder_name, build_number, step_name,
http_client):
try:
return step_util.GetWaterfallBuildStepLog(
master_name, builder_name, build_number, step_name, http_client)
except ResponseTooLargeError:
logging.exception('Log of step "%s" is too large for urlfetch.', step_name)
# If the stdio log of a step is too large, we don't want to pull
# it again in next run, because that might lead to DDoS to the
# master.
return 'Stdio log is too large for urlfetch.'
def ExtractSignalsForCompileFailure(failure_info, http_client):
signals = {}
master_name = failure_info.master_name
builder_name = failure_info.builder_name
build_number = failure_info.build_number
step_name = 'compile'
if step_name not in (failure_info.failed_steps or {}):
logging.debug(
'No compile failure found when extracting signals for failed '
'build %s/%s/%d', master_name, builder_name, build_number)
return signals
if not failure_info.failed_steps[step_name].supported:
# Bail out if the step is not supported.
logging.info('Findit could not analyze compile failure for master %s.',
master_name)
return signals
failure_log = None
# 1. Tries to get stored failure log from step.
step = (WfStep.Get(master_name, builder_name, build_number, step_name) or
WfStep.Create(master_name, builder_name, build_number, step_name))
if step.log_data:
failure_log = step.log_data
else:
# 2. Tries to get ninja_output as failure log.
from_ninja_output = False
use_ninja_output_log = (waterfall_config.GetDownloadBuildDataSettings(
).get('use_ninja_output_log'))
if use_ninja_output_log:
failure_log = step_util.GetWaterfallBuildStepLog(
master_name, builder_name, build_number, step_name,
http_client, 'json.output[ninja_info]')
from_ninja_output = True
if not failure_log:
# 3. Tries to get stdout log for compile step.
from_ninja_output = False
failure_log = extract_signal.GetStdoutLog(master_name,
builder_name,
build_number, step_name,
http_client)
try:
if not failure_log:
raise extract_signal.FailedToGetFailureLogError(
'Failed to pull failure log (stdio or ninja output) of step %s of'
' %s/%s/%d' %
(step_name, master_name, builder_name, build_number))
except extract_signal.FailedToGetFailureLogError:
return {}
# Save step log in datastore and avoid downloading again during retry.
step.log_data = extract_signal.ExtractStorablePortionOfLog(
failure_log, from_ninja_output)
try:
step.put()
except Exception as e: # pragma: no cover
# Sometimes, the step log is too large to save in datastore.
logging.exception(e)
signals[step_name] = extractors.ExtractSignal(
master_name,
builder_name,
step_name,
test_name=None,
failure_log=failure_log).ToDict()
extract_signal.SaveSignalInAnalysis(master_name, builder_name,
build_number, signals)
return signals
def testGetStepLogNotJosonLoadable(self, mocked_log, *_):
self.assertIsNone(
step_util.GetWaterfallBuildStepLog('m', 'b', 123, 's', None,
'step_metadata'))
mocked_log.assert_called_with(
'Failed to json load data for step_metadata. Data is: log.')
def testGetStepLogStdio(self, *_):
self.assertEqual(
'log1/nlog2',
step_util.GetWaterfallBuildStepLog('m', 'b', 123, 's', None))
def testLegacyGetStepMetadataStreamNone(self, *_):
step_metadata = step_util.GetWaterfallBuildStepLog(
'm', 'b', 123, 's', None, 'step_metadata')
self.assertIsNone(step_metadata)
def testMalformattedNinjaInfo(self, *_):
step_metadata = step_util.GetWaterfallBuildStepLog(
'm', 'b', 123, 's', None, 'json.output[ninja_info]')
self.assertIsNone(step_metadata)
def testLegacyGetStepMetadata(self, *_):
step_metadata = step_util.GetWaterfallBuildStepLog(
'm', 'b', 123, 's', None, 'step_metadata')
self.assertEqual(step_metadata, wf_testcase.SAMPLE_STEP_METADATA)