def testGetCompiledComponentClassifierSetting(self):
crash_config = CrashConfig.Get()
self.assertEqual(crash_config.component_classifier,
DUMMY_COMPONENT_PATTERNS)
self._VerifyTwoCompiledComponentClassifierEqual(
crash_config.compiled_component_classifier,
DUMMY_COMPILED_COMPONENT_PATTERNS)
def run(self, client_id, crash_keys, publish_to_client=False):
"""Reruns analysis for a batch of crashes.
Args:
client_id (CrashClient): The client whose crash we should iterate.
crash_keys (list): A list of urlsafe encodings of crash keys.
"""
client = PredatorForClientID(
client_id, CachedGitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
updated = []
for key in crash_keys:
key = ndb.Key(urlsafe=key)
crash = key.get()
crash.ReInitialize(client)
updated.append(crash)
ndb.put_multi(updated)
for crash in updated:
logging.info('Initialize analysis for crash %s', crash.identifiers)
if publish_to_client:
run_analysis = yield CrashAnalysisPipeline(
client_id, crash.identifiers)
with pipeline.After(run_analysis):
yield PublishResultPipeline(client_id, crash.identifiers)
else:
yield CrashAnalysisPipeline(client_id, crash.identifiers)
def testHandleGet(self, mocked_get_component_classifier_config):
mocked_get_component_classifier_config.return_value = _MOCK_CONFIG
response = self.test_app.get('/process/update-component-config',
headers={'X-AppEngine-Cron': 'true'})
self.assertEqual(response.status_int, 200)
self.assertDictEqual(_MOCK_CONFIG,
CrashConfig.Get().component_classifier)
def run(self, raw_crash_data):
"""Fire off pipelines to run the analysis and publish its results.
N.B., due to the structure of AppEngine pipelines, this method must
accept the same arguments as are passed to ``__init__``; however,
because they were already passed to ``__init__`` there's no use in
recieving them here. Thus, we discard all the arguments to this method
(except for ``self``, naturally).
"""
predator_client = PredatorForClientID(
self._client_id,
CachedGitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
crash_data = predator_client.GetCrashData(raw_crash_data)
need_analysis = predator_client.NeedsNewAnalysis(crash_data)
if need_analysis:
logging.info('New %s analysis is scheduled for %s',
self._client_id, crash_data.identifiers)
UpdateCrashAnalysisData(crash_data, predator_client)
run_analysis = yield CrashAnalysisPipeline(self._client_id,
crash_data.identifiers)
with pipeline.After(run_analysis):
yield PublishResultPipeline(self._client_id,
crash_data.identifiers)
else:
yield PublishResultPipeline(self._client_id,
crash_data.identifiers)
def __init__(self, client_id, crash_identifiers):
super(CrashBasePipeline, self).__init__(client_id, crash_identifiers)
self._crash_identifiers = crash_identifiers
self._predator = PredatorForClientID(
client_id, CachedGitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
self._predator.SetLog(self.log)
def __init__(self, client_id, crash_identifiers):
super(CrashBasePipeline, self).__init__(client_id, crash_identifiers)
self._crash_identifiers = crash_identifiers
self._findit = FinditForClientID(
client_id,
CachedGitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
def testGetCompiledComponentClassifierSettingFromCache(self):
crash_config = CrashConfig.Get()
crash_config.ClearCache()
crash_config.cached_component_classifier = DUMMY_COMPILED_COMPONENT_PATTERNS
self._VerifyTwoCompiledComponentClassifierEqual(
crash_config.compiled_component_classifier,
DUMMY_COMPILED_COMPONENT_PATTERNS)
def testClearCache(self):
crash_config = CrashConfig.Get()
crash_config.ClearCache()
self.assertIsNone(crash_config.cached_component_classifier)
self._VerifyTwoCompiledComponentClassifierEqual(
crash_config.compiled_component_classifier,
DUMMY_COMPILED_COMPONENT_PATTERNS)
def GetMockPredatorApp(self, get_repository=None, config=None,
client_id=CrashClient.FRACAS):
get_repository = (get_repository or
GitilesRepository.Factory(self.GetMockHttpClient()))
config = config or CrashConfig.Get()
class MockPredatorApp(PredatorApp): # pylint: disable=W0223
"""Overwrite abstract method of PredatorApp for testing."""
def __init__(self):
super(MockPredatorApp, self).__init__(get_repository, config)
@classmethod
def _ClientID(cls):
return client_id
def ProcessResultForPublishing(self, result, key): # pylint: disable=W0613
return result
def GetCrashData(self, crash_data):
class MockCrashData(CrashData):
@property
def regression_range(self):
return crash_data.get('regression_range')
@property
def stacktrace(self):
return None
@property
def dependencies(self):
return {}
@property
def dependency_rolls(self):
return {}
@property
def identifiers(self):
return crash_data.get('crash_identifiers')
return MockCrashData(crash_data)
def GetAnalysis(self, crash_identifiers):
return MockCrashAnalysis.Get(crash_identifiers)
def CreateAnalysis(self, crash_identifiers):
return MockCrashAnalysis.Create(crash_identifiers)
def _Predator(self):
return Predator(MockChangelistClassifier(get_repository, None, None),
self._component_classifier,
self._project_classifier)
mock_predator = MockPredatorApp()
mock_predator.SetLog(self.GetMockLog())
return mock_predator
def HandleGet(self):
settings = CrashConfigModel.Get()
data = {
'fracas': settings.fracas,
'cracas': settings.cracas,
'component_classifier': settings.component_classifier,
'project_classifier': settings.project_classifier,
}
return {'template': 'crash/crash_config.html', 'data': data}
def HandleGet(self):
# Update component_classifier with latest component/team information.
new_config_dict = {'component_classifier': GetComponentClassifierConfig(
OWNERS_MAPPING_URL, PREDATOR_MAPPING_URL, HttpClientAppengine())}
if not new_config_dict.get('component_classifier'): # pragma: no cover.
return BaseHandler.CreateError(
'Component Classifier Config Update Fail', 400)
crash_config = CrashConfig.Get()
crash_config.Update(
users.User('*****@*****.**'), True, **new_config_dict)
def testHandleGet(self, mocked_get_repo_to_dep_path):
"""Tests ``UpdateRepoToDepPath`` handler."""
mock_repo_to_dep_path = {
'https://chromium.git': 'src',
'https://chromium.v8.git': 'src/v8',
}
mocked_get_repo_to_dep_path.return_value = mock_repo_to_dep_path
response = self.test_app.get('/process/update-repo-to-dep-path',
headers={'X-AppEngine-Cron': 'true'})
self.assertEqual(response.status_int, 200)
self.assertDictEqual(mock_repo_to_dep_path,
CrashConfig.Get().repo_to_dep_path)
def testHandleGet(self):
"""Tests ``HandleGet`` method of ``CrashConfig`` handler."""
self.mock_current_user(user_email='*****@*****.**', is_admin=True)
CrashConfigModel.Get().Update(users.GetCurrentUser(), True,
**_MOCK_CONFIG)
response = self.test_app.get('/crash/config',
params={'format': 'json'})
self.assertEquals(response.status_int, 200)
self.assertDictEqual(_MOCK_CONFIG, response.json_body)
def testFindCulprit(self):
self.mock(FinditForChromeCrash, 'FindCulprit', lambda self, *_: None)
# TODO(wrengr): would be less fragile to call
# FinditForFracas.CreateAnalysis instead; though if I'm right about
# the original purpose of this test, then this is one of the few
# places where calling FracasCrashAnalysis directly would actually
# make sense.
analysis = FracasCrashAnalysis.Create({'signature': 'sig'})
findit_client = _FinditForChromeCrash(
GitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
self.assertIsNone(findit_client.FindCulprit(analysis))
def HandleGet(self):
# Update component_classifier with latest component/team information.
crash_config = CrashConfig.Get()
new_component_config = GetComponentClassifierConfig(
crash_config.component_classifier)
if not new_component_config: # pragma: no cover.
return self.CreateError('Component Classifier Config Update Fail',
400)
crash_config.Update(users.User('*****@*****.**'),
True,
component_classifier=new_component_config)
def testFindCulprit(self, mock_find_culprit):
mock_find_culprit.return_value = None
# TODO(wrengr): would be less fragile to call
# PredatorForFracas.CreateAnalysis instead; though if I'm right about
# the original purpose of this test, then this is one of the few
# places where calling FracasCrashAnalysis directly would actually
# make sense.
analysis = FracasCrashAnalysis.Create({'signature': 'sig'})
predator_client = _PredatorForChromeCrash(
GitilesRepository.Factory(HttpClientAppengine()),
CrashConfig.Get())
self.assertIsNone(predator_client.FindCulprit(analysis))
def HandleGet(self):
"""Update the repo_to_dep_path in config from the lastest DEPS."""
# Update repo_to_dep_path to the latest information.
dep_fetcher = ChromeDependencyFetcher(
CachedGitilesRepository.Factory(HttpClientAppengine()))
repo_to_dep_path = GetRepoToDepPath(dep_fetcher)
if not repo_to_dep_path: # pragma: no cover.
return self.CreateError('Fail to update repo_to_dep_path config.', 400)
crash_config = CrashConfig.Get()
crash_config.Update(users.User(app_identity.get_service_account_name()),
True, repo_to_dep_path=repo_to_dep_path)
def HandlePost(self):
data = self.request.params.get('data')
new_config_dict = json.loads(data)
if not _ConfigurationDictIsValid(new_config_dict):
return self.CreateError(
'New configuration settings are not properly formatted.', 400)
# Sort config dict in place.
_SortConfig(new_config_dict)
crash_config = CrashConfigModel.Get()
crash_config.Update(users.get_current_user(),
users.IsCurrentUserAdmin(), **new_config_dict)
return self.HandleGet()
def HandleGet(self):
settings = CrashConfigModel.Get()
data = {
'fracas': settings.fracas,
'cracas': settings.cracas,
'clusterfuzz': settings.clusterfuzz,
'uma_sampling_profiler': settings.uma_sampling_profiler,
'component_classifier': settings.component_classifier,
'project_classifier': settings.project_classifier,
'repo_to_dep_path': settings.repo_to_dep_path,
'feature_options': settings.feature_options,
}
return {'template': 'crash_config.html', 'data': data}
def GetMockFindit(self,
get_repository=None,
config=None,
client_id='mock_client'):
get_repository = (get_repository or GitilesRepository.Factory(
self.GetMockHttpClient()))
config = config or CrashConfig.Get()
class MockFindit(Findit): # pylint: disable=W0223
"""Overwrite abstract method of Findit for testing."""
def __init__(self):
super(MockFindit, self).__init__(get_repository, config)
@classmethod
def _ClientID(cls):
return client_id
def ProcessResultForPublishing(self, result, key): # pylint: disable=W0613
return result
def GetCrashData(self, crash_data):
class MockCrashData(CrashData):
@property
def regression_range(self):
return crash_data.get('regression_range')
@property
def stacktrace(self):
return None
@property
def dependencies(self):
return {}
@property
def dependency_rolls(self):
return {}
return MockCrashData(crash_data)
def GetAnalysis(self, crash_identifiers):
return CrashAnalysis.Get(crash_identifiers)
def CreateAnalysis(self, crash_identifiers):
return CrashAnalysis.Create(crash_identifiers)
return MockFindit()
def NeedNewAnalysis(json_crash_data):
"""Checks if an analysis is needed for this crash.
Args:
json_crash_data (dict): Crash information from clients.
Returns:
True if a new analysis is needed; False otherwise.
"""
if json_crash_data.get('redo'):
logging.info('Force redo crash %s',
repr(json_crash_data['crash_identifiers']))
return True
# N.B., must call FinditForClientID indirectly, for mock testing.
findit_client = crash_pipeline.FinditForClientID(
json_crash_data['client_id'],
CachedGitilesRepository.Factory(HttpClientAppengine()), CrashConfig.Get())
crash_data = findit_client.GetCrashData(json_crash_data)
# Detect the regression range, and decide if we actually need to
# run a new analysis or not.
return findit_client.NeedsNewAnalysis(crash_data)
def GetCulprits(crashes,
client_id,
app_id,
verbose=False): # pragma: no cover.
"""Run ``CrashAnalysis`` entities in parallel and returns culprits.
Args:
crashes (list): A list of ``CrashAnalysis`` entities to run Predator on and
get culprit results.
client_id (CrashClient): One of CrashClient.FRACAS, CrashClient.CRACAS and
CrashClient.CLUSTERFUZZ.
app_id (str): Project id of app engine app.
verbose (boolean): Whether to print every culprit results or not.
Returns:
A dict mapping crash id (urlsafe of entity key for Cracas/Fracas, testcase
id for Cluterfuzz) to culprit results (dict version of ``Culprit``.)
"""
# Enable remote access to app engine services.
remote_api.EnableRemoteApi(app_id)
tasks = []
lock = threading.Lock()
config = CrashConfig.Get()
id_to_culprits = {}
for crash in crashes.itervalues():
tasks.append({
'function':
StoreResults,
'args': [crash, client_id, app_id, id_to_culprits, lock, config],
'kwargs': {
'verbose': verbose
}
})
script_util.RunTasks(tasks)
return id_to_culprits
def testGetClientConfig(self):
crash_config = CrashConfig.Get()
self.assertIsNotNone(crash_config.GetClientConfig(CrashClient.FRACAS))
self.assertIsNone(crash_config.GetClientConfig('Unsupported_client'))
def setUp(self):
super(PredatorForFracasTest, self).setUp()
self._client = _PredatorForFracas(config=CrashConfig.Get())
def setUp(self):
super(CrashAnalysisTest, self).setUp()
CrashConfig.Get().Update(users.User(email='*****@*****.**'), True,
**CONFIG_DATA)
def setUp(self):
super(FinditForFracasTest, self).setUp()
self._client = _FinditForFracas(config=CrashConfig.Get())
def setUp(self):
super(PredatorForCracasTest, self).setUp()
self.predator = PredatorForCracas(MOCK_GET_REPOSITORY,
CrashConfig.Get())
def testHandleGet(self, mocked_get_component_classifier_config):
mocked_get_component_classifier_config.return_value = _MOCK_CONFIG
self.mock_current_user(user_email='*****@*****.**', is_admin=True)
response = self.test_app.get('/crash/update-component-config')
self.assertEqual(response.status_int, 200)
self.assertDictEqual(_MOCK_CONFIG, CrashConfig.Get().component_classifier)
def setUp(self):
super(PredatorForUMASamplingProfilerTest, self).setUp()
self._client = PredatorForUMASamplingProfiler(
self.GetMockRepoFactory(), CrashConfig.Get())
def setUp(self):
super(AppengineTestCase, self).setUp()
CrashConfig.Get().Update(users.User(email='*****@*****.**'), True,
**DEFAULT_CONFIG_DATA)
gae_ts_mon.reset_for_unittest(disable=True)