def test_get(self): api = SignatureFirstDate(config=self.config) # ..................................................................... # Test 1: a valid signature params = {"signatures": "hey"} res = api.post(**params) res_expected = { "hits": [{ "signature": "hey", "first_date": "2000-01-01T00:00:00+00:00", "first_build": 12, }], "total": 1 } eq_(res['total'], res_expected['total']) eq_(res, res_expected) # ..................................................................... # Test 2: several signatures params = {"signatures": ["hey", "i_just_met_you()"]} res = api.post(**params) res_expected = { "hits": [{ "signature": "hey", "first_date": "2000-01-01T00:00:00+00:00", "first_build": 12, }, { "signature": "i_just_met_you()", "first_date": "2000-01-02T00:00:00+00:00", "first_build": 12, }], "total": 2 } eq_(res['total'], res_expected['total']) eq_(res, res_expected) # ..................................................................... # Test 3: a non-existent signature params = {"signatures": "unknown"} res = api.post(**params) res_expected = {"hits": [], "total": 0} eq_(res, res_expected) # ..................................................................... # Test 4: missing argument params = {} assert_raises(MissingArgumentError, api.post, **params)
def test_get(self): api = SignatureFirstDate(config=self.config) # Test 1: a valid signature res = api.get(signatures='hey') res_expected = { "hits": [{ "signature": "hey", "first_date": datetime.datetime(2000, 1, 1, 0, 0, 0, 0, tzinfo=UTC), "first_build": "12", }], "total": 1 } assert res['total'] == res_expected['total'] assert res == res_expected # Test 2: several signatures res = api.get(signatures=["hey", "i_just_met_you()"]) res_expected = { "hits": [{ "signature": "hey", "first_date": datetime.datetime(2000, 1, 1, 0, 0, 0, 0, tzinfo=UTC), "first_build": "12", }, { "signature": "i_just_met_you()", "first_date": datetime.datetime(2000, 1, 2, 0, 0, 0, 0, tzinfo=UTC), "first_build": "12", }], "total": 2 } assert res['total'] == res_expected['total'] assert res == res_expected # Test 3: a non-existent signature res = api.get(signatures='unknown') res_expected = {"hits": [], "total": 0} assert res == res_expected # Test 4: missing argument with pytest.raises(MissingArgumentError): api.get()
def run(self, end_datetime): # Truncate to the hour end_datetime = end_datetime.replace(minute=0, second=0, microsecond=0) # Do a super search and get the signature, buildid, and date processed for # every crash in the range all_fields = SuperSearchFields(config=self.config).get() api = SuperSearch(config=self.config) start_datetime = end_datetime - datetime.timedelta(minutes=self.config.period) self.config.logger.info('Looking at %s to %s', start_datetime, end_datetime) params = { 'date': [ '>={}'.format(start_datetime.isoformat()), '<{}'.format(end_datetime.isoformat()), ], '_columns': ['signature', 'build_id', 'date'], '_facets_size': 0, '_fields': all_fields, # Set up first page '_results_offset': 0, '_results_number': MAX_PAGE, } results = {} crashids_count = 0 while True: resp = api.get(**params) hits = resp['hits'] for hit in hits: crashids_count += 1 if not hit['build_id']: # Not all crashes have a build id, so skip the ones that don't. continue if hit['signature'] in results: data = results[hit['signature']] data['build_id'] = min(data['build_id'], hit['build_id']) data['date'] = min(data['date'], hit['date']) else: data = { 'signature': hit['signature'], 'build_id': hit['build_id'], 'date': hit['date'] } results[hit['signature']] = data # If there are no more crash ids to get, we return total = resp['total'] if not hits or crashids_count >= total: break # Get the next page, but only as many results as we need params['_results_offset'] += MAX_PAGE params['_results_number'] = min( # MAX_PAGE is the maximum we can request MAX_PAGE, # The number of results Super Search can return to us that is hasn't returned so far total - crashids_count ) signature_data = results.values() # Save signature data to the db signature_first_date_api = SignatureFirstDate(config=self.config) for item in signature_data: if self.config.dry_run: self.config.logger.info( 'Inserting/updating signature (%s, %s, %s)', item['signature'], item['date'], item['build_id'] ) else: signature_first_date_api.post( signature=item['signature'], first_report=item['date'], first_build=item['build_id'] ) self.config.logger.info('Inserted/updated %d signatures.', len(signature_data))