def test_get_indices(self):
now = datetime.datetime(2001, 1, 2, 0, 0)
lastweek = now - datetime.timedelta(weeks=1)
lastmonth = now - datetime.timedelta(weeks=4)
dates = [
search_common.SearchParam('date', now, '<'),
search_common.SearchParam('date', lastweek, '>'),
]
res = self.api.get_indices(dates)
eq_(res, ['socorro_integration_test_reports'])
config = self.get_base_config(es_index='socorro_%Y%W')
api = SuperSearch(config=config)
dates = [
search_common.SearchParam('date', now, '<'),
search_common.SearchParam('date', lastweek, '>'),
]
res = api.get_indices(dates)
eq_(res, ['socorro_200052', 'socorro_200101'])
dates = [
search_common.SearchParam('date', now, '<'),
search_common.SearchParam('date', lastmonth, '>'),
]
res = api.get_indices(dates)
eq_(res, [
'socorro_200049', 'socorro_200050', 'socorro_200051',
'socorro_200052', 'socorro_200101'
])
def test_get_against_nonexistent_index(self):
config = self.get_base_config(es_index='socorro_test_reports_%W')
api = SuperSearch(config=config)
params = {'date': ['>2000-01-01T00:00:00', '<2000-01-10T00:00:00']}
res = api.get(**params)
eq_(res, {'total': 0, 'hits': [], 'facets': {}})
def setUp(self):
super(IntegrationTestSuperSearch, self).setUp()
self.api = SuperSearch(config=self.config)
self.now = datetimeutil.utc_now()
def setup_method(self):
super().setup_method()
config = self.get_base_config(cls=SuperSearch)
self.api = SuperSearch(config=config)
self.now = datetimeutil.utc_now()
def run(self, end_datetime):
# Truncate to the hour
end_datetime = end_datetime.replace(minute=0, second=0, microsecond=0)
# Do a super search and get the signature, buildid, and date processed for
# every crash in the range
all_fields = SuperSearchFieldsData().get()
api = SuperSearch(self.config)
start_datetime = end_datetime - datetime.timedelta(
minutes=self.config.period)
self.logger.info('Looking at %s to %s', start_datetime, end_datetime)
params = {
'date': [
'>={}'.format(start_datetime.isoformat()),
'<{}'.format(end_datetime.isoformat()),
],
'_columns': ['signature', 'build_id', 'date'],
'_facets_size':
0,
'_fields':
all_fields,
# Set up first page
'_results_offset':
0,
'_results_number':
MAX_PAGE,
}
results = {}
crashids_count = 0
while True:
resp = api.get(**params)
hits = resp['hits']
for hit in hits:
crashids_count += 1
if not hit['build_id']:
# Not all crashes have a build id, so skip the ones that don't.
continue
if hit['signature'] in results:
data = results[hit['signature']]
data['build_id'] = min(data['build_id'], hit['build_id'])
data['date'] = min(data['date'], hit['date'])
else:
data = {
'signature': hit['signature'],
'build_id': hit['build_id'],
'date': hit['date']
}
results[hit['signature']] = data
# If there are no more crash ids to get, we return
total = resp['total']
if not hits or crashids_count >= total:
break
# Get the next page, but only as many results as we need
params['_results_offset'] += MAX_PAGE
params['_results_number'] = min(
# MAX_PAGE is the maximum we can request
MAX_PAGE,
# The number of results Super Search can return to us that is hasn't returned so far
total - crashids_count)
signature_data = results.values()
# Save signature data to the db
for item in signature_data:
if self.config.dry_run:
self.logger.info('Inserting/updating signature (%s, %s, %s)',
item['signature'], item['date'],
item['build_id'])
else:
self.update_crashstats_signature(
signature=item['signature'],
report_date=item['date'],
report_build=item['build_id'],
)
self.logger.info('Inserted/updated %d signatures.',
len(signature_data))
