def test_get(self):
api = SignatureFirstDate(config=self.config)
# .....................................................................
# Test 1: a valid signature
params = {"signatures": "hey"}
res = api.post(**params)
res_expected = {
"hits": [{
"signature": "hey",
"first_date": "2000-01-01T00:00:00+00:00",
"first_build": 12,
}],
"total":
1
}
eq_(res['total'], res_expected['total'])
eq_(res, res_expected)
# .....................................................................
# Test 2: several signatures
params = {"signatures": ["hey", "i_just_met_you()"]}
res = api.post(**params)
res_expected = {
"hits": [{
"signature": "hey",
"first_date": "2000-01-01T00:00:00+00:00",
"first_build": 12,
}, {
"signature": "i_just_met_you()",
"first_date": "2000-01-02T00:00:00+00:00",
"first_build": 12,
}],
"total":
2
}
eq_(res['total'], res_expected['total'])
eq_(res, res_expected)
# .....................................................................
# Test 3: a non-existent signature
params = {"signatures": "unknown"}
res = api.post(**params)
res_expected = {"hits": [], "total": 0}
eq_(res, res_expected)
# .....................................................................
# Test 4: missing argument
params = {}
assert_raises(MissingArgumentError, api.post, **params)
def test_get(self):
api = SignatureFirstDate(config=self.config)
# Test 1: a valid signature
res = api.get(signatures='hey')
res_expected = {
"hits": [
{
"signature": "hey",
"first_date": datetime.datetime(
2000, 1, 1, 0, 0, 0, 0,
tzinfo=UTC
),
"first_build": "12",
}
],
"total": 1
}
assert res['total'] == res_expected['total']
assert res == res_expected
# Test 2: several signatures
res = api.get(signatures=["hey", "i_just_met_you()"])
res_expected = {
"hits": [
{
"signature": "hey",
"first_date": datetime.datetime(
2000, 1, 1, 0, 0, 0, 0,
tzinfo=UTC
),
"first_build": "12",
},
{
"signature": "i_just_met_you()",
"first_date": datetime.datetime(
2000, 1, 2, 0, 0, 0, 0,
tzinfo=UTC
),
"first_build": "12",
}
],
"total": 2
}
assert res['total'] == res_expected['total']
assert res == res_expected
# Test 3: a non-existent signature
res = api.get(signatures='unknown')
res_expected = {
"hits": [],
"total": 0
}
assert res == res_expected
# Test 4: missing argument
with pytest.raises(MissingArgumentError):
api.get()
def test_get(self):
api = SignatureFirstDate(config=self.config)
# Test 1: a valid signature
res = api.get(signatures='hey')
res_expected = {
"hits": [{
"signature":
"hey",
"first_date":
datetime.datetime(2000, 1, 1, 0, 0, 0, 0, tzinfo=UTC),
"first_build":
"12",
}],
"total":
1
}
assert res['total'] == res_expected['total']
assert res == res_expected
# Test 2: several signatures
res = api.get(signatures=["hey", "i_just_met_you()"])
res_expected = {
"hits": [{
"signature":
"hey",
"first_date":
datetime.datetime(2000, 1, 1, 0, 0, 0, 0, tzinfo=UTC),
"first_build":
"12",
}, {
"signature":
"i_just_met_you()",
"first_date":
datetime.datetime(2000, 1, 2, 0, 0, 0, 0, tzinfo=UTC),
"first_build":
"12",
}],
"total":
2
}
assert res['total'] == res_expected['total']
assert res == res_expected
# Test 3: a non-existent signature
res = api.get(signatures='unknown')
res_expected = {"hits": [], "total": 0}
assert res == res_expected
# Test 4: missing argument
with pytest.raises(MissingArgumentError):
api.get()
def test_get(self):
api = SignatureFirstDate(config=self.config)
# .....................................................................
# Test 1: a valid signature
params = {
"signatures": "hey"
}
res = api.post(**params)
res_expected = {
"hits": [
{
"signature": "hey",
"first_date": "2000-01-01T00:00:00+00:00",
"first_build": 12,
}
],
"total": 1
}
eq_(res['total'], res_expected['total'])
eq_(res, res_expected)
# .....................................................................
# Test 2: several signatures
params = {
"signatures": ["hey", "i_just_met_you()"]
}
res = api.post(**params)
res_expected = {
"hits": [
{
"signature": "hey",
"first_date": "2000-01-01T00:00:00+00:00",
"first_build": 12,
},
{
"signature": "i_just_met_you()",
"first_date": "2000-01-02T00:00:00+00:00",
"first_build": 12,
}
],
"total": 2
}
eq_(res['total'], res_expected['total'])
eq_(res, res_expected)
# .....................................................................
# Test 3: a non-existent signature
params = {
"signatures": "unknown"
}
res = api.post(**params)
res_expected = {
"hits": [],
"total": 0
}
eq_(res, res_expected)
# .....................................................................
# Test 4: missing argument
params = {}
assert_raises(MissingArgumentError, api.post, **params)
def run(self, end_datetime):
# Truncate to the hour
end_datetime = end_datetime.replace(minute=0, second=0, microsecond=0)
# Do a super search and get the signature, buildid, and date processed for
# every crash in the range
all_fields = SuperSearchFields(config=self.config).get()
api = SuperSearch(config=self.config)
start_datetime = end_datetime - datetime.timedelta(minutes=self.config.period)
self.config.logger.info('Looking at %s to %s', start_datetime, end_datetime)
params = {
'date': [
'>={}'.format(start_datetime.isoformat()),
'<{}'.format(end_datetime.isoformat()),
],
'_columns': ['signature', 'build_id', 'date'],
'_facets_size': 0,
'_fields': all_fields,
# Set up first page
'_results_offset': 0,
'_results_number': MAX_PAGE,
}
results = {}
crashids_count = 0
while True:
resp = api.get(**params)
hits = resp['hits']
for hit in hits:
crashids_count += 1
if not hit['build_id']:
# Not all crashes have a build id, so skip the ones that don't.
continue
if hit['signature'] in results:
data = results[hit['signature']]
data['build_id'] = min(data['build_id'], hit['build_id'])
data['date'] = min(data['date'], hit['date'])
else:
data = {
'signature': hit['signature'],
'build_id': hit['build_id'],
'date': hit['date']
}
results[hit['signature']] = data
# If there are no more crash ids to get, we return
total = resp['total']
if not hits or crashids_count >= total:
break
# Get the next page, but only as many results as we need
params['_results_offset'] += MAX_PAGE
params['_results_number'] = min(
# MAX_PAGE is the maximum we can request
MAX_PAGE,
# The number of results Super Search can return to us that is hasn't returned so far
total - crashids_count
)
signature_data = results.values()
# Save signature data to the db
signature_first_date_api = SignatureFirstDate(config=self.config)
for item in signature_data:
if self.config.dry_run:
self.config.logger.info(
'Inserting/updating signature (%s, %s, %s)',
item['signature'],
item['date'],
item['build_id']
)
else:
signature_first_date_api.post(
signature=item['signature'],
first_report=item['date'],
first_build=item['build_id']
)
self.config.logger.info('Inserted/updated %d signatures.', len(signature_data))
