def get_pushdate(json, data):
"""Get the pushdate from the json if the patch has not been backed out.
"""
if not json.get('backedoutby', False):
pushdate = json['pushdate'][0]
pushdate = lmdutils.as_utc(datetime.utcfromtimestamp(pushdate))
data.append(pushdate)
def handler_rev(json, data):
push = json['pushdate'][0]
push = datetime.datetime.utcfromtimestamp(push)
push = lmdutils.as_utc(push)
data['date'] = lmdutils.get_date_str(push)
data['backedout'] = utils.is_backout(json)
m = BUG_PAT.search(json['desc'])
if not m or m.group(1) != data['bugid']:
data['bugid'] = ''
def handler_rev(json, data):
push = json["pushdate"][0]
push = datetime.datetime.utcfromtimestamp(push)
push = lmdutils.as_utc(push)
data["date"] = lmdutils.get_date_str(push)
data["backedout"] = utils.is_backout(json)
m = BUG_PAT.search(json["desc"])
if not m or m.group(1) != data["bugid"]:
data["bugid"] = ""
def handler_rev(json, data):
push = json['pushdate'][0]
push = datetime.datetime.utcfromtimestamp(push)
push = lmdutils.as_utc(push)
data['date'] = lmdutils.get_date_str(push)
data['backedout'] = utils.is_backout(json)
m = BUG_PAT.search(json['desc'])
if not m or m.group(1) != data['bugid']:
data['bugid'] = ''
def get_info_from_hg(json):
res = {}
push = json["pushdate"][0]
push = datetime.datetime.utcfromtimestamp(push)
push = lmdutils.as_utc(push)
res["date"] = lmdutils.get_date_str(push)
res["backedout"] = json.get("backedoutby", "") != ""
m = BUG_PAT.search(json["desc"])
res["bugid"] = m.group(1) if m else ""
return res
def get_info_from_hg(json):
res = {}
push = json['pushdate'][0]
push = datetime.datetime.utcfromtimestamp(push)
push = lmdutils.as_utc(push)
res['date'] = lmdutils.get_date_str(push)
res['backedout'] = json.get('backedoutby', '') != ''
m = BUG_PAT.search(json['desc'])
res['bugid'] = m.group(1) if m else ''
return res
def get_info_from_hg(json):
res = {}
push = json['pushdate'][0]
push = datetime.datetime.utcfromtimestamp(push)
push = lmdutils.as_utc(push)
res['date'] = lmdutils.get_date_str(push)
res['backedout'] = json.get('backedoutby', '') != ''
m = BUG_PAT.search(json['desc'])
res['bugid'] = m.group(1) if m else ''
return res
def test_get_date_ymd(self):
self.assertIsNotNone(utils.get_date_ymd('today'))
self.assertIsNotNone(utils.get_date_ymd('yesterday'))
self.assertIsNotNone(utils.get_date_ymd('tomorrow'))
self.assertTrue(
utils.get_date_ymd('yesterday') < utils.get_date_ymd('today') <
utils.get_date_ymd('tomorrow'))
date = utils.as_utc(
datetime.datetime.strptime('1991-04-16', '%Y-%m-%d'))
self.assertEqual(utils.get_date_ymd('1991/04/16'), date)
self.assertEqual(utils.get_date_ymd('1991-04-16'), date)
self.assertEqual(utils.get_date_ymd('1991 04 16'), date)
self.assertEqual(utils.get_date_ymd('04/16/1991'), date)
self.assertEqual(utils.get_date_ymd('16/04/1991'), date)
self.assertEqual(utils.get_date_ymd('1991-04-16 12:00:00'),
utils.as_utc(datetime.datetime(1991, 4, 16, 12, 0)))
with self.assertRaises(Exception):
utils.get_date_ymd('')
with self.assertRaises(Exception):
utils.get_date_ymd('marco')
def __trend_handler(default_trend, json, data):
for facets in json['facets']['histogram_date']:
d = utils.as_utc(datetime.strptime(facets['term'], '%Y-%m-%dT00:00:00+00:00'))
s = facets['facets']['signature']
for signature in s:
count = signature['count']
sgn = signature['term']
if sgn in data:
data[sgn][d] = count
else:
trend = default_trend.copy()
trend[d] = count
data[sgn] = trend
def handler(json, data):
if not json['backedoutby']:
pushdate = json['pushdate'][0]
pushdate = lmdutils.as_utc(datetime.utcfromtimestamp(pushdate))
data.append(pushdate)
def get_uuids_for_spiking_signatures(channel,
cache=None,
product='Firefox',
date='today',
limit=10000,
max_days=3,
threshold=5):
psttz = pytz.timezone('US/Pacific')
end_date = utils.get_date_ymd(date) # 2016-10-18 UTC
end_date_moz = psttz.localize(
datetime(end_date.year, end_date.month,
end_date.day)) # 2016-10-18 PST
end_buildid = utils.get_buildid_from_date(end_date_moz) # < 20161018000000
start_date_moz = end_date_moz - timedelta(
days=max_days + 1) # 2016-10-14 PST (max_days == 3)
start_buildid = utils.get_buildid_from_date(
start_date_moz) # >= 20161014000000
search_buildid = ['>=' + start_buildid, '<' + end_buildid]
start_date = utils.as_utc(start_date_moz) # 2016-10-14 07:00:00 UTC
search_date = '>=' + utils.get_date_str(start_date)
data = defaultdict(lambda: defaultdict(lambda: 0))
buildids = {}
def handler(json, data):
if not json['errors']:
for facets in json['facets']['build_id']:
date = utils.get_date_from_buildid(
facets['term']).astimezone(psttz)
buildids[date] = facets['count']
for s in facets['facets']['signature']:
sgn = s['term']
count = s['count']
data[sgn][date] += count
socorro.SuperSearch(params={
'product': product,
'date': search_date,
'build_id': search_buildid,
'release_channel': channel,
'_aggs.build_id': 'signature',
'_facets_size': limit,
'_results_number': 0
},
handler=handler,
handlerdata=data).wait()
_data = {}
base = {
start_date_moz + timedelta(days=i): {
'buildids': {},
'total': 0
}
for i in range(max_days + 1)
} # from 2016-10-14 to 2016-10-17 PST
for sgn, info in data.items():
d = copy.deepcopy(base)
_data[sgn] = d
for bid, count in info.items():
date = psttz.localize(datetime(bid.year, bid.month, bid.day))
d[date]['buildids'][bid] = count
d[date]['total'] += count
data = _data
spiking_signatures = []
for sgn, info in data.items():
stats2 = [
i['total'] for _, i in sorted(info.items(), key=lambda p: p[0])
]
if all(i == 0 for i in stats2[:-1]) and stats2[-1] >= threshold:
spiking_signatures.append(sgn)
data = None
if spiking_signatures:
# sort the signatures to be sure to always have the same order for the test
spiking_signatures = sorted(spiking_signatures)
start_buildid = utils.get_buildid_from_date(end_date_moz -
timedelta(days=1))
search_buildid = ['>=' + start_buildid, '<' + end_buildid]
queries = []
data = defaultdict(lambda: list())
def handler(json, data):
if not json['errors']:
for facets in json['facets']['proto_signature']:
proto = facets['term']
count = facets['count']
facets = facets['facets']
sgn = facets['signature'][0]['term']
first_uuid = facets['uuid'][0]['term']
uuids = {i['term'] for i in facets['uuid']}
if cache:
i = uuids.intersection(cache['uuids'])
uuid = i.pop() if i else first_uuid
else:
uuid = first_uuid
data[sgn].append({
'proto': proto,
'uuid': uuid,
'count': count
})
for sgns in Connection.chunks(spiking_signatures, 5):
queries.append(
Query(socorro.SuperSearch.URL, {
'product': product,
'date': search_date,
'build_id': search_buildid,
'signature': ['=' + s for s in sgns],
'release_channel': channel,
'_aggs.proto_signature': ['uuid', 'signature'],
'_facets_size': 10000,
'_results_number': 0
},
handler=handler,
handlerdata=data))
socorro.SuperSearch(queries=queries).wait()
return data
def get_uuids_for_spiking_signatures(channel, cache=None, product='Firefox', date='today', limit=10000, max_days=3, threshold=5):
psttz = pytz.timezone('US/Pacific')
end_date = utils.get_date_ymd(date) # 2016-10-18 UTC
end_date_moz = psttz.localize(datetime(end_date.year, end_date.month, end_date.day)) # 2016-10-18 PST
end_buildid = utils.get_buildid_from_date(end_date_moz) # < 20161018000000
start_date_moz = end_date_moz - timedelta(days=max_days + 1) # 2016-10-14 PST (max_days == 3)
start_buildid = utils.get_buildid_from_date(start_date_moz) # >= 20161014000000
search_buildid = ['>=' + start_buildid, '<' + end_buildid]
start_date = utils.as_utc(start_date_moz) # 2016-10-14 07:00:00 UTC
search_date = '>=' + utils.get_date_str(start_date)
data = defaultdict(lambda: defaultdict(lambda: 0))
buildids = {}
def handler(json, data):
if not json['errors']:
for facets in json['facets']['build_id']:
date = utils.get_date_from_buildid(facets['term']).astimezone(psttz)
buildids[date] = facets['count']
for s in facets['facets']['signature']:
sgn = s['term']
count = s['count']
data[sgn][date] += count
socorro.SuperSearch(params={'product': product,
'date': search_date,
'build_id': search_buildid,
'release_channel': channel,
'_aggs.build_id': 'signature',
'_facets_size': limit,
'_results_number': 0},
handler=handler, handlerdata=data).wait()
_data = {}
base = {start_date_moz + timedelta(days=i): {'buildids': {}, 'total': 0} for i in range(max_days + 1)} # from 2016-10-14 to 2016-10-17 PST
for sgn, info in data.items():
d = copy.deepcopy(base)
_data[sgn] = d
for bid, count in info.items():
date = psttz.localize(datetime(bid.year, bid.month, bid.day))
d[date]['buildids'][bid] = count
d[date]['total'] += count
data = _data
spiking_signatures = []
for sgn, info in data.items():
stats2 = [i['total'] for _, i in sorted(info.items(), key=lambda p: p[0])]
if all(i == 0 for i in stats2[:-1]) and stats2[-1] >= threshold:
spiking_signatures.append(sgn)
data = None
if spiking_signatures:
# sort the signatures to be sure to always have the same order for the test
spiking_signatures = sorted(spiking_signatures)
start_buildid = utils.get_buildid_from_date(end_date_moz - timedelta(days=1))
search_buildid = ['>=' + start_buildid, '<' + end_buildid]
queries = []
data = defaultdict(lambda: list())
def handler(json, data):
if not json['errors']:
for facets in json['facets']['proto_signature']:
proto = facets['term']
count = facets['count']
facets = facets['facets']
sgn = facets['signature'][0]['term']
first_uuid = facets['uuid'][0]['term']
uuids = {i['term'] for i in facets['uuid']}
if cache:
i = uuids.intersection(cache['uuids'])
uuid = i.pop() if i else first_uuid
else:
uuid = first_uuid
data[sgn].append({'proto': proto, 'uuid': uuid, 'count': count})
for sgns in Connection.chunks(spiking_signatures, 5):
queries.append(Query(socorro.SuperSearch.URL,
{'product': product,
'date': search_date,
'build_id': search_buildid,
'signature': ['=' + s for s in sgns],
'release_channel': channel,
'_aggs.proto_signature': ['uuid', 'signature'],
'_facets_size': 10000,
'_results_number': 0},
handler=handler, handlerdata=data))
socorro.SuperSearch(queries=queries).wait()
return data
import get_bugs
import utils
# The month abbreviation should be in English.
locale.setlocale(locale.LC_TIME, 'C')
if __name__ == '__main__':
bugs = get_bugs.get_all()
uplifts = utils.get_uplifts(bugs)
months = {}
for uplift in uplifts:
for channel in utils.uplift_approved_channels(uplift):
uplift_date = utils.get_uplift_date(uplift, channel)
if uplift_date > as_utc(datetime(2016, 8, 24)):
continue
delta = relativedelta.relativedelta(uplift_date,
as_utc(datetime(2014, 7, 1)))
delta_num = delta.years * 12 + delta.months
key = (delta_num, uplift_date.strftime('%b %Y'), channel)
if key not in months:
months[key] = 0
months[key] += 1
with open('uplift_dates.csv', 'w') as output_file:
csv_writer = csv.writer(output_file)
csv_writer.writerow(['Delta', 'Month', 'Channel', 'Number_of_uplifts'])
for (delta_str, month,
channel), number in sorted(months.items(), key=lambda x: x[0][0]):
csv_writer.writerow([delta_str, month, channel, number])
