def get_pushdate(json, data): """Get the pushdate from the json if the patch has not been backed out. """ if not json.get('backedoutby', False): pushdate = json['pushdate'][0] pushdate = lmdutils.as_utc(datetime.utcfromtimestamp(pushdate)) data.append(pushdate)
def handler_rev(json, data): push = json['pushdate'][0] push = datetime.datetime.utcfromtimestamp(push) push = lmdutils.as_utc(push) data['date'] = lmdutils.get_date_str(push) data['backedout'] = utils.is_backout(json) m = BUG_PAT.search(json['desc']) if not m or m.group(1) != data['bugid']: data['bugid'] = ''
def handler_rev(json, data): push = json["pushdate"][0] push = datetime.datetime.utcfromtimestamp(push) push = lmdutils.as_utc(push) data["date"] = lmdutils.get_date_str(push) data["backedout"] = utils.is_backout(json) m = BUG_PAT.search(json["desc"]) if not m or m.group(1) != data["bugid"]: data["bugid"] = ""
def get_info_from_hg(json): res = {} push = json["pushdate"][0] push = datetime.datetime.utcfromtimestamp(push) push = lmdutils.as_utc(push) res["date"] = lmdutils.get_date_str(push) res["backedout"] = json.get("backedoutby", "") != "" m = BUG_PAT.search(json["desc"]) res["bugid"] = m.group(1) if m else "" return res
def get_info_from_hg(json): res = {} push = json['pushdate'][0] push = datetime.datetime.utcfromtimestamp(push) push = lmdutils.as_utc(push) res['date'] = lmdutils.get_date_str(push) res['backedout'] = json.get('backedoutby', '') != '' m = BUG_PAT.search(json['desc']) res['bugid'] = m.group(1) if m else '' return res
def test_get_date_ymd(self): self.assertIsNotNone(utils.get_date_ymd('today')) self.assertIsNotNone(utils.get_date_ymd('yesterday')) self.assertIsNotNone(utils.get_date_ymd('tomorrow')) self.assertTrue( utils.get_date_ymd('yesterday') < utils.get_date_ymd('today') < utils.get_date_ymd('tomorrow')) date = utils.as_utc( datetime.datetime.strptime('1991-04-16', '%Y-%m-%d')) self.assertEqual(utils.get_date_ymd('1991/04/16'), date) self.assertEqual(utils.get_date_ymd('1991-04-16'), date) self.assertEqual(utils.get_date_ymd('1991 04 16'), date) self.assertEqual(utils.get_date_ymd('04/16/1991'), date) self.assertEqual(utils.get_date_ymd('16/04/1991'), date) self.assertEqual(utils.get_date_ymd('1991-04-16 12:00:00'), utils.as_utc(datetime.datetime(1991, 4, 16, 12, 0))) with self.assertRaises(Exception): utils.get_date_ymd('') with self.assertRaises(Exception): utils.get_date_ymd('marco')
def __trend_handler(default_trend, json, data): for facets in json['facets']['histogram_date']: d = utils.as_utc(datetime.strptime(facets['term'], '%Y-%m-%dT00:00:00+00:00')) s = facets['facets']['signature'] for signature in s: count = signature['count'] sgn = signature['term'] if sgn in data: data[sgn][d] = count else: trend = default_trend.copy() trend[d] = count data[sgn] = trend
def handler(json, data): if not json['backedoutby']: pushdate = json['pushdate'][0] pushdate = lmdutils.as_utc(datetime.utcfromtimestamp(pushdate)) data.append(pushdate)
def get_uuids_for_spiking_signatures(channel, cache=None, product='Firefox', date='today', limit=10000, max_days=3, threshold=5): psttz = pytz.timezone('US/Pacific') end_date = utils.get_date_ymd(date) # 2016-10-18 UTC end_date_moz = psttz.localize( datetime(end_date.year, end_date.month, end_date.day)) # 2016-10-18 PST end_buildid = utils.get_buildid_from_date(end_date_moz) # < 20161018000000 start_date_moz = end_date_moz - timedelta( days=max_days + 1) # 2016-10-14 PST (max_days == 3) start_buildid = utils.get_buildid_from_date( start_date_moz) # >= 20161014000000 search_buildid = ['>=' + start_buildid, '<' + end_buildid] start_date = utils.as_utc(start_date_moz) # 2016-10-14 07:00:00 UTC search_date = '>=' + utils.get_date_str(start_date) data = defaultdict(lambda: defaultdict(lambda: 0)) buildids = {} def handler(json, data): if not json['errors']: for facets in json['facets']['build_id']: date = utils.get_date_from_buildid( facets['term']).astimezone(psttz) buildids[date] = facets['count'] for s in facets['facets']['signature']: sgn = s['term'] count = s['count'] data[sgn][date] += count socorro.SuperSearch(params={ 'product': product, 'date': search_date, 'build_id': search_buildid, 'release_channel': channel, '_aggs.build_id': 'signature', '_facets_size': limit, '_results_number': 0 }, handler=handler, handlerdata=data).wait() _data = {} base = { start_date_moz + timedelta(days=i): { 'buildids': {}, 'total': 0 } for i in range(max_days + 1) } # from 2016-10-14 to 2016-10-17 PST for sgn, info in data.items(): d = copy.deepcopy(base) _data[sgn] = d for bid, count in info.items(): date = psttz.localize(datetime(bid.year, bid.month, bid.day)) d[date]['buildids'][bid] = count d[date]['total'] += count data = _data spiking_signatures = [] for sgn, info in data.items(): stats2 = [ i['total'] for _, i in sorted(info.items(), key=lambda p: p[0]) ] if all(i == 0 for i in stats2[:-1]) and stats2[-1] >= threshold: spiking_signatures.append(sgn) data = None if spiking_signatures: # sort the signatures to be sure to always have the same order for the test spiking_signatures = sorted(spiking_signatures) start_buildid = utils.get_buildid_from_date(end_date_moz - timedelta(days=1)) search_buildid = ['>=' + start_buildid, '<' + end_buildid] queries = [] data = defaultdict(lambda: list()) def handler(json, data): if not json['errors']: for facets in json['facets']['proto_signature']: proto = facets['term'] count = facets['count'] facets = facets['facets'] sgn = facets['signature'][0]['term'] first_uuid = facets['uuid'][0]['term'] uuids = {i['term'] for i in facets['uuid']} if cache: i = uuids.intersection(cache['uuids']) uuid = i.pop() if i else first_uuid else: uuid = first_uuid data[sgn].append({ 'proto': proto, 'uuid': uuid, 'count': count }) for sgns in Connection.chunks(spiking_signatures, 5): queries.append( Query(socorro.SuperSearch.URL, { 'product': product, 'date': search_date, 'build_id': search_buildid, 'signature': ['=' + s for s in sgns], 'release_channel': channel, '_aggs.proto_signature': ['uuid', 'signature'], '_facets_size': 10000, '_results_number': 0 }, handler=handler, handlerdata=data)) socorro.SuperSearch(queries=queries).wait() return data
def get_uuids_for_spiking_signatures(channel, cache=None, product='Firefox', date='today', limit=10000, max_days=3, threshold=5): psttz = pytz.timezone('US/Pacific') end_date = utils.get_date_ymd(date) # 2016-10-18 UTC end_date_moz = psttz.localize(datetime(end_date.year, end_date.month, end_date.day)) # 2016-10-18 PST end_buildid = utils.get_buildid_from_date(end_date_moz) # < 20161018000000 start_date_moz = end_date_moz - timedelta(days=max_days + 1) # 2016-10-14 PST (max_days == 3) start_buildid = utils.get_buildid_from_date(start_date_moz) # >= 20161014000000 search_buildid = ['>=' + start_buildid, '<' + end_buildid] start_date = utils.as_utc(start_date_moz) # 2016-10-14 07:00:00 UTC search_date = '>=' + utils.get_date_str(start_date) data = defaultdict(lambda: defaultdict(lambda: 0)) buildids = {} def handler(json, data): if not json['errors']: for facets in json['facets']['build_id']: date = utils.get_date_from_buildid(facets['term']).astimezone(psttz) buildids[date] = facets['count'] for s in facets['facets']['signature']: sgn = s['term'] count = s['count'] data[sgn][date] += count socorro.SuperSearch(params={'product': product, 'date': search_date, 'build_id': search_buildid, 'release_channel': channel, '_aggs.build_id': 'signature', '_facets_size': limit, '_results_number': 0}, handler=handler, handlerdata=data).wait() _data = {} base = {start_date_moz + timedelta(days=i): {'buildids': {}, 'total': 0} for i in range(max_days + 1)} # from 2016-10-14 to 2016-10-17 PST for sgn, info in data.items(): d = copy.deepcopy(base) _data[sgn] = d for bid, count in info.items(): date = psttz.localize(datetime(bid.year, bid.month, bid.day)) d[date]['buildids'][bid] = count d[date]['total'] += count data = _data spiking_signatures = [] for sgn, info in data.items(): stats2 = [i['total'] for _, i in sorted(info.items(), key=lambda p: p[0])] if all(i == 0 for i in stats2[:-1]) and stats2[-1] >= threshold: spiking_signatures.append(sgn) data = None if spiking_signatures: # sort the signatures to be sure to always have the same order for the test spiking_signatures = sorted(spiking_signatures) start_buildid = utils.get_buildid_from_date(end_date_moz - timedelta(days=1)) search_buildid = ['>=' + start_buildid, '<' + end_buildid] queries = [] data = defaultdict(lambda: list()) def handler(json, data): if not json['errors']: for facets in json['facets']['proto_signature']: proto = facets['term'] count = facets['count'] facets = facets['facets'] sgn = facets['signature'][0]['term'] first_uuid = facets['uuid'][0]['term'] uuids = {i['term'] for i in facets['uuid']} if cache: i = uuids.intersection(cache['uuids']) uuid = i.pop() if i else first_uuid else: uuid = first_uuid data[sgn].append({'proto': proto, 'uuid': uuid, 'count': count}) for sgns in Connection.chunks(spiking_signatures, 5): queries.append(Query(socorro.SuperSearch.URL, {'product': product, 'date': search_date, 'build_id': search_buildid, 'signature': ['=' + s for s in sgns], 'release_channel': channel, '_aggs.proto_signature': ['uuid', 'signature'], '_facets_size': 10000, '_results_number': 0}, handler=handler, handlerdata=data)) socorro.SuperSearch(queries=queries).wait() return data
import get_bugs import utils # The month abbreviation should be in English. locale.setlocale(locale.LC_TIME, 'C') if __name__ == '__main__': bugs = get_bugs.get_all() uplifts = utils.get_uplifts(bugs) months = {} for uplift in uplifts: for channel in utils.uplift_approved_channels(uplift): uplift_date = utils.get_uplift_date(uplift, channel) if uplift_date > as_utc(datetime(2016, 8, 24)): continue delta = relativedelta.relativedelta(uplift_date, as_utc(datetime(2014, 7, 1))) delta_num = delta.years * 12 + delta.months key = (delta_num, uplift_date.strftime('%b %Y'), channel) if key not in months: months[key] = 0 months[key] += 1 with open('uplift_dates.csv', 'w') as output_file: csv_writer = csv.writer(output_file) csv_writer.writerow(['Delta', 'Month', 'Channel', 'Number_of_uplifts']) for (delta_str, month, channel), number in sorted(months.items(), key=lambda x: x[0][0]): csv_writer.writerow([delta_str, month, channel, number])