def geocode_cadastral(self, cadastral_number, retries=3):
"""Converting cadastral number into geographic coordinates using xgis-ky service
http://geoportaal.maaamet.ee/est/Teenused/Poordumine-kaardirakendusse-labi-URLi-p9.html#a13
:param cadastral_number: cadastral number to retrieve coordinates for.
"""
if not cadastral_number:
raise ValueError
result = {"cadastral": cadastral_number}
querystring = {"what": "tsentroid",
"out": "json",
"ky": cadastral_number}
response = utils.get_with_retries(retries, url=self.geocode_url, params=querystring)
if response.ok:
try:
json = response.json()["1"]
result["x"] = json["X"]
result["y"] = json["Y"]
except (ValueError, KeyError):
raise GeocodeError(cadastral_number)
wgs84 = Proj(init="epsg:4326")
lest97 = Proj(init="epsg:3301")
result["lon"], result["lat"] = transform(lest97, wgs84, json["X"], json["Y"])
return result
else:
raise HttpError(response.status_code, self.geocode_url)
def __getVersions():
"""Get the versions number for each channel
Returns:
dict: versions for each channel
"""
r = utils.get_with_retries(
'https://product-details.mozilla.org/1.0/firefox_versions.json')
if r.status_code != 200:
print(r.text)
raise Exception(r)
data = r.json()
aurora = data['FIREFOX_AURORA']
nightly = '%d.0a1' % (__get_major(aurora) + 1)
esr = data['FIREFOX_ESR_NEXT']
if not esr:
esr = data['FIREFOX_ESR']
if esr.endswith('esr'):
esr = esr[:-3]
return {
'release': data['LATEST_FIREFOX_VERSION'],
'beta': data['LATEST_FIREFOX_RELEASED_DEVEL_VERSION'],
'aurora': str(aurora),
'nightly': nightly,
'esr': esr
}
def download_crash(uuid):
response = utils.get_with_retries(
'https://crash-stats.mozilla.com/api/ProcessedCrash',
params={
'crash_id': uuid,
})
response.raise_for_status()
return response.json()
def query_dxr(q):
r = utils.get_with_retries('https://dxr.mozilla.org/mozilla-central/search', params={
'q': q,
'limit': 1000
}, headers={
'Accept': 'application/json'
})
if r.status_code != 200:
print(r.text)
raise Exception(r)
return r.json()
def get_signatures_from_bug(bug_id):
url = 'https://bugzilla.mozilla.org/rest/bug'
response = utils.get_with_retries(url, params={'id': bug_id})
response.raise_for_status()
signature = []
for sig in response.json()['bugs'][0]['cf_crash_signature'].split('\r\n'):
pos = sig.find('[@')
if pos != -1:
sig = sig[pos + 2:]
pos2 = sig.rfind(']')
if pos2 != -1:
sig = sig[:pos2]
signature.append(sig.strip())
return signature
def download_file(self, url, out_filename, block_size=1024,
timeout=20, extension_from_header=False, retries=5):
response = utils.get_with_retries(retries, url=url, timeout=timeout)
if response.ok:
if extension_from_header:
_, params = parse_header(response.headers.get("content-disposition", ""))
_, extension = os.path.splitext(params.get("filename", ""))
if extension:
out_filename = "".join([out_filename, extension])
with open(out_filename, "w") as f:
for block in response.iter_content(block_size):
f.write(block)
return out_filename
else:
raise HttpError(response.status_code, url)
def download_day_crashes(day, product='Firefox'):
crashes = []
path = file_path(day, product)
try:
crashes += read_json(path)
except IOError:
pass
finished = False
RESULTS_NUMBER = 1000
while not finished:
params = {
'product': product,
'date': ['>=' + str(day), '<' + str(day + timedelta(1))],
'_columns': [
'uuid',
'signature',
'proto_signature',
],
'_results_number': RESULTS_NUMBER,
'_results_offset': len(crashes),
'_facets_size': 0,
}
print(str(day) + ' - ' + str(len(crashes)))
response = utils.get_with_retries(
'https://crash-stats.mozilla.com/api/SuperSearch', params=params)
response.raise_for_status()
found = response.json()['hits']
crashes += found
if len(found) < RESULTS_NUMBER:
finished = True
uuids = set()
filtered_crashes = []
for crash in crashes:
if crash['uuid'] not in uuids:
uuids.add(crash['uuid'])
filtered_crashes.append(crash)
write_json(path, filtered_crashes)
def download_stack_traces_for_signature(signature, traces_num=100):
url = 'https://crash-stats.mozilla.com/api/SuperSearch'
params = {
'signature': '=' + signature,
'_facets': ['proto_signature'],
'_facets_size': traces_num,
'_results_number': 0
}
res = utils.get_with_retries(url, params)
records = res.json()['facets']['proto_signature']
traces = set()
for record in records:
traces.add(record['term'])
return traces
def get_top(number, channel, days=3, product='Firefox'):
versions = get_versions(channel, product)
url = 'https://crash-stats.mozilla.com/api/SuperSearch'
params = {
'product': product,
'date':
['>=' + str(utils.utc_today() - timedelta(days) + timedelta(1))],
'version': versions,
'_results_number': 0,
'_facets_size': number,
}
r = utils.get_with_retries(url, params=params)
if r.status_code != 200:
print(r.text)
raise Exception(r)
return [signature['term'] for signature in r.json()['facets']['signature']]
def get_versions(channel, product='Firefox'):
channel = channel.lower()
version = str(versions.get(base=True)[channel])
r = utils.get_with_retries(
'https://crash-stats.mozilla.com/api/ProductVersions',
params={
'product': product,
'active': True,
'is_rapid_beta': False,
})
if r.status_code != 200:
print(r.text)
raise Exception(r)
return [
result['version'] for result in r.json()['hits']
if result['version'].startswith(version)
and result['build_type'] == channel
]
def get_stack_traces_for_signature(fnames, signature, traces_num=100):
traces = set()
# query stack traces online
url = 'https://crash-stats.mozilla.com/api/SuperSearch'
params = {
'signature': '=' + signature,
'_facets': ['proto_signature'],
'_facets_size': traces_num,
'_results_number': 0
}
res = utils.get_with_retries(url, params)
records = res.json()['facets']['proto_signature']
for record in records:
traces.add(record['term'])
# query stack traces from downloaded data
for line in utils.read_files(fnames):
data = json.loads(line)
if data['signature'] == signature:
traces.add(data['proto_signature'])
return list(traces)
def download_stack_trace_for_crashid(crash_id):
url = 'https://crash-stats.mozilla.com/api/ProcessedCrash'
params = {'crash_id': crash_id}
res = utils.get_with_retries(url, params)
return res.json()['proto_signature']
def test_get_with_retries_raises_400_with_no_params(self):
resp = utils.get_with_retries(self.url)
self.assertEqual(resp.status_code, 400)
def test_get_with_retries(self):
bug_id = '1308863'
resp = utils.get_with_retries(self.url, params={'id': bug_id})
self.assertEqual(resp.status_code, 200)