async def uptake(product, version):
date = yesterday()
versions = await get_channel_versions(product, version)
version_params = [("versions", x) for x in versions]
params = [("start_date", date), ("end_date", date), ("product", product)]
params.extend(version_params)
url = crash_stats_query_url(params)
async with get_session() as session:
async with session.get(url) as resp:
body = await resp.json()
if not body['hits']:
# Try the day before
date = yesterday(days=2)
params = [("start_date", date), ("end_date", date),
("product", product)]
params.extend(version_params)
url = crash_stats_query_url(params)
async with session.get(url) as resp:
body = await resp.json()
if not body['hits']:
status = Status.ERROR
message = "No crash-stats ADI info for version {}".format(
versions)
return build_task_response(status, url, message)
current_version_hits = [
h for h in body['hits'] if h['version'] == version
]
if not current_version_hits:
status = Status.MISSING
message = "No crash-stats ADI hits for version {}".format(
version)
else:
version_users = current_version_hits.pop()["adi_count"]
total_users = sum([h['adi_count'] for h in body['hits']])
ratio = version_users / total_users
if ratio < 0.5:
status = Status.INCOMPLETE
else:
status = Status.EXISTS
message = 'Crash-Stats uptake for version {} is {:.2f}%'.format(
version, ratio * 100)
return build_task_response(status, url, message)
async def buildhub(product, version):
if 'build' in version:
version = version.replace('build', 'rc')
try:
build_ids = await get_build_ids_for_version(product, version)
status = True
except TaskError:
status = False
channel = get_version_channel(product, strip_candidate_info(version))
exists_message = 'Build IDs for this release: {}'
missing_message = 'Buildhub does not contain any information about this release yet.'
if status:
if channel is Channel.NIGHTLY:
last_expected_nightly = yesterday(formating='%Y%m%d')
if build_ids[0][:8] < last_expected_nightly:
status = Status.INCOMPLETE
build_ids = build_ids[:3]
else:
build_ids = [
bid for bid in build_ids if bid > last_expected_nightly
]
exists_message = exists_message.format(', '.join(build_ids))
url = get_buildhub_url(product, version, channel)
return build_task_response(status, url, exists_message, missing_message)
async def main_summary_uptake(product, version):
channel = get_version_channel(product, version)
async with get_session(headers=get_telemetry_auth_header()) as session:
# Get the build IDs for this channel
build_ids = await get_build_ids_for_version(product, version)
submission_date = yesterday(formating='%Y%m%d')
if channel is Channel.NIGHTLY:
build_ids = [bid for bid in build_ids if bid > submission_date]
version_name = "{} ({})".format(version, ", ".join(build_ids))
query_title = "Uptake {} {}"
query_title = query_title.format(product.title(), channel.value)
else:
version_name = "{} ({})".format(version, ", ".join(build_ids))
query_title = "Uptake {} {} {}"
query_title = query_title.format(product.title(), channel.value,
version_name)
# There are 100 samples, so we take the one from sample_id=${TELEMETRY_CACHED_SAMPLE}
# and we grow its value by a 100 times
query = """
WITH updated_t AS (
SELECT COUNT(DISTINCT client_id)*100 AS updated
FROM main_summary
WHERE submission_date_s3 >= date_format(current_date - interval '1' day, '%Y%m%d')
AND app_build_id IN ({build_ids})
AND normalized_channel = '{channel}'
AND sample_id = '{sample_id}'
),
total_t AS (
SELECT COUNT(DISTINCT client_id)*100 AS total
FROM main_summary
WHERE submission_date_s3 >= date_format(current_date - interval '1' day, '%Y%m%d')
AND normalized_channel = '{channel}'
AND sample_id = '{sample_id}'
)
SELECT updated * 1.0 / total as ratio, updated, total
FROM updated_t, total_t
""".format(build_ids=', '.join(["'{}'".format(bid) for bid in build_ids]),
channel=channel.value.lower(),
sample_id=TELEMETRY_CACHED_SAMPLE)
query_info = await get_query_info_from_title(session, query_title)
if query_info:
if channel is Channel.NIGHTLY:
# Update the NIGHTLY query with the last build_ids
await put_query(session,
query_title,
version_name,
query,
query_id=query_info['id'],
run=False)
# In that case the query already exists
latest_query_data_id = query_info["latest_query_data_id"]
# In case the query processing didn't start, the last_query_data_id can be None.
if latest_query_data_id is None:
url = "{}/queries/{}".format(TELEMETRY_SERVER,
query_info['id'])
return build_task_response(Status.INCOMPLETE, url,
"Query still processing.")
# Get the results if we know the query results ID
url = "{}/api/query_results/{}".format(TELEMETRY_SERVER,
latest_query_data_id)
async with session.get(url) as resp:
if resp.status != 200:
return build_task_response(
Status.MISSING, url,
"Query Result {} unavailable (HTTP {})".format(
latest_query_data_id, resp.status))
body = await resp.json()
# If no data are matching the query, we may have an empty list of results,
if not body["query_result"]["data"]["rows"]:
url = "{}/queries/{}".format(TELEMETRY_SERVER,
query_info['id'])
return build_task_response(
Status.ERROR, url, "No result found for your query.")
data = body["query_result"]["data"]["rows"][0]
# version_users = data["updated"]
# total_users = data["total"]
ratio = data["ratio"]
if ratio < 0.5:
status = Status.INCOMPLETE
else:
status = Status.EXISTS
url = "{}/queries/{}".format(TELEMETRY_SERVER, query_info["id"])
message = 'Telemetry uptake for version {} is {:.2f}%'.format(
version_name, ratio * 100)
return build_task_response(status, url, message)
query_id = await put_query(session, query_title, version_name, query)
url = "{}/queries/{}".format(TELEMETRY_SERVER, query_id)
message = 'Telemetry uptake calculation for version {} is in progress'.format(
version_name)
return build_task_response(Status.INCOMPLETE, url, message)
def test_yesterday():
utc_today = datetime.datetime.utcnow()
utc_yesterday = utc_today - datetime.timedelta(days=1)
result = yesterday(formating="%d-%H:%M")
assert result == utc_yesterday.strftime("%d-%H:%M")
async def main_summary_uptake(product, version):
channel = get_version_channel(product, version)
async with get_session(headers=get_telemetry_auth_header()) as session:
# Get the build IDs for this channel
build_ids = await get_build_ids_for_version(product, version)
submission_date = yesterday(formating='%Y%m%d')
if channel is Channel.NIGHTLY:
build_ids = [bid for bid in build_ids if bid > submission_date]
version_name = "{} ({})".format(version, ", ".join(build_ids))
query_title = "Uptake {} {}"
query_title = query_title.format(product.title(), channel.value)
else:
version_name = "{} ({})".format(version, ", ".join(build_ids))
query_title = "Uptake {} {} {}"
query_title = query_title.format(product.title(), channel.value, version_name)
url = "https://sql.telemetry.mozilla.org/queries/{}".format(TELEMETRY_UPTAKE_QUERY_ID)
saved_query = await get_saved_query_by_id(session, TELEMETRY_UPTAKE_QUERY_ID)
if not saved_query:
raise TelemetryUptakeConfigurationError(
"The saved Telemetry Uptake query can't be found. "
"({})".format(url)
)
query_data_id = saved_query["latest_query_data_id"]
query_results = await get_query_results(session, query_data_id)
rows = query_results["data"]["rows"]
if not rows:
return build_task_response(Status.ERROR, url,
"Query results contained no rows.")
# Looking up on a set is much faster.
our_build_ids = set(build_ids)
our_normalized_channel = channel.value.lower()
updated = 0
total = None
for row in rows:
if row['normalized_channel'] != our_normalized_channel:
continue
total = row['total']
if row['app_build_id'] in our_build_ids:
updated += row['updated']
if total:
ratio = updated / total
else:
ratio = 0
if ratio < 0.5:
status = Status.INCOMPLETE
else:
status = Status.EXISTS
message = 'Telemetry uptake for version {} is {:.1f}%'.format(
version_name, ratio * 100)
return build_task_response(status, url, message)