def update_metric(url, start_datetime, change_datetime, end_datetime,
stats_dict, metric):
metric_before = metric_after = 0
while start_datetime < change_datetime:
curr_day_before = to_date(start_datetime)
metric_before += stats_dict.get(url,
{}).get('metrics',
{}).get(curr_day_before,
{}).get(metric, 0)
start_datetime += timedelta(days=1)
while change_datetime < end_datetime:
curr_day_after = to_date(change_datetime)
metric_after += stats_dict.get(url,
{}).get('metrics',
{}).get(curr_day_after,
{}).get(metric, 0)
change_datetime += timedelta(days=1)
return metric_before, metric_after
def update_pageviews(user_id, view_id, start_date, finish_date, url,
ga_stats_dict, access_token):
"""
Sends parameters related to pageviews to GA function, gets response by chunks,
parses it and updates the GA stats dict
:param user_id:
:param access_token:
:param url:
:param view_id:
:param start_date:
:param finish_date:
:param ga_stats_dict:
:return:
"""
metrics = [{'expression': 'ga:pageviews'}]
dimensions = [{'name': 'ga:date'}, {'name': 'ga:pagePath'}]
custom_filter = 'ga:pagePath'
google_response = request_google_core_api(user_id=user_id,
view_id=view_id,
start_date=start_date,
finish_date=finish_date,
metrics=metrics,
dimensions=dimensions,
url_item=url,
custom_filter=custom_filter,
access_token=access_token)
try:
for row in iter(google_response):
current_date_datetime = datetime.strptime(
row.get('dimensions', [])[0], '%Y%m%d')
current_date = to_date(current_date_datetime)
pageviews = int(row.get('metrics', [])[0].get('values', [])[0])
ga_stats_dict.setdefault(url, {})
ga_stats_dict[url].setdefault('metrics', {})
ga_stats_dict[url]['metrics'].setdefault(current_date, {})
ga_stats_dict[url]['metrics'][current_date][
'pageviews'] = pageviews
except KeyError:
raise Exception(google_response)
return ga_stats_dict
def update_single_url(*args, **kwargs):
access_token = kwargs['access_token']
user_id = kwargs['user_id']
url_id = kwargs['url_id']
try:
user = User.query.get(user_id)
url_item = Url.query.get(url_id)
url_list = [url_item]
authorized_urls, not_authorized_urls = get_authorized_urls(
url_list, user_id)
total_links = 1
if total_links == 0:
_set_task_progress(100)
return False
with app.app_context():
i = 0
_set_task_progress(0)
update_stats(user_id=user_id,
url=url_item.url,
url_id=url_item.id,
old_link=url_item.old_url,
human_date=to_date(url_item.date),
period_list=json.loads(user.pref.periods),
access_token=access_token)
i += 1
_set_task_progress(100 * i // total_links)
if len(not_authorized_urls) > 0:
_inform_exceptions(
f"The following links are not authorized to check: {not_authorized_urls}"
)
except Exception as e:
_set_task_progress(100, failed=1)
_inform_exceptions(str(e))
app.logger.error('Unhandled exception', exc_info=sys.exc_info())
def update_stats(user_id,
url,
url_id,
human_date,
period_list,
access_token,
old_link=None):
"""
"""
metrics = ('pageviews', 'landing', 'bounces', 'transactions')
ga_stats_dict = {}
view_id, time_shift = detect_view(user_id, url)
yesterday = datetime.today() - timedelta(hours=time_shift) - timedelta(
days=1)
date_datetime = to_datetime(human_date)
min_value = max_value = date_datetime
for period in period_list:
if date_datetime + timedelta(days=period) <= yesterday:
max_value = date_datetime + timedelta(days=period)
min_value = date_datetime - timedelta(days=period)
start_date = to_date(min_value)
finish_date = to_date(max_value)
update_landing(user_id=user_id,
view_id=view_id,
start_date=start_date,
finish_date=finish_date,
url=url,
ga_stats_dict=ga_stats_dict,
access_token=access_token)
update_pageviews(user_id=user_id,
view_id=view_id,
start_date=start_date,
finish_date=finish_date,
url=url,
ga_stats_dict=ga_stats_dict,
access_token=access_token)
if old_link:
update_landing(user_id=user_id,
view_id=view_id,
start_date=start_date,
finish_date=finish_date,
url=old_link,
ga_stats_dict=ga_stats_dict,
access_token=access_token)
update_pageviews(user_id=user_id,
view_id=view_id,
start_date=start_date,
finish_date=finish_date,
url=old_link,
ga_stats_dict=ga_stats_dict,
access_token=access_token)
for period in period_list:
start_datetime = date_datetime - timedelta(days=period)
change_datetime = date_datetime
finish_datetime = date_datetime + timedelta(days=period)
if (yesterday - change_datetime).days < period:
continue # Skip the period, if the period is not passed yet
else:
for metric in metrics:
metric_before = metric_after = 0
if ga_stats_dict.get(url):
new_metric_before, new_metric_after = update_metric(
url=url,
start_datetime=start_datetime,
change_datetime=change_datetime,
end_datetime=finish_datetime,
stats_dict=ga_stats_dict,
metric=metric)
metric_before += new_metric_before
metric_after += new_metric_after
if old_link and ga_stats_dict.get(old_link):
old_metric_before, old_metric_after = update_metric(
url=old_link,
start_datetime=start_datetime,
change_datetime=change_datetime,
end_datetime=finish_datetime,
stats_dict=ga_stats_dict,
metric=metric)
metric_before += old_metric_before
metric_after += old_metric_after
try:
metric_change = (metric_after -
metric_before) / metric_before
except ZeroDivisionError:
if metric_after == 0:
metric_change = 0
else:
metric_change = 1
# Save metric before
update_db(url_id=url_id,
metric=metric,
period=str('-' + str(period)),
value=metric_before)
# Save metric after
update_db(url_id=url_id,
metric=metric,
period=str(period),
value=metric_after)
# Save metric change
update_db(url_id=url_id,
metric=metric + '_change',
period=str(period),
value=metric_change)
db.session.commit()
return None
def update_landing(user_id, view_id, start_date, finish_date, url,
ga_stats_dict, access_token):
"""
Sends parameters related to landing to GA function, gets response by chunks,
parses it and updates the GA stats dict
:param access_token:
:param user_id:
:param ga_stats_dict:
:param view_id:
:param start_date:
:param finish_date:
:param url:
:return:
"""
metrics = [{
'expression': 'ga:sessions'
}, {
'expression': 'ga:bounces'
}, {
'expression': 'ga:transactions'
}]
dimensions = [{'name': 'ga:date'}, {'name': 'ga:landingPagePath'}]
custom_filter = 'ga:landingPagePath'
google_response = request_google_core_api(user_id=user_id,
view_id=view_id,
start_date=start_date,
finish_date=finish_date,
metrics=metrics,
dimensions=dimensions,
url_item=url,
custom_filter=custom_filter,
access_token=access_token)
# uri = take_uri(url)
try:
for row in iter(google_response): # Iterate over yielded chunk of data
current_date_datetime = datetime.strptime(row['dimensions'][0],
'%Y%m%d')
current_date = to_date(current_date_datetime)
# if row.get('dimensions', [])[1] == uri:
current_sessions = int(
row.get('metrics', [])[0].get('values', [])[0])
current_bounces = int(
row.get('metrics', [])[0].get('values', [])[1])
current_trans = int(row.get('metrics', [])[0].get('values', [])[2])
ga_stats_dict.setdefault(url, {})
ga_stats_dict[url].setdefault('metrics', {})
ga_stats_dict[url]['metrics'].setdefault(current_date, {})
ga_stats_dict[url]['metrics'][current_date][
'landing'] = current_sessions
ga_stats_dict[url]['metrics'][current_date][
'bounces'] = current_bounces
ga_stats_dict[url]['metrics'][current_date][
'transactions'] = current_trans
except KeyError:
pprint(google_response)
return ga_stats_dict