def make_article_list(query_date, lang, project):
wiki_info = get_wiki_info(lang, project)
raw_traffic = get_traffic(query_date, lang, project)
articles = [a for a in raw_traffic if is_article(a['article'], wiki_info)]
prev_traffic = load_prev_traffic(query_date, 10, lang, project)
tweet_templates = get_tweet_templates(lang)
ret = []
for article in articles:
title = article['article']
if not prev_traffic:
prev_article = {}
else:
prev_article = prev_traffic[0].get(title, {})
permalink = PERMALINK_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day,
title=title)
streak = find_streaks(title, prev_traffic)
article.update(streak)
article['views_short'] = shorten_number(article['views'])
article['url'] = 'https://%s.%s.org/wiki/%s' % (lang, project, title)
article['title'] = title.replace('_', ' ')
article['permalink'] = permalink.encode('utf-8')
article['rank'] = len(ret) + 1
article['pviews'] = prev_article.get('views', None)
article['prank'] = prev_article.get('rank', None)
if not article['pviews'] or article['views'] > article['pviews']:
article['view_trend'] = '↑'
elif article['views'] == article['pviews']:
article['view_trend'] = None
else:
article['view_trend'] = '↓'
if not article['prank'] or article['rank'] < article['prank']:
article['rank_trend'] = '↑'
elif article['rank'] == article['prank']:
article['rank_trend'] = None
else:
article['rank_trend'] = '↓'
if article['pviews']:
view_delta = abs(article['views'] - article['pviews'])
article['view_delta'] = shorten_number(view_delta)
else:
article['view_delta'] = None
tweet = tweet_composer(article, lang, project, tweets=tweet_templates)
article['tweet'] = quote_plus(tweet.encode('utf-8'), safe=':/')
ret.append(article)
return ret
def make_article_list(query_date, lang, project):
wiki_info = get_wiki_info(lang, project)
raw_traffic = get_traffic(query_date, lang, project)
articles = [a for a in raw_traffic if is_article(a['article'], wiki_info)]
prev_traffic = load_prev_traffic(query_date, 10, lang, project)
tweet_templates = get_tweet_templates(lang)
ret = []
for article in articles:
title = article['article']
if not prev_traffic:
prev_article = {}
else:
prev_article = prev_traffic[0].get(title, {})
permalink = PERMALINK_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day,
title=title)
streak = find_streaks(title, prev_traffic)
article.update(streak)
article['views_short'] = shorten_number(article['views'])
article['url'] = 'https://%s.%s.org/wiki/%s' % (lang, project, title)
article['title'] = title.replace('_', ' ')
article['permalink'] = permalink.encode('utf-8')
article['rank'] = len(ret) + 1
article['pviews'] = prev_article.get('views', None)
article['prank'] = prev_article.get('rank', None)
if not article['pviews'] or article['views'] > article['pviews']:
article['view_trend'] = '↑'
elif article['views'] == article['pviews']:
article['view_trend'] = None
else:
article['view_trend'] = '↓'
if not article['prank'] or article['rank'] < article['prank']:
article['rank_trend'] = '↑'
elif article['rank'] == article['prank']:
article['rank_trend'] = None
else:
article['rank_trend'] = '↓'
if article['pviews']:
view_delta = abs(article['views'] - article['pviews'])
article['view_delta'] = shorten_number(view_delta)
else:
article['view_delta'] = None
tweet = tweet_composer(article, lang, project, tweets=tweet_templates)
article['tweet'] = quote_plus(tweet.encode('utf-8'), safe=':/')
ret.append(article)
return ret
def save_traffic_stats(lang, project, query_date, limit=DEFAULT_LIMIT):
'''\
1. Get articles
2. Add images and summaries
3. Prepare and save results
'''
articles = make_article_list(query_date,
lang=lang,
project=project)
total_traffic = get_project_traffic(query_date, lang, project)
articles = articles[:limit]
articles = add_extras(articles, lang=lang, project=project)
ret = {'articles': articles,
'formatted_date': format_date(query_date,
format='d MMMM yyyy',
locale=lang),
'date': {'day': query_date.day,
'month': query_date.month,
'year': query_date.year},
'lang': lang,
'full_lang': LOCAL_LANG_MAP[lang],
'total_traffic': total_traffic,
'total_traffic_short': shorten_number(total_traffic),
'examples': [articles[0],
articles[1],
articles[2],
articles[query_date.day * 2]], # haha ok..
'project': project.capitalize(),
'permalink': DATE_PERMALINK_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day),
'meta': {'fetched': datetime.utcnow().isoformat()}}
outfile_name = DATA_PATH_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day)
with tlog.critical('saving_single_day_stats') as rec:
rec['out_file'] = os.path.abspath(outfile_name)
try:
out_file = codecs.open(outfile_name, 'w')
except IOError:
mkdir_p(os.path.dirname(outfile_name))
out_file = codecs.open(outfile_name, 'w')
with out_file:
data_bytes = json.dumps(ret, indent=2, sort_keys=True)
rec['len_bytes'] = len(data_bytes)
out_file.write(data_bytes)
rec.success('wrote {len_bytes} bytes to {out_file}')
return
def save_traffic_stats(lang, project, query_date, limit=DEFAULT_LIMIT):
'''\
1. Get articles
2. Add images and summaries
3. Prepare and save results
'''
articles = make_article_list(query_date, lang=lang, project=project)
total_traffic = get_project_traffic(query_date, lang, project)
articles = articles[:limit]
articles = add_extras(articles, lang=lang, project=project)
ret = {
'articles':
articles,
'formatted_date':
format_date(query_date, format='d MMMM yyyy', locale=lang),
'date': {
'day': query_date.day,
'month': query_date.month,
'year': query_date.year
},
'lang':
lang,
'full_lang':
LOCAL_LANG_MAP[lang],
'total_traffic':
total_traffic,
'total_traffic_short':
shorten_number(total_traffic),
'examples':
[articles[0], articles[1], articles[2],
articles[query_date.day * 2]], # haha ok..
'project':
project.capitalize(),
'permalink':
DATE_PERMALINK_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day),
'meta': {
'fetched': datetime.utcnow().isoformat()
}
}
outfile_name = DATA_PATH_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day)
with tlog.critical('saving_single_day_stats') as rec:
rec['out_file'] = os.path.abspath(outfile_name)
try:
out_file = codecs.open(outfile_name, 'w')
except IOError:
mkdir_p(os.path.dirname(outfile_name))
out_file = codecs.open(outfile_name, 'w')
with out_file:
data_bytes = json.dumps(ret, indent=2, sort_keys=True)
rec['len_bytes'] = len(data_bytes)
out_file.write(data_bytes)
rec.success('wrote {len_bytes} bytes to {out_file}')
return
