def make_article_list(query_date, lang, project): wiki_info = get_wiki_info(lang, project) raw_traffic = get_traffic(query_date, lang, project) articles = [a for a in raw_traffic if is_article(a['article'], wiki_info)] prev_traffic = load_prev_traffic(query_date, 10, lang, project) tweet_templates = get_tweet_templates(lang) ret = [] for article in articles: title = article['article'] if not prev_traffic: prev_article = {} else: prev_article = prev_traffic[0].get(title, {}) permalink = PERMALINK_TMPL.format(lang=lang, project=project, year=query_date.year, month=query_date.month, day=query_date.day, title=title) streak = find_streaks(title, prev_traffic) article.update(streak) article['views_short'] = shorten_number(article['views']) article['url'] = 'https://%s.%s.org/wiki/%s' % (lang, project, title) article['title'] = title.replace('_', ' ') article['permalink'] = permalink.encode('utf-8') article['rank'] = len(ret) + 1 article['pviews'] = prev_article.get('views', None) article['prank'] = prev_article.get('rank', None) if not article['pviews'] or article['views'] > article['pviews']: article['view_trend'] = '↑' elif article['views'] == article['pviews']: article['view_trend'] = None else: article['view_trend'] = '↓' if not article['prank'] or article['rank'] < article['prank']: article['rank_trend'] = '↑' elif article['rank'] == article['prank']: article['rank_trend'] = None else: article['rank_trend'] = '↓' if article['pviews']: view_delta = abs(article['views'] - article['pviews']) article['view_delta'] = shorten_number(view_delta) else: article['view_delta'] = None tweet = tweet_composer(article, lang, project, tweets=tweet_templates) article['tweet'] = quote_plus(tweet.encode('utf-8'), safe=':/') ret.append(article) return ret
def save_traffic_stats(lang, project, query_date, limit=DEFAULT_LIMIT): '''\ 1. Get articles 2. Add images and summaries 3. Prepare and save results ''' articles = make_article_list(query_date, lang=lang, project=project) total_traffic = get_project_traffic(query_date, lang, project) articles = articles[:limit] articles = add_extras(articles, lang=lang, project=project) ret = {'articles': articles, 'formatted_date': format_date(query_date, format='d MMMM yyyy', locale=lang), 'date': {'day': query_date.day, 'month': query_date.month, 'year': query_date.year}, 'lang': lang, 'full_lang': LOCAL_LANG_MAP[lang], 'total_traffic': total_traffic, 'total_traffic_short': shorten_number(total_traffic), 'examples': [articles[0], articles[1], articles[2], articles[query_date.day * 2]], # haha ok.. 'project': project.capitalize(), 'permalink': DATE_PERMALINK_TMPL.format(lang=lang, project=project, year=query_date.year, month=query_date.month, day=query_date.day), 'meta': {'fetched': datetime.utcnow().isoformat()}} outfile_name = DATA_PATH_TMPL.format(lang=lang, project=project, year=query_date.year, month=query_date.month, day=query_date.day) with tlog.critical('saving_single_day_stats') as rec: rec['out_file'] = os.path.abspath(outfile_name) try: out_file = codecs.open(outfile_name, 'w') except IOError: mkdir_p(os.path.dirname(outfile_name)) out_file = codecs.open(outfile_name, 'w') with out_file: data_bytes = json.dumps(ret, indent=2, sort_keys=True) rec['len_bytes'] = len(data_bytes) out_file.write(data_bytes) rec.success('wrote {len_bytes} bytes to {out_file}') return
def save_traffic_stats(lang, project, query_date, limit=DEFAULT_LIMIT): '''\ 1. Get articles 2. Add images and summaries 3. Prepare and save results ''' articles = make_article_list(query_date, lang=lang, project=project) total_traffic = get_project_traffic(query_date, lang, project) articles = articles[:limit] articles = add_extras(articles, lang=lang, project=project) ret = { 'articles': articles, 'formatted_date': format_date(query_date, format='d MMMM yyyy', locale=lang), 'date': { 'day': query_date.day, 'month': query_date.month, 'year': query_date.year }, 'lang': lang, 'full_lang': LOCAL_LANG_MAP[lang], 'total_traffic': total_traffic, 'total_traffic_short': shorten_number(total_traffic), 'examples': [articles[0], articles[1], articles[2], articles[query_date.day * 2]], # haha ok.. 'project': project.capitalize(), 'permalink': DATE_PERMALINK_TMPL.format(lang=lang, project=project, year=query_date.year, month=query_date.month, day=query_date.day), 'meta': { 'fetched': datetime.utcnow().isoformat() } } outfile_name = DATA_PATH_TMPL.format(lang=lang, project=project, year=query_date.year, month=query_date.month, day=query_date.day) with tlog.critical('saving_single_day_stats') as rec: rec['out_file'] = os.path.abspath(outfile_name) try: out_file = codecs.open(outfile_name, 'w') except IOError: mkdir_p(os.path.dirname(outfile_name)) out_file = codecs.open(outfile_name, 'w') with out_file: data_bytes = json.dumps(ret, indent=2, sort_keys=True) rec['len_bytes'] = len(data_bytes) out_file.write(data_bytes) rec.success('wrote {len_bytes} bytes to {out_file}') return