def load_data(query_date, lang, project):
file_name = DATA_PATH_TMPL.format(
lang=lang, project=project, year=query_date.year, month=query_date.month, day=query_date.day
)
with open(file_name, "r") as data_file:
data = load(data_file)
return data
def load_data(query_date, lang, project):
file_name = DATA_PATH_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day)
with open(file_name, 'r') as data_file:
data = load(data_file)
return data
def save_traffic_stats(lang, project, query_date, limit=DEFAULT_LIMIT):
'''\
1. Get articles
2. Add images and summaries
3. Prepare and save results
'''
articles = make_article_list(query_date,
lang=lang,
project=project)
total_traffic = get_project_traffic(query_date, lang, project)
articles = articles[:limit]
articles = add_extras(articles, lang=lang, project=project)
ret = {'articles': articles,
'formatted_date': format_date(query_date,
format='d MMMM yyyy',
locale=lang),
'date': {'day': query_date.day,
'month': query_date.month,
'year': query_date.year},
'lang': lang,
'full_lang': LOCAL_LANG_MAP[lang],
'total_traffic': total_traffic,
'total_traffic_short': shorten_number(total_traffic),
'examples': [articles[0],
articles[1],
articles[2],
articles[query_date.day * 2]], # haha ok..
'project': project.capitalize(),
'permalink': DATE_PERMALINK_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day),
'meta': {'fetched': datetime.utcnow().isoformat()}}
outfile_name = DATA_PATH_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day)
with tlog.critical('saving_single_day_stats') as rec:
rec['out_file'] = os.path.abspath(outfile_name)
try:
out_file = codecs.open(outfile_name, 'w')
except IOError:
mkdir_p(os.path.dirname(outfile_name))
out_file = codecs.open(outfile_name, 'w')
with out_file:
data_bytes = json.dumps(ret, indent=2, sort_keys=True)
rec['len_bytes'] = len(data_bytes)
out_file.write(data_bytes)
rec.success('wrote {len_bytes} bytes to {out_file}')
return
def load_traffic(query_date, lang, project):
file_name = DATA_PATH_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day)
try:
data_file = codecs.open(file_name, 'r')
except IOError:
return None
with data_file:
return json.load(data_file)
def load_traffic(query_date, lang, project):
file_name = DATA_PATH_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day)
try:
data_file = codecs.open(file_name, 'r')
except IOError:
return None
with data_file:
return json.load(data_file)
def save(render_ctx, is_dev=DEBUG):
fargs = {'date_str': datetime.utcnow().strftime('%Y%m%d'),
'lang_shortcode': render_ctx['short_lang_name'],
'dev_flag': ''}
if is_dev:
fargs['dev_flag'] = '_dev'
out_path = DATA_PATH_TMPL.format(**fargs)
rendered = render_rc(render_ctx)
try:
out_file = open(out_path, 'w')
except IOError:
mkdir_p(os.path.dirname(out_path))
out_file = open(out_path, 'w')
# if exception, couldn't create file or parent directories
with out_file:
out_file.write(rendered)
return (out_path, len(rendered))
def save(render_ctx, is_dev=DEBUG):
fargs = {'date_str': datetime.utcnow().strftime('%Y%m%d'),
'lang_shortcode': render_ctx['short_lang_name'],
'dev_flag': ''}
if is_dev:
fargs['dev_flag'] = '_dev'
out_path = DATA_PATH_TMPL.format(**fargs)
rendered = render_rc(render_ctx)
try:
out_file = open(out_path, 'w')
except IOError:
mkdir_p(os.path.dirname(out_path))
out_file = open(out_path, 'w')
# if exception, couldn't create file or parent directories
with out_file:
out_file.write(rendered)
return (out_path, len(rendered))
def save_traffic_stats(lang, project, query_date, limit=DEFAULT_LIMIT):
'''\
1. Get articles
2. Add images and summaries
3. Prepare and save results
'''
articles = make_article_list(query_date, lang=lang, project=project)
total_traffic = get_project_traffic(query_date, lang, project)
articles = articles[:limit]
articles = add_extras(articles, lang=lang, project=project)
ret = {
'articles':
articles,
'formatted_date':
format_date(query_date, format='d MMMM yyyy', locale=lang),
'date': {
'day': query_date.day,
'month': query_date.month,
'year': query_date.year
},
'lang':
lang,
'full_lang':
LOCAL_LANG_MAP[lang],
'total_traffic':
total_traffic,
'total_traffic_short':
shorten_number(total_traffic),
'examples':
[articles[0], articles[1], articles[2],
articles[query_date.day * 2]], # haha ok..
'project':
project.capitalize(),
'permalink':
DATE_PERMALINK_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day),
'meta': {
'fetched': datetime.utcnow().isoformat()
}
}
outfile_name = DATA_PATH_TMPL.format(lang=lang,
project=project,
year=query_date.year,
month=query_date.month,
day=query_date.day)
with tlog.critical('saving_single_day_stats') as rec:
rec['out_file'] = os.path.abspath(outfile_name)
try:
out_file = codecs.open(outfile_name, 'w')
except IOError:
mkdir_p(os.path.dirname(outfile_name))
out_file = codecs.open(outfile_name, 'w')
with out_file:
data_bytes = json.dumps(ret, indent=2, sort_keys=True)
rec['len_bytes'] = len(data_bytes)
out_file.write(data_bytes)
rec.success('wrote {len_bytes} bytes to {out_file}')
return