def main():
"""Add pdf info to all indicator metadata"""
metas = glob.glob('_indicators/*.md')
print('Processing ' + str(len(metas)) + ' metadata files...')
for meta in metas:
try:
md = yamlmd.read_yamlmd(meta)
md = add_goal_pdf_info(md, meta)
yamlmd.write_yamlmd(md, meta)
except Exception as e:
print(meta, e)
def test_read(tmpdir):
"""Check that the main write function works"""
test_header = ruamel.yaml.comments.CommentedMap({"a": 1})
test_md = ["\n", "the first thing\n"]
tmp_file = tmpdir.mkdir("md").join("test.md").strpath
write_yamlmd([test_header, test_md], tmp_file)
with open(tmp_file) as f:
tmp_lines = f.readlines()
assert tmp_lines[0] == "---\n"
assert len(tmp_lines) == 5
def build_meta(inid):
"""Perform pre-processing for the metadata files"""
status = True
# Read and write paths may be different
fr = indicator_path(inid, ftype='meta', mode='r')
fw = indicator_path(inid, ftype='meta', mode='w')
meta = yamlmd.read_yamlmd(fr)
git_update = sdg.git.get_git_updates(inid)
for k in git_update.keys():
meta[0][k] = git_update[k]
yamlmd.write_yamlmd(meta, fw)
return status
df.to_csv(data_path, index=False)
# Fix the special "graph_titles" metadata field we added above.
graph_titles = []
for series in metadata[indicator_id]['graph_titles']:
graph_titles.append({
'series': 'SERIES.' + series,
'title': 'SERIES.' + series,
})
metadata[indicator_id]['graph_titles'] = graph_titles
meta_path = os.path.join('meta', slug + '.md')
meta = yamlmd.read_yamlmd(meta_path)
for field in metadata[indicator_id]:
meta[0][field] = metadata[indicator_id][field]
yamlmd.write_yamlmd(meta, meta_path)
skip_translations = drop_these_columns()
for key in things_to_translate:
if key in skip_translations:
continue
if key not in english:
english[key] = things_to_translate[key]
else:
for item in things_to_translate[key]:
if item not in english[key]:
english[key][item] = things_to_translate[key][item]
skip_translations = [
'BASE_PER',
'COMMENT_OBS',
"reporting_status": "notstarted"
},
[
'\n',
'We are still looking for a suitable data source for this indicator. Please contact us if you think you can help by emailing <a href="mailto:[email protected]">[email protected]</a>.'
]
]
md_meta = merge_meta_data(md_data[0], meta)
return [md_meta, md_data[1]]
# %% Loop over all indicators
for index, row in dft.iterrows():
new_md = update_meta_data(index, row)
new_file = os.path.join("_indicators", row['File'])
print(row['File'])
yamlmd.write_yamlmd(new_md, new_file)
# %% Find the new files
empty_data = pd.read_csv("data/indicator_1-1-1.csv")
for index, row in dft.query('New').iterrows():
csv_file = ('indicator_' + row.Goal + '-' + row.Target + '-' + row.INDN +
'.csv')
csv_path = os.path.join('data', csv_file)
if (not os.path.isfile(csv_path)):
empty_data.to_csv(os.path.join("data", csv_file), index=False)
if has_national_title and not has_global_title:
if translation_is_global(meta[0]['indicator_name']):
# In some cases the national indicators use a global indicator name.
has_global_title = True
if not has_global_title:
indicator_id_parts = indicator_id.split('-')
if len(indicator_id_parts) == 3:
has_global_title = True
if has_global_title and has_national_title:
meta[0]['indicator_name'] = translation_key_global
meta[0]['indicator_available'] = translation_key_national
meta[0]['graph_title'] = translation_key_national
elif has_global_title:
meta[0]['indicator_name'] = translation_key_global
if 'indicator_available' in meta[0]:
del meta[0]['indicator_available']
meta[0]['graph_title'] = translation_key_global
elif has_national_title:
meta[0]['indicator_name'] = translation_key_national
if 'indicator_available' in meta[0]:
del meta[0]['indicator_available']
meta[0]['graph_title'] = translation_key_national
else:
print('WARNING - indicator ' + indicator_id +
' has neither global nor national title.')
yamlmd.write_yamlmd(meta, filepath)
@author: dashton
"""
import glob
import yamlmd
import pandas as pd
# if you don't have yamlmd do
# pip install git+git://github.com/dougmet/yamlmd
goals = glob.glob('_goals/*.md')
goals_df = pd.read_csv('data/sdg_goals.csv')
for goal in goals:
ymd = yamlmd.read_yamlmd(goal)
ymd_head = dict(ymd[0])
csv_info = goals_df[goals_df['goal'] == ymd_head['sdg_goal']].to_dict(
'records')[0]
out_header = {
'title': csv_info['title'],
'short': csv_info['short'],
'sdg_goal': str(csv_info['goal']),
'color': csv_info['colorInfo/hex']
}
out_ymd = [out_header, ymd[1]]
yamlmd.write_yamlmd(out_ymd, goal)
# -*- coding: utf-8 -*-
"""
Created on Wed May 30 15:47:21 2018
@author: dashton
"""
import glob
import yamlmd
# if you don't have yamlmd do
# pip install git+git://github.com/dougmet/yamlmd
page_only_vars = ['layout', 'permalink', 'indicator']
metas = glob.glob('_indicators/*.md')
for meta in metas:
ymd = yamlmd.read_yamlmd(meta)
ymd_head = dict(ymd[0])
out_header = {k: v for (k, v) in ymd_head.items() if k in page_only_vars}
out_ymd = [out_header, ymd[1]]
yamlmd.write_yamlmd(out_ymd, meta)
