def read_meta(inid, git=True, src_dir='', git_data_dir=None):
"""Perform pre-processing for the metadata files"""
status = True
# Read and write paths may be different
fr = input_path(inid, ftype='meta', src_dir=src_dir)
meta_md = yamlmd.read_yamlmd(fr)
meta = dict(meta_md[0])
if git:
git_update = sdg.git.get_git_updates(inid,
src_dir=src_dir,
git_data_dir=git_data_dir)
for k in git_update.keys():
meta[k] = git_update[k]
meta['page_content'] = ''.join(meta_md[1])
# Now look for all subfolders of the meta folder, which may contain
# multilingual metadata, and add them as well.
meta_folder = input_path(None, ftype='meta', src_dir=src_dir)
languages = next(os.walk(meta_folder))[1]
for language in languages:
i18n_fr = os.path.join(meta_folder, language, inid + '.md')
if os.path.isfile(i18n_fr):
i18n_meta_md = yamlmd.read_yamlmd(i18n_fr)
i18n_meta = dict(i18n_meta_md[0])
meta[language] = i18n_meta
meta[language]['page_content'] = ''.join(i18n_meta_md[1])
return meta
def test_read():
"""Check that the main read function works"""
lorum_file = os.path.join(base_path, "data", "lorum.md")
lorum = read_yamlmd(lorum_file)
assert len(lorum) == 2
assert isinstance(lorum[0], ruamel.yaml.comments.CommentedMap)
assert isinstance(lorum[1], list)
def main():
"""Add pdf info to all indicator metadata"""
metas = glob.glob('_indicators/*.md')
print('Processing ' + str(len(metas)) + ' metadata files...')
for meta in metas:
try:
md = yamlmd.read_yamlmd(meta)
md = add_goal_pdf_info(md, meta)
yamlmd.write_yamlmd(md, meta)
except Exception as e:
print(meta, e)
def build_meta(inid):
"""Perform pre-processing for the metadata files"""
status = True
# Read and write paths may be different
fr = indicator_path(inid, ftype='meta', mode='r')
fw = indicator_path(inid, ftype='meta', mode='w')
meta = yamlmd.read_yamlmd(fr)
git_update = sdg.git.get_git_updates(inid)
for k in git_update.keys():
meta[0][k] = git_update[k]
yamlmd.write_yamlmd(meta, fw)
return status
def read_meta(inid, git=True, src_dir=''):
"""Perform pre-processing for the metadata files"""
status = True
# Read and write paths may be different
fr = input_path(inid, ftype='meta', src_dir=src_dir)
meta_md = yamlmd.read_yamlmd(fr)
meta = dict(meta_md[0])
if git:
git_update = sdg.git.get_git_updates(inid, src_dir=src_dir)
for k in git_update.keys():
meta[k] = git_update[k]
meta['page_content'] = ''.join(meta_md[1])
return meta
def update_meta_data(index, row):
meta = build_meta_data(index, row)
if not row['New']:
fname = os.path.join("_indicators", row['File'])
md_data = yamlmd.read_yamlmd(fname)
else:
md_data = [
{"graph_type": "line",
"graph_title": None,
"reporting_status": "notstarted"},
['\n', 'We are still looking for a suitable data source for this indicator. Please contact us if you think you can help by emailing <a href="mailto:[email protected]">[email protected]</a>.']
]
md_meta = merge_meta_data(md_data[0], meta)
return [md_meta, md_data[1]]
cols = ['Year'] + cols + ['Value']
df = df[cols]
df.to_csv(data_path, index=False)
# Fix the special "graph_titles" metadata field we added above.
graph_titles = []
for series in metadata[indicator_id]['graph_titles']:
graph_titles.append({
'series': 'SERIES.' + series,
'title': 'SERIES.' + series,
})
metadata[indicator_id]['graph_titles'] = graph_titles
meta_path = os.path.join('meta', slug + '.md')
meta = yamlmd.read_yamlmd(meta_path)
for field in metadata[indicator_id]:
meta[0][field] = metadata[indicator_id][field]
yamlmd.write_yamlmd(meta, meta_path)
skip_translations = drop_these_columns()
for key in things_to_translate:
if key in skip_translations:
continue
if key not in english:
english[key] = things_to_translate[key]
else:
for item in things_to_translate[key]:
if item not in english[key]:
english[key][item] = things_to_translate[key][item]
return True
# Load the Russian translations to decide whether each indicator has global/national names.
with open(os.path.join('translations', 'ru', 'global_indicators.yml'),
'r',
encoding='utf-8') as stream:
global_indicators = yaml.load(stream, Loader=yaml.FullLoader)
with open(os.path.join('translations', 'ru', 'national_indicators.yml'),
'r',
encoding='utf-8') as stream:
national_indicators = yaml.load(stream, Loader=yaml.FullLoader)
for filename in os.listdir('meta'):
filepath = os.path.join('meta', filename)
meta = yamlmd.read_yamlmd(filepath)
indicator_id = os.path.splitext(filename)[0]
translation_key = indicator_id + '-title'
translation_key_global = 'global_indicators.' + translation_key
translation_key_national = 'national_indicators.' + translation_key
has_global_title = translation_key in global_indicators
has_national_title = translation_key in national_indicators
if has_national_title and not has_global_title:
if translation_is_global(meta[0]['indicator_name']):
# In some cases the national indicators use a global indicator name.
has_global_title = True
if not has_global_title:
indicator_id_parts = indicator_id.split('-')
def read_meta_at_path(self, filepath):
meta_md = yamlmd.read_yamlmd(filepath)
meta = dict(meta_md[0])
meta['page_content'] = ''.join(meta_md[1])
return meta
@author: dashton
"""
import glob
import yamlmd
import pandas as pd
# if you don't have yamlmd do
# pip install git+git://github.com/dougmet/yamlmd
goals = glob.glob('_goals/*.md')
goals_df = pd.read_csv('data/sdg_goals.csv')
for goal in goals:
ymd = yamlmd.read_yamlmd(goal)
ymd_head = dict(ymd[0])
csv_info = goals_df[goals_df['goal'] == ymd_head['sdg_goal']].to_dict(
'records')[0]
out_header = {
'title': csv_info['title'],
'short': csv_info['short'],
'sdg_goal': str(csv_info['goal']),
'color': csv_info['colorInfo/hex']
}
out_ymd = [out_header, ymd[1]]
yamlmd.write_yamlmd(out_ymd, goal)
# -*- coding: utf-8 -*-
"""
Created on Wed May 30 15:47:21 2018
@author: dashton
"""
import glob
import yamlmd
# if you don't have yamlmd do
# pip install git+git://github.com/dougmet/yamlmd
page_only_vars = ['layout', 'permalink', 'indicator']
metas = glob.glob('_indicators/*.md')
for meta in metas:
ymd = yamlmd.read_yamlmd(meta)
ymd_head = dict(ymd[0])
out_header = {k: v for (k, v) in ymd_head.items() if k in page_only_vars}
out_ymd = [out_header, ymd[1]]
yamlmd.write_yamlmd(out_ymd, meta)
