def main(): """Add pdf info to all indicator metadata""" metas = glob.glob('_indicators/*.md') print('Processing ' + str(len(metas)) + ' metadata files...') for meta in metas: try: md = yamlmd.read_yamlmd(meta) md = add_goal_pdf_info(md, meta) yamlmd.write_yamlmd(md, meta) except Exception as e: print(meta, e)
def test_read(tmpdir): """Check that the main write function works""" test_header = ruamel.yaml.comments.CommentedMap({"a": 1}) test_md = ["\n", "the first thing\n"] tmp_file = tmpdir.mkdir("md").join("test.md").strpath write_yamlmd([test_header, test_md], tmp_file) with open(tmp_file) as f: tmp_lines = f.readlines() assert tmp_lines[0] == "---\n" assert len(tmp_lines) == 5
def build_meta(inid): """Perform pre-processing for the metadata files""" status = True # Read and write paths may be different fr = indicator_path(inid, ftype='meta', mode='r') fw = indicator_path(inid, ftype='meta', mode='w') meta = yamlmd.read_yamlmd(fr) git_update = sdg.git.get_git_updates(inid) for k in git_update.keys(): meta[0][k] = git_update[k] yamlmd.write_yamlmd(meta, fw) return status
df.to_csv(data_path, index=False) # Fix the special "graph_titles" metadata field we added above. graph_titles = [] for series in metadata[indicator_id]['graph_titles']: graph_titles.append({ 'series': 'SERIES.' + series, 'title': 'SERIES.' + series, }) metadata[indicator_id]['graph_titles'] = graph_titles meta_path = os.path.join('meta', slug + '.md') meta = yamlmd.read_yamlmd(meta_path) for field in metadata[indicator_id]: meta[0][field] = metadata[indicator_id][field] yamlmd.write_yamlmd(meta, meta_path) skip_translations = drop_these_columns() for key in things_to_translate: if key in skip_translations: continue if key not in english: english[key] = things_to_translate[key] else: for item in things_to_translate[key]: if item not in english[key]: english[key][item] = things_to_translate[key][item] skip_translations = [ 'BASE_PER', 'COMMENT_OBS',
"reporting_status": "notstarted" }, [ '\n', 'We are still looking for a suitable data source for this indicator. Please contact us if you think you can help by emailing <a href="mailto:[email protected]">[email protected]</a>.' ] ] md_meta = merge_meta_data(md_data[0], meta) return [md_meta, md_data[1]] # %% Loop over all indicators for index, row in dft.iterrows(): new_md = update_meta_data(index, row) new_file = os.path.join("_indicators", row['File']) print(row['File']) yamlmd.write_yamlmd(new_md, new_file) # %% Find the new files empty_data = pd.read_csv("data/indicator_1-1-1.csv") for index, row in dft.query('New').iterrows(): csv_file = ('indicator_' + row.Goal + '-' + row.Target + '-' + row.INDN + '.csv') csv_path = os.path.join('data', csv_file) if (not os.path.isfile(csv_path)): empty_data.to_csv(os.path.join("data", csv_file), index=False)
if has_national_title and not has_global_title: if translation_is_global(meta[0]['indicator_name']): # In some cases the national indicators use a global indicator name. has_global_title = True if not has_global_title: indicator_id_parts = indicator_id.split('-') if len(indicator_id_parts) == 3: has_global_title = True if has_global_title and has_national_title: meta[0]['indicator_name'] = translation_key_global meta[0]['indicator_available'] = translation_key_national meta[0]['graph_title'] = translation_key_national elif has_global_title: meta[0]['indicator_name'] = translation_key_global if 'indicator_available' in meta[0]: del meta[0]['indicator_available'] meta[0]['graph_title'] = translation_key_global elif has_national_title: meta[0]['indicator_name'] = translation_key_national if 'indicator_available' in meta[0]: del meta[0]['indicator_available'] meta[0]['graph_title'] = translation_key_national else: print('WARNING - indicator ' + indicator_id + ' has neither global nor national title.') yamlmd.write_yamlmd(meta, filepath)
@author: dashton """ import glob import yamlmd import pandas as pd # if you don't have yamlmd do # pip install git+git://github.com/dougmet/yamlmd goals = glob.glob('_goals/*.md') goals_df = pd.read_csv('data/sdg_goals.csv') for goal in goals: ymd = yamlmd.read_yamlmd(goal) ymd_head = dict(ymd[0]) csv_info = goals_df[goals_df['goal'] == ymd_head['sdg_goal']].to_dict( 'records')[0] out_header = { 'title': csv_info['title'], 'short': csv_info['short'], 'sdg_goal': str(csv_info['goal']), 'color': csv_info['colorInfo/hex'] } out_ymd = [out_header, ymd[1]] yamlmd.write_yamlmd(out_ymd, goal)
# -*- coding: utf-8 -*- """ Created on Wed May 30 15:47:21 2018 @author: dashton """ import glob import yamlmd # if you don't have yamlmd do # pip install git+git://github.com/dougmet/yamlmd page_only_vars = ['layout', 'permalink', 'indicator'] metas = glob.glob('_indicators/*.md') for meta in metas: ymd = yamlmd.read_yamlmd(meta) ymd_head = dict(ymd[0]) out_header = {k: v for (k, v) in ymd_head.items() if k in page_only_vars} out_ymd = [out_header, ymd[1]] yamlmd.write_yamlmd(out_ymd, meta)