def read_meta(inid, git=True, src_dir='', git_data_dir=None): """Perform pre-processing for the metadata files""" status = True # Read and write paths may be different fr = input_path(inid, ftype='meta', src_dir=src_dir) meta_md = yamlmd.read_yamlmd(fr) meta = dict(meta_md[0]) if git: git_update = sdg.git.get_git_updates(inid, src_dir=src_dir, git_data_dir=git_data_dir) for k in git_update.keys(): meta[k] = git_update[k] meta['page_content'] = ''.join(meta_md[1]) # Now look for all subfolders of the meta folder, which may contain # multilingual metadata, and add them as well. meta_folder = input_path(None, ftype='meta', src_dir=src_dir) languages = next(os.walk(meta_folder))[1] for language in languages: i18n_fr = os.path.join(meta_folder, language, inid + '.md') if os.path.isfile(i18n_fr): i18n_meta_md = yamlmd.read_yamlmd(i18n_fr) i18n_meta = dict(i18n_meta_md[0]) meta[language] = i18n_meta meta[language]['page_content'] = ''.join(i18n_meta_md[1]) return meta
def test_read(): """Check that the main read function works""" lorum_file = os.path.join(base_path, "data", "lorum.md") lorum = read_yamlmd(lorum_file) assert len(lorum) == 2 assert isinstance(lorum[0], ruamel.yaml.comments.CommentedMap) assert isinstance(lorum[1], list)
def main(): """Add pdf info to all indicator metadata""" metas = glob.glob('_indicators/*.md') print('Processing ' + str(len(metas)) + ' metadata files...') for meta in metas: try: md = yamlmd.read_yamlmd(meta) md = add_goal_pdf_info(md, meta) yamlmd.write_yamlmd(md, meta) except Exception as e: print(meta, e)
def build_meta(inid): """Perform pre-processing for the metadata files""" status = True # Read and write paths may be different fr = indicator_path(inid, ftype='meta', mode='r') fw = indicator_path(inid, ftype='meta', mode='w') meta = yamlmd.read_yamlmd(fr) git_update = sdg.git.get_git_updates(inid) for k in git_update.keys(): meta[0][k] = git_update[k] yamlmd.write_yamlmd(meta, fw) return status
def read_meta(inid, git=True, src_dir=''): """Perform pre-processing for the metadata files""" status = True # Read and write paths may be different fr = input_path(inid, ftype='meta', src_dir=src_dir) meta_md = yamlmd.read_yamlmd(fr) meta = dict(meta_md[0]) if git: git_update = sdg.git.get_git_updates(inid, src_dir=src_dir) for k in git_update.keys(): meta[k] = git_update[k] meta['page_content'] = ''.join(meta_md[1]) return meta
def update_meta_data(index, row): meta = build_meta_data(index, row) if not row['New']: fname = os.path.join("_indicators", row['File']) md_data = yamlmd.read_yamlmd(fname) else: md_data = [ {"graph_type": "line", "graph_title": None, "reporting_status": "notstarted"}, ['\n', 'We are still looking for a suitable data source for this indicator. Please contact us if you think you can help by emailing <a href="mailto:[email protected]">[email protected]</a>.'] ] md_meta = merge_meta_data(md_data[0], meta) return [md_meta, md_data[1]]
cols = ['Year'] + cols + ['Value'] df = df[cols] df.to_csv(data_path, index=False) # Fix the special "graph_titles" metadata field we added above. graph_titles = [] for series in metadata[indicator_id]['graph_titles']: graph_titles.append({ 'series': 'SERIES.' + series, 'title': 'SERIES.' + series, }) metadata[indicator_id]['graph_titles'] = graph_titles meta_path = os.path.join('meta', slug + '.md') meta = yamlmd.read_yamlmd(meta_path) for field in metadata[indicator_id]: meta[0][field] = metadata[indicator_id][field] yamlmd.write_yamlmd(meta, meta_path) skip_translations = drop_these_columns() for key in things_to_translate: if key in skip_translations: continue if key not in english: english[key] = things_to_translate[key] else: for item in things_to_translate[key]: if item not in english[key]: english[key][item] = things_to_translate[key][item]
return True # Load the Russian translations to decide whether each indicator has global/national names. with open(os.path.join('translations', 'ru', 'global_indicators.yml'), 'r', encoding='utf-8') as stream: global_indicators = yaml.load(stream, Loader=yaml.FullLoader) with open(os.path.join('translations', 'ru', 'national_indicators.yml'), 'r', encoding='utf-8') as stream: national_indicators = yaml.load(stream, Loader=yaml.FullLoader) for filename in os.listdir('meta'): filepath = os.path.join('meta', filename) meta = yamlmd.read_yamlmd(filepath) indicator_id = os.path.splitext(filename)[0] translation_key = indicator_id + '-title' translation_key_global = 'global_indicators.' + translation_key translation_key_national = 'national_indicators.' + translation_key has_global_title = translation_key in global_indicators has_national_title = translation_key in national_indicators if has_national_title and not has_global_title: if translation_is_global(meta[0]['indicator_name']): # In some cases the national indicators use a global indicator name. has_global_title = True if not has_global_title: indicator_id_parts = indicator_id.split('-')
def read_meta_at_path(self, filepath): meta_md = yamlmd.read_yamlmd(filepath) meta = dict(meta_md[0]) meta['page_content'] = ''.join(meta_md[1]) return meta
@author: dashton """ import glob import yamlmd import pandas as pd # if you don't have yamlmd do # pip install git+git://github.com/dougmet/yamlmd goals = glob.glob('_goals/*.md') goals_df = pd.read_csv('data/sdg_goals.csv') for goal in goals: ymd = yamlmd.read_yamlmd(goal) ymd_head = dict(ymd[0]) csv_info = goals_df[goals_df['goal'] == ymd_head['sdg_goal']].to_dict( 'records')[0] out_header = { 'title': csv_info['title'], 'short': csv_info['short'], 'sdg_goal': str(csv_info['goal']), 'color': csv_info['colorInfo/hex'] } out_ymd = [out_header, ymd[1]] yamlmd.write_yamlmd(out_ymd, goal)
# -*- coding: utf-8 -*- """ Created on Wed May 30 15:47:21 2018 @author: dashton """ import glob import yamlmd # if you don't have yamlmd do # pip install git+git://github.com/dougmet/yamlmd page_only_vars = ['layout', 'permalink', 'indicator'] metas = glob.glob('_indicators/*.md') for meta in metas: ymd = yamlmd.read_yamlmd(meta) ymd_head = dict(ymd[0]) out_header = {k: v for (k, v) in ymd_head.items() if k in page_only_vars} out_ymd = [out_header, ymd[1]] yamlmd.write_yamlmd(out_ymd, meta)