def build_files(df, meta_function, meta_map, gen_2_dir, prefix, suffix): filelist = [] countrylist = [] for iso3 in statmart_utils.get_index_set(df): try: idf = df.ix[iso3] if type(idf) == pd.Series: #idf a Series if there is only one element in it, but we want a DataFrame always idf = pd.DataFrame([idf]) idf = idf[["Year","Value","Source","Notes"]] idf.columns = ["year","value","source","notes"] idf["source"] = idf["source"].apply(lambda x : meta_map["source"]) idf["notes"] = idf["notes"].apply(lambda x : get_notes(str(x), meta_map)) filestem = prefix + "_" + iso3.lower() + "_" + suffix filename = filestem + ".csv" filepath = gen_2_dir + filename idf.to_csv(filepath, encoding="utf8", index=False) meta = meta_function(iso3, meta_map) meta.append(("file", filename)) meta.append(("filehash", statmart_utils.githash(filepath))) meta.append(("columns", "year,value,source,notes")) metafile = gen_2_dir + filestem + "_meta.csv" pd.DataFrame(meta,columns = ["key","value"]).to_csv(metafile, encoding="utf8", index=False) filelist.append([filestem]) countrylist.append(statmart_utils.get_country_by_iso3(iso3)) except Exception as strerror: print ("ERROR: Failed to build data for %s" % iso3) print (strerror) fldf = pd.DataFrame(filelist, index=countrylist).sort_index() fldf.to_csv(gen_2_dir + "_" + prefix + ".csv", encoding="utf8", float_format='%.1f', index=False, header=False) return fldf
def get_metadata(iso3, meta_map): country = statmart_utils.get_country_by_iso3(iso3) return [("name", "%s - Contraceptive use [CEPALStat]" % (country)), ("originalsource", meta_map["source"]), ("proximatesource", "CEPALStat"), ("dataset", meta_map["indicator"] + " [" + indicator_id + "]"), ("description", meta_map["definition"]), ("category", "Social"), ("type", "Health")]
def get_metadata(iso3, meta_map): country = statmart_utils.get_country_by_iso3(iso3) return [("name", "%s - %s [CEPALStat]" % (country, indicator_name)), ("originalsource", meta_map["source"]), ("proximatesource", "CEPALStat"), ("dataset", meta_map["indicator"] + " [" + indicator_id + "]"), ("description", meta_map["definition"]), ("category", indicator_category), ("type", indicator_type)]
def get_metadata(iso3, meta_map): country = statmart_utils.get_country_by_iso3(iso3) return [("name", "%s - %s [CEPALStat]" % (country, indicator_name)), ("originalsource", meta_map["source"]), ("proximatesource", "CEPALStat"), ("dataset", meta_map["indicator"] + " [" + indicator_id + "]"), ("description", meta_map["definition"]), ("category", indicator_category), ("type", indicator_type) ]
def get_metadata(iso3, meta_map): country = statmart_utils.get_country_by_iso3(iso3) return [ ("name", "%s - Contraceptive use [CEPALStat]" % (country)), ("originalsource", meta_map["source"]), ("proximatesource", "CEPALStat"), ("dataset", meta_map["indicator"] + " [" + indicator_id + "]"), ("description", meta_map["definition"]), ("category", "Social"), ("type", "Health"), ]
def build_files(df, meta_function, meta_map, gen_2_dir, prefix, suffix): filelist = [] countrylist = [] for iso3 in statmart_utils.get_index_set(df): try: idf = df.ix[iso3] if type( idf ) == pd.Series: #idf a Series if there is only one element in it, but we want a DataFrame always idf = pd.DataFrame([idf]) idf = idf[["Year", "Value", "Source", "Notes"]] idf.columns = ["year", "value", "source", "notes"] idf["source"] = idf["source"].apply(lambda x: meta_map["source"]) idf["notes"] = idf["notes"].apply( lambda x: get_notes(str(x), meta_map)) filestem = prefix + "_" + iso3.lower() + "_" + suffix filename = filestem + ".csv" filepath = gen_2_dir + filename idf.to_csv(filepath, encoding="utf8", index=False) meta = meta_function(iso3, meta_map) meta.append(("file", filename)) meta.append(("filehash", statmart_utils.githash(filepath))) meta.append(("columns", "year,value,source,notes")) metafile = gen_2_dir + filestem + "_meta.csv" pd.DataFrame(meta, columns=["key", "value"]).to_csv(metafile, encoding="utf8", index=False) filelist.append([filestem]) countrylist.append(statmart_utils.get_country_by_iso3(iso3)) except Exception as strerror: print("ERROR: Failed to build data for %s" % iso3) print(strerror) fldf = pd.DataFrame(filelist, index=countrylist).sort_index() fldf.to_csv(gen_2_dir + "_" + prefix + ".csv", encoding="utf8", float_format='%.1f', index=False, header=False) return fldf