]] df = pd.concat([df, missing]) df = df[~df.continent_code.isnull()].reset_index(drop=True) df["level"] = "country" df = df.rename( columns={ "code_col": "code", "name_es_col": "name_es", "continent_code": "parent_code", }) assert df.loc[6, "name"] is pd.np.nan df.loc[6, "name"] = u"Netherlands Antilles" regions = pd.read_table( "../Mexico/in/Mexico Country codes - continents - Continents - Regions.tsv", encoding="utf-8", ) df = pd.concat([df, regions]).reset_index(drop=True) h = Hierarchy(["region", "country"]) parent_id_table = parent_code_table_to_parent_id_table(df, h) c = Classification(parent_id_table, h) c.to_csv("out/locations_international_dane.csv") # c.to_stata("out/locations_international_dane.dta")
) if __name__ == "__main__": names = pd.read_table("./in/AgProducts_Names.tsv", encoding="utf-8", dtype={"code": str}) hierarchy = pd.read_table("./in/AgProducts_Hierarchy.tsv", encoding="utf-8") hierarchy.columns = [ "level3_code", "level2_code", "level1_code", "level0_code" ] fields = {"level0": [], "level1": [], "level2": [], "level3": []} h = Hierarchy(["level0", "level1", "level2", "level3"]) parent_code_table = repeated_table_to_parent_id_table(hierarchy, h, fields) parent_code_table.code = parent_code_table.code.astype(str) parent_code_table = parent_code_table.merge(names, on=["code", "level"]) parent_id_table = parent_code_table_to_parent_id_table( parent_code_table, h) parent_id_table["name"] = parent_id_table.name_en parent_id_table = parent_id_table[[ "code", "name", "level", "name_en", "name_es", "parent_id" ]] c = Classification(parent_id_table, h) c.table.code = c.table.code.str.lower()
four_digit["parent_code"] = four_digit.code.apply(lambda x: x[:2]) four_digit = four_digit.drop("community", axis=1) four_digit["level"] = "4digit" two_digit = hs4.iloc[1241:1339] two_digit["code"] = two_digit.code.astype(str).str.zfill(2) two_digit = two_digit.rename(columns={"community": "parent_code"}) two_digit["parent_code"] = two_digit.parent_code.astype(str).str.zfill(3) two_digit["level"] = "2digit" section = hs4.iloc[1339:].drop("community", axis=1) section["code"] = section.code.astype(str).str.zfill(3) section["parent_code"] = None section["level"] = "section" hs_clean = pd.concat([section, two_digit, four_digit]) hs_clean = hs_clean.reset_index(drop=True) h = Hierarchy(["section", "2digit", "4digit"]) hs_clean = parent_code_table_to_parent_id_table(hs_clean, h) c = Classification(hs_clean, h) # community = pd.read_table("in/hs4_community.tsv", encoding="utf-8") # hs4 = hs4.merge(community, left_on="community", right_on="code", how="inner") # weird bug where pandas infer_type was returning mixed instead of string c.table.code = c.table.code.astype(str) c.to_csv("out/hs92_atlas.csv") c.to_stata("out/hs92_atlas.dta")
df = pd.read_table("in/DIVIPOLA_20150331.txt", encoding="utf-16") df.columns = ["department_code", "municipality_code", "population_center_code", "department_name", "municipality_name", "population_center_name", "population_center_type", "longitude", "", "latitude", "district", "municipality_type", "metro_area"] df = df[["department_code", "department_name", "municipality_code", "municipality_name", "population_center_code", "population_center_name"]] df.department_code = df.department_code.astype(str).str.zfill(2) df.municipality_code = df.municipality_code.astype(str).str.zfill(5) df.population_center_code = df.population_center_code.astype(str).str.zfill(8) df.department_name = df.department_name.str.title() df.municipality_name = df.municipality_name.str.title() df.population_center_name = df.population_center_name.str.title() h = Hierarchy(["department", "municipality", "population_center"]) parent_code_table = repeated_table_to_parent_id_table(df, h) parent_id_table = parent_code_table_to_parent_id_table(parent_code_table, h) c = Classification(parent_id_table, h) c.to_csv("out/locations_colombia_dane.csv") c.to_stata("out/locations_colombia_dane.dta")
if __name__ == "__main__": assert (len(sys.argv) == 3) file_name = sys.argv[1] new_file_prefix = sys.argv[2] df = pd.read_table(file_name, encoding="utf-16") df = parse_dane(df) df = df[~df.duplicated(["code"])] df = df.reset_index(drop=True) df.columns = ["name", "level", "code"] df.name = df.name.str.title() from classification import (parent_code_table_to_parent_id_table, Classification, Hierarchy, ordered_table_to_parent_code_table) h = Hierarchy(DANE_HIERARCHY) df = ordered_table_to_parent_code_table(df, h) df = parent_code_table_to_parent_id_table(df, h) c = Classification(df, h) # weird bug where pandas infer_type was returning mixed instead of string c.table.code = c.table.code.astype(str) c.to_csv(new_file_prefix + ".csv") c.to_stata(new_file_prefix + ".dta")
from classification import ( Hierarchy, repeated_table_to_parent_id_table, parent_code_table_to_parent_id_table, spread_out_entries, sort_by_code_and_level, Classification, ) if __name__ == "__main__": df = pd.read_csv("./in/NACE_Rev2_custom_hierarchy.csv") df.columns = ["level", "code", "parent_code", "name"] df.level = df.level.astype(str) df.loc[df.level == "1", "level"] = "section" df.loc[df.level == "2", "level"] = "division" df.loc[df.level == "3", "level"] = "group" h = Hierarchy(["section", "division", "group"]) df = sort_by_code_and_level(df, h) df = parent_code_table_to_parent_id_table(df, h) level_starts = {"section": 0, "division": 100, "group": 300} df = spread_out_entries(df, level_starts, h) c = Classification(df, h) c.to_csv("./out/nace_industries.csv")
Classification) if __name__ == "__main__": sinco = pd.read_csv("in/SINCO_2011.csv", header=None, encoding="latin-1") sinco.columns = ["data"] sinco = sinco[~sinco.data.str.startswith("INEGI.")] sinco = sinco[~sinco.data.str.startswith(u"Clave DescripciĆ³n")] for index, row in reversed( list(sinco[~sinco.data.str.match("^\d* ")].iterrows())): sinco.ix[index - 1] += (" " + sinco.ix[index]) sinco = sinco[sinco.data.str.match("^\d* ")] sinco = sinco.data.str.split(" ", 1).apply(pd.Series, 1) sinco.columns = ["code", "name"] sinco["level"] = sinco["code"].apply(lambda x: str(len(x)) + "digit") h = Hierarchy(["1digit", "2digit", "3digit", "4digit"]) parent_code_table = ordered_table_to_parent_code_table(sinco, h) parent_id_table = parent_code_table_to_parent_id_table( parent_code_table, h) c = Classification(parent_id_table, h) c.to_csv("out/occupations_sinco_2011.csv") c.to_stata("out/occupations_sinco_2011.dta")
repeated_table_to_parent_id_table, parent_code_table_to_parent_id_table, Classification, ) if __name__ == "__main__": names = pd.read_table("./in/Livestock_Names.tsv", encoding="utf-8", dtype={"code": str}) hierarchy = pd.read_table("./in/Livestock_Hierarchy.tsv", encoding="utf-8") hierarchy.columns = ["level1_code", "level0_code"] fields = {"level0": [], "level1": []} h = Hierarchy(["level0", "level1"]) parent_code_table = repeated_table_to_parent_id_table(hierarchy, h, fields) parent_code_table.code = parent_code_table.code.astype(str) parent_code_table = parent_code_table.merge(names, on=["code", "level"]) parent_id_table = parent_code_table_to_parent_id_table( parent_code_table, h) parent_id_table["name"] = parent_id_table.name_en parent_id_table = parent_id_table[[ "code", "name", "level", "name_en", "name_es",
"city_name", "city_code" ]] metro_areas.columns = ["name", "code"] metro_areas["parent_code"] = metro_areas.code.str.slice(0, 2) metro_areas["level"] = "msa" df.loc[df.level == "department", "parent_code"] = "COL" df = pd.concat([pd.DataFrame(colombia).T, df, metro_areas]) df = df.sort(["level", "code"], ascending=True) df = df.reset_index(drop=True) h = Hierarchy(["country", "department", "msa", "municipality"]) parent_id_table = parent_code_table_to_parent_id_table(df, h) parent_id_table["name_es"] = parent_id_table.name parent_id_table["name_short_en"] = parent_id_table.name parent_id_table["name_short_es"] = parent_id_table.name # Work around issue where parent_code_table_to_parent_id_table breaks # because the parent of munis are not msas depts = df[df.level == "department"] depts = depts[["code"]].reset_index().set_index("code") lookup_table = depts.to_dict()["index"] def fill_parents(row): if row.level == "municipality" and pd.isnull(row.parent_id): row.parent_id = lookup_table[row.code[:2]] return row
df.municipality_code = df.state_code + df.municipality_code df.locality_code = df.municipality_code + df.locality_code df.state_name = df.state_name.str.title() df.municipality_name = df.municipality_name.str.title() df.locality_name = df.locality_name.str.title() df = df.rename( columns={ "state_name": "name_en_state", "municipality_name": "name_en_municipality", "locality_name": "name_en_locality", }) h = Hierarchy(["state", "municipality", "locality"]) parent_code_table = repeated_table_to_parent_id_table( df, h, level_fields={ "state": ["name_en_state"], "municipality": ["name_en_municipality"], "locality": ["name_en_locality"], }, ) # TODO: This isn't the official classification level name but this makes # compatibility between colombia and mexico way easier parent_code_table.loc[parent_code_table.level == "state", "level"] = "department"
"latitude", "longitude", "altitude", "map_code", "ambito", "population_total", "population_male", "population_female", "dwellings_occupied"] df = df[["state_code", "state_name", "municipality_code", "municipality_name", "locality_code", "locality_name"]] df.state_code = df.state_code.astype(str).str.zfill(2) df.municipality_code = df.municipality_code.astype(str).str.zfill(3) df.locality_code = df.locality_code.astype(str).str.zfill(4) df.municipality_code = df.state_code + df.municipality_code df.locality_code = df.municipality_code + df.locality_code df.state_name = df.state_name.str.title() df.municipality_name = df.municipality_name.str.title() df.locality_name = df.locality_name.str.title() h = Hierarchy(["state", "municipality", "locality"]) parent_code_table = repeated_table_to_parent_id_table(df, h) parent_id_table = parent_code_table_to_parent_id_table(parent_code_table, h) c = Classification(parent_id_table, h) c.to_csv("out/locations_mexico_inegi.csv") c.to_stata("out/locations_mexico_inegi.dta")
selected_rows = df.name_english.str.contains(regex) df["tag_en_" + name] = False df.loc[selected_rows, "tag_en_" + name] = True df.name_english = df.name_english.map(lambda x: re.sub(regex, "", x)) for name, regex in regexes.items(): selected_rows = df.name_spanish.str.contains(regex) df["tag_sp_" + name] = False df.loc[selected_rows, "tag_sp_" + name] = True df.name_spanish = df.name_spanish.map(lambda x: re.sub(regex, "", x)) # Replace trailing comma and space df.name_spanish = df.name_spanish.str.replace(", $", "") df.name_english = df.name_english.str.replace(", $", "") h = Hierarchy( ["twodigit", "threedigit", "fourdigit", "fivedigit", "sixdigit"]) df.loc[df.code.str.len() == 2, "level"] = "twodigit" df.loc[df.code.str.len() == 3, "level"] = "threedigit" df.loc[df.code.str.len() == 4, "level"] = "fourdigit" df.loc[df.code.str.len() == 5, "level"] = "fivedigit" df.loc[df.code.str.len() == 6, "level"] = "sixdigit" spanish = df[["code", "level", "name_spanish"]] spanish.columns = ["code", "level", "name_es"] # make sure this is the hand-fixed version assert df.loc[304, "code"] == '31' df = df[["code", "name_english", "level"]] df.columns = ["code", "name", "level"]
row["level"] = "country" row["parent_code"] = pd.np.nan elif row.code.endswith("0000"): row["level"] = "department" row["parent_code"] = "000000" elif row.code.endswith("00"): row["level"] = "province" row["parent_code"] = row["code"][:2] + "0000" else: row["level"] = "district" row["parent_code"] = row["code"][:4] + "00" return row df = df.apply(fix_levels, axis=1) h = Hierarchy(["country", "department", "province", "district"]) df.level = df.level.astype("category", categories=h, ordered=True) df = df.sort_values(by=["level", "code"]) df.level = df.level.astype(str) df = df.reset_index(drop=True) parent_id_table = parent_code_table_to_parent_id_table(df, h) # TODO: This isn't the official classification level name but this makes # compatibility between colombia and mexico way easier # parent_code_table.loc[parent_code_table.level == "state", "level"] = "department" # Drop the "locality" level since we don't use it # parent_code_table = parent_code_table[parent_code_table.level != "locality"]
import pandas as pd from classification import ( Hierarchy, repeated_table_to_parent_id_table, parent_code_table_to_parent_id_table, Classification, ) if __name__ == "__main__": df = pd.read_table("in/Col_occupations_SOC_2010 - Hierarchy.tsv", encoding="utf-8") h = Hierarchy([ "major_group", "minor_group", "broad_occupation", "detailed_occupation" ]) fields = { "major_group": ["name_en_major_group", "name_es_major_group"], "minor_group": [ "name_en_minor_group", "name_es_minor_group", "name_short_es_minor_group", ], "broad_occupation": ["name_en_broad_occupation", "name_es_broad_occupation"], "detailed_occupation": [ "name_en_detailed_occupation", "name_es_detailed_occupation", "name_short_es_detailed_occupation", ],
return x trans = trans.apply(fill_code, axis=1) # Prospedia specific trans = trans[trans.level != "section"] df = pd.read_table("./in/prospedia_hs_structure.txt") df.columns = ["4digit_code", "2digit_code", "prospedia_section_code"] df["4digit_code"] = df["4digit_code"].astype(str).str.zfill(4) df["4digit_name"] = None df["2digit_code"] = df["2digit_code"].astype(str).str.zfill(2) df["2digit_name"] = None df["prospedia_section_name"] = None df["prospedia_section_code"] = df["prospedia_section_code"].astype(str).str.zfill(1) h = Hierarchy(["prospedia_section", "2digit", "4digit"]) parent_code_table = repeated_table_to_parent_id_table(df, h) parent_id_table = parent_code_table_to_parent_id_table(parent_code_table, h) parent_id_table = parent_id_table.merge(trans, on=["level", "code"]) parent_id_table.name = parent_id_table.name_en assert parent_id_table.name.isnull().sum() == 3 parent_id_table.loc[parent_id_table.name.isnull(), "name"] = u"No name" assert parent_id_table.name.isnull().sum() == 0 c = Classification(parent_id_table, h) c.to_csv("out/products_mexico_prospedia.csv") c.to_stata("out/products_mexico_prospedia.dta")
"./in/col_industry_name_category_master - Hierarchy.tsv", encoding="utf-8") df.class_code = df.class_code.astype(int).astype(str).str.zfill(4) df.division_code = df.division_code.astype(int).astype(str).str.zfill(2) df.section_code = df.section_code.astype(int).astype(str).str.zfill(1) names = pd.read_table("./in/col_industry_name_category_master - Names.tsv", encoding="utf-8") names.loc[names.level == "section", "code"] = names.code.astype(str) names.loc[names.level == "division", "code"] = names.code.astype(str).str.zfill(2) names.loc[names.level == "class", "code"] = names.code.astype(str).str.zfill(4) h = Hierarchy(["section", "division", "class"]) parent_code_table = (repeated_table_to_parent_id_table( df, h, level_fields={ "section": [], "division": [], "class": [] }).sort_values(by=["level", "code"]).reset_index(drop=True)) parent_id_table = parent_code_table_to_parent_id_table( parent_code_table, h) parent_id_table = parent_id_table.merge(names) parent_id_table["name"] = parent_id_table.name_en c = Classification(parent_id_table, h)