"2digit_code", "section_code", ] services = pd.read_csv( "./in/Services_Hierarchy.csv", encoding="utf-8", dtype={"code": str} ) # Drop the 5-digit level. names = names[names.level != "5digit"] hierarchy = hierarchy.iloc[:, 1:].drop_duplicates() fields = {"section": [], "2digit": [], "3digit": [], "4digit": []} h = Hierarchy(["section", "2digit", "3digit", "4digit"]) parent_code_table = repeated_table_to_parent_id_table(hierarchy, h, fields) parent_code_table.code = parent_code_table.code.astype(str) parent_code_table = parent_code_table.merge(names, on=["code", "level"]) # Sort by level order (not necessarily alphabetical) parent_code_table = sort_by_code_and_level(parent_code_table, h) parent_id_table = parent_code_table_to_parent_id_table(parent_code_table, h) parent_id_table["name"] = parent_id_table.name_en parent_id_table = parent_id_table[ [ "code", "name", "level", "name_en",
df = pd.read_table("in/DIVIPOLA_20150331.txt", encoding="utf-16") df.columns = ["department_code", "municipality_code", "population_center_code", "department_name", "municipality_name", "population_center_name", "population_center_type", "longitude", "", "latitude", "district", "municipality_type", "metro_area"] df = df[["department_code", "department_name", "municipality_code", "municipality_name", "population_center_code", "population_center_name"]] df.department_code = df.department_code.astype(str).str.zfill(2) df.municipality_code = df.municipality_code.astype(str).str.zfill(5) df.population_center_code = df.population_center_code.astype(str).str.zfill(8) df.department_name = df.department_name.str.title() df.municipality_name = df.municipality_name.str.title() df.population_center_name = df.population_center_name.str.title() h = Hierarchy(["department", "municipality", "population_center"]) parent_code_table = repeated_table_to_parent_id_table(df, h) parent_id_table = parent_code_table_to_parent_id_table(parent_code_table, h) c = Classification(parent_id_table, h) c.to_csv("out/locations_colombia_dane.csv") c.to_stata("out/locations_colombia_dane.dta")
df.locality_name = df.locality_name.str.title() df = df.rename( columns={ "state_name": "name_en_state", "municipality_name": "name_en_municipality", "locality_name": "name_en_locality", }) h = Hierarchy(["state", "municipality", "locality"]) parent_code_table = repeated_table_to_parent_id_table( df, h, level_fields={ "state": ["name_en_state"], "municipality": ["name_en_municipality"], "locality": ["name_en_locality"], }, ) # TODO: This isn't the official classification level name but this makes # compatibility between colombia and mexico way easier parent_code_table.loc[parent_code_table.level == "state", "level"] = "department" # Drop the "locality" level since we don't use it parent_code_table = parent_code_table[ parent_code_table.level != "locality"] # This adds a highest level element that represents the whole country
fix_spanish_title_case, na_action="ignore") h = Hierarchy(["department", "municipality", "population_center"]) df = df.rename( columns={ "department_name": "name_department", "municipality_name": "name_municipality", "population_center_name": "name_population_center", }) parent_code_table = repeated_table_to_parent_id_table( df, h, level_fields={ "department": ["name_department"], "municipality": ["name_municipality"], "population_center": ["name_population_center"], }, ) parent_id_table = parent_code_table_to_parent_id_table( parent_code_table, h) # Reorder columns to keep diff clean parent_id_table = parent_id_table.ix[:, [ "code", "name", "level", "parent_id" ]] c = Classification(parent_id_table, h) c.to_csv("out/locations_colombia_dane.csv")
"name_short_es_minor_group", ], "broad_occupation": ["name_en_broad_occupation", "name_es_broad_occupation"], "detailed_occupation": [ "name_en_detailed_occupation", "name_es_detailed_occupation", "name_short_es_detailed_occupation", ], } # TODO: no short names for these df["name_short_es_broad_occupation"] = "" df["name_short_es_major_group"] = "" df = repeated_table_to_parent_id_table(df, h, fields) df["name_short_en"] = df["name_en"] df["name_short_es"] = df["name_short_es"].fillna(df.name_es) assert not df[df.level == "detailed_occupation"].code.str.endswith( "0").all() assert df[df.level == "broad_occupation"].code.str.endswith("0").all() assert df[df.level == "major_group"].code.str.endswith("000").all() assert df[df.level == "minor_group"].code.str.endswith("00").all() df["name"] = df.name_en df = parent_code_table_to_parent_id_table(df, h) c = Classification(df, h)
df.class_code = df.class_code.astype(int).astype(str).str.zfill(4) df.division_code = df.division_code.astype(int).astype(str).str.zfill(2) df.section_code = df.section_code.astype(int).astype(str).str.zfill(1) names = pd.read_table("./in/col_industry_name_category_master - Names.tsv", encoding="utf-8") names.loc[names.level == "section", "code"] = names.code.astype(str) names.loc[names.level == "division", "code"] = names.code.astype(str).str.zfill(2) names.loc[names.level == "class", "code"] = names.code.astype(str).str.zfill(4) h = Hierarchy(["section", "division", "class"]) parent_code_table = (repeated_table_to_parent_id_table( df, h, level_fields={ "section": [], "division": [], "class": [] }).sort_values(by=["level", "code"]).reset_index(drop=True)) parent_id_table = parent_code_table_to_parent_id_table( parent_code_table, h) parent_id_table = parent_id_table.merge(names) parent_id_table["name"] = parent_id_table.name_en c = Classification(parent_id_table, h) c.to_csv("out/industries_colombia_isic_prosperia.csv") c.to_stata("out/industries_colombia_isic_prosperia.dta")