def save_products_muni():
ret = process_dataset(trade4digit_municipality)
df = ret[('location_id', 'product_id', 'year')]
df = merge_classifications(df)
pci = process_dataset(trade4digit_country)[('product_id', 'year')][["pci"]].reset_index()
df = df.reset_index().merge(pci, on=["product_id", "year"], how="outer")
return df.set_index(['location_id', 'product_id', 'year'])
def save_products_muni():
ret = process_dataset(trade4digit_municipality)
df = ret[('location_id', 'product_id', 'year')]
df = merge_classifications(df)
pci = process_dataset(trade4digit_country)[('product_id', 'year')][["pci"]].reset_index()
df = df.reset_index().merge(pci, on=["product_id", "year"], how="outer")
return df.set_index(['location_id', 'product_id', 'year'])
def save_rcpy(rcpy_dataset):
ret = process_dataset(rcpy_dataset)
df = ret[("country_id", "location_id", "product_id", "year")]
df = merge_classifications(df)
pci = process_dataset(trade4digit_country)[('product_id', 'year')][["pci"]].reset_index()
df = df.reset_index().merge(pci, on=["product_id", "year"], how="outer")
return df.set_index(['country_id', 'location_id', 'product_id', 'year'])
def save_rcpy(rcpy_dataset):
ret = process_dataset(rcpy_dataset)
df = ret[("country_id", "location_id", "product_id", "year")]
df = merge_classifications(df)
pci = process_dataset(trade4digit_country)[('product_id', 'year')][["pci"]].reset_index()
df = df.reset_index().merge(pci, on=["product_id", "year"], how="outer")
return df.set_index(['country_id', 'location_id', 'product_id', 'year'])
def save_industries_municipality():
ret = process_dataset(industry4digit_municipality)
m = ret[('location_id', 'industry_id', 'year')]
m = merge_classifications(m)
return m
def save_occupations():
ret = process_dataset(occupation2digit_industry2digit)
m = ret[('occupation_id', 'industry_id')]
m = merge_classifications(m)
m["year"] = current_app.config["YEAR_MAX_DEMOGRAPHIC"]
return m.set_index("year")
def save_demographic():
ret = process_dataset(gdp_real_department)
gdp_real_df = ret[('location_id', 'year')]
ret = process_dataset(gdp_nominal_department)
gdp_nominal_df = ret[('location_id', 'year')]
gdp_df = gdp_real_df.join(gdp_nominal_df).reset_index()
ret = process_dataset(population)
pop_df = ret[('location_id', 'year')].reset_index()
m = gdp_df.merge(pop_df, on=["location_id", "year"], how="outer")
m = merge_classifications(m.set_index(['location_id', 'year']))
return m
def save_demographic():
ret = process_dataset(gdp_real_department)
gdp_real_df = ret[('location_id', 'year')]
ret = process_dataset(gdp_nominal_department)
gdp_nominal_df = ret[('location_id', 'year')]
gdp_df = gdp_real_df.join(gdp_nominal_df).reset_index()
ret = process_dataset(population)
pop_df = ret[('location_id', 'year')].reset_index()
m = gdp_df.merge(pop_df, on=["location_id", "year"], how="outer")
m = merge_classifications(m.set_index(['location_id', 'year']))
return m
def save_occupations():
ret = process_dataset(occupation2digit_industry2digit)
m = ret[('occupation_id', 'industry_id')]
m = merge_classifications(m)
m["year"] = 2014
return m.set_index("year")
def save_industries_municipality():
ret = process_dataset(industry4digit_municipality)
m = ret[('location_id', 'industry_id', 'year')]
m = merge_classifications(m)
return m
def save_occupations():
ret = process_dataset(occupation2digit_industry2digit)
m = ret[('occupation_id', 'industry_id')]
m = merge_classifications(m)
m["year"] = 2014
return m.set_index("year")
def save_industries_msa():
ret = process_dataset(industry4digit_msa)
dpy = ret[('location_id', 'industry_id', 'year')].reset_index()
py = ret[('industry_id', 'year')][["complexity"]].reset_index()
m = dpy.merge(py, on=["industry_id", "year"])
m = merge_classifications(m.set_index(['location_id', 'industry_id', 'year']))
return m
def save_industries_country():
ret = process_dataset(industry4digit_country)
dpy = ret[('location_id', 'industry_id', 'year')].reset_index()
py = ret[('industry_id', 'year')][["complexity"]].reset_index()
m = dpy.merge(py, on=["industry_id", "year"])
m = merge_classifications(m.set_index(['location_id', 'industry_id', 'year']))
return m
def save_industries_department():
ret = process_dataset(industry4digit_department)
dpy = ret[('location_id', 'industry_id', 'year')].reset_index()
py = ret[('industry_id', 'year')][["complexity"]].reset_index()
ly = ret[('location_id', 'year')][["industry_eci", "industry_coi"]].reset_index()
m = dpy.merge(py, on=["industry_id", "year"])
m = m.merge(ly, on=["location_id", "year"])
m = merge_classifications(m.set_index(['location_id', 'industry_id', 'year']))
return m
def save_rural(path, datasets, facet=None, prefix=""):
results = {}
for geolevel, dataset in datasets.items():
ret = process_dataset(dataset)
if facet is None:
facet = list(ret.keys())[0]
results[geolevel] = merge_classifications(ret[facet])
save(path, results[geolevel],
"{}{}".format(prefix, geolevel),
format="excel",
include_from_index=None,
)
return results
models.FarmType)
db.session.add_all(farmtype)
db.session.commit()
farmsize = classification_to_models(farmsize_classification,
models.FarmSize)
db.session.add_all(farmsize)
db.session.commit()
countries = classification_to_models(country_classification,
models.Country)
db.session.add_all(countries)
db.session.commit()
# Country product year
ret = process_dataset(trade4digit_country)
df = ret[('location_id', 'product_id', 'year')].reset_index()
df["level"] = "4digit"
df.to_sql("country_product_year",
db.engine,
index=False,
chunksize=10000,
if_exists="append")
# Department product year
ret = process_dataset(trade4digit_department)
df = ret[('product_id', 'year')].reset_index()
df["level"] = "4digit"
df.to_sql("product_year",
def save_products_country():
ret = process_dataset(trade4digit_country)
dpy = ret[('location_id', 'product_id', 'year')]
return merge_classifications(dpy)
def save_products_msa():
ret = process_dataset(trade4digit_msa)
m = region_product_year(ret)
m = merge_classifications(m)
return m
models.Industry)
db.session.add_all(industries)
db.session.commit()
occupations = classification_to_models(occupation_classification,
models.Occupation)
db.session.add_all(occupations)
db.session.commit()
countries = classification_to_models(country_classification,
models.Country)
db.session.add_all(countries)
db.session.commit()
# Country product year
ret = process_dataset(trade4digit_country)
df = ret[('location_id', 'product_id', 'year')].reset_index()
df["level"] = "4digit"
df.to_sql("country_product_year",
db.engine,
index=False,
chunksize=10000,
if_exists="append")
# Department product year
ret = process_dataset(trade4digit_department)
df = ret[('product_id', 'year')].reset_index()
df["level"] = "4digit"
df.to_sql("product_year",
def save_products_msa():
ret = process_dataset(trade4digit_msa)
m = region_product_year(ret)
m = merge_classifications(m)
return m
"gdp_real": first,
"gdp_nominal": first,
"gdp_pc_real": first,
"gdp_pc_nominal": first,
"population": first,
}
}
}
if __name__ == "__main__":
import dataset_tools
store = pd.HDFStore("data.h5", complib="blosc")
# Country Product Year
cpy = dataset_tools.process_dataset(trade4digit_country)
df = cpy[("location_id", "product_id", "year")].reset_index()
df.to_hdf(store, "country_product_year", format="table")
attrs = {
"sql_table_name": "country_product_year",
"location_level": "country",
"product_level": "4digit"
}
store.get_storer("/country_product_year").attrs.atlas_metadata = attrs
# Country Year
cy = cpy[("location_id", "year")].reset_index()
cy.to_hdf(store, "country_year", format="table")
attrs = {
db.session.add_all(industries)
db.session.commit()
occupations = classification_to_models(occupation_classification,
models.Occupation)
db.session.add_all(occupations)
db.session.commit()
countries = classification_to_models(country_classification,
models.Country)
db.session.add_all(countries)
db.session.commit()
# Country product year
ret = process_dataset(trade4digit_country)
df = ret[('location_id', 'product_id', 'year')].reset_index()
df["level"] = "4digit"
df.to_sql("country_product_year", db.engine, index=False,
chunksize=10000, if_exists="append")
# Department product year
ret = process_dataset(trade4digit_department)
df = ret[('product_id', 'year')].reset_index()
df["level"] = "4digit"
df.to_sql("product_year", db.engine, index=False,
chunksize=10000, if_exists="append")
df = ret[('location_id', 'product_id', 'year')].reset_index()
models.FarmType)
db.session.add_all(farmtype)
db.session.commit()
farmsize = classification_to_models(farmsize_classification,
models.FarmSize)
db.session.add_all(farmsize)
db.session.commit()
countries = classification_to_models(country_classification,
models.Country)
db.session.add_all(countries)
db.session.commit()
# Country product year
ret = process_dataset(trade4digit_country)
df = ret[('location_id', 'product_id', 'year')].reset_index()
df["level"] = "4digit"
df.to_sql("country_product_year", db.engine, index=False,
chunksize=10000, if_exists="append")
# Department product year
ret = process_dataset(trade4digit_department)
df = ret[('product_id', 'year')].reset_index()
df["level"] = "4digit"
df.to_sql("product_year", db.engine, index=False,
chunksize=10000, if_exists="append")
df = ret[('location_id', 'product_id', 'year')].reset_index()
