def upload_normalized_dfs(dfs: Dict[str, pandas.DataFrame], schema: str):
def set_ids(df, start_id):
df["id"] = range(start_id, len(df) + start_id)
def upload_table(table_name, df):
dtypes = {
str(column.name): column.type
for column in oep_tables[table_name].columns
}
df.to_sql(name=table_name,
con=db.engine,
schema=schema,
if_exists="append",
index=False,
dtype=dtypes)
db = setup_db_connection()
oep_tables = get_oep_tables(db)
# Upload scenario:
if len(dfs["oed_scenario"]) > 1:
raise IndexError("Scenarios can only be uploaded one by one")
scenario_id = get_next_id(db, oep_tables["oed_scenario"])
scenario = dfs["oed_scenario"]
set_ids(scenario, scenario_id)
upload_table("oed_scenario", scenario)
# Upload data:
next_id = get_next_id(db, oep_tables["oed_data"])
data = dfs["oed_data"]
data["scenario_id"] = scenario_id
set_ids(data, next_id)
upload_table("oed_data", data)
# Upload scalar:
scalar = dfs["oed_scalar"]
scalar["id"] = data[data["type"] == "scalar"]["id"]
upload_table("oed_scalar", scalar)
# Upload timeseries:
timeseries = dfs["oed_timeseries"]
timeseries["id"] = data[data["type"] == "timeseries"]["id"].reset_index(
drop=True)
upload_table("oed_timeseries", timeseries)
def test_next_id():
db = setup_db_connection()
tables = upload.get_oep_tables(db)
assert isinstance(upload.get_next_id(db, tables["oed_scenario"]), int)
from oem2orm import oep_oedialect_oem2orm as oem2orm
import os
import getpass
if __name__ == "__main__":
oem2orm.setup_logger()
os.environ["OEP_TOKEN"] = getpass.getpass('Token:')
db = oem2orm.setup_db_connection()
metadata_folder = oem2orm.select_oem_dir(oem_folder_name="metadata")
tables_orm = oem2orm.collect_tables_from_oem(db, metadata_folder)
oem2orm.create_tables(db, tables_orm)
def get_oep_tables(db=None):
db = db or setup_db_connection()
tables = collect_tables_from_oem(db, OEDATAMODEL_META_DIR)
return {table.name: table for table in tables}