def df_to_db(df, data_schema, provider, force=False):
if not df_is_not_null(df):
return
db_engine = get_db_engine(provider, data_schema=data_schema)
if force:
session = get_db_session(provider=provider, data_schema=data_schema)
ids = df["id"].tolist()
# count = len(ids)
# start = 0
# while True:
# end = min(count, start + 5000)
# sql = f'delete from {data_schema.__tablename__} where id in {tuple(ids[start:end])}'
# session.execute(sql)
# session.commit()
# if end == count:
# break
# start = end
sql = f'delete from {data_schema.__tablename__} where id in {tuple(ids)}'
session.execute(sql)
session.commit()
else:
current = get_data(data_schema=data_schema, columns=[data_schema.id], provider=provider)
if df_is_not_null(current):
df = df[~df['id'].isin(current['id'])]
df.to_sql(data_schema.__tablename__, db_engine, index=False, if_exists='append')
def df_to_db(df, data_schema, provider):
db_engine = get_db_engine(provider, data_schema=data_schema)
current = get_data(data_schema=data_schema, columns=[data_schema.id], provider=provider)
if df_is_not_null(current):
df = df[~df['id'].isin(current['id'])]
df.to_sql(data_schema.__tablename__, db_engine, index=False, if_exists='append')
def init_entities(df, entity_type='stock', provider='exchange'):
df = df.drop_duplicates(subset=['id'])
data_schema = get_entity_schema(entity_type)
store_category = get_db_name(data_schema=data_schema)
db_engine = get_db_engine(provider, db_name=store_category)
security_schema = get_entity_schema(entity_type)
current = get_entities(entity_type=entity_type, columns=[security_schema.id, security_schema.code],
provider=provider)
if df_is_not_null(current):
df = df[~df['id'].isin(current['id'])]
df.to_sql(security_schema.__tablename__, db_engine, index=False, if_exists='append')