def create_playground_schemas():
dialect = helpers.read_config('db_dialect')['dialect']
if dialect != 'snowflake':
pass
else:
connection_params = helpers.read_config('metadata')
engine = create_engine(
'snowflake://{user}:{password}@{account}/{database}?warehouse={warehouse}'
.format(user=connection_params['user'],
password=connection_params['password'],
account=connection_params['account'],
database=connection_params['database'],
warehouse=connection_params['warehouse']))
metadata_schema = connection_params['schema']
rawvault_schema = helpers.read_config('rawvault')['schema']
staging_schema = helpers.read_config('staging')['schema']
for playground_schema in [
metadata_schema, rawvault_schema, staging_schema
]:
try:
engine.execute(schema.CreateSchema(playground_schema))
except exc.ProgrammingError:
pass
engine.dispose()
def test_config_read():
sample_config = helpers.read_config('db_dialect',
config_ini="config.ini.sample")
assert sample_config['dialect'] == 'sqlite'
sample_config = helpers.read_config('metadata',
config_ini="config.ini.sample")
assert sample_config == {
"user": '******',
"password": '******',
"account": 'XXX.xxxregion',
"database": 'test_db',
"schema": 'DV_MTD',
"warehouse": 'xxx'
}
def fill_metadata_schemas(eng):
connection = eng.connect()
metadata = MetaData()
schemas = Table('SCHEMAS', metadata, autoload=True, autoload_with=eng)
insp = reflection.Inspector.from_engine(eng)
connection.execute(schemas.delete())
for role in ['metadata', 'source', 'staging', 'rawvault']:
conn_conf = helpers.read_config(role)
if eng.dialect.name == 'sqlite':
val = {
'DATABASE': 'None',
'SCHEMA': conn_conf['schema'],
'ROLE': role
}
if eng.dialect.name == 'snowflake':
val = {
'DATABASE': conn_conf['database'],
'SCHEMA': conn_conf['schema'],
'ROLE': role
}
connection.execute(schemas.insert().values(val))
connection.execute(hub_business_keys.insert().values(
[hub_business_keys_values]))
def insert_link_mappings(eng):
metadata = MetaData()
connection = eng.connect()
link_mappings = Table('LINK_MAPPINGS',
metadata,
autoload=True,
autoload_with=eng)
connection.execute(link_mappings.delete())
with open(link_mapping_csv, 'r') as ifile:
link_mapping_values = {}
rows = DictReader(ifile)
for row in rows:
connection.execute(link_mappings.insert().values(row))
if __name__ == "__main__":
dialect = helpers.read_config('db_dialect')['dialect']
if dialect == 'snowflake':
engine_target = helpers.engine_snowflake('metadata')
else:
engine_target = helpers.engine_sqlite('metadata')
insert_hub_mappings(engine_target)
insert_link_mappings(engine_target)
engine_target.dispose()