def create_playground_schemas(): dialect = helpers.read_config('db_dialect')['dialect'] if dialect != 'snowflake': pass else: connection_params = helpers.read_config('metadata') engine = create_engine( 'snowflake://{user}:{password}@{account}/{database}?warehouse={warehouse}' .format(user=connection_params['user'], password=connection_params['password'], account=connection_params['account'], database=connection_params['database'], warehouse=connection_params['warehouse'])) metadata_schema = connection_params['schema'] rawvault_schema = helpers.read_config('rawvault')['schema'] staging_schema = helpers.read_config('staging')['schema'] for playground_schema in [ metadata_schema, rawvault_schema, staging_schema ]: try: engine.execute(schema.CreateSchema(playground_schema)) except exc.ProgrammingError: pass engine.dispose()
def test_config_read(): sample_config = helpers.read_config('db_dialect', config_ini="config.ini.sample") assert sample_config['dialect'] == 'sqlite' sample_config = helpers.read_config('metadata', config_ini="config.ini.sample") assert sample_config == { "user": '******', "password": '******', "account": 'XXX.xxxregion', "database": 'test_db', "schema": 'DV_MTD', "warehouse": 'xxx' }
def fill_metadata_schemas(eng): connection = eng.connect() metadata = MetaData() schemas = Table('SCHEMAS', metadata, autoload=True, autoload_with=eng) insp = reflection.Inspector.from_engine(eng) connection.execute(schemas.delete()) for role in ['metadata', 'source', 'staging', 'rawvault']: conn_conf = helpers.read_config(role) if eng.dialect.name == 'sqlite': val = { 'DATABASE': 'None', 'SCHEMA': conn_conf['schema'], 'ROLE': role } if eng.dialect.name == 'snowflake': val = { 'DATABASE': conn_conf['database'], 'SCHEMA': conn_conf['schema'], 'ROLE': role } connection.execute(schemas.insert().values(val))
connection.execute(hub_business_keys.insert().values( [hub_business_keys_values])) def insert_link_mappings(eng): metadata = MetaData() connection = eng.connect() link_mappings = Table('LINK_MAPPINGS', metadata, autoload=True, autoload_with=eng) connection.execute(link_mappings.delete()) with open(link_mapping_csv, 'r') as ifile: link_mapping_values = {} rows = DictReader(ifile) for row in rows: connection.execute(link_mappings.insert().values(row)) if __name__ == "__main__": dialect = helpers.read_config('db_dialect')['dialect'] if dialect == 'snowflake': engine_target = helpers.engine_snowflake('metadata') else: engine_target = helpers.engine_sqlite('metadata') insert_hub_mappings(engine_target) insert_link_mappings(engine_target) engine_target.dispose()