def initialize_etl_example():
logging.info('Creating connections, pool and sql path')
session = Session()
def create_new_conn(session, attributes):
new_conn = models.Connection()
new_conn.conn_id = attributes.get("conn_id")
new_conn.conn_type = attributes.get('conn_type')
new_conn.host = attributes.get('host')
new_conn.port = attributes.get('port')
new_conn.schema = attributes.get('schema')
new_conn.login = attributes.get('login')
new_conn.set_password(attributes.get('password'))
session.add(new_conn)
session.commit()
create_new_conn(
session, {
"conn_id": "mssql",
"conn_type": "MS SQL Server",
"host": "mssql",
"port": 1433,
"schema": "master",
"login": "******",
"password": "******"
})
create_new_conn(
session, {
"conn_id": "postgres_dwh",
"conn_type": "postgres",
"host": "postgres",
"port": 5432,
"schema": "dwh",
"login": "******",
"password": "******"
})
new_var = models.Variable()
new_var.key = "sql_path"
new_var.set_val("/usr/local/airflow/sql")
session.add(new_var)
session.commit()
new_pool = models.Pool()
new_pool.pool = "postgres_dwh"
new_pool.slots = 10
new_pool.description = "Allows max. 10 connections to the DWH"
session.add(new_pool)
session.commit()
session.close()
def setUp(self):
self.pools = []
for i in range(2):
name = 'experimental_%s' % (i + 1)
pool = models.Pool(
pool=name,
slots=i,
description=name,
)
self.pools.append(pool)
with create_session() as session:
session.add_all(self.pools)
def setUp(self):
clear_db_pools()
self.pools = [Pool.get_default_pool()]
for i in range(self.USER_POOL_COUNT):
name = f'experimental_{i + 1}'
pool = models.Pool(
pool=name,
slots=i,
description=name,
)
self.pools.append(pool)
with create_session() as session:
session.add_all(self.pools)
def setUp(self):
self.session = settings.Session()
self.pools = []
for i in range(2):
name = 'experimental_%s' % (i + 1)
pool = models.Pool(
pool=name,
slots=i,
description=name,
)
self.session.add(pool)
self.pools.append(pool)
self.session.commit()
def initialize_etl_example():
logging.info('Creating connections, pool and sql path')
session = Session()
def create_new_conn(session, attributes):
new_conn = models.Connection()
new_conn.conn_id = attributes.get("conn_id")
new_conn.conn_type = attributes.get('conn_type')
new_conn.host = attributes.get('host')
new_conn.port = attributes.get('port')
new_conn.schema = attributes.get('schema')
new_conn.login = attributes.get('login')
new_conn.set_password(attributes.get('password'))
session.add(new_conn)
session.commit()
create_new_conn(session,
{"conn_id": "postgres_oltp",
"conn_type": "postgres",
"host": "postgres",
"port": 5432,
"schema": "orders",
"login": "******",
"password": "******"})
# change from dwh_svc_account to db_owner
create_new_conn(session,
{"conn_id": "postgres_dwh",
"conn_type": "postgres",
"host": "postgres",
"port": 5432,
"schema": "dwh",
"login": "******",
"password": "******"})
# Move variable setting to startup batch, to avoid console err msg when loading those dags refto this var.
# new_var = models.Variable()
# new_var.key = "sql_path"
# new_var.set_val("/usr/local/airflow/sql")
# session.add(new_var)
# session.commit()
new_pool = models.Pool()
new_pool.pool = "postgres_dwh"
new_pool.slots = 10
new_pool.description = "Allows max. 10 connections to the DWH"
session.add(new_pool)
session.commit()
session.close()
def setUp(self):
self.session = settings.Session()
clear_db_pools()
self.pools = [Pool.get_default_pool()]
for i in range(self.USER_POOL_COUNT):
name = 'experimental_%s' % (i + 1)
pool = models.Pool(
pool=name,
slots=i,
description=name,
)
self.session.add(pool)
self.pools.append(pool)
self.session.commit()
def generate_config():
logging.info('Creating connections, pool and sql path')
session = Session()
def create_new_conn(session, attributes):
if Session.query(models.Connection).filter(
models.Connection.conn_id == attributes.get(
"conn_id")).count() == 0:
new_conn = models.Connection()
new_conn.conn_id = attributes.get("conn_id")
new_conn.conn_type = attributes.get('conn_type')
new_conn.host = attributes.get('host')
new_conn.port = attributes.get('port')
new_conn.schema = attributes.get('schema')
new_conn.login = attributes.get('login')
new_conn.set_password(attributes.get('password'))
session.add(new_conn)
session.commit()
else:
logging.info('Connection {} already exists'.format(
attributes.get("conn_id")))
create_new_conn(
session, {
"conn_id": "mysql_oltp",
"conn_type": "mysql",
"host": "host.docker.internal",
"port": 3306,
"schema": "employees",
"login": "******",
"password": "******"
})
create_new_conn(
session, {
"conn_id": "mysql_dwh",
"conn_type": "mysql",
"host": "host.docker.internal",
"port": 3306,
"schema": "dwh",
"login": "******",
"password": "******"
})
create_new_conn(
session, {
"conn_id": "postgres_oltp",
"conn_type": "postgres",
"host": "host.docker.internal",
"port": 5432,
"schema": "dwh",
"login": "******",
"password": "******"
})
create_new_conn(
session, {
"conn_id": "postgres_dwh",
"conn_type": "postgres",
"host": "host.docker.internal",
"port": 5432,
"schema": "dwh",
"login": "******",
"password": "******"
})
if Session.query(models.Variable).filter(
models.Variable.key == "sql_template_paths").count() == 0:
new_var = models.Variable()
new_var.key = "sql_template_paths"
new_var.set_val("./sql_templates")
session.add(new_var)
session.commit()
else:
logging.info('Variable sql_template_paths already exists')
if Session.query(
models.Pool).filter(models.Pool.pool == "mysql_dwh").count() == 0:
new_pool = models.Pool()
new_pool.pool = "mysql_dwh"
new_pool.slots = 10
new_pool.description = "Allows max. 10 connections to the DWH"
session.add(new_pool)
session.commit()
else:
logging.info('Pool mysql_dwh already exists')
session.close()
