def setUpClass(cls):
super(TestVarImportView, cls).setUpClass()
session = Session()
session.query(models.User).delete()
session.commit()
user = models.User(username='******')
session.add(user)
session.commit()
session.close()
def setUpClass(cls):
super(TestKnownEventView, cls).setUpClass()
session = Session()
session.query(models.KnownEvent).delete()
session.query(models.User).delete()
session.commit()
user = models.User(username='******')
session.add(user)
session.commit()
cls.user_id = user.id
session.close()
def setUp(self):
self.app = application.create_app(testing=True)
session = Session()
user = models.User()
password_user = PasswordUser(user)
password_user.username = '******'
password_user.password = '******'
session.add(password_user)
session.commit()
session.close()
def setUpClass(cls):
super(TestKnownEventView, cls).setUpClass()
session = Session()
session.query(models.KnownEvent).delete()
session.query(models.User).delete()
session.commit()
user = models.User(username='******')
session.add(user)
session.commit()
cls.user_id = user.id
session.close()
def initialize_example():
logging.info('Reading config')
session = Session()
def create_new_conn(session, attributes):
new_conn = models.Connection()
new_conn.conn_id = attributes.get("conn_id")
new_conn.conn_type = attributes.get('conn_type')
new_conn.host = attributes.get('host')
new_conn.port = attributes.get('port')
new_conn.schema = attributes.get('schema')
new_conn.login = attributes.get('login')
new_conn.set_password(attributes.get('password'))
session.add(new_conn)
session.commit()
def config(filename='../.postgresql.ini'):
# create a parser
parser = ConfigParser(strict=False)
# read config file
parser.read(filename)
# get section, default to postgresql
db_config = {}
logging.info('Creating connections and sql path')
for section in parser.sections():
conn_id = section
conn_type = section.split('_')
for param in parser.items(section):
db_config[param[0]] = param[1]
create_new_conn(
session, {
"conn_id": conn_id,
"conn_type": conn_type[0],
"host": db_config['host'],
"port": db_config['port'],
"schema": db_config['dbname'],
"login": db_config['user'],
"password": db_config['password']
})
config()
new_var = models.Variable()
new_var.key = "sql_path"
new_var.set_val("/Users/jackychu/airflow/sql")
session.add(new_var)
session.commit()
session.close()
def initialize_etl_example():
logging.info('Creating connections, pool and sql path')
session = Session()
def create_new_conn(session, attributes):
new_conn = models.Connection()
new_conn.conn_id = attributes.get("conn_id")
new_conn.conn_type = attributes.get('conn_type')
new_conn.host = attributes.get('host')
new_conn.port = attributes.get('port')
new_conn.schema = attributes.get('schema')
new_conn.login = attributes.get('login')
new_conn.set_password(attributes.get('password'))
session.add(new_conn)
session.commit()
create_new_conn(session,
{"conn_id": "postgres_oltp",
"conn_type": "postgres",
"host": "postgres",
"port": 5432,
"schema": "orders",
"login": "******",
"password": "******"})
# change from dwh_svc_account to db_owner
create_new_conn(session,
{"conn_id": "postgres_dwh",
"conn_type": "postgres",
"host": "postgres",
"port": 5432,
"schema": "dwh",
"login": "******",
"password": "******"})
# Move variable setting to startup batch, to avoid console err msg when loading those dags refto this var.
# new_var = models.Variable()
# new_var.key = "sql_path"
# new_var.set_val("/usr/local/airflow/sql")
# session.add(new_var)
# session.commit()
new_pool = models.Pool()
new_pool.pool = "postgres_dwh"
new_pool.slots = 10
new_pool.description = "Allows max. 10 connections to the DWH"
session.add(new_pool)
session.commit()
session.close()
def __update_conn_extra_tokens(self, auth, connection):
from airflow.settings import Session
conn_extra = connection.extra_dejson
conn_extra['twitter_access_token'] = auth.access_token
conn_extra['twitter_access_token_secret'] = auth.access_token_secret
connection.set_extra(conn_extra)
session = Session()
session.add(connection)
session.commit()
self.log.info(f'Connection {connection.conn_id} updated with "twitter_access_token"'
f'and "twitter_access_token_secret" values.')
def create_connection(config):
"""Creates a Connection object for Airflow
Args:
config (Dictionary): A dictionary containing all information needed to create a Connection object
"""
print(f"Creating and adding connection {config['conn_id']}")
# Create connection object
conn = Connection(
conn_id=config['conn_id'],
conn_type=config['conn_type'],
#host = config['host'],
#schema = config['schema'],
#login = config['login'],
#password = config['password'],
#port = config['port'],
#extra = json.dumps(config['extra'])
)
if config['login'] is not None:
conn.login = config['login']
if config['password'] is not None:
conn.password = config['password']
if config['host'] is not None:
conn.host = config['host']
if config['schema'] is not None:
conn.schema = config['schema']
if config['port'] is not None:
conn.port = config['port']
if config['extra'] is not None:
conn.extra = json.dumps(config['extra'])
connection = Session.query(Connection).filter_by(
conn_id=conn.conn_id).first()
if connection is None:
# Add connection object to session, and commit
Session.add(conn)
Session.commit()
print(f"Connection {config['conn_id']} added and committed")
else:
print(f"Connection {config['conn_id']} already exists")
def setUp(self):
configuration.conf.set("webserver", "authenticate", "True")
configuration.conf.set("webserver", "auth_backend", "airflow.contrib.auth.backends.password_auth")
app = application.create_app()
app.config['TESTING'] = True
self.app = app.test_client()
from airflow.contrib.auth.backends.password_auth import PasswordUser
session = Session()
user = models.User()
password_user = PasswordUser(user)
password_user.username = '******'
password_user.password = '******'
print(password_user._password)
session.add(password_user)
session.commit()
session.close()
def setUp(self):
configuration.conf.set("webserver", "authenticate", "True")
configuration.conf.set("webserver", "auth_backend",
"airflow.contrib.auth.backends.password_auth")
app = application.create_app()
app.config['TESTING'] = True
self.app = app.test_client()
from airflow.contrib.auth.backends.password_auth import PasswordUser
session = Session()
user = models.User()
password_user = PasswordUser(user)
password_user.username = '******'
password_user.password = '******'
print(password_user._password)
session.add(password_user)
session.commit()
session.close()
def registered(self, driver, frameworkId, masterInfo):
logging.info("AirflowScheduler registered to mesos with framework ID %s", frameworkId.value)
if configuration.getboolean('mesos', 'CHECKPOINT') and configuration.get('mesos', 'FAILOVER_TIMEOUT'):
# Import here to work around a circular import error
from airflow.models import Connection
# Update the Framework ID in the database.
session = Session()
conn_id = FRAMEWORK_CONNID_PREFIX + get_framework_name()
connection = Session.query(Connection).filter_by(conn_id=conn_id).first()
if connection is None:
connection = Connection(conn_id=conn_id, conn_type='mesos_framework-id',
extra=frameworkId.value)
else:
connection.extra = frameworkId.value
session.add(connection)
session.commit()
Session.remove()
def registered(self, driver, frameworkId, masterInfo):
logging.info("AirflowScheduler registered to mesos with framework ID %s", frameworkId.value)
if configuration.getboolean('mesos', 'CHECKPOINT') and configuration.get('mesos', 'FAILOVER_TIMEOUT'):
# Import here to work around a circular import error
from airflow.models import Connection
# Update the Framework ID in the database.
session = Session()
conn_id = FRAMEWORK_CONNID_PREFIX + get_framework_name()
connection = Session.query(Connection).filter_by(conn_id=conn_id).first()
if connection is None:
connection = Connection(conn_id=conn_id, conn_type='mesos_framework-id',
extra=frameworkId.value)
else:
connection.extra = frameworkId.value
session.add(connection)
session.commit()
Session.remove()
def setUp(self):
configuration.load_test_config()
try:
configuration.conf.add_section("api")
except DuplicateSectionError:
pass
configuration.conf.set("api", "auth_backend",
"airflow.contrib.auth.backends.password_auth")
self.app = application.create_app(testing=True)
session = Session()
user = models.User()
password_user = PasswordUser(user)
password_user.username = '******'
password_user.password = '******'
session.add(password_user)
session.commit()
session.close()
def setUp(self):
configuration.load_test_config()
try:
configuration.conf.add_section("api")
except DuplicateSectionError:
pass
configuration.conf.set("api",
"auth_backend",
"airflow.contrib.auth.backends.password_auth")
self.app = application.create_app(testing=True)
session = Session()
user = models.User()
password_user = PasswordUser(user)
password_user.username = '******'
password_user.password = '******'
session.add(password_user)
session.commit()
session.close()
def generate_init():
"""
Function is checking names of tables in postgres other.
Then information is combined with standard params and pushed as Airflow Variable object.
Also pool is created.
:return:
"""
psql_hook = PostgresHook('airflow_docker_db')
eng = psql_hook.get_sqlalchemy_engine()
df = pd.read_sql(
"select table_name from information_schema.tables where table_schema='other';",
con=eng)
table_list = df['table_name'].tolist()
try:
pool = Pool()
pool.slots = 1
pool.description = 'How many tasks can run at once'
pool.pool = 'generate_tasks'
session = Session()
session.add(pool)
session.commit()
except Exception as ex:
logging.info(f'Could not set pool. Details: {ex}')
init_data = {
'psql_conn_id': 'airflow_docker_db',
'table_list': table_list,
'pool': 'generate_tasks'
}
try:
Variable.set(key='generate_tasks',
value=init_data,
serialize_json=True)
except Exception as ex:
logging.info(f'Could not set global variable. Details: {ex}')
def generate_config():
logging.info('Creating connections, pool and sql path')
session = Session()
def create_new_conn(session, attributes):
if Session.query(models.Connection).filter(
models.Connection.conn_id == attributes.get(
"conn_id")).count() == 0:
new_conn = models.Connection()
new_conn.conn_id = attributes.get("conn_id")
new_conn.conn_type = attributes.get('conn_type')
new_conn.host = attributes.get('host')
new_conn.port = attributes.get('port')
new_conn.schema = attributes.get('schema')
new_conn.login = attributes.get('login')
new_conn.set_password(attributes.get('password'))
session.add(new_conn)
session.commit()
else:
logging.info('Connection {} already exists'.format(
attributes.get("conn_id")))
create_new_conn(
session, {
"conn_id": "mysql_oltp",
"conn_type": "mysql",
"host": "host.docker.internal",
"port": 3306,
"schema": "employees",
"login": "******",
"password": "******"
})
create_new_conn(
session, {
"conn_id": "mysql_dwh",
"conn_type": "mysql",
"host": "host.docker.internal",
"port": 3306,
"schema": "dwh",
"login": "******",
"password": "******"
})
create_new_conn(
session, {
"conn_id": "postgres_oltp",
"conn_type": "postgres",
"host": "host.docker.internal",
"port": 5432,
"schema": "dwh",
"login": "******",
"password": "******"
})
create_new_conn(
session, {
"conn_id": "postgres_dwh",
"conn_type": "postgres",
"host": "host.docker.internal",
"port": 5432,
"schema": "dwh",
"login": "******",
"password": "******"
})
if Session.query(models.Variable).filter(
models.Variable.key == "sql_template_paths").count() == 0:
new_var = models.Variable()
new_var.key = "sql_template_paths"
new_var.set_val("./sql_templates")
session.add(new_var)
session.commit()
else:
logging.info('Variable sql_template_paths already exists')
if Session.query(
models.Pool).filter(models.Pool.pool == "mysql_dwh").count() == 0:
new_pool = models.Pool()
new_pool.pool = "mysql_dwh"
new_pool.slots = 10
new_pool.description = "Allows max. 10 connections to the DWH"
session.add(new_pool)
session.commit()
else:
logging.info('Pool mysql_dwh already exists')
session.close()
class TestPoolApiExperimental(unittest.TestCase):
@classmethod
def setUpClass(cls):
super(TestPoolApiExperimental, cls).setUpClass()
session = Session()
session.query(Pool).delete()
session.commit()
session.close()
def setUp(self):
super(TestPoolApiExperimental, self).setUp()
configuration.load_test_config()
app = application.create_app(testing=True)
self.app = app.test_client()
self.session = Session()
self.pools = []
for i in range(2):
name = 'experimental_%s' % (i + 1)
pool = Pool(
pool=name,
slots=i,
description=name,
)
self.session.add(pool)
self.pools.append(pool)
self.session.commit()
self.pool = self.pools[0]
def tearDown(self):
self.session.query(Pool).delete()
self.session.commit()
self.session.close()
super(TestPoolApiExperimental, self).tearDown()
def _get_pool_count(self):
response = self.app.get('/api/experimental/pools')
self.assertEqual(response.status_code, 200)
return len(json.loads(response.data.decode('utf-8')))
def test_get_pool(self):
response = self.app.get(
'/api/experimental/pools/{}'.format(self.pool.pool), )
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.data.decode('utf-8')),
self.pool.to_json())
def test_get_pool_non_existing(self):
response = self.app.get('/api/experimental/pools/foo')
self.assertEqual(response.status_code, 404)
self.assertEqual(
json.loads(response.data.decode('utf-8'))['error'],
"Pool 'foo' doesn't exist")
def test_get_pools(self):
response = self.app.get('/api/experimental/pools')
self.assertEqual(response.status_code, 200)
pools = json.loads(response.data.decode('utf-8'))
self.assertEqual(len(pools), 2)
for i, pool in enumerate(sorted(pools, key=lambda p: p['pool'])):
self.assertDictEqual(pool, self.pools[i].to_json())
def test_create_pool(self):
response = self.app.post(
'/api/experimental/pools',
data=json.dumps({
'name': 'foo',
'slots': 1,
'description': '',
}),
content_type='application/json',
)
self.assertEqual(response.status_code, 200)
pool = json.loads(response.data.decode('utf-8'))
self.assertEqual(pool['pool'], 'foo')
self.assertEqual(pool['slots'], 1)
self.assertEqual(pool['description'], '')
self.assertEqual(self._get_pool_count(), 3)
def test_create_pool_with_bad_name(self):
for name in ('', ' '):
response = self.app.post(
'/api/experimental/pools',
data=json.dumps({
'name': name,
'slots': 1,
'description': '',
}),
content_type='application/json',
)
self.assertEqual(response.status_code, 400)
self.assertEqual(
json.loads(response.data.decode('utf-8'))['error'],
"Pool name shouldn't be empty",
)
self.assertEqual(self._get_pool_count(), 2)
def test_delete_pool(self):
response = self.app.delete(
'/api/experimental/pools/{}'.format(self.pool.pool), )
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.data.decode('utf-8')),
self.pool.to_json())
self.assertEqual(self._get_pool_count(), 1)
def test_delete_pool_non_existing(self):
response = self.app.delete('/api/experimental/pools/foo', )
self.assertEqual(response.status_code, 404)
self.assertEqual(
json.loads(response.data.decode('utf-8'))['error'],
"Pool 'foo' doesn't exist")
def init_hive_example():
logging.info('Creating connections, pool and sql path')
session = Session()
def create_new_conn(session, attributes):
new_conn = models.Connection()
new_conn.conn_id = attributes.get("conn_id")
new_conn.conn_type = attributes.get('conn_type')
new_conn.host = attributes.get('host')
new_conn.port = attributes.get('port')
new_conn.schema = attributes.get('schema')
new_conn.login = attributes.get('login')
new_conn.set_extra(attributes.get('extra'))
new_conn.set_password(attributes.get('password'))
session.add(new_conn)
session.commit()
create_new_conn(
session, {
"conn_id": "postgres_oltp",
"conn_type": "postgres",
"host": "postgres",
"port": 5432,
"schema": "orders",
"login": "******",
"password": "******"
})
create_new_conn(
session, {
"conn_id":
"hive_staging",
"conn_type":
"hive_cli",
"host":
"hive",
"schema":
"default",
"port":
10000,
"login":
"******",
"password":
"******",
"extra":
json.dumps({
"hive_cli_params": "",
"auth": "none",
"use_beeline": "true"
})
})
new_var = models.Variable()
new_var.key = "sql_path"
new_var.set_val("/usr/local/airflow/sql")
session.add(new_var)
new_var = models.Variable()
new_var.key = "hive_sql_path"
new_var.set_val("/usr/local/airflow/hql")
session.add(new_var)
session.commit()
session.close()
class TestPoolApiExperimental(unittest.TestCase):
@classmethod
def setUpClass(cls):
super(TestPoolApiExperimental, cls).setUpClass()
session = Session()
session.query(Pool).delete()
session.commit()
session.close()
def setUp(self):
super(TestPoolApiExperimental, self).setUp()
configuration.load_test_config()
app, _ = application.create_app(testing=True)
self.app = app.test_client()
self.session = Session()
self.pools = []
for i in range(2):
name = 'experimental_%s' % (i + 1)
pool = Pool(
pool=name,
slots=i,
description=name,
)
self.session.add(pool)
self.pools.append(pool)
self.session.commit()
self.pool = self.pools[0]
def tearDown(self):
self.session.query(Pool).delete()
self.session.commit()
self.session.close()
super(TestPoolApiExperimental, self).tearDown()
def _get_pool_count(self):
response = self.app.get('/api/experimental/pools')
self.assertEqual(response.status_code, 200)
return len(json.loads(response.data.decode('utf-8')))
def test_get_pool(self):
response = self.app.get(
'/api/experimental/pools/{}'.format(self.pool.pool),
)
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.data.decode('utf-8')),
self.pool.to_json())
def test_get_pool_non_existing(self):
response = self.app.get('/api/experimental/pools/foo')
self.assertEqual(response.status_code, 404)
self.assertEqual(json.loads(response.data.decode('utf-8'))['error'],
"Pool 'foo' doesn't exist")
def test_get_pools(self):
response = self.app.get('/api/experimental/pools')
self.assertEqual(response.status_code, 200)
pools = json.loads(response.data.decode('utf-8'))
self.assertEqual(len(pools), 2)
for i, pool in enumerate(sorted(pools, key=lambda p: p['pool'])):
self.assertDictEqual(pool, self.pools[i].to_json())
def test_create_pool(self):
response = self.app.post(
'/api/experimental/pools',
data=json.dumps({
'name': 'foo',
'slots': 1,
'description': '',
}),
content_type='application/json',
)
self.assertEqual(response.status_code, 200)
pool = json.loads(response.data.decode('utf-8'))
self.assertEqual(pool['pool'], 'foo')
self.assertEqual(pool['slots'], 1)
self.assertEqual(pool['description'], '')
self.assertEqual(self._get_pool_count(), 3)
def test_create_pool_with_bad_name(self):
for name in ('', ' '):
response = self.app.post(
'/api/experimental/pools',
data=json.dumps({
'name': name,
'slots': 1,
'description': '',
}),
content_type='application/json',
)
self.assertEqual(response.status_code, 400)
self.assertEqual(
json.loads(response.data.decode('utf-8'))['error'],
"Pool name shouldn't be empty",
)
self.assertEqual(self._get_pool_count(), 2)
def test_delete_pool(self):
response = self.app.delete(
'/api/experimental/pools/{}'.format(self.pool.pool),
)
self.assertEqual(response.status_code, 200)
self.assertEqual(json.loads(response.data.decode('utf-8')),
self.pool.to_json())
self.assertEqual(self._get_pool_count(), 1)
def test_delete_pool_non_existing(self):
response = self.app.delete(
'/api/experimental/pools/foo',
)
self.assertEqual(response.status_code, 404)
self.assertEqual(json.loads(response.data.decode('utf-8'))['error'],
"Pool 'foo' doesn't exist")
