def registered(self, driver, frameworkId, masterInfo):
self.log.info(
"AirflowScheduler registered to Mesos with framework ID %s",
frameworkId.value)
if configuration.getboolean('mesos',
'CHECKPOINT') and configuration.get(
'mesos', 'FAILOVER_TIMEOUT'):
# Import here to work around a circular import error
from airflow.models import Connection
# Update the Framework ID in the database.
session = Session()
conn_id = FRAMEWORK_CONNID_PREFIX + get_framework_name()
connection = Session.query(Connection).filter_by(
conn_id=conn_id).first()
if connection is None:
connection = Connection(conn_id=conn_id,
conn_type='mesos_framework-id',
extra=frameworkId.value)
else:
connection.extra = frameworkId.value
session.add(connection)
session.commit()
Session.remove()
def create_airflow_conn(conf):
conn = Connection()
conn.conn_id = conf.get('conn_id')
conn.conn_type = conf.get('conn_type')
conn.host = conf.get('host')
conn.port = conf.get('port')
conn.login = conf.get('login')
conn.password = conf.get('password')
conn.schema = conf.get('schema')
conn.extra = conf.get('extra')
session = settings.Session()
try:
conn_name = session.\
query(Connection).\
filter(Connection.conn_id == conn.conn_id).first()
if str(conn_name) == str(conf.get('conn_id')):
logger.info(f"Connection {conn.conn_id} already exists")
logger.info(f"Deleting connection {conn.conn_id}")
session.delete(conn_name)
else:
session.add(conn)
session.commit()
session.close()
logger.info(Connection.log_info(conn))
logger.info(f'Connection {conn.conn_id} is created')
except Exception as e:
logging.info(f"Exception occured while creating connection \n {e}")
def create_connection(config):
"""Creates a Connection object for Airflow
Args:
config (Dictionary): A dictionary containing all information needed to create a Connection object
"""
print(f"Creating and adding connection {config['conn_id']}")
# Create connection object
conn = Connection(
conn_id=config['conn_id'],
conn_type=config['conn_type'],
#host = config['host'],
#schema = config['schema'],
#login = config['login'],
#password = config['password'],
#port = config['port'],
#extra = json.dumps(config['extra'])
)
if config['login'] is not None:
conn.login = config['login']
if config['password'] is not None:
conn.password = config['password']
if config['host'] is not None:
conn.host = config['host']
if config['schema'] is not None:
conn.schema = config['schema']
if config['port'] is not None:
conn.port = config['port']
if config['extra'] is not None:
conn.extra = json.dumps(config['extra'])
connection = Session.query(Connection).filter_by(
conn_id=conn.conn_id).first()
if connection is None:
# Add connection object to session, and commit
Session.add(conn)
Session.commit()
print(f"Connection {config['conn_id']} added and committed")
else:
print(f"Connection {config['conn_id']} already exists")
def create_airflow_conn(conf):
conn = Connection()
conn.conn_id = conf.get('conn_id')
conn.conn_type = conf.get('conn_type')
conn.host = conf.get('host')
conn.port = conf.get('port')
conn.login = conf.get('login')
conn.password = conf.get('password')
conn.schema = conf.get('schema')
conn.extra = conf.get('extra')
session = settings.Session()
try:
existing_conns = session.query(Connection).filter(
Connection.conn_id == conn.conn_id).delete()
finally:
session.add(conn)
session.commit()
session.close()
def registered(self, driver, frameworkId, masterInfo):
logging.info("AirflowScheduler registered to mesos with framework ID %s", frameworkId.value)
if configuration.getboolean('mesos', 'CHECKPOINT') and configuration.get('mesos', 'FAILOVER_TIMEOUT'):
# Import here to work around a circular import error
from airflow.models import Connection
# Update the Framework ID in the database.
session = Session()
conn_id = FRAMEWORK_CONNID_PREFIX + get_framework_name()
connection = Session.query(Connection).filter_by(conn_id=conn_id).first()
if connection is None:
connection = Connection(conn_id=conn_id, conn_type='mesos_framework-id',
extra=frameworkId.value)
else:
connection.extra = frameworkId.value
session.add(connection)
session.commit()
Session.remove()
def subscribed(self, driver, session=None):
"""
Subscribe to Mesos Master
:param driver: Mesos driver object
"""
from airflow.models import Connection
# Update the Framework ID in the database.
conn_id = FRAMEWORK_CONNID_PREFIX + get_framework_name()
connection = session.query(Connection).filter_by(
conn_id=conn_id).first()
if connection is None:
connection = Connection(conn_id=conn_id,
conn_type='mesos_framework-id',
extra=driver.frameworkId)
else:
connection.extra = driver.frameworkId
self.driver = driver
