def registered(self, driver, frameworkId, masterInfo): self.log.info( "AirflowScheduler registered to Mesos with framework ID %s", frameworkId.value) if configuration.getboolean('mesos', 'CHECKPOINT') and configuration.get( 'mesos', 'FAILOVER_TIMEOUT'): # Import here to work around a circular import error from airflow.models import Connection # Update the Framework ID in the database. session = Session() conn_id = FRAMEWORK_CONNID_PREFIX + get_framework_name() connection = Session.query(Connection).filter_by( conn_id=conn_id).first() if connection is None: connection = Connection(conn_id=conn_id, conn_type='mesos_framework-id', extra=frameworkId.value) else: connection.extra = frameworkId.value session.add(connection) session.commit() Session.remove()
def create_airflow_conn(conf): conn = Connection() conn.conn_id = conf.get('conn_id') conn.conn_type = conf.get('conn_type') conn.host = conf.get('host') conn.port = conf.get('port') conn.login = conf.get('login') conn.password = conf.get('password') conn.schema = conf.get('schema') conn.extra = conf.get('extra') session = settings.Session() try: conn_name = session.\ query(Connection).\ filter(Connection.conn_id == conn.conn_id).first() if str(conn_name) == str(conf.get('conn_id')): logger.info(f"Connection {conn.conn_id} already exists") logger.info(f"Deleting connection {conn.conn_id}") session.delete(conn_name) else: session.add(conn) session.commit() session.close() logger.info(Connection.log_info(conn)) logger.info(f'Connection {conn.conn_id} is created') except Exception as e: logging.info(f"Exception occured while creating connection \n {e}")
def create_connection(config): """Creates a Connection object for Airflow Args: config (Dictionary): A dictionary containing all information needed to create a Connection object """ print(f"Creating and adding connection {config['conn_id']}") # Create connection object conn = Connection( conn_id=config['conn_id'], conn_type=config['conn_type'], #host = config['host'], #schema = config['schema'], #login = config['login'], #password = config['password'], #port = config['port'], #extra = json.dumps(config['extra']) ) if config['login'] is not None: conn.login = config['login'] if config['password'] is not None: conn.password = config['password'] if config['host'] is not None: conn.host = config['host'] if config['schema'] is not None: conn.schema = config['schema'] if config['port'] is not None: conn.port = config['port'] if config['extra'] is not None: conn.extra = json.dumps(config['extra']) connection = Session.query(Connection).filter_by( conn_id=conn.conn_id).first() if connection is None: # Add connection object to session, and commit Session.add(conn) Session.commit() print(f"Connection {config['conn_id']} added and committed") else: print(f"Connection {config['conn_id']} already exists")
def create_airflow_conn(conf): conn = Connection() conn.conn_id = conf.get('conn_id') conn.conn_type = conf.get('conn_type') conn.host = conf.get('host') conn.port = conf.get('port') conn.login = conf.get('login') conn.password = conf.get('password') conn.schema = conf.get('schema') conn.extra = conf.get('extra') session = settings.Session() try: existing_conns = session.query(Connection).filter( Connection.conn_id == conn.conn_id).delete() finally: session.add(conn) session.commit() session.close()
def registered(self, driver, frameworkId, masterInfo): logging.info("AirflowScheduler registered to mesos with framework ID %s", frameworkId.value) if configuration.getboolean('mesos', 'CHECKPOINT') and configuration.get('mesos', 'FAILOVER_TIMEOUT'): # Import here to work around a circular import error from airflow.models import Connection # Update the Framework ID in the database. session = Session() conn_id = FRAMEWORK_CONNID_PREFIX + get_framework_name() connection = Session.query(Connection).filter_by(conn_id=conn_id).first() if connection is None: connection = Connection(conn_id=conn_id, conn_type='mesos_framework-id', extra=frameworkId.value) else: connection.extra = frameworkId.value session.add(connection) session.commit() Session.remove()
def subscribed(self, driver, session=None): """ Subscribe to Mesos Master :param driver: Mesos driver object """ from airflow.models import Connection # Update the Framework ID in the database. conn_id = FRAMEWORK_CONNID_PREFIX + get_framework_name() connection = session.query(Connection).filter_by( conn_id=conn_id).first() if connection is None: connection = Connection(conn_id=conn_id, conn_type='mesos_framework-id', extra=driver.frameworkId) else: connection.extra = driver.frameworkId self.driver = driver