def sync_to_db(self, session: Optional[Session] = None): """Save attributes about list of DAG to the DB.""" # To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel # Retry 'DAG.bulk_write_to_db' & 'SerializedDagModel.bulk_sync_to_db' in case # of any Operational Errors # In case of failures, provide_session handles rollback for attempt in tenacity.Retrying( retry=tenacity.retry_if_exception_type( exception_types=OperationalError), wait=tenacity.wait_random_exponential(multiplier=0.5, max=5), stop=tenacity.stop_after_attempt(settings.MAX_DB_RETRIES), before_sleep=tenacity.before_sleep_log(self.log, logging.DEBUG), reraise=True): with attempt: self.log.debug( "Running dagbag.sync_to_db with retries. Try %d of %d", attempt.retry_state.attempt_number, settings.MAX_DB_RETRIES) self.log.debug("Calling the DAG.bulk_sync_to_db method") try: DAG.bulk_write_to_db(self.dags.values(), session=session) # Write Serialized DAGs to DB self.log.debug( "Calling the SerializedDagModel.bulk_sync_to_db method" ) SerializedDagModel.bulk_sync_to_db(self.dags.values(), session=session) except OperationalError: session.rollback() raise
def test_bulk_sync_to_db(self): dags = [ DAG("dag_1"), DAG("dag_2"), DAG("dag_3"), ] with assert_queries_count(10): SDM.bulk_sync_to_db(dags)
def sync_to_db(self): """ Save attributes about list of DAG to the DB. """ # To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel DAG.bulk_sync_to_db(self.dags.values()) SerializedDagModel.bulk_sync_to_db(self.dags.values())
def sync_to_db(self, session: Optional[Session] = None): """Save attributes about list of DAG to the DB.""" # To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel self.log.debug("Calling the DAG.bulk_sync_to_db method") DAG.bulk_write_to_db(self.dags.values(), session=session) # Write Serialized DAGs to DB self.log.debug("Calling the SerializedDagModel.bulk_sync_to_db method") SerializedDagModel.bulk_sync_to_db(self.dags.values(), session=session)
def sync_to_db(self): """ Save attributes about list of DAG to the DB. """ # To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag from airflow.models.dag import DAG from airflow.models.serialized_dag import SerializedDagModel self.log.debug("Calling the DAG.bulk_sync_to_db method") DAG.bulk_sync_to_db(self.dags.values()) # Write Serialized DAGs to DB if DAG Serialization is turned on # Even though self.read_dags_from_db is False if settings.STORE_SERIALIZED_DAGS: self.log.debug("Calling the SerializedDagModel.bulk_sync_to_db method") SerializedDagModel.bulk_sync_to_db(self.dags.values())