def sync_to_db(self, session: Optional[Session] = None):
"""Save attributes about list of DAG to the DB."""
# To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag
from airflow.models.dag import DAG
from airflow.models.serialized_dag import SerializedDagModel
# Retry 'DAG.bulk_write_to_db' & 'SerializedDagModel.bulk_sync_to_db' in case
# of any Operational Errors
# In case of failures, provide_session handles rollback
for attempt in tenacity.Retrying(
retry=tenacity.retry_if_exception_type(
exception_types=OperationalError),
wait=tenacity.wait_random_exponential(multiplier=0.5, max=5),
stop=tenacity.stop_after_attempt(settings.MAX_DB_RETRIES),
before_sleep=tenacity.before_sleep_log(self.log,
logging.DEBUG),
reraise=True):
with attempt:
self.log.debug(
"Running dagbag.sync_to_db with retries. Try %d of %d",
attempt.retry_state.attempt_number,
settings.MAX_DB_RETRIES)
self.log.debug("Calling the DAG.bulk_sync_to_db method")
try:
DAG.bulk_write_to_db(self.dags.values(), session=session)
# Write Serialized DAGs to DB
self.log.debug(
"Calling the SerializedDagModel.bulk_sync_to_db method"
)
SerializedDagModel.bulk_sync_to_db(self.dags.values(),
session=session)
except OperationalError:
session.rollback()
raise
def test_bulk_sync_to_db(self):
dags = [
DAG("dag_1"),
DAG("dag_2"),
DAG("dag_3"),
]
with assert_queries_count(10):
SDM.bulk_sync_to_db(dags)
def sync_to_db(self):
"""
Save attributes about list of DAG to the DB.
"""
# To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag
from airflow.models.dag import DAG
from airflow.models.serialized_dag import SerializedDagModel
DAG.bulk_sync_to_db(self.dags.values())
SerializedDagModel.bulk_sync_to_db(self.dags.values())
def sync_to_db(self, session: Optional[Session] = None):
"""Save attributes about list of DAG to the DB."""
# To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag
from airflow.models.dag import DAG
from airflow.models.serialized_dag import SerializedDagModel
self.log.debug("Calling the DAG.bulk_sync_to_db method")
DAG.bulk_write_to_db(self.dags.values(), session=session)
# Write Serialized DAGs to DB
self.log.debug("Calling the SerializedDagModel.bulk_sync_to_db method")
SerializedDagModel.bulk_sync_to_db(self.dags.values(), session=session)
def sync_to_db(self):
"""
Save attributes about list of DAG to the DB.
"""
# To avoid circular import - airflow.models.dagbag -> airflow.models.dag -> airflow.models.dagbag
from airflow.models.dag import DAG
from airflow.models.serialized_dag import SerializedDagModel
self.log.debug("Calling the DAG.bulk_sync_to_db method")
DAG.bulk_sync_to_db(self.dags.values())
# Write Serialized DAGs to DB if DAG Serialization is turned on
# Even though self.read_dags_from_db is False
if settings.STORE_SERIALIZED_DAGS:
self.log.debug("Calling the SerializedDagModel.bulk_sync_to_db method")
SerializedDagModel.bulk_sync_to_db(self.dags.values())