def _track_task(task):
if should_not_track(task):
return
if is_instance_by_class_name(task, "SubDagOperator"):
# we do not track the execute of a SubDag, only its tasks
track_dag(task.subdag)
else:
track_operator(task)
def _track_task(task):
if should_not_track(task):
return
try:
tracking_wrapper = get_tracking_wrapper(task)
except KeyError:
return
else:
tracking_wrapper(task)
def _track_task(task):
from airflow.operators.subdag_operator import SubDagOperator
if should_not_track(task):
return
track_operator(task)
if is_instance_by_class_name(task, SubDagOperator.__name__):
# we also track the subdag's inner tasks
track_dag(task.subdag)
def _track_task(task):
from airflow.operators.subdag_operator import SubDagOperator
if should_not_track(task):
return
if is_instance_by_class_name(task, SubDagOperator.__name__):
# we do not track the execute of a SubDag, only its tasks
track_dag(task.subdag)
else:
track_operator(task)
def _track_function(function):
if not _is_function(function) or should_not_track(function):
return
decorated_function = task(function)
# We modify all modules since each module has its own pointers to local and imported functions.
# If a module has already imported the function we need to change the pointer in that module.
for module in sys.modules.copy().values():
if not _is_module(module):
continue
for k, v in module.__dict__.items():
if v is function:
module.__dict__[k] = decorated_function
def track_dag(dag):
"""
Modify operators in dag if necessary so we could track them.
Supported operators: EmrAddStepsOperator, DataProcPySparkOperator, SparkSubmitOperator, PythonOperator.
Other operators are not modified.
More details on each operator:
- EmrAddStepsOperator: modify args of spark-submit steps by adding dbnd variables using --conf
- DataProcPySparkOperator: add dbnd variables to dataproc properties
- SparkSubmitOperator: add dbnd variables to spark-submit command using --conf
- PythonOperator: wrap python function with @task
"""
try:
if should_not_track(dag):
return
for task in dag.tasks:
track_task(task)
except Exception:
logger.exception("Failed to modify %s for tracking" % dag.dag_id)