def _serialize_node(cls, op: Union[BaseOperator, MappedOperator], include_deps: bool) -> Dict[str, Any]: """Serializes operator into a JSON object.""" serialize_op = cls.serialize_to_json(op, cls._decorated_fields) serialize_op['_task_type'] = getattr(op, "_task_type", type(op).__name__) serialize_op['_task_module'] = getattr(op, "_task_module", type(op).__module__) # Used to determine if an Operator is inherited from EmptyOperator serialize_op['_is_empty'] = op.inherits_from_empty_operator if op.operator_extra_links: serialize_op['_operator_extra_links'] = cls._serialize_operator_extra_links( op.operator_extra_links ) if include_deps: serialize_op['deps'] = cls._serialize_deps(op.deps) # Store all template_fields as they are if there are JSON Serializable # If not, store them as strings if op.template_fields: for template_field in op.template_fields: value = getattr(op, template_field, None) if not cls._is_excluded(value, template_field, op): serialize_op[template_field] = serialize_template_field(value) if op.params: serialize_op['params'] = cls._serialize_params_dict(op.params) return serialize_op
def __init__(self, ti: TaskInstance, render_templates=True): self.dag_id = ti.dag_id self.task_id = ti.task_id self.task = ti.task self.execution_date = ti.execution_date self.ti = ti if render_templates: ti.render_templates() self.rendered_fields = { field: serialize_template_field(getattr(self.task, field)) for field in self.task.template_fields }
def __init__(self, ti: TaskInstance, render_templates=True): self.dag_id = ti.dag_id self.task_id = ti.task_id self.task = ti.task self.execution_date = ti.execution_date self.ti = ti if render_templates: ti.render_templates() if os.environ.get("AIRFLOW_IS_K8S_EXECUTOR_POD", None): self.k8s_pod_yaml = ti.render_k8s_pod_yaml() self.rendered_fields = { field: serialize_template_field(getattr(self.task, field)) for field in self.task.template_fields }
def __init__(self, ti: TaskInstance, render_templates=True): self.dag_id = ti.dag_id self.task_id = ti.task_id self.task = ti.task self.execution_date = ti.execution_date self.ti = ti if render_templates: ti.render_templates() if IS_K8S_OR_K8SCELERY_EXECUTOR: self.k8s_pod_yaml = ti.render_k8s_pod_yaml() self.rendered_fields = { field: serialize_template_field(getattr(self.task, field)) for field in self.task.template_fields }
def _serialize_node(cls, op: Union[BaseOperator, MappedOperator]) -> Dict[str, Any]: """Serializes operator into a JSON object.""" serialize_op = cls.serialize_to_json(op, cls._decorated_fields) serialize_op['_task_type'] = type(op).__name__ serialize_op['_task_module'] = type(op).__module__ # Used to determine if an Operator is inherited from DummyOperator serialize_op['_is_dummy'] = op.inherits_from_dummy_operator if op.operator_extra_links: serialize_op['_operator_extra_links'] = cls._serialize_operator_extra_links( op.operator_extra_links ) if op.deps is not BaseOperator.deps: # Are the deps different to BaseOperator, if so serialize the class names! # For Airflow 2.0 expediency we _only_ allow built in Dep classes. # Fix this for 2.0.x or 2.1 deps = [] for dep in op.deps: klass = type(dep) module_name = klass.__module__ if not module_name.startswith("airflow.ti_deps.deps."): assert op.dag # for type checking raise SerializationError( f"Cannot serialize {(op.dag.dag_id + '.' + op.task_id)!r} with `deps` from non-core " f"module {module_name!r}" ) deps.append(f'{module_name}.{klass.__name__}') # deps needs to be sorted here, because op.deps is a set, which is unstable when traversing, # and the same call may get different results. # When calling json.dumps(self.data, sort_keys=True) to generate dag_hash, misjudgment will occur serialize_op['deps'] = sorted(deps) # Store all template_fields as they are if there are JSON Serializable # If not, store them as strings if op.template_fields: for template_field in op.template_fields: value = getattr(op, template_field, None) if not cls._is_excluded(value, template_field, op): serialize_op[template_field] = serialize_template_field(value) if op.params: serialize_op['params'] = cls._serialize_params_dict(op.params) return serialize_op
def serialize_operator(cls, op: BaseOperator) -> dict: """Serializes operator into a JSON object.""" serialize_op = cls.serialize_to_json(op, cls._decorated_fields) serialize_op['_task_type'] = op.__class__.__name__ serialize_op['_task_module'] = op.__class__.__module__ if op.operator_extra_links: serialize_op['_operator_extra_links'] = \ cls._serialize_operator_extra_links(op.operator_extra_links) # Store all template_fields as they are if there are JSON Serializable # If not, store them as strings if op.template_fields: for template_field in op.template_fields: value = getattr(op, template_field, None) if not cls._is_excluded(value, template_field, op): serialize_op[template_field] = serialize_template_field(value) return serialize_op
def serialize_operator(cls, op: BaseOperator) -> Dict[str, Any]: """Serializes operator into a JSON object.""" serialize_op = cls.serialize_to_json(op, cls._decorated_fields) serialize_op['_task_type'] = op.__class__.__name__ serialize_op['_task_module'] = op.__class__.__module__ # Used to determine if an Operator is inherited from DummyOperator serialize_op['_is_dummy'] = op.inherits_from_dummy_operator if op.operator_extra_links: serialize_op['_operator_extra_links'] = cls._serialize_operator_extra_links( op.operator_extra_links ) if op.deps is not BaseOperator.deps: # Are the deps different to BaseOperator, if so serialize the class names! # For Airflow 2.0 expediency we _only_ allow built in Dep classes. # Fix this for 2.0.x or 2.1 deps = [] for dep in op.deps: klass = type(dep) module_name = klass.__module__ if not module_name.startswith("airflow.ti_deps.deps."): raise SerializationError( f"Cannot serialize {(op.dag.dag_id + '.' + op.task_id)!r} with `deps` from non-core " f"module {module_name!r}" ) deps.append(f'{module_name}.{klass.__name__}') serialize_op['deps'] = deps # Store all template_fields as they are if there are JSON Serializable # If not, store them as strings if op.template_fields: for template_field in op.template_fields: value = getattr(op, template_field, None) if not cls._is_excluded(value, template_field, op): serialize_op[template_field] = serialize_template_field(value) event_handler = op.get_events_handler() if event_handler is not None: serialize_op['_events_handler'] = EventHandler.serialize(op.get_events_handler()) return serialize_op