class JobSchemaV2(ApiStrictSchema): id = fields.Int() name = fields.Str() user = fields.Str() reported_by_user = fields.Str() is_archived = fields.Boolean() ui_hidden = fields.Boolean() is_airflow_synced = fields.Boolean() project_id = fields.Int() project_name = fields.Str() # computed run_states = fields.Dict() source_link = fields.Str() # joined latest_run_start_time = fields.DateTime() latest_run_state = fields.Str() latest_run_uid = fields.UUID() latest_run_root_task_run_uid = fields.UUID() latest_run_trigger = fields.Str() latest_run_env = fields.Str() source_instance_name = fields.Str() source_type = fields.Str() airflow_instance_name = fields.Str() # TODO_SERVER: API: Deprecate scheduled_job_count = fields.Number()
class RootRunInfoSchema(ApiObjectSchema): root_run_uid = fields.UUID() root_task_run_uid = fields.UUID(allow_none=True) root_run_url = fields.String(allow_none=True) @post_load def make_object(self, data, **kwargs): return _as_dotted_dict(**data)
class LogTargetMetricsSchema(_ApiCallSchema): task_run_uid = fields.UUID(required=True) task_run_attempt_uid = fields.UUID(required=True) target_path = fields.String() value_preview = fields.String(allow_none=True) data_dimensions = fields.List(fields.Integer(), allow_none=True) data_schema = fields.String(allow_none=True)
class TaskRunAttemptSchema(ApiStrictSchema): task_run_uid = fields.UUID() task_run_attempt_uid = fields.UUID() state = EnumField(TaskRunState, allow_none=True) timestamp = fields.DateTime(allow_none=True) first_error = fields.Nested(ErrorInfoSchema, allow_none=True) latest_error = fields.Nested(ErrorInfoSchema, allow_none=True) attempt_number = fields.Number(allow_none=True) source = fields.Str(allow_none=True) start_date = fields.DateTime(allow_none=True) end_date = fields.DateTime(allow_none=True) external_links_dict = fields.Dict(allow_none=True)
class TaskRunAttemptUpdateArgsSchema(ApiObjectSchema): task_run_uid = fields.UUID() task_run_attempt_uid = fields.UUID() state = EnumField(TaskRunState, allow_none=True) timestamp = fields.DateTime(allow_none=True) error = fields.Nested(ErrorInfoSchema, allow_none=True) attempt_number = fields.Number(allow_none=True) source = fields.Str(allow_none=True) start_date = fields.DateTime(allow_none=True) @post_load def make_object(self, data, **kwargs): return TaskRunAttemptUpdateArgs(**data)
class ScheduledJobSchemaV2(Schema): class Meta: strict = True uid = fields.Str(attribute="DbndScheduledJob.uid", allow_none=True) name = fields.Str(attribute="DbndScheduledJob.name", required=True) cmd = fields.Str(attribute="DbndScheduledJob.cmd", required=True) schedule_interval = fields.Str( attribute="DbndScheduledJob.schedule_interval", required=True) start_date = fields.DateTime(allow_none=True, attribute="DbndScheduledJob.start_date", format="iso") end_date = fields.DateTime(allow_none=True, attribute="DbndScheduledJob.end_date", format="iso") readable_schedule_interval = fields.Str( attribute="DbndScheduledJob.readable_schedule_interval", allow_none=True) scheduled_interval_in_seconds = fields.Integer( attribute="DbndScheduledJob.scheduled_interval_in_seconds", allow_none=True) catchup = fields.Boolean(allow_none=True, attribute="DbndScheduledJob.catchup") depends_on_past = fields.Boolean( allow_none=True, attribute="DbndScheduledJob.depends_on_past") retries = fields.Int(allow_none=True, attribute="DbndScheduledJob.retries") active = fields.Boolean(allow_none=True, attribute="DbndScheduledJob.active") create_user = fields.Str(allow_none=True, attribute="DbndScheduledJob.create_user") create_time = fields.DateTime(allow_none=True, attribute="DbndScheduledJob.create_time") update_user = fields.Str(allow_none=True, attribute="DbndScheduledJob.update_user") update_time = fields.DateTime(allow_none=True, attribute="DbndScheduledJob.update_time") from_file = fields.Boolean(allow_none=True, attribute="DbndScheduledJob.from_file") deleted_from_file = fields.Boolean( allow_none=True, attribute="DbndScheduledJob.deleted_from_file") next_job_date = fields.DateTime(attribute="DbndScheduledJob.next_job_date", allow_none=True) alerts = fields.List( fields.Nested(AlertEventSchema), attribute="DbndScheduledJob.alerts", allow_none=True, ) job_name = fields.Str(dump_only=True, attribute="DbndScheduledJob.job_name") last_run_uid = fields.UUID(dump_only=True) last_run_job = fields.Str(dump_only=True) last_job_date = fields.DateTime(dump_only=True) last_run_state = fields.Str(dump_only=True) is_airflow_synced = fields.Bool(dump_only=True) list_order = fields.Integer(attribute="DbndScheduledJob.list_order", allow_none=True) validation_errors = fields.Str( allow_none=True, attribute="DbndScheduledJob.validation_errors")
class LogTargetSchema(_ApiCallSchema): run_uid = fields.UUID(required=True) task_run_uid = fields.UUID(required=True) task_run_name = fields.String() task_run_attempt_uid = fields.UUID(required=True) target_path = fields.String() param_name = fields.String(allow_none=True) task_def_uid = fields.UUID(allow_none=True) operation_type = EnumField(DbndTargetOperationType) operation_status = EnumField(DbndTargetOperationStatus) value_preview = fields.String(allow_none=True) data_dimensions = fields.List(fields.Integer(), allow_none=True) data_schema = fields.String(allow_none=True) data_hash = fields.String(allow_none=True)
class UpdateAirflowMonitorStateRequestSchema(ApiStrictSchema): airflow_version = fields.String(required=False, allow_none=True) airflow_export_version = fields.String(required=False, allow_none=True) airflow_monitor_version = fields.String(required=False, allow_none=True) monitor_status = fields.String(required=False, allow_none=True) monitor_error_message = fields.String(required=False, allow_none=True) airflow_instance_uid = fields.UUID(required=False, allow_none=True) api_mode = fields.String(required=False, allow_none=True)
class InitRunArgsSchema(ApiObjectSchema): run_uid = fields.UUID() root_run_uid = fields.UUID() driver_task_uid = fields.UUID(allow_none=True) task_run_env = fields.Nested(TaskRunEnvInfoSchema) task_runs_info = fields.Nested(TaskRunsInfoSchema) new_run_info = fields.Nested(RunInfoSchema, allow_none=True) scheduled_run_info = fields.Nested(ScheduledRunInfoSchema, allow_none=True) update_existing = fields.Boolean() source = fields.Str(allow_none=True) @post_load def make_init_run_args(self, data, **kwargs): return InitRunArgs(**data)
class TaskRunInfoSchema(ApiObjectSchema): task_run_uid = fields.UUID() task_run_attempt_uid = fields.UUID() task_definition_uid = fields.UUID() run_uid = fields.UUID() task_id = fields.String() task_signature = fields.String() task_signature_source = fields.String() task_af_id = fields.String() execution_date = fields.DateTime() name = fields.String() env = fields.String() command_line = fields.String() functional_call = fields.String() has_downstreams = fields.Boolean() has_upstreams = fields.Boolean() is_reused = fields.Boolean() is_dynamic = fields.Boolean() is_system = fields.Boolean() is_skipped = fields.Boolean() is_root = fields.Boolean() output_signature = fields.String() state = EnumField(TaskRunState) target_date = fields.Date(allow_none=True) log_local = fields.String(allow_none=True) log_remote = fields.String(allow_none=True) version = fields.String() task_run_params = fields.Nested(TaskRunParamSchema, many=True) external_links = fields.Dict(allow_none=True) @post_load def make_task_run(self, data, **kwargs): return TaskRunInfo(**data)
class RunInfoSchema(ApiStrictSchema): root_run_uid = fields.UUID() run_uid = fields.UUID() job_name = fields.String() project_name = fields.String(allow_none=True) user = fields.String() name = fields.String() description = fields.String(allow_none=True) state = EnumField(RunState) start_time = fields.DateTime() end_time = fields.DateTime(allow_none=True) # deprecate dag_id = fields.String() cmd_name = fields.String(allow_none=True) execution_date = fields.DateTime() # move to task target_date = fields.Date(allow_none=True) version = fields.String(allow_none=True) driver_name = fields.String() is_archived = fields.Boolean() env_name = fields.String(allow_none=True) cloud_type = fields.String() trigger = fields.String() task_executor = fields.String(allow_none=True) root_run = fields.Nested(RootRunInfoSchema) scheduled_run = fields.Nested(ScheduledRunInfoSchema, allow_none=True) sends_heartbeat = fields.Boolean(default=False, allow_none=True) scheduled_job_name = fields.String(allow_none=True) scheduled_date = fields.DateTime(allow_none=True) external_links = fields.Dict(allow_none=True) @post_load def make_run_info(self, data, **kwargs): return _as_dotted_dict(**data)
class ScheduledRunInfoSchema(ApiObjectSchema): scheduled_job_uid = fields.UUID(allow_none=True) scheduled_date = fields.DateTime(allow_none=True) scheduled_job_dag_run_id = fields.String(allow_none=True) scheduled_job_name = fields.String(allow_none=True) @post_load def make_object(self, data, **kwargs): return _as_dotted_dict(**data)
class TargetInfoSchema(ApiObjectSchema): parameter_name = fields.String() path = fields.String() created_date = fields.DateTime(allow_none=True) task_run_uid = fields.UUID(allow_none=True) @post_load def make_target(self, data, **kwargs): return TargetInfo(**data)
class AirflowTaskContextSchema(ApiObjectSchema): dag_id = fields.String() execution_date = fields.String() task_id = fields.String() try_number = fields.Integer(allow_none=True) airflow_instance_uid = fields.UUID() @post_load def make_run_info(self, data, **kwargs): return AirflowTaskContext(**data)
class TaskRunsInfoSchema(ApiObjectSchema): run_uid = fields.UUID() root_run_uid = fields.UUID() task_run_env_uid = fields.UUID() parent_child_map = fields.List(fields.List(fields.UUID())) task_runs = fields.Nested(TaskRunInfoSchema, many=True, exclude=("task_signature_source", )) upstreams_map = fields.List(fields.List(fields.UUID())) dynamic_task_run_update = fields.Boolean() targets = fields.Nested(TargetInfoSchema, many=True) task_definitions = fields.Nested(TaskDefinitionInfoSchema, many=True) af_context = fields.Nested(AirflowTaskContextSchema, allow_none=True) @post_load def make_run_info(self, data, **kwargs): return _as_dotted_dict(**data)
class SyncedTransactionOperationSchema(ApiStrictSchema): op_type = fields.String() started_date = fields.DateTime() # type: datetime records_count = fields.Integer() dataset_uri = fields.String() dataset_uid = fields.UUID(allow_none=True) @post_load def make_object(self, data): return SyncedTransactionOperation(**data)
class AirflowTaskInfoSchema(_ApiCallSchema): execution_date = fields.DateTime() last_sync = fields.DateTime(allow_none=True) dag_id = fields.String() task_id = fields.String() task_run_attempt_uid = fields.UUID() retry_number = fields.Integer(required=False, allow_none=True) @post_load def make_object(self, data, **kwargs): return AirflowTaskInfo(**data)
class JobSchemaV2(ApiObjectSchema): id = fields.Int() name = fields.Str() user = fields.Str() ui_hidden = fields.Boolean() is_airflow_synced = fields.Boolean() # computed run_states = fields.Dict() airflow_link = fields.Str() # joined latest_run_start_time = fields.DateTime() latest_run_state = fields.Str() latest_run_uid = fields.UUID() latest_run_root_task_run_uid = fields.UUID() latest_run_trigger = fields.Str() latest_run_env = fields.Str() scheduled_job_count = fields.Number()
class DatasetsReportSchema(ApiStrictSchema): sync_event_uid = fields.UUID() monitor_state = fields.Nested(DatasourceMonitorStateSchema) source_type = fields.String(allow_none=True) # bigquery / snowflake / etc syncer_type = fields.String(allow_none=True) sync_event_timestamp = fields.DateTime(required=False, allow_none=True) datasets = fields.Nested(SyncedDatasetSchema, many=True) @post_load def make_object(self, data): return DatasetsReport(**data)
class SyncedTransactionsReportSchema(ApiStrictSchema): sync_event_uid = fields.UUID() sync_event_timestamp = fields.DateTime() source_type = fields.String(allow_none=True) # bigquery / snowflake / etc syncer_type = fields.String(allow_none=True) monitor_state = fields.Nested(DatasourceMonitorStateSchema) transactions = fields.Nested(SyncedTransactionSchema, many=True) @post_load def make_object(self, data): return SyncedTransactionsReport(**data)
class SyncedDatasetSchema(ApiStrictSchema): uri = fields.String() # {storage_type://region/project_id/scheme_name/table_name} created_date = fields.DateTime() last_modified_date = fields.DateTime(allow_none=True) metadata = fields.Nested(SyncedDatasetMetadataSchema) uid = fields.UUID(allow_none=True) @post_load def make_object(self, data): return SyncedDataset(**data)
class TaskRunEnvInfoSchema(ApiObjectSchema): uid = fields.UUID() cmd_line = fields.String() user = fields.String() machine = fields.String() databand_version = fields.String() user_code_version = fields.String() user_code_committed = fields.Boolean() project_root = fields.String() user_data = fields.String() heartbeat = fields.DateTime() @post_load def make_object(self, data, **kwargs): return TaskRunEnvInfo(**data)
class TaskDefinitionInfoSchema(ApiObjectSchema): task_definition_uid = fields.UUID() name = fields.String() class_version = fields.String() family = fields.String() module_source = fields.String(allow_none=True) module_source_hash = fields.String(allow_none=True) source = fields.String(allow_none=True) source_hash = fields.String(allow_none=True) type = fields.String() task_param_definitions = fields.Nested(TaskDefinitionParamSchema, many=True) @post_load def make_task_definition(self, data, **kwargs): return TaskDefinitionInfo(**data)
class ScheduledJobInfoSchema(ApiObjectSchema): uid = fields.UUID() name = fields.String() cmd = fields.String() start_date = fields.DateTime() create_user = fields.String() create_time = fields.DateTime() end_date = fields.DateTime(allow_none=True) schedule_interval = fields.String(allow_none=True) catchup = fields.Boolean(allow_none=True) depends_on_past = fields.Boolean(allow_none=True) retries = fields.Integer(allow_none=True) active = fields.Boolean(allow_none=True) update_user = fields.String(allow_none=True) update_time = fields.DateTime(allow_none=True) from_file = fields.Boolean(allow_none=True) deleted_from_file = fields.Boolean(allow_none=True) list_order = fields.Integer(allow_none=True) job_name = fields.String(allow_none=True) @post_load def make_object(self, data, **kwargs): return ScheduledJobInfo(**data)
class HeartbeatSchema(_ApiCallSchema): run_uid = fields.UUID()
class SetUnfinishedTasksStateShcmea(_ApiCallSchema): run_uid = fields.UUID() state = EnumField(TaskRunState) timestamp = fields.DateTime()
class LogArtifactSchema(_ApiCallSchema): task_run_attempt_uid = fields.UUID(required=True) name = fields.String() path = fields.String()
class LogMetricSchema(_ApiCallSchema): task_run_attempt_uid = fields.UUID(required=True) target_meta_uid = fields.UUID(allow_none=True) metric = fields.Nested(MetricSchema, allow_none=True) source = fields.String(allow_none=True)
class SaveExternalLinksSchema(_ApiCallSchema): task_run_attempt_uid = fields.UUID(required=True) external_links_dict = fields.Dict( name=fields.Str(), url=fields.Str(), required=True )
class SaveTaskRunLogSchema(_ApiCallSchema): task_run_attempt_uid = fields.UUID(required=True) log_body = fields.String(allow_none=True)