class AirflowServerInfoSchema(ApiStrictSchema): airflow_version = fields.String(allow_none=True) airflow_export_version = fields.String(allow_none=True) airflow_monitor_version = fields.String(allow_none=True) last_sync_time = fields.DateTime(allow_none=True) monitor_error_message = fields.String(allow_none=True) synced_to = fields.DateTime(allow_none=True) api_mode = fields.String(allow_none=True) sync_interval = fields.Integer(allow_none=True) is_sync_enabled = fields.Boolean(allow_none=True) system_alert_definitions = fields.Dict() fetcher = fields.String(allow_none=True) composer_client_id = fields.String(allow_none=True) dag_ids = fields.String(allow_none=True) base_url = fields.String() external_url = fields.String(allow_none=True) source_instance_uid = fields.String(allow_none=True) tracking_source_uid = fields.String() airflow_instance_uid = fields.String( allow_none=True) # TODO_API: deprecate name = fields.String(allow_none=True) env = fields.String(allow_none=True) monitor_status = fields.String(allow_none=True) monitor_config = fields.Nested(MonitorConfigSchema, allow_none=True) airflow_environment = fields.String(allow_none=True) last_seen_dag_run_id = fields.Integer(allow_none=True) last_seen_log_id = fields.Integer(allow_none=True) @post_load def make_object(self, data, **kwargs): return AirflowServerInfo(**data)
class AirflowServerInfoSchema(_ApiCallSchema): base_url = fields.String() external_url = fields.String(allow_none=True) airflow_instance_uid = fields.String(allow_none=True) airflow_version = fields.String(allow_none=True) airflow_export_version = fields.String(allow_none=True) airflow_monitor_version = fields.String(allow_none=True) dags_path = fields.String(allow_none=True) logs_path = fields.String(allow_none=True) last_sync_time = fields.DateTime(allow_none=True) monitor_status = fields.String(allow_none=True) monitor_error_message = fields.String(allow_none=True) monitor_start_time = fields.DateTime(allow_none=True) synced_from = fields.DateTime(allow_none=True) synced_to = fields.DateTime(allow_none=True) api_mode = fields.String(allow_none=True) sync_interval = fields.Integer(allow_none=True) is_sync_enabled = fields.Boolean(allow_none=True) fetcher = fields.String(allow_none=True) composer_client_id = fields.String(allow_none=True) active_dags = fields.Dict(allow_none=True) name = fields.String(allow_none=True) env = fields.String(allow_none=True) include_logs = fields.Boolean(allow_none=True) include_task_args = fields.Boolean(allow_none=True) fetch_quantity = fields.Integer(allow_none=True) oldest_incomplete_data_in_days = fields.Integer(allow_none=True) @post_load def make_object(self, data, **kwargs): return AirflowServerInfo(**data)
class DagRunSchema(ApiStrictSchema): dag_id = fields.String(allow_none=True) run_id = fields.String(required=False) dagrun_id = fields.Integer() start_date = fields.DateTime(allow_none=True) state = fields.String() end_date = fields.DateTime(allow_none=True) execution_date = fields.DateTime(allow_none=True) task_args = fields.Dict()
class TaskInstanceSchema(ApiStrictSchema): execution_date = fields.DateTime() dag_id = fields.String() state = fields.String(allow_none=True) try_number = fields.Integer() task_id = fields.String() start_date = fields.DateTime(allow_none=True) end_date = fields.DateTime(allow_none=True) log_body = fields.String(allow_none=True) xcom_dict = fields.Dict()
class AirflowTaskInfoSchema(_ApiCallSchema): execution_date = fields.DateTime() last_sync = fields.DateTime(allow_none=True) dag_id = fields.String() task_id = fields.String() task_run_attempt_uid = fields.UUID() retry_number = fields.Integer(required=False, allow_none=True) @post_load def make_object(self, data, **kwargs): return AirflowTaskInfo(**data)
class DatasourceMonitorStateSchema(ApiStrictSchema): datasource_monitor_version = fields.String(required=False, allow_none=True) monitor_status = fields.String(required=False, allow_none=True) monitor_error_message = fields.String(required=False, allow_none=True) monitor_start_time = fields.DateTime(required=False, allow_none=True) last_sync_time = fields.DateTime(required=False, allow_none=True) last_transaction_sync_time = fields.DateTime(required=False, allow_none=True) @post_load def make_object(self, data): return DatasourceMonitorState(**data)
class SyncedDatasetSchema(ApiStrictSchema): uri = fields.String() # {storage_type://region/project_id/scheme_name/table_name} created_date = fields.DateTime() last_modified_date = fields.DateTime(allow_none=True) metadata = fields.Nested(SyncedDatasetMetadataSchema) uid = fields.UUID(allow_none=True) @post_load def make_object(self, data): return SyncedDataset(**data)
class TaskRunAttemptSchema(ApiStrictSchema): task_run_uid = fields.UUID() task_run_attempt_uid = fields.UUID() state = EnumField(TaskRunState, allow_none=True) timestamp = fields.DateTime(allow_none=True) first_error = fields.Nested(ErrorInfoSchema, allow_none=True) latest_error = fields.Nested(ErrorInfoSchema, allow_none=True) attempt_number = fields.Number(allow_none=True) source = fields.Str(allow_none=True) start_date = fields.DateTime(allow_none=True) end_date = fields.DateTime(allow_none=True) external_links_dict = fields.Dict(allow_none=True)
class TaskRunAttemptUpdateArgsSchema(ApiObjectSchema): task_run_uid = fields.UUID() task_run_attempt_uid = fields.UUID() state = EnumField(TaskRunState, allow_none=True) timestamp = fields.DateTime(allow_none=True) error = fields.Nested(ErrorInfoSchema, allow_none=True) attempt_number = fields.Number(allow_none=True) source = fields.Str(allow_none=True) start_date = fields.DateTime(allow_none=True) @post_load def make_object(self, data, **kwargs): return TaskRunAttemptUpdateArgs(**data)
class JobSchemaV2(ApiStrictSchema): id = fields.Int() name = fields.Str() user = fields.Str() reported_by_user = fields.Str() is_archived = fields.Boolean() ui_hidden = fields.Boolean() is_airflow_synced = fields.Boolean() project_id = fields.Int() project_name = fields.Str() # computed run_states = fields.Dict() source_link = fields.Str() # joined latest_run_start_time = fields.DateTime() latest_run_state = fields.Str() latest_run_uid = fields.UUID() latest_run_root_task_run_uid = fields.UUID() latest_run_trigger = fields.Str() latest_run_env = fields.Str() source_instance_name = fields.Str() source_type = fields.Str() airflow_instance_name = fields.Str() # TODO_SERVER: API: Deprecate scheduled_job_count = fields.Number()
class AirflowServerInfoSchema(_ApiCallSchema): base_url = fields.String() airflow_version = fields.String(allow_none=True) airflow_export_version = fields.String(allow_none=True) airflow_monitor_version = fields.String(allow_none=True) dags_path = fields.String(allow_none=True) logs_path = fields.String(allow_none=True) last_sync_time = fields.DateTime(allow_none=True) monitor_status = fields.String(allow_none=True) monitor_error_message = fields.String(allow_none=True) monitor_start_time = fields.DateTime(allow_none=True) synced_from = fields.DateTime(allow_none=True) synced_to = fields.DateTime(allow_none=True) @post_load def make_object(self, data, **kwargs): return AirflowServerInfo(**data)
class SyncedTransactionSchema(ApiStrictSchema): datasource_transaction_id = fields.String() created_date = fields.DateTime() started_date = fields.DateTime() ended_date = fields.DateTime() write_operation = fields.Nested(SyncedTransactionOperationSchema) read_operations = fields.Nested( SyncedTransactionOperationSchema, many=True, allow_none=True ) query_string = fields.String(allow_none=True) data_schema = fields.Nested(StructuredDataSchema, allow_none=True) @post_load def make_object(self, data): return SyncedTransaction(**data)
class RunInfoSchema(ApiStrictSchema): root_run_uid = fields.UUID() run_uid = fields.UUID() job_name = fields.String() project_name = fields.String(allow_none=True) user = fields.String() name = fields.String() description = fields.String(allow_none=True) state = EnumField(RunState) start_time = fields.DateTime() end_time = fields.DateTime(allow_none=True) # deprecate dag_id = fields.String() cmd_name = fields.String(allow_none=True) execution_date = fields.DateTime() # move to task target_date = fields.Date(allow_none=True) version = fields.String(allow_none=True) driver_name = fields.String() is_archived = fields.Boolean() env_name = fields.String(allow_none=True) cloud_type = fields.String() trigger = fields.String() task_executor = fields.String(allow_none=True) root_run = fields.Nested(RootRunInfoSchema) scheduled_run = fields.Nested(ScheduledRunInfoSchema, allow_none=True) sends_heartbeat = fields.Boolean(default=False, allow_none=True) scheduled_job_name = fields.String(allow_none=True) scheduled_date = fields.DateTime(allow_none=True) external_links = fields.Dict(allow_none=True) @post_load def make_run_info(self, data, **kwargs): return _as_dotted_dict(**data)
class ScheduledRunInfoSchema(ApiObjectSchema): scheduled_job_uid = fields.UUID(allow_none=True) scheduled_date = fields.DateTime(allow_none=True) scheduled_job_dag_run_id = fields.String(allow_none=True) scheduled_job_name = fields.String(allow_none=True) @post_load def make_object(self, data, **kwargs): return _as_dotted_dict(**data)
class JobFromFileSchema(Schema): class Meta: strict = True name = fields.Str(required=True) cmd = fields.Str(required=True) schedule_interval = fields.Str(required=True) start_date = fields.DateTime(allow_none=False, required=True, format="iso") owner = fields.Str(allow_none=False) end_date = fields.DateTime(allow_none=True, format="iso") depends_on_past = fields.Boolean(allow_none=True) catchup = fields.Boolean(allow_none=True) retries = fields.Int(allow_none=True) list_order = fields.Integer(allow_none=True) active = fields.Boolean(allow_none=True)
class TargetInfoSchema(ApiObjectSchema): parameter_name = fields.String() path = fields.String() created_date = fields.DateTime(allow_none=True) task_run_uid = fields.UUID(allow_none=True) @post_load def make_target(self, data, **kwargs): return TargetInfo(**data)
class SyncedTransactionOperationSchema(ApiStrictSchema): op_type = fields.String() started_date = fields.DateTime() # type: datetime records_count = fields.Integer() dataset_uri = fields.String() dataset_uid = fields.UUID(allow_none=True) @post_load def make_object(self, data): return SyncedTransactionOperation(**data)
class SyncedTransactionsReportSchema(ApiStrictSchema): sync_event_uid = fields.UUID() sync_event_timestamp = fields.DateTime() source_type = fields.String(allow_none=True) # bigquery / snowflake / etc syncer_type = fields.String(allow_none=True) monitor_state = fields.Nested(DatasourceMonitorStateSchema) transactions = fields.Nested(SyncedTransactionSchema, many=True) @post_load def make_object(self, data): return SyncedTransactionsReport(**data)
class MetricSchema(ApiObjectSchema): key = fields.String() value = fields.String(allow_none=True) value_int = fields.Integer(allow_none=True) value_float = fields.Float(allow_none=True) timestamp = fields.DateTime() @post_load def make_object(self, data, **kwargs): from dbnd._core.tracking.metrics import Metric return Metric(**data)
class DatasetsReportSchema(ApiStrictSchema): sync_event_uid = fields.UUID() monitor_state = fields.Nested(DatasourceMonitorStateSchema) source_type = fields.String(allow_none=True) # bigquery / snowflake / etc syncer_type = fields.String(allow_none=True) sync_event_timestamp = fields.DateTime(required=False, allow_none=True) datasets = fields.Nested(SyncedDatasetSchema, many=True) @post_load def make_object(self, data): return DatasetsReport(**data)
class MetricSchema(ApiStrictSchema): key = fields.String() value = fields.Raw(allow_none=True) value_str = fields.String(allow_none=True) value_json = fields.Raw(allow_none=True) value_int = fields.Integer(allow_none=True) value_float = fields.Float(allow_none=True) timestamp = fields.DateTime() @post_load def make_object(self, data, **kwargs): return Metric(**data)
class ScheduledJobSchemaV2(Schema): class Meta: strict = True uid = fields.Str(attribute="DbndScheduledJob.uid", allow_none=True) name = fields.Str(attribute="DbndScheduledJob.name", required=True) cmd = fields.Str(attribute="DbndScheduledJob.cmd", required=True) schedule_interval = fields.Str( attribute="DbndScheduledJob.schedule_interval", required=True) start_date = fields.DateTime(allow_none=True, attribute="DbndScheduledJob.start_date", format="iso") end_date = fields.DateTime(allow_none=True, attribute="DbndScheduledJob.end_date", format="iso") readable_schedule_interval = fields.Str( attribute="DbndScheduledJob.readable_schedule_interval", allow_none=True) scheduled_interval_in_seconds = fields.Integer( attribute="DbndScheduledJob.scheduled_interval_in_seconds", allow_none=True) catchup = fields.Boolean(allow_none=True, attribute="DbndScheduledJob.catchup") depends_on_past = fields.Boolean( allow_none=True, attribute="DbndScheduledJob.depends_on_past") retries = fields.Int(allow_none=True, attribute="DbndScheduledJob.retries") active = fields.Boolean(allow_none=True, attribute="DbndScheduledJob.active") create_user = fields.Str(allow_none=True, attribute="DbndScheduledJob.create_user") create_time = fields.DateTime(allow_none=True, attribute="DbndScheduledJob.create_time") update_user = fields.Str(allow_none=True, attribute="DbndScheduledJob.update_user") update_time = fields.DateTime(allow_none=True, attribute="DbndScheduledJob.update_time") from_file = fields.Boolean(allow_none=True, attribute="DbndScheduledJob.from_file") deleted_from_file = fields.Boolean( allow_none=True, attribute="DbndScheduledJob.deleted_from_file") next_job_date = fields.DateTime(attribute="DbndScheduledJob.next_job_date", allow_none=True) alerts = fields.List( fields.Nested(AlertEventSchema), attribute="DbndScheduledJob.alerts", allow_none=True, ) job_name = fields.Str(dump_only=True, attribute="DbndScheduledJob.job_name") last_run_uid = fields.UUID(dump_only=True) last_run_job = fields.Str(dump_only=True) last_job_date = fields.DateTime(dump_only=True) last_run_state = fields.Str(dump_only=True) is_airflow_synced = fields.Bool(dump_only=True) list_order = fields.Integer(attribute="DbndScheduledJob.list_order", allow_none=True) validation_errors = fields.Str( allow_none=True, attribute="DbndScheduledJob.validation_errors")
class DagSchema(ApiStrictSchema): description = fields.String() root_task_ids = fields.List(fields.String()) tasks = fields.Nested(TaskSchema, many=True) owner = fields.String() dag_id = fields.String() schedule_interval = fields.String() catchup = fields.Boolean(allow_none=True) start_date = fields.DateTime(allow_none=True) end_date = fields.DateTime(allow_none=True) is_committed = fields.Boolean() git_commit = fields.String() dag_folder = fields.String() hostname = fields.String() source_code = fields.String(allow_none=True) module_source_hash = fields.String(allow_none=True) is_subdag = fields.Boolean() tags = fields.List(fields.String(), allow_none=True) task_type = fields.String() task_args = fields.Dict() is_active = fields.Boolean(allow_none=True) is_paused = fields.Boolean(allow_none=True)
class AirflowServerInfoSchema(_ApiCallSchema): base_url = fields.String() external_url = fields.String() airflow_version = fields.String(allow_none=True) airflow_export_version = fields.String(allow_none=True) airflow_monitor_version = fields.String(allow_none=True) dags_path = fields.String(allow_none=True) logs_path = fields.String(allow_none=True) last_sync_time = fields.DateTime(allow_none=True) monitor_status = fields.String(allow_none=True) monitor_error_message = fields.String(allow_none=True) monitor_start_time = fields.DateTime(allow_none=True) synced_from = fields.DateTime(allow_none=True) synced_to = fields.DateTime(allow_none=True) rbac_enabled = fields.Boolean(allow_none=True) sync_interval = fields.Integer(allow_none=True) is_sync_enabled = fields.Boolean(allow_none=True) fetcher = fields.String(allow_none=True) composer_client_id = fields.String(allow_none=True) @post_load def make_object(self, data, **kwargs): return AirflowServerInfo(**data)
class ScheduledJobInfoSchema(ApiObjectSchema): uid = fields.UUID() name = fields.String() cmd = fields.String() start_date = fields.DateTime() create_user = fields.String() create_time = fields.DateTime() end_date = fields.DateTime(allow_none=True) schedule_interval = fields.String(allow_none=True) catchup = fields.Boolean(allow_none=True) depends_on_past = fields.Boolean(allow_none=True) retries = fields.Integer(allow_none=True) active = fields.Boolean(allow_none=True) update_user = fields.String(allow_none=True) update_time = fields.DateTime(allow_none=True) from_file = fields.Boolean(allow_none=True) deleted_from_file = fields.Boolean(allow_none=True) list_order = fields.Integer(allow_none=True) job_name = fields.String(allow_none=True) @post_load def make_object(self, data, **kwargs): return ScheduledJobInfo(**data)
class TaskRunInfoSchema(ApiObjectSchema): task_run_uid = fields.UUID() task_run_attempt_uid = fields.UUID() task_definition_uid = fields.UUID() run_uid = fields.UUID() task_id = fields.String() task_signature = fields.String() task_signature_source = fields.String() task_af_id = fields.String() execution_date = fields.DateTime() name = fields.String() env = fields.String() command_line = fields.String() functional_call = fields.String() has_downstreams = fields.Boolean() has_upstreams = fields.Boolean() is_reused = fields.Boolean() is_dynamic = fields.Boolean() is_system = fields.Boolean() is_skipped = fields.Boolean() is_root = fields.Boolean() output_signature = fields.String() state = EnumField(TaskRunState) target_date = fields.Date(allow_none=True) log_local = fields.String(allow_none=True) log_remote = fields.String(allow_none=True) version = fields.String() task_run_params = fields.Nested(TaskRunParamSchema, many=True) external_links = fields.Dict(allow_none=True) @post_load def make_task_run(self, data, **kwargs): return TaskRunInfo(**data)
class TaskRunEnvInfoSchema(ApiObjectSchema): uid = fields.UUID() cmd_line = fields.String() user = fields.String() machine = fields.String() databand_version = fields.String() user_code_version = fields.String() user_code_committed = fields.Boolean() project_root = fields.String() user_data = fields.String() heartbeat = fields.DateTime() @post_load def make_object(self, data, **kwargs): return TaskRunEnvInfo(**data)
class JobSchemaV2(ApiObjectSchema): id = fields.Int() name = fields.Str() user = fields.Str() ui_hidden = fields.Boolean() is_airflow_synced = fields.Boolean() # computed run_states = fields.Dict() airflow_link = fields.Str() # joined latest_run_start_time = fields.DateTime() latest_run_state = fields.Str() latest_run_uid = fields.UUID() latest_run_root_task_run_uid = fields.UUID() latest_run_trigger = fields.Str() latest_run_env = fields.Str() scheduled_job_count = fields.Number()
class AlertDefsSchema(ApiObjectSchema): uid = fields.Str() original_uid = fields.Str() custom_name = fields.Str() custom_description = fields.Str() summary = fields.Str() severity = fields.Str() type = fields.Str() task_name = fields.Str() user_metric = fields.Str() operator = fields.Str() is_str_value = fields.Bool() value = fields.Str() advanced_json = fields.Str() created_at = fields.DateTime() scheduled_job_uid = fields.Str() scheduled_job_name = fields.Str(attribute="scheduled_job.name") job_name = fields.Str()
class AirflowMonitorDataSchema(ApiObjectSchema): airflow_export_data = fields.Str() airflow_base_url = fields.Str() last_sync_time = fields.DateTime()