class JobFromFileSchema(Schema):
class Meta:
strict = True
name = fields.Str(required=True)
cmd = fields.Str(required=True)
schedule_interval = fields.Str(required=True)
start_date = fields.DateTime(allow_none=False, required=True, format="iso")
owner = fields.Str(allow_none=False)
end_date = fields.DateTime(allow_none=True, format="iso")
depends_on_past = fields.Boolean(allow_none=True)
catchup = fields.Boolean(allow_none=True)
retries = fields.Int(allow_none=True)
list_order = fields.Integer(allow_none=True)
active = fields.Boolean(allow_none=True)
class TaskRunAttemptSchema(ApiStrictSchema):
task_run_uid = fields.UUID()
task_run_attempt_uid = fields.UUID()
state = EnumField(TaskRunState, allow_none=True)
timestamp = fields.DateTime(allow_none=True)
first_error = fields.Nested(ErrorInfoSchema, allow_none=True)
latest_error = fields.Nested(ErrorInfoSchema, allow_none=True)
attempt_number = fields.Number(allow_none=True)
source = fields.Str(allow_none=True)
start_date = fields.DateTime(allow_none=True)
end_date = fields.DateTime(allow_none=True)
external_links_dict = fields.Dict(allow_none=True)
class TaskRunAttemptUpdateArgsSchema(ApiObjectSchema):
task_run_uid = fields.UUID()
task_run_attempt_uid = fields.UUID()
state = EnumField(TaskRunState, allow_none=True)
timestamp = fields.DateTime(allow_none=True)
error = fields.Nested(ErrorInfoSchema, allow_none=True)
attempt_number = fields.Number(allow_none=True)
source = fields.Str(allow_none=True)
start_date = fields.DateTime(allow_none=True)
@post_load
def make_object(self, data, **kwargs):
return TaskRunAttemptUpdateArgs(**data)
class InitRunArgsSchema(ApiObjectSchema):
run_uid = fields.UUID()
root_run_uid = fields.UUID()
driver_task_uid = fields.UUID(allow_none=True)
task_run_env = fields.Nested(TaskRunEnvInfoSchema)
task_runs_info = fields.Nested(TaskRunsInfoSchema)
new_run_info = fields.Nested(RunInfoSchema, allow_none=True)
scheduled_run_info = fields.Nested(ScheduledRunInfoSchema, allow_none=True)
update_existing = fields.Boolean()
source = fields.Str(allow_none=True)
@post_load
def make_init_run_args(self, data, **kwargs):
return InitRunArgs(**data)
class JobSchemaV2(ApiObjectSchema):
id = fields.Int()
name = fields.Str()
user = fields.Str()
is_archived = fields.Boolean()
ui_hidden = fields.Boolean()
is_airflow_synced = fields.Boolean()
# computed
run_states = fields.Dict()
airflow_link = fields.Str()
# joined
latest_run_start_time = fields.DateTime()
latest_run_state = fields.Str()
latest_run_uid = fields.UUID()
latest_run_root_task_run_uid = fields.UUID()
latest_run_trigger = fields.Str()
latest_run_env = fields.Str()
airflow_instance_name = fields.Str()
scheduled_job_count = fields.Number()
class AirflowMonitorDataSchema(ApiObjectSchema):
airflow_export_data = fields.Str()
airflow_base_url = fields.Str()
last_sync_time = fields.DateTime()
class SaveExternalLinksSchema(_ApiCallSchema):
task_run_attempt_uid = fields.UUID(required=True)
external_links_dict = fields.Dict(
name=fields.Str(), url=fields.Str(), required=True
)
class ExternalUrlSchema(ApiObjectSchema):
id = fields.UUID(attribute="uid")
name = fields.Str()
url = fields.Str()
class JobSchemaV2(ApiStrictSchema):
id = fields.Int()
name = fields.Str()
user = fields.Str()
reported_by_user = fields.Str()
is_archived = fields.Boolean()
ui_hidden = fields.Boolean()
is_airflow_synced = fields.Boolean()
project_id = fields.Int()
project_name = fields.Str()
# computed
run_states = fields.Dict()
source_link = fields.Str()
# joined
latest_run_start_time = fields.DateTime()
latest_run_state = fields.Str()
latest_run_uid = fields.UUID()
latest_run_root_task_run_uid = fields.UUID()
latest_run_trigger = fields.Str()
latest_run_env = fields.Str()
source_instance_name = fields.Str()
source_type = fields.Str()
airflow_instance_name = fields.Str() # TODO_SERVER: API: Deprecate
scheduled_job_count = fields.Number()
class AlertDefsSchema(ApiObjectSchema):
uid = fields.Str()
original_uid = fields.Str()
custom_name = fields.Str()
custom_description = fields.Str()
summary = fields.Str()
severity = fields.Str()
type = fields.Str()
task_name = fields.Str()
user_metric = fields.Str()
operator = fields.Str()
is_str_value = fields.Bool()
value = fields.Str()
advanced_json = fields.Str()
created_at = fields.DateTime()
scheduled_job_uid = fields.Str()
scheduled_job_name = fields.Str(attribute="scheduled_job.name")
job_name = fields.Str()
class AlertEventSchema(Schema):
name = fields.Str()
description = fields.Str()
run_uid = fields.Str()
class ScheduledJobSchemaV2(Schema):
class Meta:
strict = True
uid = fields.Str(attribute="DbndScheduledJob.uid", allow_none=True)
name = fields.Str(attribute="DbndScheduledJob.name", required=True)
cmd = fields.Str(attribute="DbndScheduledJob.cmd", required=True)
schedule_interval = fields.Str(
attribute="DbndScheduledJob.schedule_interval", required=True)
start_date = fields.DateTime(allow_none=True,
attribute="DbndScheduledJob.start_date",
format="iso")
end_date = fields.DateTime(allow_none=True,
attribute="DbndScheduledJob.end_date",
format="iso")
readable_schedule_interval = fields.Str(
attribute="DbndScheduledJob.readable_schedule_interval",
allow_none=True)
scheduled_interval_in_seconds = fields.Integer(
attribute="DbndScheduledJob.scheduled_interval_in_seconds",
allow_none=True)
catchup = fields.Boolean(allow_none=True,
attribute="DbndScheduledJob.catchup")
depends_on_past = fields.Boolean(
allow_none=True, attribute="DbndScheduledJob.depends_on_past")
retries = fields.Int(allow_none=True, attribute="DbndScheduledJob.retries")
active = fields.Boolean(allow_none=True,
attribute="DbndScheduledJob.active")
create_user = fields.Str(allow_none=True,
attribute="DbndScheduledJob.create_user")
create_time = fields.DateTime(allow_none=True,
attribute="DbndScheduledJob.create_time")
update_user = fields.Str(allow_none=True,
attribute="DbndScheduledJob.update_user")
update_time = fields.DateTime(allow_none=True,
attribute="DbndScheduledJob.update_time")
from_file = fields.Boolean(allow_none=True,
attribute="DbndScheduledJob.from_file")
deleted_from_file = fields.Boolean(
allow_none=True, attribute="DbndScheduledJob.deleted_from_file")
next_job_date = fields.DateTime(attribute="DbndScheduledJob.next_job_date",
allow_none=True)
alerts = fields.List(
fields.Nested(AlertEventSchema),
attribute="DbndScheduledJob.alerts",
allow_none=True,
)
job_name = fields.Str(dump_only=True,
attribute="DbndScheduledJob.job_name")
last_run_uid = fields.UUID(dump_only=True)
last_run_job = fields.Str(dump_only=True)
last_job_date = fields.DateTime(dump_only=True)
last_run_state = fields.Str(dump_only=True)
is_airflow_synced = fields.Bool(dump_only=True)
list_order = fields.Integer(attribute="DbndScheduledJob.list_order",
allow_none=True)
validation_errors = fields.Str(
allow_none=True, attribute="DbndScheduledJob.validation_errors")
class AlertDefsSchema(ApiStrictSchema):
severity = fields.Str(required=True)
type = fields.Str(required=True)
user_metric = fields.Str()
operator = fields.Str()
is_str_value = fields.Bool()
created_at = fields.DateTime()
scheduled_job_name = fields.Str(attribute="scheduled_job.name")
source_instance_name = fields.Method("get_tracking_source_name")
env = fields.Method("get_tracking_source_env")
# TODO_CORE: API: Deprecate airflow_server_info
airflow_instance_name = fields.Method("get_tracking_source_name")
project_id = fields.Int(attribute="job.project_id")
project_name = fields.Str(attribute="job.project.name")
alert_on_historical_runs = fields.Bool()
group_uid = fields.Str(allow_none=True, load_from="alert_group_uid")
root_group_uid = fields.Str(allow_none=True,
load_from="alert_root_group_uid")
uid = fields.Str(allow_none=True)
value = fields.Str(allow_none=True)
job_id = fields.Int(allow_none=True)
summary = fields.Str(allow_none=True)
job_name = fields.Str(attribute="job.name", allow_none=True)
task_repr = fields.Str(allow_none=True)
task_name = fields.Str(allow_none=True)
custom_name = fields.Str(allow_none=True)
original_uid = fields.Str(allow_none=True)
advanced_json = fields.Str(allow_none=True)
scheduled_job_uid = fields.Str(allow_none=True)
custom_description = fields.Str(allow_none=True)
ml_alert = fields.Nested(MLAlert, allow_none=True)
# Fields for DatasetSlaAlert/DatasetSlaAdvancedAlert alert
# --------------------------------------
seconds_delta = fields.Int(
allow_none=True) # Converts to datetime.timedelta
dataset_partial_name = fields.Str(allow_none=True)
datasets_uids = fields.List(fields.Str(), allow_none=True)
# Fields for OperationColumnStatAdvancedAlert alert
# --------------------------------------
dataset_uid = fields.Str(allow_none=True)
# Operation type (e.g. "read", "write", None=any) to filter stats by
operation_type = fields.Str(allow_none=True)
# Type of MetricRule, found in dbnd_web. Used to build advanced_json
metrics_rules = fields.List(fields.Dict(), allow_none=True)
# Used only used by the UI
affected_datasets = fields.List(fields.Dict(),
allow_none=True,
dump_only=True)
assigned_jobs = fields.Method(serialize="get_assigned_jobs",
dump_only=True)
is_system = fields.Function(lambda alert_def: alert_def.owner == "system",
dump_only=True)
def get_tracking_source_name(self, obj):
return self._get_tracking_source_instance(obj).name
def get_tracking_source_env(self, obj):
return self._get_tracking_source_instance(obj).env
def _get_tracking_source_instance(self, obj):
if obj.job:
return obj.job.tracking_source
return obj.tracking_source
@pre_load
def prepere(self, data: dict, **kwargs):
value = data.get("value", None)
if value is not None:
data["value"] = str(data["value"])
return data
def get_assigned_jobs(self, alert_def):
self_job = (alert_def.job_id, alert_def.job_name)
sub_alerts_jobs = ((sub_alert.job_id, sub_alert.job_name)
for sub_alert in alert_def.sub_alert_definitions)
alert_jobs = chain(sub_alerts_jobs, [self_job])
alert_jobs = set(filter(lambda l: l != (None, None), alert_jobs))
serialized_assigned_jobs = [{
"job_id": job_id,
"job_name": job_name
} for job_id, job_name in alert_jobs]
return serialized_assigned_jobs
