def job_to_dag(self, job): # type: (dict) -> Union[DAG, None]
start_day = convert_to_utc(job.get("start_date", None))
end_date = convert_to_utc(job.get("end_date", None))
default_args = {
"owner": job.get("create_user", None),
"depends_on_past": job.get("depends_on_past", False),
"start_date": job["start_date"],
"end_date": end_date,
}
job_name = clean_job_name(job["name"])
dag = DAG(
"dbnd_launcher__%s" % job_name,
start_date=start_day,
default_args=default_args,
schedule_interval=job.get("schedule_interval", None),
catchup=job.get("catchup", False),
)
DbndSchedulerOperator(
scheduled_cmd=job["cmd"],
scheduled_job_name=job_name,
scheduled_job_uid=job.get("uid", None),
shell=config.getboolean("scheduler", "shell_cmd"),
task_id="launcher",
dag=dag,
retries=job.get("retries", self.default_retries)
or self.default_retries,
)
return dag
def job_to_dag(self, job): # type: (dict) -> Union[DAG, None]
# convert_to_utc usage might be dangerous, as there is the same function at airflow
# however, that one use pendulum not from _vendorized
default_args = {}
if job.get("depends_on_past"):
default_args["depends_on_past"] = job.get("depends_on_past")
start_date = convert_to_utc(job.get("start_date"))
if start_date:
default_args["start_day"] = start_date
if job.get("end_date"):
default_args["end_date"] = convert_to_utc(job.get("end_date"))
if job.get("owner"):
default_args["owner"] = job.get("create_user")
job_name = clean_job_name(job["name"])
dag = DAG(
"%s" % job_name,
start_date=start_date,
default_args=default_args,
schedule_interval=job.get("schedule_interval", None),
catchup=job.get("catchup", False),
)
custom_operator_class = self.custom_operator_class or DbndSchedulerOperator
custom_operator_class(
scheduled_cmd=job["cmd"],
scheduled_job_name=job_name,
extra_args=job.get("extra_args", None),
with_name=False,
scheduled_job_uid=job.get("uid", None),
shell=config.getboolean("scheduler", "shell_cmd"),
task_id="launcher",
dag=dag,
retries=job.get("retries") or self.default_retries,
)
return dag
def job_to_dag(self, job): # type: (dict) -> Union[DAG, None]
default_args = {}
if job.get("depends_on_past"):
default_args["depends_on_past"] = job.get("depends_on_past")
start_date = convert_to_utc(job.get("start_date"))
if start_date:
default_args["start_day"] = start_date
if job.get("end_date"):
default_args["end_date"] = convert_to_utc(job.get("end_date"))
if job.get("owner"):
default_args["owner"] = job.get("owner")
job_name = clean_job_name(job["name"])
dag = DAG(
"%s" % job_name,
start_date=start_date,
default_args=default_args,
schedule_interval=job.get("schedule_interval", None),
catchup=job.get("catchup", False),
)
DbndSchedulerOperator(
task_id="launcher",
dag=dag,
retries=job.get("retries") or self.default_retries,
scheduled_cmd=job["cmd"],
scheduled_job_name=job_name,
with_name=False,
scheduled_job_uid=job.get("uid", None),
shell=config.getboolean("scheduler", "shell_cmd"),
)
return dag
def test_convert_to_utc_with_00_00(self):
target = datetime.datetime.now()
converted_value = convert_to_utc(target)
converted_again = convert_to_utc(converted_value)
converted_again = convert_to_utc(converted_again)
assert converted_again