def test_get_qualified_dags_should_skip_dags_with_future_start_date(
self, mock_dag_bag):
base_dttm = pendulum_datetime(2020, 1, 2)
test_dag = DAG('test-dag', start_date=pendulum_datetime(3020, 1, 1))
mock_dag_bag.return_value.dags = {'test-dag': test_dag}
dags = get_qualified_dags(base_dttm)
assert dags == []
def test_get_qualified_dags(self, mock_dag_bag):
test_dag_1 = DAG('test-dag-1',
start_date=pendulum_datetime(2020, 1, 1))
test_dag_2 = DAG('test-dag-2',
start_date=pendulum_datetime(2020, 1, 2))
test_dag_3 = DAG('test-dag-2',
start_date=pendulum_datetime(2020, 1, 3))
mock_dag_bag.return_value.dags = {
'test-dag-1': test_dag_1,
'test-dag-2': test_dag_2,
'test-dag-3': test_dag_3,
}
base_dttm = pendulum_datetime(2020, 1, 2)
dags = get_qualified_dags(base_dttm)
assert dags == [test_dag_1, test_dag_2]
def test_get_qualified_dags_should_skip_dags_with_schedule_interval_none(
self, mock_dag_bag):
base_dttm = pendulum_datetime(2020, 1, 2)
test_dag = DAG('test-dag', schedule_interval=None)
mock_dag_bag.return_value.dags = {'test-dag': test_dag}
dags = get_qualified_dags(base_dttm)
assert dags == []
def test_get_qualified_dags_should_skip_sub_dags(self, mock_dag_bag):
base_dttm = pendulum_datetime(2020, 1, 2)
test_dag = DAG('test-dag')
test_dag.is_subdag = True
mock_dag_bag.return_value.dags = {'test-dag': test_dag}
dags = get_qualified_dags(base_dttm)
assert dags == []
def test_get_dag_schedule_for_airflow_cron_preset(self,
mock_get_qualified_dags):
from_dttm = pendulum_datetime(2020, 1, 5)
to_dttm = pendulum_datetime(2020, 1, 6)
mock_get_qualified_dags.return_value = [
DAG(dag_id='test-dag-1',
start_date=from_dttm,
schedule_interval='@weekly'),
DAG(dag_id='test-dag-2',
start_date=from_dttm,
schedule_interval='@daily'),
]
expected = {
pendulum_datetime(2020, 1, 5, 0, 0).timestamp() * 1000:
['test-dag-1', 'test-dag-2'],
pendulum_datetime(2020, 1, 6, 0, 0).timestamp() * 1000:
['test-dag-2']
}
assert get_dag_schedules(from_dttm, to_dttm) == expected
def from_pendulum_format(
datetime_string: str,
fmt: str,
tz: Optional[pendulum_timezone] = None,
locale=None,
) -> TSDatetime:
"""
Creates a DateTime instance from a specific format.
"""
subseconds = None
parts = _formatter.parse(datetime_string, fmt, now(), locale=locale)
if not isinstance(parts, dict):
raise TypeError(f"Could not match any datetime tokens in '{fmt}'.")
if parts["tz"] is None:
parts["tz"] = tz
if "S" in fmt:
subseconds = parts["microsecond"]
parts["microsecond"] = 0
ts_date_time = TSDatetime(
datetime_=pendulum_datetime(**parts), subseconds=subseconds
)
return ts_date_time
def test_get_dag_schedule_for_timedelta(self, mock_get_qualified_dags):
from_dttm = pendulum_datetime(2020, 1, 1)
to_dttm = pendulum_datetime(2020, 1, 2)
mock_get_qualified_dags.return_value = [
DAG(dag_id='test-dag-1',
start_date=from_dttm,
schedule_interval=timedelta(hours=8)),
DAG(dag_id='test-dag-2',
start_date=from_dttm,
schedule_interval=timedelta(hours=16)),
]
expected = {
pendulum_datetime(2020, 1, 1, 0).timestamp() * 1000:
['test-dag-1', 'test-dag-2'],
pendulum_datetime(2020, 1, 1, 8).timestamp() * 1000:
['test-dag-1'],
pendulum_datetime(2020, 1, 1, 16).timestamp() * 1000:
['test-dag-1', 'test-dag-2'],
pendulum_datetime(2020, 1, 2, 0).timestamp() * 1000:
['test-dag-1'],
}
assert get_dag_schedules(from_dttm, to_dttm) == expected
def test_get_dag_schedule_for_cron_expression(self,
mock_get_qualified_dags):
from_dttm = pendulum_datetime(2020, 1, 1)
to_dttm = pendulum_datetime(2020, 1, 2)
mock_get_qualified_dags.return_value = [
DAG('test-dag-1',
start_date=from_dttm,
schedule_interval='0 */8 * * *'),
DAG('test-dag-2',
start_date=from_dttm,
schedule_interval='0 */16 * * *')
]
expected = {
pendulum_datetime(2020, 1, 1, 0).timestamp() * 1000:
['test-dag-1', 'test-dag-2'],
pendulum_datetime(2020, 1, 1, 8).timestamp() * 1000:
['test-dag-1'],
pendulum_datetime(2020, 1, 1, 16).timestamp() * 1000:
['test-dag-1', 'test-dag-2'],
pendulum_datetime(2020, 1, 2, 0).timestamp() * 1000:
['test-dag-1', 'test-dag-2']
}
assert get_dag_schedules(from_dttm, to_dttm) == expected
"MM/DD/YYYY hh:mm:ss.SSSSSS A z",
"DD/MM/YYYY hh:mm:ss.SSSSSS A z",
]
unambiguous_formats = [
"dddd, MMMM Do YYYY hh:mm:ss A zz z",
"dddd, MMMM Do YYYY hh:mm:ss A zz z",
"dddd, MMMM Do YYYY hh:mm:ss A z",
"dddd, MMMM Do YYYY hh:mm:ss A zz",
"dddd, MMMM DD YYYY hh:mm:ss A z",
"dddd, MMMM Do, YYYY hh:mm:ss A z",
"dddd, MMMM DD YYYY hh:mm:ss.SSSSSS A z",
"dddd, MMMM Do, YYYY hh:mm:ss.SSSSSS A z",
]
ambiguous_datetime = pendulum_datetime(2001, 2, 3, 4, 5, 6, 7)
forgiving_ambiguous_datetime = pendulum_datetime(2001, 1, 1, 1, 1, 1, 1)
def test_unambiguous_formats():
datetime_str = "Saturday, February 3rd 2001 04:05:06 AM UTC"
parse_with_formats(datetime_str, formats=unambiguous_formats)
def test_require_unambiguous_formats_with_unambiguous_formats():
datetime_str = "Saturday, February 3rd 2001 04:05:06 AM UTC"
expected = "2001-02-03T04:05:06Z"
datetime_config = DatetimeConfig(require_unambiguous_formats=True)
result = parse_with_formats(datetime_str,
formats=unambiguous_formats,
config=datetime_config)