def test_make_naive(self): self.assertEqual( timezone.make_naive(datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT), EAT), datetime.datetime(2011, 9, 1, 13, 20, 30)) self.assertEqual( timezone.make_naive(datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=ICT), EAT), datetime.datetime(2011, 9, 1, 13, 20, 30)) with self.assertRaises(ValueError): timezone.make_naive(datetime.datetime(2011, 9, 1, 13, 20, 30), EAT)
def test_make_naive(self): assert timezone.make_naive( datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT), EAT ) == datetime.datetime(2011, 9, 1, 13, 20, 30) assert timezone.make_naive( datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=ICT), EAT ) == datetime.datetime(2011, 9, 1, 13, 20, 30) with pytest.raises(ValueError): timezone.make_naive(datetime.datetime(2011, 9, 1, 13, 20, 30), EAT)
def test_make_naive(self): self.assertEqual( timezone.make_naive( datetime.datetime(2011, 9, 1, 13, 20, 30, tzinfo=EAT), EAT), datetime.datetime(2011, 9, 1, 13, 20, 30)) self.assertEqual( timezone.make_naive( datetime.datetime(2011, 9, 1, 17, 20, 30, tzinfo=ICT), EAT), datetime.datetime(2011, 9, 1, 13, 20, 30)) with self.assertRaises(ValueError): timezone.make_naive(datetime.datetime(2011, 9, 1, 13, 20, 30), EAT)
def choose_branch(self, context: Context) -> Union[str, Iterable[str]]: if self.use_task_execution_date is True: now = timezone.make_naive(context["logical_date"], self.dag.timezone) else: now = timezone.make_naive(timezone.utcnow(), self.dag.timezone) lower, upper = target_times_as_dates(now, self.target_lower, self.target_upper) if upper is not None and upper < now: return self.follow_task_ids_if_false if lower is not None and lower > now: return self.follow_task_ids_if_false return self.follow_task_ids_if_true
def test_set_dag_run_state_edge_cases(self): # Dag does not exist altered = set_dag_run_state_to_success(None, self.execution_dates[0]) assert len(altered) == 0 altered = set_dag_run_state_to_failed(None, self.execution_dates[0]) assert len(altered) == 0 altered = set_dag_run_state_to_running(None, self.execution_dates[0]) assert len(altered) == 0 # Invalid execution date altered = set_dag_run_state_to_success(self.dag1, None) assert len(altered) == 0 altered = set_dag_run_state_to_failed(self.dag1, None) assert len(altered) == 0 altered = set_dag_run_state_to_running(self.dag1, None) assert len(altered) == 0 # This will throw ValueError since dag.latest_execution_date # need to be 0 does not exist. with pytest.raises(ValueError): set_dag_run_state_to_success( self.dag2, timezone.make_naive(self.execution_dates[0])) # altered = set_dag_run_state_to_success(self.dag1, self.execution_dates[0]) # DagRun does not exist # This will throw ValueError since dag.latest_execution_date does not exist with pytest.raises(ValueError): set_dag_run_state_to_success(self.dag2, self.execution_dates[0])
def test_set_dag_run_state_edge_cases(self): # Dag does not exist altered = set_dag_run_state_to_success(None, self.execution_dates[0]) self.assertEqual(len(altered), 0) altered = set_dag_run_state_to_failed(None, self.execution_dates[0]) self.assertEqual(len(altered), 0) altered = set_dag_run_state_to_running(None, self.execution_dates[0]) self.assertEqual(len(altered), 0) # Invalid execution date altered = set_dag_run_state_to_success(self.dag1, None) self.assertEqual(len(altered), 0) altered = set_dag_run_state_to_failed(self.dag1, None) self.assertEqual(len(altered), 0) altered = set_dag_run_state_to_running(self.dag1, None) self.assertEqual(len(altered), 0) # This will throw ValueError since dag.latest_execution_date # need to be 0 does not exist. self.assertRaises(ValueError, set_dag_run_state_to_success, self.dag2, timezone.make_naive(self.execution_dates[0])) # altered = set_dag_run_state_to_success(self.dag1, self.execution_dates[0]) # DagRun does not exist # This will throw ValueError since dag.latest_execution_date does not exist self.assertRaises(ValueError, set_dag_run_state_to_success, self.dag2, self.execution_dates[0])
def test_set_dag_run_state_edge_cases(self): # Dag does not exist altered = set_dag_run_state_to_success(None, self.execution_dates[0]) self.assertEqual(len(altered), 0) altered = set_dag_run_state_to_failed(None, self.execution_dates[0]) self.assertEqual(len(altered), 0) altered = set_dag_run_state_to_running(None, self.execution_dates[0]) self.assertEqual(len(altered), 0) # Invalid execution date altered = set_dag_run_state_to_success(self.dag1, None) self.assertEqual(len(altered), 0) altered = set_dag_run_state_to_failed(self.dag1, None) self.assertEqual(len(altered), 0) altered = set_dag_run_state_to_running(self.dag1, None) self.assertEqual(len(altered), 0) # This will throw AssertionError since dag.latest_execution_date # need to be 0 does not exist. self.assertRaises(AssertionError, set_dag_run_state_to_success, self.dag1, timezone.make_naive(self.execution_dates[0])) # altered = set_dag_run_state_to_success(self.dag1, self.execution_dates[0]) # DagRun does not exist # This will throw AssertionError since dag.latest_execution_date does not exist self.assertRaises(AssertionError, set_dag_run_state_to_success, self.dag1, self.execution_dates[0])
def __init__(self, *, target_time, **kwargs): super().__init__(**kwargs) self.target_time = target_time current_time = timezone.make_naive(timezone.utcnow(), self.dag.timezone) todays_date = current_time.date() self.target_datetime = datetime.datetime.combine( todays_date, self.target_time, current_time.tzinfo)
def choose_branch(self, context: Context) -> Union[str, Iterable[str]]: if self.use_task_execution_day: now = context["logical_date"] else: now = timezone.make_naive(timezone.utcnow(), self.dag.timezone) if now.isoweekday() in self._week_day_num: return self.follow_task_ids_if_true return self.follow_task_ids_if_false
def _get_prev(self, current: DateTime) -> DateTime: """Get the first schedule before specified time, with DST fixed.""" naive = make_naive(current, self._timezone) cron = croniter(self._expression, start_time=naive) scheduled = cron.get_prev(datetime.datetime) if not self._should_fix_dst: return convert_to_utc(make_aware(scheduled, self._timezone)) delta = naive - scheduled return convert_to_utc(current.in_timezone(self._timezone) - delta)
def process_bind_param(self, value, dialect): if value is not None: if not isinstance(value, datetime.datetime): raise TypeError('expected datetime.datetime, not ' + repr(value)) elif value.tzinfo is None: raise ValueError('naive datetime is disallowed') # For mysql we should store timestamps as naive values # Timestamp in MYSQL is not timezone aware. In MySQL 5.6 # timezone added at the end is ignored but in MySQL 5.7 # inserting timezone value fails with 'invalid-date' # See https://issues.apache.org/jira/browse/AIRFLOW-7001 if using_mysql: from airflow.utils.timezone import make_naive return make_naive(value, timezone=utc) return value.astimezone(utc)
def date_range(start_date, end_date=None, num=None, delta=None): """ Get a set of dates as a list based on a start, end and delta, delta can be something that can be added to ``datetime.datetime`` or a cron expression as a ``str`` :param start_date: anchor date to start the series from :type start_date: datetime.datetime :param end_date: right boundary for the date range :type end_date: datetime.datetime :param num: alternatively to end_date, you can specify the number of number of entries you want in the range. This number can be negative, output will always be sorted regardless :type num: int >>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1)) [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0), datetime.datetime(2016, 1, 3, 0, 0)] >>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta='0 0 * * *') [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0), datetime.datetime(2016, 1, 3, 0, 0)] >>> date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *") [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0), datetime.datetime(2016, 3, 1, 0, 0)] """ if not delta: return [] if end_date and start_date > end_date: raise Exception("Wait. start_date needs to be before end_date") if end_date and num: raise Exception("Wait. Either specify end_date OR num") if not end_date and not num: end_date = timezone.utcnow() delta_iscron = False if isinstance(delta, six.string_types): delta_iscron = True tz = start_date.tzinfo start_date = timezone.make_naive(start_date, tz) cron = croniter(delta, start_date) elif isinstance(delta, timedelta): delta = abs(delta) l = [] if end_date: while start_date <= end_date: if timezone.is_naive(start_date): l.append(timezone.make_aware(start_date, tz)) else: l.append(start_date) if delta_iscron: start_date = cron.get_next(datetime) else: start_date += delta else: for _ in range(abs(num)): if timezone.is_naive(start_date): l.append(timezone.make_aware(start_date, tz)) else: l.append(start_date) if delta_iscron: if num > 0: start_date = cron.get_next(datetime) else: start_date = cron.get_prev(datetime) else: if num > 0: start_date += delta else: start_date -= delta return sorted(l)
def round_time(dt, delta, start_date=timezone.make_aware(datetime.min)): """ Returns the datetime of the form start_date + i * delta which is closest to dt for any non-negative integer i. Note that delta may be a datetime.timedelta or a dateutil.relativedelta >>> round_time(datetime(2015, 1, 1, 6), timedelta(days=1)) datetime.datetime(2015, 1, 1, 0, 0) >>> round_time(datetime(2015, 1, 2), relativedelta(months=1)) datetime.datetime(2015, 1, 1, 0, 0) >>> round_time(datetime(2015, 9, 16, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0)) datetime.datetime(2015, 9, 16, 0, 0) >>> round_time(datetime(2015, 9, 15, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0)) datetime.datetime(2015, 9, 15, 0, 0) >>> round_time(datetime(2015, 9, 14, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0)) datetime.datetime(2015, 9, 14, 0, 0) >>> round_time(datetime(2015, 9, 13, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0)) datetime.datetime(2015, 9, 14, 0, 0) """ if isinstance(delta, six.string_types): # It's cron based, so it's easy tz = start_date.tzinfo start_date = timezone.make_naive(start_date, tz) cron = croniter(delta, start_date) prev = cron.get_prev(datetime) if prev == start_date: return timezone.make_aware(start_date, tz) else: return timezone.make_aware(prev, tz) # Ignore the microseconds of dt dt -= timedelta(microseconds=dt.microsecond) # We are looking for a datetime in the form start_date + i * delta # which is as close as possible to dt. Since delta could be a relative # delta we don't know it's exact length in seconds so we cannot rely on # division to find i. Instead we employ a binary search algorithm, first # finding an upper and lower limit and then disecting the interval until # we have found the closest match. # We first search an upper limit for i for which start_date + upper * delta # exceeds dt. upper = 1 while start_date + upper * delta < dt: # To speed up finding an upper limit we grow this exponentially by a # factor of 2 upper *= 2 # Since upper is the first value for which start_date + upper * delta # exceeds dt, upper // 2 is below dt and therefore forms a lower limited # for the i we are looking for lower = upper // 2 # We now continue to intersect the interval between # start_date + lower * delta and start_date + upper * delta # until we find the closest value while True: # Invariant: start + lower * delta < dt <= start + upper * delta # If start_date + (lower + 1)*delta exceeds dt, then either lower or # lower+1 has to be the solution we are searching for if start_date + (lower + 1) * delta >= dt: # Check if start_date + (lower + 1)*delta or # start_date + lower*delta is closer to dt and return the solution if ((start_date + (lower + 1) * delta) - dt <= dt - (start_date + lower * delta)): return start_date + (lower + 1) * delta else: return start_date + lower * delta # We intersect the interval and either replace the lower or upper # limit with the candidate candidate = lower + (upper - lower) // 2 if start_date + candidate * delta >= dt: upper = candidate else: lower = candidate
def round_time(dt, delta, start_date=timezone.make_aware(datetime.min)): """ Returns the datetime of the form start_date + i * delta which is closest to dt for any non-negative integer i. Note that delta may be a datetime.timedelta or a dateutil.relativedelta >>> round_time(datetime(2015, 1, 1, 6), timedelta(days=1)) datetime.datetime(2015, 1, 1, 0, 0) >>> round_time(datetime(2015, 1, 2), relativedelta(months=1)) datetime.datetime(2015, 1, 1, 0, 0) >>> round_time(datetime(2015, 9, 16, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0)) datetime.datetime(2015, 9, 16, 0, 0) >>> round_time(datetime(2015, 9, 15, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0)) datetime.datetime(2015, 9, 15, 0, 0) >>> round_time(datetime(2015, 9, 14, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0)) datetime.datetime(2015, 9, 14, 0, 0) >>> round_time(datetime(2015, 9, 13, 0, 0), timedelta(1), datetime(2015, 9, 14, 0, 0)) datetime.datetime(2015, 9, 14, 0, 0) """ if isinstance(delta, six.string_types): # It's cron based, so it's easy tz = start_date.tzinfo start_date = timezone.make_naive(start_date, tz) cron = croniter(delta, start_date) prev = cron.get_prev(datetime) if prev == start_date: return timezone.make_aware(start_date, tz) else: return timezone.make_aware(prev, tz) # Ignore the microseconds of dt dt -= timedelta(microseconds=dt.microsecond) # We are looking for a datetime in the form start_date + i * delta # which is as close as possible to dt. Since delta could be a relative # delta we don't know its exact length in seconds so we cannot rely on # division to find i. Instead we employ a binary search algorithm, first # finding an upper and lower limit and then disecting the interval until # we have found the closest match. # We first search an upper limit for i for which start_date + upper * delta # exceeds dt. upper = 1 while start_date + upper * delta < dt: # To speed up finding an upper limit we grow this exponentially by a # factor of 2 upper *= 2 # Since upper is the first value for which start_date + upper * delta # exceeds dt, upper // 2 is below dt and therefore forms a lower limited # for the i we are looking for lower = upper // 2 # We now continue to intersect the interval between # start_date + lower * delta and start_date + upper * delta # until we find the closest value while True: # Invariant: start + lower * delta < dt <= start + upper * delta # If start_date + (lower + 1)*delta exceeds dt, then either lower or # lower+1 has to be the solution we are searching for if start_date + (lower + 1) * delta >= dt: # Check if start_date + (lower + 1)*delta or # start_date + lower*delta is closer to dt and return the solution if ( (start_date + (lower + 1) * delta) - dt <= dt - (start_date + lower * delta)): return start_date + (lower + 1) * delta else: return start_date + lower * delta # We intersect the interval and either replace the lower or upper # limit with the candidate candidate = lower + (upper - lower) // 2 if start_date + candidate * delta >= dt: upper = candidate else: lower = candidate
def date_range( start_date: datetime, end_date: Optional[datetime] = None, num: Optional[int] = None, delta: Optional[Union[str, timedelta, relativedelta]] = None, ) -> List[datetime]: """ Get a set of dates as a list based on a start, end and delta, delta can be something that can be added to `datetime.datetime` or a cron expression as a `str` .. code-block:: pycon >>> from airflow.utils.dates import datterange >>> from datetime import datetime, timedelta >>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1)) [datetime.datetime(2016, 1, 1, 0, 0, tzinfo=Timezone('UTC')), datetime.datetime(2016, 1, 2, 0, 0, tzinfo=Timezone('UTC')), datetime.datetime(2016, 1, 3, 0, 0, tzinfo=Timezone('UTC'))] >>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta="0 0 * * *") [datetime.datetime(2016, 1, 1, 0, 0, tzinfo=Timezone('UTC')), datetime.datetime(2016, 1, 2, 0, 0, tzinfo=Timezone('UTC')), datetime.datetime(2016, 1, 3, 0, 0, tzinfo=Timezone('UTC'))] >>> date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *") [datetime.datetime(2016, 1, 1, 0, 0, tzinfo=Timezone('UTC')), datetime.datetime(2016, 2, 1, 0, 0, tzinfo=Timezone('UTC')), datetime.datetime(2016, 3, 1, 0, 0, tzinfo=Timezone('UTC'))] :param start_date: anchor date to start the series from :param end_date: right boundary for the date range :param num: alternatively to end_date, you can specify the number of number of entries you want in the range. This number can be negative, output will always be sorted regardless :param delta: step length. It can be datetime.timedelta or cron expression as string """ warnings.warn( "`airflow.utils.dates.date_range()` is deprecated. Please use `airflow.timetables`.", category=DeprecationWarning, stacklevel=2, ) if not delta: return [] if end_date: if start_date > end_date: raise Exception("Wait. start_date needs to be before end_date") if num: raise Exception("Wait. Either specify end_date OR num") if not end_date and not num: end_date = timezone.utcnow() delta_iscron = False time_zone = start_date.tzinfo abs_delta: Union[timedelta, relativedelta] if isinstance(delta, str): delta_iscron = True if timezone.is_localized(start_date): start_date = timezone.make_naive(start_date, time_zone) cron = croniter(cron_presets.get(delta, delta), start_date) elif isinstance(delta, timedelta): abs_delta = abs(delta) elif isinstance(delta, relativedelta): abs_delta = abs(delta) else: raise Exception( "Wait. delta must be either datetime.timedelta or cron expression as str" ) dates = [] if end_date: if timezone.is_naive(start_date) and not timezone.is_naive(end_date): end_date = timezone.make_naive(end_date, time_zone) while start_date <= end_date: # type: ignore if timezone.is_naive(start_date): dates.append(timezone.make_aware(start_date, time_zone)) else: dates.append(start_date) if delta_iscron: start_date = cron.get_next(datetime) else: start_date += abs_delta else: num_entries: int = num # type: ignore for _ in range(abs(num_entries)): if timezone.is_naive(start_date): dates.append(timezone.make_aware(start_date, time_zone)) else: dates.append(start_date) if delta_iscron and num_entries > 0: start_date = cron.get_next(datetime) elif delta_iscron: start_date = cron.get_prev(datetime) elif num_entries > 0: start_date += abs_delta else: start_date -= abs_delta return sorted(dates)
def date_range( start_date, end_date=None, num=None, delta=None): """ Get a set of dates as a list based on a start, end and delta, delta can be something that can be added to ``datetime.datetime`` or a cron expression as a ``str`` :param start_date: anchor date to start the series from :type start_date: datetime.datetime :param end_date: right boundary for the date range :type end_date: datetime.datetime :param num: alternatively to end_date, you can specify the number of number of entries you want in the range. This number can be negative, output will always be sorted regardless :type num: int >>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1)) [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0), datetime.datetime(2016, 1, 3, 0, 0)] >>> date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta='0 0 * * *') [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0), datetime.datetime(2016, 1, 3, 0, 0)] >>> date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *") [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0), datetime.datetime(2016, 3, 1, 0, 0)] """ if not delta: return [] if end_date and start_date > end_date: raise Exception("Wait. start_date needs to be before end_date") if end_date and num: raise Exception("Wait. Either specify end_date OR num") if not end_date and not num: end_date = timezone.utcnow() delta_iscron = False tz = start_date.tzinfo if isinstance(delta, six.string_types): delta_iscron = True start_date = timezone.make_naive(start_date, tz) cron = croniter(delta, start_date) elif isinstance(delta, timedelta): delta = abs(delta) dates = [] if end_date: if timezone.is_naive(start_date): end_date = timezone.make_naive(end_date, tz) while start_date <= end_date: if timezone.is_naive(start_date): dates.append(timezone.make_aware(start_date, tz)) else: dates.append(start_date) if delta_iscron: start_date = cron.get_next(datetime) else: start_date += delta else: for _ in range(abs(num)): if timezone.is_naive(start_date): dates.append(timezone.make_aware(start_date, tz)) else: dates.append(start_date) if delta_iscron: if num > 0: start_date = cron.get_next(datetime) else: start_date = cron.get_prev(datetime) else: if num > 0: start_date += delta else: start_date -= delta return sorted(dates)
def poke(self, context: Context): self.log.info('Checking if the time (%s) has come', self.target_time) return timezone.make_naive(timezone.utcnow(), self.dag.timezone).time() > self.target_time
def date_range(start_date, end_date=None, num=None, delta=None): # pylint: disable=too-many-branches """ Get a set of dates as a list based on a start, end and delta, delta can be something that can be added to `datetime.datetime` or a cron expression as a `str` .. code-block:: python date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta=timedelta(1)) [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0), datetime.datetime(2016, 1, 3, 0, 0)] date_range(datetime(2016, 1, 1), datetime(2016, 1, 3), delta='0 0 * * *') [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 1, 2, 0, 0), datetime.datetime(2016, 1, 3, 0, 0)] date_range(datetime(2016, 1, 1), datetime(2016, 3, 3), delta="0 0 0 * *") [datetime.datetime(2016, 1, 1, 0, 0), datetime.datetime(2016, 2, 1, 0, 0), datetime.datetime(2016, 3, 1, 0, 0)] :param start_date: anchor date to start the series from :type start_date: datetime.datetime :param end_date: right boundary for the date range :type end_date: datetime.datetime :param num: alternatively to end_date, you can specify the number of number of entries you want in the range. This number can be negative, output will always be sorted regardless :type num: int :param delta: step length. It can be datetime.timedelta or cron expression as string :type delta: datetime.timedelta or str """ if not delta: return [] if end_date and start_date > end_date: raise Exception("Wait. start_date needs to be before end_date") if end_date and num: raise Exception("Wait. Either specify end_date OR num") if not end_date and not num: end_date = timezone.utcnow() if delta in cron_presets: delta = cron_presets.get(delta) delta_iscron = False time_zone = start_date.tzinfo if isinstance(delta, str): delta_iscron = True if timezone.is_localized(start_date): start_date = timezone.make_naive(start_date, time_zone) cron = croniter(delta, start_date) elif isinstance(delta, timedelta): delta = abs(delta) else: raise Exception( "Wait. delta must be either datetime.timedelta or cron expression as str" ) dates = [] if end_date: if timezone.is_naive(start_date) and not timezone.is_naive(end_date): end_date = timezone.make_naive(end_date, time_zone) while start_date <= end_date: if timezone.is_naive(start_date): dates.append(timezone.make_aware(start_date, time_zone)) else: dates.append(start_date) if delta_iscron: start_date = cron.get_next(datetime) else: start_date += delta else: for _ in range(abs(num)): if timezone.is_naive(start_date): dates.append(timezone.make_aware(start_date, time_zone)) else: dates.append(start_date) if delta_iscron and num > 0: start_date = cron.get_next(datetime) elif delta_iscron: start_date = cron.get_prev(datetime) elif num > 0: start_date += delta else: start_date -= delta return sorted(dates)