def add_java_args(context):
params = context['params']
fixed_duration_strategy = params['retry_extra_params'][
'fixed_duration_strategy']
interval = params['retry_extra_params']['schedule_interval']
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
if not is_execution_date_valid(execution_date, fixed_duration_strategy,
interval):
logging.info(
'The execution date {} is not the last interval of fixed duration {}.'
.format(execution_date, fixed_duration_strategy))
start_date = floor_time(execution_date,
time_delta=fixed_duration_strategy)
end_date = floor_time(execution_date + interval,
time_delta=fixed_duration_strategy)
utc_start_date = convert_to_utc(start_date)
utc_end_date = convert_to_utc(end_date)
java_args = {'start_date': utc_start_date, 'end_date': utc_end_date}
java_args = ' '.join(SpringBootJarOperator.java_args_prefix + '%s %s' %
(key, val)
for (key, val) in java_args.iteritems())
return java_args
def execute(self, context):
"""
Checks if execution_date is last interval of fixed duration, then creates java args, otherwise skip the task.
java args include start_date, end_date and fixed_duration_strategy
:raise InvalidExecutionDateError - Raise error if the execution_date is not the last interval of fixed duration.
"""
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
if not is_execution_date_valid(
execution_date, self.fixed_duration_strategy, self.interval):
# e.g: execution_date = datetime(2014, 11, 28, 13, 50, 0)
# interval = timedelta(minutes=5)
# fixed_duration = timedelta(days=1)
self.log.info(
'The execution date {} is not the last interval of fixed duration {}.'
.format(execution_date, self.fixed_duration_strategy))
start_date = floor_time(execution_date,
time_delta=self.fixed_duration_strategy)
end_date = floor_time(execution_date + self.interval,
time_delta=self.fixed_duration_strategy)
utc_start_date = convert_to_utc(start_date)
utc_end_date = convert_to_utc(end_date)
java_args = {'start_date': utc_start_date, 'end_date': utc_end_date}
super(FixedDurationJarOperator, self).update_java_args(java_args)
super(FixedDurationJarOperator, self).execute(context)
def test_weekly_round_time():
"""
Test weekly round_time method
:return:
"""
logging.info('Test weekly round time method')
assert floor_time(DEFAULT_DATE, timedelta(weeks=1)) == datetime(2014, 11, 27, 0, 0, 0)
def test_hourly_round_time():
"""
Test hourly round_time method
:return:
"""
logging.info('Test hourly round time method')
assert floor_time(DEFAULT_DATE, timedelta(hours=1)) == datetime(2014, 11, 28, 10, 0, 0)
def is_execution_date_valid(date_time, fixed_duration_strategy, interval):
"""
If interval greater than fixed_duration the date_time is valid,
Otherwise checks whether the given date_time is the last interval of fixed_duration_strategy
e.g:
valid date_time:
date_time = datetime(2014, 11, 28, 13, 55, 0)
interval = timedelta(minutes=5)
fixed_duration = timedelta(days=1)
invalid date_time:
date_time = datetime(2014, 11, 28, 13, 40, 0)
interval = timedelta(minutes=5)
fixed_duration = timedelta(days=1)
:param date_time: The timestamp to check
:type date_time: datetime.datetime
:param fixed_duration_strategy:
:type fixed_duration_strategy: datetime.timedelta
:param interval:
:type interval: datetime.timedelta
:return: boolean
"""
is_last = True
interval = interval.total_seconds()
if interval < fixed_duration_strategy.total_seconds():
date = datetime_to_epoch(date_time)
date_round = floor_time(date_time, time_delta=fixed_duration_strategy)
time = datetime_to_epoch(
date_round) + fixed_duration_strategy.total_seconds() - interval
is_last = date >= time
return is_last
def _get_input_pre_processor_arguments(self, context):
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
arguments = self.static_arguments.copy()
if START_INSTANCE_DYNAMIC_ARGUMENT in self.dynamic_arguments:
start_date = floor_time(execution_date,
time_delta=FIX_DURATION_STRATEGY_DAILY)
utc_start_date = convert_to_utc(start_date)
arguments.update({START_INSTANCE_DYNAMIC_ARGUMENT: utc_start_date})
if END_INSTANCE_DYNAMIC_ARGUMENT in self.dynamic_arguments:
end_date = floor_time(execution_date + timedelta(days=1),
time_delta=FIX_DURATION_STRATEGY_DAILY)
utc_end_date = convert_to_utc(end_date)
arguments.update({END_INSTANCE_DYNAMIC_ARGUMENT: utc_end_date})
return arguments
def poke(self, context):
'''
@return: bool - whether there are tasks to wait for.
'''
execution_date = context['execution_date']
execution_date = floor_time(
execution_date,
time_delta=self.fixed_duration_strategy) - self.interval
execution_date = execution_date.replace(tzinfo=pytz.utc)
self.log.info('Poking for all dag instances of '
'{self._dag_ids} with time '
'execution_date ... '.format(**locals()))
return self._is_finished_wait_for_gapped_dag(execution_date)
