def add_java_args(context):
params = context['params']
fixed_duration_strategy = params['retry_extra_params'][
'fixed_duration_strategy']
interval = params['retry_extra_params']['schedule_interval']
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
if not is_execution_date_valid(execution_date, fixed_duration_strategy,
interval):
logging.info(
'The execution date {} is not the last interval of fixed duration {}.'
.format(execution_date, fixed_duration_strategy))
start_date = floor_time(execution_date,
time_delta=fixed_duration_strategy)
end_date = floor_time(execution_date + interval,
time_delta=fixed_duration_strategy)
utc_start_date = convert_to_utc(start_date)
utc_end_date = convert_to_utc(end_date)
java_args = {'start_date': utc_start_date, 'end_date': utc_end_date}
java_args = ' '.join(SpringBootJarOperator.java_args_prefix + '%s %s' %
(key, val)
for (key, val) in java_args.iteritems())
return java_args
def execute(self, context):
"""
Checks if execution_date is last interval of fixed duration, then creates java args, otherwise skip the task.
java args include start_date, end_date and fixed_duration_strategy
:raise InvalidExecutionDateError - Raise error if the execution_date is not the last interval of fixed duration.
"""
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
if not is_execution_date_valid(
execution_date, self.fixed_duration_strategy, self.interval):
# e.g: execution_date = datetime(2014, 11, 28, 13, 50, 0)
# interval = timedelta(minutes=5)
# fixed_duration = timedelta(days=1)
self.log.info(
'The execution date {} is not the last interval of fixed duration {}.'
.format(execution_date, self.fixed_duration_strategy))
start_date = floor_time(execution_date,
time_delta=self.fixed_duration_strategy)
end_date = floor_time(execution_date + self.interval,
time_delta=self.fixed_duration_strategy)
utc_start_date = convert_to_utc(start_date)
utc_end_date = convert_to_utc(end_date)
java_args = {'start_date': utc_start_date, 'end_date': utc_end_date}
super(FixedDurationJarOperator, self).update_java_args(java_args)
super(FixedDurationJarOperator, self).execute(context)
def test_convert_to_utc_pacific_tz():
"""
Test convert_to_utc method
:return:
"""
logging.info('Test convert_to_utc method')
date = datetime(2014, 5, 13, 13, 56, 2, tzinfo=pytz.timezone("US/Pacific"))
with pytest.raises(Exception):
convert_to_utc(date)
def get_java_args(self, context):
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
start_date = execution_date
end_date = execution_date + self.interval
java_args = {
'start_date': convert_to_utc(start_date),
'end_date': convert_to_utc(end_date)
}
return java_args
def test_convert_to_utc_no_tz():
"""
Test convert_to_utc method
:return:
"""
logging.info('Test convert_to_utc method')
date = datetime(2014, 5, 13, 13, 56, 2)
assert convert_to_utc(date) == '2014-05-13T13:56:02Z'
def test_convert_to_utc_with_utc():
"""
Test convert_to_utc method
:return:
"""
logging.info('Test convert_to_utc method')
date = datetime(2014, 5, 13, 13, 56, 2, tzinfo=pytz.utc)
assert convert_to_utc(date) == '2014-05-13T13:56:02Z'
def _get_input_pre_processor_arguments(self, context):
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
arguments = self.static_arguments.copy()
if START_INSTANCE_DYNAMIC_ARGUMENT in self.dynamic_arguments:
start_date = floor_time(execution_date,
time_delta=FIX_DURATION_STRATEGY_DAILY)
utc_start_date = convert_to_utc(start_date)
arguments.update({START_INSTANCE_DYNAMIC_ARGUMENT: utc_start_date})
if END_INSTANCE_DYNAMIC_ARGUMENT in self.dynamic_arguments:
end_date = floor_time(execution_date + timedelta(days=1),
time_delta=FIX_DURATION_STRATEGY_DAILY)
utc_end_date = convert_to_utc(end_date)
arguments.update({END_INSTANCE_DYNAMIC_ARGUMENT: utc_end_date})
return arguments
def add_java_args(context):
params = context['params']
interval = params['retry_extra_params']['schedule_interval']
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
end_date = execution_date + interval
java_args = {
'end_date': convert_to_utc(end_date)
}
java_args = ' '.join(
SpringBootJarOperator.java_args_prefix + '%s %s' % (key, val) for (key, val) in java_args.iteritems()
)
return java_args
def add_java_args(self, context):
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
end_date = execution_date
java_args = {
'end_date': convert_to_utc(end_date)
}
java_args = ' '.join(
SpringBootJarOperator.java_args_prefix + '%s %s' % (key, val) for (key, val) in java_args.iteritems()
)
return java_args
def execute(self, context):
"""
Add end_date to java_args
"""
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
until_date = execution_date
java_args = {
'end_date': convert_to_utc(until_date),
}
super(RetentionOperator, self).update_java_args(java_args)
super(RetentionOperator, self).execute(context)
def execute(self, context):
"""
Add end_date to java_args
"""
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
end_date = execution_date + +timedelta(hours=1)
java_args = {'end_date': convert_to_utc(end_date)}
super(HourIsReadyAccordingToS3NWGatewaySensorOperator,
self).update_java_args(java_args)
super(HourIsReadyAccordingToS3NWGatewaySensorOperator,
self).execute(context)
def execute(self, context):
"""
Runs the model jar with end date which equals to execution_date + schedule_interval
:raise InvalidExecutionDateError - Raise error if the execution_date is not the last interval of fixed duration.
"""
context_wrapper = ContextWrapper(context)
execution_date = context_wrapper.get_execution_date()
end_date = execution_date + timedelta(hours=1)
java_args = {
'end_date': convert_to_utc(end_date)
}
super(ModelOperator, self).update_java_args(java_args)
super(ModelOperator, self).execute(context)
