def test_scheduler_multi_instance(self, method):
method.side_effect = self.target_method
second_scheduler = legacy_scheduler.LegacyScheduler(CONF.scheduler)
second_scheduler.start()
self.addCleanup(second_scheduler.stop, True)
job = sched_base.SchedulerJob(
run_after=DELAY,
func_name=TARGET_METHOD_PATH,
func_args={
'name': 'task',
'id': '321'
},
)
second_scheduler.schedule(job)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
self.queue.get()
method.assert_called_once_with(name='task', id='321')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_without_factory(self, method):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
FACTORY_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
call = self._assert_single_item(
calls,
target_method_name=FACTORY_METHOD_PATH
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
method.assert_called_once_with(name='task', id='321')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_with_factory(self, factory):
target_method_name = 'run_something'
factory.return_value = type('something', (object, ), {
target_method_name:
mock.MagicMock(side_effect=self.target_method)
})
job = sched_base.SchedulerJob(
run_after=DELAY,
target_factory_func_name=TARGET_METHOD_PATH,
func_name=target_method_name,
func_args={
'name': 'task',
'id': '123'
})
self.scheduler.schedule(job)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(calls,
target_method_name=target_method_name)
self.assertIn('name', call['method_arguments'])
self.queue.get()
factory().run_something.assert_called_once_with(name='task', id='123')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_without_factory(self, method):
method.side_effect = self.target_method
job = sched_base.SchedulerJob(run_after=DELAY,
func_name=TARGET_METHOD_PATH,
func_args={
'name': 'task',
'id': '321'
},
key='my_job_key')
self.scheduler.schedule(job)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(calls,
target_method_name=TARGET_METHOD_PATH,
key='my_job_key')
self.assertIn('name', call['method_arguments'])
self.queue.get()
method.assert_called_once_with(name='task', id='321')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_without_factory(self, method):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
FACTORY_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
call = self._assert_single_item(
calls,
target_method_name=FACTORY_METHOD_PATH
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
method.assert_called_once_with(name='task', id='321')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_with_serializer(self, factory):
target_method_name = 'run_something'
factory.return_value = type('something', (object, ), {
target_method_name:
mock.MagicMock(side_effect=self.target_method)
})
task_result = ml_actions.Result('data', 'error')
method_args = {'name': 'task', 'id': '123', 'result': task_result}
serializers = {'result': 'mistral.workflow.utils.ResultSerializer'}
scheduler.schedule_call(TARGET_METHOD_PATH,
target_method_name,
DELAY,
serializers=serializers,
**method_args)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(calls,
target_method_name=target_method_name)
self.assertIn('name', call['method_arguments'])
self.queue.get()
result = factory().run_something.call_args[1].get('result')
self.assertIsInstance(result, ml_actions.Result)
self.assertEqual('data', result.data)
self.assertEqual('error', result.error)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_with_factory(self, factory):
target_method = 'run_something'
method_args = {'name': 'task', 'id': '123'}
scheduler.schedule_call(
FACTORY_METHOD_PATH,
target_method,
DELAY,
**method_args
)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=2)
)
call = self._assert_single_item(
calls,
target_method_name=target_method
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
factory().run_something.assert_called_once_with(name='task', id='123')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_with_factory(self, factory):
target_method_name = 'run_something'
factory.return_value = type(
'something',
(object,),
{
target_method_name:
mock.MagicMock(side_effect=self.target_method)
}
)
scheduler.schedule_call(
TARGET_METHOD_PATH,
target_method_name,
DELAY,
**{'name': 'task', 'id': '123'}
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(
calls,
target_method_name=target_method_name
)
self.assertIn('name', call['method_arguments'])
self.queue.get()
factory().run_something.assert_called_once_with(name='task', id='123')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_multi_instance(self, method):
def stop_thread_groups():
[tg.stop() for tg in self.tgs]
self.tgs = [scheduler.setup(), scheduler.setup()]
self.addCleanup(stop_thread_groups)
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
eventlet.sleep(WAIT)
method.assert_called_once_with(name='task', id='321')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_multi_instance(self, method):
scheds = [scheduler.start(), scheduler.start()]
def stop_schedulers():
[scheduler.stop_scheduler(s, True) for s in scheds]
self.addCleanup(stop_schedulers)
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **method_args)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
eventlet.sleep(WAIT)
method.assert_called_once_with(name='task', id='321')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_with_factory(self, factory):
target_method = 'run_something'
method_args = {'name': 'task', 'id': '123'}
scheduler.schedule_call(
FACTORY_METHOD_PATH,
target_method,
DELAY,
**method_args
)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=2)
)
call = self._assert_single_item(
calls,
target_method_name=target_method
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
factory().run_something.assert_called_once_with(name='task', id='123')
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
def test_scheduler_with_serializer(self, factory):
target_method = 'run_something'
task_result = wf_utils.Result('data', 'error')
method_args = {
'name': 'task',
'id': '123',
'result': task_result
}
serializers = {
'result': 'mistral.workflow.utils.ResultSerializer'
}
delay = 1.5
scheduler.schedule_call(
FACTORY_METHOD_NAME,
target_method,
delay,
serializers=serializers,
**method_args
)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=2)
)
call = self._assert_single_item(
calls,
target_method_name=target_method
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(delay)
result = factory().run_something.call_args[1].get('result')
self.assertIsInstance(result, wf_utils.Result)
self.assertEqual('data', result.data)
self.assertEqual('error', result.error)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=1)
)
self.assertEqual(0, len(calls))
def test_scheduler_with_serializer(self, factory):
target_method_name = 'run_something'
factory.return_value = type(
'something',
(object,),
{
target_method_name:
mock.MagicMock(side_effect=self.target_method)
}
)
task_result = ml_actions.Result('data', 'error')
method_args = {
'name': 'task',
'id': '123',
'result': task_result
}
serializers = {
'result': 'mistral.workflow.utils.ResultSerializer'
}
scheduler.schedule_call(
TARGET_METHOD_PATH,
target_method_name,
DELAY,
serializers=serializers,
**method_args
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(
calls,
target_method_name=target_method_name
)
self.assertIn('name', call['method_arguments'])
self.queue.get()
result = factory().run_something.call_args[1].get('result')
self.assertIsInstance(result, ml_actions.Result)
self.assertEqual('data', result.data)
self.assertEqual('error', result.error)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_doesnt_handle_calls_the_failed_on_update(
self,
update_delayed_call):
def update_call_failed(id, values, query_filter):
self.queue.put("item")
return None, 0
update_delayed_call.side_effect = update_call_failed
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'name': 'task', 'id': '321'}
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.queue.get()
eventlet.sleep(1)
update_delayed_call.assert_called_with(
id=calls[0].id,
values=mock.ANY,
query_filter=mock.ANY
)
# If the scheduler does handel calls that failed on update
# DBEntityNotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def test_scheduler_doesnt_handel_calls_the_failed_on_update(self):
def stop_thread_groups():
[tg.stop() for tg in self.tgs]
self.tgs = [scheduler.setup(), scheduler.setup()]
self.addCleanup(stop_thread_groups)
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
TARGET_METHOD_NAME,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
eventlet.sleep(WAIT)
# If the scheduler does handel calls that failed on update
# NotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def run_delayed_calls(self, ctx=None):
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
# Wrap delayed calls processing in transaction to
# guarantee that calls will be processed just once.
# Do delete query to DB first to force hanging up all
# parallel transactions.
# It should work on isolation level 'READ-COMMITTED',
# 'REPEATABLE-READ' and above.
#
# 'REPEATABLE-READ' is by default in MySQL and
# 'READ-COMMITTED is by default in PostgreSQL.
delayed_calls = []
with db_api.transaction():
for call in db_api.get_delayed_calls_to_start(time_filter):
# Delete this delayed call from DB before the making call in
# order to prevent calling from parallel transaction.
db_api.delete_delayed_call(call.id)
LOG.debug('Processing next delayed call: %s', call)
context.set_ctx(context.MistralContext(call.auth_context))
if call.factory_method_path:
factory = importutils.import_class(
call.factory_method_path)
target_method = getattr(factory(), call.target_method_name)
else:
target_method = importutils.import_class(
call.target_method_name)
method_args = copy.copy(call.method_arguments)
if call.serializers:
# Deserialize arguments.
for arg_name, ser_path in call.serializers.items():
serializer = importutils.import_class(ser_path)()
deserialized = serializer.deserialize(
method_args[arg_name])
method_args[arg_name] = deserialized
delayed_calls.append((target_method, method_args))
# TODO(m4dcoder): Troubleshoot deadlocks with PostgreSQL and MySQL.
# The queries in the target method such as
# mistral.engine.task_handler.run_action can deadlock
# with delete_delayed_call. Please keep the scope of the
# transaction short.
for (target_method, method_args) in delayed_calls:
with db_api.transaction():
try:
# Call the method.
target_method(**method_args)
except Exception as e:
LOG.debug("Delayed call failed [call=%s, exception=%s]",
call, e)
def _capture_calls(batch_size):
"""Captures delayed calls eligible for processing (based on time).
The intention of this method is to select delayed calls based on time
criteria and mark them in DB as being processed so that no other
threads could process them in parallel.
:return: A list of delayed calls captured for further processing.
"""
result = []
time_filter = utils.utc_now_sec() + datetime.timedelta(seconds=1)
with db_api.transaction():
candidates = db_api.get_delayed_calls_to_start(
time_filter, batch_size)
for call in candidates:
# Mark this delayed call has been processed in order to
# prevent calling from parallel transaction.
db_call, updated_cnt = db_api.update_delayed_call(
id=call.id,
values={'processing': True},
query_filter={'processing': False})
# If updated_cnt != 1 then another scheduler
# has already updated it.
if updated_cnt == 1:
result.append(db_call)
LOG.debug("Scheduler captured %s delayed calls.", len(result))
return result
def test_scheduler_delete_calls(self, method):
def stop_thread_groups():
[tg.stop() for tg in self.tgs]
self.tgs = [scheduler.setup(), scheduler.setup()]
self.addCleanup(stop_thread_groups)
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
TARGET_METHOD_NAME,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=TARGET_METHOD_NAME)
eventlet.sleep(WAIT)
self.assertRaises(exc.NotFoundException,
db_api.get_delayed_call,
calls[0].id
)
def test_scheduler_doesnt_handle_calls_the_failed_on_update(
self, update_delayed_call):
def update_call_failed(id, values, query_filter):
self.queue.put("item")
return None, 0
update_delayed_call.side_effect = update_call_failed
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **{
'name': 'task',
'id': '321'
})
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.queue.get()
eventlet.sleep(1)
update_delayed_call.assert_called_with(id=calls[0].id,
values=mock.ANY,
query_filter=mock.ANY)
# If the scheduler does handel calls that failed on update
# DBEntityNotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def test_scheduler_with_serializer(self, factory):
target_method = 'run_something'
task_result = wf_utils.Result('data', 'error')
method_args = {
'name': 'task',
'id': '123',
'result': task_result
}
serializers = {
'result': 'mistral.workflow.utils.ResultSerializer'
}
scheduler.schedule_call(
FACTORY_METHOD_PATH,
target_method,
DELAY,
serializers=serializers,
**method_args
)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=WAIT)
)
call = self._assert_single_item(
calls,
target_method_name=target_method
)
self.assertIn('name', call['method_arguments'])
eventlet.sleep(WAIT)
result = factory().run_something.call_args[1].get('result')
self.assertIsInstance(result, wf_utils.Result)
self.assertEqual('data', result.data)
self.assertEqual('error', result.error)
calls = db_api.get_delayed_calls_to_start(
datetime.datetime.now() + datetime.timedelta(seconds=1)
)
self.assertEqual(0, len(calls))
def test_scheduler_without_factory(self, method):
method.side_effect = self.target_method
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **{
'name': 'task',
'id': '321'
})
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(calls,
target_method_name=TARGET_METHOD_PATH)
self.assertIn('name', call['method_arguments'])
self.queue.get()
method.assert_called_once_with(name='task', id='321')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_multi_instance(self, method):
method.side_effect = self.target_method
second_scheduler = scheduler.Scheduler(1, 1, None)
second_scheduler.start()
self.addCleanup(second_scheduler.stop, True)
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **{
'name': 'task',
'id': '321'
})
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
self.queue.get()
method.assert_called_once_with(name='task', id='321')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def run_delayed_calls(self, ctx=None):
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
# Wrap delayed calls processing in transaction to
# guarantee that calls will be processed just once.
# Do delete query to DB first to force hanging up all
# parallel transactions.
# It should work on isolation level 'READ-COMMITTED',
# 'REPEATABLE-READ' and above.
#
# 'REPEATABLE-READ' is by default in MySQL and
# 'READ-COMMITTED is by default in PostgreSQL.
with db_api.transaction():
delayed_calls = db_api.get_delayed_calls_to_start(time_filter)
for call in delayed_calls:
# Delete this delayed call from DB before the making call in
# order to prevent calling from parallel transaction.
db_api.delete_delayed_call(call.id)
LOG.debug('Processing next delayed call: %s', call)
context.set_ctx(context.MistralContext(call.auth_context))
if call.factory_method_path:
factory = importutils.import_class(
call.factory_method_path
)
target_method = getattr(factory(), call.target_method_name)
else:
target_method = importutils.import_class(
call.target_method_name
)
method_args = copy.copy(call.method_arguments)
if call.serializers:
# Deserialize arguments.
for arg_name, ser_path in call.serializers.items():
serializer = importutils.import_class(ser_path)()
deserialized = serializer.deserialize(
method_args[arg_name]
)
method_args[arg_name] = deserialized
try:
# Call the method.
target_method(**method_args)
except Exception as e:
LOG.debug(
"Delayed call failed [call=%s, exception=%s]", call, e
)
def test_scheduler_multi_instance(self, method):
method.side_effect = self.target_method
second_scheduler = scheduler.Scheduler(1, 1, None)
second_scheduler.start()
self.addCleanup(second_scheduler.stop, True)
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'name': 'task', 'id': '321'}
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
self.queue.get()
method.assert_called_once_with(name='task', id='321')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_without_factory(self, method):
method.side_effect = self.target_method
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'name': 'task', 'id': '321'}
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
call = self._assert_single_item(
calls,
target_method_name=TARGET_METHOD_PATH
)
self.assertIn('name', call['method_arguments'])
self.queue.get()
method.assert_called_once_with(name='task', id='321')
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self.assertEqual(0, len(calls))
def test_scheduler_delete_calls(self, method):
method.side_effect = self.target_method
scheduler.schedule_call(None, TARGET_METHOD_PATH, DELAY, **{
'name': 'task',
'id': '321'
})
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
self.queue.get()
self.assertRaises(exc.DBEntityNotFoundError, db_api.get_delayed_call,
calls[0].id)
def test_scheduler_delete_calls(self, method):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(None, FACTORY_METHOD_PATH, DELAY,
**method_args)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=FACTORY_METHOD_PATH)
eventlet.sleep(WAIT)
self.assertRaises(exc.DBEntityNotFoundError, db_api.get_delayed_call,
calls[0].id)
def test_scheduler_doesnt_handle_calls_the_failed_on_update(self):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(None, FACTORY_METHOD_PATH, DELAY,
**method_args)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
eventlet.sleep(WAIT)
# If the scheduler does handel calls that failed on update
# DBEntityNotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def test_processing_true_does_not_return_in_get_delayed_calls_to_start(
self, method):
method.side_effect = self.target_method
values = {
'factory_method_path': None,
'target_method_name': TARGET_METHOD_PATH,
'execution_time': get_time_delay(),
'auth_context': None,
'serializers': None,
'method_arguments': None,
'processing': True
}
call = db_api.create_delayed_call(values)
calls = db_api.get_delayed_calls_to_start(get_time_delay(10))
self.assertEqual(0, len(calls))
db_api.delete_delayed_call(call.id)
def test_scheduler_doesnt_handle_calls_the_failed_on_update(self):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
FACTORY_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
eventlet.sleep(WAIT)
# If the scheduler does handel calls that failed on update
# DBEntityNotFoundException will raise.
db_api.get_delayed_call(calls[0].id)
db_api.delete_delayed_call(calls[0].id)
def test_processing_true_does_not_return_in_get_delayed_calls_to_start(
self,
method):
method.side_effect = self.target_method
values = {
'factory_method_path': None,
'target_method_name': TARGET_METHOD_PATH,
'execution_time': get_time_delay(),
'auth_context': None,
'serializers': None,
'method_arguments': None,
'processing': True
}
call = db_api.create_delayed_call(values)
calls = db_api.get_delayed_calls_to_start(get_time_delay(10))
self.assertEqual(0, len(calls))
db_api.delete_delayed_call(call.id)
def test_scheduler_delete_calls(self, method):
method_args = {'name': 'task', 'id': '321'}
scheduler.schedule_call(
None,
FACTORY_METHOD_PATH,
DELAY,
**method_args
)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=2)
calls = db_api.get_delayed_calls_to_start(time_filter)
self._assert_single_item(calls, target_method_name=FACTORY_METHOD_PATH)
eventlet.sleep(WAIT)
self.assertRaises(exc.DBEntityNotFoundException,
db_api.get_delayed_call,
calls[0].id
)
def test_processing_true_does_not_return_in_get_delayed_calls_to_start(
self, method):
execution_time = (datetime.datetime.now() +
datetime.timedelta(seconds=DELAY))
values = {
'factory_method_path': None,
'target_method_name': FACTORY_METHOD_PATH,
'execution_time': execution_time,
'auth_context': None,
'serializers': None,
'method_arguments': None,
'processing': True
}
call = db_api.create_delayed_call(values)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=10)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
db_api.delete_delayed_call(call.id)
def test_scheduler_delete_calls(self, method):
method.side_effect = self.target_method
scheduler.schedule_call(
None,
TARGET_METHOD_PATH,
DELAY,
**{'name': 'task', 'id': '321'}
)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
self.queue.get()
eventlet.sleep(0.1)
self.assertRaises(
exc.DBEntityNotFoundError,
db_api.get_delayed_call,
calls[0].id
)
def test_scheduler_delete_calls(self, method):
method.side_effect = self.target_method
job = sched_base.SchedulerJob(
run_after=DELAY,
func_name=TARGET_METHOD_PATH,
func_args={
'name': 'task',
'id': '321'
},
)
self.scheduler.schedule(job)
calls = db_api.get_delayed_calls_to_start(get_time_delay())
self._assert_single_item(calls, target_method_name=TARGET_METHOD_PATH)
self.queue.get()
eventlet.sleep(0.1)
self.assertRaises(exc.DBEntityNotFoundError, db_api.get_delayed_call,
calls[0].id)
def test_processing_true_does_not_return_in_get_delayed_calls_to_start(
self,
method):
execution_time = (datetime.datetime.now() +
datetime.timedelta(seconds=DELAY))
values = {
'factory_method_path': None,
'target_method_name': FACTORY_METHOD_PATH,
'execution_time': execution_time,
'auth_context': None,
'serializers': None,
'method_arguments': None,
'processing': True
}
call = db_api.create_delayed_call(values)
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=10)
calls = db_api.get_delayed_calls_to_start(time_filter)
self.assertEqual(0, len(calls))
db_api.delete_delayed_call(call.id)
def _capture_calls(batch_size):
"""Captures delayed calls eligible for processing (based on time).
The intention of this method is to select delayed calls based on time
criteria and mark them in DB as being processed so that no other
threads could process them in parallel.
:return: A list of delayed calls captured for further processing.
"""
result = []
time_filter = utils.utc_now_sec() + datetime.timedelta(seconds=1)
with db_api.transaction():
candidates = db_api.get_delayed_calls_to_start(
time_filter,
batch_size
)
for call in candidates:
# Mark this delayed call has been processed in order to
# prevent calling from parallel transaction.
db_call, updated_cnt = db_api.update_delayed_call(
id=call.id,
values={'processing': True},
query_filter={'processing': False}
)
# If updated_cnt != 1 then another scheduler
# has already updated it.
if updated_cnt == 1:
result.append(db_call)
LOG.debug("Scheduler captured %s delayed calls.", len(result))
return result
def run_delayed_calls(self, ctx=None):
time_filter = datetime.datetime.now() + datetime.timedelta(
seconds=1)
# Wrap delayed calls processing in transaction to
# guarantee that calls will be processed just once.
# Do delete query to DB first to force hanging up all
# parallel transactions.
# It should work on isolation level 'READ-COMMITTED',
# 'REPEATABLE-READ' and above.
#
# 'REPEATABLE-READ' is by default in MySQL and
# 'READ-COMMITTED is by default in PostgreSQL.
delayed_calls = []
with db_api.transaction():
candidate_calls = db_api.get_delayed_calls_to_start(
time_filter
)
calls_to_make = []
for call in candidate_calls:
# Mark this delayed call has been processed in order to
# prevent calling from parallel transaction.
result, number_of_updated = db_api.update_delayed_call(
id=call.id,
values={'processing': True},
query_filter={"processing": False}
)
# If number_of_updated != 1 other scheduler already
# updated.
if number_of_updated == 1:
calls_to_make.append(result)
for call in calls_to_make:
LOG.debug('Processing next delayed call: %s', call)
target_auth_context = copy.deepcopy(call.auth_context)
if call.factory_method_path:
factory = importutils.import_class(
call.factory_method_path
)
target_method = getattr(factory(), call.target_method_name)
else:
target_method = importutils.import_class(
call.target_method_name
)
method_args = copy.deepcopy(call.method_arguments)
if call.serializers:
# Deserialize arguments.
for arg_name, ser_path in call.serializers.items():
serializer = importutils.import_class(ser_path)()
deserialized = serializer.deserialize(
method_args[arg_name]
)
method_args[arg_name] = deserialized
delayed_calls.append(
(target_auth_context, target_method, method_args)
)
for (target_auth_context, target_method, method_args) in delayed_calls:
try:
# Set the correct context for the method.
context.set_ctx(
context.MistralContext(target_auth_context)
)
# Call the method.
target_method(**method_args)
except Exception as e:
LOG.exception(
"Delayed call failed, method: %s, exception: %s",
target_method,
e
)
finally:
# Remove context.
context.set_ctx(None)
with db_api.transaction():
for call in calls_to_make:
try:
# Delete calls that were processed.
db_api.delete_delayed_call(call.id)
except Exception as e:
LOG.error(
"failed to delete call [call=%s, "
"exception=%s]", call, e
)
def run_delayed_calls(self, ctx=None):
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
# Wrap delayed calls processing in transaction to
# guarantee that calls will be processed just once.
# Do delete query to DB first to force hanging up all
# parallel transactions.
# It should work on isolation level 'READ-COMMITTED',
# 'REPEATABLE-READ' and above.
#
# 'REPEATABLE-READ' is by default in MySQL and
# 'READ-COMMITTED is by default in PostgreSQL.
delayed_calls = []
with db_api.transaction():
candidate_calls = db_api.get_delayed_calls_to_start(time_filter)
calls_to_make = []
for call in candidate_calls:
# Mark this delayed call has been processed in order to
# prevent calling from parallel transaction.
result, number_of_updated = db_api.update_delayed_call(
id=call.id,
values={'processing': True},
query_filter={"processing": False})
# If number_of_updated != 1 other scheduler already
# updated.
if number_of_updated == 1:
calls_to_make.append(result)
for call in calls_to_make:
LOG.debug('Processing next delayed call: %s', call)
target_auth_context = copy.deepcopy(call.auth_context)
if call.factory_method_path:
factory = importutils.import_class(call.factory_method_path)
target_method = getattr(factory(), call.target_method_name)
else:
target_method = importutils.import_class(
call.target_method_name)
method_args = copy.deepcopy(call.method_arguments)
if call.serializers:
# Deserialize arguments.
for arg_name, ser_path in call.serializers.items():
serializer = importutils.import_class(ser_path)()
deserialized = serializer.deserialize(
method_args[arg_name])
method_args[arg_name] = deserialized
delayed_calls.append(
(target_auth_context, target_method, method_args))
for (target_auth_context, target_method, method_args) in delayed_calls:
# Transaction is needed here because some of the
# target_method can use the DB
with db_api.transaction():
try:
# Set the correct context for the method.
context.set_ctx(
context.MistralContext(target_auth_context))
# Call the method.
target_method(**method_args)
except Exception as e:
LOG.error("Delayed call failed [exception=%s]", e)
finally:
# Remove context.
context.set_ctx(None)
with db_api.transaction():
for call in calls_to_make:
try:
# Delete calls that were processed.
db_api.delete_delayed_call(call.id)
except Exception as e:
LOG.error(
"failed to delete call [call=%s, "
"exception=%s]", call, e)
def run_delayed_calls(self, ctx=None):
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
# Wrap delayed calls processing in transaction to guarantee that calls
# will be processed just once. Do delete query to DB first to force
# hanging up all parallel transactions.
# It should work with transactions which run at least 'READ-COMMITTED'
# mode.
delayed_calls = []
with db_api.transaction():
candidate_calls = db_api.get_delayed_calls_to_start(time_filter)
calls_to_make = []
for call in candidate_calls:
# Mark this delayed call has been processed in order to
# prevent calling from parallel transaction.
result, number_of_updated = db_api.update_delayed_call(
id=call.id,
values={'processing': True},
query_filter={'processing': False})
# If number_of_updated != 1 other scheduler already
# updated.
if number_of_updated == 1:
calls_to_make.append(result)
for call in calls_to_make:
LOG.debug('Processing next delayed call: %s', call)
target_auth_context = copy.deepcopy(call.auth_context)
if call.factory_method_path:
factory = importutils.import_class(call.factory_method_path)
target_method = getattr(factory(), call.target_method_name)
else:
target_method = importutils.import_class(
call.target_method_name)
method_args = copy.deepcopy(call.method_arguments)
if call.serializers:
# Deserialize arguments.
for arg_name, ser_path in call.serializers.items():
serializer = importutils.import_class(ser_path)()
deserialized = serializer.deserialize(
method_args[arg_name])
method_args[arg_name] = deserialized
delayed_calls.append(
(target_auth_context, target_method, method_args))
ctx_serializer = context.RpcContextSerializer(
context.JsonPayloadSerializer())
for (target_auth_context, target_method, method_args) in delayed_calls:
try:
# Set the correct context for the method.
ctx_serializer.deserialize_context(target_auth_context)
# Call the method.
target_method(**method_args)
except Exception as e:
LOG.exception("Delayed call failed, method: %s, exception: %s",
target_method, e)
finally:
# Remove context.
context.set_ctx(None)
with db_api.transaction():
for call in calls_to_make:
try:
# Delete calls that were processed.
db_api.delete_delayed_call(call.id)
except Exception as e:
LOG.error(
"failed to delete call [call=%s, "
"exception=%s]", call, e)
