def _invoke_calls(delayed_calls):
"""Invokes prepared delayed calls.
:param delayed_calls: Prepared delayed calls represented as tuples
(target_auth_context, target_method, method_args).
"""
ctx_serializer = context.RpcContextSerializer()
for (target_auth_context, target_method, method_args) in delayed_calls:
try:
# Set the correct context for the method.
ctx_serializer.deserialize_context(target_auth_context)
# Invoke the method.
target_method(**method_args)
except Exception as e:
LOG.exception(
"Delayed call failed, method: %s, exception: %s",
target_method,
e
)
finally:
# Remove context.
context.set_ctx(None)
def launch_executor(transport):
target = messaging.Target(topic=cfg.CONF.executor.topic,
server=cfg.CONF.executor.host)
executor_v2 = def_executor.DefaultExecutor(rpc.get_engine_client())
endpoints = [rpc.ExecutorServer(executor_v2)]
server = messaging.get_rpc_server(transport,
target,
endpoints,
executor='eventlet',
serializer=ctx.RpcContextSerializer(
ctx.JsonPayloadSerializer()))
executor_v2.register_membership()
try:
server.start()
while True:
time.sleep(604800)
except (KeyboardInterrupt, SystemExit):
pass
finally:
print("Stopping executor service...")
server.stop()
server.wait()
def schedule_call(factory_method_path, target_method_name,
run_after, serializers=None, key=None, **method_args):
"""Schedules call and lately invokes target_method.
Add this call specification to DB, and then after run_after
seconds service CallScheduler invokes the target_method.
:param factory_method_path: Full python-specific path to
factory method that creates a target object that the call will be
made against.
:param target_method_name: Name of a method which will be invoked.
:param run_after: Value in seconds.
:param serializers: map of argument names and their serializer class
paths. Use when an argument is an object of specific type, and needs
to be serialized. Example:
{ "result": "mistral.utils.serializer.ResultSerializer"}
Serializer for the object type must implement serializer interface
in mistral/utils/serializer.py
:param key: Key which can potentially be used for squashing similar
delayed calls.
:param method_args: Target method keyword arguments.
"""
ctx_serializer = context.RpcContextSerializer()
ctx = (
ctx_serializer.serialize_context(context.ctx())
if context.has_ctx() else {}
)
execution_time = (datetime.datetime.now() +
datetime.timedelta(seconds=run_after))
if serializers:
for arg_name, serializer_path in serializers.items():
if arg_name not in method_args:
raise exc.MistralException(
"Serializable method argument %s"
" not found in method_args=%s"
% (arg_name, method_args))
try:
serializer = importutils.import_class(serializer_path)()
except ImportError as e:
raise ImportError(
"Cannot import class %s: %s" % (serializer_path, e)
)
method_args[arg_name] = serializer.serialize(method_args[arg_name])
values = {
'factory_method_path': factory_method_path,
'target_method_name': target_method_name,
'execution_time': execution_time,
'auth_context': ctx,
'serializers': serializers,
'key': key,
'method_arguments': method_args,
'processing': False
}
db_api.create_delayed_call(values)
def launch_engine(transport):
target = messaging.Target(topic=cfg.CONF.engine.topic,
server=cfg.CONF.engine.host)
engine_v2 = def_eng.DefaultEngine(rpc.get_engine_client())
endpoints = [rpc.EngineServer(engine_v2)]
# Setup scheduler in engine.
db_api.setup_db()
scheduler.setup()
# Setup expiration policy
expiration_policy.setup()
server = messaging.get_rpc_server(transport,
target,
endpoints,
executor='eventlet',
serializer=ctx.RpcContextSerializer(
ctx.JsonPayloadSerializer()))
engine_v2.register_membership()
try:
server.start()
while True:
time.sleep(604800)
except (KeyboardInterrupt, SystemExit):
pass
finally:
print("Stopping engine service...")
server.stop()
server.wait()
def setUp(self):
super(EngineProfilerTest, self).setUp()
# Configure the osprofiler.
self.mock_profiler_log_func = mock.Mock(return_value=None)
osprofiler.notifier.set(self.mock_profiler_log_func)
self.ctx_serializer = context.RpcContextSerializer()
def __init__(self, conf):
super(OsloRPCClient, self).__init__(conf)
self.topic = conf.topic
serializer = auth_ctx.RpcContextSerializer()
self._client = messaging.RPCClient(rpc.get_transport(),
messaging.Target(topic=self.topic),
serializer=serializer)
def __init__(self, transport):
"""Constructs an RPC client for engine.
:param transport: Messaging transport.
"""
serializer = auth_ctx.RpcContextSerializer(
auth_ctx.JsonPayloadSerializer())
self._client = messaging.RPCClient(
transport,
messaging.Target(topic=cfg.CONF.engine.topic),
serializer=serializer)
def launch_executor_server(transport, executor):
target = messaging.Target(topic=cfg.CONF.executor.topic,
server=cfg.CONF.executor.host)
server = messaging.get_rpc_server(transport,
target, [rpc.ExecutorServer(executor)],
executor='blocking',
serializer=ctx.RpcContextSerializer(
ctx.JsonPayloadSerializer()))
server.start()
server.wait()
def run(self, executor='blocking'):
target = messaging.Target(topic=self.topic, server=self.server_id)
server = messaging.get_rpc_server(rpc.get_transport(),
target,
self.endpoints,
executor=executor,
serializer=ctx.RpcContextSerializer(
ctx.JsonPayloadSerializer()))
server.start()
server.wait()
def run(self, executor='blocking'):
target = messaging.Target(topic=self.topic, server=self.server_id)
# TODO(rakhmerov): rpc.get_transport() should be in oslo.messaging
# related module.
self.oslo_server = messaging.get_rpc_server(
rpc.get_transport(),
target,
self.endpoints,
executor=executor,
serializer=ctx.RpcContextSerializer(ctx.JsonPayloadSerializer()))
self.oslo_server.start()
def _invoke_job(auth_ctx, func, args):
ctx_serializer = context.RpcContextSerializer()
try:
# Set the correct context for the function.
ctx_serializer.deserialize_context(auth_ctx)
# Invoke the function.
func(**args)
except Exception as e:
LOG.exception("Scheduled job failed, method: %s, exception: %s",
func, e)
finally:
# Remove context.
context.set_ctx(None)
def run(self, executor='eventlet'):
target = messaging.Target(topic=self.topic, server=self.server_id)
# TODO(rakhmerov): rpc.get_transport() should be in oslo.messaging
# related module.
access_policy = dispatcher.DefaultRPCAccessPolicy
self.oslo_server = messaging.get_rpc_server(
rpc.get_transport(),
target,
self.endpoints,
executor=executor,
serializer=ctx.RpcContextSerializer(),
access_policy=access_policy)
self.oslo_server.start()
def launch_executor(transport):
target = messaging.Target(topic=cfg.CONF.executor.topic,
server=cfg.CONF.executor.host)
executor_v2 = def_executor.DefaultExecutor(rpc.get_engine_client())
endpoints = [rpc.ExecutorServer(executor_v2)]
server = messaging.get_rpc_server(transport,
target,
endpoints,
executor='eventlet',
serializer=ctx.RpcContextSerializer(
ctx.JsonPayloadSerializer()))
server.start()
server.wait()
def launch_executor_server(transport, executor):
target = messaging.Target(topic=cfg.CONF.executor.topic,
server=cfg.CONF.executor.host)
server = messaging.get_rpc_server(transport,
target, [rpc.ExecutorServer(executor)],
executor='blocking',
serializer=ctx.RpcContextSerializer(
ctx.JsonPayloadSerializer()))
try:
server.start()
while True:
eventlet.sleep(604800)
except (KeyboardInterrupt, SystemExit):
LOG.info("Stopping executor service...")
finally:
server.stop()
server.wait()
def _persist_job(job):
ctx_serializer = context.RpcContextSerializer()
ctx = (
ctx_serializer.serialize_context(context.ctx())
if context.has_ctx() else {}
)
execute_at = (utils.utc_now_sec() +
datetime.timedelta(seconds=job.run_after))
args = job.func_args
arg_serializers = job.func_arg_serializers
if arg_serializers:
for arg_name, serializer_path in arg_serializers.items():
if arg_name not in args:
raise exc.MistralException(
"Serializable function argument %s"
" not found in func_args=%s"
% (arg_name, args))
try:
serializer = importutils.import_class(serializer_path)()
except ImportError as e:
raise ImportError(
"Cannot import class %s: %s" % (serializer_path, e)
)
args[arg_name] = serializer.serialize(args[arg_name])
values = {
'run_after': job.run_after,
'target_factory_func_name': job.target_factory_func_name,
'func_name': job.func_name,
'func_args': args,
'func_arg_serializers': arg_serializers,
'auth_ctx': ctx,
'execute_at': execute_at,
'captured_at': None,
'key': job.key
}
return db_api.create_scheduled_job(values)
def _invoke_job(auth_ctx, func, args):
# Scheduler runs jobs in an separate thread that's neither related
# to an RPC nor a REST request processing thread. So we need to
# initialize a profiler specifically for this thread.
if cfg.CONF.profiler.enabled:
profiler.init(cfg.CONF.profiler.hmac_keys)
ctx_serializer = context.RpcContextSerializer()
try:
# Set the correct context for the function.
ctx_serializer.deserialize_context(auth_ctx)
# Invoke the function.
func(**args)
except Exception as e:
LOG.exception("Scheduled job failed, method: %s, exception: %s",
func, e)
finally:
# Remove context.
context.set_ctx(None)
def launch_engine(transport):
target = messaging.Target(topic=cfg.CONF.engine.topic,
server=cfg.CONF.engine.host)
engine_v2 = def_eng.DefaultEngine(rpc.get_engine_client())
endpoints = [rpc.EngineServer(engine_v2)]
# Setup scheduler in engine.
db_api.setup_db()
scheduler.setup()
server = messaging.get_rpc_server(transport,
target,
endpoints,
executor='eventlet',
serializer=ctx.RpcContextSerializer(
ctx.JsonPayloadSerializer()))
server.start()
server.wait()
def schedule_call(factory_method_path,
target_method_name,
run_after,
serializers=None,
unique_key=None,
**method_args):
"""Schedules call and lately invokes target_method.
Add this call specification to DB, and then after run_after
seconds service CallScheduler invokes the target_method.
:param factory_method_path: Full python-specific path to
factory method that creates a target object that the call will be
made against.
:param target_method_name: Name of a method which will be invoked.
:param run_after: Value in seconds.
:param serializers: map of argument names and their serializer class
paths. Use when an argument is an object of specific type, and needs
to be serialized. Example:
{ "result": "mistral.utils.serializer.ResultSerializer"}
Serializer for the object type must implement serializer interface
in mistral/utils/serializer.py
:param unique_key: Unique key which in combination with 'processing'
flag restricts a number of delayed calls if it's passed. For example,
if we schedule two calls but pass the same unique key for them then
we won't get two of them in DB if both have same value of 'processing'
flag.
:param method_args: Target method keyword arguments.
"""
ctx_serializer = context.RpcContextSerializer(
context.JsonPayloadSerializer())
ctx = (ctx_serializer.serialize_context(context.ctx())
if context.has_ctx() else {})
execution_time = (datetime.datetime.now() +
datetime.timedelta(seconds=run_after))
if serializers:
for arg_name, serializer_path in serializers.items():
if arg_name not in method_args:
raise exc.MistralException("Serializable method argument %s"
" not found in method_args=%s" %
(arg_name, method_args))
try:
serializer = importutils.import_class(serializer_path)()
except ImportError as e:
raise ImportError("Cannot import class %s: %s" %
(serializer_path, e))
method_args[arg_name] = serializer.serialize(method_args[arg_name])
values = {
'factory_method_path': factory_method_path,
'target_method_name': target_method_name,
'execution_time': execution_time,
'auth_context': ctx,
'serializers': serializers,
'unique_key': unique_key,
'method_arguments': method_args,
'processing': False
}
db_api.insert_or_ignore_delayed_call(values)
def run_delayed_calls(self, ctx=None):
time_filter = datetime.datetime.now() + datetime.timedelta(seconds=1)
# Wrap delayed calls processing in transaction to guarantee that calls
# will be processed just once. Do delete query to DB first to force
# hanging up all parallel transactions.
# It should work with transactions which run at least 'READ-COMMITTED'
# mode.
delayed_calls = []
with db_api.transaction():
candidate_calls = db_api.get_delayed_calls_to_start(time_filter)
calls_to_make = []
for call in candidate_calls:
# Mark this delayed call has been processed in order to
# prevent calling from parallel transaction.
result, number_of_updated = db_api.update_delayed_call(
id=call.id,
values={'processing': True},
query_filter={'processing': False})
# If number_of_updated != 1 other scheduler already
# updated.
if number_of_updated == 1:
calls_to_make.append(result)
for call in calls_to_make:
LOG.debug('Processing next delayed call: %s', call)
target_auth_context = copy.deepcopy(call.auth_context)
if call.factory_method_path:
factory = importutils.import_class(call.factory_method_path)
target_method = getattr(factory(), call.target_method_name)
else:
target_method = importutils.import_class(
call.target_method_name)
method_args = copy.deepcopy(call.method_arguments)
if call.serializers:
# Deserialize arguments.
for arg_name, ser_path in call.serializers.items():
serializer = importutils.import_class(ser_path)()
deserialized = serializer.deserialize(
method_args[arg_name])
method_args[arg_name] = deserialized
delayed_calls.append(
(target_auth_context, target_method, method_args))
ctx_serializer = context.RpcContextSerializer(
context.JsonPayloadSerializer())
for (target_auth_context, target_method, method_args) in delayed_calls:
try:
# Set the correct context for the method.
ctx_serializer.deserialize_context(target_auth_context)
# Call the method.
target_method(**method_args)
except Exception as e:
LOG.exception("Delayed call failed, method: %s, exception: %s",
target_method, e)
finally:
# Remove context.
context.set_ctx(None)
with db_api.transaction():
for call in calls_to_make:
try:
# Delete calls that were processed.
db_api.delete_delayed_call(call.id)
except Exception as e:
LOG.error(
"failed to delete call [call=%s, "
"exception=%s]", call, e)
