def __init__(self, config, connection):
self._lock = Lock()
self._batch = queue.Queue()
self._last_commit = time()
self.config = config
self._amqp_connection = connection
self._started = queue.Queue()
self._reset_cache()
# exception stored here will be raised by the main thread
self.error_exit = None
def channel_method(self,
method,
channel=None,
wait=True,
timeout=None,
**kwargs):
"""Schedule a channel method to be called from the connection thread.
Use this to schedule a channel method such as .publish or .basic_ack
to be called from the connection thread.
"""
if wait and self._consumer_thread \
and self._consumer_thread is threading.current_thread():
# when sending from the connection thread, we can't wait because
# then we wouldn't allow the actual send loop (._process_publish)
# to run, because we'd block on the err_queue here
raise RuntimeError(
'Cannot wait when sending from the connection thread')
# the message is going to be sent from another thread (the .consume
# thread). If an error happens there, we must have a way to get it
# back out, so we pass a Queue together with the message, that will
# contain either an exception instance, or None
err_queue = queue.Queue() if wait else None
envelope = {
'method': method,
'message': kwargs,
'err_queue': err_queue,
'channel': channel
}
self._connection_tasks_queue.put(envelope)
if err_queue:
err = err_queue.get(timeout=timeout)
if isinstance(err, Exception):
raise err
def __init__(self, ctx, port=None):
port = port or get_unused_port()
socket_url = 'http://localhost:{0}'.format(port)
super(HTTPCtxProxy, self).__init__(ctx, socket_url)
self.port = port
self._started = queue.Queue(1)
self.thread = self._start_server()
self._started.get(timeout=5)
def __init__(self, *client_args, **client_kwargs):
self.client_started = Event()
self._connect_lock = RLock()
self._callers = 0
self._thread = None
self._queue = queue.Queue()
self._client_args = client_args
self._client_kwargs = client_kwargs
def _restore_inter_deployment_dependencies(self):
# managers older than 4.6.0 didn't have the support get_capability.
# manager newer than 5.0.5 have the inter deployment dependencies as
# part of the database dump
if (self._snapshot_version < V_4_6_0
or self._snapshot_version > V_5_0_5):
return
ctx.logger.info('Restoring inter deployment dependencies')
update_service_composition = (self._snapshot_version == V_5_0_5)
deployment_contexts = utils.get_dep_contexts(self._snapshot_version)
deployments_queue = queue.Queue()
failed_deployments_queue = queue.Queue()
for tenant_name, deployments in deployment_contexts:
for dep_id in deployments:
deployments_queue.put((tenant_name, dep_id))
wf_context = current_workflow_ctx.get_ctx()
context_params = current_workflow_ctx.get_parameters()
threads = []
for i in range(
min(self._config.snapshot_restore_threads,
deployments_queue.qsize())):
t = threading.Thread(
target=self._create_inter_deployment_dependencies,
args=(deployments_queue, failed_deployments_queue, wf_context,
context_params, update_service_composition))
threads.append(t)
t.start()
for t in threads:
t.join()
if not failed_deployments_queue.empty():
deployments = list(failed_deployments_queue.queue)
raise NonRecoverableError('Failed to restore snapshot, could not '
'create the inter deployment '
'dependencies from the following '
'deployments {0}. See exception '
'tracebacks logged above for more '
'details'.format(deployments))
ctx.logger.info('Successfully restored inter deployment dependencies.')
def _restore_deployment_envs(self, postgres):
deps = utils.get_dep_contexts(self._snapshot_version)
token_info = postgres.get_deployment_creator_ids_and_tokens()
deps_with_failed_plugins = queue.Queue()
failed_deployments = queue.Queue()
threads = list()
for tenant, deployments in deps:
ctx.logger.info(
'Restoring deployment environments for {tenant}'.format(
tenant=tenant,
)
)
tenant_client = get_rest_client(tenant=tenant)
for deployment_id, dep_ctx in deployments.items():
# Task graph is created and executed by threads to
# shorten restore time significantly
wf_ctx = current_workflow_ctx.get_ctx()
wf_parameters = current_workflow_ctx.get_parameters()
self._semaphore.acquire()
t = threading.Thread(target=self._get_and_execute_task_graph,
args=(token_info, deployment_id, dep_ctx,
tenant, tenant_client, wf_ctx,
wf_parameters,
deps_with_failed_plugins,
failed_deployments)
)
t.setDaemon(True)
threads.append(t)
t.start()
for t in threads:
t.join()
if not failed_deployments.empty():
deployments = list(failed_deployments.queue)
raise NonRecoverableError('Failed to restore snapshot, the '
'following deployment environments were'
' not restored: {0}. See exception'
' tracebacks logged above for more'
' details.'.format(deployments))
self._log_final_information(deps_with_failed_plugins)
def _create_connections():
acks_queue = queue.Queue()
cfy_config = config.instance
port = BROKER_PORT_SSL if cfy_config.amqp_ca_path else BROKER_PORT_NO_SSL
amqp_client = get_client(amqp_host=cfy_config.amqp_host,
amqp_user=cfy_config.amqp_username,
amqp_pass=cfy_config.amqp_password,
amqp_vhost='/',
amqp_port=port,
ssl_enabled=bool(cfy_config.amqp_ca_path),
ssl_cert_path=cfy_config.amqp_ca_path,
cls=AckingAMQPConnection)
amqp_client.acks_queue = acks_queue
db_publisher = DBLogEventPublisher(config.instance, amqp_client)
amqp_consumer = AMQPLogsEventsConsumer(
message_processor=db_publisher.process)
amqp_client.add_handler(amqp_consumer)
db_publisher.start()
return amqp_client, db_publisher
def __init__(self,
handlers,
name=None,
amqp_params=None,
connect_timeout=10):
self._handlers = handlers
self.name = name
self._connection_params = self._get_connection_params()
self._reconnect_backoff = 1
self._closed = False
self._amqp_params = amqp_params or AMQPParams()
self._pika_connection = None
self._consumer_thread = None
self.connect_wait = threading.Event()
self._connect_timeout = connect_timeout
self._error = None
self._daemon_factory = _get_daemon_factory()
# use this queue to schedule methods to be called on the pika channel
# from the connection thread - for sending data to rabbitmq, eg.
# publishing messages or sending ACKs, which needs to be done from
# the connection thread
self._connection_tasks_queue = queue.Queue()
def __init__(self, *args, **kwargs):
super(BlockingRequestResponseHandler, self).__init__(*args, **kwargs)
self._response = queue.Queue()
def _handle_remote_workflow(self):
tenant = self.ctx._context['tenant'].get('original_name',
self.ctx.tenant_name)
rest = get_rest_client(tenant=tenant)
execution = rest.executions.get(self.ctx.execution_id,
_include=['status'])
if execution.status == Execution.STARTED:
self.ctx.resume = True
try:
amqp_client_utils.init_events_publisher()
try:
self._workflow_started()
except InvalidExecutionUpdateStatus:
self._workflow_cancelled()
return api.EXECUTION_CANCELLED_RESULT
result_queue = queue.Queue()
t = AMQPWrappedThread(target=self._remote_workflow_child_thread,
args=(result_queue, ),
name='Workflow-Child')
t.start()
# while the child thread is executing the workflow, the parent
# thread is polling for 'cancel' requests while also waiting for
# messages from the child thread
result = None
while True:
# check if child thread sent a message
try:
data = result_queue.get(timeout=5)
if 'result' in data:
# child thread has terminated
result = data['result']
break
else:
# error occurred in child thread
raise data['error']
except queue.Empty:
pass
# A very hacky way to solve an edge case when trying to poll
# for the execution status while the DB is downgraded during
# a snapshot restore
if self.cloudify_context['workflow_id'] == 'restore_snapshot':
continue
# check for 'cancel' requests
execution = rest.executions.get(self.ctx.execution_id,
_include=['status'])
if execution.status in [
Execution.CANCELLING, Execution.FORCE_CANCELLING,
Execution.KILL_CANCELLING
]:
# send a 'cancel' message to the child thread. It is up to
# the workflow implementation to check for this message
# and act accordingly (by stopping and raising an
# api.ExecutionCancelled error, or by returning the
# deprecated api.EXECUTION_CANCELLED_RESULT as result).
# parent thread then goes back to polling for messages from
# child thread or possibly 'force-cancelling' requests
api.set_cancel_request()
if execution.status == Execution.KILL_CANCELLING:
# if a custom workflow function must attempt some cleanup,
# it might attempt to catch SIGTERM, and confirm using this
# flag that it is being kill-cancelled
api.set_kill_request()
if execution.status in [
Execution.FORCE_CANCELLING, Execution.KILL_CANCELLING
]:
# force-cancel additionally stops this loop immediately
result = api.EXECUTION_CANCELLED_RESULT
break
if result == api.EXECUTION_CANCELLED_RESULT:
self._workflow_cancelled()
else:
self._workflow_succeeded()
return result
except exceptions.WorkflowFailed as e:
self._workflow_failed(e)
raise
except BaseException as e:
self._workflow_failed(e, traceback.format_exc())
raise
finally:
amqp_client_utils.close_amqp_client()
def _handle_remote_workflow(self):
"""Run the workflow function.
This runs the workflow in a background thread. The main thread will
wait for the background thread to finish, and poll the execution
status, to check if the execution was cancelled. If so, the cancel
flag is set, which allows the workflow function to clean up.
If the force-cancel flag is set, then this function will return
early, without waiting for the background thread to finish.
"""
tenant = self.ctx._context['tenant'].get('original_name',
self.ctx.tenant_name)
rest = get_rest_client(tenant=tenant)
try:
try:
self._workflow_started()
except InvalidExecutionUpdateStatus:
self._workflow_cancelled()
return api.EXECUTION_CANCELLED_RESULT
result_queue = queue.Queue()
t = threading.Thread(target=self._remote_workflow_child_thread,
args=(result_queue, ),
name='Workflow-Child')
t.daemon = True
t.start()
# while the child thread is executing the workflow, the parent
# thread is polling for 'cancel' requests while also waiting for
# messages from the child thread
result = None
while True:
# check if child thread sent a message
try:
data = result_queue.get(timeout=5)
if 'result' in data:
# child thread has terminated
result = data['result']
break
else:
# error occurred in child thread
raise data['error']
except queue.Empty:
pass
# A very hacky way to solve an edge case when trying to poll
# for the execution status while the DB is downgraded during
# a snapshot restore
if self.cloudify_context['workflow_id'] == 'restore_snapshot':
continue
# check for 'cancel' requests
execution = rest.executions.get(self.ctx.execution_id,
_include=['status'])
if execution.status in [
Execution.CANCELLING, Execution.FORCE_CANCELLING,
Execution.KILL_CANCELLING
]:
# send a 'cancel' message to the child thread. It is up to
# the workflow implementation to check for this message
# and act accordingly (by stopping and raising an
# api.ExecutionCancelled error, or by returning the
# deprecated api.EXECUTION_CANCELLED_RESULT as result).
# parent thread then goes back to polling for messages from
# child thread or possibly 'force-cancelling' requests
api.set_cancel_request()
if execution.status == Execution.KILL_CANCELLING:
# if a custom workflow function must attempt some cleanup,
# it might attempt to catch SIGTERM, and confirm using this
# flag that it is being kill-cancelled
api.set_kill_request()
if execution.status in [
Execution.FORCE_CANCELLING, Execution.KILL_CANCELLING
]:
# force-cancel additionally stops this loop immediately
result = api.EXECUTION_CANCELLED_RESULT
break
if result == api.EXECUTION_CANCELLED_RESULT:
self._workflow_cancelled()
else:
self._workflow_succeeded()
return result
except _WorkflowFuncError as e:
self._workflow_failed(e.wrapped_exc, e.wrapped_tb)
# `raise e.wrapped_exc from None` - but syntax that won't break py2
e.wrapped_exc.__suppress_context__ = True
raise e.wrapped_exc
except BaseException as e:
self._workflow_failed(e, traceback.format_exc())
raise
