def initialize_worker(sender, instance, **kwargs):
"""
This function performs all the necessary initialization of the Celery worker.
It starts by cleaning up old state if this worker was previously running, but died unexpectedly.
In such cases, any Pulp tasks that were running or waiting on this worker will show incorrect
state. Any reserved_resource reservations associated with the previous worker will also be
removed along with the worker entry in the database itself. The working directory specified in
/etc/pulp/server.conf (/var/cache/pulp/<worker_name>) by default is removed and recreated. This
is called early in the worker start process, and later when it's fully online, pulp_celerybeat
will discover the worker as usual to allow new work to arrive at this worker. If there is no
previous work to cleanup, this method still runs, but has no effect on the database.
After cleaning up old state, it ensures the existence of the worker's working directory.
Lastly, this function makes the call to Pulp's initialization code.
It uses the celeryd_after_setup signal[0] so that it gets called by Celery after logging is
initialized, but before Celery starts to run tasks.
[0] http://celery.readthedocs.org/en/latest/userguide/signals.html#celeryd-after-setup
:param sender: The hostname of the worker
:type sender: basestring
:param instance: The Worker instance to be initialized (unused)
:type instance: celery.apps.worker.Worker
:param kwargs: Other params (unused)
:type kwargs: dict
"""
initialization.initialize()
tasks._delete_worker(sender, normal_shutdown=True)
# Create a new working directory for worker that is starting now
common_utils.create_worker_working_directory(sender)
def initialize_worker(sender, instance, **kwargs):
"""
This function performs all the necessary initialization of the Celery worker.
It starts by cleaning up old state if this worker was previously running, but died unexpectedly.
In such cases, any Pulp tasks that were running or waiting on this worker will show incorrect
state. Any reserved_resource reservations associated with the previous worker will also be
removed along with the worker entry in the database itself. The working directory specified in
/etc/pulp/server.conf (/var/cache/pulp/<worker_name>) by default is removed and recreated. This
is called early in the worker start process, and later when it's fully online, pulp_celerybeat
will discover the worker as usual to allow new work to arrive at this worker. If there is no
previous work to cleanup, this method still runs, but has no effect on the database.
After cleaning up old state, it ensures the existence of the worker's working directory.
Lastly, this function makes the call to Pulp's initialization code.
It uses the celeryd_after_setup signal[0] so that it gets called by Celery after logging is
initialized, but before Celery starts to run tasks.
[0] http://celery.readthedocs.org/en/latest/userguide/signals.html#celeryd-after-setup
:param sender: The hostname of the worker
:type sender: basestring
:param instance: The Worker instance to be initialized (unused)
:type instance: celery.apps.worker.Worker
:param kwargs: Other params (unused)
:type kwargs: dict
"""
initialization.initialize()
tasks._delete_worker(sender, normal_shutdown=True)
# Create a new working directory for worker that is starting now
common_utils.create_worker_working_directory(sender)
def test_initialize_does_not_call_manager_factory_if_plugin_api_raises_Exception(self):
self.mock_plugin_api.initialize.side_effect = OSError("my message")
try:
initialize()
except Exception:
pass
self.assertTrue(not self.mock_manager_factory.called)
def test_initialize_does_not_call_manager_factory_if_plugin_api_raises_Exception(
self):
self.mock_plugin_api.initialize.side_effect = OSError('my message')
try:
initialize()
except Exception:
pass
self.assertTrue(not self.mock_manager_factory.called)
def initialize_worker(sender, instance, **kwargs):
"""
This function performs all the necessary initialization of the Celery worker.
We clean up old state in case this worker was previously running, but died unexpectedly.
In such cases, any Pulp tasks that were running or waiting on this worker will show incorrect
state. Any reserved_resource reservations associated with the previous worker will also be
removed along with the worker entry in the database itself. The working directory specified in
/etc/pulp/server.conf (/var/cache/pulp/<worker_name>) by default is removed and recreated. This
is called early in the worker start process, and later when it's fully online, pulp_celerybeat
will discover the worker as usual to allow new work to arrive at this worker. If there is no
previous work to cleanup, this method still runs, but has no effect on the database.
After cleaning up old state, it ensures the existence of the worker's working directory.
Lastly, this function makes the call to Pulp's initialization code.
It uses the celeryd_after_setup signal[0] so that it gets called by Celery after logging is
initialized, but before Celery starts to run tasks.
If the worker is a resource manager, it tries to acquire a lock stored within the database.
If the lock cannot be acquired immediately, it will wait until the currently active instance
becomes unavailable, at which point the worker cleanup routine will clear the lock for us to
acquire. While the worker remains in this waiting state, it is not connected to the broker and
will not attempt to do any work. A side effect of this is that, if terminated while in this
state, the process will not send the "worker-offline" signal used by the EventMonitor to
immediately clean up terminated workers. Therefore, we override the SIGTERM signal handler
while in this state so that cleanup is done properly.
[0] http://celery.readthedocs.org/en/latest/userguide/signals.html#celeryd-after-setup
:param sender: The hostname of the worker
:type sender: basestring
:param instance: The Worker instance to be initialized (unused)
:type instance: celery.apps.worker.Worker
:param kwargs: Other params (unused)
:type kwargs: dict
"""
initialization.initialize()
# Delete any potential old state
tasks._delete_worker(sender, normal_shutdown=True)
# Create a new working directory for worker that is starting now
common_utils.delete_worker_working_directory(sender)
common_utils.create_worker_working_directory(sender)
# If the worker is a resource manager, try to acquire the lock, or wait until it
# can be acquired
if sender.startswith(constants.RESOURCE_MANAGER_WORKER_NAME):
get_resource_manager_lock(sender)
def initialize_worker(sender, instance, **kwargs):
"""
This function performs all the necessary initialization of the Celery worker.
We clean up old state in case this worker was previously running, but died unexpectedly.
In such cases, any Pulp tasks that were running or waiting on this worker will show incorrect
state. Any reserved_resource reservations associated with the previous worker will also be
removed along with the worker entry in the database itself. The working directory specified in
/etc/pulp/server.conf (/var/cache/pulp/<worker_name>) by default is removed and recreated. This
is called early in the worker start process, and later when it's fully online, pulp_celerybeat
will discover the worker as usual to allow new work to arrive at this worker. If there is no
previous work to cleanup, this method still runs, but has no effect on the database.
After cleaning up old state, it ensures the existence of the worker's working directory.
Lastly, this function makes the call to Pulp's initialization code.
It uses the celeryd_after_setup signal[0] so that it gets called by Celery after logging is
initialized, but before Celery starts to run tasks.
If the worker is a resource manager, it tries to acquire a lock stored within the database.
If the lock cannot be acquired immediately, it will wait until the currently active instance
becomes unavailable, at which point the worker cleanup routine will clear the lock for us to
acquire. While the worker remains in this waiting state, it is not connected to the broker and
will not attempt to do any work. A side effect of this is that, if terminated while in this
state, the process will not send the "worker-offline" signal used by the EventMonitor to
immediately clean up terminated workers. Therefore, we override the SIGTERM signal handler
while in this state so that cleanup is done properly.
[0] http://celery.readthedocs.org/en/latest/userguide/signals.html#celeryd-after-setup
:param sender: The hostname of the worker
:type sender: basestring
:param instance: The Worker instance to be initialized (unused)
:type instance: celery.apps.worker.Worker
:param kwargs: Other params (unused)
:type kwargs: dict
"""
initialization.initialize()
# Delete any potential old state
tasks._delete_worker(sender, normal_shutdown=True)
# Create a new working directory for worker that is starting now
common_utils.delete_worker_working_directory(sender)
common_utils.create_worker_working_directory(sender)
# If the worker is a resource manager, try to acquire the lock, or wait until it
# can be acquired
if sender.startswith(constants.RESOURCE_MANAGER_WORKER_NAME):
get_resource_manager_lock(sender)
def wsgi_application():
"""
Application factory to create, configure, and return a WSGI application
using the django framework
:return: wsgi application callable
"""
try:
logger = logging.getLogger(__name__)
logs.start_logging()
initialization.initialize()
except initialization.InitializationException, e:
logger.fatal('*************************************************************')
logger.fatal('The Pulp Puppet Forge server failed to start due to the following reasons:')
logger.exception(' ' + e.message)
logger.fatal('*************************************************************')
raise e
def _initialize_web_services():
"""
This function initializes Pulp for webservices.
"""
# This initialization order is very sensitive, and each touches a number of
# sub-systems in pulp. If you get this wrong, you will have pulp tripping
# over itself on start up.
global _IS_INITIALIZED, STACK_TRACER
if _IS_INITIALIZED:
return
logs.start_logging()
# Run the common initialization code that all processes should share. This will start the
# database connection, initialize plugins, and initialize the manager factory.
initialization.initialize()
# configure agent services
AgentServices.init()
# Verify the database has been migrated to the correct version. This is
# very likely a reason the server will fail to start.
try:
migration_models.check_package_versions()
except Exception:
msg = "The database has not been migrated to the current version. "
msg += "Run pulp-manage-db and restart the application."
raise initialization.InitializationException(msg), None, sys.exc_info()[2]
# There's a significantly smaller chance the following calls will fail.
# The previous two are likely user errors, but the remainder represent
# something gone horribly wrong. As such, I'm not going to account for each
# and instead simply let the exception itself bubble up.
# start agent services
AgentServices.start()
# If we got this far, it was successful, so flip the flag
_IS_INITIALIZED = True
def _initialize_web_services():
"""
This function initializes Pulp for webservices.
"""
# This initialization order is very sensitive, and each touches a number of
# sub-systems in pulp. If you get this wrong, you will have pulp tripping
# over itself on start up.
global _IS_INITIALIZED, STACK_TRACER
if _IS_INITIALIZED:
return
logs.start_logging()
# Run the common initialization code that all processes should share. This will start the
# database connection, initialize plugins, and initialize the manager factory.
initialization.initialize()
# configure agent services
AgentServices.init()
# Verify the database has been migrated to the correct version. This is
# very likely a reason the server will fail to start.
try:
migration_models.check_package_versions()
except Exception:
msg = 'The database has not been migrated to the current version. '
msg += 'Run pulp-manage-db and restart the application.'
raise initialization.InitializationException(msg), None, sys.exc_info()[2]
# There's a significantly smaller chance the following calls will fail.
# The previous two are likely user errors, but the remainder represent
# something gone horribly wrong. As such, I'm not going to account for each
# and instead simply let the exception itself bubble up.
# start agent services
AgentServices.start()
# If we got this far, it was successful, so flip the flag
_IS_INITIALIZED = True
def wsgi_application():
"""
Application factory to create, configure, and return a WSGI application
using the django framework
:return: wsgi application callable
"""
try:
logger = logging.getLogger(__name__)
logs.start_logging()
initialization.initialize()
except initialization.InitializationException, e:
logger.fatal(
'*************************************************************')
logger.fatal(
'The Pulp Puppet Forge server failed to start due to the following reasons:'
)
logger.exception(' ' + e.message)
logger.fatal(
'*************************************************************')
raise e
def test_initialize_calls_plugin_api_initialize(self):
initialize()
self.mock_plugin_api.initialize.assert_called_once_with()
def test_global__IS_INITIALIZED_is_set_to_True(self):
initialize()
self.assertTrue(init_module._IS_INITIALIZED)
def test_initialize_calls_manager_factory(self):
initialize()
self.mock_manager_factory.initialize.assert_called_once_with()
# -*- coding: utf-8 -*- # This import will load our configs from pulp.server import config from pulp.server import initialization # We need this import so that the Celery setup_logging signal gets registered from pulp.server import logs # This import is here so that Celery will find our application instance from pulp.server. async .celery_instance import celery initialization.initialize()
def test_initialize_calls_plugin_api_initialize(self):
initialize()
self.mock_plugin_api.initialize.assert_called_once_with()
def test_initialize_only_does_nothing_if__IS_INITIALIZED_is_True(self):
self.mock__IS_INITIALIZED = True
initialize()
self.assertTrue(not self.mock_db_connection.called)
def test_global__IS_INITIALIZED_is_set_to_True(self):
initialize()
self.assertTrue(init_module._IS_INITIALIZED)
def test_initialize_calls_manager_factory(self):
initialize()
self.mock_manager_factory.initialize.assert_called_once_with()
# -*- coding: utf-8 -*- # # Copyright © 2013 Red Hat, Inc. # # This software is licensed to you under the GNU General Public # License as published by the Free Software Foundation; either version # 2 of the License (GPLv2) or (at your option) any later version. # There is NO WARRANTY for this software, express or implied, # including the implied warranties of MERCHANTABILITY, # NON-INFRINGEMENT, or FITNESS FOR A PARTICULAR PURPOSE. You should # have received a copy of GPLv2 along with this software; if not, see # http://www.gnu.org/licenses/old-licenses/gpl-2.0.txt. # This import will load our configs from pulp.server import config from pulp.server import initialization # We need this import so that the Celery setup_logging signal gets registered from pulp.server import logs # This import is here so that Celery will find our application instance from pulp.server.async.celery_instance import celery initialization.initialize()
def test_initialize_only_does_nothing_if__IS_INITIALIZED_is_True(self):
self.mock__IS_INITIALIZED = True
initialize()
self.assertTrue(not self.mock_db_connection.called)
