def create_mongo_config():
"""
Inspects the pulp config's mongodb settings and returns a data structure
that can be passed to celery for it's mongodb result backend config.
:return: dictionary with keys 'host' and 'database', and optionally with
keys 'user' and 'password', that can be passed to celery as the
config for a mongodb result backend
:rtype: dict
"""
db_name = config.get('database', 'name')
# celery 3.1 doesn't support multiple seeds, so we just use the first one
seeds = config.get('database', 'seeds')
seed = seeds.split(',')[0].strip()
host = seed.split(':')[0]
port = seed.split(':')[1] if ':' in seed else None
mongo_config = {'host': host, 'database': db_name}
if port:
mongo_config['port'] = port
if config.has_option('database', 'username') and config.has_option('database', 'password'):
mongo_config['user'] = config.get('database', 'username')
mongo_config['password'] = config.get('database', 'password')
if config.getboolean('database', 'ssl'):
mongo_config['ssl'] = True
ssl_keyfile = config.get('database', 'ssl_keyfile')
ssl_certfile = config.get('database', 'ssl_certfile')
if ssl_keyfile:
mongo_config['ssl_keyfile'] = ssl_keyfile
if ssl_certfile:
mongo_config['ssl_certfile'] = ssl_certfile
verify_ssl = config.getboolean('database', 'verify_ssl')
mongo_config['ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE
mongo_config['ssl_ca_certs'] = config.get('database', 'ca_path')
return mongo_config
def __call__(self, *args, **kwargs):
"""
This overrides PulpTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is 'canceled'.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status[
'state'] == constants.CALL_CANCELED_STATE:
_logger.debug("Task cancel received for task-id : [%s]" %
self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Skip updating status for eagerly executed tasks, since we don't want to track
# synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
if config.getboolean('profiling', 'enabled') is True:
self.pr = cProfile.Profile()
self.pr.enable()
return super(Task, self).__call__(*args, **kwargs)
def handle_event(notifier_config, event):
"""
If email is enabled in the server settings, sends an email to each recipient
listed in the notifier_config.
:param notifier_config: dictionary with keys 'subject', which defines the
subject of each email message, and 'addresses',
which is a list of strings that are email addresses
that should receive this notification.
:type notifier_config: dict
:param event: Event instance
:type event: pulp.server.event.data.event
:return: None
"""
if not config.getboolean('email', 'enabled'):
return
body = json.dumps(event.data(), indent=2, default=json_util.default)
subject = notifier_config['subject']
addresses = notifier_config['addresses']
for address in addresses:
thread = threading.Thread(target=_send_email,
args=(subject, body, address))
thread.daemon = True
thread.start()
def _check_username_password_ldap(self, username, password=None):
"""
Check a username and password against the ldap server.
Return None if the username and password are not valid
:type username: str
:param username: the login of the user
:type password: str or None
:param password: password of the user, None => do not validate the password
:rtype: L{pulp.server.db.model.auth.User} instance or None
:return: user corresponding to the credentials
"""
ldap_uri = config.get('ldap', 'uri')
ldap_base = config.get('ldap', 'base')
ldap_tls = config.getboolean('ldap', 'tls')
ldap_filter = None
if config.has_option('ldap', 'filter'):
ldap_filter = config.get('ldap', 'filter')
ldap_server = ldap_connection.LDAPConnection(server=ldap_uri, tls=ldap_tls)
ldap_server.connect()
user = ldap_server.authenticate_user(ldap_base, username, password,
filter=ldap_filter)
return user
def _check_username_password_ldap(self, username, password=None):
"""
Check a username and password against the ldap server.
Return None if the username and password are not valid
:type username: str
:param username: the login of the user
:type password: str or None
:param password: password of the user, None => do not validate the password
:rtype: L{pulp.server.db.model.auth.User} instance or None
:return: user corresponding to the credentials
"""
ldap_uri = config.get('ldap', 'uri')
ldap_base = config.get('ldap', 'base')
ldap_tls = config.getboolean('ldap', 'tls')
ldap_filter = None
if config.has_option('ldap', 'filter'):
ldap_filter = config.get('ldap', 'filter')
ldap_server = ldap_connection.LDAPConnection(server=ldap_uri,
tls=ldap_tls)
ldap_server.connect()
user = ldap_server.authenticate_user(ldap_base,
username,
password,
filter=ldap_filter)
return user
def send(document, routing_key=None):
"""
Attempt to send a message to the AMQP broker.
If we cannot obtain a new connection then the message will be dropped. Note
that we do not block when waiting for a connection.
:param document: the taskstatus Document we want to send
:type document: mongoengine.Document
:param routing_key: The routing key for the message
:type routing_key: str
"""
# if the user has not enabled notifications, just bail
event_notifications_enabled = config.getboolean("messaging", "event_notifications_enabled")
if not event_notifications_enabled:
return
try:
payload = document.to_json()
except TypeError:
_logger.warn("unable to convert document to JSON; event message not sent")
return
broker_url = config.get("messaging", "event_notification_url")
notification_topic = Exchange(name=DEFAULT_EXCHANGE_NAME, type="topic")
with Connection(broker_url) as connection:
producer = Producer(connection)
producer.maybe_declare(notification_topic)
producer.publish(payload, exchange=notification_topic, routing_key=routing_key)
def _handle_cProfile(self, task_id):
"""
If cProfiling is enabled, stop the profiler and write out the data.
:param task_id: the id of the task
:type task_id: unicode
"""
if config.getboolean('profiling', 'enabled') is True:
self.pr.disable()
profile_directory = config.get('profiling', 'directory')
misc.mkdir(profile_directory, mode=0755)
self.pr.dump_stats("%s/%s" % (profile_directory, task_id))
def configure_SSL():
"""
Configures the celery object with BROKER_USE_SSL options
"""
if config.getboolean('tasks', 'celery_require_ssl'):
BROKER_USE_SSL = {
'ca_certs': config.get('tasks', 'cacert'),
'keyfile': config.get('tasks', 'keyfile'),
'certfile': config.get('tasks', 'certfile'),
'cert_reqs': ssl.CERT_REQUIRED,
}
celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def configure_SSL():
"""
Configures the celery object with BROKER_USE_SSL options
"""
if config.getboolean("tasks", "celery_require_ssl"):
BROKER_USE_SSL = {
"ca_certs": config.get("tasks", "cacert"),
"keyfile": config.get("tasks", "keyfile"),
"certfile": config.get("tasks", "certfile"),
"cert_reqs": ssl.CERT_REQUIRED,
}
celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def configure_SSL():
"""
Configures the celery object with BROKER_USE_SSL options
"""
if config.getboolean('tasks', 'celery_require_ssl'):
BROKER_USE_SSL = {
'ca_certs': config.get('tasks', 'cacert'),
'keyfile': config.get('tasks', 'keyfile'),
'certfile': config.get('tasks', 'certfile'),
'cert_reqs': ssl.CERT_REQUIRED,
}
celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def create_mongo_config():
"""
Inspects the pulp config's mongodb settings and returns a data structure
that can be passed to celery for it's mongodb result backend config.
:return: dictionary with keys 'host' and 'database', and optionally with
keys 'user' and 'password', that can be passed to celery as the
config for a mongodb result backend
:rtype: dict
"""
db_name = config.get('database', 'name')
# celery 3.1 doesn't support multiple seeds, so we just use the first one
seeds = config.get('database', 'seeds')
seed = seeds.split(',')[0].strip()
host = seed.split(':')[0]
port = seed.split(':')[1] if ':' in seed else None
mongo_config = {'host': host, 'database': db_name}
if port:
mongo_config['port'] = port
if config.has_option('database', 'username') and config.has_option(
'database', 'password'):
mongo_config['user'] = config.get('database', 'username')
mongo_config['password'] = config.get('database', 'password')
if config.getboolean('database', 'ssl'):
mongo_config['ssl'] = True
ssl_keyfile = config.get('database', 'ssl_keyfile')
ssl_certfile = config.get('database', 'ssl_certfile')
if ssl_keyfile:
mongo_config['ssl_keyfile'] = ssl_keyfile
if ssl_certfile:
mongo_config['ssl_certfile'] = ssl_certfile
verify_ssl = config.getboolean('database', 'verify_ssl')
mongo_config[
'ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE
mongo_config['ssl_ca_certs'] = config.get('database', 'ca_path')
return mongo_config
def _handle_cProfile(self, task_id):
"""
If cProfiling is enabled, stop the profiler and write out the data.
:param task_id: the id of the task
:type task_id: unicode
"""
if config.getboolean('profiling', 'enabled') is True:
self.pr.disable()
profile_directory = config.get('profiling', 'directory')
try:
os.makedirs(profile_directory, 0755)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
self.pr.dump_stats("%s/%s" % (profile_directory, task_id))
def _handle_cProfile(self, task_id):
"""
If cProfiling is enabled, stop the profiler and write out the data.
:param task_id: the id of the task
:type task_id: unicode
"""
if config.getboolean('profiling', 'enabled') is True:
self.pr.disable()
profile_directory = config.get('profiling', 'directory')
try:
os.makedirs(profile_directory, 0755)
except OSError as exc:
if exc.errno != errno.EEXIST:
raise
self.pr.dump_stats("%s/%s" % (profile_directory, task_id))
def __call__(self, *args, **kwargs):
"""
This overrides PulpTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is in one of complete states.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status[
'state'] in constants.CALL_COMPLETE_STATES:
_logger.debug("Task is in the %s state, task-id : [%s]" %
(task_status['state'], self.request.id))
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Also update the worker_name to cover cases where apply_async was called without
# providing the worker name up-front. Skip updating status for eagerly executed tasks,
# since we don't want to track synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
worker_name = self.request.hostname
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
try:
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
set__worker_name=worker_name,
upsert=True)
except NotUniqueError:
# manually retry the upsert. see https://jira.mongodb.org/browse/SERVER-14322
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
set__worker_name=worker_name,
upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
if config.getboolean('profiling', 'enabled') is True:
self.pr = cProfile.Profile()
self.pr.enable()
return super(Task, self).__call__(*args, **kwargs)
def oauth_authentication():
if not config.getboolean('oauth', 'enabled'):
return None, False
username = http.request_info('HTTP_PULP_USER')
auth = http.http_authorization()
cert_pem = http.ssl_client_cert()
if username is None or auth is None:
if cert_pem is not None:
raise AuthenticationFailed(auth_utils.CODE_INVALID_SSL_CERT)
return None, False
meth = http.request_info('REQUEST_METHOD')
url = http.request_url()
query = http.request_info('QUERY_STRING')
userid, is_consumer = factory.authentication_manager().check_oauth(username, meth, url, auth, query)
if userid is None:
raise AuthenticationFailed(auth_utils.CODE_OAUTH)
_LOG.debug("User authenticated with Oauth: %s" % userid)
return userid, is_consumer
def oauth_authentication():
if not config.getboolean('oauth', 'enabled'):
return None, False
username = http.request_info('HTTP_PULP_USER')
auth = http.http_authorization()
cert_pem = http.ssl_client_cert()
if username is None or auth is None:
if cert_pem is not None:
raise PulpCodedAuthenticationException(error_code=error_codes.PLP0027, user=username)
return None, False
meth = http.request_info('REQUEST_METHOD')
url = http.request_url()
query = http.request_info('QUERY_STRING')
userid, is_consumer = factory.authentication_manager().check_oauth(username, meth, url, auth, query)
if userid is None:
raise PulpCodedAuthenticationException(error_code=error_codes.PLP0028, user=username)
_LOG.debug("User authenticated with Oauth: %s" % userid)
return userid, is_consumer
def check_username_password(self, username, password=None):
"""
Check username and password.
Return None if the username and password are not valid
:type username: str
:param username: the login of the user
:type password: str or None
:param password: password of the user, None => do not validate the password
:rtype: str or None
:return: user login corresponding to the credentials
"""
user = self._check_username_password_local(username, password)
if user is None and config.getboolean('ldap', 'enabled'):
user = self._check_username_password_ldap(username, password)
if user is not None:
return user['login']
return None
def check_username_password(self, username, password=None):
"""
Check username and password.
Return None if the username and password are not valid
:type username: str
:param username: the login of the user
:type password: str or None
:param password: password of the user, None => do not validate the password
:rtype: str or None
:return: user login corresponding to the credentials
"""
user = self._check_username_password_local(username, password)
if user is None and config.getboolean('ldap', 'enabled'):
user = self._check_username_password_ldap(username, password)
if user is not None:
return user['login']
return None
def oauth_authentication():
if not config.getboolean('oauth', 'enabled'):
return None, False
username = http.request_info('HTTP_PULP_USER')
auth = http.http_authorization()
cert_pem = http.ssl_client_cert()
if username is None or auth is None:
if cert_pem is not None:
raise AuthenticationFailed(auth_utils.CODE_INVALID_SSL_CERT)
return None, False
meth = http.request_info('REQUEST_METHOD')
url = http.request_url()
query = http.request_info('QUERY_STRING')
userid, is_consumer = factory.authentication_manager().check_oauth(
username, meth, url, auth, query)
if userid is None:
raise AuthenticationFailed(auth_utils.CODE_OAUTH)
_LOG.debug("User authenticated with Oauth: %s" % userid)
return userid, is_consumer
def oauth_authentication():
if not config.getboolean('oauth', 'enabled'):
return None, False
username = http.request_info('HTTP_PULP_USER')
auth = http.http_authorization()
cert_pem = http.ssl_client_cert()
if username is None or auth is None:
if cert_pem is not None:
raise PulpCodedAuthenticationException(
error_code=error_codes.PLP0027, user=username)
return None, False
meth = http.request_info('REQUEST_METHOD')
url = http.request_url()
query = http.request_info('QUERY_STRING')
userid, is_consumer = factory.authentication_manager().check_oauth(
username, meth, url, auth, query)
if userid is None:
raise PulpCodedAuthenticationException(error_code=error_codes.PLP0028,
user=username)
_logger.debug("User authenticated with Oauth: %s" % userid)
return userid, is_consumer
def _check_username_password_ldap(username, password=None):
"""
Check a username and password against the ldap server.
Return None if the username and password are not valid
@type username: str
@param username: the login of the user
@type password: str or None
@param password: password of the user, None => do not validate the password
@rtype: L{pulp.server.db.model.User} instance or None
@return: user corresponding to the credentials
"""
ldap_uri = "ldap://localhost"
if config.has_option('ldap', 'uri'):
ldap_uri = config.get("ldap", "uri")
else:
_log.info("No valid server found, default to localhost")
ldap_base = "dc=localhost"
if config.has_option('ldap', 'base'):
ldap_base = config.get('ldap', 'base')
else:
_log.info("No valid base found, default to localhost")
ldap_filter = None
if config.has_option('ldap', 'filter'):
ldap_filter = config.get('ldap', 'filter')
ldap_tls = False
if config.has_option('ldap', 'tls'):
ldap_tls = config.getboolean('ldap', 'tls')
ldap_server = ldap_connection.LDAPConnection(server=ldap_uri, tls=ldap_tls)
ldap_server.connect()
user = None
if password is not None:
user = ldap_server.authenticate_user(ldap_base, username, password,
filter=ldap_filter)
else:
user_query_manager = factory.user_query_manager()
user = user_query_manager.find_by_login(username)
if user is None:
return None
return user
def send(document, routing_key=None):
"""
Attempt to send a message to the AMQP broker.
If we cannot obtain a new connection then the message will be dropped. Note
that we do not block when waiting for a connection.
:param document: the taskstatus Document we want to send
:type document: mongoengine.Document
:param routing_key: The routing key for the message
:type routing_key: str
"""
# if the user has not enabled notifications, just bail
event_notifications_enabled = config.getboolean(
'messaging', 'event_notifications_enabled')
if not event_notifications_enabled:
return
try:
payload = document.to_json()
except TypeError:
_logger.warn(
"unable to convert document to JSON; event message not sent")
return
broker_url = config.get('messaging', 'event_notification_url')
notification_topic = Exchange(name=DEFAULT_EXCHANGE_NAME, type='topic')
with Connection(broker_url) as connection:
producer = Producer(connection)
producer.maybe_declare(notification_topic)
producer.publish(payload,
exchange=notification_topic,
routing_key=routing_key)
def handle_event(notifier_config, event):
"""
If email is enabled in the server settings, sends an email to each recipient
listed in the notifier_config.
:param notifier_config: dictionary with keys 'subject', which defines the
subject of each email message, and 'addresses',
which is a list of strings that are email addresses
that should receive this notification.
:type notifier_config: dict
:param event: Event instance
:type event: pulp.server.event.data.event
:return: None
"""
if not config.getboolean('email', 'enabled'):
return
body = json.dumps(event.payload, indent=2)
subject = notifier_config['subject']
addresses = notifier_config['addresses']
for address in addresses:
thread = threading.Thread(target=_send_email, args=(subject, body, address))
thread.daemon = True
thread.start()
def __call__(self, *args, **kwargs):
"""
This overrides PulpTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is 'canceled'.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status['state'] == constants.CALL_CANCELED_STATE:
_logger.debug("Task cancel received for task-id : [%s]" % self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Also update the worker_name to cover cases where apply_async was called without
# providing the worker name up-front. Skip updating status for eagerly executed tasks,
# since we don't want to track synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
worker_name = self.request.hostname
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
set__worker_name=worker_name,
upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
if config.getboolean('profiling', 'enabled') is True:
self.pr = cProfile.Profile()
self.pr.enable()
return super(Task, self).__call__(*args, **kwargs)
db_name = config.get('database', 'name')
# celery 3.1 doesn't support multiple seeds, so we just use the first one
seeds = config.get('database', 'seeds')
seed = seeds.split(',')[0].strip()
host = seed.split(':')[0]
port = seed.split(':')[1] if ':' in seed else None
mongo_config = {'host': host, 'database': db_name}
if port:
mongo_config['port'] = port
if config.has_option('database', 'user') and config.has_option('database', 'password'):
mongo_config['user'] = config.get('database', 'user')
mongo_config['password'] = config.get('database', 'password')
return mongo_config
celery.conf.update(CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE)
celery.conf.update(CELERYBEAT_SCHEDULER='pulp.server.async.scheduler.Scheduler')
celery.conf.update(CELERY_RESULT_BACKEND='mongodb')
celery.conf.update(CELERY_MONGODB_BACKEND_SETTINGS=create_mongo_config())
celery.conf.update(CELERY_WORKER_DIRECT=True)
if config.getboolean('tasks', 'celery_require_ssl'):
BROKER_USE_SSL = {
'ca_certs': config.get('tasks', 'cacert'),
'keyfile': config.get('tasks', 'keyfile'),
'certfile': config.get('tasks', 'certfile'),
'cert_reqs': ssl.CERT_REQUIRED,
}
celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
# celery 3.1 doesn't support multiple seeds, so we just use the first one
seeds = config.get('database', 'seeds')
seed = seeds.split(',')[0].strip()
host = seed.split(':')[0]
port = seed.split(':')[1] if ':' in seed else None
mongo_config = {'host': host, 'database': db_name}
if port:
mongo_config['port'] = port
if config.has_option('database', 'username') and config.has_option(
'database', 'password'):
mongo_config['user'] = config.get('database', 'username')
mongo_config['password'] = config.get('database', 'password')
return mongo_config
celery.conf.update(CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE)
celery.conf.update(
CELERYBEAT_SCHEDULER='pulp.server.async.scheduler.Scheduler')
celery.conf.update(CELERY_RESULT_BACKEND='mongodb')
celery.conf.update(CELERY_MONGODB_BACKEND_SETTINGS=create_mongo_config())
celery.conf.update(CELERY_WORKER_DIRECT=True)
if config.getboolean('tasks', 'celery_require_ssl'):
BROKER_USE_SSL = {
'ca_certs': config.get('tasks', 'cacert'),
'keyfile': config.get('tasks', 'keyfile'),
'certfile': config.get('tasks', 'certfile'),
'cert_reqs': ssl.CERT_REQUIRED,
}
celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
