def create_mongo_config(): """ Inspects the pulp config's mongodb settings and returns a data structure that can be passed to celery for it's mongodb result backend config. :return: dictionary with keys 'host' and 'database', and optionally with keys 'user' and 'password', that can be passed to celery as the config for a mongodb result backend :rtype: dict """ db_name = config.get('database', 'name') # celery 3.1 doesn't support multiple seeds, so we just use the first one seeds = config.get('database', 'seeds') seed = seeds.split(',')[0].strip() host = seed.split(':')[0] port = seed.split(':')[1] if ':' in seed else None mongo_config = {'host': host, 'database': db_name} if port: mongo_config['port'] = port if config.has_option('database', 'username') and config.has_option('database', 'password'): mongo_config['user'] = config.get('database', 'username') mongo_config['password'] = config.get('database', 'password') if config.getboolean('database', 'ssl'): mongo_config['ssl'] = True ssl_keyfile = config.get('database', 'ssl_keyfile') ssl_certfile = config.get('database', 'ssl_certfile') if ssl_keyfile: mongo_config['ssl_keyfile'] = ssl_keyfile if ssl_certfile: mongo_config['ssl_certfile'] = ssl_certfile verify_ssl = config.getboolean('database', 'verify_ssl') mongo_config['ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE mongo_config['ssl_ca_certs'] = config.get('database', 'ca_path') return mongo_config
def __call__(self, *args, **kwargs): """ This overrides PulpTask's __call__() method. We use this method for task state tracking of Pulp tasks. """ # Check task status and skip running the task if task state is 'canceled'. try: task_status = TaskStatus.objects.get(task_id=self.request.id) except DoesNotExist: task_status = None if task_status and task_status[ 'state'] == constants.CALL_CANCELED_STATE: _logger.debug("Task cancel received for task-id : [%s]" % self.request.id) return # Update start_time and set the task state to 'running' for asynchronous tasks. # Skip updating status for eagerly executed tasks, since we don't want to track # synchronous tasks in our database. if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) # Using 'upsert' to avoid a possible race condition described in the apply_async method # above. TaskStatus.objects(task_id=self.request.id).update_one( set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, upsert=True) # Run the actual task _logger.debug("Running task : [%s]" % self.request.id) if config.getboolean('profiling', 'enabled') is True: self.pr = cProfile.Profile() self.pr.enable() return super(Task, self).__call__(*args, **kwargs)
def handle_event(notifier_config, event): """ If email is enabled in the server settings, sends an email to each recipient listed in the notifier_config. :param notifier_config: dictionary with keys 'subject', which defines the subject of each email message, and 'addresses', which is a list of strings that are email addresses that should receive this notification. :type notifier_config: dict :param event: Event instance :type event: pulp.server.event.data.event :return: None """ if not config.getboolean('email', 'enabled'): return body = json.dumps(event.data(), indent=2, default=json_util.default) subject = notifier_config['subject'] addresses = notifier_config['addresses'] for address in addresses: thread = threading.Thread(target=_send_email, args=(subject, body, address)) thread.daemon = True thread.start()
def _check_username_password_ldap(self, username, password=None): """ Check a username and password against the ldap server. Return None if the username and password are not valid :type username: str :param username: the login of the user :type password: str or None :param password: password of the user, None => do not validate the password :rtype: L{pulp.server.db.model.auth.User} instance or None :return: user corresponding to the credentials """ ldap_uri = config.get('ldap', 'uri') ldap_base = config.get('ldap', 'base') ldap_tls = config.getboolean('ldap', 'tls') ldap_filter = None if config.has_option('ldap', 'filter'): ldap_filter = config.get('ldap', 'filter') ldap_server = ldap_connection.LDAPConnection(server=ldap_uri, tls=ldap_tls) ldap_server.connect() user = ldap_server.authenticate_user(ldap_base, username, password, filter=ldap_filter) return user
def send(document, routing_key=None): """ Attempt to send a message to the AMQP broker. If we cannot obtain a new connection then the message will be dropped. Note that we do not block when waiting for a connection. :param document: the taskstatus Document we want to send :type document: mongoengine.Document :param routing_key: The routing key for the message :type routing_key: str """ # if the user has not enabled notifications, just bail event_notifications_enabled = config.getboolean("messaging", "event_notifications_enabled") if not event_notifications_enabled: return try: payload = document.to_json() except TypeError: _logger.warn("unable to convert document to JSON; event message not sent") return broker_url = config.get("messaging", "event_notification_url") notification_topic = Exchange(name=DEFAULT_EXCHANGE_NAME, type="topic") with Connection(broker_url) as connection: producer = Producer(connection) producer.maybe_declare(notification_topic) producer.publish(payload, exchange=notification_topic, routing_key=routing_key)
def _handle_cProfile(self, task_id): """ If cProfiling is enabled, stop the profiler and write out the data. :param task_id: the id of the task :type task_id: unicode """ if config.getboolean('profiling', 'enabled') is True: self.pr.disable() profile_directory = config.get('profiling', 'directory') misc.mkdir(profile_directory, mode=0755) self.pr.dump_stats("%s/%s" % (profile_directory, task_id))
def configure_SSL(): """ Configures the celery object with BROKER_USE_SSL options """ if config.getboolean('tasks', 'celery_require_ssl'): BROKER_USE_SSL = { 'ca_certs': config.get('tasks', 'cacert'), 'keyfile': config.get('tasks', 'keyfile'), 'certfile': config.get('tasks', 'certfile'), 'cert_reqs': ssl.CERT_REQUIRED, } celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def configure_SSL(): """ Configures the celery object with BROKER_USE_SSL options """ if config.getboolean("tasks", "celery_require_ssl"): BROKER_USE_SSL = { "ca_certs": config.get("tasks", "cacert"), "keyfile": config.get("tasks", "keyfile"), "certfile": config.get("tasks", "certfile"), "cert_reqs": ssl.CERT_REQUIRED, } celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def create_mongo_config(): """ Inspects the pulp config's mongodb settings and returns a data structure that can be passed to celery for it's mongodb result backend config. :return: dictionary with keys 'host' and 'database', and optionally with keys 'user' and 'password', that can be passed to celery as the config for a mongodb result backend :rtype: dict """ db_name = config.get('database', 'name') # celery 3.1 doesn't support multiple seeds, so we just use the first one seeds = config.get('database', 'seeds') seed = seeds.split(',')[0].strip() host = seed.split(':')[0] port = seed.split(':')[1] if ':' in seed else None mongo_config = {'host': host, 'database': db_name} if port: mongo_config['port'] = port if config.has_option('database', 'username') and config.has_option( 'database', 'password'): mongo_config['user'] = config.get('database', 'username') mongo_config['password'] = config.get('database', 'password') if config.getboolean('database', 'ssl'): mongo_config['ssl'] = True ssl_keyfile = config.get('database', 'ssl_keyfile') ssl_certfile = config.get('database', 'ssl_certfile') if ssl_keyfile: mongo_config['ssl_keyfile'] = ssl_keyfile if ssl_certfile: mongo_config['ssl_certfile'] = ssl_certfile verify_ssl = config.getboolean('database', 'verify_ssl') mongo_config[ 'ssl_cert_reqs'] = ssl.CERT_REQUIRED if verify_ssl else ssl.CERT_NONE mongo_config['ssl_ca_certs'] = config.get('database', 'ca_path') return mongo_config
def _handle_cProfile(self, task_id): """ If cProfiling is enabled, stop the profiler and write out the data. :param task_id: the id of the task :type task_id: unicode """ if config.getboolean('profiling', 'enabled') is True: self.pr.disable() profile_directory = config.get('profiling', 'directory') try: os.makedirs(profile_directory, 0755) except OSError as exc: if exc.errno != errno.EEXIST: raise self.pr.dump_stats("%s/%s" % (profile_directory, task_id))
def __call__(self, *args, **kwargs): """ This overrides PulpTask's __call__() method. We use this method for task state tracking of Pulp tasks. """ # Check task status and skip running the task if task state is in one of complete states. try: task_status = TaskStatus.objects.get(task_id=self.request.id) except DoesNotExist: task_status = None if task_status and task_status[ 'state'] in constants.CALL_COMPLETE_STATES: _logger.debug("Task is in the %s state, task-id : [%s]" % (task_status['state'], self.request.id)) return # Update start_time and set the task state to 'running' for asynchronous tasks. # Also update the worker_name to cover cases where apply_async was called without # providing the worker name up-front. Skip updating status for eagerly executed tasks, # since we don't want to track synchronous tasks in our database. if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) worker_name = self.request.hostname # Using 'upsert' to avoid a possible race condition described in the apply_async method # above. try: TaskStatus.objects(task_id=self.request.id).update_one( set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, set__worker_name=worker_name, upsert=True) except NotUniqueError: # manually retry the upsert. see https://jira.mongodb.org/browse/SERVER-14322 TaskStatus.objects(task_id=self.request.id).update_one( set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, set__worker_name=worker_name, upsert=True) # Run the actual task _logger.debug("Running task : [%s]" % self.request.id) if config.getboolean('profiling', 'enabled') is True: self.pr = cProfile.Profile() self.pr.enable() return super(Task, self).__call__(*args, **kwargs)
def oauth_authentication(): if not config.getboolean('oauth', 'enabled'): return None, False username = http.request_info('HTTP_PULP_USER') auth = http.http_authorization() cert_pem = http.ssl_client_cert() if username is None or auth is None: if cert_pem is not None: raise AuthenticationFailed(auth_utils.CODE_INVALID_SSL_CERT) return None, False meth = http.request_info('REQUEST_METHOD') url = http.request_url() query = http.request_info('QUERY_STRING') userid, is_consumer = factory.authentication_manager().check_oauth(username, meth, url, auth, query) if userid is None: raise AuthenticationFailed(auth_utils.CODE_OAUTH) _LOG.debug("User authenticated with Oauth: %s" % userid) return userid, is_consumer
def oauth_authentication(): if not config.getboolean('oauth', 'enabled'): return None, False username = http.request_info('HTTP_PULP_USER') auth = http.http_authorization() cert_pem = http.ssl_client_cert() if username is None or auth is None: if cert_pem is not None: raise PulpCodedAuthenticationException(error_code=error_codes.PLP0027, user=username) return None, False meth = http.request_info('REQUEST_METHOD') url = http.request_url() query = http.request_info('QUERY_STRING') userid, is_consumer = factory.authentication_manager().check_oauth(username, meth, url, auth, query) if userid is None: raise PulpCodedAuthenticationException(error_code=error_codes.PLP0028, user=username) _LOG.debug("User authenticated with Oauth: %s" % userid) return userid, is_consumer
def check_username_password(self, username, password=None): """ Check username and password. Return None if the username and password are not valid :type username: str :param username: the login of the user :type password: str or None :param password: password of the user, None => do not validate the password :rtype: str or None :return: user login corresponding to the credentials """ user = self._check_username_password_local(username, password) if user is None and config.getboolean('ldap', 'enabled'): user = self._check_username_password_ldap(username, password) if user is not None: return user['login'] return None
def oauth_authentication(): if not config.getboolean('oauth', 'enabled'): return None, False username = http.request_info('HTTP_PULP_USER') auth = http.http_authorization() cert_pem = http.ssl_client_cert() if username is None or auth is None: if cert_pem is not None: raise AuthenticationFailed(auth_utils.CODE_INVALID_SSL_CERT) return None, False meth = http.request_info('REQUEST_METHOD') url = http.request_url() query = http.request_info('QUERY_STRING') userid, is_consumer = factory.authentication_manager().check_oauth( username, meth, url, auth, query) if userid is None: raise AuthenticationFailed(auth_utils.CODE_OAUTH) _LOG.debug("User authenticated with Oauth: %s" % userid) return userid, is_consumer
def oauth_authentication(): if not config.getboolean('oauth', 'enabled'): return None, False username = http.request_info('HTTP_PULP_USER') auth = http.http_authorization() cert_pem = http.ssl_client_cert() if username is None or auth is None: if cert_pem is not None: raise PulpCodedAuthenticationException( error_code=error_codes.PLP0027, user=username) return None, False meth = http.request_info('REQUEST_METHOD') url = http.request_url() query = http.request_info('QUERY_STRING') userid, is_consumer = factory.authentication_manager().check_oauth( username, meth, url, auth, query) if userid is None: raise PulpCodedAuthenticationException(error_code=error_codes.PLP0028, user=username) _logger.debug("User authenticated with Oauth: %s" % userid) return userid, is_consumer
def _check_username_password_ldap(username, password=None): """ Check a username and password against the ldap server. Return None if the username and password are not valid @type username: str @param username: the login of the user @type password: str or None @param password: password of the user, None => do not validate the password @rtype: L{pulp.server.db.model.User} instance or None @return: user corresponding to the credentials """ ldap_uri = "ldap://localhost" if config.has_option('ldap', 'uri'): ldap_uri = config.get("ldap", "uri") else: _log.info("No valid server found, default to localhost") ldap_base = "dc=localhost" if config.has_option('ldap', 'base'): ldap_base = config.get('ldap', 'base') else: _log.info("No valid base found, default to localhost") ldap_filter = None if config.has_option('ldap', 'filter'): ldap_filter = config.get('ldap', 'filter') ldap_tls = False if config.has_option('ldap', 'tls'): ldap_tls = config.getboolean('ldap', 'tls') ldap_server = ldap_connection.LDAPConnection(server=ldap_uri, tls=ldap_tls) ldap_server.connect() user = None if password is not None: user = ldap_server.authenticate_user(ldap_base, username, password, filter=ldap_filter) else: user_query_manager = factory.user_query_manager() user = user_query_manager.find_by_login(username) if user is None: return None return user
def send(document, routing_key=None): """ Attempt to send a message to the AMQP broker. If we cannot obtain a new connection then the message will be dropped. Note that we do not block when waiting for a connection. :param document: the taskstatus Document we want to send :type document: mongoengine.Document :param routing_key: The routing key for the message :type routing_key: str """ # if the user has not enabled notifications, just bail event_notifications_enabled = config.getboolean( 'messaging', 'event_notifications_enabled') if not event_notifications_enabled: return try: payload = document.to_json() except TypeError: _logger.warn( "unable to convert document to JSON; event message not sent") return broker_url = config.get('messaging', 'event_notification_url') notification_topic = Exchange(name=DEFAULT_EXCHANGE_NAME, type='topic') with Connection(broker_url) as connection: producer = Producer(connection) producer.maybe_declare(notification_topic) producer.publish(payload, exchange=notification_topic, routing_key=routing_key)
def handle_event(notifier_config, event): """ If email is enabled in the server settings, sends an email to each recipient listed in the notifier_config. :param notifier_config: dictionary with keys 'subject', which defines the subject of each email message, and 'addresses', which is a list of strings that are email addresses that should receive this notification. :type notifier_config: dict :param event: Event instance :type event: pulp.server.event.data.event :return: None """ if not config.getboolean('email', 'enabled'): return body = json.dumps(event.payload, indent=2) subject = notifier_config['subject'] addresses = notifier_config['addresses'] for address in addresses: thread = threading.Thread(target=_send_email, args=(subject, body, address)) thread.daemon = True thread.start()
def __call__(self, *args, **kwargs): """ This overrides PulpTask's __call__() method. We use this method for task state tracking of Pulp tasks. """ # Check task status and skip running the task if task state is 'canceled'. try: task_status = TaskStatus.objects.get(task_id=self.request.id) except DoesNotExist: task_status = None if task_status and task_status['state'] == constants.CALL_CANCELED_STATE: _logger.debug("Task cancel received for task-id : [%s]" % self.request.id) return # Update start_time and set the task state to 'running' for asynchronous tasks. # Also update the worker_name to cover cases where apply_async was called without # providing the worker name up-front. Skip updating status for eagerly executed tasks, # since we don't want to track synchronous tasks in our database. if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) worker_name = self.request.hostname # Using 'upsert' to avoid a possible race condition described in the apply_async method # above. TaskStatus.objects(task_id=self.request.id).update_one( set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, set__worker_name=worker_name, upsert=True) # Run the actual task _logger.debug("Running task : [%s]" % self.request.id) if config.getboolean('profiling', 'enabled') is True: self.pr = cProfile.Profile() self.pr.enable() return super(Task, self).__call__(*args, **kwargs)
db_name = config.get('database', 'name') # celery 3.1 doesn't support multiple seeds, so we just use the first one seeds = config.get('database', 'seeds') seed = seeds.split(',')[0].strip() host = seed.split(':')[0] port = seed.split(':')[1] if ':' in seed else None mongo_config = {'host': host, 'database': db_name} if port: mongo_config['port'] = port if config.has_option('database', 'user') and config.has_option('database', 'password'): mongo_config['user'] = config.get('database', 'user') mongo_config['password'] = config.get('database', 'password') return mongo_config celery.conf.update(CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE) celery.conf.update(CELERYBEAT_SCHEDULER='pulp.server.async.scheduler.Scheduler') celery.conf.update(CELERY_RESULT_BACKEND='mongodb') celery.conf.update(CELERY_MONGODB_BACKEND_SETTINGS=create_mongo_config()) celery.conf.update(CELERY_WORKER_DIRECT=True) if config.getboolean('tasks', 'celery_require_ssl'): BROKER_USE_SSL = { 'ca_certs': config.get('tasks', 'cacert'), 'keyfile': config.get('tasks', 'keyfile'), 'certfile': config.get('tasks', 'certfile'), 'cert_reqs': ssl.CERT_REQUIRED, } celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
# celery 3.1 doesn't support multiple seeds, so we just use the first one seeds = config.get('database', 'seeds') seed = seeds.split(',')[0].strip() host = seed.split(':')[0] port = seed.split(':')[1] if ':' in seed else None mongo_config = {'host': host, 'database': db_name} if port: mongo_config['port'] = port if config.has_option('database', 'username') and config.has_option( 'database', 'password'): mongo_config['user'] = config.get('database', 'username') mongo_config['password'] = config.get('database', 'password') return mongo_config celery.conf.update(CELERYBEAT_SCHEDULE=CELERYBEAT_SCHEDULE) celery.conf.update( CELERYBEAT_SCHEDULER='pulp.server.async.scheduler.Scheduler') celery.conf.update(CELERY_RESULT_BACKEND='mongodb') celery.conf.update(CELERY_MONGODB_BACKEND_SETTINGS=create_mongo_config()) celery.conf.update(CELERY_WORKER_DIRECT=True) if config.getboolean('tasks', 'celery_require_ssl'): BROKER_USE_SSL = { 'ca_certs': config.get('tasks', 'cacert'), 'keyfile': config.get('tasks', 'keyfile'), 'certfile': config.get('tasks', 'certfile'), 'cert_reqs': ssl.CERT_REQUIRED, } celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)