def _send_email(subject, body, to_address): """ Send a text email to one recipient :param subject: email subject :type subject: basestring :param body: text body of the email :type body: basestring :param to_address: email address to send to :type to_address: basestring :return: None """ host = config.get('email', 'host') port = config.getint('email', 'port') from_address = config.get('email', 'from') message = MIMEText(body) message['Subject'] = subject message['From'] = from_address message['To'] = to_address try: connection = smtplib.SMTP(host=host, port=port) except smtplib.SMTPConnectError: logger.exception('SMTP connection failed to %s on %s' % (host, port)) return try: connection.sendmail(from_address, to_address, message.as_string()) except smtplib.SMTPException, e: try: logger.exception('Error sending mail.') except AttributeError: logger.error('SMTP error while sending mail')
def create_consumer_payload(self, repo, config): payload = {} ##TODO for jdob: load the pulp.conf and make it accessible to distributor payload["repo_name"] = repo.display_name payload["server_name"] = pulp_server_config.get("server", "server_name") ssl_ca_path = pulp_server_config.get("security", "ssl_ca_certificate") if os.path.exists(ssl_ca_path): payload["ca_cert"] = open(pulp_server_config.get("security", "ssl_ca_certificate")).read() else: payload["ca_cert"] = config.get("https_ca") payload["relative_path"] = "/".join((RELATIVE_URL, self.get_repo_relative_path(repo, config))) payload["protocols"] = [] if config.get("http"): payload["protocols"].append("http") if config.get("https"): payload["protocols"].append("https") payload["gpg_keys"] = [] if config.get("gpgkey") is not None: payload["gpg_keys"] = config.get("gpgkey") payload["client_cert"] = None if config.get("auth_cert") and config.get("auth_ca"): payload["client_cert"] = config.get("auth_cert") else: # load the global auth if enabled repo_auth_config = load_config() global_cert_dir = repo_auth_config.get("repos", "global_cert_location") global_auth_cert = os.path.join(global_cert_dir, "pulp-global-repo.cert") global_auth_key = os.path.join(global_cert_dir, "pulp-global-repo.key") global_auth_ca = os.path.join(global_cert_dir, "pulp-global-repo.ca") if os.path.exists(global_auth_ca) and os.path.exists(global_auth_cert): payload["global_auth_cert"] = open(global_auth_cert).read() payload["global_auth_key"] = open(global_auth_key).read() payload["global_auth_ca"] = open(global_auth_ca).read() return payload
def init(): url = Services.get_url() broker = Broker(url) broker.ssl.ca_certificate = config.get('messaging', 'cacert') broker.ssl.client_certificate = config.get('messaging', 'clientcert') Domain.broker.add(broker) _logger.info(_('AMQP broker configured: %(b)s'), {'b': broker})
def create_mongo_config(): """ Inspects the pulp config's mongodb settings and returns a data structure that can be passed to celery for it's mongodb result backend config. :return: dictionary with keys 'host' and 'database', and optionally with keys 'user' and 'password', that can be passed to celery as the config for a mongodb result backend :rtype: dict """ db_name = config.get('database', 'name') # celery 3.1 doesn't support multiple seeds, so we just use the first one seeds = config.get('database', 'seeds') seed = seeds.split(',')[0].strip() host = seed.split(':')[0] port = seed.split(':')[1] if ':' in seed else None mongo_config = {'host': host, 'database': db_name} if port: mongo_config['port'] = port if config.has_option('database', 'username') and config.has_option( 'database', 'password'): mongo_config['user'] = config.get('database', 'username') mongo_config['password'] = config.get('database', 'password') return mongo_config
def _send_email(subject, body, to_address): """ Send a text email to one recipient :param subject: email subject :type subject: basestring :param body: text body of the email :type body: basestring :param to_address: email address to send to :type to_address: basestring :return: None """ host = config.get('email', 'host') port = config.getint('email', 'port') from_address = config.get('email', 'from') message = MIMEText(body) message['Subject'] = subject message['From'] = from_address message['To'] = to_address try: connection = smtplib.SMTP(host=host, port=port) except smtplib.SMTPConnectError: logger.error('SMTP connection failed to %s on %s' % (host, port)) return try: connection.sendmail(from_address, to_address, message.as_string()) except smtplib.SMTPException, e: try: logger.error('Error sending mail: %s' % e.message) except AttributeError: logger.error('SMTP error while sending mail')
def _get_streamer_url(catalog_entry, signing_key): """ Build a URL that can be used to retrieve the file in the catalog entry from the lazy streamer. :param catalog_entry: The catalog entry to get the URL for. :type catalog_entry: pulp.server.db.model.LazyCatalogEntry :param signing_key: The server private RSA key to sign the url with. :type signing_key: M2Crypto.RSA.RSA :return: The signed streamer URL which corresponds to the catalog entry. :rtype: str """ try: https_retrieval = parse_bool(pulp_conf.get('lazy', 'https_retrieval')) except Unparsable: raise PulpCodedTaskException(error_codes.PLP1014, section='lazy', key='https_retrieval', reason=_('The value is not boolean')) retrieval_scheme = 'https' if https_retrieval else 'http' host = pulp_conf.get('lazy', 'redirect_host') port = pulp_conf.get('lazy', 'redirect_port') path_prefix = pulp_conf.get('lazy', 'redirect_path') netloc = (host + ':' + port) if port else host path = os.path.join(path_prefix, catalog_entry.path.lstrip('/')) unsigned_url = urlunsplit((retrieval_scheme, netloc, path, None, None)) # Sign the URL for a year to avoid the URL expiring before the task completes return str(URL(unsigned_url).sign(signing_key, expiration=31536000))
def pulp_bindings(): """ Get a pulp bindings object for this node. Properties defined in the pulp server configuration are used when not defined in the node configuration. :return: A pulp bindings object. :rtype: Bindings """ node_conf = read_config() oauth = node_conf.oauth verify_ssl = parse_bool(node_conf.main.verify_ssl) ca_path = node_conf.main.ca_path host = pulp_conf.get('server', 'server_name') key = pulp_conf.get('oauth', 'oauth_key') secret = pulp_conf.get('oauth', 'oauth_secret') connection = PulpConnection( host=host, port=443, oauth_key=key, oauth_secret=secret, oauth_user=oauth.user_id, verify_ssl=verify_ssl, ca_path=ca_path) bindings = Bindings(connection) return bindings
def start(cls): url = config.get('messaging', 'url') transport = config.get('messaging', 'transport') # asynchronous reply cls.reply_handler = ReplyHandler(url, transport) cls.reply_handler.start() _logger.info(_('AMQP reply handler started'))
def redirect(request, key): """ Redirected GET request. :param request: The WSGI request object. :type request: django.core.handlers.wsgi.WSGIRequest :param key: A private RSA key. :type key: RSA.RSA :return: A redirect or not-found reply. :rtype: django.http.HttpResponse """ path = os.path.realpath(request.path_info) scheme = request.environ['REQUEST_SCHEME'] host = request.environ['SERVER_NAME'] port = request.environ['SERVER_PORT'] query = request.environ['QUERY_STRING'] remote_ip = request.environ['REMOTE_ADDR'] redirect_host = pulp_conf.get('lazy', 'redirect_host') redirect_port = pulp_conf.get('lazy', 'redirect_port') redirect_path = pulp_conf.get('lazy', 'redirect_path') redirect = ContentView.urljoin(scheme, redirect_host or host, redirect_port or port, redirect_path, path, query) url = URL(redirect) signed = url.sign(key, remote_ip=remote_ip) return HttpResponseRedirect(str(signed))
def redirect(request, key): """ Redirected GET request. :param request: The WSGI request object. :type request: django.core.handlers.wsgi.WSGIRequest :param key: A private RSA key. :type key: RSA.RSA :return: A redirect or not-found reply. :rtype: django.http.HttpResponse """ path = os.path.realpath(request.path_info) scheme = request.environ['REQUEST_SCHEME'] host = request.environ['SERVER_NAME'] port = request.environ['SERVER_PORT'] query = request.environ['QUERY_STRING'] remote_ip = request.environ['REMOTE_ADDR'] redirect_host = pulp_conf.get('lazy', 'redirect_host') redirect_port = pulp_conf.get('lazy', 'redirect_port') redirect_path = pulp_conf.get('lazy', 'redirect_path') redirect = ContentView.urljoin( scheme, redirect_host or host, redirect_port or port, redirect_path, path, query) url = URL(redirect) signed = url.sign(key, remote_ip=remote_ip) return HttpResponseRedirect(str(signed))
def init(): url = config.get('messaging', 'url') transport = config.get('messaging', 'transport') broker = Broker(url, transport=transport) broker.cacert = config.get('messaging', 'cacert') broker.clientcert = config.get('messaging', 'clientcert') _logger.info(_('AMQP broker configured: %(b)s'), {'b': broker})
def start(cls): url = config.get('messaging', 'url') transport = config.get('messaging', 'transport') # asynchronous reply cls.reply_handler = ReplyHandler(url, transport) cls.reply_handler.start() log.info(_('AMQP reply handler started'))
def _check_username_password_ldap(self, username, password=None): """ Check a username and password against the ldap server. Return None if the username and password are not valid :type username: str :param username: the login of the user :type password: str or None :param password: password of the user, None => do not validate the password :rtype: L{pulp.server.db.model.auth.User} instance or None :return: user corresponding to the credentials """ ldap_uri = config.get('ldap', 'uri') ldap_base = config.get('ldap', 'base') ldap_tls = config.getboolean('ldap', 'tls') ldap_filter = None if config.has_option('ldap', 'filter'): ldap_filter = config.get('ldap', 'filter') ldap_server = ldap_connection.LDAPConnection(server=ldap_uri, tls=ldap_tls) ldap_server.connect() user = ldap_server.authenticate_user(ldap_base, username, password, filter=ldap_filter) return user
def init(): url = config.get('messaging', 'url') transport = config.get('messaging', 'transport') broker = Broker(url, transport=transport) broker.cacert = config.get('messaging', 'cacert') broker.clientcert = config.get('messaging', 'clientcert') log.info(_('AMQP broker configured: %(b)s'), {'b': broker})
def init(): url = config.get('messaging', 'url') transport = config.get('messaging', 'transport') broker = Broker(url, transport=transport) broker.cacert = config.get('messaging', 'cacert') broker.clientcert = config.get('messaging', 'clientcert') log.info('AMQP broker configured: %s', broker)
def test_init(self): Services.init() url = pulp_conf.get('messaging', 'url') ca_cert = pulp_conf.get('messaging', 'cacert') client_cert = pulp_conf.get('messaging', 'clientcert') broker = Broker(url) self.assertEqual(broker.url, URL(url)) self.assertEqual(broker.cacert, ca_cert) self.assertEqual(broker.clientcert, client_cert)
def on_success(self, retval, task_id, args, kwargs): """ This overrides the success handler run by the worker when the task executes successfully. It updates state, finish_time and traceback of the relevant task status for asynchronous tasks. Skip updating status for synchronous tasks. :param retval: The return value of the task. :param task_id: Unique id of the executed task. :param args: Original arguments for the executed task. :param kwargs: Original keyword arguments for the executed task. """ _logger.debug("Task successful : [%s]" % task_id) if kwargs.get('scheduled_call_id') is not None: if not isinstance(retval, AsyncResult): _logger.info( _('resetting consecutive failure count for schedule %(id)s' ) % {'id': kwargs['scheduled_call_id']}) utils.reset_failure_count(kwargs['scheduled_call_id']) if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) finish_time = dateutils.format_iso8601_datetime(now) task_status = TaskStatus.objects.get(task_id=task_id) task_status['finish_time'] = finish_time task_status['result'] = retval # Only set the state to finished if it's not already in a complete state. This is # important for when the task has been canceled, so we don't move the task from canceled # to finished. if task_status['state'] not in constants.CALL_COMPLETE_STATES: task_status['state'] = constants.CALL_FINISHED_STATE if isinstance(retval, TaskResult): task_status['result'] = retval.return_value if retval.error: task_status['error'] = retval.error.to_dict() if retval.spawned_tasks: task_list = [] for spawned_task in retval.spawned_tasks: if isinstance(spawned_task, AsyncResult): task_list.append(spawned_task.task_id) elif isinstance(spawned_task, dict): task_list.append(spawned_task['task_id']) task_status['spawned_tasks'] = task_list if isinstance(retval, AsyncResult): task_status['spawned_tasks'] = [ retval.task_id, ] task_status['result'] = None task_status.save() if config.get('profiling', 'enabled') is True: profile_directory = config.get('profiling', 'directory') self.pr.disable() self.pr.dump_stats("%s/%s" % (profile_directory, task_id)) common_utils.delete_working_directory()
def create_consumer_payload(self, repo, config, binding_config): """ Called when a consumer binds to a repository using this distributor. This call should return a dictionary describing all data the consumer will need to access the repository. :param repo: metadata describing the repository :type repo: pulp.plugins.model.Repository :param config: plugin configuration :type config: pulp.plugins.config.PluginCallConfiguration :param binding_config: configuration applicable only for the specific consumer the payload is generated for; this will be None if there are no specific options for the consumer in question :type binding_config: object or None :return: dictionary of relevant data :rtype: dict """ payload = dict() payload['repo_name'] = repo.display_name payload['server_name'] = pulp_server_config.get('server', 'server_name') ssl_ca_path = pulp_server_config.get('security', 'ssl_ca_certificate') try: payload['ca_cert'] = open(ssl_ca_path).read() except (OSError, IOError): payload['ca_cert'] = config.get('https_ca') payload['relative_path'] = \ '/'.join([RELATIVE_URL, configuration.get_repo_relative_path(repo, config)]) payload['protocols'] = [] if config.get('http'): payload['protocols'].append('http') if config.get('https'): payload['protocols'].append('https') payload['gpg_keys'] = [] if config.get('gpgkey') is not None: payload['gpg_keys'] = {'pulp.key': config.get('gpgkey')} payload['client_cert'] = None if config.get('auth_cert') and config.get('auth_ca'): payload['client_cert'] = config.get('auth_cert') else: # load the global auth if enabled repo_auth_config = configuration.load_config(constants.REPO_AUTH_CONFIG_FILE) global_cert_dir = repo_auth_config.get('repos', 'global_cert_location') global_auth_cert = os.path.join(global_cert_dir, 'pulp-global-repo.cert') global_auth_key = os.path.join(global_cert_dir, 'pulp-global-repo.key') global_auth_ca = os.path.join(global_cert_dir, 'pulp-global-repo.ca') if os.path.exists(global_auth_ca) and \ os.path.exists(global_auth_cert) and \ os.path.exists(global_auth_key): payload['global_auth_cert'] = open(global_auth_cert).read() payload['global_auth_key'] = open(global_auth_key).read() payload['global_auth_ca'] = open(global_auth_ca).read() return payload
def test_init(self, mock_broker): Services.init() url = pulp_conf.get('messaging', 'url') transport = pulp_conf.get('messaging', 'transport') ca_cert = pulp_conf.get('messaging', 'cacert') client_cert = pulp_conf.get('messaging', 'clientcert') mock_broker.assert_called_with(url, transport=transport) broker = mock_broker() self.assertEqual(broker.cacert, ca_cert) self.assertEqual(broker.clientcert, client_cert)
def add_connector(): """ Configure and add the gofer connector used to connect to the message broker. This call is idempotent. """ url = get_url() connector = Connector(url) connector.ssl.ca_certificate = config.get('messaging', 'cacert') connector.ssl.client_certificate = config.get('messaging', 'clientcert') connector.add()
def on_success(self, retval, task_id, args, kwargs): """ This overrides the success handler run by the worker when the task executes successfully. It updates state, finish_time and traceback of the relevant task status for asynchronous tasks. Skip updating status for synchronous tasks. :param retval: The return value of the task. :param task_id: Unique id of the executed task. :param args: Original arguments for the executed task. :param kwargs: Original keyword arguments for the executed task. """ _logger.debug("Task successful : [%s]" % task_id) if kwargs.get('scheduled_call_id') is not None: if not isinstance(retval, AsyncResult): _logger.info(_('resetting consecutive failure count for schedule %(id)s') % {'id': kwargs['scheduled_call_id']}) utils.reset_failure_count(kwargs['scheduled_call_id']) if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) finish_time = dateutils.format_iso8601_datetime(now) task_status = TaskStatus.objects.get(task_id=task_id) task_status['finish_time'] = finish_time task_status['result'] = retval # Only set the state to finished if it's not already in a complete state. This is # important for when the task has been canceled, so we don't move the task from canceled # to finished. if task_status['state'] not in constants.CALL_COMPLETE_STATES: task_status['state'] = constants.CALL_FINISHED_STATE if isinstance(retval, TaskResult): task_status['result'] = retval.return_value if retval.error: task_status['error'] = retval.error.to_dict() if retval.spawned_tasks: task_list = [] for spawned_task in retval.spawned_tasks: if isinstance(spawned_task, AsyncResult): task_list.append(spawned_task.task_id) elif isinstance(spawned_task, dict): task_list.append(spawned_task['task_id']) task_status['spawned_tasks'] = task_list if isinstance(retval, AsyncResult): task_status['spawned_tasks'] = [retval.task_id, ] task_status['result'] = None task_status.save() if config.get('profiling', 'enabled') is True: profile_directory = config.get('profiling', 'directory') self.pr.disable() self.pr.dump_stats("%s/%s" % (profile_directory, task_id)) common_utils.delete_working_directory()
def __init__(self, **kwargs): super(ContentView, self).__init__(**kwargs) self.key = Key.load(pulp_conf.get('authentication', 'rsa_key')) # Make sure all requested paths fall under these sub-directories, otherwise # we might find ourselves serving private keys to all and sundry. local_storage = pulp_conf.get('server', 'storage_dir') self.safe_serving_paths = [os.path.realpath(os.path.join(local_storage, subdir)) for subdir in SAFE_STORAGE_SUBDIRS] logger.debug(_("Serving Pulp content from {paths}; ensure " "Apache's mod_xsendfile is configured to serve from " "these paths as well").format(paths=str(self.safe_serving_paths)))
def get_url(): """ This constructs a gofer 2.x URL and is intended to maintain configuration file backwards compatibility until pulp 3.0 :return: A gofer 2.x broker URL. :rtype: str """ url = config.get('messaging', 'url') adapter = config.get('messaging', 'transport') return '+'.join((adapter, url))
def configure_SSL(): """ Configures the celery object with BROKER_USE_SSL options """ if config.getboolean('tasks', 'celery_require_ssl'): BROKER_USE_SSL = { 'ca_certs': config.get('tasks', 'cacert'), 'keyfile': config.get('tasks', 'keyfile'), 'certfile': config.get('tasks', 'certfile'), 'cert_reqs': ssl.CERT_REQUIRED, } celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def configure_SSL(): """ Configures the celery object with BROKER_USE_SSL options """ if config.getboolean("tasks", "celery_require_ssl"): BROKER_USE_SSL = { "ca_certs": config.get("tasks", "cacert"), "keyfile": config.get("tasks", "keyfile"), "certfile": config.get("tasks", "certfile"), "cert_reqs": ssl.CERT_REQUIRED, } celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def check_oauth(self, username, method, url, auth, query): """ Check OAuth header credentials. Return None if the credentials are invalid :type username: str :param username: username corresponding to credentials :type method: str :param method: http method :type url: str :param url: request url :type auth: str :param auth: http authorization header value :type query: str :param query: http request query string :rtype: str or None :return: user login corresponding to the credentials """ is_consumer = False headers = {'Authorization': auth} req = oauth2.Request.from_request(method, url, headers, query_string=query) if not req: return None, is_consumer if not (config.has_option('oauth', 'oauth_key') and config.has_option('oauth', 'oauth_secret')): _logger.error( _("Attempting OAuth authentication and you do not have oauth_key and " "oauth_secret in pulp.conf")) return None, is_consumer key = config.get('oauth', 'oauth_key') secret = config.get('oauth', 'oauth_secret') consumer = oauth2.Consumer(key=key, secret=secret) server = oauth2.Server() server.add_signature_method(oauth2.SignatureMethod_HMAC_SHA1()) try: # this call has a return value, but failures are noted by the exception server.verify_request(req, consumer, None) except oauth2.Error, e: _logger.error('error verifying OAuth signature: %s' % e) return None, is_consumer
def check_oauth(self, username, method, url, auth, query): """ Check OAuth header credentials. Return None if the credentials are invalid :type username: str :param username: username corresponding to credentials :type method: str :param method: http method :type url: str :param url: request url :type auth: str :param auth: http authorization header value :type query: str :param query: http request query string :rtype: str or None :return: user login corresponding to the credentials """ is_consumer = False headers = {'Authorization': auth} req = oauth2.Request.from_request(method, url, headers, query_string=query) if not req: return None, is_consumer if not (config.has_option('oauth', 'oauth_key') and config.has_option('oauth', 'oauth_secret')): _logger.error(_("Attempting OAuth authentication and you do not have oauth_key and " "oauth_secret in pulp.conf")) return None, is_consumer key = config.get('oauth', 'oauth_key') secret = config.get('oauth', 'oauth_secret') consumer = oauth2.Consumer(key=key, secret=secret) server = oauth2.Server() server.add_signature_method(oauth2.SignatureMethod_HMAC_SHA1()) try: # this call has a return value, but failures are noted by the exception server.verify_request(req, consumer, None) except oauth2.Error, e: _logger.error('error verifying OAuth signature: %s' % e) return None, is_consumer
def setUpClass(cls): if not os.path.exists(cls.TMP_ROOT): os.makedirs(cls.TMP_ROOT) stop_logging() path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data', 'pulp.conf') pulp_conf.read(path) start_logging() storage_dir = pulp_conf.get('server', 'storage_dir') if not os.path.exists(storage_dir): os.makedirs(storage_dir) shutil.rmtree(storage_dir + '/*', ignore_errors=True) name = pulp_conf.get('database', 'name') connection.initialize(name) managers.initialize()
def __init__(self, **kwargs): super(ContentView, self).__init__(**kwargs) self.key = Key.load(pulp_conf.get('authentication', 'rsa_key')) # Make sure all requested paths fall under these sub-directories, otherwise # we might find ourselves serving private keys to all and sundry. local_storage = pulp_conf.get('server', 'storage_dir') self.safe_serving_paths = [ os.path.realpath(os.path.join(local_storage, subdir)) for subdir in SAFE_STORAGE_SUBDIRS ] logger.debug( _("Serving Pulp content from {paths}; ensure " "Apache's mod_xsendfile is configured to serve from " "these paths as well").format( paths=str(self.safe_serving_paths)))
def tearDownClass(cls): name = pulp_conf.get('database', 'name') db = pymongo.database.Database(connection._CONNECTION, name) for name in db.collection_names(): if name[:7] == 'system.': continue db.drop_collection(name)
def _missing_units(self, unit_inventory): """ Determine the list of units defined upstream inventory that are not in the local inventory. :param unit_inventory: The inventory of both upstream and local content units. :type unit_inventory: UnitInventory :return: The list of units to be added. Each item: (unit_upstream, unit_to_be_added) :rtype: list """ new_units = [] storage_dir = pulp_conf.get("server", "storage_dir") for unit in unit_inventory.upstream_only(): unit["metadata"].pop("_id") unit["metadata"].pop("_ns") type_id = unit["type_id"] unit_key = unit["unit_key"] metadata = unit["metadata"] storage_path = unit.get("storage_path") if storage_path: relative_path = unit["_relative_path"] storage_path = "/".join((storage_dir, relative_path)) unit_in = Unit(type_id, unit_key, metadata, storage_path) new_units.append((unit, unit_in)) return new_units
def associated_unit(association, unit): """ Create a dictionary that is a composite of a unit association and the unit. :param association: A unit association DB record. :type association: dict :param unit: A DB unit record. :type unit: dict :return: A composite of the unit association and the unit. :rtype: dict """ unit_key = {} unit_id = unit.pop('_id') type_id = association['unit_type_id'] for key in get_unit_key_fields_for_type(type_id): unit_key[key] = unit.pop(key, None) storage_dir = pulp_conf.get('server', 'storage_dir') storage_path = unit.pop('_storage_path', None) last_updated = unit.pop('_last_updated', 0.0) if storage_path: relative_path = storage_path[len(storage_dir):].lstrip('/') else: relative_path = None return dict(unit_id=unit_id, type_id=type_id, unit_key=unit_key, storage_path=storage_path, relative_path=relative_path, last_updated=last_updated, metadata=unit)
def get(self, request): """ Process the GET content request. :param request: The WSGI request object. :type request: django.core.handlers.wsgi.WSGIRequest :return: An appropriate HTTP reply :rtype: django.http.HttpResponse """ host = request.get_host() path = os.path.realpath(request.path_info) lazy_enabled = parse_bool(pulp_conf.get('lazy', 'enabled')) # Authorization if not allow_access(request.environ, host): # Not Authorized return HttpResponseForbidden() # Already downloaded if os.path.exists(path): return self.x_send(path) # Redirect if lazy_enabled: return self.redirect(request, self.key) # NotFound return HttpResponseNotFound(request.path_info)
def __call__(self, *args, **kwargs): """ This overrides PulpTask's __call__() method. We use this method for task state tracking of Pulp tasks. """ # Check task status and skip running the task if task state is 'canceled'. try: task_status = TaskStatus.objects.get(task_id=self.request.id) except DoesNotExist: task_status = None if task_status and task_status['state'] == constants.CALL_CANCELED_STATE: _logger.debug("Task cancel received for task-id : [%s]" % self.request.id) return # Update start_time and set the task state to 'running' for asynchronous tasks. # Skip updating status for eagerly executed tasks, since we don't want to track # synchronous tasks in our database. if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) # Using 'upsert' to avoid a possible race condition described in the apply_async method # above. TaskStatus.objects(task_id=self.request.id).update_one( set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, upsert=True) # Run the actual task _logger.debug("Running task : [%s]" % self.request.id) if config.get('profiling', 'enabled') is True: self.pr = cProfile.Profile() self.pr.enable() return super(Task, self).__call__(*args, **kwargs)
def get(self, request): """ Process the GET content request. :param request: The WSGI request object. :type request: django.core.handlers.wsgi.WSGIRequest :return: An appropriate HTTP reply :rtype: django.http.HttpResponse """ host = request.get_host() path = os.path.realpath(request.path_info) lazy_enabled = parse_bool(pulp_conf.get('lazy', 'enabled')) # Authorization if not allow_access(request.environ, host): # Not Authorized return HttpResponseForbidden() # Immediately 404 if the symbolic link doesn't even exist if not os.path.lexists(request.path_info): return HttpResponseNotFound(request.path_info) # Already downloaded if os.path.exists(path): return self.x_send(path) # Redirect if lazy is on if lazy_enabled: return self.redirect(request, self.key) # NotFound return HttpResponseNotFound(request.path_info)
def associated_unit(association, unit): """ Create a dictionary that is a composite of a unit association and the unit. :param association: A unit association DB record. :type association: dict :param unit: A DB unit record. :type unit: dict :return: A composite of the unit association and the unit. :rtype: dict """ unit_key = {} unit_id = unit.pop('_id') type_id = association['unit_type_id'] for key in get_unit_key_fields_for_type(type_id): unit_key[key] = unit.pop(key, None) storage_dir = pulp_conf.get('server', 'storage_dir') storage_path = unit.pop('_storage_path', None) last_updated = unit.pop('_last_updated', 0.0) if storage_path: relative_path = storage_path[len(storage_dir):].lstrip('/') else: relative_path = None return dict( unit_id=unit_id, type_id=type_id, unit_key=unit_key, storage_path=storage_path, relative_path=relative_path, last_updated=last_updated, metadata=unit)
def test_email_enabled(self, mock_smtp, mock_getbool): mail.handle_event(self.notifier_config, self.event) # verify self.assertEqual(mock_smtp.call_count, 2) mock_sendmail = mock_smtp.return_value.sendmail self.assertEqual(mock_sendmail.call_args[0][0], config.get('email', 'from')) self.assertTrue(mock_sendmail.call_args[0][1] in self.notifier_config['addresses']) # verify message attributes message = Parser().parsestr(mock_sendmail.call_args[0][2]) self.assertEqual(json.loads(message.get_payload()), self.event.payload) self.assertEqual(message.get('Subject', None), self.notifier_config['subject']) self.assertEqual(message.get('From', None), config.get('email', 'from')) self.assertTrue(message.get('To', None) in self.notifier_config['addresses'])
def test_email_enabled(self, mock_smtp, mock_getbool): mail.handle_event(self.notifier_config, self.event) #verify self.assertEqual(mock_smtp.call_count, 2) mock_sendmail = mock_smtp.return_value.sendmail self.assertEqual(mock_sendmail.call_args[0][0], config.get('email', 'from')) self.assertTrue(mock_sendmail.call_args[0][1] in self.notifier_config['addresses']) # verify message attributes message = Parser().parsestr(mock_sendmail.call_args[0][2]) self.assertEqual(json.loads(message.get_payload()), self.event.payload) self.assertEqual(message.get('Subject', None), self.notifier_config['subject']) self.assertEqual(message.get('From', None), config.get('email', 'from')) self.assertTrue(message.get('To', None) in self.notifier_config['addresses'])
def configure_login_method(): """ Configures the celery object with BROKER_LOGIN_METHOD if not default. """ login_method = config.get('tasks', 'login_method') if login_method is not '': celery.conf.update(BROKER_LOGIN_METHOD=login_method)
def _missing_units(self, unit_inventory): """ Determine the list of units defined parent inventory that are not in the child inventory. :param unit_inventory: The inventory of both parent and child content units. :type unit_inventory: UnitInventory :return: The list of units to be added. Each item: (parent_unit, unit_to_be_added) :rtype: list """ new_units = [] storage_dir = pulp_conf.get('server', 'storage_dir') for unit in unit_inventory.parent_only(): unit['metadata'].pop('_id') unit['metadata'].pop('_ns') type_id = unit['type_id'] unit_key = unit['unit_key'] metadata = unit['metadata'] storage_path = unit.get('storage_path') if storage_path: relative_path = unit['_relative_path'] storage_path = '/'.join((storage_dir, relative_path)) unit_in = Unit(type_id, unit_key, metadata, storage_path) new_units.append((unit, unit_in)) return new_units
def __init__(self, consumer, **details): """ :param consumer: A consumer DB model object. :type consumer: dict :param details: A dictionary of information to be round-tripped. Primarily used to correlate asynchronous replies. :type details: dict """ self.agent_id = 'pulp.agent.%s' % consumer['id'] self.secret = str(consumer['_id']) self.url = pulp_conf.get('messaging', 'url') self.transport = pulp_conf.get('messaging', 'transport') self.details = details self.reply_queue = Services.REPLY_QUEUE self.authenticator = Authenticator() self.authenticator.load()
def __call__(self, *args, **kwargs): """ This overrides PulpTask's __call__() method. We use this method for task state tracking of Pulp tasks. """ # Check task status and skip running the task if task state is 'canceled'. try: task_status = TaskStatus.objects.get(task_id=self.request.id) except DoesNotExist: task_status = None if task_status and task_status[ 'state'] == constants.CALL_CANCELED_STATE: _logger.debug("Task cancel received for task-id : [%s]" % self.request.id) return # Update start_time and set the task state to 'running' for asynchronous tasks. # Skip updating status for eagerly executed tasks, since we don't want to track # synchronous tasks in our database. if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) # Using 'upsert' to avoid a possible race condition described in the apply_async method # above. TaskStatus.objects(task_id=self.request.id).update_one( set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, upsert=True) # Run the actual task _logger.debug("Running task : [%s]" % self.request.id) if config.get('profiling', 'enabled') is True: self.pr = cProfile.Profile() self.pr.enable() return super(Task, self).__call__(*args, **kwargs)
def send(document, routing_key=None): """ Attempt to send a message to the AMQP broker. If we cannot obtain a new connection then the message will be dropped. Note that we do not block when waiting for a connection. :param document: the taskstatus Document we want to send :type document: mongoengine.Document :param routing_key: The routing key for the message :type routing_key: str """ # if the user has not enabled notifications, just bail event_notifications_enabled = config.getboolean("messaging", "event_notifications_enabled") if not event_notifications_enabled: return try: payload = document.to_json() except TypeError: _logger.warn("unable to convert document to JSON; event message not sent") return broker_url = config.get("messaging", "event_notification_url") notification_topic = Exchange(name=DEFAULT_EXCHANGE_NAME, type="topic") with Connection(broker_url) as connection: producer = Producer(connection) producer.maybe_declare(notification_topic) producer.publish(payload, exchange=notification_topic, routing_key=routing_key)
def add_units(self, begin, end): units = [] storage_dir = os.path.join(pulp_conf.get('server', 'storage_dir'), 'content') if not os.path.exists(storage_dir): os.makedirs(storage_dir) for n in range(begin, end): unit_id = self.UNIT_ID % n unit = dict(self.UNIT_KEY) unit.update(self.UNIT_METADATA) unit['N'] = n # add unit file storage_path = os.path.join(storage_dir, '.'.join( (unit_id, self.UNIT_TYPE_ID))) if n % 2 == 0: # even numbered has file associated unit['_storage_path'] = storage_path if n == 0: # 1st one is a directory of files os.makedirs(storage_path) dist_path = os.path.join(os.path.dirname(__file__), 'data/distribution.tar') tb = tarfile.open(dist_path) tb.extractall(path=storage_path) tb.close() else: with open(storage_path, 'w+') as fp: fp.write(unit_id) # add unit manager = managers.content_manager() manager.add_content_unit(self.UNIT_TYPE_ID, unit_id, unit) manager = managers.repo_unit_association_manager() # associate unit manager.associate_unit_by_id(self.REPO_ID, self.UNIT_TYPE_ID, unit_id) units.append(unit) return units
def __init__(self, step_type, step_description, lazy_status_conduit, download_requests): """ Initializes a Step that downloads all the download requests provided. :param lazy_status_conduit: Conduit used to update the task status. :type lazy_status_conduit: LazyStatusConduit :param download_requests: List of download requests to process. :type download_requests: list of nectar.request.DownloadRequest """ super(LazyUnitDownloadStep, self).__init__( step_type=step_type, status_conduit=lazy_status_conduit, ) self.description = step_description self.download_requests = download_requests self.download_config = { MAX_CONCURRENT: int(pulp_conf.get('lazy', 'download_concurrency')), HEADERS: { PULP_STREAM_REQUEST_HEADER: 'true' }, SSL_VALIDATION: True } self.downloader = HTTPThreadedDownloader( DownloaderConfig(**self.download_config), self)
def storage_dir(): """ The root storage path. :return: The root storage path. :rtype: str """ return config.get('server', 'storage_dir')
def __init__(self, consumer): self.uuid = consumer['id'] self.url = config.get('messaging', 'url') certificate = consumer.get('certificate') hash = hashlib.sha256() hash.update(certificate.strip()) self.secret = hash.hexdigest() self.call_request_id = factory.context().call_request_id
def test_construction(self, mock_consumer, mock_queue, mock_auth): url = 'http://broker' transport = pulp_conf.get('messaging', 'transport') handler = ReplyHandler(url, transport) mock_queue.assert_called_with(Services.REPLY_QUEUE, transport=transport) mock_consumer.assert_called_with( mock_queue(), url=url, transport=transport, authenticator=mock_auth()) self.assertEqual(handler.consumer, mock_consumer())