def _send_email(subject, body, to_address):
"""
Send a text email to one recipient
:param subject: email subject
:type subject: basestring
:param body: text body of the email
:type body: basestring
:param to_address: email address to send to
:type to_address: basestring
:return: None
"""
host = config.get('email', 'host')
port = config.getint('email', 'port')
from_address = config.get('email', 'from')
message = MIMEText(body)
message['Subject'] = subject
message['From'] = from_address
message['To'] = to_address
try:
connection = smtplib.SMTP(host=host, port=port)
except smtplib.SMTPConnectError:
logger.exception('SMTP connection failed to %s on %s' % (host, port))
return
try:
connection.sendmail(from_address, to_address, message.as_string())
except smtplib.SMTPException, e:
try:
logger.exception('Error sending mail.')
except AttributeError:
logger.error('SMTP error while sending mail')
def create_consumer_payload(self, repo, config):
payload = {}
##TODO for jdob: load the pulp.conf and make it accessible to distributor
payload["repo_name"] = repo.display_name
payload["server_name"] = pulp_server_config.get("server", "server_name")
ssl_ca_path = pulp_server_config.get("security", "ssl_ca_certificate")
if os.path.exists(ssl_ca_path):
payload["ca_cert"] = open(pulp_server_config.get("security", "ssl_ca_certificate")).read()
else:
payload["ca_cert"] = config.get("https_ca")
payload["relative_path"] = "/".join((RELATIVE_URL, self.get_repo_relative_path(repo, config)))
payload["protocols"] = []
if config.get("http"):
payload["protocols"].append("http")
if config.get("https"):
payload["protocols"].append("https")
payload["gpg_keys"] = []
if config.get("gpgkey") is not None:
payload["gpg_keys"] = config.get("gpgkey")
payload["client_cert"] = None
if config.get("auth_cert") and config.get("auth_ca"):
payload["client_cert"] = config.get("auth_cert")
else:
# load the global auth if enabled
repo_auth_config = load_config()
global_cert_dir = repo_auth_config.get("repos", "global_cert_location")
global_auth_cert = os.path.join(global_cert_dir, "pulp-global-repo.cert")
global_auth_key = os.path.join(global_cert_dir, "pulp-global-repo.key")
global_auth_ca = os.path.join(global_cert_dir, "pulp-global-repo.ca")
if os.path.exists(global_auth_ca) and os.path.exists(global_auth_cert):
payload["global_auth_cert"] = open(global_auth_cert).read()
payload["global_auth_key"] = open(global_auth_key).read()
payload["global_auth_ca"] = open(global_auth_ca).read()
return payload
def init():
url = Services.get_url()
broker = Broker(url)
broker.ssl.ca_certificate = config.get('messaging', 'cacert')
broker.ssl.client_certificate = config.get('messaging', 'clientcert')
Domain.broker.add(broker)
_logger.info(_('AMQP broker configured: %(b)s'), {'b': broker})
def create_mongo_config():
"""
Inspects the pulp config's mongodb settings and returns a data structure
that can be passed to celery for it's mongodb result backend config.
:return: dictionary with keys 'host' and 'database', and optionally with
keys 'user' and 'password', that can be passed to celery as the
config for a mongodb result backend
:rtype: dict
"""
db_name = config.get('database', 'name')
# celery 3.1 doesn't support multiple seeds, so we just use the first one
seeds = config.get('database', 'seeds')
seed = seeds.split(',')[0].strip()
host = seed.split(':')[0]
port = seed.split(':')[1] if ':' in seed else None
mongo_config = {'host': host, 'database': db_name}
if port:
mongo_config['port'] = port
if config.has_option('database', 'username') and config.has_option(
'database', 'password'):
mongo_config['user'] = config.get('database', 'username')
mongo_config['password'] = config.get('database', 'password')
return mongo_config
def _send_email(subject, body, to_address):
"""
Send a text email to one recipient
:param subject: email subject
:type subject: basestring
:param body: text body of the email
:type body: basestring
:param to_address: email address to send to
:type to_address: basestring
:return: None
"""
host = config.get('email', 'host')
port = config.getint('email', 'port')
from_address = config.get('email', 'from')
message = MIMEText(body)
message['Subject'] = subject
message['From'] = from_address
message['To'] = to_address
try:
connection = smtplib.SMTP(host=host, port=port)
except smtplib.SMTPConnectError:
logger.error('SMTP connection failed to %s on %s' % (host, port))
return
try:
connection.sendmail(from_address, to_address, message.as_string())
except smtplib.SMTPException, e:
try:
logger.error('Error sending mail: %s' % e.message)
except AttributeError:
logger.error('SMTP error while sending mail')
def _get_streamer_url(catalog_entry, signing_key):
"""
Build a URL that can be used to retrieve the file in the catalog entry from
the lazy streamer.
:param catalog_entry: The catalog entry to get the URL for.
:type catalog_entry: pulp.server.db.model.LazyCatalogEntry
:param signing_key: The server private RSA key to sign the url with.
:type signing_key: M2Crypto.RSA.RSA
:return: The signed streamer URL which corresponds to the catalog entry.
:rtype: str
"""
try:
https_retrieval = parse_bool(pulp_conf.get('lazy', 'https_retrieval'))
except Unparsable:
raise PulpCodedTaskException(error_codes.PLP1014, section='lazy', key='https_retrieval',
reason=_('The value is not boolean'))
retrieval_scheme = 'https' if https_retrieval else 'http'
host = pulp_conf.get('lazy', 'redirect_host')
port = pulp_conf.get('lazy', 'redirect_port')
path_prefix = pulp_conf.get('lazy', 'redirect_path')
netloc = (host + ':' + port) if port else host
path = os.path.join(path_prefix, catalog_entry.path.lstrip('/'))
unsigned_url = urlunsplit((retrieval_scheme, netloc, path, None, None))
# Sign the URL for a year to avoid the URL expiring before the task completes
return str(URL(unsigned_url).sign(signing_key, expiration=31536000))
def pulp_bindings():
"""
Get a pulp bindings object for this node.
Properties defined in the pulp server configuration are used
when not defined in the node configuration.
:return: A pulp bindings object.
:rtype: Bindings
"""
node_conf = read_config()
oauth = node_conf.oauth
verify_ssl = parse_bool(node_conf.main.verify_ssl)
ca_path = node_conf.main.ca_path
host = pulp_conf.get('server', 'server_name')
key = pulp_conf.get('oauth', 'oauth_key')
secret = pulp_conf.get('oauth', 'oauth_secret')
connection = PulpConnection(
host=host,
port=443,
oauth_key=key,
oauth_secret=secret,
oauth_user=oauth.user_id,
verify_ssl=verify_ssl,
ca_path=ca_path)
bindings = Bindings(connection)
return bindings
def start(cls):
url = config.get('messaging', 'url')
transport = config.get('messaging', 'transport')
# asynchronous reply
cls.reply_handler = ReplyHandler(url, transport)
cls.reply_handler.start()
_logger.info(_('AMQP reply handler started'))
def redirect(request, key):
"""
Redirected GET request.
:param request: The WSGI request object.
:type request: django.core.handlers.wsgi.WSGIRequest
:param key: A private RSA key.
:type key: RSA.RSA
:return: A redirect or not-found reply.
:rtype: django.http.HttpResponse
"""
path = os.path.realpath(request.path_info)
scheme = request.environ['REQUEST_SCHEME']
host = request.environ['SERVER_NAME']
port = request.environ['SERVER_PORT']
query = request.environ['QUERY_STRING']
remote_ip = request.environ['REMOTE_ADDR']
redirect_host = pulp_conf.get('lazy', 'redirect_host')
redirect_port = pulp_conf.get('lazy', 'redirect_port')
redirect_path = pulp_conf.get('lazy', 'redirect_path')
redirect = ContentView.urljoin(scheme, redirect_host or host,
redirect_port or port, redirect_path,
path, query)
url = URL(redirect)
signed = url.sign(key, remote_ip=remote_ip)
return HttpResponseRedirect(str(signed))
def redirect(request, key):
"""
Redirected GET request.
:param request: The WSGI request object.
:type request: django.core.handlers.wsgi.WSGIRequest
:param key: A private RSA key.
:type key: RSA.RSA
:return: A redirect or not-found reply.
:rtype: django.http.HttpResponse
"""
path = os.path.realpath(request.path_info)
scheme = request.environ['REQUEST_SCHEME']
host = request.environ['SERVER_NAME']
port = request.environ['SERVER_PORT']
query = request.environ['QUERY_STRING']
remote_ip = request.environ['REMOTE_ADDR']
redirect_host = pulp_conf.get('lazy', 'redirect_host')
redirect_port = pulp_conf.get('lazy', 'redirect_port')
redirect_path = pulp_conf.get('lazy', 'redirect_path')
redirect = ContentView.urljoin(
scheme,
redirect_host or host,
redirect_port or port,
redirect_path,
path,
query)
url = URL(redirect)
signed = url.sign(key, remote_ip=remote_ip)
return HttpResponseRedirect(str(signed))
def init():
url = config.get('messaging', 'url')
transport = config.get('messaging', 'transport')
broker = Broker(url, transport=transport)
broker.cacert = config.get('messaging', 'cacert')
broker.clientcert = config.get('messaging', 'clientcert')
_logger.info(_('AMQP broker configured: %(b)s'), {'b': broker})
def start(cls):
url = config.get('messaging', 'url')
transport = config.get('messaging', 'transport')
# asynchronous reply
cls.reply_handler = ReplyHandler(url, transport)
cls.reply_handler.start()
log.info(_('AMQP reply handler started'))
def init():
url = Services.get_url()
broker = Broker(url)
broker.ssl.ca_certificate = config.get('messaging', 'cacert')
broker.ssl.client_certificate = config.get('messaging', 'clientcert')
Domain.broker.add(broker)
_logger.info(_('AMQP broker configured: %(b)s'), {'b': broker})
def _check_username_password_ldap(self, username, password=None):
"""
Check a username and password against the ldap server.
Return None if the username and password are not valid
:type username: str
:param username: the login of the user
:type password: str or None
:param password: password of the user, None => do not validate the password
:rtype: L{pulp.server.db.model.auth.User} instance or None
:return: user corresponding to the credentials
"""
ldap_uri = config.get('ldap', 'uri')
ldap_base = config.get('ldap', 'base')
ldap_tls = config.getboolean('ldap', 'tls')
ldap_filter = None
if config.has_option('ldap', 'filter'):
ldap_filter = config.get('ldap', 'filter')
ldap_server = ldap_connection.LDAPConnection(server=ldap_uri,
tls=ldap_tls)
ldap_server.connect()
user = ldap_server.authenticate_user(ldap_base,
username,
password,
filter=ldap_filter)
return user
def init():
url = config.get('messaging', 'url')
transport = config.get('messaging', 'transport')
broker = Broker(url, transport=transport)
broker.cacert = config.get('messaging', 'cacert')
broker.clientcert = config.get('messaging', 'clientcert')
log.info(_('AMQP broker configured: %(b)s'), {'b': broker})
def init():
url = config.get('messaging', 'url')
transport = config.get('messaging', 'transport')
broker = Broker(url, transport=transport)
broker.cacert = config.get('messaging', 'cacert')
broker.clientcert = config.get('messaging', 'clientcert')
log.info('AMQP broker configured: %s', broker)
def _check_username_password_ldap(self, username, password=None):
"""
Check a username and password against the ldap server.
Return None if the username and password are not valid
:type username: str
:param username: the login of the user
:type password: str or None
:param password: password of the user, None => do not validate the password
:rtype: L{pulp.server.db.model.auth.User} instance or None
:return: user corresponding to the credentials
"""
ldap_uri = config.get('ldap', 'uri')
ldap_base = config.get('ldap', 'base')
ldap_tls = config.getboolean('ldap', 'tls')
ldap_filter = None
if config.has_option('ldap', 'filter'):
ldap_filter = config.get('ldap', 'filter')
ldap_server = ldap_connection.LDAPConnection(server=ldap_uri, tls=ldap_tls)
ldap_server.connect()
user = ldap_server.authenticate_user(ldap_base, username, password,
filter=ldap_filter)
return user
def _get_streamer_url(catalog_entry, signing_key):
"""
Build a URL that can be used to retrieve the file in the catalog entry from
the lazy streamer.
:param catalog_entry: The catalog entry to get the URL for.
:type catalog_entry: pulp.server.db.model.LazyCatalogEntry
:param signing_key: The server private RSA key to sign the url with.
:type signing_key: M2Crypto.RSA.RSA
:return: The signed streamer URL which corresponds to the catalog entry.
:rtype: str
"""
try:
https_retrieval = parse_bool(pulp_conf.get('lazy', 'https_retrieval'))
except Unparsable:
raise PulpCodedTaskException(error_codes.PLP1014,
section='lazy',
key='https_retrieval',
reason=_('The value is not boolean'))
retrieval_scheme = 'https' if https_retrieval else 'http'
host = pulp_conf.get('lazy', 'redirect_host')
port = pulp_conf.get('lazy', 'redirect_port')
path_prefix = pulp_conf.get('lazy', 'redirect_path')
netloc = (host + ':' + port) if port else host
path = os.path.join(path_prefix, catalog_entry.path.lstrip('/'))
unsigned_url = urlunsplit((retrieval_scheme, netloc, path, None, None))
# Sign the URL for a year to avoid the URL expiring before the task completes
return str(URL(unsigned_url).sign(signing_key, expiration=31536000))
def test_init(self):
Services.init()
url = pulp_conf.get('messaging', 'url')
ca_cert = pulp_conf.get('messaging', 'cacert')
client_cert = pulp_conf.get('messaging', 'clientcert')
broker = Broker(url)
self.assertEqual(broker.url, URL(url))
self.assertEqual(broker.cacert, ca_cert)
self.assertEqual(broker.clientcert, client_cert)
def on_success(self, retval, task_id, args, kwargs):
"""
This overrides the success handler run by the worker when the task
executes successfully. It updates state, finish_time and traceback
of the relevant task status for asynchronous tasks. Skip updating status
for synchronous tasks.
:param retval: The return value of the task.
:param task_id: Unique id of the executed task.
:param args: Original arguments for the executed task.
:param kwargs: Original keyword arguments for the executed task.
"""
_logger.debug("Task successful : [%s]" % task_id)
if kwargs.get('scheduled_call_id') is not None:
if not isinstance(retval, AsyncResult):
_logger.info(
_('resetting consecutive failure count for schedule %(id)s'
) % {'id': kwargs['scheduled_call_id']})
utils.reset_failure_count(kwargs['scheduled_call_id'])
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
finish_time = dateutils.format_iso8601_datetime(now)
task_status = TaskStatus.objects.get(task_id=task_id)
task_status['finish_time'] = finish_time
task_status['result'] = retval
# Only set the state to finished if it's not already in a complete state. This is
# important for when the task has been canceled, so we don't move the task from canceled
# to finished.
if task_status['state'] not in constants.CALL_COMPLETE_STATES:
task_status['state'] = constants.CALL_FINISHED_STATE
if isinstance(retval, TaskResult):
task_status['result'] = retval.return_value
if retval.error:
task_status['error'] = retval.error.to_dict()
if retval.spawned_tasks:
task_list = []
for spawned_task in retval.spawned_tasks:
if isinstance(spawned_task, AsyncResult):
task_list.append(spawned_task.task_id)
elif isinstance(spawned_task, dict):
task_list.append(spawned_task['task_id'])
task_status['spawned_tasks'] = task_list
if isinstance(retval, AsyncResult):
task_status['spawned_tasks'] = [
retval.task_id,
]
task_status['result'] = None
task_status.save()
if config.get('profiling', 'enabled') is True:
profile_directory = config.get('profiling', 'directory')
self.pr.disable()
self.pr.dump_stats("%s/%s" % (profile_directory, task_id))
common_utils.delete_working_directory()
def create_consumer_payload(self, repo, config, binding_config):
"""
Called when a consumer binds to a repository using this distributor.
This call should return a dictionary describing all data the consumer
will need to access the repository.
:param repo: metadata describing the repository
:type repo: pulp.plugins.model.Repository
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
:param binding_config: configuration applicable only for the specific
consumer the payload is generated for; this will be None
if there are no specific options for the consumer in question
:type binding_config: object or None
:return: dictionary of relevant data
:rtype: dict
"""
payload = dict()
payload['repo_name'] = repo.display_name
payload['server_name'] = pulp_server_config.get('server', 'server_name')
ssl_ca_path = pulp_server_config.get('security', 'ssl_ca_certificate')
try:
payload['ca_cert'] = open(ssl_ca_path).read()
except (OSError, IOError):
payload['ca_cert'] = config.get('https_ca')
payload['relative_path'] = \
'/'.join([RELATIVE_URL, configuration.get_repo_relative_path(repo, config)])
payload['protocols'] = []
if config.get('http'):
payload['protocols'].append('http')
if config.get('https'):
payload['protocols'].append('https')
payload['gpg_keys'] = []
if config.get('gpgkey') is not None:
payload['gpg_keys'] = {'pulp.key': config.get('gpgkey')}
payload['client_cert'] = None
if config.get('auth_cert') and config.get('auth_ca'):
payload['client_cert'] = config.get('auth_cert')
else:
# load the global auth if enabled
repo_auth_config = configuration.load_config(constants.REPO_AUTH_CONFIG_FILE)
global_cert_dir = repo_auth_config.get('repos', 'global_cert_location')
global_auth_cert = os.path.join(global_cert_dir, 'pulp-global-repo.cert')
global_auth_key = os.path.join(global_cert_dir, 'pulp-global-repo.key')
global_auth_ca = os.path.join(global_cert_dir, 'pulp-global-repo.ca')
if os.path.exists(global_auth_ca) and \
os.path.exists(global_auth_cert) and \
os.path.exists(global_auth_key):
payload['global_auth_cert'] = open(global_auth_cert).read()
payload['global_auth_key'] = open(global_auth_key).read()
payload['global_auth_ca'] = open(global_auth_ca).read()
return payload
def test_init(self, mock_broker):
Services.init()
url = pulp_conf.get('messaging', 'url')
transport = pulp_conf.get('messaging', 'transport')
ca_cert = pulp_conf.get('messaging', 'cacert')
client_cert = pulp_conf.get('messaging', 'clientcert')
mock_broker.assert_called_with(url, transport=transport)
broker = mock_broker()
self.assertEqual(broker.cacert, ca_cert)
self.assertEqual(broker.clientcert, client_cert)
def add_connector():
"""
Configure and add the gofer connector used to connect
to the message broker. This call is idempotent.
"""
url = get_url()
connector = Connector(url)
connector.ssl.ca_certificate = config.get('messaging', 'cacert')
connector.ssl.client_certificate = config.get('messaging', 'clientcert')
connector.add()
def test_init(self, mock_broker):
Services.init()
url = pulp_conf.get('messaging', 'url')
transport = pulp_conf.get('messaging', 'transport')
ca_cert = pulp_conf.get('messaging', 'cacert')
client_cert = pulp_conf.get('messaging', 'clientcert')
mock_broker.assert_called_with(url, transport=transport)
broker = mock_broker()
self.assertEqual(broker.cacert, ca_cert)
self.assertEqual(broker.clientcert, client_cert)
def on_success(self, retval, task_id, args, kwargs):
"""
This overrides the success handler run by the worker when the task
executes successfully. It updates state, finish_time and traceback
of the relevant task status for asynchronous tasks. Skip updating status
for synchronous tasks.
:param retval: The return value of the task.
:param task_id: Unique id of the executed task.
:param args: Original arguments for the executed task.
:param kwargs: Original keyword arguments for the executed task.
"""
_logger.debug("Task successful : [%s]" % task_id)
if kwargs.get('scheduled_call_id') is not None:
if not isinstance(retval, AsyncResult):
_logger.info(_('resetting consecutive failure count for schedule %(id)s')
% {'id': kwargs['scheduled_call_id']})
utils.reset_failure_count(kwargs['scheduled_call_id'])
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
finish_time = dateutils.format_iso8601_datetime(now)
task_status = TaskStatus.objects.get(task_id=task_id)
task_status['finish_time'] = finish_time
task_status['result'] = retval
# Only set the state to finished if it's not already in a complete state. This is
# important for when the task has been canceled, so we don't move the task from canceled
# to finished.
if task_status['state'] not in constants.CALL_COMPLETE_STATES:
task_status['state'] = constants.CALL_FINISHED_STATE
if isinstance(retval, TaskResult):
task_status['result'] = retval.return_value
if retval.error:
task_status['error'] = retval.error.to_dict()
if retval.spawned_tasks:
task_list = []
for spawned_task in retval.spawned_tasks:
if isinstance(spawned_task, AsyncResult):
task_list.append(spawned_task.task_id)
elif isinstance(spawned_task, dict):
task_list.append(spawned_task['task_id'])
task_status['spawned_tasks'] = task_list
if isinstance(retval, AsyncResult):
task_status['spawned_tasks'] = [retval.task_id, ]
task_status['result'] = None
task_status.save()
if config.get('profiling', 'enabled') is True:
profile_directory = config.get('profiling', 'directory')
self.pr.disable()
self.pr.dump_stats("%s/%s" % (profile_directory, task_id))
common_utils.delete_working_directory()
def __init__(self, **kwargs):
super(ContentView, self).__init__(**kwargs)
self.key = Key.load(pulp_conf.get('authentication', 'rsa_key'))
# Make sure all requested paths fall under these sub-directories, otherwise
# we might find ourselves serving private keys to all and sundry.
local_storage = pulp_conf.get('server', 'storage_dir')
self.safe_serving_paths = [os.path.realpath(os.path.join(local_storage, subdir))
for subdir in SAFE_STORAGE_SUBDIRS]
logger.debug(_("Serving Pulp content from {paths}; ensure "
"Apache's mod_xsendfile is configured to serve from "
"these paths as well").format(paths=str(self.safe_serving_paths)))
def get_url():
"""
This constructs a gofer 2.x URL and is intended to maintain
configuration file backwards compatibility until pulp 3.0
:return: A gofer 2.x broker URL.
:rtype: str
"""
url = config.get('messaging', 'url')
adapter = config.get('messaging', 'transport')
return '+'.join((adapter, url))
def get_url():
"""
This constructs a gofer 2.x URL and is intended to maintain
configuration file backwards compatibility until pulp 3.0
:return: A gofer 2.x broker URL.
:rtype: str
"""
url = config.get('messaging', 'url')
adapter = config.get('messaging', 'transport')
return '+'.join((adapter, url))
def configure_SSL():
"""
Configures the celery object with BROKER_USE_SSL options
"""
if config.getboolean('tasks', 'celery_require_ssl'):
BROKER_USE_SSL = {
'ca_certs': config.get('tasks', 'cacert'),
'keyfile': config.get('tasks', 'keyfile'),
'certfile': config.get('tasks', 'certfile'),
'cert_reqs': ssl.CERT_REQUIRED,
}
celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def configure_SSL():
"""
Configures the celery object with BROKER_USE_SSL options
"""
if config.getboolean("tasks", "celery_require_ssl"):
BROKER_USE_SSL = {
"ca_certs": config.get("tasks", "cacert"),
"keyfile": config.get("tasks", "keyfile"),
"certfile": config.get("tasks", "certfile"),
"cert_reqs": ssl.CERT_REQUIRED,
}
celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def configure_SSL():
"""
Configures the celery object with BROKER_USE_SSL options
"""
if config.getboolean('tasks', 'celery_require_ssl'):
BROKER_USE_SSL = {
'ca_certs': config.get('tasks', 'cacert'),
'keyfile': config.get('tasks', 'keyfile'),
'certfile': config.get('tasks', 'certfile'),
'cert_reqs': ssl.CERT_REQUIRED,
}
celery.conf.update(BROKER_USE_SSL=BROKER_USE_SSL)
def check_oauth(self, username, method, url, auth, query):
"""
Check OAuth header credentials.
Return None if the credentials are invalid
:type username: str
:param username: username corresponding to credentials
:type method: str
:param method: http method
:type url: str
:param url: request url
:type auth: str
:param auth: http authorization header value
:type query: str
:param query: http request query string
:rtype: str or None
:return: user login corresponding to the credentials
"""
is_consumer = False
headers = {'Authorization': auth}
req = oauth2.Request.from_request(method,
url,
headers,
query_string=query)
if not req:
return None, is_consumer
if not (config.has_option('oauth', 'oauth_key')
and config.has_option('oauth', 'oauth_secret')):
_logger.error(
_("Attempting OAuth authentication and you do not have oauth_key and "
"oauth_secret in pulp.conf"))
return None, is_consumer
key = config.get('oauth', 'oauth_key')
secret = config.get('oauth', 'oauth_secret')
consumer = oauth2.Consumer(key=key, secret=secret)
server = oauth2.Server()
server.add_signature_method(oauth2.SignatureMethod_HMAC_SHA1())
try:
# this call has a return value, but failures are noted by the exception
server.verify_request(req, consumer, None)
except oauth2.Error, e:
_logger.error('error verifying OAuth signature: %s' % e)
return None, is_consumer
def check_oauth(self, username, method, url, auth, query):
"""
Check OAuth header credentials.
Return None if the credentials are invalid
:type username: str
:param username: username corresponding to credentials
:type method: str
:param method: http method
:type url: str
:param url: request url
:type auth: str
:param auth: http authorization header value
:type query: str
:param query: http request query string
:rtype: str or None
:return: user login corresponding to the credentials
"""
is_consumer = False
headers = {'Authorization': auth}
req = oauth2.Request.from_request(method, url, headers, query_string=query)
if not req:
return None, is_consumer
if not (config.has_option('oauth', 'oauth_key') and
config.has_option('oauth', 'oauth_secret')):
_logger.error(_("Attempting OAuth authentication and you do not have oauth_key and "
"oauth_secret in pulp.conf"))
return None, is_consumer
key = config.get('oauth', 'oauth_key')
secret = config.get('oauth', 'oauth_secret')
consumer = oauth2.Consumer(key=key, secret=secret)
server = oauth2.Server()
server.add_signature_method(oauth2.SignatureMethod_HMAC_SHA1())
try:
# this call has a return value, but failures are noted by the exception
server.verify_request(req, consumer, None)
except oauth2.Error, e:
_logger.error('error verifying OAuth signature: %s' % e)
return None, is_consumer
def setUpClass(cls):
if not os.path.exists(cls.TMP_ROOT):
os.makedirs(cls.TMP_ROOT)
stop_logging()
path = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'data',
'pulp.conf')
pulp_conf.read(path)
start_logging()
storage_dir = pulp_conf.get('server', 'storage_dir')
if not os.path.exists(storage_dir):
os.makedirs(storage_dir)
shutil.rmtree(storage_dir + '/*', ignore_errors=True)
name = pulp_conf.get('database', 'name')
connection.initialize(name)
managers.initialize()
def __init__(self, **kwargs):
super(ContentView, self).__init__(**kwargs)
self.key = Key.load(pulp_conf.get('authentication', 'rsa_key'))
# Make sure all requested paths fall under these sub-directories, otherwise
# we might find ourselves serving private keys to all and sundry.
local_storage = pulp_conf.get('server', 'storage_dir')
self.safe_serving_paths = [
os.path.realpath(os.path.join(local_storage, subdir))
for subdir in SAFE_STORAGE_SUBDIRS
]
logger.debug(
_("Serving Pulp content from {paths}; ensure "
"Apache's mod_xsendfile is configured to serve from "
"these paths as well").format(
paths=str(self.safe_serving_paths)))
def tearDownClass(cls):
name = pulp_conf.get('database', 'name')
db = pymongo.database.Database(connection._CONNECTION, name)
for name in db.collection_names():
if name[:7] == 'system.':
continue
db.drop_collection(name)
def _missing_units(self, unit_inventory):
"""
Determine the list of units defined upstream inventory that are
not in the local inventory.
:param unit_inventory: The inventory of both upstream and local content units.
:type unit_inventory: UnitInventory
:return: The list of units to be added.
Each item: (unit_upstream, unit_to_be_added)
:rtype: list
"""
new_units = []
storage_dir = pulp_conf.get("server", "storage_dir")
for unit in unit_inventory.upstream_only():
unit["metadata"].pop("_id")
unit["metadata"].pop("_ns")
type_id = unit["type_id"]
unit_key = unit["unit_key"]
metadata = unit["metadata"]
storage_path = unit.get("storage_path")
if storage_path:
relative_path = unit["_relative_path"]
storage_path = "/".join((storage_dir, relative_path))
unit_in = Unit(type_id, unit_key, metadata, storage_path)
new_units.append((unit, unit_in))
return new_units
def associated_unit(association, unit):
"""
Create a dictionary that is a composite of a unit association and the unit.
:param association: A unit association DB record.
:type association: dict
:param unit: A DB unit record.
:type unit: dict
:return: A composite of the unit association and the unit.
:rtype: dict
"""
unit_key = {}
unit_id = unit.pop('_id')
type_id = association['unit_type_id']
for key in get_unit_key_fields_for_type(type_id):
unit_key[key] = unit.pop(key, None)
storage_dir = pulp_conf.get('server', 'storage_dir')
storage_path = unit.pop('_storage_path', None)
last_updated = unit.pop('_last_updated', 0.0)
if storage_path:
relative_path = storage_path[len(storage_dir):].lstrip('/')
else:
relative_path = None
return dict(unit_id=unit_id,
type_id=type_id,
unit_key=unit_key,
storage_path=storage_path,
relative_path=relative_path,
last_updated=last_updated,
metadata=unit)
def get(self, request):
"""
Process the GET content request.
:param request: The WSGI request object.
:type request: django.core.handlers.wsgi.WSGIRequest
:return: An appropriate HTTP reply
:rtype: django.http.HttpResponse
"""
host = request.get_host()
path = os.path.realpath(request.path_info)
lazy_enabled = parse_bool(pulp_conf.get('lazy', 'enabled'))
# Authorization
if not allow_access(request.environ, host):
# Not Authorized
return HttpResponseForbidden()
# Already downloaded
if os.path.exists(path):
return self.x_send(path)
# Redirect
if lazy_enabled:
return self.redirect(request, self.key)
# NotFound
return HttpResponseNotFound(request.path_info)
def __call__(self, *args, **kwargs):
"""
This overrides PulpTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is 'canceled'.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status['state'] == constants.CALL_CANCELED_STATE:
_logger.debug("Task cancel received for task-id : [%s]" % self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Skip updating status for eagerly executed tasks, since we don't want to track
# synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
if config.get('profiling', 'enabled') is True:
self.pr = cProfile.Profile()
self.pr.enable()
return super(Task, self).__call__(*args, **kwargs)
def get(self, request):
"""
Process the GET content request.
:param request: The WSGI request object.
:type request: django.core.handlers.wsgi.WSGIRequest
:return: An appropriate HTTP reply
:rtype: django.http.HttpResponse
"""
host = request.get_host()
path = os.path.realpath(request.path_info)
lazy_enabled = parse_bool(pulp_conf.get('lazy', 'enabled'))
# Authorization
if not allow_access(request.environ, host):
# Not Authorized
return HttpResponseForbidden()
# Immediately 404 if the symbolic link doesn't even exist
if not os.path.lexists(request.path_info):
return HttpResponseNotFound(request.path_info)
# Already downloaded
if os.path.exists(path):
return self.x_send(path)
# Redirect if lazy is on
if lazy_enabled:
return self.redirect(request, self.key)
# NotFound
return HttpResponseNotFound(request.path_info)
def tearDownClass(cls):
name = pulp_conf.get('database', 'name')
db = pymongo.database.Database(connection._CONNECTION, name)
for name in db.collection_names():
if name[:7] == 'system.':
continue
db.drop_collection(name)
def associated_unit(association, unit):
"""
Create a dictionary that is a composite of a unit association and the unit.
:param association: A unit association DB record.
:type association: dict
:param unit: A DB unit record.
:type unit: dict
:return: A composite of the unit association and the unit.
:rtype: dict
"""
unit_key = {}
unit_id = unit.pop('_id')
type_id = association['unit_type_id']
for key in get_unit_key_fields_for_type(type_id):
unit_key[key] = unit.pop(key, None)
storage_dir = pulp_conf.get('server', 'storage_dir')
storage_path = unit.pop('_storage_path', None)
last_updated = unit.pop('_last_updated', 0.0)
if storage_path:
relative_path = storage_path[len(storage_dir):].lstrip('/')
else:
relative_path = None
return dict(
unit_id=unit_id,
type_id=type_id,
unit_key=unit_key,
storage_path=storage_path,
relative_path=relative_path,
last_updated=last_updated,
metadata=unit)
def test_email_enabled(self, mock_smtp, mock_getbool):
mail.handle_event(self.notifier_config, self.event)
# verify
self.assertEqual(mock_smtp.call_count, 2)
mock_sendmail = mock_smtp.return_value.sendmail
self.assertEqual(mock_sendmail.call_args[0][0],
config.get('email', 'from'))
self.assertTrue(mock_sendmail.call_args[0][1] in self.notifier_config['addresses'])
# verify message attributes
message = Parser().parsestr(mock_sendmail.call_args[0][2])
self.assertEqual(json.loads(message.get_payload()), self.event.payload)
self.assertEqual(message.get('Subject', None), self.notifier_config['subject'])
self.assertEqual(message.get('From', None), config.get('email', 'from'))
self.assertTrue(message.get('To', None) in self.notifier_config['addresses'])
def test_email_enabled(self, mock_smtp, mock_getbool):
mail.handle_event(self.notifier_config, self.event)
#verify
self.assertEqual(mock_smtp.call_count, 2)
mock_sendmail = mock_smtp.return_value.sendmail
self.assertEqual(mock_sendmail.call_args[0][0],
config.get('email', 'from'))
self.assertTrue(mock_sendmail.call_args[0][1] in self.notifier_config['addresses'])
# verify message attributes
message = Parser().parsestr(mock_sendmail.call_args[0][2])
self.assertEqual(json.loads(message.get_payload()), self.event.payload)
self.assertEqual(message.get('Subject', None), self.notifier_config['subject'])
self.assertEqual(message.get('From', None), config.get('email', 'from'))
self.assertTrue(message.get('To', None) in self.notifier_config['addresses'])
def configure_login_method():
"""
Configures the celery object with BROKER_LOGIN_METHOD if not default.
"""
login_method = config.get('tasks', 'login_method')
if login_method is not '':
celery.conf.update(BROKER_LOGIN_METHOD=login_method)
def configure_login_method():
"""
Configures the celery object with BROKER_LOGIN_METHOD if not default.
"""
login_method = config.get('tasks', 'login_method')
if login_method is not '':
celery.conf.update(BROKER_LOGIN_METHOD=login_method)
def _missing_units(self, unit_inventory):
"""
Determine the list of units defined parent inventory that are
not in the child inventory.
:param unit_inventory: The inventory of both parent and child content units.
:type unit_inventory: UnitInventory
:return: The list of units to be added.
Each item: (parent_unit, unit_to_be_added)
:rtype: list
"""
new_units = []
storage_dir = pulp_conf.get('server', 'storage_dir')
for unit in unit_inventory.parent_only():
unit['metadata'].pop('_id')
unit['metadata'].pop('_ns')
type_id = unit['type_id']
unit_key = unit['unit_key']
metadata = unit['metadata']
storage_path = unit.get('storage_path')
if storage_path:
relative_path = unit['_relative_path']
storage_path = '/'.join((storage_dir, relative_path))
unit_in = Unit(type_id, unit_key, metadata, storage_path)
new_units.append((unit, unit_in))
return new_units
def __init__(self, consumer, **details):
"""
:param consumer: A consumer DB model object.
:type consumer: dict
:param details: A dictionary of information to be round-tripped.
Primarily used to correlate asynchronous replies.
:type details: dict
"""
self.agent_id = 'pulp.agent.%s' % consumer['id']
self.secret = str(consumer['_id'])
self.url = pulp_conf.get('messaging', 'url')
self.transport = pulp_conf.get('messaging', 'transport')
self.details = details
self.reply_queue = Services.REPLY_QUEUE
self.authenticator = Authenticator()
self.authenticator.load()
def __call__(self, *args, **kwargs):
"""
This overrides PulpTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is 'canceled'.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status[
'state'] == constants.CALL_CANCELED_STATE:
_logger.debug("Task cancel received for task-id : [%s]" %
self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Skip updating status for eagerly executed tasks, since we don't want to track
# synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
if config.get('profiling', 'enabled') is True:
self.pr = cProfile.Profile()
self.pr.enable()
return super(Task, self).__call__(*args, **kwargs)
def send(document, routing_key=None):
"""
Attempt to send a message to the AMQP broker.
If we cannot obtain a new connection then the message will be dropped. Note
that we do not block when waiting for a connection.
:param document: the taskstatus Document we want to send
:type document: mongoengine.Document
:param routing_key: The routing key for the message
:type routing_key: str
"""
# if the user has not enabled notifications, just bail
event_notifications_enabled = config.getboolean("messaging", "event_notifications_enabled")
if not event_notifications_enabled:
return
try:
payload = document.to_json()
except TypeError:
_logger.warn("unable to convert document to JSON; event message not sent")
return
broker_url = config.get("messaging", "event_notification_url")
notification_topic = Exchange(name=DEFAULT_EXCHANGE_NAME, type="topic")
with Connection(broker_url) as connection:
producer = Producer(connection)
producer.maybe_declare(notification_topic)
producer.publish(payload, exchange=notification_topic, routing_key=routing_key)
def add_units(self, begin, end):
units = []
storage_dir = os.path.join(pulp_conf.get('server', 'storage_dir'),
'content')
if not os.path.exists(storage_dir):
os.makedirs(storage_dir)
for n in range(begin, end):
unit_id = self.UNIT_ID % n
unit = dict(self.UNIT_KEY)
unit.update(self.UNIT_METADATA)
unit['N'] = n
# add unit file
storage_path = os.path.join(storage_dir, '.'.join(
(unit_id, self.UNIT_TYPE_ID)))
if n % 2 == 0: # even numbered has file associated
unit['_storage_path'] = storage_path
if n == 0: # 1st one is a directory of files
os.makedirs(storage_path)
dist_path = os.path.join(os.path.dirname(__file__),
'data/distribution.tar')
tb = tarfile.open(dist_path)
tb.extractall(path=storage_path)
tb.close()
else:
with open(storage_path, 'w+') as fp:
fp.write(unit_id)
# add unit
manager = managers.content_manager()
manager.add_content_unit(self.UNIT_TYPE_ID, unit_id, unit)
manager = managers.repo_unit_association_manager()
# associate unit
manager.associate_unit_by_id(self.REPO_ID, self.UNIT_TYPE_ID,
unit_id)
units.append(unit)
return units
def __init__(self, step_type, step_description, lazy_status_conduit,
download_requests):
"""
Initializes a Step that downloads all the download requests provided.
:param lazy_status_conduit: Conduit used to update the task status.
:type lazy_status_conduit: LazyStatusConduit
:param download_requests: List of download requests to process.
:type download_requests: list of nectar.request.DownloadRequest
"""
super(LazyUnitDownloadStep, self).__init__(
step_type=step_type,
status_conduit=lazy_status_conduit,
)
self.description = step_description
self.download_requests = download_requests
self.download_config = {
MAX_CONCURRENT: int(pulp_conf.get('lazy', 'download_concurrency')),
HEADERS: {
PULP_STREAM_REQUEST_HEADER: 'true'
},
SSL_VALIDATION: True
}
self.downloader = HTTPThreadedDownloader(
DownloaderConfig(**self.download_config), self)
def storage_dir():
"""
The root storage path.
:return: The root storage path.
:rtype: str
"""
return config.get('server', 'storage_dir')
def __init__(self, consumer):
self.uuid = consumer['id']
self.url = config.get('messaging', 'url')
certificate = consumer.get('certificate')
hash = hashlib.sha256()
hash.update(certificate.strip())
self.secret = hash.hexdigest()
self.call_request_id = factory.context().call_request_id
def test_construction(self, mock_consumer, mock_queue, mock_auth):
url = 'http://broker'
transport = pulp_conf.get('messaging', 'transport')
handler = ReplyHandler(url, transport)
mock_queue.assert_called_with(Services.REPLY_QUEUE, transport=transport)
mock_consumer.assert_called_with(
mock_queue(), url=url, transport=transport, authenticator=mock_auth())
self.assertEqual(handler.consumer, mock_consumer())
def storage_dir():
"""
The root storage path.
:return: The root storage path.
:rtype: str
"""
return config.get('server', 'storage_dir')
