def unit_key_str(unit_key_dict):
"""
Converts the unit key dict form into a single string that can be
used as the key in a dict lookup.
"""
template = '%s-%s-%s'
return template % (encode_unicode(unit_key_dict['name']),
encode_unicode(unit_key_dict['version']),
encode_unicode(unit_key_dict['author']))
def unit_key_str(unit_key_dict):
"""
Converts the unit key dict form into a single string that can be
used as the key in a dict lookup.
"""
template = '%s-%s-%s'
return template % (encode_unicode(unit_key_dict['name']),
encode_unicode(unit_key_dict['version']),
encode_unicode(unit_key_dict['author']))
def test_create_i18n(self):
# Setup
i18n_text = 'Brasília'
# Test
self.manager.create_repo('repo-i18n', display_name=i18n_text, description=i18n_text)
# Verify
repo = Repo.get_collection().find_one({'id' : 'repo-i18n'})
self.assertTrue(repo is not None)
self.assertEqual(encode_unicode(repo['display_name']), i18n_text)
self.assertEqual(encode_unicode(repo['description']), i18n_text)
def test_create_i18n(self):
# Setup
i18n_text = 'Brasília'
# Test
self.manager.create_repo('repo-i18n', display_name=i18n_text, description=i18n_text)
# Verify
repo = Repo.get_collection().find_one({'id': 'repo-i18n'})
self.assertTrue(repo is not None)
self.assertEqual(encode_unicode(repo['display_name']), i18n_text)
self.assertEqual(encode_unicode(repo['description']), i18n_text)
def deserialize(cls, data):
"""
Deserialize the data returned from a serialize call into a call request
@param data: serialized call request
@type data: dict
@return: deserialized call request
@rtype: CallRequest
"""
if data is None:
return None
constructor_kwargs = dict(data)
constructor_kwargs.pop('callable_name') # added for search
for key, value in constructor_kwargs.items():
constructor_kwargs[encode_unicode(key)] = constructor_kwargs.pop(key)
try:
for field in cls.pickled_fields:
constructor_kwargs[field] = pickle.loads(data[field].encode('ascii'))
except Exception, e:
_LOG.exception(e)
return None
def _download_file(self, url, destination):
"""
Downloads the content at the given URL into the given destination.
The object passed into destination must have a method called "update"
that accepts a single parameter (the buffer that was read).
:param url: location to download
:type url: str
:param destination: object
@return:
"""
curl = self._create_and_configure_curl()
url = encode_unicode(
url) # because of how the config is stored in pulp
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, destination.update)
curl.perform()
status = curl.getinfo(curl.HTTP_CODE)
curl.close()
if status == 401:
raise exceptions.UnauthorizedException(url)
elif status == 404:
raise exceptions.FileNotFoundException(url)
elif status != 200:
raise exceptions.FileRetrievalException(url)
def deserialize(cls, data):
"""
Deserialize the data returned from a serialize call into a call request
@param data: serialized call request
@type data: dict
@return: deserialized call request
@rtype: CallRequest
"""
if data is None:
return None
constructor_kwargs = dict(data)
constructor_kwargs.pop('callable_name', None) # added for search
for key, value in constructor_kwargs.items():
constructor_kwargs[encode_unicode(key)] = constructor_kwargs.pop(
key)
try:
for field in cls.pickled_fields:
constructor_kwargs[field] = pickle.loads(
data[field].encode('ascii'))
except Exception, e:
_LOG.exception(e)
return None
def __init__(self, sync_conduit, config):
"""
Initialize an ISOSyncRun.
:param sync_conduit: the sync conduit to use for this sync run.
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
"""
self.sync_conduit = sync_conduit
self.config = config
self._remove_missing_units = config.get(
importer_constants.KEY_UNITS_REMOVE_MISSING,
default=constants.CONFIG_UNITS_REMOVE_MISSING_DEFAULT)
self._validate_downloads = config.get(importer_constants.KEY_VALIDATE,
default=constants.CONFIG_VALIDATE_DEFAULT)
self._repo_url = encode_unicode(config.get(importer_constants.KEY_FEED))
# The _repo_url must end in a trailing slash, because we will use urljoin to determine
# the path to
# PULP_MANIFEST later
if self._repo_url[-1] != '/':
self._repo_url = self._repo_url + '/'
# Cast our config parameters to the correct types and use them to build a Downloader
max_speed = config.get(importer_constants.KEY_MAX_SPEED)
if max_speed is not None:
max_speed = float(max_speed)
max_downloads = config.get(importer_constants.KEY_MAX_DOWNLOADS)
if max_downloads is not None:
max_downloads = int(max_downloads)
else:
max_downloads = constants.CONFIG_MAX_DOWNLOADS_DEFAULT
ssl_validation = config.get_boolean(importer_constants.KEY_SSL_VALIDATION)
ssl_validation = ssl_validation if ssl_validation is not None else \
constants.CONFIG_VALIDATE_DEFAULT
downloader_config = {
'max_speed': max_speed,
'max_concurrent': max_downloads,
'ssl_client_cert': config.get(importer_constants.KEY_SSL_CLIENT_CERT),
'ssl_client_key': config.get(importer_constants.KEY_SSL_CLIENT_KEY),
'ssl_ca_cert': config.get(importer_constants.KEY_SSL_CA_CERT),
'ssl_validation': ssl_validation,
'proxy_url': config.get(importer_constants.KEY_PROXY_HOST),
'proxy_port': config.get(importer_constants.KEY_PROXY_PORT),
'proxy_username': config.get(importer_constants.KEY_PROXY_USER),
'proxy_password': config.get(importer_constants.KEY_PROXY_PASS),
'basic_auth_username': config.get(importer_constants.KEY_BASIC_AUTH_USER),
'basic_auth_password': config.get(importer_constants.KEY_BASIC_AUTH_PASS),
'working_dir': common_utils.get_working_directory()}
downloader_config = DownloaderConfig(**downloader_config)
# We will pass self as the event_listener, so that we can receive the callbacks in this
# class
if self._repo_url.lower().startswith('file'):
self.downloader = LocalFileDownloader(downloader_config, self)
else:
self.downloader = HTTPThreadedDownloader(downloader_config, self)
self.progress_report = SyncProgressReport(sync_conduit)
self.repo_units = []
def __init__(self, sync_conduit, config):
self.sync_conduit = sync_conduit
self._remove_missing_units = config.get(constants.CONFIG_REMOVE_MISSING_UNITS, default=False)
self._repo_url = encode_unicode(config.get(constants.CONFIG_FEED_URL))
self._validate_downloads = config.get(constants.CONFIG_VALIDATE_DOWNLOADS, default=True)
# Cast our config parameters to the correct types and use them to build a Downloader
max_speed = config.get(constants.CONFIG_MAX_SPEED)
if max_speed is not None:
max_speed = float(max_speed)
num_threads = config.get(constants.CONFIG_NUM_THREADS)
if num_threads is not None:
num_threads = int(num_threads)
else:
num_threads = constants.DEFAULT_NUM_THREADS
downloader_config = {
'max_speed': max_speed, 'num_threads': num_threads,
'ssl_client_cert': config.get(constants.CONFIG_SSL_CLIENT_CERT),
'ssl_client_key': config.get(constants.CONFIG_SSL_CLIENT_KEY),
'ssl_ca_cert': config.get(constants.CONFIG_SSL_CA_CERT), 'ssl_verify_host': 1,
'ssl_verify_peer': 1, 'proxy_url': config.get(constants.CONFIG_PROXY_URL),
'proxy_port': config.get(constants.CONFIG_PROXY_PORT),
'proxy_user': config.get(constants.CONFIG_PROXY_USER),
'proxy_password': config.get(constants.CONFIG_PROXY_PASSWORD)}
downloader_config = DownloaderConfig(protocol='https', **downloader_config)
# We will pass self as the event_listener, so that we can receive the callbacks in this class
self.downloader = factory.get_downloader(downloader_config, self)
self.progress_report = SyncProgressReport(sync_conduit)
def __init__(self, sync_conduit, config):
"""
Initialize an ISOSyncRun.
:param sync_conduit: the sync conduit to use for this sync run.
:type sync_conduit: pulp.plugins.conduits.repo_sync.RepoSyncConduit
:param config: plugin configuration
:type config: pulp.plugins.config.PluginCallConfiguration
"""
self.sync_conduit = sync_conduit
self._remove_missing_units = config.get(
importer_constants.KEY_UNITS_REMOVE_MISSING,
default=constants.CONFIG_UNITS_REMOVE_MISSING_DEFAULT)
self._validate_downloads = config.get(
importer_constants.KEY_VALIDATE,
default=constants.CONFIG_VALIDATE_DEFAULT)
self._repo_url = encode_unicode(config.get(
importer_constants.KEY_FEED))
# The _repo_url must end in a trailing slash, because we will use urljoin to determine
# the path to
# PULP_MANIFEST later
if self._repo_url[-1] != '/':
self._repo_url = self._repo_url + '/'
# Cast our config parameters to the correct types and use them to build a Downloader
max_speed = config.get(importer_constants.KEY_MAX_SPEED)
if max_speed is not None:
max_speed = float(max_speed)
max_downloads = config.get(importer_constants.KEY_MAX_DOWNLOADS)
if max_downloads is not None:
max_downloads = int(max_downloads)
else:
max_downloads = constants.CONFIG_MAX_DOWNLOADS_DEFAULT
ssl_validation = config.get_boolean(
importer_constants.KEY_SSL_VALIDATION)
ssl_validation = ssl_validation if ssl_validation is not None else \
constants.CONFIG_VALIDATE_DEFAULT
downloader_config = {
'max_speed': max_speed,
'max_concurrent': max_downloads,
'ssl_client_cert':
config.get(importer_constants.KEY_SSL_CLIENT_CERT),
'ssl_client_key':
config.get(importer_constants.KEY_SSL_CLIENT_KEY),
'ssl_ca_cert': config.get(importer_constants.KEY_SSL_CA_CERT),
'ssl_validation': ssl_validation,
'proxy_url': config.get(importer_constants.KEY_PROXY_HOST),
'proxy_port': config.get(importer_constants.KEY_PROXY_PORT),
'proxy_username': config.get(importer_constants.KEY_PROXY_USER),
'proxy_password': config.get(importer_constants.KEY_PROXY_PASS)
}
downloader_config = DownloaderConfig(**downloader_config)
# We will pass self as the event_listener, so that we can receive the callbacks in this
# class
if self._repo_url.lower().startswith('file'):
self.downloader = LocalFileDownloader(downloader_config, self)
else:
self.downloader = HTTPThreadedDownloader(downloader_config, self)
self.progress_report = SyncProgressReport(sync_conduit)
def _download_file(self, url, destination):
"""
Downloads the content at the given URL into the given destination.
The object passed into destination must have a method called "update"
that accepts a single parameter (the buffer that was read).
:param url: location to download
:type url: str
:param destination: object
@return:
"""
curl = self._create_and_configure_curl()
url = encode_unicode(url) # because of how the config is stored in pulp
curl.setopt(pycurl.URL, url)
curl.setopt(pycurl.WRITEFUNCTION, destination.update)
curl.perform()
status = curl.getinfo(curl.HTTP_CODE)
curl.close()
if status == 401:
raise exceptions.UnauthorizedException(url)
elif status == 404:
raise exceptions.FileNotFoundException(url)
elif status != 200:
raise exceptions.FileRetrievalException(url)
def process_relative_url(self, repo_id, importer_config,
distributor_config):
"""
During create (but not update), if the relative path isn't specified it
is derived from the feed_url.
Ultimately, this belongs in the distributor itself. When we rewrite the
yum distributor, we'll remove this entirely from the client.
jdob, May 10, 2013
"""
if constants.PUBLISH_RELATIVE_URL_KEYWORD not in distributor_config:
if importer_constants.KEY_FEED in importer_config:
if importer_config[importer_constants.KEY_FEED] is None:
self.prompt.render_failure_message(
_('Given repository feed URL is invalid.'))
return
url_parse = urlparse(encode_unicode(
importer_config[importer_constants.KEY_FEED]))
if url_parse[2] in ('', '/'):
relative_path = '/' + repo_id
else:
relative_path = url_parse[2]
distributor_config[constants.PUBLISH_RELATIVE_URL_KEYWORD] = relative_path # noqa
else:
distributor_config[constants.PUBLISH_RELATIVE_URL_KEYWORD] = repo_id # noqa
def process_relative_url(self, repo_id, importer_config,
yum_distributor_config):
"""
During create (but not update), if the relative path isn't specified it is derived
from the feed_url.
Ultimately, this belongs in the distributor itself. When we rewrite the yum distributor,
we'll remove this entirely from the client. jdob, May 10, 2013
"""
if 'relative_url' not in yum_distributor_config:
if importer_constants.KEY_FEED in importer_config:
if importer_config[importer_constants.KEY_FEED] is None:
self.prompt.render_failure_message(
_('Given repository feed URL is invalid.'))
return
url_parse = urlparse(
encode_unicode(
importer_config[importer_constants.KEY_FEED]))
if url_parse[2] in ('', '/'):
relative_path = '/' + repo_id
else:
relative_path = url_parse[2]
yum_distributor_config['relative_url'] = relative_path
else:
yum_distributor_config['relative_url'] = repo_id
def make_cert(self, cn, expiration, uid=None):
"""
Generate an x509 certificate with the Subject set to the cn passed into this method:
Subject: CN=someconsumer.example.com
@param cn: ID to be embedded in the certificate
@type cn: string
@param uid: The optional userid. In pulp, this is the DB document _id
for both users and consumers.
@type uid: str
@return: tuple of PEM encoded private key and certificate
@rtype: (str, str)
"""
# Ensure we are dealing with a string and not unicode
try:
cn = str(cn)
except UnicodeEncodeError:
cn = encode_unicode(cn)
_logger.debug("make_cert: [%s]" % cn)
# Make a private key
# Don't use M2Crypto directly as it leads to segfaults when trying to convert
# the key to a PEM string. Instead create the key with openssl and return the PEM string
# Sorta hacky but necessary.
# rsa = RSA.gen_key(1024, 65537, callback=passphrase_callback)
private_key_pem = _make_priv_key()
rsa = RSA.load_key_string(private_key_pem,
callback=util.no_passphrase_callback)
# Make the Cert Request
req, pub_key = _make_cert_request(cn, rsa, uid=uid)
# Sign it with the Pulp server CA
# We can't do this in m2crypto either so we have to shell out
ca_cert = config.config.get('security', 'cacert')
ca_key = config.config.get('security', 'cakey')
sn = SerialNumber()
serial = sn.next()
cmd = 'openssl x509 -req -sha1 -CA %s -CAkey %s -set_serial %s -days %d' % \
(ca_cert, ca_key, serial, expiration)
p = subprocess.Popen(cmd,
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
output = p.communicate(input=req.as_pem())[0]
p.wait()
exit_code = p.returncode
if exit_code != 0:
raise Exception("error signing cert request: %s" % output)
cert_pem_string = output[output.index("-----BEGIN CERTIFICATE-----"):]
return private_key_pem, cert_pem_string
def write(self,
content,
new_line=True,
center=False,
color=None,
tag=None,
skip_wrap=False):
content = encode_unicode(content)
Prompt.write(self, content, new_line, center, color, tag, skip_wrap)
def request(self, method, url, body):
headers = dict(self.pulp_connection.headers
) # copy so we don't affect the calling method
# Create a new connection each time since HTTPSConnection has problems
# reusing a connection for multiple calls (lame).
ssl_context = None
if self.pulp_connection.username and self.pulp_connection.password:
raw = ':'.join(
(self.pulp_connection.username, self.pulp_connection.password))
encoded = base64.encodestring(raw)[:-1]
headers['Authorization'] = 'Basic ' + encoded
elif self.pulp_connection.cert_filename:
ssl_context = SSL.Context('sslv3')
ssl_context.set_session_timeout(self.pulp_connection.timeout)
ssl_context.load_cert(self.pulp_connection.cert_filename)
# oauth configuration
if self.pulp_connection.oauth_key and self.pulp_connection.oauth_secret:
oauth_consumer = oauth.Consumer(self.pulp_connection.oauth_key,
self.pulp_connection.oauth_secret)
oauth_request = oauth.Request.from_consumer_and_token(
oauth_consumer,
http_method=method,
http_url='https://%s:%d%s' %
(self.pulp_connection.host, self.pulp_connection.port, url))
oauth_request.sign_request(oauth.SignatureMethod_HMAC_SHA1(),
oauth_consumer, None)
oauth_header = oauth_request.to_header()
# unicode header values causes m2crypto to do odd things.
for k, v in oauth_header.items():
oauth_header[k] = encode_unicode(v)
headers.update(oauth_header)
headers['pulp-user'] = self.pulp_connection.oauth_user
# Can't pass in None, so need to decide between two signatures (also lame)
if ssl_context is not None:
connection = httpslib.HTTPSConnection(self.pulp_connection.host,
self.pulp_connection.port,
ssl_context=ssl_context)
else:
connection = httpslib.HTTPSConnection(self.pulp_connection.host,
self.pulp_connection.port)
# Request against the server
connection.request(method, url, body=body, headers=headers)
try:
response = connection.getresponse()
except SSL.SSLError, err:
# Translate stale login certificate to an auth exception
if 'sslv3 alert certificate expired' == str(err):
raise exceptions.ClientSSLException(
self.pulp_connection.cert_filename)
else:
raise exceptions.ConnectionException(None, str(err), None)
def perform_sync(self, repo, sync_conduit, config):
"""
Perform the sync operation accoring to the config for the given repo, and return a report.
The sync progress will be reported through the sync_conduit.
:param repo: Metadata describing the repository
:type repo: pulp.server.plugins.model.Repository
:param sync_conduit: The sync_conduit that gives us access to the local repository
:type sync_conduit: pulp.server.conduits.repo_sync.RepoSyncConduit
:param config: The configuration for the importer
:type config: pulp.server.plugins.config.PluginCallConfiguration
:return: The sync report
:rtype: pulp.plugins.model.SyncReport
"""
# Build the progress report and set it to the running state
progress_report = SyncProgressReport(sync_conduit)
# Cast our config parameters to the correct types and use them to build an ISOBumper
max_speed = config.get(constants.CONFIG_MAX_SPEED)
if max_speed is not None:
max_speed = float(max_speed)
num_threads = config.get(constants.CONFIG_NUM_THREADS)
if num_threads is not None:
num_threads = int(num_threads)
else:
num_threads = constants.DEFAULT_NUM_THREADS
progress_report.metadata_state = STATE_RUNNING
progress_report.update_progress()
self.bumper = ISOBumper(
repo_url=encode_unicode(config.get(constants.CONFIG_FEED_URL)),
working_path=repo.working_dir,
max_speed=max_speed, num_threads=num_threads,
ssl_client_cert=config.get(constants.CONFIG_SSL_CLIENT_CERT),
ssl_client_key=config.get(constants.CONFIG_SSL_CLIENT_KEY),
ssl_ca_cert=config.get(constants.CONFIG_SSL_CA_CERT),
proxy_url=config.get(constants.CONFIG_PROXY_URL),
proxy_port=config.get(constants.CONFIG_PROXY_PORT),
proxy_user=config.get(constants.CONFIG_PROXY_USER),
proxy_password=config.get(constants.CONFIG_PROXY_PASSWORD))
# Get the manifest and download the ISOs that we are missing
manifest = self.bumper.get_manifest()
progress_report.metadata_state = STATE_COMPLETE
progress_report.modules_state = STATE_RUNNING
progress_report.update_progress()
missing_isos = self._filter_missing_isos(sync_conduit, manifest)
new_isos = self.bumper.download_resources(missing_isos)
# Move the downloaded stuff and junk to the permanent location
self._create_units(sync_conduit, new_isos)
# Report that we are finished
progress_report.modules_state = STATE_COMPLETE
report = progress_report.build_final_report()
return report
def make_cert(self, cn, expiration, uid=None):
"""
Generate an x509 certificate with the Subject set to the cn passed into this method:
Subject: CN=someconsumer.example.com
@param cn: ID to be embedded in the certificate
@type cn: string
@param uid: The optional userid. In pulp, this is the DB document _id
for both users and consumers.
@type uid: str
@return: tuple of PEM encoded private key and certificate
@rtype: (str, str)
"""
# Ensure we are dealing with a string and not unicode
try:
cn = str(cn)
except UnicodeEncodeError:
cn = encode_unicode(cn)
log.debug("make_cert: [%s]" % cn)
#Make a private key
# Don't use M2Crypto directly as it leads to segfaults when trying to convert
# the key to a PEM string. Instead create the key with openssl and return the PEM string
# Sorta hacky but necessary.
# rsa = RSA.gen_key(1024, 65537, callback=passphrase_callback)
private_key_pem = _make_priv_key()
rsa = RSA.load_key_string(private_key_pem,
callback=util.no_passphrase_callback)
# Make the Cert Request
req, pub_key = _make_cert_request(cn, rsa, uid=uid)
# Sign it with the Pulp server CA
# We can't do this in m2crypto either so we have to shell out
ca_cert = config.config.get('security', 'cacert')
ca_key = config.config.get('security', 'cakey')
sn = SerialNumber()
serial = sn.next()
cmd = 'openssl x509 -req -sha1 -CA %s -CAkey %s -set_serial %s -days %d' % \
(ca_cert, ca_key, serial, expiration)
p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output = p.communicate(input=req.as_pem())[0]
p.wait()
exit_code = p.returncode
if exit_code != 0:
raise Exception("error signing cert request: %s" % output)
cert_pem_string = output[output.index("-----BEGIN CERTIFICATE-----"):]
return private_key_pem, cert_pem_string
def __init__(self, id):
'''
Creates a new instance, populating itself with the default values for all
properties defined in PROPERTIES.
@param id: unique identifier for the repo
@type id: string
'''
self.id = encode_unicode(id)
for k, d in self.PROPERTIES:
self[k] = d
def __init__(self, id):
"""
Creates a new instance, populating itself with the default values for all
properties defined in PROPERTIES.
@param id: unique identifier for the repo
@type id: string
"""
self.id = encode_unicode(id)
for k, d in self.PROPERTIES:
self[k] = d
def request(self, method, url, body):
headers = dict(self.pulp_connection.headers) # copy so we don't affect the calling method
# Create a new connection each time since HTTPSConnection has problems
# reusing a connection for multiple calls (lame).
# Create SSL.Context and set it to verify peer SSL certificate against system CA certs.
ssl_context = SSL.Context('sslv3')
if self.pulp_connection.validate_ssl_ca:
ssl_context.set_verify(SSL.verify_peer, 1)
ssl_context.load_verify_locations(capath=self.pulp_connection.system_ca_dir)
ssl_context.set_session_timeout(self.pulp_connection.timeout)
if self.pulp_connection.username and self.pulp_connection.password:
raw = ':'.join((self.pulp_connection.username, self.pulp_connection.password))
encoded = base64.encodestring(raw)[:-1]
headers['Authorization'] = 'Basic ' + encoded
elif self.pulp_connection.cert_filename:
ssl_context.load_cert(self.pulp_connection.cert_filename)
# oauth configuration. This block is only True if oauth is not None, so it won't run on RHEL
# 5.
if self.pulp_connection.oauth_key and self.pulp_connection.oauth_secret and oauth:
oauth_consumer = oauth.Consumer(
self.pulp_connection.oauth_key,
self.pulp_connection.oauth_secret)
oauth_request = oauth.Request.from_consumer_and_token(
oauth_consumer,
http_method=method,
http_url='https://%s:%d%s' % (self.pulp_connection.host, self.pulp_connection.port, url))
oauth_request.sign_request(oauth.SignatureMethod_HMAC_SHA1(), oauth_consumer, None)
oauth_header = oauth_request.to_header()
# unicode header values causes m2crypto to do odd things.
for k, v in oauth_header.items():
oauth_header[k] = encode_unicode(v)
headers.update(oauth_header)
headers['pulp-user'] = self.pulp_connection.oauth_user
connection = httpslib.HTTPSConnection(
self.pulp_connection.host, self.pulp_connection.port, ssl_context=ssl_context)
try:
# Request against the server
connection.request(method, url, body=body, headers=headers)
response = connection.getresponse()
except SSL.SSLError, err:
# Translate stale login certificate to an auth exception
if 'sslv3 alert certificate expired' == str(err):
raise exceptions.ClientSSLException(self.pulp_connection.cert_filename)
else:
raise exceptions.ConnectionException(None, str(err), None)
def _ensure_input_encoding(self, input):
"""
Recursively traverse any input structures and ensure any strings are
encoded as utf-8
@param input: input data
@return: input data with strings encoded as utf-8
"""
if isinstance(input, (list, set, tuple)):
return [self._ensure_input_encoding(i) for i in input]
if isinstance(input, dict):
return dict((self._ensure_input_encoding(k), self._ensure_input_encoding(v)) for k, v in input.items())
try:
return encode_unicode(decode_unicode(input))
except (UnicodeDecodeError, UnicodeEncodeError):
raise InputEncodingError(input), None, sys.exc_info()[2]
def _ensure_input_encoding(self, input):
"""
Recursively traverse any input structures and ensure any strings are
encoded as utf-8
@param input: input data
@return: input data with strings encoded as utf-8
"""
if isinstance(input, (list, set, tuple)):
return [self._ensure_input_encoding(i) for i in input]
if isinstance(input, dict):
return dict((self._ensure_input_encoding(k),
self._ensure_input_encoding(v))
for k, v in input.items())
try:
return encode_unicode(decode_unicode(input))
except (UnicodeDecodeError, UnicodeEncodeError):
raise InputEncodingError(input), None, sys.exc_info()[2]
def __get_groups_xml_info(repo_dir):
groups_xml_path = None
repodata_file = os.path.join(repo_dir, "repodata", "repomd.xml")
repodata_file = encode_unicode(repodata_file)
if os.path.isfile(repodata_file):
ftypes = util.get_repomd_filetypes(repodata_file)
_LOG.debug("repodata has filetypes of %s" % (ftypes))
if "group" in ftypes:
comps_ftype = util.get_repomd_filetype_path(
repodata_file, "group")
filetype_path = os.path.join(repo_dir, comps_ftype)
# createrepo uses filename as mdtype, rename to type.<ext>
# to avoid filename too long errors
renamed_filetype_path = os.path.join(os.path.dirname(comps_ftype),
"comps" + '.' + '.'.join(os.path.basename(comps_ftype).split('.')[1:]))
groups_xml_path = os.path.join(repo_dir, renamed_filetype_path)
os.rename(filetype_path, groups_xml_path)
return groups_xml_path
def _backup_existing_repodata(self):
"""
Takes a backup of any existing repodata files. This is used in the final
step where other file types in repomd.xml such as presto, updateinfo, comps
are copied back to the repodata.
"""
current_repo_dir = os.path.join(self.repodir, "repodata")
# Note: backup_repo_dir is used to store presto metadata and possibly other custom metadata types
# they will be copied back into new 'repodata' if needed.
current_repo_dir = encode_unicode(current_repo_dir)
if os.path.exists(current_repo_dir):
_LOG.info("existing metadata found; taking backup.")
self.backup_repodata_dir = os.path.join(self.repodir, "repodata.old")
if os.path.exists(self.backup_repodata_dir):
_LOG.debug("clean up any stale dirs")
shutil.rmtree(self.backup_repodata_dir)
shutil.copytree(current_repo_dir, self.backup_repodata_dir)
os.system("chmod -R u+wX %s" % self.backup_repodata_dir)
def _create_repo(dir, groups=None, checksum_type=DEFAULT_CHECKSUM):
if not groups:
cmd = "createrepo --database --checksum %s --update %s " % (checksum_type, dir)
else:
try:
cmd = "createrepo --database --checksum %s -g %s --update %s " % (checksum_type, groups, dir)
except UnicodeDecodeError:
groups = decode_unicode(groups)
cmd = "createrepo --database --checksum %s -g %s --update %s " % (checksum_type, groups, dir)
# shlex now can handle unicode strings as well
cmd = encode_unicode(cmd)
try:
cmd = shlex.split(cmd.encode('ascii', 'ignore'))
except:
cmd = shlex.split(cmd)
_LOG.info("started repo metadata update: %s" % (cmd))
handle = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
return handle
def subject(self):
"""
Get the certificate subject.
note: Missing NID mapping for UID added to patch openssl.
@return: A dictionary of subject fields.
@rtype: dict
"""
d = {}
subject = self.x509.get_subject()
subject.nid['UID'] = 458
for key, nid in subject.nid.items():
entry = subject.get_entries_by_nid(nid)
if len(entry):
asn1 = entry[0].get_data()
try:
d[key] = str(asn1)
except UnicodeEncodeError:
d[key] = encode_unicode(asn1)
continue
return d
def subject(self):
"""
Get the certificate subject.
note: Missing NID mapping for UID added to patch openssl.
@return: A dictionary of subject fields.
@rtype: dict
"""
d = {}
subject = self.x509.get_subject()
subject.nid['UID'] = 458
for key, nid in subject.nid.items():
entry = subject.get_entries_by_nid(nid)
if len(entry):
asn1 = entry[0].get_data()
try:
d[key] = str(asn1)
except UnicodeEncodeError:
d[key] = encode_unicode(asn1)
continue
return d
def initialize(self):
"""
Set up the nectar downloader
Originally based on the ISO sync setup
"""
config = self.get_config()
self._validate_downloads = config.get(importer_constants.KEY_VALIDATE, default=True)
self._repo_url = encode_unicode(config.get(importer_constants.KEY_FEED))
# The _repo_url must end in a trailing slash, because we will use
# urljoin to determine the path later
if self._repo_url[-1] != '/':
self._repo_url = self._repo_url + '/'
downloader_config = importer_config_to_nectar_config(config.flatten())
# We will pass self as the event_listener, so that we can receive the
# callbacks in this class
if self._repo_url.lower().startswith('file'):
self.downloader = LocalFileDownloader(downloader_config, self)
else:
self.downloader = HTTPThreadedDownloader(downloader_config, self)
def initialize(self):
"""
Set up the nectar downloader
Originally based on the ISO sync setup
"""
config = self.get_config()
self._validate_downloads = config.get(importer_constants.KEY_VALIDATE, default=True)
self._repo_url = encode_unicode(config.get(importer_constants.KEY_FEED))
# The _repo_url must end in a trailing slash, because we will use
# urljoin to determine the path later
if self._repo_url[-1] != '/':
self._repo_url = self._repo_url + '/'
downloader_config = importer_config_to_nectar_config(config.flatten())
# We will pass self as the event_listener, so that we can receive the
# callbacks in this class
if self._repo_url.lower().startswith('file'):
self.downloader = LocalFileDownloader(downloader_config, self)
else:
self.downloader = HTTPThreadedDownloader(downloader_config, self)
def process_relative_url(self, repo_id, importer_config, yum_distributor_config):
"""
During create (but not update), if the relative path isn't specified it is derived
from the feed_url.
Ultimately, this belongs in the distributor itself. When we rewrite the yum distributor,
we'll remove this entirely from the client. jdob, May 10, 2013
"""
if "relative_url" not in yum_distributor_config:
if importer_constants.KEY_FEED in importer_config:
if importer_config[importer_constants.KEY_FEED] is None:
self.prompt.render_failure_message(_("Given repository feed URL is invalid."))
return
url_parse = urlparse(encode_unicode(importer_config[importer_constants.KEY_FEED]))
if url_parse[2] in ("", "/"):
relative_path = "/" + repo_id
else:
relative_path = url_parse[2]
yum_distributor_config["relative_url"] = relative_path
else:
yum_distributor_config["relative_url"] = repo_id
def request(self, method, url, body):
"""
Make the request against the Pulp server, returning a tuple of (status_code, respose_body).
:param method: The HTTP method to be used for the request (GET, POST, etc.)
:type method: str
:param url: The Pulp URL to make the request against
:type url: str
:param body: The body to pass with the request
:type body: str
:return: A 2-tuple of the status_code and response_body. status_code is the HTTP
status code (200, 404, etc.). If the server's response is valid json,
it will be parsed and response_body will be a dictionary. If not, it will be
returned as a string.
:rtype: tuple
"""
headers = dict(self.pulp_connection.headers) # copy so we don't affect the calling method
# Create a new connection each time since HTTPSConnection has problems
# reusing a connection for multiple calls (lame).
ssl_context = SSL.Context('sslv3')
if self.pulp_connection.verify_ssl:
ssl_context.set_verify(SSL.verify_peer, depth=100)
# We need to stat the ca_path to see if it exists (error if it doesn't), and if so
# whether it is a file or a directory. m2crypto has different directives depending on
# which type it is.
if os.path.isfile(self.pulp_connection.ca_path):
ssl_context.load_verify_locations(cafile=self.pulp_connection.ca_path)
elif os.path.isdir(self.pulp_connection.ca_path):
ssl_context.load_verify_locations(capath=self.pulp_connection.ca_path)
else:
# If it's not a file and it's not a directory, it's not a valid setting
raise exceptions.MissingCAPathException(self.pulp_connection.ca_path)
ssl_context.set_session_timeout(self.pulp_connection.timeout)
if self.pulp_connection.username and self.pulp_connection.password:
raw = ':'.join((self.pulp_connection.username, self.pulp_connection.password))
encoded = base64.encodestring(raw)[:-1]
headers['Authorization'] = 'Basic ' + encoded
elif self.pulp_connection.cert_filename:
ssl_context.load_cert(self.pulp_connection.cert_filename)
# oauth configuration. This block is only True if oauth is not None, so it won't run on RHEL
# 5.
if self.pulp_connection.oauth_key and self.pulp_connection.oauth_secret and oauth:
oauth_consumer = oauth.Consumer(
self.pulp_connection.oauth_key,
self.pulp_connection.oauth_secret)
oauth_request = oauth.Request.from_consumer_and_token(
oauth_consumer,
http_method=method,
http_url='https://%s:%d%s' % (self.pulp_connection.host, self.pulp_connection.port, url))
oauth_request.sign_request(oauth.SignatureMethod_HMAC_SHA1(), oauth_consumer, None)
oauth_header = oauth_request.to_header()
# unicode header values causes m2crypto to do odd things.
for k, v in oauth_header.items():
oauth_header[k] = encode_unicode(v)
headers.update(oauth_header)
headers['pulp-user'] = self.pulp_connection.oauth_user
connection = httpslib.HTTPSConnection(
self.pulp_connection.host, self.pulp_connection.port, ssl_context=ssl_context)
try:
# Request against the server
connection.request(method, url, body=body, headers=headers)
response = connection.getresponse()
except SSL.SSLError, err:
# Translate stale login certificate to an auth exception
if 'sslv3 alert certificate expired' == str(err):
raise exceptions.ClientCertificateExpiredException(
self.pulp_connection.cert_filename)
elif 'certificate verify failed' in str(err):
raise exceptions.CertificateVerificationException()
else:
raise exceptions.ConnectionException(None, str(err), None)
def request(self, method, url, body):
"""
Make the request against the Pulp server, returning a tuple of (status_code, respose_body).
This method creates a new connection each time since HTTPSConnection has problems
reusing a connection for multiple calls (as claimed by a prior comment in this module).
:param method: The HTTP method to be used for the request (GET, POST, etc.)
:type method: str
:param url: The Pulp URL to make the request against
:type url: str
:param body: The body to pass with the request
:type body: str
:return: A 2-tuple of the status_code and response_body. status_code is the HTTP
status code (200, 404, etc.). If the server's response is valid json,
it will be parsed and response_body will be a dictionary. If not, it will be
returned as a string.
:rtype: tuple
"""
headers = dict(self.pulp_connection.headers
) # copy so we don't affect the calling method
# Despite the confusing name, 'sslv23' configures m2crypto to use any available protocol in
# the underlying openssl implementation.
ssl_context = SSL.Context('sslv23')
# This restricts the protocols we are willing to do by configuring m2 not to do SSLv2.0 or
# SSLv3.0. EL 5 does not have support for TLS > v1.0, so we have to leave support for
# TLSv1.0 enabled.
ssl_context.set_options(m2.SSL_OP_NO_SSLv2 | m2.SSL_OP_NO_SSLv3)
if self.pulp_connection.verify_ssl:
ssl_context.set_verify(SSL.verify_peer, depth=100)
# We need to stat the ca_path to see if it exists (error if it doesn't), and if so
# whether it is a file or a directory. m2crypto has different directives depending on
# which type it is.
if os.path.isfile(self.pulp_connection.ca_path):
ssl_context.load_verify_locations(
cafile=self.pulp_connection.ca_path)
elif os.path.isdir(self.pulp_connection.ca_path):
ssl_context.load_verify_locations(
capath=self.pulp_connection.ca_path)
else:
# If it's not a file and it's not a directory, it's not a valid setting
raise exceptions.MissingCAPathException(
self.pulp_connection.ca_path)
ssl_context.set_session_timeout(self.pulp_connection.timeout)
if self.pulp_connection.username and self.pulp_connection.password:
raw = ':'.join(
(self.pulp_connection.username, self.pulp_connection.password))
encoded = base64.encodestring(raw)[:-1]
headers['Authorization'] = 'Basic ' + encoded
elif self.pulp_connection.cert_filename:
ssl_context.load_cert(self.pulp_connection.cert_filename)
# oauth configuration. This block is only True if oauth is not None, so it won't run on RHEL
# 5.
if self.pulp_connection.oauth_key and self.pulp_connection.oauth_secret and oauth:
oauth_consumer = oauth.Consumer(self.pulp_connection.oauth_key,
self.pulp_connection.oauth_secret)
oauth_request = oauth.Request.from_consumer_and_token(
oauth_consumer,
http_method=method,
http_url='https://%s:%d%s' %
(self.pulp_connection.host, self.pulp_connection.port, url))
oauth_request.sign_request(oauth.SignatureMethod_HMAC_SHA1(),
oauth_consumer, None)
oauth_header = oauth_request.to_header()
# unicode header values causes m2crypto to do odd things.
for k, v in oauth_header.items():
oauth_header[k] = encode_unicode(v)
headers.update(oauth_header)
headers['pulp-user'] = self.pulp_connection.oauth_user
connection = httpslib.HTTPSConnection(self.pulp_connection.host,
self.pulp_connection.port,
ssl_context=ssl_context)
try:
# Request against the server
connection.request(method, url, body=body, headers=headers)
response = connection.getresponse()
except SSL.SSLError, err:
# Translate stale login certificate to an auth exception
if 'sslv3 alert certificate expired' == str(err):
raise exceptions.ClientCertificateExpiredException(
self.pulp_connection.cert_filename)
elif 'certificate verify failed' in str(err):
raise exceptions.CertificateVerificationException()
else:
raise exceptions.ConnectionException(None, str(err), None)
def _write_cert_bundle(self, file_prefix, cert_dir, bundle):
'''
Writes the files represented by the cert bundle to a directory on the
Pulp server unique to the given repo. If certificates already exist in the
repo's certificate directory, they will be overwritten. If the value for
any bundle component is None, the associated file will be erased if one
exists. The file prefix will be used to differentiate between files that
belong to the feed bundle v. those that belong to the consumer bundle.
@param file_prefix: used in the filename of the bundle item to differentiate it
from other bundles; cannot be None
@type file_prefix: str
@param cert_dir: absolute path to the location in which the cert bundle should
be written; cannot be None
@type cert_dir: str
@param bundle: cert bundle (see module docs for more information on format)
@type bundle: dict {str, str}
@raises ValueError: if bundle is invalid (see validate_cert_bundle)
@return: mapping of cert bundle item (see module docs) to the absolute path
to where it is stored on disk
'''
file_prefix = encode_unicode(file_prefix)
cert_dir = encode_unicode(cert_dir)
WRITE_LOCK.acquire()
try:
# Create the cert directory if it doesn't exist
if not os.path.exists(cert_dir):
os.makedirs(cert_dir)
# For each item in the cert bundle, save it to disk using the given prefix
# to identify the type of bundle it belongs to. If the value is None, the
# item is being deleted.
cert_files = {}
for key, value in bundle.items():
filename = os.path.join(cert_dir, '%s.%s' % (file_prefix, key))
try:
if value is None:
if os.path.exists(filename):
LOG.info('Removing repo cert file [%s]' % filename)
os.remove(filename)
cert_files[key] = None
else:
LOG.info('Storing repo cert file [%s]' % filename)
f = open(filename, 'w')
f.write(value)
f.close()
cert_files[key] = str(filename)
except Exception:
LOG.exception('Error storing certificate file [%s]' %
filename)
raise Exception('Error storing certificate file [%s]' %
filename)
return cert_files
finally:
WRITE_LOCK.release()
def __init__(self, msg=None):
if msg is None:
class_name = self.__class__.__name__
msg = _('Pulp auth exception occurred: %(c)s') % {'c': class_name}
self.msg = encode_unicode(msg)
def create_repo(dir, groups=None, checksum_type=DEFAULT_CHECKSUM, skip_metadata_types=[]):
handle = None
# Lock the lookup and launch of a new createrepo process
# Lock is released once createrepo is launched
if not os.path.exists(dir):
_LOG.warning("create_repo invoked on a directory which doesn't exist: %s" % dir)
CREATE_REPO_PROCESS_LOOKUP_LOCK.acquire()
try:
if CREATE_REPO_PROCESS_LOOKUP.has_key(dir):
raise CreateRepoAlreadyRunningError()
current_repo_dir = os.path.join(dir, "repodata")
# Note: backup_repo_dir is used to store presto metadata and possibly other custom metadata types
# they will be copied back into new 'repodata' if needed.
backup_repo_dir = None
current_repo_dir = encode_unicode(current_repo_dir)
if os.path.exists(current_repo_dir):
_LOG.info("metadata found; taking backup.")
backup_repo_dir = os.path.join(dir, "repodata.old")
if os.path.exists(backup_repo_dir):
_LOG.debug("clean up any stale dirs")
shutil.rmtree(backup_repo_dir)
shutil.copytree(current_repo_dir, backup_repo_dir)
os.system("chmod -R u+wX %s" % (backup_repo_dir))
handle = _create_repo(dir, groups=groups, checksum_type=checksum_type)
if not handle:
raise CreateRepoError("Unable to execute createrepo on %s" % (dir))
os.system("chmod -R ug+wX %s" % (dir))
_LOG.info("Createrepo process with pid %s running on directory %s" % (handle.pid, dir))
CREATE_REPO_PROCESS_LOOKUP[dir] = handle
finally:
CREATE_REPO_PROCESS_LOOKUP_LOCK.release()
# Ensure we clean up CREATE_REPO_PROCESS_LOOKUP, surround all ops with try/finally
try:
# Block on process till complete (Note it may be async terminated)
out_msg, err_msg = handle.communicate(None)
if handle.returncode != 0:
try:
# Cleanup createrepo's temporary working directory
cleanup_dir = os.path.join(dir, ".repodata")
if os.path.exists(cleanup_dir):
shutil.rmtree(cleanup_dir)
except Exception, e:
_LOG.exception(e)
_LOG.warn("Unable to remove temporary createrepo dir: %s" % (cleanup_dir))
if handle.returncode == -9:
_LOG.warn("createrepo on %s was killed" % (dir))
raise CancelException()
else:
_LOG.error("createrepo on %s failed with returncode <%s>" % (dir, handle.returncode))
_LOG.error("createrepo stdout:\n%s" % (out_msg))
_LOG.error("createrepo stderr:\n%s" % (err_msg))
raise CreateRepoError(err_msg)
_LOG.info("createrepo on %s finished" % (dir))
if not backup_repo_dir:
_LOG.info("Nothing further to check; we got our fresh metadata")
return
#check if presto metadata exist in the backup
repodata_file = os.path.join(backup_repo_dir, "repomd.xml")
ftypes = util.get_repomd_filetypes(repodata_file)
base_ftypes = ['primary', 'primary_db', 'filelists_db', 'filelists', 'other', 'other_db', 'group', 'group_gz']
for ftype in ftypes:
if ftype in base_ftypes:
# no need to process these again
continue
if ftype in skip_metadata_types and not skip_metadata_types[ftype]:
_LOG.info("mdtype %s part of skip metadata; skipping" % ftype)
continue
filetype_path = os.path.join(backup_repo_dir, os.path.basename(util.get_repomd_filetype_path(repodata_file, ftype)))
# modifyrepo uses filename as mdtype, rename to type.<ext>
renamed_filetype_path = os.path.join(os.path.dirname(filetype_path), \
ftype + '.' + '.'.join(os.path.basename(filetype_path).split('.')[1:]))
os.rename(filetype_path, renamed_filetype_path)
if renamed_filetype_path.endswith('.gz'):
# if file is gzipped, decompress before passing to modifyrepo
data = gzip.open(renamed_filetype_path).read().decode("utf-8", "replace")
renamed_filetype_path = '.'.join(renamed_filetype_path.split('.')[:-1])
open(renamed_filetype_path, 'w').write(data.encode("UTF-8"))
if os.path.isfile(renamed_filetype_path):
_LOG.info("Modifying repo for %s metadata" % ftype)
modify_repo(current_repo_dir, renamed_filetype_path, checksum_type=checksum_type)
def _write_cert_bundle(self, file_prefix, cert_dir, bundle):
'''
Writes the files represented by the cert bundle to a directory on the
Pulp server unique to the given repo. If certificates already exist in the
repo's certificate directory, they will be overwritten. If the value for
any bundle component is None, the associated file will be erased if one
exists. The file prefix will be used to differentiate between files that
belong to the feed bundle v. those that belong to the consumer bundle.
@param file_prefix: used in the filename of the bundle item to differentiate it
from other bundles; cannot be None
@type file_prefix: str
@param cert_dir: absolute path to the location in which the cert bundle should
be written; cannot be None
@type cert_dir: str
@param bundle: cert bundle (see module docs for more information on format)
@type bundle: dict {str, str}
@raises ValueError: if bundle is invalid (see validate_cert_bundle)
@return: mapping of cert bundle item (see module docs) to the absolute path
to where it is stored on disk
'''
file_prefix = encode_unicode(file_prefix)
cert_dir = encode_unicode(cert_dir)
WRITE_LOCK.acquire()
try:
# Create the cert directory if it doesn't exist
if not os.path.exists(cert_dir):
os.makedirs(cert_dir)
# For each item in the cert bundle, save it to disk using the given prefix
# to identify the type of bundle it belongs to. If the value is None, the
# item is being deleted.
cert_files = {}
for key, value in bundle.items():
filename = os.path.join(cert_dir, '%s.%s' % (file_prefix, key))
try:
if value is None:
if os.path.exists(filename):
LOG.info('Removing repo cert file [%s]' % filename)
os.remove(filename)
cert_files[key] = None
else:
LOG.info('Storing repo cert file [%s]' % filename)
f = open(filename, 'w')
f.write(value)
f.close()
cert_files[key] = str(filename)
except:
LOG.exception('Error storing certificate file [%s]' % filename)
raise Exception('Error storing certificate file [%s]' % filename)
return cert_files
finally:
WRITE_LOCK.release()
def write(self, content, new_line=True, center=False, color=None, tag=None,
skip_wrap=False):
content = encode_unicode(content)
Prompt.write(self, content, new_line, center, color, tag, skip_wrap)
def render_document_list(self, items, filters=None, order=None, spaces_between_cols=1, indent=0,
step=2, omit_hidden=True, header_func=None, num_separator_spaces=1):
"""
Prints a list of JSON documents retrieved from the REST bindings (more
generally, will print any list of dicts). The data will be output as
an aligned series of key-value pairs. Keys will be capitalized and all
unicode markers inserted from the JSON serialization (i.e. u'text')
will be stripped automatically.
If filters are specified, only keys in the list of filters will be output.
Thus the data does not need to be pre-stripped of unwanted fields, this
call will skip them.
The order argument is a list of keys in the order they should be
rendered. Any keys not in the given list but that have passed the filter
test described above will be rendered in alphabetical order following
the ordered items.
If specified, the header_func must be a function that accepts a single
parameter and returns a string. The parameter will be the item (document)
about to be rendered. The returned value will be rendered prior to
rendering the document itself, providing a way to output a header or
separator between items for UI clarity.
:param items: list of items (each a dict) to render
:type items: list
:param filters: list of fields in each dict to display
:type filters: list
:param order: list of fields specifying the order in which they are rendered
:type order: list
:param spaces_between_cols: number of spaces between the key and value columns
:type spaces_between_cols: int
:param header_func: function to be applied to the item before it is
rendered; the results will be printed prior to rendering the item
:type header_func: function
:param num_separator_spaces: number of blank lines to include after each
item in the list
:type num_separator_spaces: int
"""
# Punch out early if the items list is empty; we access the first
# element later for max width calculation so we need there to be at
# least one item.
if len(items) is 0:
return
all_keys = items[0].keys()
# If no filters were specified, consider the filter to be all keys. This
# will make later calculations a ton easier.
if filters is None:
filters = all_keys
# Apply the filters
filtered_items = []
for i in items:
filtered = dict([(k, v) for k, v in i.items() if k in filters])
filtered_items.append(filtered)
# Determine the order to display the items
if order is None:
ordered_keys = sorted(filters)
else:
# Remove any keys from the order that weren't in the filter
filtered_order = [o for o in order if o in filters]
# The order may only be a subset of filtered keys, so figure out
# which ones are missing and tack them onto the end
not_ordered = [k for k in filters if k not in filtered_order]
# Assemble the pieces: ordered keys + not ordered keys
ordered_keys = order + sorted(not_ordered)
# Generate a list of tuples of key to pretty-formatted key
ordered_formatted_keys = []
for k in ordered_keys:
formatted_key = None
# Don't apply the fancy _ stripping logic to values that start with _
# These values probably shouldn't be in the returned document, but
# let's not rely on that.
if k.startswith('_'):
if omit_hidden:
continue
else:
formatted_key = k
else:
for part in k.split('_'):
part = str(part)
if formatted_key is None:
formatted_part = part.capitalize()
if formatted_part in CAPITALIZE_WORD_EXCEPTIONS:
formatted_part = CAPITALIZE_WORD_EXCEPTIONS[formatted_part]
formatted_key = formatted_part
else:
formatted_part = part.capitalize()
if formatted_part in CAPITALIZE_WORD_EXCEPTIONS:
formatted_part = CAPITALIZE_WORD_EXCEPTIONS[formatted_part]
formatted_key += ' '
formatted_key += formatted_part
ordered_formatted_keys.append((k, formatted_key))
# Generate template using the formatted key values for proper length checking
# +1 for the : appended later
max_key_length = reduce(
lambda x, y: max(x, len(y)), [o[1] for o in ordered_formatted_keys], 0
) + 1
line_template = (' ' * indent) + '%-' + str(max_key_length) + 's' + \
(' ' * spaces_between_cols) + '%s'
# Print each item
for i in filtered_items:
if not i:
continue
if header_func is not None:
h = header_func(i)
self.write(h)
for k, formatted_k in ordered_formatted_keys:
# If a filter was specified for a value that's not there, that's
# ok, just skip it
if k not in i:
continue
v = i[k]
if isinstance(v, dict):
self.write(line_template % (formatted_k + ':', ''))
self.render_document_list([v], indent=indent + step, omit_hidden=omit_hidden)
continue
# If the value is a list, pretty it up
if isinstance(v, (tuple, list)):
if len(v) > 0 and isinstance(v[0], dict):
self.write(line_template % (formatted_k + ':', ''))
self.render_document_list(v, indent=indent + step, omit_hidden=omit_hidden)
continue
else:
try:
v = ', '.join(v)
except TypeError:
# This is ugly, but it's the quickest way to get around
# lists of other lists.
pass
else:
if isinstance(v, (str, unicode)):
v = v.replace('\n', ' ')
line = line_template % (formatted_k + ':', encode_unicode(v))
long_value_indent = max_key_length + spaces_between_cols + indent
line = self.wrap(line, remaining_line_indent=long_value_indent)
# get rid of anything that isn't an ascii character
line = line.decode('ascii', errors='ignore')
self.write(line, tag=TAG_DOCUMENT, skip_wrap=True)
# Only add a space if we're at the highest level of the rendering
if indent is 0:
self.render_spacer(lines=num_separator_spaces)
# Only add a space if we're at the highest level of the rendering
if indent is 0:
self.render_spacer()
def render_document_list(self, items, filters=None, order=None,
spaces_between_cols=1, indent=0, step=2,
omit_hidden=True, header_func=None,
num_separator_spaces=1):
"""
Prints a list of JSON documents retrieved from the REST bindings (more
generally, will print any list of dicts). The data will be output as
an aligned series of key-value pairs. Keys will be capitalized and all
unicode markers inserted from the JSON serialization (i.e. u'text')
will be stripped automatically.
If filters are specified, only keys in the list of filters will be output.
Thus the data does not need to be pre-stripped of unwanted fields, this
call will skip them.
The order argument is a list of keys in the order they should be
rendered. Any keys not in the given list but that have passed the filter
test described above will be rendered in alphabetical order following
the ordered items.
If specified, the header_func must be a function that accepts a single
parameter and returns a string. The parameter will be the item (document)
about to be rendered. The returned value will be rendered prior to
rendering the document itself, providing a way to output a header or
separator between items for UI clarity.
:param items: list of items (each a dict) to render
:type items: list
:param filters: list of fields in each dict to display
:type filters: list
:param order: list of fields specifying the order in which they are rendered
:type order: list
:param spaces_between_cols: number of spaces between the key and value columns
:type spaces_between_cols: int
:param header_func: function to be applied to the item before it is
rendered; the results will be printed prior to rendering the item
:type header_func: function
:param num_separator_spaces: number of blank lines to include after each
item in the list
:type num_separator_spaces: int
"""
# Punch out early if the items list is empty; we access the first
# element later for max width calculation so we need there to be at
# least one item.
if len(items) is 0:
return
all_keys = items[0].keys()
# If no filters were specified, consider the filter to be all keys. This
# will make later calculations a ton easier.
if filters is None:
filters = all_keys
# Apply the filters
filtered_items = []
for i in items:
filtered = dict([(k, v) for k, v in i.items() if k in filters])
filtered_items.append(filtered)
# Determine the order to display the items
if order is None:
ordered_keys = sorted(filters)
else:
# Remove any keys from the order that weren't in the filter
filtered_order = [o for o in order if o in filters]
# The order may only be a subset of filtered keys, so figure out
# which ones are missing and tack them onto the end
not_ordered = [k for k in filters if k not in filtered_order]
# Assemble the pieces: ordered keys + not ordered keys
ordered_keys = order + sorted(not_ordered)
# Generate a list of tuples of key to pretty-formatted key
ordered_formatted_keys = []
for k in ordered_keys:
formatted_key = None
# Don't apply the fancy _ stripping logic to values that start with _
# These values probably shouldn't be in the returned document, but
# let's not rely on that.
if k.startswith('_'):
if omit_hidden:
continue
else:
formatted_key = k
else:
for part in k.split('_'):
part = str(part)
if formatted_key is None:
formatted_part = part.capitalize()
if formatted_part in CAPITALIZE_WORD_EXCEPTIONS:
formatted_part = CAPITALIZE_WORD_EXCEPTIONS[formatted_part]
formatted_key = formatted_part
else:
formatted_part = part.capitalize()
if formatted_part in CAPITALIZE_WORD_EXCEPTIONS:
formatted_part = CAPITALIZE_WORD_EXCEPTIONS[formatted_part]
formatted_key += ' '
formatted_key += formatted_part
ordered_formatted_keys.append((k, formatted_key))
# Generate template using the formatted key values for proper length checking
max_key_length = reduce(lambda x, y: max(x, len(y)), [o[1] for o in ordered_formatted_keys], 0) + 1 # +1 for the : appended later
line_template = (' ' * indent) + '%-' + str(max_key_length) + 's' + (' ' * spaces_between_cols) + '%s'
# Print each item
for i in filtered_items:
if not i:
continue
if header_func is not None:
h = header_func(i)
self.write(h)
for k, formatted_k in ordered_formatted_keys:
# If a filter was specified for a value that's not there, that's
# ok, just skip it
if k not in i:
continue
v = i[k]
if isinstance(v, dict):
self.write(line_template % (formatted_k + ':', ''))
self.render_document_list([v], indent=indent+step)
continue
# If the value is a list, pretty it up
if isinstance(v, (tuple, list)):
if len(v) > 0 and isinstance(v[0], dict):
self.write(line_template % (formatted_k + ':', ''))
self.render_document_list(v, indent=indent+step)
continue
else:
try:
v = ', '.join(v)
except TypeError:
# This is ugly, but it's the quickest way to get around
# lists of other lists.
pass
else:
if isinstance(v, (str, unicode)):
v = v.replace('\n', ' ')
line = line_template % (formatted_k + ':', encode_unicode(v))
long_value_indent = max_key_length + spaces_between_cols + indent
line = self.wrap(line, remaining_line_indent=long_value_indent)
self.write(line, tag=TAG_DOCUMENT, skip_wrap=True)
# Only add a space if we're at the highest level of the rendering
if indent is 0:
self.render_spacer(lines=num_separator_spaces)
# Only add a space if we're at the highest level of the rendering
if indent is 0:
self.render_spacer()
