def cmpNEVR(nevr1, nevr2):
'''Compare two RPM version identifiers in NEVR format.
:param str nevr1: RPM identifier in N(E)VR format
:param str nevr2: RPM identifier in N(E)VR format
:return: ``-1``/``0``/``1`` if ``nevr1 < nevr2`` / ``nevr1 == nevr2`` /
``nevr1 > nevr2``
:rtype: int
:raise CheckbValueError: if name in ``nevr1`` doesn't match name in
``nevr2``
'''
rpmver1 = hawkey.split_nevra(nevr1 + '.noarch')
rpmver2 = hawkey.split_nevra(nevr2 + '.noarch')
if rpmver1.name != rpmver2.name:
raise exc.CheckbValueError("Name in nevr1 doesn't match name in "
"nevr2: %s, %s" % (nevr1, nevr2))
# sack is needed for the comparison, because it can be influence the
# comparison (for example epoch can be set to be ignored). A default empty
# sack should match Fedora customs
sack = hawkey.Sack()
# we need evr_cmp to return int so we can use it as comparison function
# in python's sorted
return int(rpmver1.evr_cmp(rpmver2, sack))
def _collection_of(obj, collection_cls, item_type=None):
'''The same as :func:`iterable` or :func:`sequence`, but the abstract
collection class can be specified dynamically with ``collection_cls``.
'''
if not isinstance(obj, collection_cls) or isinstance(obj, basestring):
return False
if item_type is not None and isinstance(obj, abc.Iterable):
try:
return all([isinstance(item, item_type) for item in obj])
except TypeError as e:
raise exc.CheckbValueError(
"'item_type' must be a type definition, not '%r': %s" %
(item_type, e))
return True
def get_dist_tag(rpmstr):
'''Parse disttag from an RPM package version string.
:param str rpmstr: string to be manipulated in a format of N(E)VR
(``foo-1.2-3.fc20`` or ``bar-4:1.2-3.fc20``) or N(E)VRA
(``foo-1.2-3.fc20.x86_64`` or ``bar-4:1.2-3.fc20.i686``)
:return: string containing dist tag (``fc20``)
:raise CheckbValueError: if ``rpmstr`` does not contain dist tag
'''
release = rpmformat(rpmstr, 'r')
matches = re.findall(r'\.(fc\d{1,2})\.?', release)
if not matches:
raise exc.CheckbValueError('Could not parse disttag from %s' % rpmstr)
# there might be e.g. fc22 in git commit hash as part of the release,
# so just take the last match which should be disttag
return matches[-1]
def popen_rt(cmd, stderr=subprocess.STDOUT, bufsize=1, **kwargs):
"""This is similar to :func:`subprocess.check_output`, but with real-time printing to console
as well. It is useful for longer-running tasks for which you'd like to both capture the output
and also see it printed in terminal continuously.
Please note that by default both ``stdout`` and ``stderr`` are merged together. You can use
``stderr=subprocess.PIPE``, and it will be returned to you separated, but it won't be printed
out to console (only ``stdout`` will).
The parameters are the same as for :class:`subprocess.Popen`. You can't use ``stdout``
parameter, that one is hardcoded to :attr:`subprocess.PIPE`.
:return: tuple of ``(stdoutdata, stderrdata)``. ``stderrdata`` will be ``None`` by default
(because ``stderr`` is redirected to ``stdout``).
:raise CheckbValueError: if you provide ``stdout`` parameter
:raise subprocess.CalledProcessError: if the command exits with non-zero exit code (helpful
attributes are provided, study its documentation)
"""
if 'stdout' in kwargs:
raise exc.CheckbValueError('stdout parameter not allowed, it will be overridden')
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, bufsize=bufsize,
universal_newlines=True, **kwargs)
output = []
for line in iter(proc.stdout.readline, ''):
print(line, end='')
output.append(line)
(stdoutdata, stderrdata) = proc.communicate()
assert proc.returncode is not None
assert not stdoutdata
if stderr == subprocess.STDOUT:
assert not stderrdata
stdoutdata = ''.join(output)
if proc.returncode:
raise subprocess.CalledProcessError(proc.returncode, cmd, stdoutdata)
return (stdoutdata, stderrdata)
def rpmformat(rpmstr, fmt='nvr', end_arch=False):
'''
Parse and convert an RPM package version string into a different format.
String identifiers: N - name, E - epoch, V - version, R - release, A -
architecture.
:param str rpmstr: string to be manipulated in a format of N(E)VR
(``foo-1.2-3.fc20`` or ``bar-4:1.2-3.fc20``) or N(E)VRA
(``foo-1.2-3.fc20.x86_64`` or ``bar-4:1.2-3.fc20.i686``)
:param str fmt: desired format of the string to be returned. Allowed options
are: ``nvr``, ``nevr``, ``nvra``, ``nevra``, ``n``, ``e``,
``v``, ``r``, ``a``. If arch is not present in ``rpmstr``
but requested in ``fmt``, ``noarch`` is used. Epoch is
provided only when specifically requested (e.g.
``fmt='nevr'``) **and** being non-zero; otherwise it's
supressed (the only exception is ``fmt='e'``, where you
receive ``0`` for zero epoch).
:param bool end_arch: set this to ``True`` if ``rpmstr`` ends with an
architecture identifier (``foo-1.2-3.fc20.x86_64``).
It's not possible to reliably distinguish that case
automatically.
:return: string based on the specified format, or integer if ``fmt='e'``
:raise CheckbValueError: if ``fmt`` value is not supported
'''
fmt = fmt.lower()
supported_formats = [
'nvr', 'nevr', 'nvra', 'nevra', 'n', 'e', 'v', 'r', 'a'
]
if fmt not in supported_formats:
raise exc.CheckbValueError("Format '%s' not in supported formats "
"(%s)" %
(fmt, ', '.join(supported_formats)))
# add arch if not present
if not end_arch:
rpmstr += '.noarch'
# split rpmstr
nevra = hawkey.split_nevra(rpmstr)
# return simple fmt
if len(fmt) == 1:
return {
'n': nevra.name,
'e': nevra.epoch,
'v': nevra.version,
'r': nevra.release,
'a': nevra.arch
}[fmt]
# return complex fmt
evr = nevra.evr()
# supress epoch if appropriate
if 'e' not in fmt or nevra.epoch == 0:
evr = evr[evr.find(':') + 1:] # remove 'epoch:' from the beginning
result = '%s-%s' % (nevra.name, evr)
# append arch if requested
if 'a' in fmt:
result += '.' + nevra.arch
return result
def build2update(self, builds, strict=False):
'''Find matching Bodhi updates for provided builds.
:param builds: builds to search for in N(E)VR format (``foo-1.2-3.fc20``
or ``foo-4:1.2-3.fc20``)
:type builds: iterable of str
:param bool strict: if ``False``, incomplete Bodhi updates are allowed.
If ``True``, every Bodhi update will be compared
with the set of provided builds. If there is an
Bodhi update which contains builds not provided in
``builds``, that update is marked as incomplete and
removed from the result - i.e. all builds from
``builds`` that were part of this incomplete update
are placed in the second dictionary of the result
tuple.
:return: a tuple of two dictionaries:
* The first dict provides mapping between ``builds`` and their
updates where no error occured.
``{build (string): Bodhi update (Munch)}``
* The second dict provides mapping between ``builds`` and their
updates where some error occured. The value is ``None`` if
the matching Bodhi update could not be found (the only
possible cause of failure if ``strict=False``). Or the value
is a Bodhi update that was incomplete (happens only if
``strict=True``).
``{build (string): Bodhi update (Munch) or None}``
* The set of keys in both these dictionaries correspond exactly
to ``builds``. For every build provided in ``builds`` you'll
get an answer in either the first or the second dictionary,
and there will be no extra builds that you haven't specified.
:raise CheckbValueError: if ``builds`` type is incorrect
'''
# validate input params
if not python_utils.iterable(builds, basestring):
raise exc.CheckbValueError(
"Param 'builds' must be an iterable of strings, and yours was: %s"
% type(builds))
updates = []
build2update = {}
failures = {}
# Bodhi works with NVR only, but we have to ensure we receive and return
# even NEVR format. So we need to convert internally.
builds_nvr = set(
[rpm_utils.rpmformat(build, 'nvr') for build in builds])
builds_queue = list(builds_nvr)
log.info('Querying Bodhi to map %d builds to their updates...',
len(builds))
# retrieve all update data
while builds_queue:
builds_chunk = builds_queue[:self._MULTICALL_REQUEST_SIZE]
builds_chunk = ' '.join(builds_chunk)
res = self.client.query(builds=builds_chunk)
updates.extend(res['updates'])
builds_queue = builds_queue[self._MULTICALL_REQUEST_SIZE:]
# don't query for builds which were already found
for update in res['updates']:
for build in update['builds']:
if build['nvr'] in builds_queue:
builds_queue.remove(build['nvr'])
log.info('Bodhi queries done: %d/%d',
len(builds_nvr) - len(builds_queue), len(builds_nvr))
# separate builds into OK and failures
for update in updates:
# all builds listed in the update
bodhi_builds = set([build['nvr'] for build in update['builds']])
# builds *not* provided in @param builds but part of the update (NVRs)
missing_builds = bodhi_builds.difference(builds_nvr)
# builds provided in @param builds and part of the update
matched_builds = [
build for build in builds
if rpm_utils.rpmformat(build, 'nvr') in bodhi_builds
]
# reject incomplete updates when strict
if missing_builds and strict:
for build in matched_builds:
failures[build] = update
continue
# otherwise the update is complete or we don't care
for build in matched_builds:
build2update[build] = update
# mark builds without any associated update as a failure
for build in builds:
if build not in build2update and build not in failures:
failures[build] = None
diff = set(builds).symmetric_difference(
set(build2update.keys()).union(set(failures.keys())))
assert not diff, "Returned NVRs different from input NVRs: %s" % diff
return (build2update, failures)
def process(self, params, arg_data):
if ('package' not in params and 'nvr' not in params) or 'path' not in params \
or 'target_dir' not in params:
detected_args = ', '.join(params.keys())
raise exc.CheckbDirectiveError(
"The distgit directive requires 'package' (or 'nvr') and 'path' and 'target_dir' arguments."
"Detected arguments: %s" % detected_args)
package = None
gitref = None
namespace = None
if 'nvr' in params:
nvr = params['nvr']
package = rpm_utils.rpmformat(nvr, fmt='n')
gitref = rpm_utils.get_dist_tag(nvr).replace('c', '')
rawhide_tag = yumrepoinfo.YumRepoInfo(resolve_baseurl=False).get(
'rawhide', 'tag')
if gitref == rawhide_tag:
gitref = 'master'
namespace = 'rpms'
# Assign defaults
package = params.get('package', package)
gitref = params.get('gitref', gitref or 'master')
namespace = params.get('namespace', namespace or 'rpms')
baseurl = params.get('baseurl', BASEURL)
target_dir = params['target_dir']
ignore_missing = params.get('ignore_missing', False)
if not python_utils.iterable(params['path']):
raise exc.CheckbValueError(
"Incorrect value type of the 'path' argument: "
"%s" % type(params['path']))
target_path = params['path']
output_data = {}
if 'localpath' in params:
if not python_utils.iterable(params['localpath']):
raise exc.CheckbValueError(
"Incorrect value type of the 'localpath' argument: "
"%s" % type(params['path']))
if not len(params['path']) == len(params['localpath']):
raise exc.CheckbValueError(
'path and localpath lists must be of the same '
'length.')
target_path = params['localpath']
format_fields = {
'package': package,
'gitref': gitref,
'namespace': namespace,
'baseurl': baseurl,
}
output_data['downloaded_files'] = []
for path, localpath in zip(params['path'], target_path):
localpath = os.path.join(target_dir, localpath)
file_utils.makedirs(os.path.dirname(localpath))
url = URL_FMT.format(path=path, **format_fields)
try:
output_data['downloaded_files'].append(
file_utils.download(url, '.', localpath))
except exc.CheckbRemoteError as e:
if e.errno == 404 and ignore_missing:
log.debug('File not found, ignoring: %s', url)
else:
raise e
return output_data