def cmpNEVR(nevr1, nevr2): '''Compare two RPM version identifiers in NEVR format. :param str nevr1: RPM identifier in N(E)VR format :param str nevr2: RPM identifier in N(E)VR format :return: ``-1``/``0``/``1`` if ``nevr1 < nevr2`` / ``nevr1 == nevr2`` / ``nevr1 > nevr2`` :rtype: int :raise CheckbValueError: if name in ``nevr1`` doesn't match name in ``nevr2`` ''' rpmver1 = hawkey.split_nevra(nevr1 + '.noarch') rpmver2 = hawkey.split_nevra(nevr2 + '.noarch') if rpmver1.name != rpmver2.name: raise exc.CheckbValueError("Name in nevr1 doesn't match name in " "nevr2: %s, %s" % (nevr1, nevr2)) # sack is needed for the comparison, because it can be influence the # comparison (for example epoch can be set to be ignored). A default empty # sack should match Fedora customs sack = hawkey.Sack() # we need evr_cmp to return int so we can use it as comparison function # in python's sorted return int(rpmver1.evr_cmp(rpmver2, sack))
def _collection_of(obj, collection_cls, item_type=None): '''The same as :func:`iterable` or :func:`sequence`, but the abstract collection class can be specified dynamically with ``collection_cls``. ''' if not isinstance(obj, collection_cls) or isinstance(obj, basestring): return False if item_type is not None and isinstance(obj, abc.Iterable): try: return all([isinstance(item, item_type) for item in obj]) except TypeError as e: raise exc.CheckbValueError( "'item_type' must be a type definition, not '%r': %s" % (item_type, e)) return True
def get_dist_tag(rpmstr): '''Parse disttag from an RPM package version string. :param str rpmstr: string to be manipulated in a format of N(E)VR (``foo-1.2-3.fc20`` or ``bar-4:1.2-3.fc20``) or N(E)VRA (``foo-1.2-3.fc20.x86_64`` or ``bar-4:1.2-3.fc20.i686``) :return: string containing dist tag (``fc20``) :raise CheckbValueError: if ``rpmstr`` does not contain dist tag ''' release = rpmformat(rpmstr, 'r') matches = re.findall(r'\.(fc\d{1,2})\.?', release) if not matches: raise exc.CheckbValueError('Could not parse disttag from %s' % rpmstr) # there might be e.g. fc22 in git commit hash as part of the release, # so just take the last match which should be disttag return matches[-1]
def popen_rt(cmd, stderr=subprocess.STDOUT, bufsize=1, **kwargs): """This is similar to :func:`subprocess.check_output`, but with real-time printing to console as well. It is useful for longer-running tasks for which you'd like to both capture the output and also see it printed in terminal continuously. Please note that by default both ``stdout`` and ``stderr`` are merged together. You can use ``stderr=subprocess.PIPE``, and it will be returned to you separated, but it won't be printed out to console (only ``stdout`` will). The parameters are the same as for :class:`subprocess.Popen`. You can't use ``stdout`` parameter, that one is hardcoded to :attr:`subprocess.PIPE`. :return: tuple of ``(stdoutdata, stderrdata)``. ``stderrdata`` will be ``None`` by default (because ``stderr`` is redirected to ``stdout``). :raise CheckbValueError: if you provide ``stdout`` parameter :raise subprocess.CalledProcessError: if the command exits with non-zero exit code (helpful attributes are provided, study its documentation) """ if 'stdout' in kwargs: raise exc.CheckbValueError('stdout parameter not allowed, it will be overridden') proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr, bufsize=bufsize, universal_newlines=True, **kwargs) output = [] for line in iter(proc.stdout.readline, ''): print(line, end='') output.append(line) (stdoutdata, stderrdata) = proc.communicate() assert proc.returncode is not None assert not stdoutdata if stderr == subprocess.STDOUT: assert not stderrdata stdoutdata = ''.join(output) if proc.returncode: raise subprocess.CalledProcessError(proc.returncode, cmd, stdoutdata) return (stdoutdata, stderrdata)
def rpmformat(rpmstr, fmt='nvr', end_arch=False): ''' Parse and convert an RPM package version string into a different format. String identifiers: N - name, E - epoch, V - version, R - release, A - architecture. :param str rpmstr: string to be manipulated in a format of N(E)VR (``foo-1.2-3.fc20`` or ``bar-4:1.2-3.fc20``) or N(E)VRA (``foo-1.2-3.fc20.x86_64`` or ``bar-4:1.2-3.fc20.i686``) :param str fmt: desired format of the string to be returned. Allowed options are: ``nvr``, ``nevr``, ``nvra``, ``nevra``, ``n``, ``e``, ``v``, ``r``, ``a``. If arch is not present in ``rpmstr`` but requested in ``fmt``, ``noarch`` is used. Epoch is provided only when specifically requested (e.g. ``fmt='nevr'``) **and** being non-zero; otherwise it's supressed (the only exception is ``fmt='e'``, where you receive ``0`` for zero epoch). :param bool end_arch: set this to ``True`` if ``rpmstr`` ends with an architecture identifier (``foo-1.2-3.fc20.x86_64``). It's not possible to reliably distinguish that case automatically. :return: string based on the specified format, or integer if ``fmt='e'`` :raise CheckbValueError: if ``fmt`` value is not supported ''' fmt = fmt.lower() supported_formats = [ 'nvr', 'nevr', 'nvra', 'nevra', 'n', 'e', 'v', 'r', 'a' ] if fmt not in supported_formats: raise exc.CheckbValueError("Format '%s' not in supported formats " "(%s)" % (fmt, ', '.join(supported_formats))) # add arch if not present if not end_arch: rpmstr += '.noarch' # split rpmstr nevra = hawkey.split_nevra(rpmstr) # return simple fmt if len(fmt) == 1: return { 'n': nevra.name, 'e': nevra.epoch, 'v': nevra.version, 'r': nevra.release, 'a': nevra.arch }[fmt] # return complex fmt evr = nevra.evr() # supress epoch if appropriate if 'e' not in fmt or nevra.epoch == 0: evr = evr[evr.find(':') + 1:] # remove 'epoch:' from the beginning result = '%s-%s' % (nevra.name, evr) # append arch if requested if 'a' in fmt: result += '.' + nevra.arch return result
def build2update(self, builds, strict=False): '''Find matching Bodhi updates for provided builds. :param builds: builds to search for in N(E)VR format (``foo-1.2-3.fc20`` or ``foo-4:1.2-3.fc20``) :type builds: iterable of str :param bool strict: if ``False``, incomplete Bodhi updates are allowed. If ``True``, every Bodhi update will be compared with the set of provided builds. If there is an Bodhi update which contains builds not provided in ``builds``, that update is marked as incomplete and removed from the result - i.e. all builds from ``builds`` that were part of this incomplete update are placed in the second dictionary of the result tuple. :return: a tuple of two dictionaries: * The first dict provides mapping between ``builds`` and their updates where no error occured. ``{build (string): Bodhi update (Munch)}`` * The second dict provides mapping between ``builds`` and their updates where some error occured. The value is ``None`` if the matching Bodhi update could not be found (the only possible cause of failure if ``strict=False``). Or the value is a Bodhi update that was incomplete (happens only if ``strict=True``). ``{build (string): Bodhi update (Munch) or None}`` * The set of keys in both these dictionaries correspond exactly to ``builds``. For every build provided in ``builds`` you'll get an answer in either the first or the second dictionary, and there will be no extra builds that you haven't specified. :raise CheckbValueError: if ``builds`` type is incorrect ''' # validate input params if not python_utils.iterable(builds, basestring): raise exc.CheckbValueError( "Param 'builds' must be an iterable of strings, and yours was: %s" % type(builds)) updates = [] build2update = {} failures = {} # Bodhi works with NVR only, but we have to ensure we receive and return # even NEVR format. So we need to convert internally. builds_nvr = set( [rpm_utils.rpmformat(build, 'nvr') for build in builds]) builds_queue = list(builds_nvr) log.info('Querying Bodhi to map %d builds to their updates...', len(builds)) # retrieve all update data while builds_queue: builds_chunk = builds_queue[:self._MULTICALL_REQUEST_SIZE] builds_chunk = ' '.join(builds_chunk) res = self.client.query(builds=builds_chunk) updates.extend(res['updates']) builds_queue = builds_queue[self._MULTICALL_REQUEST_SIZE:] # don't query for builds which were already found for update in res['updates']: for build in update['builds']: if build['nvr'] in builds_queue: builds_queue.remove(build['nvr']) log.info('Bodhi queries done: %d/%d', len(builds_nvr) - len(builds_queue), len(builds_nvr)) # separate builds into OK and failures for update in updates: # all builds listed in the update bodhi_builds = set([build['nvr'] for build in update['builds']]) # builds *not* provided in @param builds but part of the update (NVRs) missing_builds = bodhi_builds.difference(builds_nvr) # builds provided in @param builds and part of the update matched_builds = [ build for build in builds if rpm_utils.rpmformat(build, 'nvr') in bodhi_builds ] # reject incomplete updates when strict if missing_builds and strict: for build in matched_builds: failures[build] = update continue # otherwise the update is complete or we don't care for build in matched_builds: build2update[build] = update # mark builds without any associated update as a failure for build in builds: if build not in build2update and build not in failures: failures[build] = None diff = set(builds).symmetric_difference( set(build2update.keys()).union(set(failures.keys()))) assert not diff, "Returned NVRs different from input NVRs: %s" % diff return (build2update, failures)
def process(self, params, arg_data): if ('package' not in params and 'nvr' not in params) or 'path' not in params \ or 'target_dir' not in params: detected_args = ', '.join(params.keys()) raise exc.CheckbDirectiveError( "The distgit directive requires 'package' (or 'nvr') and 'path' and 'target_dir' arguments." "Detected arguments: %s" % detected_args) package = None gitref = None namespace = None if 'nvr' in params: nvr = params['nvr'] package = rpm_utils.rpmformat(nvr, fmt='n') gitref = rpm_utils.get_dist_tag(nvr).replace('c', '') rawhide_tag = yumrepoinfo.YumRepoInfo(resolve_baseurl=False).get( 'rawhide', 'tag') if gitref == rawhide_tag: gitref = 'master' namespace = 'rpms' # Assign defaults package = params.get('package', package) gitref = params.get('gitref', gitref or 'master') namespace = params.get('namespace', namespace or 'rpms') baseurl = params.get('baseurl', BASEURL) target_dir = params['target_dir'] ignore_missing = params.get('ignore_missing', False) if not python_utils.iterable(params['path']): raise exc.CheckbValueError( "Incorrect value type of the 'path' argument: " "%s" % type(params['path'])) target_path = params['path'] output_data = {} if 'localpath' in params: if not python_utils.iterable(params['localpath']): raise exc.CheckbValueError( "Incorrect value type of the 'localpath' argument: " "%s" % type(params['path'])) if not len(params['path']) == len(params['localpath']): raise exc.CheckbValueError( 'path and localpath lists must be of the same ' 'length.') target_path = params['localpath'] format_fields = { 'package': package, 'gitref': gitref, 'namespace': namespace, 'baseurl': baseurl, } output_data['downloaded_files'] = [] for path, localpath in zip(params['path'], target_path): localpath = os.path.join(target_dir, localpath) file_utils.makedirs(os.path.dirname(localpath)) url = URL_FMT.format(path=path, **format_fields) try: output_data['downloaded_files'].append( file_utils.download(url, '.', localpath)) except exc.CheckbRemoteError as e: if e.errno == 404 and ignore_missing: log.debug('File not found, ignoring: %s', url) else: raise e return output_data