Beispiel #1
0
 def _fetch_epoch_mapping(self):
     epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True)
     if not epoch_map:
         epoch_map = {}
     built_epochs = {}
     for name in self.python_names:
         if name in epoch_map:
             built_epochs[name] = epoch_map.pop(name)
         else:
             built_epochs[name] = self.OPENSTACK_EPOCH
     # Exclude names from the epoch map that we never downloaded in the
     # first place (since these are not useful and should not be set in
     # the first place).
     try:
         raw_downloaded = sh.load_file(self.build_requires_filename)
         downloaded_reqs = pip_helper.parse_requirements(raw_downloaded)
     except IOError as e:
         if e.errno != errno.ENOENT:
             raise
     else:
         downloaded_names = set([req.key for req in downloaded_reqs])
         tmp_epoch_map = {}
         for (name, epoch) in six.iteritems(epoch_map):
             if name.lower() in downloaded_names:
                 tmp_epoch_map[name] = epoch
             else:
                 LOG.debug("Discarding %s:%s from the epoch mapping since"
                           " it was not part of the downloaded build"
                           " requirements", name, epoch)
         epoch_map = tmp_epoch_map
     epoch_map.update(built_epochs)
     return epoch_map
Beispiel #2
0
 def _fetch_epoch_mapping(self):
     epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True)
     if not epoch_map:
         epoch_map = {}
     built_epochs = {}
     for name in self.python_names:
         if name in epoch_map:
             built_epochs[name] = epoch_map.pop(name)
         else:
             built_epochs[name] = self.OPENSTACK_EPOCH
     # Exclude names from the epoch map that we never downloaded in the
     # first place (since these are not useful and should not be set in
     # the first place).
     try:
         raw_downloaded = sh.load_file(self.build_requires_filename)
         downloaded_reqs = pip_helper.parse_requirements(raw_downloaded)
     except IOError as e:
         if e.errno != errno.ENOENT:
             raise
     else:
         downloaded_names = set([req.key for req in downloaded_reqs])
         tmp_epoch_map = {}
         for (name, epoch) in six.iteritems(epoch_map):
             if name.lower() in downloaded_names:
                 tmp_epoch_map[name] = epoch
             else:
                 LOG.debug(
                     "Discarding %s:%s from the epoch mapping since"
                     " it was not part of the downloaded build"
                     " requirements", name, epoch)
         epoch_map = tmp_epoch_map
     epoch_map.update(built_epochs)
     return epoch_map
 def _extract_pip_requires(self, fn):
     if not sh.isfile(fn):
         return []
     LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn))
     pips_needed = pip_helper.parse_requirements(sh.load_file(fn))
     matchings = []
     for req in pips_needed:
         (pkg_info, from_pip) = self._match_pip_requires(req)
         matchings.append({"requirement": req, "package": pkg_info, "from_pip": from_pip, "needed_by": fn})
     return matchings
Beispiel #4
0
 def _extract_pip_requires(self, fn):
     if not sh.isfile(fn):
         return []
     LOG.debug("Resolving dependencies from %s.", colorizer.quote(fn))
     pips_needed = pip_helper.parse_requirements(sh.load_file(fn))
     matchings = []
     for req in pips_needed:
         (pkg_info, from_pip) = self._match_pip_requires(req)
         matchings.append({
             'requirement': req,
             'package': pkg_info,
             'from_pip': from_pip,
             'needed_by': fn,
         })
     return matchings
def main():
    if len(sys.argv) < 3:
        print("%s distro_yaml root_dir ..." % sys.argv[0])
        return 1
    root_dirs = sys.argv[2:]
    yaml_fn = sh.abspth(sys.argv[1])

    requires_files = []
    for d in root_dirs:
        all_contents = sh.listdir(d, recursive=True, files_only=True)
        requires_files = [sh.abspth(f) for f in all_contents
                          if re.search(r"(test|pip)[-]requires$", f, re.I)]

    requires_files = sorted(list(set(requires_files)))
    requirements = []
    source_requirements = {}
    for fn in requires_files:
        source_requirements[fn] = []
        for req in pip_helper.parse_requirements(sh.load_file(fn)):
            requirements.append(req.key.lower().strip())
            source_requirements[fn].append(req.key.lower().strip())

    print("Comparing pips/pip2pkgs in %s to those found in %s" % (yaml_fn, root_dirs))
    for fn in sorted(requires_files):
        print(" + " + str(fn))

    requirements = set(requirements)
    print("All known requirements:")
    for r in sorted(requirements):
        print("+ " + str(r))

    distro_yaml = utils.load_yaml(yaml_fn)
    components = distro_yaml.get('components', {})
    all_known_names = []
    components_pips = {}
    for (c, details) in components.items():
        components_pips[c] = []
        pip2pkgs = details.get('pip_to_package', [])
        pips = details.get('pips', [])
        known_names = []
        for item in pip2pkgs:
            known_names.append(item['name'].lower().strip())
        for item in pips:
            known_names.append(item['name'].lower().strip())
        components_pips[c].extend(known_names)
        all_known_names.extend(known_names)

    all_known_names = sorted(list(set(all_known_names)))
    not_needed = []
    for n in all_known_names:
        if n not in requirements:
            not_needed.append(n)
    if not_needed:
        print("The following distro yaml mappings may not be needed:")
        for n in sorted(not_needed):
            msg = "  + %s (" % (n)
            # Find which components said they need this...
            for (c, known_names) in components_pips.items():
                if n in known_names:
                    msg += c + ","
            msg += ")"
            print(msg)
    not_found = []
    for n in requirements:
        name = n.lower().strip()
        if name not in all_known_names:
            not_found.append(name)
    not_found = sorted(list(set(not_found)))
    if not_found:
        print("The following distro yaml mappings may be required but were not found:")
        for n in sorted(not_found):
            msg = "  + %s" % (n)
            msg += " ("
            # Find which file/s said they need this...
            for (fn, reqs) in source_requirements.items():
                matched = False
                for r in reqs:
                    if r.lower().strip() == name:
                        matched = True
                if matched:
                    msg += fn + ","
            msg += ")"
            print(msg)
    return len(not_found) + len(not_needed)
Beispiel #6
0
def main():
    if len(sys.argv) < 3:
        print("%s distro_yaml root_dir ..." % sys.argv[0])
        return 1
    root_dirs = sys.argv[2:]
    yaml_fn = sh.abspth(sys.argv[1])

    requires_files = []
    for d in root_dirs:
        all_contents = sh.listdir(d, recursive=True, files_only=True)
        requires_files = [
            sh.abspth(f) for f in all_contents
            if re.search(r"(test|pip)[-]requires$", f, re.I)
        ]

    requires_files = sorted(list(set(requires_files)))
    requirements = []
    source_requirements = {}
    for fn in requires_files:
        source_requirements[fn] = []
        for req in pip_helper.parse_requirements(sh.load_file(fn)):
            requirements.append(req.key.lower().strip())
            source_requirements[fn].append(req.key.lower().strip())

    print("Comparing pips/pip2pkgs in %s to those found in %s" %
          (yaml_fn, root_dirs))
    for fn in sorted(requires_files):
        print(" + " + str(fn))

    requirements = set(requirements)
    print("All known requirements:")
    for r in sorted(requirements):
        print("+ " + str(r))

    distro_yaml = utils.load_yaml(yaml_fn)
    components = distro_yaml.get('components', {})
    all_known_names = []
    components_pips = {}
    for (c, details) in components.items():
        components_pips[c] = []
        pip2pkgs = details.get('pip_to_package', [])
        pips = details.get('pips', [])
        known_names = []
        for item in pip2pkgs:
            known_names.append(item['name'].lower().strip())
        for item in pips:
            known_names.append(item['name'].lower().strip())
        components_pips[c].extend(known_names)
        all_known_names.extend(known_names)

    all_known_names = sorted(list(set(all_known_names)))
    not_needed = []
    for n in all_known_names:
        if n not in requirements:
            not_needed.append(n)
    if not_needed:
        print("The following distro yaml mappings may not be needed:")
        for n in sorted(not_needed):
            msg = "  + %s (" % (n)
            # Find which components said they need this...
            for (c, known_names) in components_pips.items():
                if n in known_names:
                    msg += c + ","
            msg += ")"
            print(msg)
    not_found = []
    for n in requirements:
        name = n.lower().strip()
        if name not in all_known_names:
            not_found.append(name)
    not_found = sorted(list(set(not_found)))
    if not_found:
        print(
            "The following distro yaml mappings may be required but were not found:"
        )
        for n in sorted(not_found):
            msg = "  + %s" % (n)
            msg += " ("
            # Find which file/s said they need this...
            for (fn, reqs) in source_requirements.items():
                matched = False
                for r in reqs:
                    if r.lower().strip() == name:
                        matched = True
                if matched:
                    msg += fn + ","
            msg += ")"
            print(msg)
    return len(not_found) + len(not_needed)
Beispiel #7
0
 def _fetch_epoch_mapping(self):
     epoch_map = self.distro.get_dependency_config("epoch_map", quiet=True)
     if not epoch_map:
         epoch_map = {}
     epoch_skips = self.distro.get_dependency_config("epoch_skips",
                                                     quiet=True)
     if not epoch_skips:
         epoch_skips = _DEFAULT_SKIP_EPOCHS
     if not isinstance(epoch_skips, (list, tuple)):
         epoch_skips = [i.strip() for i in epoch_skips.split(",")]
     built_epochs = {}
     for name in self.python_names:
         if name in epoch_map:
             built_epochs[name] = str(epoch_map.pop(name))
         else:
             built_epochs[name] = str(self.OPENSTACK_EPOCH)
     # Ensure epochs set by a yum searching (that are not in the list of
     # epochs to provide) are correctly set when building dependent
     # packages...
     keep_names = set()
     try:
         yum_satisfies = sh.load_file(self.yum_satisfies_filename)
     except IOError as e:
         if e.errno != errno.ENOENT:
             raise
     else:
         for line in yum_satisfies.splitlines():
             raw_req_rpm = utils.parse_json(line)
             req = pip_helper.extract_requirement(raw_req_rpm['requirement'])
             if req.key in epoch_map:
                 LOG.debug("Ensuring manually set epoch is retained for"
                           " requirement '%s' with epoch %s", req,
                           epoch_map[req.key])
                 keep_names.add(req.key)
             else:
                 rpm_info = raw_req_rpm['rpm']
                 rpm_epoch = rpm_info.get('epoch')
                 if rpm_epoch and str(rpm_epoch) not in epoch_skips:
                     LOG.debug("Adding in yum satisfiable package %s for"
                               " requirement '%s' with epoch %s from repo %s",
                               rpm_info['name'], req, rpm_epoch, rpm_info['repo'])
                     keep_names.add(req.key)
                     epoch_map[req.key] = str(rpm_epoch)
     # Exclude names from the epoch map that we never downloaded in the
     # first place or that we did not just set automatically (since these
     # are not useful and should not be set in the first place).
     try:
         raw_downloaded = sh.load_file(self.build_requires_filename)
         downloaded_reqs = pip_helper.parse_requirements(raw_downloaded)
     except IOError as e:
         if e.errno != errno.ENOENT:
             raise
     else:
         downloaded_names = set([req.key for req in downloaded_reqs])
         tmp_epoch_map = {}
         for (name, epoch) in six.iteritems(epoch_map):
             name = name.lower()
             if name in downloaded_names or name in keep_names:
                 tmp_epoch_map[name] = str(epoch)
             else:
                 LOG.debug("Discarding %s:%s from the epoch mapping since"
                           " it was not part of the downloaded (or automatically"
                           " included) build requirements", name, epoch)
         epoch_map = tmp_epoch_map
     epoch_map.update(built_epochs)
     return epoch_map