def _anything_there(self, pip): wanted_pip = extract_requirement(pip) pip_there = self.helper.get_installed(wanted_pip.name) if not pip_there: # Nothing installed return None if wanted_pip.version is not None: # Check if version wanted will work with whats installed if str(wanted_pip.version) not in pip_there: is_upgrading = False for o in ['-U', '--upgrade']: if o in pip.get('options', []): is_upgrading = True if is_upgrading and (wanted_pip.name not in self.upgraded): # Upgrade should hopefully get that package to the right version.... LOG.warn( "Upgrade is occuring for %s, even though %s is installed.", wanted_pip, pip_there) # Mark it so that we don't keep on flip-flopping on upgrading this # package (ie install new, install old, install new....) self.upgraded[wanted_pip.name] = wanted_pip return None else: msg = ("Pip %s is already installed" " and it is not compatible with desired" " pip %s") msg = msg % (pip_there, wanted_pip) raise excp.DependencyException(msg) return pip_there
def _verify_pip_requires(self): all_pips = self.pip_requires for details in all_pips: req = details['requirement'] needed_by = details['needed_by'] pkg_info = details['package'] if not pkg_info: raise excp.DependencyException(("Pip dependency '%s' needed by '%s' is not translatable to a listed" " (from this or previously activated components) pip package" ' or a pip->package mapping!') % (req, needed_by))
def _verify_pip_requires(self): all_pips = [] for fn in self.requires_files: all_pips.extend(self._extract_pip_requires(fn)) for (requirement, (pkg_info, _from_pip)) in all_pips: if not pkg_info: raise excp.DependencyException(( "Pip dependency '%s' is not translatable to a listed" " (from this or previously activated components) pip package" ' or a pip->package mapping!') % (requirement))
def _gather_pips_to_install(self, requires_files, extra_pips=None): """Analyze requires_files and extra_pips. Updates `self.forced_pips` and `self.pips_to_install`. Writes requirements to `self.gathered_requires_filename`. """ ignore_pips = set(self.python_names) ignore_pips.update(self.ignore_pips) forced_pips = set() forced_distro_pips = self.distro.get_dependency_config("forced_pips", quiet=True) if forced_distro_pips: forced_pips.update(forced_distro_pips) compatibles, incompatibles = self.multipip.resolve( extra_pips, requires_files, ignore_pips, forced_pips) self.pips_to_install = compatibles sh.write_file(self.gathered_requires_filename, "\n".join(self.pips_to_install)) pip_requirements, raw_requirements = pip_helper.read_requirement_files( [self.gathered_requires_filename]) pips_to_install = sorted(raw_requirements, cmp=sort_req) utils.log_iterable(pips_to_install, logger=LOG, header="Full known python dependency list") for (name, lines) in incompatibles.items(): LOG.warn("Incompatible requirements found for %s", colorizer.quote(name, quote_color='red')) for line in lines: LOG.warn(line) if not self.pips_to_install: LOG.error("No valid dependencies found. Something went wrong.") raise exc.DependencyException("No valid dependencies found") # Translate those that we altered requirements for into a set of forced # requirements file (and associated list). self.forced_pips = [] forced_pip_keys = [] for req in [ pip_helper.extract_requirement(line) for line in self.pips_to_install ]: if req.key in incompatibles and req.key not in forced_pip_keys: self.forced_pips.append(req) forced_pip_keys.append(req.key) self.forced_pips = sorted(self.forced_pips, cmp=sort_req) forced_pips = [str(req) for req in self.forced_pips] utils.log_iterable(forced_pips, logger=LOG, header="Automatically forced python dependencies") sh.write_file(self.forced_requires_filename, "\n".join(forced_pips))
def _anything_there(self, pip): wanted_pip = extract_requirement(pip) pip_there = self.helper.get_installed(wanted_pip.name) if not pip_there: # Nothing installed return None if wanted_pip.version is not None: # Check if version wanted will work with whats installed if str(wanted_pip.version) not in pip_there: msg = ("Pip %s is already installed" " and it is not compatible with desired" " pip %s") msg = msg % (pip_there, wanted_pip) raise excp.DependencyException(msg) return pip_there
def validate_requirement(filename, source_req): install_egg = None for egg_info in own_eggs: if egg_info['name'] == source_req.key: install_egg = egg_info break if not install_egg: return # Ensure what we are about to install/create will actually work # with the desired version. If it is not compatible then we should # abort and someone should update the tag/branch in the origin # file (or fix it via some other mechanism). if install_egg['version'] not in source_req: msg = ("Can not satisfy '%s' with '%s', version" " conflict found in %s") raise exc.DependencyException( msg % (source_req, install_egg['req'], filename))
def _all_rpm_names(self): # This file should have all the requirements (including test ones) # that we need to install (and which should have been built as rpms # in the previous build stages). gathered_requires = sh.load_file( self.gathered_requires_filename).splitlines() gathered_requires = [ line.strip() for line in gathered_requires if line.strip() ] req_names = [] reqs = [] for line in gathered_requires: req = pip_helper.extract_requirement(line) if req.key in req_names: continue req_names.append(req.key) reqs.append(req) rpm_names = self._convert_names_python2rpm(req_names) # Ensure we select the right versions that is required and not a # version that doesn't match the requirements. desired_rpms = [] desired_rpm_names = set() desired_rpms_formatted = [] def format_name(rpm_name, py_req): full_name = str(rpm_name).strip() if py_req is not None: full_name += ",%s" % (py_req) return full_name def capture_rpm(rpm_name, py_req): if rpm_name in desired_rpm_names or not rpm_name: return desired_rpms_formatted.append(format_name(rpm_name, py_req)) desired_rpms.append((rpm_name, py_req)) desired_rpm_names.add(rpm_name) for (rpm_name, req) in zip(rpm_names, reqs): capture_rpm(rpm_name, req) for inst in self.instances: if sh.isdir(inst.get_option("app_dir")): req = None rpm_name = None try: (rpm_name, _tpl) = self._get_template_and_rpm_name(inst) req = inst.egg_info['req'] except AttributeError: pass capture_rpm(rpm_name, req) for rpm_name in inst.package_names(): capture_rpm(rpm_name, None) for rpm_name in self.requirements["requires"]: capture_rpm(rpm_name, None) cmd = [self.yumfind_executable, '-j'] desired_rpms_formatted = sorted(desired_rpms_formatted) for p in desired_rpms_formatted: cmd.extend(['-p', p]) header = "Validating %s required packages are still available" % ( len(desired_rpms)) utils.log_iterable(desired_rpms_formatted, header=header, logger=LOG) rpms_located = [] rpm_names_located = set() for matched in sh.execute(cmd)[0].splitlines(): matched = matched.strip() if matched: pkg = json.loads(matched) if isinstance(pkg, dict): rpm_names_located.add(pkg['name']) rpms_located.append(pkg) rpm_names_missing = desired_rpm_names - rpm_names_located if rpm_names_missing: # Include the python version required information (if applicable) missing_formatted = [] for n in sorted(rpm_names_missing): source_found = False for (n2, py_req) in desired_rpms: if n2 == n: missing_formatted.append(format_name(n2, py_req)) source_found = True break if not source_found: missing_formatted.append(format_name(n, None)) msg = "Could not find available rpm packages: %s" msg = msg % (", ".join(missing_formatted)) raise excp.DependencyException(msg) LOG.info("All %s required packages are still available!", len(desired_rpms)) desired_rpms = [] for pkg in rpms_located: LOG.debug("Found %s", pkg) desired_rpms.append("%s,%s" % (pkg['name'], pkg['version'])) return list(sorted(desired_rpms))
def _gather_pips_to_install(self, requires_files, extra_pips=None): """Analyze requires_files and extra_pips. Updates `self.forced_packages` and `self.pips_to_install`. Writes requirements to `self.gathered_requires_filename`. """ extra_pips = extra_pips or [] cmdline = [ self.multipip_executable, "--skip-requirements-regex", "python.*client", "--pip", self.pip_executable ] cmdline = cmdline + extra_pips + ["-r"] + requires_files cmdline.extend(["--ignore-package"]) cmdline.extend(OPENSTACK_PACKAGES) cmdline.extend(SKIP_PACKAGE_NAMES) cmdline.extend(self.python_names) stdout, stderr = sh.execute(cmdline, check_exit_code=False) self.pips_to_install = list(utils.splitlines_not_empty(stdout)) sh.write_file(self.gathered_requires_filename, "\n".join(self.pips_to_install)) utils.log_iterable(sorted(self.pips_to_install), logger=LOG, header="Full known python dependency list") incompatibles = collections.defaultdict(list) if stderr: current_name = '' for line in stderr.strip().splitlines(): if line.endswith(": incompatible requirements"): current_name = line.split(":", 1)[0].lower().strip() if current_name not in incompatibles: incompatibles[current_name] = [] else: incompatibles[current_name].append(line) for (name, lines) in incompatibles.items(): if not name: continue LOG.warn("Incompatible requirements found for %s", colorizer.quote(name, quote_color='red')) for line in lines: LOG.warn(line) if not self.pips_to_install: LOG.error("No dependencies for OpenStack found." "Something went wrong. Please check:") LOG.error("'%s'" % "' '".join(cmdline)) raise exc.DependencyException( "No dependencies for OpenStack found") # Translate those that we altered requirements for into a set of forced # requirements file (and associated list). self.forced_packages = [] for req in [ pip_helper.extract_requirement(line) for line in self.pips_to_install ]: if req.key in incompatibles: self.forced_packages.append(req) sh.write_file(self.forced_requires_filename, "\n".join([str(req) for req in self.forced_packages]))