def get_requirement_info_by_name(self, req, requirements_by_name=dict()): """ Given a requirement, it provides its valid information to be included in the final file """ if self.use_uniform_requirements: # If it is a repo and not a package, this will be True if req.link: requirement_name = parse_requirement(_get_valid_link_req(req.req)).name else: requirement_name = parse_requirement(str(req.req)).name # If the requirement is not in the dict, it is because it was not needed as a dependency in the original # set containing all requirements if requirement_name in requirements_by_name.keys(): # In case we want to show the source for inherited dependencies if self.show_comes_from_info and type(req.comes_from) == InstallRequirement: return "{}\t\t\t# From: {}".format(requirements_by_name[requirement_name].requirement, str(req.comes_from.req)) else: return requirements_by_name[requirement_name].requirement else: return None else: # If not using uniform versions, we just need to provide the information based on wether it is # a repository or a package if self.show_comes_from_info and type(req.comes_from) == InstallRequirement: return "{}\t\t\t# From: {}".format(format_requirement(req) if not req.link else str(req.req), str(req.comes_from.req)) else: return format_requirement(req) if not req.link else str(req.req)
def parse_requires_data(data): """Create a list of dependencies from a requires.txt file. *data*: the contents of a setuptools-produced requires.txt file. """ reqs = [] lines = data.splitlines() for line in lines: line = line.strip() if line.startswith('['): logger.warning('Unexpected line: quitting requirement scan: %r', line) break r = parse_requirement(line) if not r: logger.warning('Not recognised as a requirement: %r', line) continue if r.extras: logger.warning('extra requirements in requires.txt are ' 'not supported') if not r.constraints: reqs.append(r.name) else: cons = ', '.join('%s%s' % c for c in r.constraints) reqs.append('%s (%s)' % (r.name, cons)) return reqs
def matches_requirement(self, req): """ Say if this instance matches (fulfills) a requirement. :param req: The requirement to match. :rtype req: str :return: True if it matches, else False. """ # Requirement may contain extras - parse to lose those # from what's passed to the matcher r = parse_requirement(req) scheme = get_scheme(self.metadata.scheme) try: matcher = scheme.matcher(r.requirement) except UnsupportedVersionError: # XXX compat-mode if cannot read the version logger.warning('could not read version %r - using name only', req) name = req.split()[0] matcher = scheme.matcher(name) name = matcher.key # case-insensitive result = False for p in self.provides: p_name, p_ver = parse_name_and_version(p) if p_name != name: continue try: result = matcher.match(p_ver) break except UnsupportedVersionError: pass return result
def set_theme(self, theme): """ Sets the theme. Prefer using the .theme property. :param theme: Requirement for theme """ self._theme = parse_requirement(theme)
def _get_valid_link_req(req): """ Formats the repo based dependencies, which can be dirty in case of collision between repo based and package based declarations (for similar packages) """ tokens = str(req).split("@") req_name = parse_requirement(tokens[0]) return req_name.name + "@ " + "@".join(tokens[1:])
def __add_requirements(self, reqs, extra=None): if len(reqs): if extra not in self._requirements: self._requirements[extra] = {} for req in reqs: req = parse_requirement(req) if req.name in self._requirements[extra]: raise ValueError("Duplicate requirement for {}.".format(req.name)) self._requirements[extra][req.name] = req
def check_duplicate_dependencies_nonuniform(self, extra, resolver): """ Checks there are not duplicate dependencies, when use_uniform_requirements==False """ requirements_by_name = {} for req in resolver.resolve(max_rounds=10): requirements_by_name[parse_requirement(str(req.req)).name] = SimpleNamespace( requirement=format_requirement(req).strip().replace(" ", ""), url=req.link ) self._check_duplicate_dependencies_by_extra(extra, requirements_by_name)
def locate(self, requirement, prereleases=False): """ Find the most recent distribution which matches the given requirement. :param requirement: A requirement of the form 'foo (1.0)' or perhaps 'foo (>= 1.0, < 2.0, != 1.3)' :param prereleases: If ``True``, allow pre-release versions to be located. Otherwise, pre-release versions are not returned. :return: A :class:`Distribution` instance, or ``None`` if no such distribution could be located. """ result = None r = parse_requirement(requirement) if r is None: raise DistlibException('Not a valid requirement: %r' % requirement) scheme = get_scheme(self.scheme) self.matcher = matcher = scheme.matcher(r.requirement) logger.debug('matcher: %s (%s)', matcher, type(matcher).__name__) versions = self.get_project(r.name) if versions: # sometimes, versions are invalid slist = [] vcls = matcher.version_class for k in versions: try: if not matcher.match(k): logger.debug('%s did not match %r', matcher, k) else: if prereleases or not vcls(k).is_prerelease: slist.append(k) else: logger.debug( 'skipping pre-release ' 'version %s of %s', k, matcher.name) except Exception: logger.warning('error matching %s with %r', matcher, k) pass # slist.append(k) if len(slist) > 1: slist = sorted(slist, key=scheme.key) if slist: logger.debug('sorted list: %s', slist) result = versions[slist[-1]] if result and r.extras: result.extras = r.extras self.matcher = None return result
def locate(self, requirement, prereleases=False): """ Find the most recent distribution which matches the given requirement. :param requirement: A requirement of the form 'foo (1.0)' or perhaps 'foo (>= 1.0, < 2.0, != 1.3)' :param prereleases: If ``True``, allow pre-release versions to be located. Otherwise, pre-release versions are not returned. :return: A :class:`Distribution` instance, or ``None`` if no such distribution could be located. """ result = None r = parse_requirement(requirement) if r is None: raise DistlibException('Not a valid requirement: %r' % requirement) scheme = get_scheme(self.scheme) self.matcher = matcher = scheme.matcher(r.requirement) logger.debug('matcher: %s (%s)', matcher, type(matcher).__name__) versions = self.get_project(r.name) if versions: # sometimes, versions are invalid slist = [] vcls = matcher.version_class for k in versions: try: if not matcher.match(k): logger.debug('%s did not match %r', matcher, k) else: if prereleases or not vcls(k).is_prerelease: slist.append(k) else: logger.debug('skipping pre-release ' 'version %s of %s', k, matcher.name) except Exception: logger.warning('error matching %s with %r', matcher, k) pass # slist.append(k) if len(slist) > 1: slist = sorted(slist, key=scheme.key) if slist: logger.debug('sorted list: %s', slist) result = versions[slist[-1]] if result and r.extras: result.extras = r.extras self.matcher = None return result
def can_be_updated(cls, dist, latest_version): """Determine whether package can be updated or not.""" scheme = get_scheme('default') name = dist.project_name dependants = cls.get_dependants(name) for dependant in dependants: requires = dependant.requires() for requirement in cls.get_requirement(name, requires): req = parse_requirement(requirement) # Ignore error if version in requirement spec can't be parsed try: matcher = scheme.matcher(req.requirement) except UnsupportedVersionError: continue if not matcher.match(str(latest_version)): return False return True
def set_theme(self, theme): self._theme = parse_requirement(theme)
def get_python_rec(anchor_file): package_type, sp_reference, pydist = get_pydist(anchor_file) if pydist is None: return None # x.provides => [u'skdata (0.0.4)'] # x.run_requires => set([u'joblib', u'scikit-learn', u'lockfile', u'numpy', u'nose (>=1.0)']) # NOQA # >>> list(x.list_installed_files()) => [(u'skdata/__init__.py', u'sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU', u'0'), (u'skdata/base.py', u'sha256=04MW02dky5T4nZb6Q0M351aRbAwLxd8voCK3nrAU-g0', u'5019'), (u'skdata/brodatz.py', u'sha256=NIPWLawJ59Fr037r0oT_gHe46WCo3UivuQ-cwxRU3ow', u'8492'), (u'skdata/caltech.py', u'sha256=cIfyMMRYggZ3Jkgc15tsYi_ZsZ7NpRqWh7mZ8bl6Fo0', u'8047'), (u'skdata/data_home.py', u'sha256=o5ChOI4v3Jd16JM3qWZlhrs5q-g_0yKa5-Oq44HC_K4', u'1297'), (u'skdata/diabetes.py', u'sha256=ny5Ihpc_eiIRYgzFn3Lm81fV0SZ1nyZQnqEmwb2PrS0', u'995'), (u'skdata/digits.py', u'sha256=DipeWAb3APpjXfmKmSumkfEFzuBW8XJ0 # NOQA # TODO: normalize names against '.', '-', '_' # TODO: ensure that this dist is *actually* the dist that matches conda-meta if package_type == PackageType.SHADOW_PYTHON_EGG_INFO_FILE: paths_data = None elif package_type == PackageType.SHADOW_PYTHON_DIST_INFO: _paths_data = [] for _path, _hash, _size in pydist.list_installed_files(): if _hash: assert _hash.startswith('sha256='), (anchor_file, _hash) sha256 = _hash[7:] else: sha256 = None _size = int(_size) if _size else None _paths_data.append(PathDataV1( _path=_path, path_type=PathType.hardlink, sha256=sha256, size_in_bytes=_size )) paths_data = PathsData(paths_version=1, paths=_paths_data) elif package_type == PackageType.SHADOW_PYTHON_EGG_INFO_DIR: _paths_data = [] # TODO: Don't use list_installed_files() here. Read SOURCES.txt directly. for _path, _, _ in pydist.list_installed_files(): _paths_data.append(PathData( _path=_path, path_type=PathType.hardlink, )) paths_data = PathsData(paths_version=1, paths=_paths_data) else: raise NotImplementedError() # TODO: need to add entry points, "exports," and other files that might not be in RECORD # NOQA depends = tuple( req.name for req in # vars(req) => {'source': u'nose (>=1.0)', 'requirement': u'nose (>= 1.0)', 'extras': None, 'name': u'nose', 'url': None, 'constraints': [(u'>=', u'1.0')]} # NOQA (parse_requirement(r) for r in pydist.run_requires) ) # TODO: need to add python (with version?) to deps python_rec = PrefixRecord( package_type=package_type, namespace='python', name=pydist.name.lower(), version=pydist.version, channel=Channel('pypi'), subdir='pypi', fn=sp_reference, build='pypi_0', build_number=0, paths_data=paths_data, depends=depends, ) return python_rec
def get_python_rec(anchor_file): package_type, sp_reference, pydist = get_pydist(anchor_file) if pydist is None: return None # x.provides => [u'skdata (0.0.4)'] # x.run_requires => set([u'joblib', u'scikit-learn', u'lockfile', u'numpy', u'nose (>=1.0)']) # NOQA # >>> list(x.list_installed_files()) => [(u'skdata/__init__.py', u'sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU', u'0'), (u'skdata/base.py', u'sha256=04MW02dky5T4nZb6Q0M351aRbAwLxd8voCK3nrAU-g0', u'5019'), (u'skdata/brodatz.py', u'sha256=NIPWLawJ59Fr037r0oT_gHe46WCo3UivuQ-cwxRU3ow', u'8492'), (u'skdata/caltech.py', u'sha256=cIfyMMRYggZ3Jkgc15tsYi_ZsZ7NpRqWh7mZ8bl6Fo0', u'8047'), (u'skdata/data_home.py', u'sha256=o5ChOI4v3Jd16JM3qWZlhrs5q-g_0yKa5-Oq44HC_K4', u'1297'), (u'skdata/diabetes.py', u'sha256=ny5Ihpc_eiIRYgzFn3Lm81fV0SZ1nyZQnqEmwb2PrS0', u'995'), (u'skdata/digits.py', u'sha256=DipeWAb3APpjXfmKmSumkfEFzuBW8XJ0 # NOQA # TODO: normalize names against '.', '-', '_' # TODO: ensure that this dist is *actually* the dist that matches conda-meta if package_type == PackageType.SHADOW_PYTHON_EGG_INFO_FILE: paths_data = None elif package_type == PackageType.SHADOW_PYTHON_DIST_INFO: _paths_data = [] for _path, _hash, _size in pydist.list_installed_files(): if _hash: assert _hash.startswith('sha256='), (anchor_file, _hash) sha256 = _hash[7:] else: sha256 = None _size = int(_size) if _size else None _paths_data.append(PathDataV1( _path=_path, path_type=PathType.hardlink, sha256=sha256, size_in_bytes=_size )) paths_data = PathsData(paths_version=1, paths=_paths_data) elif package_type == PackageType.SHADOW_PYTHON_EGG_INFO_DIR: _paths_data = [] # TODO: Don't use list_installed_files() here. Read SOURCES.txt directly. for _path, _, _ in pydist.list_installed_files(): _paths_data.append(PathData( _path=_path, path_type=PathType.hardlink, )) paths_data = PathsData(paths_version=1, paths=_paths_data) else: raise NotImplementedError() # TODO: need to add entry points, "exports," and other files that might not be in RECORD # NOQA depends = tuple( req.name for req in # vars(req) => {'source': u'nose (>=1.0)', 'requirement': u'nose (>= 1.0)', 'extras': None, 'name': u'nose', 'url': None, 'constraints': [(u'>=', u'1.0')]} # NOQA (parse_requirement(r) for r in pydist.run_requires) ) # TODO: need to add python (with version?) to deps python_rec = PrefixRecord( package_type=package_type, name=pydist.name.lower(), version=pydist.version, channel=Channel('pypi'), subdir='pypi', fn=sp_reference, build='pypi_0', build_number=0, paths_data=paths_data, depends=depends, ) return python_rec
def on_end(self, event): # Our config object python_config = event.config["python"] # Pip / PyPI repository = PyPIRepository([], cache_dir=CACHE_DIR) # We just need to construct this structure if use_uniform_requirements == True requirements_by_name = {} if python_config.use_uniform_requirements: tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) for extra in itertools.chain((None,), python_config.get_extras()): tmpfile.write("\n".join(python_config.get_requirements(extra=extra)) + "\n") tmpfile.flush() constraints = list( parse_requirements( tmpfile.name, finder=repository.finder, session=repository.session, options=repository.options ) ) # This resolver is able to evaluate ALL the dependencies along the extras resolver = Resolver( constraints, repository, cache=DependencyCache(CACHE_DIR), # cache=DependencyCache(tempfile.tempdir), prereleases=False, clear_caches=False, allow_unsafe=False, ) for req in resolver.resolve(max_rounds=10): requirements_by_name[parse_requirement(str(req.req)).name] = SimpleNamespace( requirement=format_requirement(req).strip().replace(" ", ""), url=req.link ) python_config.check_duplicate_dependencies_uniform(requirements_by_name) # Now it iterates along the versions in extras and looks for the requirements and its dependencies, using the # structure created above to select the unified versions (unless the flag indicates otherwise). for extra in itertools.chain((None,), python_config.get_extras()): requirements_file = "requirements{}.txt".format("-" + extra if extra else "") if python_config.override_requirements or not os.path.exists(requirements_file): tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) tmpfile.write("\n".join(python_config.get_requirements(extra=extra)) + "\n") tmpfile.flush() constraints = list( parse_requirements( tmpfile.name, finder=repository.finder, session=repository.session, options=repository.options ) ) resolver = Resolver( constraints, repository, cache=DependencyCache(CACHE_DIR), prereleases=False, clear_caches=False, allow_unsafe=False, ) if not python_config.use_uniform_requirements: python_config.check_duplicate_dependencies_nonuniform(extra, resolver) requirements_list = [] for req in resolver.resolve(max_rounds=10): if req.name != python_config.get("name"): requirement = python_config.get_requirement_info_by_name(req, requirements_by_name) if requirement: requirements_list.append(requirement) self.render_file_inline( requirements_file, "\n".join( ( "-e .{}".format("[" + extra + "]" if extra else ""), *(("-r requirements.txt",) if extra else ()), *python_config.get_vendors(extra=extra), *sorted(requirements_list), ) ), override=python_config.override_requirements, ) # Updates setup file setup = python_config.get_setup() context = { "url": setup.pop("url", ""), "download_url": setup.pop("download_url", ""), } for k, v in context.items(): context[k] = context[k].format(name=setup["name"], user=getuser(), version="{version}") context.update( { "entry_points": setup.pop("entry_points", {}), "extras_require": python_config.get("extras_require"), "install_requires": python_config.get("install_requires"), "python": python_config, "setup": setup, "banner": get_override_warning_banner(), } ) # Render (with overwriting) the allmighty setup.py self.render_file("setup.py", "python/setup.py.j2", context, override=True)