def gen_clauses(self, groups, trackers, specs): C = Clauses() def push_MatchSpec(ms): name = self.ms_to_v(ms) m = C.from_name(name) if m is None: libs = [fn for fn in self.find_matches_group(ms, groups, trackers)] m = C.Any(libs, polarity=None if ms.optional else True, name=name) return m # Create package variables for group in itervalues(groups): for fn in group: C.new_var(fn) # Create spec variables for ms in specs: push_MatchSpec(ms) # Create feature variables for name in iterkeys(trackers): push_MatchSpec(MatchSpec('@' + name)) # Add dependency relationships for group in itervalues(groups): C.Require(C.AtMostOne_NSQ, group) for fn in group: for ms in self.ms_depends(fn): if not ms.optional: C.Require(C.Or, C.Not(fn), push_MatchSpec(ms)) return C
def test_clean_source_cache(self): cache_dirs = { 'source cache': text_type(context.src_cache), 'git cache': text_type(context.git_cache), 'hg cache': text_type(context.hg_cache), 'svn cache': text_type(context.svn_cache), } assert all(isdir(d) for d in itervalues(cache_dirs)) run_command(Commands.CLEAN, '', "--source-cache --yes") assert not all(isdir(d) for d in itervalues(cache_dirs))
def gen_clauses(self, groups, trackers, specs): C = Clauses() # Creates a variable that represents the proposition: # Does the package set include a package that matches MatchSpec "ms"? def push_MatchSpec(ms): name = self.ms_to_v(ms) m = C.from_name(name) if m is None: libs = [ fn for fn in self.find_matches_group(ms, groups, trackers) ] # If the MatchSpec is optional, then there may be cases where we want # to assert that it is *not* True. This requires polarity=None. m = C.Any(libs, polarity=None if ms.optional else True, name=name) return m # Creates a variable that represents the proposition: # Does the package set include package "fn"? for group in itervalues(groups): for fn in group: C.new_var(fn) # Install no more than one version of each package C.Require(C.AtMostOne, group) # Create a variable that represents the proposition: # Is the feature "name" active in this package set? # We mark this as "optional" below because sometimes we need to be able to # assert the proposition is False during the feature minimization pass. for name in iterkeys(trackers): ms = MatchSpec('@' + name) ms.optional = True push_MatchSpec(ms) # Create a variable that represents the proposition: # Is the MatchSpec "ms" satisfied by the current package set? for ms in specs: push_MatchSpec(ms) # Create propositions that assert: # If package "fn" is installed, its dependencie must be satisfied for group in itervalues(groups): for fn in group: for ms in self.ms_depends(fn): if not ms.optional: C.Require(C.Or, C.Not(fn), push_MatchSpec(ms)) return C
def gen_clauses(self, v, dists, specs, features): groups = defaultdict(list) # map name to list of filenames for fn in dists: groups[self.index[fn]['name']].append(fn) for filenames in itervalues(groups): # ensure packages with the same name conflict for fn1 in filenames: v1 = v[fn1] for fn2 in filenames: v2 = v[fn2] if v1 < v2: # NOT (fn1 AND fn2) # e.g. NOT (numpy-1.6 AND numpy-1.7) yield (-v1, -v2) for fn1 in dists: for ms in self.ms_depends(fn1): # ensure dependencies are installed # e.g. numpy-1.7 IMPLIES (python-2.7.3 OR python-2.7.4 OR ...) clause = [-v[fn1]] for fn2 in self.find_matches(ms): if fn2 in dists: clause.append(v[fn2]) assert len(clause) > 1, '%s %r' % (fn1, ms) yield tuple(clause) for feat in features: # ensure that a package (with required name) which has # the feature is installed # e.g. numpy-1.7 IMPLIES (numpy-1.8[mkl] OR numpy-1.7[mkl]) clause = [-v[fn1]] for fn2 in groups[ms.name]: if feat in self.features(fn2): clause.append(v[fn2]) if len(clause) > 1: yield tuple(clause) for spec in specs: ms = MatchSpec(spec) # ensure that a matching package with the feature is installed for feat in features: # numpy-1.7[mkl] OR numpy-1.8[mkl] clause = [v[fn] for fn in self.find_matches(ms) if fn in dists and feat in self.features(fn)] if len(clause) > 0: yield tuple(clause) # Don't install any package that has a feature that wasn't requested. for fn in self.find_matches(ms): if fn in dists and self.features(fn) - features: yield (-v[fn],) # finally, ensure a matching package itself is installed # numpy-1.7-py27 OR numpy-1.7-py26 OR numpy-1.7-py33 OR # numpy-1.7-py27[mkl] OR ... clause = [v[fn] for fn in self.find_matches(ms) if fn in dists] assert len(clause) >= 1, ms yield tuple(clause)
def get_index(channel_urls=(), prepend=True, platform=None, use_cache=False, unknown=False, offline=False, prefix=None): """ Return the index of packages available on the channels If prepend=False, only the channels passed in as arguments are used. If platform=None, then the current platform is used. If prefix is supplied, then the packages installed in that prefix are added. """ channel_urls = config.normalize_urls(channel_urls, platform, offline) if prepend: pri0 = max(itervalues(channel_urls)) if channel_urls else 0 for url, rec in iteritems(config.get_channel_urls(platform, offline)): channel_urls[url] = (rec[0], rec[1] + pri0) index = fetch_index(channel_urls, use_cache=use_cache, unknown=unknown) if prefix: for dist, info in iteritems(install.linked_data(prefix)): fn = dist + '.tar.bz2' channel = info.get('channel', '') if channel not in channel_urls: channel_urls[channel] = (config.canonical_channel_name(channel, True, True), 0) url_s, priority = channel_urls[channel] key = url_s + '::' + fn if url_s else fn if key not in index: # only if the package in not in the repodata, use local # conda-meta (with 'depends' defaulting to []) info.setdefault('depends', []) info['fn'] = fn info['schannel'] = url_s info['channel'] = channel info['url'] = channel + fn info['priority'] = priority index[key] = info return index
def fetch_index(channel_urls, use_cache=False, unknown=False): log.debug('channel_urls=' + repr(channel_urls)) index = {} stdoutlog.info("Fetching package metadata: ") for url in reversed(channel_urls): repodata = fetch_repodata(url, use_cache=use_cache) if repodata is None: continue new_index = repodata['packages'] for info in itervalues(new_index): info['channel'] = url index.update(new_index) stdoutlog.info('\n') if unknown: for pkgs_dir in config.pkgs_dirs: if not isdir(pkgs_dir): continue for dn in os.listdir(pkgs_dir): fn = dn + '.tar.bz2' if fn in index: continue try: with open(join(pkgs_dir, dn, 'info', 'index.json')) as fi: meta = json.load(fi) except IOError: continue if 'depends' not in meta: continue log.debug("adding cached pkg to index: %s" % fn) index[fn] = meta return index
def app_get_index(all_version=False): """ return the index of available applications on the channels By default only the latest version of each app is included in the result, unless all_version is set to True. """ import sys pyxx = 'py%d%d' % sys.version_info[:2] def filter_build(build): return bool(pyxx in build) if 'py' in build else True index = {fn: info for fn, info in iteritems(get_index()) if info.get('type') == 'app' and filter_build(info['build'])} if all_version: return index d = defaultdict(list) # name -> list of Package objects for fn, info in iteritems(index): d[_name_fn(fn)].append(Package(fn, info)) res = {} for pkgs in itervalues(d): pkg = max(pkgs) res[pkg.fn] = index[pkg.fn] return res
def get_index(channel_urls=(), prepend=True, platform=None, use_local=False, use_cache=False, unknown=False, offline=False, prefix=None): """ Return the index of packages available on the channels If prepend=False, only the channels passed in as arguments are used. If platform=None, then the current platform is used. If prefix is supplied, then the packages installed in that prefix are added. """ if use_local: channel_urls = ['local'] + list(channel_urls) channel_urls = normalize_urls(channel_urls, platform, offline) if prepend: channel_urls.extend(get_channel_urls(platform, offline)) channel_urls = prioritize_channels(channel_urls) index = fetch_index(channel_urls, use_cache=use_cache, unknown=unknown) if prefix: priorities = {c: p for c, p in itervalues(channel_urls)} for dist, info in iteritems(install.linked_data(prefix)): fn = info['fn'] schannel = info['schannel'] prefix = '' if schannel == 'defaults' else schannel + '::' priority = priorities.get(schannel, 0) key = prefix + fn if key in index: # Copy the link information so the resolver knows this is installed index[key]['link'] = info.get('link') else: # only if the package in not in the repodata, use local # conda-meta (with 'depends' defaulting to []) info.setdefault('depends', []) info['priority'] = priority index[key] = info return index
def generate_version_eq(self, v, dists, include0=False): groups = defaultdict(list) # map name to list of filenames for fn in sorted(dists): groups[self.index[fn]['name']].append(fn) eq = [] max_rhs = 0 for filenames in sorted(itervalues(groups)): pkgs = sorted(filenames, key=lambda i: dists[i], reverse=True) i = 0 prev = pkgs[0] for pkg in pkgs: try: if (dists[pkg].name, dists[pkg].norm_version, dists[pkg].build_number) != (dists[prev].name, dists[prev].norm_version, dists[prev].build_number): i += 1 except TypeError: i += 1 if i or include0: eq += [(i, v[pkg])] prev = pkg max_rhs += i return eq, max_rhs
def generate_version_eq(self, v, dists, include0=False): groups = defaultdict(list) # map name to list of filenames for fn in sorted(dists): groups[self.index[fn]['name']].append(fn) eq = [] max_rhs = 0 for filenames in sorted(itervalues(groups)): pkgs = sorted(filenames, key=lambda i: dists[i], reverse=True) i = 0 prev = pkgs[0] for pkg in pkgs: try: if (dists[pkg].name, dists[pkg].norm_version, dists[pkg].build_number) != ( dists[prev].name, dists[prev].norm_version, dists[prev].build_number): i += 1 except TypeError: i += 1 if i or include0: eq += [(i, v[pkg])] prev = pkg max_rhs += i return eq, max_rhs
def gen_clauses(self, specs): C = Clauses() # Creates a variable that represents the proposition: # Does the package set include package "fn"? for name, group in iteritems(self.groups): for fkey in group: C.new_var(fkey) # Install no more than one version of each package C.Require(C.AtMostOne, group) # Create an on/off variable for the entire group name = self.ms_to_v(name) C.name_var(C.Any(group, polarity=None, name=name), name+'?') # Creates a variable that represents the proposition: # Does the package set include track_feature "feat"? for name, group in iteritems(self.trackers): name = self.ms_to_v('@' + name) C.name_var(C.Any(group, polarity=None, name=name), name+'?') # Create propositions that assert: # If package "fn" is installed, its dependencie must be satisfied for group in itervalues(self.groups): for fkey in group: nkey = C.Not(fkey) for ms in self.ms_depends(fkey): if not ms.optional: C.Require(C.Or, nkey, self.push_MatchSpec(C, ms)) return C
def bad_installed(self, installed, new_specs): if not installed: return [] dists = {fn: self.index[fn] for fn in installed} specs = [ MatchSpec(' % s %s %s' % (rec['name'], rec['version'], rec['build'])) for rec in itervalues(dists) ] groups, trackers = build_groups(dists) C = self.gen_clauses(groups, trackers, specs) constraints = self.generate_spec_constraints(C, specs) solution = C.sat(constraints) if solution: return [] def get_(name, snames): if name not in snames: snames.add(name) for fn in self.groups.get(name, []): for ms in self.ms_depends(fn): get_(ms.name, snames) snames = set() for spec in new_specs: get_(MatchSpec(spec).name, snames) return set(s.name for s in specs if s.name not in snames)
def filter_group(matches): # If we are here, then this dependency is mandatory, # so add it to the master list. That way it is still # participates in the pruning even if one of its # parents is pruned away match1 = next(ms for ms in matches) isopt = all(ms.optional for ms in matches) name = match1.name isfeat = name[0] == '@' first = name not in snames if isfeat: assert len(matches) == 1 and match1.strictness == 1 group = self.trackers.get(name[1:], []) else: group = self.groups.get(name, []) # Prune packages that don't match any of the patterns # or which have unsatisfiable dependencies nold = nnew = 0 for fkey in group: if filter.setdefault(fkey, True): nold += 1 sat = isfeat or self.match_any(matches, fkey) sat = sat and all(any(filter.get(f2, True) for f2 in self.find_matches(ms)) for ms in self.ms_depends(fkey)) filter[fkey] = sat nnew += sat # Quick exit if we detect unsatisfiability reduced = nnew < nold if reduced: log.debug('%s: pruned from %d -> %d' % (name, nold, nnew)) if nnew == 0: if name in snames: snames.remove(name) if not isopt: raise BadPrune(name) return nnew != 0 if not reduced and not first or isopt or isfeat: return reduced # Perform the same filtering steps on any dependencies shared across # *all* packages in the group. Even if just one of the packages does # not have a particular dependency, it must be ignored in this pass. if first: snames.add(name) cdeps = defaultdict(list) for fkey in group: if filter[fkey]: for m2 in self.ms_depends(fkey): if m2.name[0] != '@' and not m2.optional: cdeps[m2.name].append(m2) cdeps = {mname: set(deps) for mname, deps in iteritems(cdeps) if len(deps) >= nnew} if cdeps: matches = [(ms,) for ms in matches] if sum(filter_group(deps) for deps in itervalues(cdeps)): reduced = True return reduced
def fetch_index(channel_urls, use_cache=False, unknown=False): log.debug('channel_urls=' + repr(channel_urls)) # pool = ThreadPool(5) index = {} stdoutlog.info("Fetching package metadata: ") session = CondaSession() for url in reversed(channel_urls): if config.allowed_channels and url not in config.allowed_channels: sys.exit(""" Error: URL '%s' not in allowed channels. Allowed channels are: - %s """ % (url, '\n - '.join(config.allowed_channels))) try: import concurrent.futures from collections import OrderedDict repodatas = [] with concurrent.futures.ThreadPoolExecutor(10) as executor: future_to_url = OrderedDict([(executor.submit(fetch_repodata, url, use_cache=use_cache, session=session), url) for url in reversed(channel_urls)]) for future in future_to_url: url = future_to_url[future] repodatas.append((url, future.result())) except ImportError: # concurrent.futures is only available in Python 3 repodatas = map(lambda url: (url, fetch_repodata(url, use_cache=use_cache, session=session)), reversed(channel_urls)) for url, repodata in repodatas: if repodata is None: continue new_index = repodata['packages'] for info in itervalues(new_index): info['channel'] = url index.update(new_index) stdoutlog.info('\n') if unknown: for pkgs_dir in config.pkgs_dirs: if not isdir(pkgs_dir): continue for dn in os.listdir(pkgs_dir): fn = dn + '.tar.bz2' if fn in index: continue try: with open(join(pkgs_dir, dn, 'info', 'index.json')) as fi: meta = json.load(fi) except IOError: continue if 'depends' not in meta: meta['depends'] = [] log.debug("adding cached pkg to index: %s" % fn) index[fn] = meta return index
def fetch_index(channel_urls, use_cache=False, unknown=False): log.debug('channel_urls=' + repr(channel_urls)) # pool = ThreadPool(5) index = {} stdoutlog.info("Fetching package metadata: ") session = CondaSession() for url in reversed(channel_urls): if config.allowed_channels and url not in config.allowed_channels: sys.exit(""" Error: URL '%s' not in allowed channels. Allowed channels are: - %s """ % (url, '\n - '.join(config.allowed_channels))) try: import concurrent.futures from collections import OrderedDict repodatas = [] with concurrent.futures.ThreadPoolExecutor(10) as executor: future_to_url = OrderedDict([(executor.submit(fetch_repodata, url, use_cache=use_cache, session=session), url) for url in reversed(channel_urls)]) for future in concurrent.futures.as_completed(future_to_url): url = future_to_url[future] repodatas.append((url, future.result())) except ImportError: # concurrent.futures is only available in Python 3 repodatas = map(lambda url: (url, fetch_repodata(url, use_cache=use_cache, session=session)), reversed(channel_urls)) for url, repodata in repodatas: if repodata is None: continue new_index = repodata['packages'] for info in itervalues(new_index): info['channel'] = url index.update(new_index) stdoutlog.info('\n') if unknown: for pkgs_dir in config.pkgs_dirs: if not isdir(pkgs_dir): continue for dn in os.listdir(pkgs_dir): fn = dn + '.tar.bz2' if fn in index: continue try: with open(join(pkgs_dir, dn, 'info', 'index.json')) as fi: meta = json.load(fi) except IOError: continue if 'depends' not in meta: continue log.debug("adding cached pkg to index: %s" % fn) index[fn] = meta return index
def gen_clauses(self, groups, trackers, specs): C = Clauses() # Creates a variable that represents the proposition: # Does the package set include a package that matches MatchSpec "ms"? def push_MatchSpec(ms): name = self.ms_to_v(ms) m = C.from_name(name) if m is None: libs = [fn for fn in self.find_matches_group(ms, groups, trackers)] # If the MatchSpec is optional, then there may be cases where we want # to assert that it is *not* True. This requires polarity=None. m = C.Any(libs, polarity=None if ms.optional else True, name=name) return m # Creates a variable that represents the proposition: # Does the package set include package "fn"? for group in itervalues(groups): for fn in group: C.new_var(fn) # Install no more than one version of each package C.Require(C.AtMostOne, group) # Create a variable that represents the proposition: # Is the feature "name" active in this package set? # We mark this as "optional" below because sometimes we need to be able to # assert the proposition is False during the feature minimization pass. for name in iterkeys(trackers): ms = MatchSpec('@' + name) ms.optional = True push_MatchSpec(ms) # Create a variable that represents the proposition: # Is the MatchSpec "ms" satisfied by the current package set? for ms in specs: push_MatchSpec(ms) # Create propositions that assert: # If package "fn" is installed, its dependencie must be satisfied for group in itervalues(groups): for fn in group: for ms in self.ms_depends(fn): if not ms.optional: C.Require(C.Or, C.Not(fn), push_MatchSpec(ms)) return C
def fetch_index(channel_urls): index = {} for url in reversed(channel_urls): repodata = fetch_repodata(url) new_index = repodata['packages'] for info in itervalues(new_index): info['channel'] = url index.update(new_index) return index
def gen_clauses(self, v, dists, specs, features): groups = defaultdict(list) # map name to list of filenames for fn in dists: groups[self.index[fn]['name']].append(fn) for filenames in itervalues(groups): # ensure packages with the same name conflict for fn1 in filenames: v1 = v[fn1] for fn2 in filenames: v2 = v[fn2] if v1 < v2: # NOT (fn1 AND fn2) # e.g. NOT (numpy-1.6 AND numpy-1.7) yield [-v1, -v2] for fn1 in dists: for ms in self.ms_depends(fn1): # ensure dependencies are installed # e.g. numpy-1.7 IMPLIES (python-2.7.3 OR python-2.7.4 OR ...) clause = [-v[fn1]] for fn2 in self.find_matches(ms): if fn2 in dists: clause.append(v[fn2]) assert len(clause) > 1, '%s %r' % (fn1, ms) yield clause for feat in features: # ensure that a package (with required name) which has # the feature is installed # e.g. numpy-1.7 IMPLIES (numpy-1.8[mkl] OR numpy-1.7[mkl]) clause = [-v[fn1]] for fn2 in groups[ms.name]: if feat in self.features(fn2): clause.append(v[fn2]) if len(clause) > 1: yield clause for spec in specs: ms = MatchSpec(spec) # ensure that a matching package with the feature is installed for feat in features: # numpy-1.7[mkl] OR numpy-1.8[mkl] clause = [ v[fn] for fn in self.find_matches(ms) if fn in dists and feat in self.features(fn) ] if len(clause) > 0: yield clause # finally, ensure a matching package itself is installed # numpy-1.7-py27 OR numpy-1.7-py26 OR numpy-1.7-py33 OR # numpy-1.7-py27[mkl] OR ... clause = [v[fn] for fn in self.find_matches(ms) if fn in dists] assert len(clause) >= 1 yield clause
def get_site_packages_dir(installed_pkgs): for info in itervalues(installed_pkgs): if info['name'] == 'python': if sys.platform == 'win32': stdlib_dir = 'Lib' else: py_ver = info['version'][:3] stdlib_dir = 'lib/python%s' % py_ver return join(stdlib_dir, 'site-packages') return None
def test_tarball_install_and_bad_metadata(self): with make_temp_env("python flask=0.10.1") as prefix: assert_package_is_installed(prefix, 'flask-0.10.1') flask_data = [ p for p in itervalues(linked_data(prefix)) if p['name'] == 'flask' ][0] run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0.10.1') assert_package_is_installed(prefix, 'python') # Regression test for 2812 # install from local channel from conda.config import pkgs_dirs flask_fname = flask_data['fn'] tar_old_path = join(pkgs_dirs[0], flask_fname) for field in ('url', 'channel', 'schannel'): del flask_data[field] repodata = {'info': {}, 'packages': {flask_fname: flask_data}} with make_temp_env() as channel: subchan = join(channel, subdir) channel = url_path(channel) os.makedirs(subchan) tar_new_path = join(subchan, flask_fname) copyfile(tar_old_path, tar_new_path) with bz2.BZ2File(join(subchan, 'repodata.json.bz2'), 'w') as f: f.write(json.dumps(repodata).encode('utf-8')) run_command(Commands.INSTALL, prefix, '-c', channel, 'flask') assert_package_is_installed(prefix, channel + '::' + 'flask-') # regression test for #2626 # install tarball with full path tar_new_path = join(prefix, flask_fname) copyfile(tar_old_path, tar_new_path) run_command(Commands.INSTALL, prefix, tar_new_path) assert_package_is_installed(prefix, 'flask-0') run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0') # regression test for #2626 # install tarball with relative path tar_new_path = relpath(tar_new_path) run_command(Commands.INSTALL, prefix, tar_new_path) assert_package_is_installed(prefix, 'flask-0.') # regression test for #2599 linked_data_.clear() flask_metadata = glob( join(prefix, 'conda-meta', flask_fname[:-8] + '.json'))[-1] bad_metadata = join(prefix, 'conda-meta', 'flask.json') copyfile(flask_metadata, bad_metadata) assert not package_is_installed(prefix, 'flask', exact=True) assert_package_is_installed(prefix, 'flask-0.')
def get_index(channel_urls=(), prepend=True, platform=None, use_local=False, use_cache=False, unknown=False, offline=False, prefix=None): """ Return the index of packages available on the channels If prepend=False, only the channels passed in as arguments are used. If platform=None, then the current platform is used. If prefix is supplied, then the packages installed in that prefix are added. """ if use_local: channel_urls = ['local'] + list(channel_urls) channel_urls = normalize_urls(channel_urls, platform, offline) if prepend: channel_urls.extend(get_channel_urls(platform, offline)) channel_urls = prioritize_channels(channel_urls) index = fetch_index(channel_urls, use_cache=use_cache, unknown=unknown) if prefix: priorities = {c: p for c, p in itervalues(channel_urls)} maxp = max(itervalues(priorities)) + 1 if priorities else 1 for dist, info in iteritems(install.linked_data(prefix)): fn = info['fn'] schannel = info['schannel'] prefix = '' if schannel == 'defaults' else schannel + '::' priority = priorities.get(schannel, maxp) key = prefix + fn if key in index: # Copy the link information so the resolver knows this is installed index[key]['link'] = info.get('link') else: # only if the package in not in the repodata, use local # conda-meta (with 'depends' defaulting to []) info.setdefault('depends', []) info['priority'] = priority index[key] = info return index
def test_install_tarball_from_local_channel(self): with make_temp_env("python flask=0.10.1") as prefix: assert_package_is_installed(prefix, 'flask-0.10.1') flask_data = [ p for p in itervalues(linked_data(prefix)) if p['name'] == 'flask' ][0] run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0.10.1') assert_package_is_installed(prefix, 'python') flask_fname = flask_data['fn'] tar_old_path = join(context.pkgs_dirs[0], flask_fname) # Regression test for #2812 # install from local channel flask_data = flask_data.dump() for field in ('url', 'channel', 'schannel'): del flask_data[field] repodata = { 'info': {}, 'packages': { flask_fname: Record(**flask_data) } } with make_temp_env() as channel: subchan = join(channel, context.subdir) channel = path_to_url(channel) os.makedirs(subchan) tar_new_path = join(subchan, flask_fname) copyfile(tar_old_path, tar_new_path) with bz2.BZ2File(join(subchan, 'repodata.json.bz2'), 'w') as f: f.write( json.dumps(repodata, cls=EntityEncoder).encode('utf-8')) run_command(Commands.INSTALL, prefix, '-c', channel, 'flask', '--json') assert_package_is_installed(prefix, channel + '::' + 'flask-') run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0') # Regression test for 2970 # install from build channel as a tarball conda_bld = join(sys.prefix, 'conda-bld') conda_bld_sub = join(conda_bld, context.subdir) if not isdir(conda_bld_sub): os.makedirs(conda_bld_sub) tar_bld_path = join(conda_bld_sub, flask_fname) copyfile(tar_new_path, tar_bld_path) # CondaFileNotFoundError: '/home/travis/virtualenv/python2.7.9/conda-bld/linux-64/flask-0.10.1-py27_2.tar.bz2'. run_command(Commands.INSTALL, prefix, tar_bld_path) assert_package_is_installed(prefix, 'flask-')
def fetch_index(channel_urls, use_cache=False, unknown=False): log.debug("channel_urls=" + repr(channel_urls)) # pool = ThreadPool(5) index = {} stdoutlog.info("Fetching package metadata: ") session = CondaSession() for url in reversed(channel_urls): if config.allowed_channels and url not in config.allowed_channels: sys.exit( """ Error: URL '%s' not in allowed channels. Allowed channels are: - %s """ % (url, "\n - ".join(config.allowed_channels)) ) try: import concurrent.futures from collections import OrderedDict repodatas = [] with concurrent.futures.ThreadPoolExecutor(10) as executor: future_to_url = OrderedDict( [ (executor.submit(fetch_repodata, url, use_cache=use_cache, session=session), url) for url in reversed(channel_urls) ] ) for future in future_to_url: url = future_to_url[future] repodatas.append((url, future.result())) except ImportError: # concurrent.futures is only available in Python 3 repodatas = map( lambda url: (url, fetch_repodata(url, use_cache=use_cache, session=session)), reversed(channel_urls) ) for url, repodata in repodatas: if repodata is None: continue new_index = repodata["packages"] for info in itervalues(new_index): info["channel"] = url index.update(new_index) stdoutlog.info("\n") if unknown: add_unknown(index) add_pip_dependency(index) return index
def fetch_index(channel_urls): log.debug('channel_urls=' + repr(channel_urls)) index = {} for url in reversed(channel_urls): repodata = fetch_repodata(url) if repodata is None: continue new_index = repodata['packages'] for info in itervalues(new_index): info['channel'] = url index.update(new_index) return index
def fetch_index(channel_urls, use_cache=False): log.debug('channel_urls=' + repr(channel_urls)) index = {} for url in reversed(channel_urls): repodata = fetch_repodata(url, use_cache=use_cache) if repodata is None: continue new_index = repodata['packages'] for info in itervalues(new_index): info['channel'] = url index.update(new_index) return index
def fetch_index(channel_urls, use_cache=False, unknown=False): log.debug('channel_urls=' + repr(channel_urls)) # pool = ThreadPool(5) index = {} stdoutlog.info("Fetching package metadata ...") session = CondaSession() for url in reversed(channel_urls): if config.allowed_channels and url not in config.allowed_channels: sys.exit(""" Error: URL '%s' not in allowed channels. Allowed channels are: - %s """ % (url, '\n - '.join(config.allowed_channels))) try: import concurrent.futures from collections import OrderedDict repodatas = [] with concurrent.futures.ThreadPoolExecutor(10) as executor: future_to_url = OrderedDict([(executor.submit(fetch_repodata, url, use_cache=use_cache, session=session), url) for url in reversed(channel_urls)]) for future in future_to_url: url = future_to_url[future] repodatas.append((url, future.result())) except ImportError: # concurrent.futures is only available in Python 3 repodatas = map( lambda url: (url, fetch_repodata(url, use_cache=use_cache, session=session)), reversed(channel_urls)) for url, repodata in repodatas: if repodata is None: continue new_index = repodata['packages'] for info in itervalues(new_index): info['channel'] = url index.update(new_index) stdoutlog.info('\n') if unknown: add_unknown(index) if config.add_pip_as_python_dependency: add_pip_dependency(index) return index
def test_tarball_install_and_bad_metadata(self): with make_temp_env("python flask=0.10.1") as prefix: assert_package_is_installed(prefix, 'flask-0.10.1') flask_data = [p for p in itervalues(linked_data(prefix)) if p['name'] == 'flask'][0] run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0.10.1') assert_package_is_installed(prefix, 'python') # Regression test for 2812 # install from local channel from conda.config import pkgs_dirs flask_fname = flask_data['fn'] tar_old_path = join(pkgs_dirs[0], flask_fname) for field in ('url', 'channel', 'schannel'): del flask_data[field] repodata = {'info': {}, 'packages':{flask_fname: flask_data}} with make_temp_env() as channel: subchan = join(channel, subdir) channel = url_path(channel) os.makedirs(subchan) tar_new_path = join(subchan, flask_fname) copyfile(tar_old_path, tar_new_path) with bz2.BZ2File(join(subchan, 'repodata.json.bz2'), 'w') as f: f.write(json.dumps(repodata).encode('utf-8')) run_command(Commands.INSTALL, prefix, '-c', channel, 'flask') assert_package_is_installed(prefix, channel + '::' + 'flask-') # regression test for #2626 # install tarball with full path tar_new_path = join(prefix, flask_fname) copyfile(tar_old_path, tar_new_path) run_command(Commands.INSTALL, prefix, tar_new_path) assert_package_is_installed(prefix, 'flask-0') run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0') # regression test for #2626 # install tarball with relative path tar_new_path = relpath(tar_new_path) run_command(Commands.INSTALL, prefix, tar_new_path) assert_package_is_installed(prefix, 'flask-0.') # regression test for #2599 linked_data_.clear() flask_metadata = glob(join(prefix, 'conda-meta', flask_fname[:-8] + '.json'))[-1] bad_metadata = join(prefix, 'conda-meta', 'flask.json') copyfile(flask_metadata, bad_metadata) assert not package_is_installed(prefix, 'flask', exact=True) assert_package_is_installed(prefix, 'flask-0.')
def test_tarball_install_and_bad_metadata(self): with make_temp_env("python flask=0.10.1 --json") as prefix: assert_package_is_installed(prefix, 'flask-0.10.1') flask_data = [ p for p in itervalues(linked_data(prefix)) if p['name'] == 'flask' ][0] run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0.10.1') assert_package_is_installed(prefix, 'python') flask_fname = flask_data['fn'] tar_old_path = join(context.pkgs_dirs[0], flask_fname) # regression test for #2886 (part 1 of 2) # install tarball from package cache, default channel run_command(Commands.INSTALL, prefix, tar_old_path) assert_package_is_installed(prefix, 'flask-0.') # regression test for #2626 # install tarball with full path, outside channel tar_new_path = join(prefix, flask_fname) copyfile(tar_old_path, tar_new_path) run_command(Commands.INSTALL, prefix, tar_new_path) assert_package_is_installed(prefix, 'flask-0') # regression test for #2626 # install tarball with relative path, outside channel run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0.10.1') tar_new_path = relpath(tar_new_path) run_command(Commands.INSTALL, prefix, tar_new_path) assert_package_is_installed(prefix, 'flask-0.') # regression test for #2886 (part 2 of 2) # install tarball from package cache, local channel run_command(Commands.REMOVE, prefix, 'flask', '--json') assert not package_is_installed(prefix, 'flask-0') run_command(Commands.INSTALL, prefix, tar_old_path) # The last install was from the `local::` channel assert_package_is_installed(prefix, 'flask-') # regression test for #2599 linked_data_.clear() flask_metadata = glob( join(prefix, 'conda-meta', flask_fname[:-8] + '.json'))[-1] bad_metadata = join(prefix, 'conda-meta', 'flask.json') copyfile(flask_metadata, bad_metadata) assert not package_is_installed(prefix, 'flask', exact=True) assert_package_is_installed(prefix, 'flask-0.')
def create_bundle(prefix=None, data_path=None, bundle_name=None, extra_meta=None): """ Create a "bundle" of the environment located in `prefix`, and return the full path to the created package, which is going to be located in the current working directory, unless specified otherwise. """ meta = dict( name='bundle', build='0', build_number=0, type='bundle', bundle_name=bundle_name, creator=os.getenv('USER'), platform=platform, arch=arch_name, ctime=time.strftime(ISO8601), depends=[], ) meta['version'] = get_version(meta) tar_path = join('bundle-%(version)s-0.tar.bz2' % meta) t = tarfile.open(tar_path, 'w:bz2') if prefix: prefix = abspath(prefix) if not prefix.startswith('/opt/anaconda'): for f in sorted(untracked(prefix, exclude_self_build=True)): if f.startswith(BDP): raise RuntimeError('bad untracked file: %s' % f) if f.startswith('info/'): continue path = join(prefix, f) add_file(t, path, f) meta['bundle_prefix'] = prefix meta['depends'] = [ '%(name)s %(version)s %(build)s' % info for info in itervalues(install.linked_data(prefix)) ] if data_path: add_data(t, data_path) if extra_meta: meta.update(extra_meta) add_info_files(t, meta) t.close() return tar_path
def gen_clauses(self, groups, trackers, specs): C = Clauses() polarities = {} def push_MatchSpec(ms, polarity=None): name = self.ms_to_v(ms) m = C.from_name(name) if m is None: m = C.Any(self.find_matches_group(ms, groups, trackers), polarity=polarity, name=name) return m # Create package variables for group in itervalues(groups): for fn in group: C.new_var(fn) # Create spec variables for ms in specs: push_MatchSpec(ms, polarity=None if ms.optional else True) # Create feature variables for name in iterkeys(trackers): push_MatchSpec(MatchSpec('@' + name), polarity=True) # Add dependency relationships for group in itervalues(groups): C.Require(C.AtMostOne_NSQ, group) for fn in group: for ms in self.ms_depends(fn): if not ms.optional: C.Require(C.Or, C.Not(fn), push_MatchSpec(ms, polarity=True)) return C
def gen_clauses(self, v, dists, specs, features): groups = defaultdict(list) # map name to list of filenames for fn in dists: groups[self.index[fn]['name']].append(fn) for filenames in itervalues(groups): # ensure packages with the same name conflict for fn1 in filenames: v1 = v[fn1] for fn2 in filenames: v2 = v[fn2] if v1 < v2: yield [-v1, -v2] for fn1 in dists: for ms in self.ms_depends(fn1): # ensure dependencies are installed clause = [-v[fn1]] for fn2 in self.find_matches(ms): if fn2 in dists: clause.append(v[fn2]) assert len(clause) > 1, '%s %r' % (fn1, ms) yield clause for feat in features: # ensure that a package (with required name) which has # the feature is installed clause = [-v[fn1]] for fn2 in groups[ms.name]: if feat in self.features(fn2): clause.append(v[fn2]) if len(clause) > 1: yield clause for spec in specs: ms = MatchSpec(spec) # ensure that a matching package which the feature is installed for feat in features: clause = [v[fn] for fn in self.find_matches(ms) if fn in dists and feat in self.features(fn)] if len(clause) > 0: yield clause # finally, ensure a matching package itself is installed clause = [v[fn] for fn in self.find_matches(ms) if fn in dists] assert len(clause) >= 1 yield clause
def generate_version_eq(self, v, dists, installed_dists, specs, include0=False, update_deps=True): groups = defaultdict(list) # map name to list of filenames for fn in sorted(dists): groups[self.index[fn]['name']].append(fn) eq = [] max_rhs = 0 for filenames in sorted(itervalues(groups)): pkgs = sorted(filenames, key=lambda i: dists[i], reverse=True) if (not update_deps and not any(s.split()[0] == pkgs[0].rsplit('-', 2)[0] for s in specs)): rearrange = True for d in installed_dists: if d in pkgs: break else: # It isn't already installed rearrange = False if rearrange: idx = pkgs.index(d) # For example, if the versions are 1.0, 2.0, 3.0, 4.0, and # 5.0, and 3.0 is installed, this prefers 3.0 > 4.0 > 5.0 # > 2.0 > 1.0. pkgs = [d] + list(reversed(pkgs[:idx])) + pkgs[idx + 1:] i = 0 prev = pkgs[0] for pkg in pkgs: try: if (dists[pkg].name, dists[pkg].norm_version, dists[pkg].build_number) != ( dists[prev].name, dists[prev].norm_version, dists[prev].build_number): i += 1 except TypeError: i += 1 if i or include0: eq += [(i, v[pkg])] prev = pkg max_rhs += i return eq, max_rhs
def create_bundle(prefix=None, data_path=None, bundle_name=None, extra_meta=None): """ Create a "bundle" of the environment located in `prefix`, and return the full path to the created package, which is going to be located in the current working directory, unless specified otherwise. """ meta = dict( name='bundle', build='0', build_number=0, type='bundle', bundle_name=bundle_name, creator=os.getenv('USER'), platform=platform, arch=arch_name, ctime=time.strftime(ISO8601), depends=[], ) meta['version'] = get_version(meta) tar_path = join('bundle-%(version)s-0.tar.bz2' % meta) t = tarfile.open(tar_path, 'w:bz2') if prefix: prefix = abspath(prefix) if not prefix.startswith('/opt/anaconda'): for f in sorted(untracked(prefix, exclude_self_build=True)): if f.startswith(BDP): raise RuntimeError('bad untracked file: %s' % f) if f.startswith('info/'): continue path = join(prefix, f) add_file(t, path, f) meta['bundle_prefix'] = prefix meta['depends'] = ['%(name)s %(version)s %(build)s' % info for info in itervalues(install.linked_data(prefix))] if data_path: add_data(t, data_path) if extra_meta: meta.update(extra_meta) add_info_files(t, meta) t.close() return tar_path
def test_tarball_install_and_bad_metadata(self): with make_temp_env("python flask=0.10.1 --json") as prefix: assert_package_is_installed(prefix, 'flask-0.10.1') flask_data = [p for p in itervalues(linked_data(prefix)) if p['name'] == 'flask'][0] run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0.10.1') assert_package_is_installed(prefix, 'python') flask_fname = flask_data['fn'] tar_old_path = join(context.pkgs_dirs[0], flask_fname) # regression test for #2886 (part 1 of 2) # install tarball from package cache, default channel run_command(Commands.INSTALL, prefix, tar_old_path) assert_package_is_installed(prefix, 'flask-0.') # regression test for #2626 # install tarball with full path, outside channel tar_new_path = join(prefix, flask_fname) copyfile(tar_old_path, tar_new_path) run_command(Commands.INSTALL, prefix, tar_new_path) assert_package_is_installed(prefix, 'flask-0') # regression test for #2626 # install tarball with relative path, outside channel run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0.10.1') tar_new_path = relpath(tar_new_path) run_command(Commands.INSTALL, prefix, tar_new_path) assert_package_is_installed(prefix, 'flask-0.') # regression test for #2886 (part 2 of 2) # install tarball from package cache, local channel run_command(Commands.REMOVE, prefix, 'flask', '--json') assert not package_is_installed(prefix, 'flask-0') run_command(Commands.INSTALL, prefix, tar_old_path) # The last install was from the `local::` channel assert_package_is_installed(prefix, 'flask-') # regression test for #2599 linked_data_.clear() flask_metadata = glob(join(prefix, 'conda-meta', flask_fname[:-8] + '.json'))[-1] bad_metadata = join(prefix, 'conda-meta', 'flask.json') copyfile(flask_metadata, bad_metadata) assert not package_is_installed(prefix, 'flask', exact=True) assert_package_is_installed(prefix, 'flask-0.')
def test_install_tarball_from_local_channel(self): with make_temp_env("python flask=0.10.1") as prefix: assert_package_is_installed(prefix, 'flask-0.10.1') flask_data = [p for p in itervalues(linked_data(prefix)) if p['name'] == 'flask'][0] run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0.10.1') assert_package_is_installed(prefix, 'python') flask_fname = flask_data['fn'] tar_old_path = join(context.pkgs_dirs[0], flask_fname) # Regression test for #2812 # install from local channel for field in ('url', 'channel', 'schannel'): del flask_data[field] repodata = {'info': {}, 'packages': {flask_fname: flask_data}} with make_temp_env() as channel: subchan = join(channel, context.subdir) channel = path_to_url(channel) os.makedirs(subchan) tar_new_path = join(subchan, flask_fname) copyfile(tar_old_path, tar_new_path) with bz2.BZ2File(join(subchan, 'repodata.json.bz2'), 'w') as f: f.write(json.dumps(repodata, cls=EntityEncoder).encode('utf-8')) run_command(Commands.INSTALL, prefix, '-c', channel, 'flask') assert_package_is_installed(prefix, channel + '::' + 'flask-') run_command(Commands.REMOVE, prefix, 'flask') assert not package_is_installed(prefix, 'flask-0') # Regression test for 2970 # install from build channel as a tarball conda_bld = join(sys.prefix, 'conda-bld') conda_bld_sub = join(conda_bld, context.subdir) tar_bld_path = join(conda_bld_sub, flask_fname) if os.path.exists(conda_bld): try: os.rename(tar_new_path, tar_bld_path) except OSError: pass else: os.makedirs(conda_bld) os.rename(subchan, conda_bld_sub) run_command(Commands.INSTALL, prefix, tar_bld_path) assert_package_is_installed(prefix, 'flask-')
def fetch_index(channel_urls, use_cache=False, unknown=False): log.debug('channel_urls=' + repr(channel_urls)) # pool = ThreadPool(5) index = {} stdoutlog.info("Fetching package metadata: ") session = CondaSession() for url in reversed(channel_urls): if config.allowed_channels and url not in config.allowed_channels: sys.exit(""" Error: URL '%s' not in allowed channels. Allowed channels are: - %s """ % (url, '\n - '.join(config.allowed_channels))) repodatas = map(lambda url: (url, fetch_repodata(url, use_cache=use_cache, session=session)), reversed(channel_urls)) for url, repodata in repodatas: if repodata is None: continue new_index = repodata['packages'] for info in itervalues(new_index): info['channel'] = url index.update(new_index) stdoutlog.info('\n') if unknown: for pkgs_dir in config.pkgs_dirs: if not isdir(pkgs_dir): continue for dn in os.listdir(pkgs_dir): fn = dn + '.tar.bz2' if fn in index: continue try: with open(join(pkgs_dir, dn, 'info', 'index.json')) as fi: meta = json.load(fi) except IOError: continue if 'depends' not in meta: continue log.debug("adding cached pkg to index: %s" % fn) index[fn] = meta return index
def app_get_index(all_version=False): """ return the index of available applications on the channels By default only the latest version of each app is included in the result, unless all_version is set to True. """ index = {fn: info for fn, info in iteritems(get_index()) if info.get("type") == "app"} if all_version: return index d = defaultdict(list) # name -> list of Package objects for fn, info in iteritems(index): d[_name_fn(fn)].append(Package(fn, info)) res = {} for pkgs in itervalues(d): pkg = max(pkgs) res[pkg.fn] = index[pkg.fn] return res
def add_unknown(index, priorities): maxpri = max(itervalues(priorities)) + 1 for pkgs_dir in pkgs_dirs: if not isdir(pkgs_dir): continue for dn in os.listdir(pkgs_dir): fn = dn + '.tar.bz2' if fn in index: continue try: with open(join(pkgs_dir, dn, 'info', 'index.json')) as fi: meta = json.load(fi) except IOError: continue channel = meta.setdefault('channel', '') url = meta[url] = channel + fn meta.setdefault('depends', []) meta.setdefault('priority', priorities.get(channel, maxpri)) log.debug("adding cached pkg to index: %s" % url) index[url] = meta
def add_unknown(index, priorities): maxpri = max(itervalues(priorities)) + 1 for pkgs_dir in config.pkgs_dirs: if not isdir(pkgs_dir): continue for dn in os.listdir(pkgs_dir): fn = dn + '.tar.bz2' if fn in index: continue try: with open(join(pkgs_dir, dn, 'info', 'index.json')) as fi: meta = json.load(fi) except IOError: continue channel = meta.setdefault('channel', '') url = meta[url] = channel + fn meta.setdefault('depends', []) meta.setdefault('priority', priorities.get(channel, maxpri)) log.debug("adding cached pkg to index: %s" % url) index[url] = meta
def generate_version_eq(self, v, dists, installed_dists, specs, include0=False, update_deps=True): groups = defaultdict(list) # map name to list of filenames for fn in sorted(dists): groups[self.index[fn]["name"]].append(fn) eq = [] max_rhs = 0 for filenames in sorted(itervalues(groups)): pkgs = sorted(filenames, key=lambda i: dists[i], reverse=True) if not update_deps and not any(s.split()[0] == pkgs[0].rsplit("-", 2)[0] for s in specs): rearrange = True for d in installed_dists: if d in pkgs: break else: # It isn't already installed rearrange = False if rearrange: idx = pkgs.index(d) # For example, if the versions are 1.0, 2.0, 3.0, 4.0, and # 5.0, and 3.0 is installed, this prefers 3.0 > 4.0 > 5.0 # > 2.0 > 1.0. pkgs = [d] + list(reversed(pkgs[:idx])) + pkgs[idx + 1 :] i = 0 prev = pkgs[0] for pkg in pkgs: try: if (dists[pkg].name, dists[pkg].norm_version, dists[pkg].build_number) != ( dists[prev].name, dists[prev].norm_version, dists[prev].build_number, ): i += 1 except TypeError: i += 1 if i or include0: eq += [(i, v[pkg])] prev = pkg max_rhs += i return eq, max_rhs
def app_get_index(all_version=False): """ return the index of available applications on the channels By default only the latest version of each app is included in the result, unless all_version is set to True. """ index = {fn: info for fn, info in iteritems(get_index()) if info.get('type') == 'app'} if all_version: return index d = defaultdict(list) # name -> list of Package objects for fn, info in iteritems(index): d[_name_fn(fn)].append(Package(fn, info)) res = {} for pkgs in itervalues(d): pkg = max(pkgs) res[pkg.fn] = index[pkg.fn] return res
def get_egg_info(prefix, all_pkgs=False): """ Return a set of canonical names of all Python packages (in `prefix`), by inspecting the .egg-info files inside site-packages. By default, only untracked (not conda installed) .egg-info files are considered. Setting `all_pkgs` to True changes this. """ installed_pkgs = linked_data(prefix) sp_dir = get_site_packages_dir(installed_pkgs) if sp_dir is None: return set() conda_files = set() for info in itervalues(installed_pkgs): conda_files.update(info.get('files', [])) res = set() for path in get_egg_info_files(join(prefix, sp_dir)): f = rel_path(prefix, path) if all_pkgs or f not in conda_files: dist = parse_egg_info(path) if dist: res.add(dist) return res
def bad_installed(self, installed, new_specs): if not installed: return [] dists = {fn: self.index[fn] for fn in installed} specs = [MatchSpec(' % s %s %s' % (rec['name'], rec['version'], rec['build'])) for rec in itervalues(dists)] groups, trackers = build_groups(dists) C = self.gen_clauses(groups, trackers, specs) constraints = self.generate_spec_constraints(C, specs) solution = C.sat(constraints) if solution: return [] def get_(name, snames): if name not in snames: snames.add(name) for fn in self.groups.get(name, []): for ms in self.ms_depends(fn): get_(ms.name, snames) snames = set() for spec in new_specs: get_(MatchSpec(spec).name, snames) return set(s.name for s in specs if s.name not in snames)
def add_unknown(index, priorities): maxp = max(itervalues(priorities)) + 1 if priorities else 1 for fkey, info in iteritems(package_cache()): if fkey in index or not info['dirs']: continue try: with open(join(info['dirs'][0], 'info', 'index.json')) as fi: meta = json.load(fi) except IOError: continue fname = dist2filename(fkey) if info['urls']: url = info['urls'][0] elif 'url' in meta: url = meta['url'] else: url = meta.get('channel', '<unknown>/') + fname channel, schannel = url_channel(url) priority = priorities.get(schannel, maxp) meta.update({'fn': fname, 'url': url, 'channel': channel, 'schannel': channel, 'priority': priority}) meta.setdefault('depends', []) log.debug("adding cached pkg to index: %s" % url) index[url] = meta
def clone_env(prefix1, prefix2, verbose=True, quiet=False, fetch_args=None): """ clone existing prefix1 into new prefix2 """ untracked_files = untracked(prefix1) # Discard conda and any package that depends on it drecs = install.linked_data(prefix1) filter = {} found = True while found: found = False for dist, info in iteritems(drecs): name = info['name'] if name in filter: continue if name == 'conda': filter['conda'] = dist found = True break for dep in info.get('depends', []): if MatchSpec(dep).name in filter: filter[name] = dist found = True if not quiet and filter: print( 'The following packages cannot be cloned out of the root environment:' ) for pkg in itervalues(filter): print(' - ' + pkg) # Assemble the URL and channel list urls = {} index = {} for dist, info in iteritems(drecs): if info['name'] in filter: continue url = info.get('url') if url is None: sys.exit('Error: no URL found for package: %s' % dist) _, schannel = url_channel(url) index[dist + '.tar.bz2'] = info urls[dist] = url r = Resolve(index) dists = r.dependency_sort(urls.keys()) urls = [urls[d] for d in dists] if verbose: print('Packages: %d' % len(dists)) print('Files: %d' % len(untracked_files)) for f in untracked_files: src = join(prefix1, f) dst = join(prefix2, f) dst_dir = dirname(dst) if islink(dst_dir) or isfile(dst_dir): os.unlink(dst_dir) if not isdir(dst_dir): os.makedirs(dst_dir) if islink(src): os.symlink(os.readlink(src), dst) continue try: with open(src, 'rb') as fi: data = fi.read() except IOError: continue try: s = data.decode('utf-8') s = s.replace(prefix1, prefix2) data = s.encode('utf-8') except UnicodeDecodeError: # data is binary pass with open(dst, 'wb') as fo: fo.write(data) shutil.copystat(src, dst) actions = explicit(urls, prefix2, verbose=not quiet, force_extract=False, fetch_args=fetch_args) return actions, untracked_files
def filter_group(matches, chains=None): # If we are here, then this dependency is mandatory, # so add it to the master list. That way it is still # participates in the pruning even if one of its # parents is pruned away if unsat: return False match1 = next(ms for ms in matches) name = match1.name first = name not in snames group = self.groups.get(name, []) # Prune packages that don't match any of the patterns # or which have unsatisfiable dependencies nold = 0 bad_deps = [] for fkey in group: if filter.setdefault(fkey, True): nold += 1 sat = self.match_any(matches, fkey) sat = sat and all(any(filter.get(f2, True) for f2 in self.find_matches(ms)) for ms in self.ms_depends(fkey)) filter[fkey] = sat if not sat: bad_deps.append(fkey) # Build dependency chains if we detect unsatisfiability nnew = nold - len(bad_deps) reduced = nnew < nold if reduced: log.debug('%s: pruned from %d -> %d' % (name, nold, nnew)) if nnew == 0: if name in snames: snames.remove(name) bad_deps = [fkey for fkey in bad_deps if self.match_any(matches, fkey)] matches = [(ms,) for ms in matches] chains = [a + b for a in chains for b in matches] if chains else matches if bad_deps: dep2 = set() for fkey in bad_deps: for ms in self.ms_depends(fkey): if not any(filter.get(f2, True) for f2 in self.find_matches(ms)): dep2.add(ms) chains = [a + (b,) for a in chains for b in dep2] unsat.update(chains) return nnew != 0 if not reduced and not first: return False # Perform the same filtering steps on any dependencies shared across # *all* packages in the group. Even if just one of the packages does # not have a particular dependency, it must be ignored in this pass. if first: snames.add(name) if match1 not in specs: nspecs.add(MatchSpec(name)) cdeps = defaultdict(list) for fkey in group: if filter[fkey]: for m2 in self.ms_depends(fkey): if m2.name[0] != '@' and not m2.optional: cdeps[m2.name].append(m2) cdeps = {mname: set(deps) for mname, deps in iteritems(cdeps) if len(deps) >= nnew} if cdeps: matches = [(ms,) for ms in matches] if chains: matches = [a + b for a in chains for b in matches] if sum(filter_group(deps, chains) for deps in itervalues(cdeps)): reduced = True return reduced
def filter_group(matches, top): # If no packages exist with this name, it's a fatal error match1 = next(x for x in matches) name = match1.name group = self.groups.get(name,[]) if not group: bad_deps.append((matches,top)) return False # If we are here, then this dependency is mandatory, # so add it to the master list. That way it is still # participates in the pruning even if one of its # parents is pruned away if all(name != ms.name for ms in specs): specs.append(MatchSpec(name, parent=str(top))) # Prune packages that don't match any of the patterns # or which may be missing dependencies nold = nnew = 0 first = False notfound = set() for fn in group: sat = valid.get(fn, None) if sat is None: first = sat = valid[fn] = True nold += sat if sat: if name[-1] == '@': sat = name[:-1] in self.track_features(fn) else: sat = self.match_any(matches, fn) if sat: sat = all(any(valid.get(f2, True) for f2 in self.find_matches(ms)) for ms in self.ms_depends(fn) if not ms.optional) if not sat: notfound.update(ms for ms in self.ms_depends(fn) if ms.name not in self.groups) nnew += sat valid[fn] = sat reduced = nnew < nold if reduced: log.debug('%s: pruned from %d -> %d' % (name, nold, nnew)) if nnew == 0: if notfound: bad_deps.append((notfound,matches)) unsat.extend(matches) return True elif not first: return False # Perform the same filtering steps on any dependencies shared across # *all* packages in the group. Even if just one of the packages does # not have a particular dependency, it must be ignored in this pass. cdeps = defaultdict(list) for fn in group: if valid[fn]: for m2 in self.ms_depends(fn): cdeps[m2.name].append(m2) if top is None: top = match1 cdeps = {mname:set(deps) for mname,deps in iteritems(cdeps) if len(deps)==nnew} if cdeps: top = top if top else match1 if sum(filter_group(deps, top) for deps in itervalues(cdeps)): reduced = True return reduced
def add_pip_dependency(index): for info in itervalues(index): if (info['name'] == 'python' and info['version'].startswith( ('2.', '3.'))): info.setdefault('depends', []).append('pip')