def load_linked_data(prefix, dist, rec=None): schannel, dname = dist2pair(dist) meta_file = join(prefix, 'conda-meta', dname + '.json') if rec is None: try: with open(meta_file) as fi: rec = json.load(fi) except IOError: return None else: linked_data(prefix) url = rec.get('url') fn = rec.get('fn') if not fn: fn = rec['fn'] = url.rsplit('/', 1)[-1] if url else dname + '.tar.bz2' if fn[:-8] != dname: log.debug('Ignoring invalid package metadata file: %s' % meta_file) return None channel = rec.get('channel') if channel: channel = channel.rstrip('/') if not url or (url.startswith('file:') and channel[0] != '<unknown>'): url = rec['url'] = channel + '/' + fn channel, schannel = url_channel(url) rec['url'] = url rec['channel'] = channel rec['schannel'] = schannel rec['link'] = rec.get('link') or True cprefix = '' if schannel == 'defaults' else schannel + '::' linked_data_[prefix][str(cprefix + dname)] = rec return rec
def load_linked_data(prefix, dist, rec=None): schannel, dname = dist2pair(dist) meta_file = join(prefix, "conda-meta", dname + ".json") if rec is None: try: with open(meta_file) as fi: rec = json.load(fi) except IOError: return None else: linked_data(prefix) url = rec.get("url") fn = rec.get("fn") if not fn: fn = rec["fn"] = url.rsplit("/", 1)[-1] if url else dname + ".tar.bz2" if fn[:-8] != dname: log.debug("Ignoring invalid package metadata file: %s" % meta_file) return None channel = rec.get("channel") if channel: channel = channel.rstrip("/") if not url or (url.startswith("file:") and channel[0] != "<unknown>"): url = rec["url"] = channel + "/" + fn channel, schannel = url_channel(url) rec["url"] = url rec["channel"] = channel rec["schannel"] = schannel rec["link"] = rec.get("link") or True cprefix = "" if schannel == "defaults" else schannel + "::" linked_data_[prefix][str(cprefix + dname)] = rec return rec
def load_linked_data(prefix, dist, rec=None): schannel, dname = dist2pair(dist) meta_file = join(prefix, 'conda-meta', dname + '.json') if rec is None: try: with open(meta_file) as fi: rec = json.load(fi) except IOError: return None else: linked_data(prefix) url = rec.get('url') if 'fn' not in rec: rec['fn'] = url.rsplit('/', 1)[-1] if url else dname + '.tar.bz2' if not url and 'channel' in rec: url = rec['url'] = rec['channel'] + rec['fn'] if rec['fn'][:-8] != dname: log.debug('Ignoring invalid package metadata file: %s' % meta_file) return None channel, schannel = url_channel(url) rec['channel'] = channel rec['schannel'] = schannel cprefix = '' if schannel == 'defaults' else schannel + '::' linked_data_[prefix][str(cprefix + dname)] = rec return rec
def add_cached_package(pdir, url, overwrite=False, urlstxt=False): """ Adds a new package to the cache. The URL is used to determine the package filename and channel, and the directory pdir is scanned for both a compressed and an extracted version of that package. If urlstxt=True, this URL will be appended to the urls.txt file in the cache, so that subsequent runs will correctly identify the package. """ package_cache() if '/' in url: dist = url.rsplit('/', 1)[-1] else: dist = url url = None if dist.endswith('.tar.bz2'): fname = dist dist = dist[:-8] else: fname = dist + '.tar.bz2' xpkg = join(pdir, fname) if not overwrite and xpkg in fname_table_: return if not isfile(xpkg): xpkg = None xdir = join(pdir, dist) if not (isdir(xdir) and isfile(join(xdir, 'info', 'files')) and isfile(join(xdir, 'info', 'index.json'))): xdir = None if not (xpkg or xdir): return if url: url = remove_binstar_tokens(url) _, schannel = url_channel(url) prefix = '' if schannel == 'defaults' else schannel + '::' xkey = xpkg or (xdir + '.tar.bz2') fname_table_[xkey] = fname_table_[url_path(xkey)] = prefix fkey = prefix + dist rec = package_cache_.get(fkey) if rec is None: rec = package_cache_[fkey] = dict(files=[], dirs=[], urls=[]) if url and url not in rec['urls']: rec['urls'].append(url) if xpkg and xpkg not in rec['files']: rec['files'].append(xpkg) if xdir and xdir not in rec['dirs']: rec['dirs'].append(xdir) if urlstxt: try: with open(join(pdir, 'urls.txt'), 'a') as fa: fa.write('%s\n' % url) except IOError: pass
def add_cached_package(pdir, url, overwrite=False, urlstxt=False): """ Adds a new package to the cache. The URL is used to determine the package filename and channel, and the directory pdir is scanned for both a compressed and an extracted version of that package. If urlstxt=True, this URL will be appended to the urls.txt file in the cache, so that subsequent runs will correctly identify the package. """ package_cache() if "/" in url: dist = url.rsplit("/", 1)[-1] else: dist = url url = None if dist.endswith(".tar.bz2"): fname = dist dist = dist[:-8] else: fname = dist + ".tar.bz2" xpkg = join(pdir, fname) if not overwrite and xpkg in fname_table_: return if not isfile(xpkg): xpkg = None xdir = join(pdir, dist) if not (isdir(xdir) and isfile(join(xdir, "info", "files")) and isfile(join(xdir, "info", "index.json"))): xdir = None if not (xpkg or xdir): return if url: url = remove_binstar_tokens(url) _, schannel = url_channel(url) prefix = "" if schannel == "defaults" else schannel + "::" xkey = xpkg or (xdir + ".tar.bz2") fname_table_[xkey] = fname_table_[path_to_url(xkey)] = prefix fkey = prefix + dist rec = package_cache_.get(fkey) if rec is None: rec = package_cache_[fkey] = dict(files=[], dirs=[], urls=[]) if url and url not in rec["urls"]: rec["urls"].append(url) if xpkg and xpkg not in rec["files"]: rec["files"].append(xpkg) if xdir and xdir not in rec["dirs"]: rec["dirs"].append(xdir) if urlstxt: try: with open(join(pdir, "urls.txt"), "a") as fa: fa.write("%s\n" % url) except IOError: pass
def load_linked_data(prefix, dist, rec=None): schannel, dname = _dist2pair(dist) if rec is None: meta_file = join(prefix, 'conda-meta', dname + '.json') try: with open(meta_file) as fi: rec = json.load(fi) except IOError: return None _, schannel = url_channel(rec.get('url')) else: linked_data(prefix) rec['schannel'] = schannel cprefix = '' if schannel == 'defaults' else schannel + '::' rec['fn'] = dname + '.tar.bz2' linked_data_[prefix][str(cprefix + dname)] = rec return rec
def load_linked_data(prefix, dist, rec=None): schannel, dname = _dist2pair(dist) if rec is None: meta_file = join(prefix, 'conda-meta', dname + '.json') try: with open(meta_file) as fi: rec = json.load(fi) except IOError: return None else: linked_data(prefix) url = rec.get('url') channel, schannel = url_channel(url) if 'fn' not in rec: rec['fn'] = url.rsplit('/', 1)[-1] if url else dname + '.tar.bz2' rec['channel'] = channel rec['schannel'] = schannel cprefix = '' if schannel == 'defaults' else schannel + '::' linked_data_[prefix][str(cprefix + dname)] = rec return rec
def add_unknown(index, priorities): maxp = max(itervalues(priorities)) + 1 if priorities else 1 for fkey, info in iteritems(package_cache()): if fkey in index or not info['dirs']: continue try: with open(join(info['dirs'][0], 'info', 'index.json')) as fi: meta = json.load(fi) except IOError: continue fname = dist2filename(fkey) if info['urls']: url = info['urls'][0] elif 'url' in meta: url = meta['url'] else: url = meta.get('channel', '<unknown>/') + fname channel, schannel = url_channel(url) priority = priorities.get(schannel, maxp) meta.update({'fn': fname, 'url': url, 'channel': channel, 'schannel': channel, 'priority': priority}) meta.setdefault('depends', []) log.debug("adding cached pkg to index: %s" % url) index[url] = meta
def explicit(specs, prefix, verbose=False, force_extract=True, fetch_args=None): actions = defaultdict(list) actions['PREFIX'] = prefix actions['op_order'] = RM_FETCHED, FETCH, RM_EXTRACTED, EXTRACT, UNLINK, LINK linked = {install.name_dist(dist): dist for dist in install.linked(prefix)} fetch_args = fetch_args or {} index = {} verifies = [] channels = {} for spec in specs: if spec == '@EXPLICIT': continue # Format: (url|path)(:#md5)? m = url_pat.match(spec) if m is None: sys.exit('Could not parse explicit URL: %s' % spec) url, md5 = m.group('url') + '/' + m.group('fn'), m.group('md5') if not is_url(url): if not isfile(url): sys.exit('Error: file not found: %s' % url) url = utils.url_path(url) url_p, fn = url.rsplit('/', 1) # See if the URL refers to a package in our cache prefix = pkg_path = dir_path = None if url_p.startswith('file://'): prefix = install.cached_url(url) # If not, determine the channel name from the URL if prefix is None: _, schannel = url_channel(url) prefix = '' if schannel == 'defaults' else schannel + '::' fn = prefix + fn dist = fn[:-8] pkg_path = install.is_fetched(dist) dir_path = install.is_extracted(dist) # Don't re-fetch unless there is an MD5 mismatch if pkg_path and (md5 and md5_file(pkg_path) != md5): # This removes any extracted copies as well actions[RM_FETCHED].append(dist) pkg_path = dir_path = None # Don't re-extract unless forced, or if we can't check the md5 if dir_path and (force_extract or md5 and not pkg_path): actions[RM_EXTRACTED].append(dist) dir_path = None if not dir_path: if not pkg_path: _, conflict = install.find_new_location(dist) if conflict: actions[RM_FETCHED].append(conflict) actions[FETCH].append(dist) if md5: # Need to verify against the package index verifies.append((dist + '.tar.bz2', md5)) channels[url_p + '/'] = (schannel, 0) actions[EXTRACT].append(dist) # unlink any installed package with that name name = install.name_dist(dist) if name in linked: actions[UNLINK].append(linked[name]) actions[LINK].append(dist) # Finish the MD5 verification if verifies: index = fetch_index(channels, **fetch_args) for fn, md5 in verifies: info = index.get(fn) if info is None: sys.exit("Error: no package '%s' in index" % fn) if 'md5' not in info: sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn) if info['md5'] != md5: sys.exit( 'MD5 mismatch for: %s\n spec: %s\n repo: %s' % (fn, md5, info['md5'])) execute_actions(actions, index=index, verbose=verbose) return actions
def clone_env(prefix1, prefix2, verbose=True, quiet=False, fetch_args=None): """ clone existing prefix1 into new prefix2 """ untracked_files = untracked(prefix1) # Discard conda and any package that depends on it drecs = install.linked_data(prefix1) filter = {} found = True while found: found = False for dist, info in iteritems(drecs): name = info['name'] if name in filter: continue if name == 'conda': filter['conda'] = dist found = True break for dep in info.get('depends', []): if MatchSpec(dep).name in filter: filter[name] = dist found = True if not quiet and filter: print('The following packages cannot be cloned out of the root environment:') for pkg in itervalues(filter): print(' - ' + pkg) # Assemble the URL and channel list urls = {} index = {} for dist, info in iteritems(drecs): if info['name'] in filter: continue url = info.get('url') if url is None: sys.exit('Error: no URL found for package: %s' % dist) _, schannel = url_channel(url) index[dist + '.tar.bz2'] = info urls[dist] = url r = Resolve(index) dists = r.dependency_sort(urls.keys()) urls = [urls[d] for d in dists] if verbose: print('Packages: %d' % len(dists)) print('Files: %d' % len(untracked_files)) for f in untracked_files: src = join(prefix1, f) dst = join(prefix2, f) dst_dir = dirname(dst) if islink(dst_dir) or isfile(dst_dir): os.unlink(dst_dir) if not isdir(dst_dir): os.makedirs(dst_dir) if islink(src): os.symlink(os.readlink(src), dst) continue try: with open(src, 'rb') as fi: data = fi.read() except IOError: continue try: s = data.decode('utf-8') s = s.replace(prefix1, prefix2) data = s.encode('utf-8') except UnicodeDecodeError: # data is binary pass with open(dst, 'wb') as fo: fo.write(data) shutil.copystat(src, dst) actions = explicit(urls, prefix2, verbose=not quiet, force_extract=False, fetch_args=fetch_args) return actions, untracked_files
def explicit(specs, prefix, verbose=False, force_extract=True, fetch_args=None): actions = defaultdict(list) actions['PREFIX'] = prefix actions[ 'op_order'] = RM_FETCHED, FETCH, RM_EXTRACTED, EXTRACT, UNLINK, LINK linked = {install.name_dist(dist): dist for dist in install.linked(prefix)} fetch_args = fetch_args or {} index = {} verifies = [] channels = {} for spec in specs: if spec == '@EXPLICIT': continue # Format: (url|path)(:#md5)? m = url_pat.match(spec) if m is None: sys.exit('Could not parse explicit URL: %s' % spec) url, md5 = m.group('url') + '/' + m.group('fn'), m.group('md5') if not is_url(url): if not isfile(url): sys.exit('Error: file not found: %s' % url) url = utils.url_path(url) url_p, fn = url.rsplit('/', 1) # See if the URL refers to a package in our cache prefix = pkg_path = dir_path = None if url_p.startswith('file://'): prefix = install.cached_url(url) # If not, determine the channel name from the URL if prefix is None: _, schannel = url_channel(url) prefix = '' if schannel == 'defaults' else schannel + '::' fn = prefix + fn dist = fn[:-8] pkg_path = install.is_fetched(dist) dir_path = install.is_extracted(dist) # Don't re-fetch unless there is an MD5 mismatch if pkg_path and (md5 and md5_file(pkg_path) != md5): # This removes any extracted copies as well actions[RM_FETCHED].append(dist) pkg_path = dir_path = None # Don't re-extract unless forced, or if we can't check the md5 if dir_path and (force_extract or md5 and not pkg_path): actions[RM_EXTRACTED].append(dist) dir_path = None if not dir_path: if not pkg_path: _, conflict = install.find_new_location(dist) if conflict: actions[RM_FETCHED].append(conflict) channels[url_p + '/'] = (schannel, 0) actions[FETCH].append(dist) verifies.append((dist + '.tar.bz2', md5)) actions[EXTRACT].append(dist) # unlink any installed package with that name name = install.name_dist(dist) if name in linked: actions[UNLINK].append(linked[name]) actions[LINK].append(dist) # Pull the repodata for channels we are using if channels: index.update(fetch_index(channels, **fetch_args)) # Finish the MD5 verification for fn, md5 in verifies: info = index.get(fn) if info is None: sys.exit("Error: no package '%s' in index" % fn) if md5 and 'md5' not in info: sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn) if md5 and info['md5'] != md5: sys.exit('MD5 mismatch for: %s\n spec: %s\n repo: %s' % (fn, md5, info['md5'])) execute_actions(actions, index=index, verbose=verbose) return actions
def clone_env(prefix1, prefix2, verbose=True, quiet=False, fetch_args=None): """ clone existing prefix1 into new prefix2 """ untracked_files = untracked(prefix1) # Discard conda and any package that depends on it drecs = install.linked_data(prefix1) filter = {} found = True while found: found = False for dist, info in iteritems(drecs): name = info['name'] if name in filter: continue if name == 'conda': filter['conda'] = dist found = True break for dep in info.get('depends', []): if MatchSpec(dep).name in filter: filter[name] = dist found = True if not quiet and filter: print( 'The following packages cannot be cloned out of the root environment:' ) for pkg in itervalues(filter): print(' - ' + pkg) # Assemble the URL and channel list urls = {} index = {} for dist, info in iteritems(drecs): if info['name'] in filter: continue url = info.get('url') if url is None: sys.exit('Error: no URL found for package: %s' % dist) _, schannel = url_channel(url) index[dist + '.tar.bz2'] = info urls[dist] = url r = Resolve(index) dists = r.dependency_sort(urls.keys()) urls = [urls[d] for d in dists] if verbose: print('Packages: %d' % len(dists)) print('Files: %d' % len(untracked_files)) for f in untracked_files: src = join(prefix1, f) dst = join(prefix2, f) dst_dir = dirname(dst) if islink(dst_dir) or isfile(dst_dir): os.unlink(dst_dir) if not isdir(dst_dir): os.makedirs(dst_dir) if islink(src): os.symlink(os.readlink(src), dst) continue try: with open(src, 'rb') as fi: data = fi.read() except IOError: continue try: s = data.decode('utf-8') s = s.replace(prefix1, prefix2) data = s.encode('utf-8') except UnicodeDecodeError: # data is binary pass with open(dst, 'wb') as fo: fo.write(data) shutil.copystat(src, dst) actions = explicit(urls, prefix2, verbose=not quiet, force_extract=False, fetch_args=fetch_args) return actions, untracked_files