def old_clone_bundle(path, prefix): """ Clone the bundle (located at `path`) by creating a new environment at `prefix`. The directory `path` is located in should be some temp directory or some other directory OUTSITE /opt/anaconda (this function handles copying the of the file if necessary for you). After calling this funtion, the original file (at `path`) may be removed. """ assert not abspath(path).startswith(abspath(config.root_dir)) assert not isdir(prefix) fn = basename(path) assert re.match(r'share-[0-9a-f]{40}-\d+\.tar\.bz2$', fn), fn dist = fn[:-8] pkgs_dir = config.pkgs_dirs[0] if not install.is_extracted(pkgs_dir, dist): shutil.copyfile(path, join(pkgs_dir, dist + '.tar.bz2')) inst.execute_instructions([(inst.EXTRACT, (dist,))]) assert install.is_extracted(pkgs_dir, dist) with open(join(pkgs_dir, dist, 'info', 'index.json')) as fi: meta = json.load(fi) # for backwards compatibility, use "requires" when "depends" is not there dists = ['-'.join(r.split()) for r in meta.get('depends', meta.get('requires', [])) if not r.startswith('conda ')] dists.append(dist) actions = plan.ensure_linked_actions(dists, prefix) index = get_index() plan.execute_actions(actions, index, verbose=False) os.unlink(join(prefix, 'conda-meta', dist + '.json'))
def clone_bundle(path, prefix): """ Clone the bundle (located at `path`) by creating a new environment at `prefix`. The directory `path` is located in should be some temp directory or some other directory OUTSITE /opt/anaconda (this function handles copying the of the file if necessary for you). After calling this funtion, the original file (at `path`) may be removed. """ assert not abspath(path).startswith(abspath(config.root_dir)) assert not isdir(prefix) fn = basename(path) assert re.match(r'share-[0-9a-f]{40}-\d+\.tar\.bz2$', fn), fn dist = fn[:-8] if not install.is_extracted(config.pkgs_dir, dist): shutil.copyfile(path, join(config.pkgs_dir, dist + '.tar.bz2')) plan.execute_plan(['%s %s' % (plan.EXTRACT, dist)]) assert install.is_extracted(config.pkgs_dir, dist) with open(join(config.pkgs_dir, dist, 'info', 'index.json')) as fi: meta = json.load(fi) # for backwards compatibility, use "requires" when "depends" is not there dists = ['-'.join(r.split()) for r in meta.get('depends', meta.get('requires')) if not r.startswith('conda ')] dists.append(dist) actions = plan.ensure_linked_actions(dists, prefix) index = get_index() plan.display_actions(actions, index) plan.execute_actions(actions, index, verbose=True) os.unlink(join(prefix, 'conda-meta', dist + '.json'))
def ensure_linked_actions(dists, prefix): actions = defaultdict(list) actions[PREFIX] = prefix for dist in dists: if install.is_linked(prefix, dist): continue actions[LINK].append(dist) if install.is_extracted(config.pkgs_dir, dist): continue actions[EXTRACT].append(dist) if install.is_fetched(config.pkgs_dir, dist): continue actions[FETCH].append(dist) return actions
def _create_env_conda_42(prefix, index, full_list_of_packages): assert CONDA_VERSION_MAJOR_MINOR < (4, 3) from conda.install import is_extracted, is_fetched, extract, link from conda.fetch import fetch_pkg for tar_name in full_list_of_packages: pkg_info = index[tar_name] dist_name = tar_name[:-len('.tar.bz2')] log.info('Resolved package: {}'.format(tar_name)) # We force a lock on retrieving anything which needs access to a distribution of this # name. If other requests come in to get the exact same package they will have to wait # for this to finish (good). If conda itself it fetching these pacakges then there is # the potential for a race condition (bad) - there is no solution to this unless # conda/conda is updated to be more precise with its locks. lock_name = os.path.join(conda_execute.config.pkg_dir, dist_name) with Locked(lock_name): if not is_extracted(dist_name): if not is_fetched(dist_name): log.info('Fetching {}'.format(dist_name)) fetch_pkg(pkg_info, conda_execute.config.pkg_dir) extract(dist_name) link(prefix, dist_name)
def extracted_where(dist): for pkgs_dir in config.pkgs_dirs: if install.is_extracted(pkgs_dir, dist): return pkgs_dir return None
def explicit(specs, prefix, verbose=False, force_extract=True, fetch_args=None): actions = defaultdict(list) actions['PREFIX'] = prefix actions['op_order'] = RM_FETCHED, FETCH, RM_EXTRACTED, EXTRACT, UNLINK, LINK linked = {install.name_dist(dist): dist for dist in install.linked(prefix)} fetch_args = fetch_args or {} index = {} verifies = [] channels = {} for spec in specs: if spec == '@EXPLICIT': continue # Format: (url|path)(:#md5)? m = url_pat.match(spec) if m is None: sys.exit('Could not parse explicit URL: %s' % spec) url, md5 = m.group('url') + '/' + m.group('fn'), m.group('md5') if not is_url(url): if not isfile(url): sys.exit('Error: file not found: %s' % url) url = utils.url_path(url) url_p, fn = url.rsplit('/', 1) # See if the URL refers to a package in our cache prefix = pkg_path = dir_path = None if url_p.startswith('file://'): prefix = install.cached_url(url) # If not, determine the channel name from the URL if prefix is None: _, schannel = url_channel(url) prefix = '' if schannel == 'defaults' else schannel + '::' fn = prefix + fn dist = fn[:-8] pkg_path = install.is_fetched(dist) dir_path = install.is_extracted(dist) # Don't re-fetch unless there is an MD5 mismatch if pkg_path and (md5 and md5_file(pkg_path) != md5): # This removes any extracted copies as well actions[RM_FETCHED].append(dist) pkg_path = dir_path = None # Don't re-extract unless forced, or if we can't check the md5 if dir_path and (force_extract or md5 and not pkg_path): actions[RM_EXTRACTED].append(dist) dir_path = None if not dir_path: if not pkg_path: _, conflict = install.find_new_location(dist) if conflict: actions[RM_FETCHED].append(conflict) actions[FETCH].append(dist) if md5: # Need to verify against the package index verifies.append((dist + '.tar.bz2', md5)) channels[url_p + '/'] = (schannel, 0) actions[EXTRACT].append(dist) # unlink any installed package with that name name = install.name_dist(dist) if name in linked: actions[UNLINK].append(linked[name]) actions[LINK].append(dist) # Finish the MD5 verification if verifies: index = fetch_index(channels, **fetch_args) for fn, md5 in verifies: info = index.get(fn) if info is None: sys.exit("Error: no package '%s' in index" % fn) if 'md5' not in info: sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn) if info['md5'] != md5: sys.exit( 'MD5 mismatch for: %s\n spec: %s\n repo: %s' % (fn, md5, info['md5'])) execute_actions(actions, index=index, verbose=verbose) return actions
def ensure_linked_actions(dists, prefix, index=None, force=False, always_copy=False): actions = defaultdict(list) actions[inst.PREFIX] = prefix actions['op_order'] = (inst.RM_FETCHED, inst.FETCH, inst.RM_EXTRACTED, inst.EXTRACT, inst.UNLINK, inst.LINK) for dist in dists: fetched_in = install.is_fetched(dist) extracted_in = install.is_extracted(dist) if fetched_in and index is not None: # Test the MD5, and possibly re-fetch fn = dist + '.tar.bz2' try: if md5_file(fetched_in) != index[fn]['md5']: # RM_FETCHED now removes the extracted data too actions[inst.RM_FETCHED].append(dist) # Re-fetch, re-extract, re-link fetched_in = extracted_in = None force = True except KeyError: sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn) if not force and install.is_linked(prefix, dist): continue if extracted_in and force: # Always re-extract in the force case actions[inst.RM_EXTRACTED].append(dist) extracted_in = None # Otherwise we need to extract, and possibly fetch if not extracted_in and not fetched_in: # If there is a cache conflict, clean it up fetched_in, conflict = install.find_new_location(dist) if conflict is not None: actions[inst.RM_FETCHED].append(conflict) actions[inst.FETCH].append(dist) if not extracted_in: actions[inst.EXTRACT].append(dist) fetched_dist = extracted_in or fetched_in[:-8] fetched_dir = dirname(fetched_dist) try: # Determine what kind of linking is necessary if not extracted_in: # If not already extracted, create some dummy # data to test with install.rm_rf(fetched_dist) ppath = join(fetched_dist, 'info') os.makedirs(ppath) index_json = join(ppath, 'index.json') with open(index_json, 'w'): pass if config.always_copy or always_copy: lt = install.LINK_COPY elif install.try_hard_link(fetched_dir, prefix, dist): lt = install.LINK_HARD elif config.allow_softlinks and sys.platform != 'win32': lt = install.LINK_SOFT else: lt = install.LINK_COPY actions[inst.LINK].append('%s %d' % (dist, lt)) except (OSError, IOError): actions[inst.LINK].append(dist) finally: if not extracted_in: # Remove the dummy data try: install.rm_rf(fetched_dist) except (OSError, IOError): pass return actions
def explicit(specs, prefix, verbose=False, force_extract=True, fetch_args=None): actions = defaultdict(list) actions['PREFIX'] = prefix actions[ 'op_order'] = RM_FETCHED, FETCH, RM_EXTRACTED, EXTRACT, UNLINK, LINK linked = {install.name_dist(dist): dist for dist in install.linked(prefix)} fetch_args = fetch_args or {} index = {} verifies = [] channels = {} for spec in specs: if spec == '@EXPLICIT': continue # Format: (url|path)(:#md5)? m = url_pat.match(spec) if m is None: sys.exit('Could not parse explicit URL: %s' % spec) url, md5 = m.group('url') + '/' + m.group('fn'), m.group('md5') if not is_url(url): if not isfile(url): sys.exit('Error: file not found: %s' % url) url = utils.url_path(url) url_p, fn = url.rsplit('/', 1) # See if the URL refers to a package in our cache prefix = pkg_path = dir_path = None if url_p.startswith('file://'): prefix = install.cached_url(url) # If not, determine the channel name from the URL if prefix is None: _, schannel = url_channel(url) prefix = '' if schannel == 'defaults' else schannel + '::' fn = prefix + fn dist = fn[:-8] pkg_path = install.is_fetched(dist) dir_path = install.is_extracted(dist) # Don't re-fetch unless there is an MD5 mismatch if pkg_path and (md5 and md5_file(pkg_path) != md5): # This removes any extracted copies as well actions[RM_FETCHED].append(dist) pkg_path = dir_path = None # Don't re-extract unless forced, or if we can't check the md5 if dir_path and (force_extract or md5 and not pkg_path): actions[RM_EXTRACTED].append(dist) dir_path = None if not dir_path: if not pkg_path: _, conflict = install.find_new_location(dist) if conflict: actions[RM_FETCHED].append(conflict) channels[url_p + '/'] = (schannel, 0) actions[FETCH].append(dist) verifies.append((dist + '.tar.bz2', md5)) actions[EXTRACT].append(dist) # unlink any installed package with that name name = install.name_dist(dist) if name in linked: actions[UNLINK].append(linked[name]) actions[LINK].append(dist) # Pull the repodata for channels we are using if channels: index.update(fetch_index(channels, **fetch_args)) # Finish the MD5 verification for fn, md5 in verifies: info = index.get(fn) if info is None: sys.exit("Error: no package '%s' in index" % fn) if md5 and 'md5' not in info: sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn) if md5 and info['md5'] != md5: sys.exit('MD5 mismatch for: %s\n spec: %s\n repo: %s' % (fn, md5, info['md5'])) execute_actions(actions, index=index, verbose=verbose) return actions
def EXTRACT_CMD(state, arg): if not install.is_extracted(config.pkgs_dirs[0], arg): install.extract(config.pkgs_dirs[0], arg)
def EXTRACT_CMD(state, arg): if not install.is_extracted(arg): install.extract(arg)
dist_tar = os.path.join(src_distro_dir, conda.config.subdir, dist) + '.tar.bz2' if not os.path.exists(dist_tar): raise IOError('Could not find {} at ({})\n' 'It may be that the content is out of synch. ' 'Have you run a build of this environment?\n' "One of the designs of the conda-manager is " 'that a MANIFEST does not guarantee that all ' 'of the distributions \nhave come from the existing ' "recipes (particularly if a recipe's version has " "been decreased).".format(dist, dist_tar)) src_pkgs = os.path.join(pkgs_dir, source_name) if not os.path.exists(src_pkgs): os.makedirs(src_pkgs) if not cinstall.is_extracted(src_pkgs, dist): placed_dist_file = os.path.join(src_pkgs, dist + '.tar.bz2') if not os.path.exists(placed_dist_file): import shutil shutil.copy(dist_tar, placed_dist_file) cinstall.extract(src_pkgs, dist) # Tidy up immediately by removing the distribution. os.remove(placed_dist_file) # Remove any packages which are no longer needed. for dist in ci.linked(prefix): if dist not in for_installation: cinstall.unlink(prefix, dist) # Install the packages. for source_name, packages in packages_by_source.items():
def ensure_linked_actions(dists, prefix, index=None, force=False, always_copy=False): actions = defaultdict(list) actions[inst.PREFIX] = prefix actions['op_order'] = (inst.RM_FETCHED, inst.FETCH, inst.RM_EXTRACTED, inst.EXTRACT, inst.UNLINK, inst.LINK) for dist in dists: fetched_in = install.is_fetched(dist) extracted_in = install.is_extracted(dist) if fetched_in and index is not None: # Test the MD5, and possibly re-fetch fn = dist + '.tar.bz2' try: if md5_file(fetched_in) != index[fn]['md5']: # RM_FETCHED now removes the extracted data too actions[inst.RM_FETCHED].append(dist) # Re-fetch, re-extract, re-link fetched_in = extracted_in = None force = True except KeyError: sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn) if not force and install.is_linked(prefix, dist): continue if extracted_in and force: # Always re-extract in the force case actions[inst.RM_EXTRACTED].append(dist) extracted_in = None # Otherwise we need to extract, and possibly fetch if not extracted_in and not fetched_in: # If there is a cache conflict, clean it up fetched_in, conflict = install.find_new_location(dist) fetched_in = join(fetched_in, install._dist2filename(dist)) if conflict is not None: actions[inst.RM_FETCHED].append(conflict) actions[inst.FETCH].append(dist) if not extracted_in: actions[inst.EXTRACT].append(dist) fetched_dist = extracted_in or fetched_in[:-8] fetched_dir = dirname(fetched_dist) try: # Determine what kind of linking is necessary if not extracted_in: # If not already extracted, create some dummy # data to test with install.rm_rf(fetched_dist) ppath = join(fetched_dist, 'info') os.makedirs(ppath) index_json = join(ppath, 'index.json') with open(index_json, 'w'): pass if config_always_copy or always_copy: lt = install.LINK_COPY elif install.try_hard_link(fetched_dir, prefix, dist): lt = install.LINK_HARD elif allow_softlinks and sys.platform != 'win32': lt = install.LINK_SOFT else: lt = install.LINK_COPY actions[inst.LINK].append('%s %d' % (dist, lt)) except (OSError, IOError): actions[inst.LINK].append(dist) finally: if not extracted_in: # Remove the dummy data try: install.rm_rf(fetched_dist) except (OSError, IOError): pass return actions