def old_clone_bundle(path, prefix):
"""
Clone the bundle (located at `path`) by creating a new environment at
`prefix`.
The directory `path` is located in should be some temp directory or
some other directory OUTSITE /opt/anaconda (this function handles
copying the of the file if necessary for you). After calling this
funtion, the original file (at `path`) may be removed.
"""
assert not abspath(path).startswith(abspath(config.root_dir))
assert not isdir(prefix)
fn = basename(path)
assert re.match(r'share-[0-9a-f]{40}-\d+\.tar\.bz2$', fn), fn
dist = fn[:-8]
pkgs_dir = config.pkgs_dirs[0]
if not install.is_extracted(pkgs_dir, dist):
shutil.copyfile(path, join(pkgs_dir, dist + '.tar.bz2'))
inst.execute_instructions([(inst.EXTRACT, (dist,))])
assert install.is_extracted(pkgs_dir, dist)
with open(join(pkgs_dir, dist, 'info', 'index.json')) as fi:
meta = json.load(fi)
# for backwards compatibility, use "requires" when "depends" is not there
dists = ['-'.join(r.split())
for r in meta.get('depends', meta.get('requires', []))
if not r.startswith('conda ')]
dists.append(dist)
actions = plan.ensure_linked_actions(dists, prefix)
index = get_index()
plan.execute_actions(actions, index, verbose=False)
os.unlink(join(prefix, 'conda-meta', dist + '.json'))
def clone_bundle(path, prefix):
"""
Clone the bundle (located at `path`) by creating a new environment at
`prefix`.
The directory `path` is located in should be some temp directory or
some other directory OUTSITE /opt/anaconda (this function handles
copying the of the file if necessary for you). After calling this
funtion, the original file (at `path`) may be removed.
"""
assert not abspath(path).startswith(abspath(config.root_dir))
assert not isdir(prefix)
fn = basename(path)
assert re.match(r'share-[0-9a-f]{40}-\d+\.tar\.bz2$', fn), fn
dist = fn[:-8]
if not install.is_extracted(config.pkgs_dir, dist):
shutil.copyfile(path, join(config.pkgs_dir, dist + '.tar.bz2'))
plan.execute_plan(['%s %s' % (plan.EXTRACT, dist)])
assert install.is_extracted(config.pkgs_dir, dist)
with open(join(config.pkgs_dir, dist, 'info', 'index.json')) as fi:
meta = json.load(fi)
# for backwards compatibility, use "requires" when "depends" is not there
dists = ['-'.join(r.split())
for r in meta.get('depends', meta.get('requires'))
if not r.startswith('conda ')]
dists.append(dist)
actions = plan.ensure_linked_actions(dists, prefix)
index = get_index()
plan.display_actions(actions, index)
plan.execute_actions(actions, index, verbose=True)
os.unlink(join(prefix, 'conda-meta', dist + '.json'))
def ensure_linked_actions(dists, prefix):
actions = defaultdict(list)
actions[PREFIX] = prefix
for dist in dists:
if install.is_linked(prefix, dist):
continue
actions[LINK].append(dist)
if install.is_extracted(config.pkgs_dir, dist):
continue
actions[EXTRACT].append(dist)
if install.is_fetched(config.pkgs_dir, dist):
continue
actions[FETCH].append(dist)
return actions
def _create_env_conda_42(prefix, index, full_list_of_packages):
assert CONDA_VERSION_MAJOR_MINOR < (4, 3)
from conda.install import is_extracted, is_fetched, extract, link
from conda.fetch import fetch_pkg
for tar_name in full_list_of_packages:
pkg_info = index[tar_name]
dist_name = tar_name[:-len('.tar.bz2')]
log.info('Resolved package: {}'.format(tar_name))
# We force a lock on retrieving anything which needs access to a distribution of this
# name. If other requests come in to get the exact same package they will have to wait
# for this to finish (good). If conda itself it fetching these pacakges then there is
# the potential for a race condition (bad) - there is no solution to this unless
# conda/conda is updated to be more precise with its locks.
lock_name = os.path.join(conda_execute.config.pkg_dir, dist_name)
with Locked(lock_name):
if not is_extracted(dist_name):
if not is_fetched(dist_name):
log.info('Fetching {}'.format(dist_name))
fetch_pkg(pkg_info, conda_execute.config.pkg_dir)
extract(dist_name)
link(prefix, dist_name)
def extracted_where(dist):
for pkgs_dir in config.pkgs_dirs:
if install.is_extracted(pkgs_dir, dist):
return pkgs_dir
return None
def extracted_where(dist):
for pkgs_dir in config.pkgs_dirs:
if install.is_extracted(pkgs_dir, dist):
return pkgs_dir
return None
def explicit(specs, prefix, verbose=False, force_extract=True, fetch_args=None):
actions = defaultdict(list)
actions['PREFIX'] = prefix
actions['op_order'] = RM_FETCHED, FETCH, RM_EXTRACTED, EXTRACT, UNLINK, LINK
linked = {install.name_dist(dist): dist for dist in install.linked(prefix)}
fetch_args = fetch_args or {}
index = {}
verifies = []
channels = {}
for spec in specs:
if spec == '@EXPLICIT':
continue
# Format: (url|path)(:#md5)?
m = url_pat.match(spec)
if m is None:
sys.exit('Could not parse explicit URL: %s' % spec)
url, md5 = m.group('url') + '/' + m.group('fn'), m.group('md5')
if not is_url(url):
if not isfile(url):
sys.exit('Error: file not found: %s' % url)
url = utils.url_path(url)
url_p, fn = url.rsplit('/', 1)
# See if the URL refers to a package in our cache
prefix = pkg_path = dir_path = None
if url_p.startswith('file://'):
prefix = install.cached_url(url)
# If not, determine the channel name from the URL
if prefix is None:
_, schannel = url_channel(url)
prefix = '' if schannel == 'defaults' else schannel + '::'
fn = prefix + fn
dist = fn[:-8]
pkg_path = install.is_fetched(dist)
dir_path = install.is_extracted(dist)
# Don't re-fetch unless there is an MD5 mismatch
if pkg_path and (md5 and md5_file(pkg_path) != md5):
# This removes any extracted copies as well
actions[RM_FETCHED].append(dist)
pkg_path = dir_path = None
# Don't re-extract unless forced, or if we can't check the md5
if dir_path and (force_extract or md5 and not pkg_path):
actions[RM_EXTRACTED].append(dist)
dir_path = None
if not dir_path:
if not pkg_path:
_, conflict = install.find_new_location(dist)
if conflict:
actions[RM_FETCHED].append(conflict)
actions[FETCH].append(dist)
if md5:
# Need to verify against the package index
verifies.append((dist + '.tar.bz2', md5))
channels[url_p + '/'] = (schannel, 0)
actions[EXTRACT].append(dist)
# unlink any installed package with that name
name = install.name_dist(dist)
if name in linked:
actions[UNLINK].append(linked[name])
actions[LINK].append(dist)
# Finish the MD5 verification
if verifies:
index = fetch_index(channels, **fetch_args)
for fn, md5 in verifies:
info = index.get(fn)
if info is None:
sys.exit("Error: no package '%s' in index" % fn)
if 'md5' not in info:
sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn)
if info['md5'] != md5:
sys.exit(
'MD5 mismatch for: %s\n spec: %s\n repo: %s'
% (fn, md5, info['md5']))
execute_actions(actions, index=index, verbose=verbose)
return actions
def ensure_linked_actions(dists, prefix, index=None, force=False, always_copy=False):
actions = defaultdict(list)
actions[inst.PREFIX] = prefix
actions['op_order'] = (inst.RM_FETCHED, inst.FETCH, inst.RM_EXTRACTED,
inst.EXTRACT, inst.UNLINK, inst.LINK)
for dist in dists:
fetched_in = install.is_fetched(dist)
extracted_in = install.is_extracted(dist)
if fetched_in and index is not None:
# Test the MD5, and possibly re-fetch
fn = dist + '.tar.bz2'
try:
if md5_file(fetched_in) != index[fn]['md5']:
# RM_FETCHED now removes the extracted data too
actions[inst.RM_FETCHED].append(dist)
# Re-fetch, re-extract, re-link
fetched_in = extracted_in = None
force = True
except KeyError:
sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn)
if not force and install.is_linked(prefix, dist):
continue
if extracted_in and force:
# Always re-extract in the force case
actions[inst.RM_EXTRACTED].append(dist)
extracted_in = None
# Otherwise we need to extract, and possibly fetch
if not extracted_in and not fetched_in:
# If there is a cache conflict, clean it up
fetched_in, conflict = install.find_new_location(dist)
if conflict is not None:
actions[inst.RM_FETCHED].append(conflict)
actions[inst.FETCH].append(dist)
if not extracted_in:
actions[inst.EXTRACT].append(dist)
fetched_dist = extracted_in or fetched_in[:-8]
fetched_dir = dirname(fetched_dist)
try:
# Determine what kind of linking is necessary
if not extracted_in:
# If not already extracted, create some dummy
# data to test with
install.rm_rf(fetched_dist)
ppath = join(fetched_dist, 'info')
os.makedirs(ppath)
index_json = join(ppath, 'index.json')
with open(index_json, 'w'):
pass
if config.always_copy or always_copy:
lt = install.LINK_COPY
elif install.try_hard_link(fetched_dir, prefix, dist):
lt = install.LINK_HARD
elif config.allow_softlinks and sys.platform != 'win32':
lt = install.LINK_SOFT
else:
lt = install.LINK_COPY
actions[inst.LINK].append('%s %d' % (dist, lt))
except (OSError, IOError):
actions[inst.LINK].append(dist)
finally:
if not extracted_in:
# Remove the dummy data
try:
install.rm_rf(fetched_dist)
except (OSError, IOError):
pass
return actions
def explicit(specs,
prefix,
verbose=False,
force_extract=True,
fetch_args=None):
actions = defaultdict(list)
actions['PREFIX'] = prefix
actions[
'op_order'] = RM_FETCHED, FETCH, RM_EXTRACTED, EXTRACT, UNLINK, LINK
linked = {install.name_dist(dist): dist for dist in install.linked(prefix)}
fetch_args = fetch_args or {}
index = {}
verifies = []
channels = {}
for spec in specs:
if spec == '@EXPLICIT':
continue
# Format: (url|path)(:#md5)?
m = url_pat.match(spec)
if m is None:
sys.exit('Could not parse explicit URL: %s' % spec)
url, md5 = m.group('url') + '/' + m.group('fn'), m.group('md5')
if not is_url(url):
if not isfile(url):
sys.exit('Error: file not found: %s' % url)
url = utils.url_path(url)
url_p, fn = url.rsplit('/', 1)
# See if the URL refers to a package in our cache
prefix = pkg_path = dir_path = None
if url_p.startswith('file://'):
prefix = install.cached_url(url)
# If not, determine the channel name from the URL
if prefix is None:
_, schannel = url_channel(url)
prefix = '' if schannel == 'defaults' else schannel + '::'
fn = prefix + fn
dist = fn[:-8]
pkg_path = install.is_fetched(dist)
dir_path = install.is_extracted(dist)
# Don't re-fetch unless there is an MD5 mismatch
if pkg_path and (md5 and md5_file(pkg_path) != md5):
# This removes any extracted copies as well
actions[RM_FETCHED].append(dist)
pkg_path = dir_path = None
# Don't re-extract unless forced, or if we can't check the md5
if dir_path and (force_extract or md5 and not pkg_path):
actions[RM_EXTRACTED].append(dist)
dir_path = None
if not dir_path:
if not pkg_path:
_, conflict = install.find_new_location(dist)
if conflict:
actions[RM_FETCHED].append(conflict)
channels[url_p + '/'] = (schannel, 0)
actions[FETCH].append(dist)
verifies.append((dist + '.tar.bz2', md5))
actions[EXTRACT].append(dist)
# unlink any installed package with that name
name = install.name_dist(dist)
if name in linked:
actions[UNLINK].append(linked[name])
actions[LINK].append(dist)
# Pull the repodata for channels we are using
if channels:
index.update(fetch_index(channels, **fetch_args))
# Finish the MD5 verification
for fn, md5 in verifies:
info = index.get(fn)
if info is None:
sys.exit("Error: no package '%s' in index" % fn)
if md5 and 'md5' not in info:
sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn)
if md5 and info['md5'] != md5:
sys.exit('MD5 mismatch for: %s\n spec: %s\n repo: %s' %
(fn, md5, info['md5']))
execute_actions(actions, index=index, verbose=verbose)
return actions
def EXTRACT_CMD(state, arg):
if not install.is_extracted(config.pkgs_dirs[0], arg):
install.extract(config.pkgs_dirs[0], arg)
def EXTRACT_CMD(state, arg):
if not install.is_extracted(arg):
install.extract(arg)
def EXTRACT_CMD(state, arg):
if not install.is_extracted(config.pkgs_dirs[0], arg):
install.extract(config.pkgs_dirs[0], arg)
dist_tar = os.path.join(src_distro_dir,
conda.config.subdir,
dist) + '.tar.bz2'
if not os.path.exists(dist_tar):
raise IOError('Could not find {} at ({})\n'
'It may be that the content is out of synch. '
'Have you run a build of this environment?\n'
"One of the designs of the conda-manager is "
'that a MANIFEST does not guarantee that all '
'of the distributions \nhave come from the existing '
"recipes (particularly if a recipe's version has "
"been decreased).".format(dist, dist_tar))
src_pkgs = os.path.join(pkgs_dir, source_name)
if not os.path.exists(src_pkgs):
os.makedirs(src_pkgs)
if not cinstall.is_extracted(src_pkgs, dist):
placed_dist_file = os.path.join(src_pkgs, dist + '.tar.bz2')
if not os.path.exists(placed_dist_file):
import shutil
shutil.copy(dist_tar, placed_dist_file)
cinstall.extract(src_pkgs, dist)
# Tidy up immediately by removing the distribution.
os.remove(placed_dist_file)
# Remove any packages which are no longer needed.
for dist in ci.linked(prefix):
if dist not in for_installation:
cinstall.unlink(prefix, dist)
# Install the packages.
for source_name, packages in packages_by_source.items():
def ensure_linked_actions(dists,
prefix,
index=None,
force=False,
always_copy=False):
actions = defaultdict(list)
actions[inst.PREFIX] = prefix
actions['op_order'] = (inst.RM_FETCHED, inst.FETCH, inst.RM_EXTRACTED,
inst.EXTRACT, inst.UNLINK, inst.LINK)
for dist in dists:
fetched_in = install.is_fetched(dist)
extracted_in = install.is_extracted(dist)
if fetched_in and index is not None:
# Test the MD5, and possibly re-fetch
fn = dist + '.tar.bz2'
try:
if md5_file(fetched_in) != index[fn]['md5']:
# RM_FETCHED now removes the extracted data too
actions[inst.RM_FETCHED].append(dist)
# Re-fetch, re-extract, re-link
fetched_in = extracted_in = None
force = True
except KeyError:
sys.stderr.write('Warning: cannot lookup MD5 of: %s' % fn)
if not force and install.is_linked(prefix, dist):
continue
if extracted_in and force:
# Always re-extract in the force case
actions[inst.RM_EXTRACTED].append(dist)
extracted_in = None
# Otherwise we need to extract, and possibly fetch
if not extracted_in and not fetched_in:
# If there is a cache conflict, clean it up
fetched_in, conflict = install.find_new_location(dist)
fetched_in = join(fetched_in, install._dist2filename(dist))
if conflict is not None:
actions[inst.RM_FETCHED].append(conflict)
actions[inst.FETCH].append(dist)
if not extracted_in:
actions[inst.EXTRACT].append(dist)
fetched_dist = extracted_in or fetched_in[:-8]
fetched_dir = dirname(fetched_dist)
try:
# Determine what kind of linking is necessary
if not extracted_in:
# If not already extracted, create some dummy
# data to test with
install.rm_rf(fetched_dist)
ppath = join(fetched_dist, 'info')
os.makedirs(ppath)
index_json = join(ppath, 'index.json')
with open(index_json, 'w'):
pass
if config_always_copy or always_copy:
lt = install.LINK_COPY
elif install.try_hard_link(fetched_dir, prefix, dist):
lt = install.LINK_HARD
elif allow_softlinks and sys.platform != 'win32':
lt = install.LINK_SOFT
else:
lt = install.LINK_COPY
actions[inst.LINK].append('%s %d' % (dist, lt))
except (OSError, IOError):
actions[inst.LINK].append(dist)
finally:
if not extracted_in:
# Remove the dummy data
try:
install.rm_rf(fetched_dist)
except (OSError, IOError):
pass
return actions