def test_fetchrepodata_httperror(self):
with pytest.raises(CondaHTTPError) as execinfo:
url = DEFAULT_CHANNEL_ALIAS
user = remove_binstar_tokens(url).split(DEFAULT_CHANNEL_ALIAS)[1].split("/")[0]
msg = 'Could not find anaconda.org user %s' % user
filename = 'repodata.json'
responses.add(responses.GET, url+filename, body='{"error": "not found"}', status=404,
content_type='application/json')
fetch_repodata(url)
assert msg in str(execinfo), str(execinfo)
def add_cached_package(pdir, url, overwrite=False, urlstxt=False):
"""
Adds a new package to the cache. The URL is used to determine the
package filename and channel, and the directory pdir is scanned for
both a compressed and an extracted version of that package. If
urlstxt=True, this URL will be appended to the urls.txt file in the
cache, so that subsequent runs will correctly identify the package.
"""
package_cache()
if '/' in url:
dist = url.rsplit('/', 1)[-1]
else:
dist = url
url = None
if dist.endswith('.tar.bz2'):
fname = dist
dist = dist[:-8]
else:
fname = dist + '.tar.bz2'
xpkg = join(pdir, fname)
if not overwrite and xpkg in fname_table_:
return
if not isfile(xpkg):
xpkg = None
xdir = join(pdir, dist)
if not (isdir(xdir) and
isfile(join(xdir, 'info', 'files')) and
isfile(join(xdir, 'info', 'index.json'))):
xdir = None
if not (xpkg or xdir):
return
if url:
url = remove_binstar_tokens(url)
_, schannel = Channel(url).url_channel_wtf
prefix = '' if schannel == 'defaults' else schannel + '::'
xkey = xpkg or (xdir + '.tar.bz2')
fname_table_[xkey] = fname_table_[path_to_url(xkey)] = prefix
fkey = prefix + dist
rec = package_cache_.get(fkey)
if rec is None:
rec = package_cache_[fkey] = dict(files=[], dirs=[], urls=[])
if url and url not in rec['urls']:
rec['urls'].append(url)
if xpkg and xpkg not in rec['files']:
rec['files'].append(xpkg)
if xdir and xdir not in rec['dirs']:
rec['dirs'].append(xdir)
if urlstxt:
try:
with open(join(pdir, 'urls.txt'), 'a') as fa:
fa.write('%s\n' % url)
except IOError:
pass
def add_cached_package(pdir, url, overwrite=False, urlstxt=False):
"""
Adds a new package to the cache. The URL is used to determine the
package filename and channel, and the directory pdir is scanned for
both a compressed and an extracted version of that package. If
urlstxt=True, this URL will be appended to the urls.txt file in the
cache, so that subsequent runs will correctly identify the package.
"""
package_cache()
if '/' in url:
dist = url.rsplit('/', 1)[-1]
else:
dist = url
url = None
if dist.endswith('.tar.bz2'):
fname = dist
dist = dist[:-8]
else:
fname = dist + '.tar.bz2'
xpkg = join(pdir, fname)
if not overwrite and xpkg in fname_table_:
return
if not isfile(xpkg):
xpkg = None
xdir = join(pdir, dist)
if not (isdir(xdir) and
isfile(join(xdir, 'info', 'files')) and
isfile(join(xdir, 'info', 'index.json'))):
xdir = None
if not (xpkg or xdir):
return
if url:
url = remove_binstar_tokens(url)
_, schannel = url_channel(url)
prefix = '' if schannel == 'defaults' else schannel + '::'
xkey = xpkg or (xdir + '.tar.bz2')
fname_table_[xkey] = fname_table_[url_path(xkey)] = prefix
fkey = prefix + dist
rec = package_cache_.get(fkey)
if rec is None:
rec = package_cache_[fkey] = dict(files=[], dirs=[], urls=[])
if url and url not in rec['urls']:
rec['urls'].append(url)
if xpkg and xpkg not in rec['files']:
rec['files'].append(xpkg)
if xdir and xdir not in rec['dirs']:
rec['dirs'].append(xdir)
if urlstxt:
try:
with open(join(pdir, 'urls.txt'), 'a') as fa:
fa.write('%s\n' % url)
except IOError:
pass
def add_cached_package(pdir, url, overwrite=False, urlstxt=False):
"""
Adds a new package to the cache. The URL is used to determine the
package filename and channel, and the directory pdir is scanned for
both a compressed and an extracted version of that package. If
urlstxt=True, this URL will be appended to the urls.txt file in the
cache, so that subsequent runs will correctly identify the package.
"""
package_cache()
if "/" in url:
dist = url.rsplit("/", 1)[-1]
else:
dist = url
url = None
if dist.endswith(".tar.bz2"):
fname = dist
dist = dist[:-8]
else:
fname = dist + ".tar.bz2"
xpkg = join(pdir, fname)
if not overwrite and xpkg in fname_table_:
return
if not isfile(xpkg):
xpkg = None
xdir = join(pdir, dist)
if not (isdir(xdir) and isfile(join(xdir, "info", "files")) and isfile(join(xdir, "info", "index.json"))):
xdir = None
if not (xpkg or xdir):
return
if url:
url = remove_binstar_tokens(url)
_, schannel = url_channel(url)
prefix = "" if schannel == "defaults" else schannel + "::"
xkey = xpkg or (xdir + ".tar.bz2")
fname_table_[xkey] = fname_table_[path_to_url(xkey)] = prefix
fkey = prefix + dist
rec = package_cache_.get(fkey)
if rec is None:
rec = package_cache_[fkey] = dict(files=[], dirs=[], urls=[])
if url and url not in rec["urls"]:
rec["urls"].append(url)
if xpkg and xpkg not in rec["files"]:
rec["files"].append(xpkg)
if xdir and xdir not in rec["dirs"]:
rec["dirs"].append(xdir)
if urlstxt:
try:
with open(join(pdir, "urls.txt"), "a") as fa:
fa.write("%s\n" % url)
except IOError:
pass
def link(pkgs_dir, prefix, dist, linktype=LINK_HARD, index=None):
'''
Set up a package in a specified (environment) prefix. We assume that
the package has been extracted (using extract() above).
'''
index = index or {}
log.debug('pkgs_dir=%r, prefix=%r, dist=%r, linktype=%r' %
(pkgs_dir, prefix, dist, linktype))
if (on_win and abspath(prefix) == abspath(sys.prefix) and
name_dist(dist) in win_ignore_root):
# on Windows we have the file lock problem, so don't allow
# linking or unlinking some packages
log.warn('Ignored: %s' % dist)
return
source_dir = join(pkgs_dir, dist)
if not run_script(source_dir, dist, 'pre-link', prefix):
sys.exit('Error: pre-link failed: %s' % dist)
info_dir = join(source_dir, 'info')
files = list(yield_lines(join(info_dir, 'files')))
has_prefix_files = read_has_prefix(join(info_dir, 'has_prefix'))
no_link = read_no_link(info_dir)
with Locked(prefix), Locked(pkgs_dir):
for f in files:
src = join(source_dir, f)
dst = join(prefix, f)
dst_dir = dirname(dst)
if not isdir(dst_dir):
os.makedirs(dst_dir)
if os.path.exists(dst):
log.warn("file already exists: %r" % dst)
try:
os.unlink(dst)
except OSError:
log.error('failed to unlink: %r' % dst)
lt = linktype
if f in has_prefix_files or f in no_link or islink(src):
lt = LINK_COPY
try:
_link(src, dst, lt)
except OSError as e:
log.error('failed to link (src=%r, dst=%r, type=%r, error=%r)' %
(src, dst, lt, e))
if name_dist(dist) == '_cache':
return
for f in sorted(has_prefix_files):
placeholder, mode = has_prefix_files[f]
try:
update_prefix(join(prefix, f), prefix, placeholder, mode)
except PaddingError:
sys.exit("ERROR: placeholder '%s' too short in: %s\n" %
(placeholder, dist))
mk_menus(prefix, files, remove=False)
if not run_script(prefix, dist, 'post-link'):
sys.exit("Error: post-link failed for: %s" % dist)
# Make sure the script stays standalone for the installer
try:
from conda.config import remove_binstar_tokens
except ImportError:
# There won't be any binstar tokens in the installer anyway
def remove_binstar_tokens(url):
return url
meta_dict = index.get(dist + '.tar.bz2', {})
meta_dict['url'] = read_url(pkgs_dir, dist)
if meta_dict['url']:
meta_dict['url'] = remove_binstar_tokens(meta_dict['url'])
meta_dict['files'] = files
meta_dict['link'] = {'source': source_dir,
'type': link_name_map.get(linktype)}
if 'channel' in meta_dict:
meta_dict['channel'] = remove_binstar_tokens(meta_dict['channel'])
create_meta(prefix, dist, info_dir, meta_dict)
def fetch_repodata(url, cache_dir=None, use_cache=False, session=None):
if not config.ssl_verify:
try:
from requests.packages.urllib3.connectionpool import InsecureRequestWarning
except ImportError:
pass
else:
warnings.simplefilter('ignore', InsecureRequestWarning)
session = session or CondaSession()
cache_path = join(cache_dir or create_cache_dir(), cache_fn_url(url))
try:
cache = json.load(open(cache_path))
except (IOError, ValueError):
cache = {'packages': {}}
if use_cache:
return cache
headers = {}
if "_tag" in cache:
headers["If-None-Match"] = cache["_etag"]
if "_mod" in cache:
headers["If-Modified-Since"] = cache["_mod"]
try:
resp = session.get(url + 'repodata.json.bz2',
headers=headers, proxies=session.proxies,
verify=config.ssl_verify)
resp.raise_for_status()
if resp.status_code != 304:
cache = json.loads(bz2.decompress(resp.content).decode('utf-8'))
except ValueError as e:
raise RuntimeError("Invalid index file: %srepodata.json.bz2: %s" %
(config.remove_binstar_tokens(url), e))
except requests.exceptions.HTTPError as e:
if e.response.status_code == 407: # Proxy Authentication Required
handle_proxy_407(url, session)
# Try again
return fetch_repodata(url, cache_dir=cache_dir,
use_cache=use_cache, session=session)
if e.response.status_code == 404:
if url.startswith(config.DEFAULT_CHANNEL_ALIAS):
msg = ('Could not find Binstar user %s' %
config.remove_binstar_tokens(url).split(config.DEFAULT_CHANNEL_ALIAS)[1].split('/')[0])
else:
msg = 'Could not find URL: %s' % config.remove_binstar_tokens(url)
elif (e.response.status_code == 401 and config.rc.get('channel_alias',
config.DEFAULT_CHANNEL_ALIAS) in url):
# Note, this will not trigger if the binstar configured url does
# not match the conda configured one.
msg = ("Warning: you may need to login to binstar again with "
"'binstar login' to access private packages(%s, %s)" %
(config.hide_binstar_tokens(url), e))
stderrlog.info(msg)
return fetch_repodata(config.remove_binstar_tokens(url), cache_dir=cache_dir, use_cache=use_cache, session=session)
else:
msg = "HTTPError: %s: %s\n" % (e, config.remove_binstar_tokens(url))
log.debug(msg)
raise RuntimeError(msg)
except requests.exceptions.ConnectionError as e:
# requests isn't so nice here. For whatever reason, https gives this
# error and http gives the above error. Also, there is no status_code
# attribute here. We have to just check if it looks like 407. See
# https://github.com/kennethreitz/requests/issues/2061.
if "407" in str(e): # Proxy Authentication Required
handle_proxy_407(url, session)
# Try again
return fetch_repodata(url, cache_dir=cache_dir,
use_cache=use_cache, session=session)
msg = "Connection error: %s: %s\n" % (e, config.remove_binstar_tokens(url))
stderrlog.info('Could not connect to %s\n' % config.remove_binstar_tokens(url))
log.debug(msg)
if fail_unknown_host:
raise RuntimeError(msg)
cache['_url'] = config.remove_binstar_tokens(url)
try:
with open(cache_path, 'w') as fo:
json.dump(cache, fo, indent=2, sort_keys=True)
except IOError:
pass
return cache or None
def fetch_repodata(url, cache_dir=None, use_cache=False, session=None):
if not config.ssl_verify:
try:
from requests.packages.urllib3.connectionpool import InsecureRequestWarning
except ImportError:
pass
else:
warnings.simplefilter('ignore', InsecureRequestWarning)
session = session or CondaSession()
cache_path = join(cache_dir or create_cache_dir(), cache_fn_url(url))
try:
cache = json.load(open(cache_path))
except (IOError, ValueError):
cache = {'packages': {}}
if use_cache:
return cache
headers = {}
if "_tag" in cache:
headers["If-None-Match"] = cache["_etag"]
if "_mod" in cache:
headers["If-Modified-Since"] = cache["_mod"]
try:
resp = session.get(url + 'repodata.json.bz2',
headers=headers, proxies=session.proxies,
verify=config.ssl_verify)
resp.raise_for_status()
if resp.status_code != 304:
cache = json.loads(bz2.decompress(resp.content).decode('utf-8'))
except ValueError as e:
raise RuntimeError("Invalid index file: %srepodata.json.bz2: %s" %
(config.remove_binstar_tokens(url), e))
except requests.exceptions.HTTPError as e:
if e.response.status_code == 407: # Proxy Authentication Required
handle_proxy_407(url, session)
# Try again
return fetch_repodata(url, cache_dir=cache_dir,
use_cache=use_cache, session=session)
if e.response.status_code == 404:
if url.startswith(config.DEFAULT_CHANNEL_ALIAS):
msg = ('Could not find Binstar user %s' %
config.remove_binstar_tokens(url).split(config.DEFAULT_CHANNEL_ALIAS)[1].split('/')[0])
else:
msg = 'Could not find URL: %s' % config.remove_binstar_tokens(url)
elif (e.response.status_code == 401 and config.rc.get('channel_alias',
config.DEFAULT_CHANNEL_ALIAS) in url):
# Note, this will not trigger if the binstar configured url does
# not match the conda configured one.
msg = ("Warning: you may need to login to binstar again with "
"'binstar login' to access private packages(%s, %s)" %
(config.hide_binstar_tokens(url), e))
stderrlog.info(msg)
return fetch_repodata(config.remove_binstar_tokens(url), cache_dir=cache_dir, use_cache=use_cache, session=session)
else:
msg = "HTTPError: %s: %s\n" % (e, config.remove_binstar_tokens(url))
log.debug(msg)
raise RuntimeError(msg)
except requests.exceptions.ConnectionError as e:
# requests isn't so nice here. For whatever reason, https gives this
# error and http gives the above error. Also, there is no status_code
# attribute here. We have to just check if it looks like 407. See
# https://github.com/kennethreitz/requests/issues/2061.
if "407" in str(e): # Proxy Authentication Required
handle_proxy_407(url, session)
# Try again
return fetch_repodata(url, cache_dir=cache_dir,
use_cache=use_cache, session=session)
msg = "Connection error: %s: %s\n" % (e, config.remove_binstar_tokens(url))
stderrlog.info('Could not connect to %s\n' % config.remove_binstar_tokens(url))
log.debug(msg)
if fail_unknown_host:
raise RuntimeError(msg)
cache['_url'] = config.remove_binstar_tokens(url)
try:
with open(cache_path, 'w') as fo:
json.dump(cache, fo, indent=2, sort_keys=True)
except IOError:
pass
return cache or None
def link(pkgs_dir, prefix, dist, linktype=LINK_HARD, index=None):
'''
Set up a package in a specified (environment) prefix. We assume that
the package has been extracted (using extract() above).
'''
index = index or {}
log.debug('pkgs_dir=%r, prefix=%r, dist=%r, linktype=%r' %
(pkgs_dir, prefix, dist, linktype))
source_dir = join(pkgs_dir, dist)
if not run_script(source_dir, dist, 'pre-link', prefix):
sys.exit('Error: pre-link failed: %s' % dist)
info_dir = join(source_dir, 'info')
files = list(yield_lines(join(info_dir, 'files')))
has_prefix_files = read_has_prefix(join(info_dir, 'has_prefix'))
no_link = read_no_link(info_dir)
with Locked(prefix), Locked(pkgs_dir):
for f in files:
src = join(source_dir, f)
dst = join(prefix, f)
dst_dir = dirname(dst)
if not isdir(dst_dir):
os.makedirs(dst_dir)
if os.path.exists(dst):
log.warn("file already exists: %r" % dst)
try:
os.unlink(dst)
except OSError:
log.error('failed to unlink: %r' % dst)
if on_win:
try:
move_path_to_trash(dst)
except ImportError:
# This shouldn't be an issue in the installer anyway
pass
lt = linktype
if f in has_prefix_files or f in no_link or islink(src):
lt = LINK_COPY
try:
_link(src, dst, lt)
except OSError as e:
log.error('failed to link (src=%r, dst=%r, type=%r, error=%r)' %
(src, dst, lt, e))
if name_dist(dist) == '_cache':
return
for f in sorted(has_prefix_files):
placeholder, mode = has_prefix_files[f]
try:
update_prefix(join(prefix, f), prefix, placeholder, mode)
except PaddingError:
sys.exit("ERROR: placeholder '%s' too short in: %s\n" %
(placeholder, dist))
mk_menus(prefix, files, remove=False)
if not run_script(prefix, dist, 'post-link'):
sys.exit("Error: post-link failed for: %s" % dist)
# Make sure the script stays standalone for the installer
try:
from conda.config import remove_binstar_tokens
except ImportError:
# There won't be any binstar tokens in the installer anyway
def remove_binstar_tokens(url):
return url
meta_dict = index.get(dist + '.tar.bz2', {})
meta_dict['url'] = read_url(pkgs_dir, dist)
if meta_dict['url']:
meta_dict['url'] = remove_binstar_tokens(meta_dict['url'])
try:
alt_files_path = join(prefix, 'conda-meta', dist + '.files')
meta_dict['files'] = list(yield_lines(alt_files_path))
os.unlink(alt_files_path)
except IOError:
meta_dict['files'] = files
meta_dict['link'] = {'source': source_dir,
'type': link_name_map.get(linktype)}
if 'channel' in meta_dict:
meta_dict['channel'] = remove_binstar_tokens(meta_dict['channel'])
if 'icon' in meta_dict:
meta_dict['icondata'] = read_icondata(source_dir)
create_meta(prefix, dist, info_dir, meta_dict)
verify=config.ssl_verify, stream=True)
resp.raise_for_status()
if resp.status_code != 304:
cache = json.loads(bz2.decompress(resp.raw.read()).decode('utf-8'))
>>>>>>> princeofdarkness76/gzip
=======
headers=headers, proxies=session.proxies,
verify=config.ssl_verify, stream=True)
resp.raise_for_status()
if resp.status_code != 304:
cache = json.loads(bz2.decompress(resp.raw.read()).decode('utf-8'))
>>>>>>> origin/gzip
except ValueError as e:
raise RuntimeError("Invalid index file: %srepodata.json.bz2: %s" %
(config.remove_binstar_tokens(url), e))
except requests.exceptions.HTTPError as e:
if e.response.status_code == 407: # Proxy Authentication Required
handle_proxy_407(url, session)
# Try again
return fetch_repodata(url, cache_dir=cache_dir,
use_cache=use_cache, session=session)
if e.response.status_code == 404:
if url.startswith(config.DEFAULT_CHANNEL_ALIAS):
msg = ('Could not find anaconda.org user %s' %
config.remove_binstar_tokens(url).split(
config.DEFAULT_CHANNEL_ALIAS)[1].split('/')[0])
else:
if url.endswith('/noarch/'): # noarch directory might not exist
def link(pkgs_dir,
prefix,
dist,
linktype=LINK_HARD,
index=None,
target_prefix=None):
'''
Set up a package in a specified (environment) prefix. We assume that
the package has been extracted (using extract() above).
'''
if target_prefix is None:
target_prefix = prefix
index = index or {}
log.debug(
'pkgs_dir=%r, prefix=%r, target_prefix=%r, dist=%r, linktype=%r' %
(pkgs_dir, prefix, target_prefix, dist, linktype))
if (on_win and abspath(prefix) == abspath(sys.prefix)
and name_dist(dist) in win_ignore_root):
# on Windows we have the file lock problem, so don't allow
# linking or unlinking some packages
log.warn('Ignored: %s' % dist)
return
source_dir = join(pkgs_dir, dist)
if not run_script(prefix, dist, 'pre-link', target_prefix):
sys.exit('Error: pre-link failed: %s' % dist)
info_dir = join(source_dir, 'info')
files = list(yield_lines(join(info_dir, 'files')))
has_prefix_files = read_has_prefix(join(info_dir, 'has_prefix'))
no_link = read_no_link(info_dir)
noarch = False
# If the distribution is noarch, it will contain a `link.json` file in
# the info_dir
with open(join(info_dir, 'index.json'), 'r') as fh:
index_data = json.loads(fh.read())
if 'noarch' in index_data:
noarch = index_data['noarch']
elif 'noarch_python' in index_data:
# `noarch_python` has been deprecated.
if index_data['noarch_python'] is True:
noarch = 'python'
if noarch == 'python':
if on_win:
raise ValueError('Windows is not supported.')
link_json = join(info_dir, 'link.json')
if exists(link_json):
with open(link_json, 'r') as fh:
link_data = json.loads(fh.read())
if 'noarch' in link_data:
noarch_json = link_data['noarch']
target_py_version = get_python_version(prefix)
target_python_short_path = join('bin',
'python{}'.format(target_py_version))
target_site_packages = join('lib',
'python{}'.format(target_py_version),
'site-packages')
# A list of the files, including pyc files and entrypoints, that will be
# added to the metadata.
all_files = []
with Locked(prefix), Locked(pkgs_dir):
for f in files:
src = join(source_dir, f)
if noarch == 'python':
noarch_f = get_python_noarch_target_path(
f, target_site_packages)
dst = join(prefix, noarch_f)
all_files.append(noarch_f)
# Non-noarch packages do not need special handling of the
# site-packages
else:
dst = join(prefix, f)
all_files.append(f)
dst_dir = dirname(dst)
if not isdir(dst_dir):
os.makedirs(dst_dir)
if os.path.exists(dst):
log.warn("file already exists: %r" % dst)
try:
os.unlink(dst)
except OSError:
log.error('failed to unlink: %r' % dst)
lt = linktype
if f in has_prefix_files or f in no_link or islink(src):
lt = LINK_COPY
try:
_link(src, dst, lt)
except OSError as e:
log.error(
'failed to link (src=%r, dst=%r, type=%r, error=%r)' %
(src, dst, lt, e))
# noarch package specific installation steps
if noarch == 'python':
# Create entrypoints
if 'entry_points' in noarch_json:
for entry_point in noarch_json['entry_points']:
command, module, func = parse_entry_point_def(entry_point)
entry_point_file = create_python_entry_point(
join(prefix, 'bin', command),
join(prefix, target_python_short_path), module, func)
all_files.append(entry_point_file)
# Compile pyc files
for f in all_files:
if f.endswith('.py'):
py_path = join(prefix, f)
pyc_filepath = compile_pyc(
join(prefix, target_python_short_path), py_path,
pyc_path(py_path, target_py_version))
if pyc_filepath.startswith(prefix):
all_files.append(pyc_filepath[len(prefix):])
if name_dist(dist) == '_cache':
return
for f in sorted(has_prefix_files):
placeholder, mode = has_prefix_files[f]
try:
update_prefix(join(prefix, f), target_prefix, placeholder,
mode)
except PaddingError:
sys.exit("ERROR: placeholder '%s' too short in: %s\n" %
(placeholder, dist))
mk_menus(prefix, files, remove=False)
if not run_script(prefix, dist, 'post-link', target_prefix):
sys.exit("Error: post-link failed for: %s" % dist)
# Make sure the script stays standalone for the installer
try:
from conda.config import remove_binstar_tokens
except ImportError:
# There won't be any binstar tokens in the installer anyway
def remove_binstar_tokens(url):
return url
meta_dict = index.get(dist + '.tar.bz2', {})
meta_dict['url'] = read_url(pkgs_dir, dist)
if meta_dict['url']:
meta_dict['url'] = remove_binstar_tokens(meta_dict['url'])
try:
alt_files_path = join(prefix, 'conda-meta', dist + '.files')
meta_dict['files'] = list(yield_lines(alt_files_path))
os.unlink(alt_files_path)
except IOError:
meta_dict['files'] = all_files
meta_dict['link'] = {
'source': source_dir,
'type': link_name_map.get(linktype)
}
if 'channel' in meta_dict:
meta_dict['channel'] = remove_binstar_tokens(meta_dict['channel'])
if 'icon' in meta_dict:
meta_dict['icondata'] = read_icondata(source_dir)
create_meta(prefix, dist, info_dir, meta_dict)
