def is_archive_file(name):
# type: (str) -> bool
"""Return True if `name` is a considered as an archive file."""
ext = splitext(name)[1].lower()
if ext in ARCHIVE_EXTENSIONS:
return True
return False
def main():
usage = "usage: %prog release"
parser = OptionParser(usage)
(options, args) = parser.parse_args()
if len(args) != 1:
parser.error("Please provide a release version, e.g. 0.7.0beta1")
release = args[0]
scripts_dir = os.path.dirname(__file__)
download_dir = os.path.normpath(
os.path.join(scripts_dir, '..', 'requirements', release))
if not os.path.exists(download_dir):
os.makedirs(download_dir)
release_reqs = os.path.join(download_dir, 'release.txt')
if not os.path.exists(release_reqs):
parser.error("%s could not be found. Please make sure you created the file before." % release_reqs)
complete_log = []
level = 1
level = pip.Logger.level_for_integer(4-level)
logger = pip.Logger([(level, sys.stdout), (pip.Logger.DEBUG, complete_log.append)])
pip.logger = logger
finder = pip.PackageFinder(
find_links=['http://pypi.pinaxproject.com'], index_urls=[pip.pypi_url])
req_set = pip.RequirementSet(build_dir=download_dir, src_dir=download_dir)
unmet_requirements= []
for req in pip.parse_requirements(release_reqs, finder=finder):
req_set.add_requirement(req)
reqs = list(req_set.unnamed_requirements) + req_set.requirements.values()
if not reqs:
parser.error("No requirements were found in %s. Are you sure it contains something?" % release_reqs)
for req in reqs:
try:
if req.name:
logger.notify('Searching for: %s' % req.name)
if req.editable:
unmet_requirements.append(req)
else:
location = req.build_location(download_dir)
if req.url is None:
link = finder.find_requirement(req, upgrade=False)
else:
link = pip.Link(req.url)
if link:
try:
md5_hash = link.md5_hash
target_url = link.url.split('#', 1)[0]
target_file = None
try:
resp = urllib2.urlopen(target_url)
except urllib2.HTTPError, e:
logger.fatal("HTTP error %s while getting %s" % (e.code, link))
raise
except IOError, e:
# Typically an FTP error
logger.fatal("Error %s while getting %s" % (e, link))
raise
content_type = resp.info()['content-type']
if content_type.startswith('text/html'):
unmet_requirements.append(req)
continue
filename = link.filename
ext = pip.splitext(filename)
if not ext:
ext = mimetypes.guess_extension(content_type)
filename += ext
temp_location = os.path.join(download_dir, filename)
if os.path.exists(temp_location):
logger.notify('Skipping %s. File exists.' % filename)
continue
fp = open(temp_location, 'wb')
if md5_hash:
download_hash = md5()
try:
total_length = int(resp.info()['content-length'])
except (ValueError, KeyError):
total_length = 0
downloaded = 0
show_progress = total_length > 40*1000 or not total_length
show_url = link.show_url
try:
if show_progress:
if total_length:
logger.start_progress('Downloading %s (%s): ' % (show_url, pip.format_size(total_length)))
else:
logger.start_progress('Downloading %s (unknown size): ' % show_url)
else:
logger.notify('Downloading %s' % show_url)
logger.debug('Downloading from URL %s' % link)
while 1:
chunk = resp.read(4096)
if not chunk:
break
downloaded += len(chunk)
if show_progress:
if not total_length:
logger.show_progress('%s' % pip.format_size(downloaded))
else:
logger.show_progress('%3i%% %s' % (100*downloaded/total_length, pip.format_size(downloaded)))
if md5_hash:
download_hash.update(chunk)
fp.write(chunk)
fp.close()
finally:
if show_progress:
logger.end_progress('%s downloaded' % pip.format_size(downloaded))
if md5_hash:
download_hash = download_hash.hexdigest()
if download_hash != md5_hash:
logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!"
% (link, download_hash, md5_hash))
raise pip.InstallationError('Bad MD5 hash for package %s' % link)
except urllib2.HTTPError, e:
logger.fatal('Could not install requirement %s because of error %s'
% (req, e))
raise InstallationError(
'Could not install requirement %s because of HTTP error %s for URL %s'
% (req, e, url))
def splitext(self):
# type: () -> Tuple[str, str]
return splitext(posixpath.basename(self.path.rstrip('/')))
def main():
usage = "usage: %prog release"
parser = OptionParser(usage)
(options, args) = parser.parse_args()
if len(args) != 1:
parser.error("Please provide a release version, e.g. 0.7.0beta3")
release = args[0]
scripts_dir = os.path.dirname(__file__)
download_dir = os.path.normpath(
os.path.join(scripts_dir, '..', 'requirements', release))
if not os.path.exists(download_dir):
os.makedirs(download_dir)
release_reqs = os.path.join(download_dir, 'release.txt')
if not os.path.exists(release_reqs):
parser.error("%s could not be found. Please make sure you created the file before." % release_reqs)
complete_log = []
level = 1
level = pip.Logger.level_for_integer(4-level)
logger = pip.Logger([(level, sys.stdout), (pip.Logger.DEBUG, complete_log.append)])
pip.logger = logger
finder = pip.PackageFinder(
find_links=['http://pypi.pinaxproject.com'], index_urls=[pip.pypi_url])
req_set = pip.RequirementSet(build_dir=download_dir, src_dir=download_dir)
unmet_requirements= []
for req in pip.parse_requirements(release_reqs, finder=finder):
req_set.add_requirement(req)
reqs = list(req_set.unnamed_requirements) + req_set.requirements.values()
if not reqs:
parser.error("No requirements were found in %s. Are you sure it contains something?" % release_reqs)
for req in reqs:
try:
if req.name:
logger.notify('Searching for: %s' % req.name)
if req.editable:
unmet_requirements.append(req)
else:
location = req.build_location(download_dir)
if req.url is None:
link = finder.find_requirement(req, upgrade=False)
else:
link = pip.Link(req.url)
if link:
try:
md5_hash = link.md5_hash
target_url = link.url.split('#', 1)[0]
target_file = None
try:
resp = urllib2.urlopen(target_url)
except urllib2.HTTPError, e:
logger.fatal("HTTP error %s while getting %s" % (e.code, link))
raise
except IOError, e:
# Typically an FTP error
logger.fatal("Error %s while getting %s" % (e, link))
raise
content_type = resp.info()['content-type']
if content_type.startswith('text/html'):
unmet_requirements.append(req)
continue
filename = link.filename
ext = pip.splitext(filename)
if not ext:
ext = mimetypes.guess_extension(content_type)
filename += ext
temp_location = os.path.join(download_dir, filename)
if os.path.exists(temp_location):
logger.notify('Skipping %s. File exists.' % filename)
continue
fp = open(temp_location, 'wb')
if md5_hash:
download_hash = md5()
try:
total_length = int(resp.info()['content-length'])
except (ValueError, KeyError):
total_length = 0
downloaded = 0
show_progress = total_length > 40*1000 or not total_length
show_url = link.show_url
try:
if show_progress:
if total_length:
logger.start_progress('Downloading %s (%s): ' % (show_url, pip.format_size(total_length)))
else:
logger.start_progress('Downloading %s (unknown size): ' % show_url)
else:
logger.notify('Downloading %s' % show_url)
logger.debug('Downloading from URL %s' % link)
while 1:
chunk = resp.read(4096)
if not chunk:
break
downloaded += len(chunk)
if show_progress:
if not total_length:
logger.show_progress('%s' % pip.format_size(downloaded))
else:
logger.show_progress('%3i%% %s' % (100*downloaded/total_length, pip.format_size(downloaded)))
if md5_hash:
download_hash.update(chunk)
fp.write(chunk)
fp.close()
finally:
if show_progress:
logger.end_progress('%s downloaded' % pip.format_size(downloaded))
if md5_hash:
download_hash = download_hash.hexdigest()
if download_hash != md5_hash:
logger.fatal("MD5 hash of the package %s (%s) doesn't match the expected hash %s!"
% (link, download_hash, md5_hash))
raise pip.InstallationError('Bad MD5 hash for package %s' % link)
except urllib2.HTTPError, e:
logger.fatal('Could not install requirement %s because of error %s'
% (req, e))
raise InstallationError(
'Could not install requirement %s because of HTTP error %s for URL %s'
% (req, e, url))
def _download_http_url(
link, # type: Link
session, # type: PipSession
temp_dir, # type: str
hashes, # type: Hashes
progress_bar # type: str
):
# type: (...) -> Tuple[str, str]
"""Download link url into temp_dir using provided session"""
target_url = link.url.split('#', 1)[0]
try:
resp = session.get(
target_url,
# We use Accept-Encoding: identity here because requests
# defaults to accepting compressed responses. This breaks in
# a variety of ways depending on how the server is configured.
# - Some servers will notice that the file isn't a compressible
# file and will leave the file alone and with an empty
# Content-Encoding
# - Some servers will notice that the file is already
# compressed and will leave the file alone and will add a
# Content-Encoding: gzip header
# - Some servers won't notice anything at all and will take
# a file that's already been compressed and compress it again
# and set the Content-Encoding: gzip header
# By setting this to request only the identity encoding We're
# hoping to eliminate the third case. Hopefully there does not
# exist a server which when given a file will notice it is
# already compressed and that you're not asking for a
# compressed file and will then decompress it before sending
# because if that's the case I don't think it'll ever be
# possible to make this work.
headers={"Accept-Encoding": "identity"},
stream=True,
)
resp.raise_for_status()
except requests.HTTPError as exc:
logger.critical(
"HTTP error %s while getting %s",
exc.response.status_code,
link,
)
raise
content_type = resp.headers.get('content-type', '')
filename = link.filename # fallback
# Have a look at the Content-Disposition header for a better guess
content_disposition = resp.headers.get('content-disposition')
if content_disposition:
type, params = cgi.parse_header(content_disposition)
# We use ``or`` here because we don't want to use an "empty" value
# from the filename param.
filename = params.get('filename') or filename
ext = splitext(filename)[1]
if not ext:
ext = mimetypes.guess_extension(content_type)
if ext:
filename += ext
if not ext and link.url != resp.url:
ext = os.path.splitext(resp.url)[1]
if ext:
filename += ext
file_path = os.path.join(temp_dir, filename)
with open(file_path, 'wb') as content_file:
_download_url(resp, link, content_file, hashes, progress_bar)
return file_path, content_type
target_file = None
try:
resp = urllib2.urlopen(target_url)
except urllib2.HTTPError, e:
logger.fatal("HTTP error %s while getting %s" % (e.code, link))
raise
except IOError, e:
# Typically an FTP error
logger.fatal("Error %s while getting %s" % (e, link))
raise
content_type = resp.info()['content-type']
if content_type.startswith('text/html'):
unmet_requirements.append(req)
continue
filename = link.filename
ext = pip.splitext(filename)
if not ext:
ext = mimetypes.guess_extension(content_type)
filename += ext
temp_location = os.path.join(download_dir, filename)
if os.path.exists(temp_location):
logger.notify('Skipping %s. File exists.' % filename)
continue
fp = open(temp_location, 'wb')
if md5_hash:
download_hash = md5()
try:
total_length = int(resp.info()['content-length'])
except (ValueError, KeyError):
total_length = 0
downloaded = 0
