def from_line(cls, name, comes_from=None, prereleases=None):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
url = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
if is_url(name):
link = Link(name)
elif (os.path.isdir(path)
and (os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(path):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' not "
"found." % name
)
link = Link(path_to_url(name))
elif is_archive_file(path):
if not os.path.isfile(path):
logger.warn(
'Requirement %r looks like a filename, but the file does '
'not exist',
name
)
link = Link(path_to_url(name))
# If the line has an egg= definition, but isn't editable, pull the
# requirement out. Otherwise, assume the name is the req for the non
# URL/path/archive case.
if link and req is None:
url = link.url_without_fragment
# when fragment is None, this will become an 'unnamed' requirement
req = link.egg_fragment
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', url):
url = path_to_url(os.path.normpath(os.path.abspath(link.path)))
# fail early for invalid or unsupported wheels
if link.ext == wheel_ext:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
if not wheel.supported():
raise UnsupportedWheel(
"%s is not a supported wheel on this platform." %
wheel.filename
)
else:
req = name
return cls(req, comes_from, url=url, prereleases=prereleases)
def _sort_locations(locations, expand_dir=False):
"""
Sort locations into "files" (archives) and "urls", and return
a pair of lists (files,urls)
"""
files = []
urls = []
# puts the url for the given file path into the appropriate list
def sort_path(path):
url = path_to_url(path)
if mimetypes.guess_type(url, strict=False)[0] == 'text/html':
urls.append(url)
else:
files.append(url)
for url in locations:
is_local_path = os.path.exists(url)
is_file_url = url.startswith('file:')
if is_local_path or is_file_url:
if is_local_path:
path = url
else:
path = url_to_path(url)
if os.path.isdir(path):
if expand_dir:
path = os.path.realpath(path)
for item in os.listdir(path):
sort_path(os.path.join(path, item))
elif is_file_url:
urls.append(url)
elif os.path.isfile(path):
sort_path(path)
else:
logger.warning(
"Url '%s' is ignored: it is neither a file "
"nor a directory.", url)
elif is_url(url):
# Only add url with clear scheme
urls.append(url)
else:
logger.warning(
"Url '%s' is ignored. It is either a non-existing "
"path or lacks a specific scheme.", url)
return files, urls
def from_line(cls, name, comes_from=None, isolated=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
url = None
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
if is_url(name):
link = Link(name)
elif (os.path.isdir(path)
and (os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(path):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' not "
"found." % name
)
link = Link(path_to_url(name))
elif is_archive_file(path):
if not os.path.isfile(path):
logger.warning(
'Requirement %r looks like a filename, but the file does '
'not exist',
name
)
link = Link(path_to_url(name))
# it's a local file, dir, or url
if link:
url = link.url
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', url):
url = path_to_url(os.path.normpath(os.path.abspath(link.path)))
# wheel file
if link.ext == wheel_ext:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
if not wheel.supported():
raise UnsupportedWheel(
"%s is not a supported wheel on this platform." %
wheel.filename
)
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
return cls(req, comes_from, url=url, markers=markers,
isolated=isolated)
def from_line(
cls, name, comes_from=None, isolated=False, options=None,
wheel_cache=None, constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
if (os.path.isdir(p) and
(os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' "
"not found." % name
)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist',
name
)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Normalize URLs
link.normalize()
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
options = options if options else {}
res = cls(req, comes_from, link=link, markers=markers,
isolated=isolated, options=options,
wheel_cache=wheel_cache, constraint=constraint)
if extras:
res.extras = Requirement('placeholder' + extras).extras
return res
def get_local_directory( self, cuppa_env, location, sub_dir, branch, full_url ):
offline = cuppa_env['offline']
local_directory = None
base = cuppa_env['download_root']
if not os.path.isabs( base ):
base = os.path.join( cuppa_env['working_dir'], base )
if location.startswith( 'file:' ):
location = pip_download.url_to_path( location )
if not pip_download.is_url( location ):
if pip_download.is_archive_file( location ):
self._local_folder = self.folder_name_from_path( location, cuppa_env )
local_directory = os.path.join( base, self._local_folder )
local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" )
if os.path.exists( local_dir_with_sub_dir ):
try:
os.rmdir( local_dir_with_sub_dir )
except:
return local_directory
self.extract( location, local_dir_with_sub_dir )
logger.debug( "(local archive) Location = [{}]".format( as_info( location ) ) )
logger.debug( "(local archive) Local folder = [{}]".format( as_info( self._local_folder ) ) )
else:
local_directory = branch and os.path.join( location, branch ) or location
self._local_folder = self.folder_name_from_path( location, cuppa_env )
logger.debug( "(local file) Location = [{}]".format( as_info( location ) ) )
logger.debug( "(local file) Local folder = [{}]".format( as_info( self._local_folder ) ) )
return local_directory
else:
self._local_folder = self.folder_name_from_path( full_url, cuppa_env )
local_directory = os.path.join( base, self._local_folder )
if full_url.scheme.startswith( 'http' ) and self.url_is_download_archive_url( full_url.path ):
logger.debug( "[{}] is an archive download".format( as_info( location ) ) )
local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" )
# First we check to see if we already downloaded and extracted this archive before
if os.path.exists( local_dir_with_sub_dir ):
try:
# If not empty this will fail
os.rmdir( local_dir_with_sub_dir )
except:
# Not empty so we'll return this as the local_directory
logger.debug( "(already present) Location = [{}]".format( as_info( location ) ) )
logger.debug( "(already present) Local folder = [{}]".format( as_info( str(self._local_folder) ) ) )
return local_directory
if cuppa_env['dump'] or cuppa_env['clean']:
return local_directory
# If not we then check to see if we cached the download
cached_archive = self.get_cached_archive( cuppa_env['cache_root'], self._local_folder )
if cached_archive:
logger.debug( "Cached archive [{}] found for [{}]".format(
as_info( cached_archive ),
as_info( location )
) )
self.extract( cached_archive, local_dir_with_sub_dir )
else:
logger.info( "Downloading [{}]...".format( as_info( location ) ) )
try:
report_hook = None
if logger.isEnabledFor( logging.INFO ):
report_hook = ReportDownloadProgress()
filename, headers = urllib.urlretrieve( location, reporthook=report_hook )
name, extension = os.path.splitext( filename )
logger.info( "[{}] successfully downloaded to [{}]".format(
as_info( location ),
as_info( filename )
) )
self.extract( filename, local_dir_with_sub_dir )
if cuppa_env['cache_root']:
cached_archive = os.path.join( cuppa_env['cache_root'], self._local_folder )
logger.debug( "Caching downloaded file as [{}]".format( as_info( cached_archive ) ) )
shutil.copyfile( filename, cached_archive )
except urllib.ContentTooShortError as error:
logger.error( "Download of [{}] failed with error [{}]".format(
as_error( location ),
as_error( str(error) )
) )
raise LocationException( error )
elif '+' in full_url.scheme:
vc_type = location.split('+', 1)[0]
backend = pip_vcs.vcs.get_backend( vc_type )
if backend:
vcs_backend = backend( self.expand_secret( location ) )
local_dir_with_sub_dir = os.path.join( local_directory, sub_dir and sub_dir or "" )
if cuppa_env['dump'] or cuppa_env['clean']:
return local_directory
if os.path.exists( local_directory ):
url, repository, branch, remote, revision = self.get_info( location, local_dir_with_sub_dir, full_url, vc_type )
rev_options = self.get_rev_options( vc_type, vcs_backend, local_remote=remote )
version = self.ver_rev_summary( branch, revision, self._full_url.path )[0]
if not offline:
logger.info( "Updating [{}] in [{}]{} at [{}]".format(
as_info( location ),
as_notice( local_dir_with_sub_dir ),
( rev_options and " on {}".format( as_notice( str(rev_options) ) ) or "" ),
as_info( version )
) )
try:
update( vcs_backend, local_dir_with_sub_dir, rev_options )
logger.debug( "Successfully updated [{}]".format( as_info( location ) ) )
except pip_exceptions.PipError as error:
logger.warn( "Could not update [{}] in [{}]{} due to error [{}]".format(
as_warning( location ),
as_warning( local_dir_with_sub_dir ),
( rev_options and " at {}".format( as_warning( str(rev_options) ) ) or "" ),
as_warning( str(error) )
) )
else:
logger.debug( "Skipping update for [{}] as running in offline mode".format( as_info( location ) ) )
else:
rev_options = self.get_rev_options( vc_type, vcs_backend )
action = "Cloning"
if vc_type == "svn":
action = "Checking out"
max_attempts = 2
attempt = 1
while attempt <= max_attempts:
logger.info( "{} [{}] into [{}]{}".format(
action,
as_info( location ),
as_info( local_dir_with_sub_dir ),
attempt > 1 and "(attempt {})".format( str(attempt) ) or ""
) )
try:
vcs_backend.obtain( local_dir_with_sub_dir )
logger.debug( "Successfully retrieved [{}]".format( as_info( location ) ) )
break
except pip_exceptions.PipError as error:
attempt = attempt + 1
log_as = logger.warn
if attempt > max_attempts:
log_as = logger.error
log_as( "Could not retrieve [{}] into [{}]{} due to error [{}]".format(
as_info( location ),
as_notice( local_dir_with_sub_dir ),
( rev_options and " to {}".format( as_notice( str(rev_options) ) ) or ""),
as_error( str(error) )
) )
if attempt > max_attempts:
raise LocationException( str(error) )
logger.debug( "(url path) Location = [{}]".format( as_info( location ) ) )
logger.debug( "(url path) Local folder = [{}]".format( as_info( self._local_folder ) ) )
return local_directory
def from_line(
cls, name, comes_from=None, isolated=False, options=None,
wheel_cache=None, constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = Marker(markers)
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
if (os.path.isdir(p) and
(os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' "
"not found." % name
)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist',
name
)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
if extras:
extras = Requirement("placeholder" + extras.lower()).extras
else:
extras = ()
if req is not None:
try:
req = Requirement(req)
except InvalidRequirement:
if os.path.sep in req:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req)
elif '=' in req and not any(op in req for op in operators):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = traceback.format_exc()
raise InstallationError(
"Invalid requirement: '%s'\n%s" % (req, add_msg))
return cls(
req, comes_from, link=link, markers=markers,
isolated=isolated,
options=options if options else {},
wheel_cache=wheel_cache,
constraint=constraint,
extras=extras,
)
def from_line(cls, name, comes_from=None, isolated=False, options=None, wheel_cache=None, constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
if is_url(name):
marker_sep = "; "
else:
marker_sep = ";"
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
if os.path.isdir(p) and (os.path.sep in name or name.startswith(".")):
if not is_installable_dir(p):
raise InstallationError("Directory %r is not installable. File 'setup.py' " "not found." % name)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning("Requirement %r looks like a filename, but the " "file does not exist", name)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == "file" and re.search(r"\.\./", link.url):
link = Link(path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
if not wheel.supported():
raise UnsupportedWheel("%s is not a supported wheel on this platform." % wheel.filename)
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
options = options if options else {}
res = cls(
req,
comes_from,
link=link,
markers=markers,
isolated=isolated,
options=options,
wheel_cache=wheel_cache,
constraint=constraint,
)
if extras:
res.extras = pkg_resources.Requirement.parse("__placeholder__" + extras).extras
return res
def from_line(cls,
name,
comes_from=None,
isolated=False,
options=None,
wheel_cache=None,
constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = Marker(markers)
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
if (os.path.isdir(p)
and (os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' "
"not found." % name)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist', name)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
options = options if options else {}
res = cls(req,
comes_from,
link=link,
markers=markers,
isolated=isolated,
options=options,
wheel_cache=wheel_cache,
constraint=constraint)
if extras:
res.extras = _safe_extras(
Requirement('placeholder' + extras).extras)
return res
def from_line(cls,
name,
comes_from=None,
isolated=False,
options=None,
wheel_cache=None,
constraint=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = Marker(markers)
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
extras = None
if is_url(name):
link = Link(name)
else:
p, extras = _strip_extras(path)
looks_like_dir = os.path.isdir(p) and (os.path.sep in name or
(os.path.altsep is not None
and os.path.altsep in name)
or name.startswith('.'))
if looks_like_dir:
if not is_installable_dir(p):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' "
"not found." % name)
link = Link(path_to_url(p))
elif is_archive_file(p):
if not os.path.isfile(p):
logger.warning(
'Requirement %r looks like a filename, but the '
'file does not exist', name)
link = Link(path_to_url(p))
# it's a local file, dir, or url
if link:
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', link.url):
link = Link(
path_to_url(os.path.normpath(os.path.abspath(link.path))))
# wheel file
if link.is_wheel:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
if extras:
extras = Requirement("placeholder" + extras.lower()).extras
else:
extras = ()
if req is not None:
try:
req = Requirement(req)
except InvalidRequirement:
if os.path.sep in req:
add_msg = "It looks like a path."
add_msg += deduce_helpful_msg(req)
elif '=' in req and not any(op in req for op in operators):
add_msg = "= is not a valid operator. Did you mean == ?"
else:
add_msg = traceback.format_exc()
raise InstallationError("Invalid requirement: '%s'\n%s" %
(req, add_msg))
return cls(
req,
comes_from,
link=link,
markers=markers,
isolated=isolated,
options=options if options else {},
wheel_cache=wheel_cache,
constraint=constraint,
extras=extras,
)
def from_line(cls, name, comes_from=None, isolated=False):
"""Creates an InstallRequirement from a name, which might be a
requirement, directory containing 'setup.py', filename, or URL.
"""
from pip.index import Link
url = None
if is_url(name):
marker_sep = '; '
else:
marker_sep = ';'
if marker_sep in name:
name, markers = name.split(marker_sep, 1)
markers = markers.strip()
if not markers:
markers = None
else:
markers = None
name = name.strip()
req = None
path = os.path.normpath(os.path.abspath(name))
link = None
if is_url(name):
link = Link(name)
elif (os.path.isdir(path)
and (os.path.sep in name or name.startswith('.'))):
if not is_installable_dir(path):
raise InstallationError(
"Directory %r is not installable. File 'setup.py' not "
"found." % name)
link = Link(path_to_url(name))
elif is_archive_file(path):
if not os.path.isfile(path):
logger.warning(
'Requirement %r looks like a filename, but the file does '
'not exist', name)
link = Link(path_to_url(name))
# it's a local file, dir, or url
if link:
url = link.url
# Handle relative file URLs
if link.scheme == 'file' and re.search(r'\.\./', url):
url = path_to_url(os.path.normpath(os.path.abspath(link.path)))
# wheel file
if link.ext == wheel_ext:
wheel = Wheel(link.filename) # can raise InvalidWheelFilename
if not wheel.supported():
raise UnsupportedWheel(
"%s is not a supported wheel on this platform." %
wheel.filename)
req = "%s==%s" % (wheel.name, wheel.version)
else:
# set the req to the egg fragment. when it's not there, this
# will become an 'unnamed' requirement
req = link.egg_fragment
# a requirement specifier
else:
req = name
return cls(req,
comes_from,
url=url,
markers=markers,
isolated=isolated)
def get_local_directory(self, cuppa_env, location, sub_dir, branch,
full_url):
offline = cuppa_env['offline']
local_directory = None
base = cuppa_env['download_root']
if not os.path.isabs(base):
base = os.path.join(cuppa_env['working_dir'], base)
if location.startswith('file:'):
location = pip_download.url_to_path(location)
if not pip_download.is_url(location):
if pip_download.is_archive_file(location):
self._local_folder = self.folder_name_from_path(
location, cuppa_env)
local_directory = os.path.join(base, self._local_folder)
local_dir_with_sub_dir = os.path.join(
local_directory, sub_dir and sub_dir or "")
if os.path.exists(local_dir_with_sub_dir):
try:
os.rmdir(local_dir_with_sub_dir)
except:
return local_directory
self.extract(location, local_dir_with_sub_dir)
logger.debug("(local archive) Location = [{}]".format(
as_info(location)))
logger.debug("(local archive) Local folder = [{}]".format(
as_info(self._local_folder)))
else:
local_directory = branch and os.path.join(location,
branch) or location
self._local_folder = self.folder_name_from_path(
location, cuppa_env)
logger.debug("(local file) Location = [{}]".format(
as_info(location)))
logger.debug("(local file) Local folder = [{}]".format(
as_info(self._local_folder)))
return local_directory
else:
self._local_folder = self.folder_name_from_path(
full_url, cuppa_env)
local_directory = os.path.join(base, self._local_folder)
if full_url.scheme.startswith(
'http') and self.url_is_download_archive_url(
full_url.path):
logger.debug("[{}] is an archive download".format(
as_info(location)))
local_dir_with_sub_dir = os.path.join(
local_directory, sub_dir and sub_dir or "")
# First we check to see if we already downloaded and extracted this archive before
if os.path.exists(local_dir_with_sub_dir):
try:
# If not empty this will fail
os.rmdir(local_dir_with_sub_dir)
except:
# Not empty so we'll return this as the local_directory
logger.debug(
"(already present) Location = [{}]".format(
as_info(location)))
logger.debug(
"(already present) Local folder = [{}]".format(
as_info(str(self._local_folder))))
return local_directory
if cuppa_env['dump'] or cuppa_env['clean']:
return local_directory
# If not we then check to see if we cached the download
cached_archive = self.get_cached_archive(
cuppa_env['cache_root'], self._local_folder)
if cached_archive:
logger.debug("Cached archive [{}] found for [{}]".format(
as_info(cached_archive), as_info(location)))
self.extract(cached_archive, local_dir_with_sub_dir)
else:
logger.info("Downloading [{}]...".format(
as_info(location)))
try:
report_hook = None
if logger.isEnabledFor(logging.INFO):
report_hook = ReportDownloadProgress()
filename, headers = urllib.urlretrieve(
location, reporthook=report_hook)
name, extension = os.path.splitext(filename)
logger.info(
"[{}] successfully downloaded to [{}]".format(
as_info(location), as_info(filename)))
self.extract(filename, local_dir_with_sub_dir)
if cuppa_env['cache_root']:
cached_archive = os.path.join(
cuppa_env['cache_root'], self._local_folder)
logger.debug(
"Caching downloaded file as [{}]".format(
as_info(cached_archive)))
shutil.copyfile(filename, cached_archive)
except urllib.ContentTooShortError as error:
logger.error(
"Download of [{}] failed with error [{}]".format(
as_error(location), as_error(str(error))))
raise LocationException(error)
elif '+' in full_url.scheme:
vc_type = location.split('+', 1)[0]
backend = pip_vcs.vcs.get_backend(vc_type)
if backend:
vcs_backend = backend(self.expand_secret(location))
local_dir_with_sub_dir = os.path.join(
local_directory, sub_dir and sub_dir or "")
if cuppa_env['dump'] or cuppa_env['clean']:
return local_directory
if os.path.exists(local_directory):
url, repository, branch, remote, revision = self.get_info(
location, local_dir_with_sub_dir, full_url,
vc_type)
rev_options = self.get_rev_options(vc_type,
vcs_backend,
local_remote=remote)
version = self.ver_rev_summary(branch, revision,
self._full_url.path)[0]
if not offline:
logger.info(
"Updating [{}] in [{}]{} at [{}]".format(
as_info(location),
as_notice(local_dir_with_sub_dir),
(rev_options and " on {}".format(
as_notice(str(rev_options))) or ""),
as_info(version)))
try:
update(vcs_backend, local_dir_with_sub_dir,
rev_options)
logger.debug(
"Successfully updated [{}]".format(
as_info(location)))
except pip_exceptions.PipError as error:
logger.warn(
"Could not update [{}] in [{}]{} due to error [{}]"
.format(as_warning(location),
as_warning(local_dir_with_sub_dir),
(rev_options and " at {}".format(
as_warning(str(rev_options)))
or ""), as_warning(str(error))))
else:
logger.debug(
"Skipping update for [{}] as running in offline mode"
.format(as_info(location)))
else:
rev_options = self.get_rev_options(
vc_type, vcs_backend)
action = "Cloning"
if vc_type == "svn":
action = "Checking out"
max_attempts = 2
attempt = 1
while attempt <= max_attempts:
logger.info("{} [{}] into [{}]{}".format(
action, as_info(location),
as_info(local_dir_with_sub_dir), attempt > 1
and "(attempt {})".format(str(attempt)) or ""))
try:
vcs_backend.obtain(local_dir_with_sub_dir)
logger.debug(
"Successfully retrieved [{}]".format(
as_info(location)))
break
except pip_exceptions.PipError as error:
attempt = attempt + 1
log_as = logger.warn
if attempt > max_attempts:
log_as = logger.error
log_as(
"Could not retrieve [{}] into [{}]{} due to error [{}]"
.format(as_info(location),
as_notice(local_dir_with_sub_dir),
(rev_options and " to {}".format(
as_notice(str(rev_options)))
or ""), as_error(str(error))))
if attempt > max_attempts:
raise LocationException(str(error))
logger.debug("(url path) Location = [{}]".format(
as_info(location)))
logger.debug("(url path) Local folder = [{}]".format(
as_info(self._local_folder)))
return local_directory