def download(url, msg):
def hook(blocks, block_size, total_size):
'''
Callback function for `urllib.urlretrieve` that is called when connection is
created and then once for each block.
Display the amount of data transferred so far and it percentage.
Use sys.stdout.write() instead of "print,", because it allows one more
symbol at the line end without linefeed on Windows
:param blocks: Number of blocks transferred so far.
:param block_size: Size of each block in bytes.
:param total_size: Total size of the HTTP object in bytes. Can be -1 if server doesn't return it.
'''
if block_size > total_size:
logger.info('\r{0} [100% - {1}]', msg, convert_bytes(total_size), addn=False)
return
downloaded = block_size * blocks
ratio = downloaded / float(total_size)
## When the last block makes the downloaded size greater than the total size
if ratio > 1:
ratio = 1
downloaded = total_size
## Calculate elapsed and remaining time
elapsed = func() - starttime
speed = downloaded / float(elapsed)
try:
remaining = (total_size - downloaded) / float(speed)
except ZeroDivisionError:
remaining = ''
if ratio == 1:
## When we finish the download we want this string to hide
remaining = ''
logger.info('\r{0} [{1:.0%} - {2} / {3}] {4}', msg, ratio, convert_bytes(downloaded), \
convert_bytes(total_size), format_time(remaining), addn=False)
if is_windows():
## On Windows time.clock should be more precise.
func = time.clock
else:
func = time.time
starttime = func()
path = urllib.urlretrieve(url, reporthook=hook)[0]
logger.newline()
with open(path) as f:
return cStringIO.StringIO(f.read())
def _download_and_install(self, url, filename, packname, e, hash=None):
fobj = download(url, 'Downloading {0}'.format(self.name))
if hash is not None:
logger.info('Checking md5 sum')
if md5(fobj.getvalue()).hexdigest() != hash:
logger.fatal('Error: {0} appears to be corrupted', self.name)
return
if e in ('.tar.gz', '.tar.bz2', '.zip'):
installer = Archive(fobj, e, packname, self.reqset)
elif e == '.egg':
installer = Egg(fobj, filename, self.reqset, packname)
elif is_windows() and e in WINDOWS_EXT:
installer = Binary(fobj, e, packname)
else:
logger.error('Error: unknown filetype: {0}', e, exc=InstallationError)
## There is no need to catch exceptions now, this will be done by `pyg.inst.Installer.install`
installer.install()
self.success = True
def from_file(filepath, packname=None):
packname = packname or os.path.basename(filepath).split('-')[0]
reqset = ReqSet(packname)
e = ext(filepath)
path = os.path.abspath(filepath)
if e in ('.tar.gz', '.tar.bz2', '.zip'):
installer = Archive(open(path), e, packname, reqset)
elif e in ('.pybundle', '.pyb'):
installer = Bundle(filepath)
elif e == '.egg':
installer = Egg(open(path), path, reqset)
elif e in ('.exe', '.msi') and is_windows():
installer = Binary(open(path), e, packname)
else:
if tarfile.is_tarfile(path):
installer = Archive(open(path), None, packname, reqset)
elif zipfile.is_zipfile(path):
installer = Archive(open(path), '.zip', packname, reqset)
else:
logger.fatal('Error: Cannot install {0}: unknown filetype', packname, exc=InstallationError)
installer.install()
Installer._install_deps(reqset, packname)
logger.success('{0} installed successfully', packname)
def find_files(self):
_un_re = re.compile(r'{0}(-(\d\.?)+(\-py\d\.\d)?\.(egg|egg\-info))?$'.format(self.name), re.I)
_un2_re = re.compile(r'{0}(?:(\.py|\.pyc))'.format(self.name), re.I)
_un3_re = re.compile(r'{0}.*\.so'.format(self.name), re.I)
_uninstall_re = [_un_re, _un2_re, _un3_re]
to_del = set()
try:
dist = pkg_resources.get_distribution(self.name)
except pkg_resources.DistributionNotFound:
logger.debug('debug: Distribution not found: {0}', self.name)
## Create a fake distribution
## In Python2.6 we can only use site.USER_SITE
class FakeDist(object):
def __init__(self, o):
self._orig_o = o
def __getattr__(self, a):
if a == 'location':
return USER_SITE
elif a == 'egg_name':
return (lambda *a: self._orig_o.name + '.egg')
return (lambda *a: False)
dist = FakeDist(self)
pkg_loc = dist.location
glob_folder = False
if pkg_loc in ALL_SITE_PACKAGES:
# try to detect the real package location
if dist.has_metadata('top_level.txt'):
pkg_loc = os.path.join( pkg_loc,
dist.get_metadata_lines('top_level.txt').next())
else:
glob_folder = True
# detect egg-info location
_base_name = dist.egg_name().split('-')
for n in range(len(_base_name) + 1):
egg_info_dir = os.path.join(
dist.location,
'-'.join(_base_name[:-n if n else None]) + '.egg-info'
)
if os.path.exists(egg_info_dir):
for file in os.listdir(egg_info_dir):
if any(u_re.match(file) for u_re in _uninstall_re):
to_del.add(os.path.join(egg_info_dir, file))
to_del.add(egg_info_dir)
break
if glob_folder:
# track individual files inside that folder
for file in os.listdir(pkg_loc):
if any(u_re.match(file) for u_re in _uninstall_re):
to_del.add(os.path.join(pkg_loc, file))
else: # specific folder (non site-packages)
if os.path.isdir(pkg_loc):
to_del.add(pkg_loc)
# finding package's files into that folder
if os.path.isdir(pkg_loc):
for file in os.listdir(pkg_loc):
if any(u_re.match(file) for u_re in _uninstall_re):
to_del.add(os.path.join(pkg_loc, file))
else:
# single file installation
for ext in '.py .pyc .pyo'.split():
_p = pkg_loc + ext
if os.path.exists(_p):
to_del.add(_p)
## Checking for package's scripts...
if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'):
for script in dist.metadata_listdir('scripts'):
to_del.add(os.path.join(BIN, script))
## If we are on Windows we have to remove *.bat files too
if is_windows():
to_del.add(os.path.join(BIN, script) + '.bat')
## Very important!
## We want to remove console scripts too.
if dist.has_metadata('entry_points.txt'):
config = ConfigParser.ConfigParser()
config.readfp(File(dist.get_metadata_lines('entry_points.txt')))
win32 = sys.platform == 'win32'
if config.has_section('console_scripts'):
for name, value in config.items('console_scripts'):
n = os.path.join(BIN, name)
## Searches in the local path
if not os.path.exists(n) and n.startswith('/usr/bin'):
n = os.path.join('/usr/local/bin', name)
## Check existance before adding to `to-del` set.
if os.path.exists(n):
to_del.add(n)
elif win32 and os.path.exists(n + '.exe'):
to_del.add(n + '.exe')
to_del.add(n + '.exe.manifest')
to_del.add(n + '-script.py')
return to_del
from setuptools.package_index import PackageIndex
from pkgtools.pypi import PyPIXmlRpc, PyPIJson, real_name
from pyg.core import Version, args_manager
from pyg.utils import FileMapper, name, ext, right_egg, version_egg, is_windows
from pyg.log import logger
__all__ = ['PREFERENCES', 'ReqManager', 'get_versions', 'get_links', \
'highest_version', 'request']
## This constant holds files priority
PREFERENCES = ('.egg', '.tar.gz', '.tar.bz2', '.zip')
if is_windows():
PREFERENCES = ('.exe', '.msi') + PREFERENCES
def get_versions(req):
'''
Return all versions the given requirement can match.
For example, if requirement is `pyg>=0.6` it will return: [0.6, 0.7].
When a package has no files on PyPI (but at least a release) we have to
look for version manually, with regular expressions.
`req` should be a Requirement object (from pyg.core).
'''
_version_re = r'{0}-(\d[\d\w.-]*).*'
name = req.name
pypi = PyPIXmlRpc()
import urllib2
import operator
import urlparse
from hashlib import md5
from pyg.utils import ext, right_egg, is_windows
from pyg.web import ReqManager, get_links, download
from pyg.core import Version, Egg, Archive, Binary, ReqSet, InstallationError, args_manager
from pyg.log import logger
__all__ = ['Requirement', 'WINDOWS_EXT']
WINDOWS_EXT = ('.exe', '.msi') if is_windows() else ()
class Requirement(object):
OPMAP = {'==': operator.eq,
'>=': operator.ge,
'>': operator.gt,
'<=': operator.le,
'<': operator.lt,
'!=': lambda a,b: a != b,
None: lambda a,b: True, ##FIXME: does None really work?
}
version = op = None