def from_req_file(filepath):
path = os.path.abspath(filepath)
not_installed = set()
parser = init_parser()
with open(path) as f:
logger.info('{0}:', path)
for line in f:
line = line.strip()
if line.startswith('#'):
logger.debug('debug: Comment found: {0}', line)
continue
try:
logger.indent = 8
logger.info('Installing: {0}', line)
logger.indent = 16
parser.dispatch(argv=['install'] + line.split())
except AlreadyInstalled:
continue
except InstallationError:
not_installed.add(line)
except SystemExit as e:
if e.code != 0:
logger.warn('W: {0} tried to raise SystemExit: skipping installation')
else:
logger.info('{0} tried to raise SystemExit, but the exit code was 0')
if not_installed:
logger.warn('These packages have not been installed:')
logger.indent = 8
for req in not_installed:
logger.warn(req)
logger.indent = 0
raise InstallationError()
def remove_files(self, package):
uninst = Uninstaller(package, yes=True)
to_del = uninst.find_files()
if not to_del:
logger.info('No files to remove found')
return
# XXX: tempfile.mktemp is deprecated, but all other functions create
# the directory and shutil does not want that.
tempdir = tempfile.mktemp()
self.removed[package] = {}
self.removed[package][tempdir] = []
for path in to_del:
self.removed[package][tempdir].append(path)
# We store files-to-delete into a temporary directory:
# if something goes wrong during the upgrading we can
# restore the original files.
p = os.path.join(tempdir, os.path.basename(path))
try:
shutil.copy2(path, p)
# It is a directory
except IOError:
try:
shutil.copytree(path, p)
except OSError:
logger.debug('debug: shutil.copytree raised OSError')
continue
logger.enabled = False
uninst.uninstall()
logger.enabled = True
def _clean(self):
logger.debug('debug: Removing temporary directories')
for package, dirs in self.removed.iteritems():
try:
shutil.rmtree(dirs.keys()[0])
except shutil.Error:
logger.debug('debug: Error while removing {0}', dirs.keys()[0])
continue
def _clean(dir):
'''
Clean the `dir` directory: it removes all top-level files, leaving only sub-directories.
'''
logger.debug('debug: bundle: cleaning build dir')
for file in (d for d in os.listdir(dir) if os.path.isfile(os.path.join(dir, d))):
logger.debug('debug: Removing: {0}', file)
os.remove(os.path.join(dir, file))
def __init__(self, skip=False):
## You should use skip=True when you want to upgrade a single package.
## Just do:
##>>> u = Updater(skip=True)
##>>> u.upgrade(package_name, json, version)
if not skip:
logger.debug('Loading list of installed packages... ', addn=False)
self.working_set = list(iter(pkg_resources.working_set))
logger.info('{0} packages loaded', len(self.working_set))
self.removed = {}
def restore_files(self, package):
try:
package = self.removed[package]
except KeyError:
logger.debug('debug: `{0}` not found in self.removed', package)
return
tempdir = package.keys()[0]
for path in package[tempdir]:
p = os.path.join(tempdir, os.path.basename(path))
try:
shutil.copy2(p, path)
## It is a directory
except IOError:
shutil.copytree(p, path)
def _download_all(self, dir):
reqs = list(self.reqs)
already_downloaded = set()
while reqs:
r = reqs.pop()
# Hack for virtualenvs
if r.name.lower == 'python':
continue
if any(r.name == rq.name for rq in already_downloaded):
logger.debug('debug: Already downloaded: {0}', r)
continue
if any(r == rq for rq in self.exclude):
logger.info('Excluding {0}', r)
continue
logger.indent = 0
logger.info('{0}:', r)
logger.indent = 8
try:
dist = SDist(self._download(dir, r))
self.callback(r, dist)
except ConfigParser.MissingSectionHeaderError:
continue
try:
logger.info('Looking for {0} dependencies', r)
logger.indent += 8
found = False
try:
requirements = dist.file('requires.txt')
except KeyError:
requirements = []
for requirement in requirements:
rq = Requirement(requirement)
if rq not in already_downloaded:
logger.info('Found: {0}', requirement)
reqs.append(rq)
found = True
if not found:
logger.info('None found')
except KeyError:
logger.debug('debug: requires.txt not found for {0}', dist)
try:
as_req = dist.as_req
except KeyError:
as_req = str(r)
already_downloaded.add(Requirement(as_req))
logger.indent = 0
logger.success('Finished processing dependencies')
def load_options():
import os.path
from pyg.core import args_manager
from pyg.locations import CFG_FILES
from pyg.log import logger
if CFG_FILES:
for cfg in CFG_FILES:
if os.path.exists(cfg):
logger.debug('Loading options from {0}', cfg)
## This is for potential warnings
logger.indent = 8
args_manager.load(cfg)
logger.indent = 0
break
def call_setup(path, a):
'''
Call the `setup.py` file under the specified path with the given arguments.
Note that `path` must be the directory in which the setup file is located,
not the direct path to the file. For example, `/home/user/packages/pyg-0.7/'
is right (assuming there is a `setup.py` file in it), while
'/home/user/packages/pyg-0.7/setup.py` is not.
'''
code = SETUP_PY_TEMPLATE.format(os.path.join(path, 'setup.py'))
args = [sys.executable, '-c', code] + a
if under_virtualenv():
logger.debug('debug: virtualenv detected')
headers = os.path.join(sys.prefix, 'include', 'site', 'python' + PYTHON_VERSION)
args += ['--install-headers', headers]
return call_subprocess(args, cwd=path)
def install(self):
'''
Given a pathname containing a :file:`setup.py` file, install the package.
First runs `setup.py egg_info` to find out requirements, then runs ``setup.py install``.
'''
if self.reqset is not None:
logger.info('Running setup.py egg_info for {0}', self.name)
call_setup(self.path, ['egg_info', '--egg-base', self.tempdir])
try:
dist = DirTools(glob.glob(os.path.join(self.path, '*egg-info'))[0])
except (IndexError, ValueError):
pass
else:
try:
for r in dist.file('requires.txt'):
self.reqset.add(r)
except (KeyError, ConfigParser.MissingSectionHeaderError):
logger.debug('debug: requires.txt not found')
#try:
# for r in dist.file('dependency_links.txt'):
# self.reqset.add(r)
#except (KeyError, ConfigParser.MissingSectionHeaderError):
# logger.debug('debug: dependency_links.txt not found')
args = []
if args_manager['install']['install_dir'] != INSTALL_DIR:
dir = os.path.abspath(args_manager['install']['install_dir'])
if not os.path.exists(dir):
os.makedirs(dir)
## Setuptools would not install the package without this hack
os.putenv('PYTHONPATH', dir)
args += ['--install-purelib', dir, '--install-platlib', dir]
if args_manager['install']['no_scripts']:
args += ['--install-scripts', self.tempdir]
if args_manager['install']['no_data']:
args += ['--install-data', self.tempdir]
if args_manager['install']['user']:
args += ['--user']
run_setup(self.path, self.name, args=args, exc=InstallationError)
def install(self):
eggpath = os.path.join(self.idir, self.eggname)
if os.path.exists(eggpath) and not args_manager['install']['upgrade']:
logger.info('{0} is already installed', self.packname)
raise AlreadyInstalled
logger.info('Installing {0} egg file', self.packname)
with ZipFile(self.fobj) as z:
z.extractall(eggpath)
logger.info('Adding egg file to sys.path')
with open(EASY_INSTALL) as f: ## TODO: Fix the opening mode to read and write simultaneously
lines = f.readlines()
with open(EASY_INSTALL, 'w') as f:
try:
f.writelines(lines[:-1])
f.write('./' + self.eggname + '\n')
f.write(lines[-1])
## When using this file for the first time
except IndexError:
pass
dist = EggDir(eggpath)
## Install scripts in the setuptools' way
if not args_manager['install']['no_scripts']:
for name, content, mode in script_args(dist):
logger.info('Installing {0} script to {1}', name, BIN)
target = os.path.join(BIN, name)
with open(target, 'w' + mode) as f:
f.write(content)
os.chmod(target, 0755)
else:
logger.info('Scripts not installed')
logger.info('Looking for requirements...')
try:
for req in dist.file('requires.txt'):
self.reqset.add(req)
except KeyError:
logger.debug('requires.txt not found')
def download(self, dest):
dest = os.path.abspath(dest)
files = self.files()
## We need a placeholder because of the nested for loops
success = False
for p in self.pref:
if success:
break
if not files[p]:
logger.warn('{0} files not found. Continue searching...', p)
continue
for v, name, hash, url in files[p]:
if success:
break
if p == '.egg' and not right_egg(name):
logger.info('Found egg file for another Python version: {0}. Continue searching...', version_egg(name))
continue
try:
data = download(url, 'Retrieving data for {0}'.format(self.name)).getvalue()
except (urllib2.URLError, urllib2.HTTPError) as e:
logger.debug('urllib2 error: {0}', e.args)
continue
if not data:
logger.debug('debug: Request failed')
continue
if not os.path.exists(dest):
os.makedirs(dest)
try:
# Fix for packages with no version in the name
if '-' not in name:
name = '{0}-{1}{2}'.format(name, v, p)
logger.info('Writing data into {0}', name)
with open(os.path.join(dest, name), 'w') as f:
f.write(data)
except (IOError, OSError):
logger.debug('debug: Error while writing data')
continue
logger.success('{0} downloaded successfully', self.name)
success = True
self.downloaded_name = name
self.downloaded_version = v
def bundle(self, include_manifest=True, build_dir=True, additional_files=[]):
'''
Create a bundle of the specified package:
1. Download all required packages (included dependencies)
2. Clean the build directory
3. Collect the packages in a single zip file (bundle)
4. Add the manifest file
5. Move the bundle from the build dir to the destination
'''
def _add_to_archive(zfile, dir):
for file in os.listdir(dir):
path = os.path.join(dir, file)
if os.path.isfile(path):
zfile.write(path, os.path.join(dir, file)[len(tempdir):])
elif os.path.isdir(path):
_add_to_archive(zfile, path)
with TempDir() as tempdir, TempDir() as bundle_dir:
## Step 1: we create the `build` directory
## If you choose to create a bundle without the build directory,
## be aware that your bundle will not be compatible with Pip.
#####
if build_dir:
build = os.path.join(tempdir, 'build')
os.mkdir(build)
else:
build = tempdir
tmp_bundle = os.path.join(bundle_dir, self.bundle_name)
## Step 2: we *recursively* download all required packages
#####
reqs = list(self.reqs)
already_downloaded = set()
while reqs:
r = reqs.pop()
if any(rq.name == r.name and rq.match(r.version) for rq in self.exclude):
logger.info('Excluding {0}', r)
continue
logger.indent = 0
logger.info('{0}:', r)
logger.indent = 8
try:
dist = SDist(self._download(build, r))
except ConfigParser.MissingSectionHeaderError:
continue
try:
logger.info('Looking for {0} dependencies', r)
logger.indent += 8
for requirement in dist.file('requires.txt'):
if requirement not in already_downloaded:
logger.info('Found: {0}', requirement)
reqs.append(Requirement(requirement))
except KeyError:
logger.debug('debug: requires.txt not found for {0}', dist)
try:
as_req = dist.as_req
except KeyError:
as_req = str(r)
already_downloaded.add(as_req)
logger.indent = 0
logger.success('Finished processing dependencies')
## Step 3: we remove all files in the build directory, so we make sure
## that when we collect packages we collect only dirs
#####
self._clean(build)
## Step 4: we collect the downloaded packages and bundle all together
## in a single file (zipped)
#####
logger.info('Adding packages to the bundle')
bundle = zipfile.ZipFile(tmp_bundle, mode='w')
_add_to_archive(bundle, build)
## Step 4: add the manifest file
if include_manifest:
logger.info('Adding the manifest file')
bundle.writestr('pyg-manifest.txt', Bundler.MANIFEST.format('\n'.join(self.bundled)))
# Additional files to add
for path in additional_files:
try:
_add_to_archive(bundle, path)
except (IOError, OSError):
logger.debug('debug: Error while adding an additional file: {0}', path)
bundle.close()
## Last step: move the bundle to the current working directory
dest = os.path.join(os.getcwd(), self.bundle_name)
if os.path.exists(dest):
logger.debug('debug: dest already exists, removing it')
os.remove(dest)
shutil.move(tmp_bundle, os.getcwd())
def find_files(self):
_un_re = re.compile(r'{0}(-(\d\.?)+(\-py\d\.\d)?\.(egg|egg\-info))?$'.format(self.name), re.I)
_un2_re = re.compile(r'{0}(?:(\.py|\.pyc))'.format(self.name), re.I)
_un3_re = re.compile(r'{0}.*\.so'.format(self.name), re.I)
_uninstall_re = [_un_re, _un2_re, _un3_re]
to_del = set()
try:
dist = pkg_resources.get_distribution(self.name)
except pkg_resources.DistributionNotFound:
logger.debug('debug: Distribution not found: {0}', self.name)
## Create a fake distribution
## In Python2.6 we can only use site.USER_SITE
class FakeDist(object):
def __init__(self, o):
self._orig_o = o
def __getattr__(self, a):
if a == 'location':
return USER_SITE
elif a == 'egg_name':
return (lambda *a: self._orig_o.name + '.egg')
return (lambda *a: False)
dist = FakeDist(self)
pkg_loc = dist.location
glob_folder = False
if pkg_loc in ALL_SITE_PACKAGES:
# try to detect the real package location
if dist.has_metadata('top_level.txt'):
pkg_loc = os.path.join( pkg_loc,
dist.get_metadata_lines('top_level.txt').next())
else:
glob_folder = True
# detect egg-info location
_base_name = dist.egg_name().split('-')
for n in range(len(_base_name) + 1):
egg_info_dir = os.path.join(
dist.location,
'-'.join(_base_name[:-n if n else None]) + '.egg-info'
)
if os.path.exists(egg_info_dir):
for file in os.listdir(egg_info_dir):
if any(u_re.match(file) for u_re in _uninstall_re):
to_del.add(os.path.join(egg_info_dir, file))
to_del.add(egg_info_dir)
break
if glob_folder:
# track individual files inside that folder
for file in os.listdir(pkg_loc):
if any(u_re.match(file) for u_re in _uninstall_re):
to_del.add(os.path.join(pkg_loc, file))
else: # specific folder (non site-packages)
if os.path.isdir(pkg_loc):
to_del.add(pkg_loc)
# finding package's files into that folder
if os.path.isdir(pkg_loc):
for file in os.listdir(pkg_loc):
if any(u_re.match(file) for u_re in _uninstall_re):
to_del.add(os.path.join(pkg_loc, file))
else:
# single file installation
for ext in '.py .pyc .pyo'.split():
_p = pkg_loc + ext
if os.path.exists(_p):
to_del.add(_p)
## Checking for package's scripts...
if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'):
for script in dist.metadata_listdir('scripts'):
to_del.add(os.path.join(BIN, script))
## If we are on Windows we have to remove *.bat files too
if is_windows():
to_del.add(os.path.join(BIN, script) + '.bat')
## Very important!
## We want to remove console scripts too.
if dist.has_metadata('entry_points.txt'):
config = ConfigParser.ConfigParser()
config.readfp(File(dist.get_metadata_lines('entry_points.txt')))
win32 = sys.platform == 'win32'
if config.has_section('console_scripts'):
for name, value in config.items('console_scripts'):
n = os.path.join(BIN, name)
## Searches in the local path
if not os.path.exists(n) and n.startswith('/usr/bin'):
n = os.path.join('/usr/local/bin', name)
## Check existance before adding to `to-del` set.
if os.path.exists(n):
to_del.add(n)
elif win32 and os.path.exists(n + '.exe'):
to_del.add(n + '.exe')
to_del.add(n + '.exe.manifest')
to_del.add(n + '-script.py')
return to_del
def bundle(self, dest=None, include_manifest=True, build_dir=True, additional_files=[], add_func=None):
'''
Create a bundle of the specified package:
1. Download all required packages (included dependencies)
2. Clean the build directory
3. Collect the packages in a single zip file (bundle)
4. Add the manifest file
5. Move the bundle from the build dir to the destination
'''
destination = dest or os.getcwd()
with TempDir() as tempdir:
with TempDir() as bundle_dir:
## Step 0: we create the `build` directory
## If you choose to create a bundle without the build directory,
## be aware that your bundle will not be compatible with Pip.
#####
if build_dir:
build = os.path.join(tempdir, 'build')
os.mkdir(build)
else:
build = tempdir
tmp_bundle = os.path.join(bundle_dir, self.bundle_name)
## Step 1: we *recursively* download all required packages
#####
self._download_all(build)
## Step 2: we remove all files in the build directory, so we make sure
## that when we collect packages we collect only dirs
#####
self._clean(build)
## Step 3: we collect the downloaded packages and bundle all together
## in a single file (zipped)
#####
logger.info('Adding packages to the bundle')
bundle = zipfile.ZipFile(tmp_bundle, mode='w')
self._add_to_archive(bundle, build, len(tempdir))
## Step 4: add the manifest file
if include_manifest:
logger.info('Adding the manifest file')
bundle.writestr('pyg-manifest.txt', Bundler.MANIFEST.format('\n'.join(self.bundled)))
# Additional files to add
for path in additional_files:
try:
self._add_to_archive(bundle, path, len(tempdir))
except (IOError, OSError):
logger.debug('debug: Error while adding an additional file: {0}', path)
if add_func is not None:
self._add_to_archive(bundle, add_func(), len(tempdir))
bundle.close()
## Last step: move the bundle to the current working directory
dest = os.path.join(destination, self.bundle_name)
if os.path.exists(dest):
logger.debug('debug: dest already exists, removing it')
os.remove(dest)
shutil.move(tmp_bundle, destination)