Beispiel #1
0
 def from_req_file(filepath):
     path = os.path.abspath(filepath)
     not_installed = set()
     parser = init_parser()
     with open(path) as f:
         logger.info('{0}:', path)
         for line in f:
             line = line.strip()
             if line.startswith('#'):
                 logger.debug('debug: Comment found: {0}', line)
                 continue
             try:
                 logger.indent = 8
                 logger.info('Installing: {0}', line)
                 logger.indent = 16
                 parser.dispatch(argv=['install'] + line.split())
             except AlreadyInstalled:
                 continue
             except InstallationError:
                 not_installed.add(line)
             except SystemExit as e:
                 if e.code != 0:
                     logger.warn('W: {0} tried to raise SystemExit: skipping installation')
                 else:
                     logger.info('{0} tried to raise SystemExit, but the exit code was 0')
     if not_installed:
         logger.warn('These packages have not been installed:')
         logger.indent = 8
         for req in not_installed:
             logger.warn(req)
         logger.indent = 0
         raise InstallationError()
Beispiel #2
0
    def remove_files(self, package):
        uninst = Uninstaller(package, yes=True)
        to_del = uninst.find_files()
        if not to_del:
            logger.info('No files to remove found')
            return

        # XXX: tempfile.mktemp is deprecated, but all other functions create
        # the directory and shutil does not want that.
        tempdir = tempfile.mktemp()
        self.removed[package] = {}
        self.removed[package][tempdir] = []
        for path in to_del:
            self.removed[package][tempdir].append(path)

            # We store files-to-delete into a temporary directory:
            # if something goes wrong during the upgrading we can
            # restore the original files.
            p = os.path.join(tempdir, os.path.basename(path))
            try:
                shutil.copy2(path, p)
            # It is a directory
            except IOError:
                try:
                    shutil.copytree(path, p)
                except OSError:
                    logger.debug('debug: shutil.copytree raised OSError')
                    continue
        logger.enabled = False
        uninst.uninstall()
        logger.enabled = True
Beispiel #3
0
 def _clean(self):
     logger.debug('debug: Removing temporary directories')
     for package, dirs in self.removed.iteritems():
         try:
             shutil.rmtree(dirs.keys()[0])
         except shutil.Error:
             logger.debug('debug: Error while removing {0}', dirs.keys()[0])
             continue
Beispiel #4
0
    def _clean(dir):
        '''
        Clean the `dir` directory: it removes all top-level files, leaving only sub-directories.
        '''

        logger.debug('debug: bundle: cleaning build dir')
        for file in (d for d in os.listdir(dir) if os.path.isfile(os.path.join(dir, d))):
            logger.debug('debug: Removing: {0}', file)
            os.remove(os.path.join(dir, file))
Beispiel #5
0
    def __init__(self, skip=False):

        ## You should use skip=True when you want to upgrade a single package.
        ## Just do:
        ##>>> u = Updater(skip=True)
        ##>>> u.upgrade(package_name, json, version)
        if not skip:
            logger.debug('Loading list of installed packages... ', addn=False)
            self.working_set = list(iter(pkg_resources.working_set))
            logger.info('{0} packages loaded', len(self.working_set))
        self.removed = {}
Beispiel #6
0
 def restore_files(self, package):
     try:
         package = self.removed[package]
     except KeyError:
         logger.debug('debug: `{0}` not found in self.removed', package)
         return
     tempdir = package.keys()[0]
     for path in package[tempdir]:
         p = os.path.join(tempdir, os.path.basename(path))
         try:
             shutil.copy2(p, path)
         ## It is a directory
         except IOError:
             shutil.copytree(p, path)
Beispiel #7
0
Datei: inst.py Projekt: Malex/pyg
 def _download_all(self, dir):
     reqs = list(self.reqs)
     already_downloaded = set()
     while reqs:
         r = reqs.pop()
         # Hack for virtualenvs
         if r.name.lower == 'python':
             continue
         if any(r.name == rq.name for rq in already_downloaded):
             logger.debug('debug: Already downloaded: {0}', r)
             continue
         if any(r == rq for rq in self.exclude):
             logger.info('Excluding {0}', r)
             continue
         logger.indent = 0
         logger.info('{0}:', r)
         logger.indent = 8
         try:
             dist = SDist(self._download(dir, r))
             self.callback(r, dist)
         except ConfigParser.MissingSectionHeaderError:
             continue
         try:
             logger.info('Looking for {0} dependencies', r)
             logger.indent += 8
             found = False
             try:
                 requirements = dist.file('requires.txt')
             except KeyError:
                 requirements = []
             for requirement in requirements:
                 rq = Requirement(requirement)
                 if rq not in already_downloaded:
                     logger.info('Found: {0}', requirement)
                     reqs.append(rq)
                     found = True
             if not found:
                 logger.info('None found')
         except KeyError:
             logger.debug('debug: requires.txt not found for {0}', dist)
         try:
             as_req = dist.as_req
         except KeyError:
             as_req = str(r)
         already_downloaded.add(Requirement(as_req))
     logger.indent = 0
     logger.success('Finished processing dependencies')
Beispiel #8
0
def load_options():
    import os.path

    from pyg.core import args_manager
    from pyg.locations import CFG_FILES
    from pyg.log import logger

    if CFG_FILES:
        for cfg in CFG_FILES:
            if os.path.exists(cfg):
                logger.debug('Loading options from {0}', cfg)

                ## This is for potential warnings
                logger.indent = 8
                args_manager.load(cfg)
                logger.indent = 0
                break
Beispiel #9
0
def call_setup(path, a):
    '''
    Call the `setup.py` file under the specified path with the given arguments.

    Note that `path` must be the directory in which the setup file is located,
    not the direct path to the file. For example, `/home/user/packages/pyg-0.7/'
    is right (assuming there is a `setup.py` file in it), while
    '/home/user/packages/pyg-0.7/setup.py` is not.
    '''

    code = SETUP_PY_TEMPLATE.format(os.path.join(path, 'setup.py'))
    args =  [sys.executable, '-c', code] + a
    if under_virtualenv():
        logger.debug('debug: virtualenv detected')
        headers = os.path.join(sys.prefix, 'include', 'site', 'python' + PYTHON_VERSION)
        args += ['--install-headers', headers]
    return call_subprocess(args, cwd=path)
Beispiel #10
0
Datei: core.py Projekt: Malex/pyg
    def install(self):
        '''
        Given a pathname containing a :file:`setup.py` file, install the package.
        First runs `setup.py egg_info` to find out requirements, then runs ``setup.py install``.
        '''

        if self.reqset is not None:
            logger.info('Running setup.py egg_info for {0}', self.name)
            call_setup(self.path, ['egg_info', '--egg-base', self.tempdir])
            try:
                dist = DirTools(glob.glob(os.path.join(self.path, '*egg-info'))[0])
            except (IndexError, ValueError):
                pass
            else:
                try:
                    for r in dist.file('requires.txt'):
                        self.reqset.add(r)
                except (KeyError, ConfigParser.MissingSectionHeaderError):
                    logger.debug('debug: requires.txt not found')
                #try:
                #    for r in dist.file('dependency_links.txt'):
                #        self.reqset.add(r)
                #except (KeyError, ConfigParser.MissingSectionHeaderError):
                #    logger.debug('debug: dependency_links.txt not found')
        args = []
        if args_manager['install']['install_dir'] != INSTALL_DIR:
            dir = os.path.abspath(args_manager['install']['install_dir'])
            if not os.path.exists(dir):
                os.makedirs(dir)

            ## Setuptools would not install the package without this hack
            os.putenv('PYTHONPATH', dir)
            args += ['--install-purelib', dir, '--install-platlib', dir]
        if args_manager['install']['no_scripts']:
            args += ['--install-scripts', self.tempdir]
        if args_manager['install']['no_data']:
            args += ['--install-data', self.tempdir]
        if args_manager['install']['user']:
            args += ['--user']
        run_setup(self.path, self.name, args=args, exc=InstallationError)
Beispiel #11
0
Datei: core.py Projekt: Malex/pyg
    def install(self):
        eggpath = os.path.join(self.idir, self.eggname)
        if os.path.exists(eggpath) and not args_manager['install']['upgrade']:
            logger.info('{0} is already installed', self.packname)
            raise AlreadyInstalled
        logger.info('Installing {0} egg file', self.packname)
        with ZipFile(self.fobj) as z:
            z.extractall(eggpath)
        logger.info('Adding egg file to sys.path')
        with open(EASY_INSTALL) as f: ## TODO: Fix the opening mode to read and write simultaneously
            lines = f.readlines()
        with open(EASY_INSTALL, 'w') as f:
            try:
                f.writelines(lines[:-1])
                f.write('./' + self.eggname + '\n')
                f.write(lines[-1])
            ## When using this file for the first time
            except IndexError:
                pass
        dist = EggDir(eggpath)

        ## Install scripts in the setuptools' way
        if not args_manager['install']['no_scripts']:
            for name, content, mode in script_args(dist):
                logger.info('Installing {0} script to {1}', name, BIN)
                target = os.path.join(BIN, name)
                with open(target, 'w' + mode) as f:
                    f.write(content)
                    os.chmod(target, 0755)
        else:
            logger.info('Scripts not installed')
        logger.info('Looking for requirements...')
        try:
            for req in dist.file('requires.txt'):
                self.reqset.add(req)
        except KeyError:
            logger.debug('requires.txt not found')
Beispiel #12
0
Datei: web.py Projekt: Malex/pyg
    def download(self, dest):
        dest = os.path.abspath(dest)
        files = self.files()

        ## We need a placeholder because of the nested for loops
        success = False

        for p in self.pref:
            if success:
                break
            if not files[p]:
                logger.warn('{0} files not found. Continue searching...', p)
                continue
            for v, name, hash, url in files[p]:
                if success:
                    break
                if p == '.egg' and not right_egg(name):
                    logger.info('Found egg file for another Python version: {0}. Continue searching...',                               version_egg(name))
                    continue
                try:
                    data = download(url, 'Retrieving data for {0}'.format(self.name)).getvalue()
                except (urllib2.URLError, urllib2.HTTPError) as e:
                    logger.debug('urllib2 error: {0}', e.args)
                    continue
                if not data:
                    logger.debug('debug: Request failed')
                    continue
                if not os.path.exists(dest):
                    os.makedirs(dest)
                try:
                    # Fix for packages with no version in the name
                    if '-' not in name:
                        name = '{0}-{1}{2}'.format(name, v, p)
                    logger.info('Writing data into {0}', name)
                    with open(os.path.join(dest, name), 'w') as f:
                        f.write(data)
                except (IOError, OSError):
                    logger.debug('debug: Error while writing data')
                    continue
                logger.success('{0} downloaded successfully', self.name)
                success = True
            self.downloaded_name = name
            self.downloaded_version = v
Beispiel #13
0
    def bundle(self, include_manifest=True, build_dir=True, additional_files=[]):
        '''
        Create a bundle of the specified package:

            1. Download all required packages (included dependencies)
            2. Clean the build directory
            3. Collect the packages in a single zip file (bundle)
            4. Add the manifest file
            5. Move the bundle from the build dir to the destination
        '''

        def _add_to_archive(zfile, dir):
            for file in os.listdir(dir):
                path = os.path.join(dir, file)
                if os.path.isfile(path):
                    zfile.write(path, os.path.join(dir, file)[len(tempdir):])
                elif os.path.isdir(path):
                    _add_to_archive(zfile, path)

        with TempDir() as tempdir, TempDir() as bundle_dir:
            ## Step 1: we create the `build` directory
            ## If you choose to create a bundle without the build directory,
            ## be aware that your bundle will not be compatible with Pip.
            #####
            if build_dir:
                build = os.path.join(tempdir, 'build')
                os.mkdir(build)
            else:
                build = tempdir
            tmp_bundle = os.path.join(bundle_dir, self.bundle_name)

            ## Step 2: we *recursively* download all required packages
            #####
            reqs = list(self.reqs)
            already_downloaded = set()
            while reqs:
                r = reqs.pop()
                if any(rq.name == r.name and rq.match(r.version) for rq in self.exclude):
                    logger.info('Excluding {0}', r)
                    continue
                logger.indent = 0
                logger.info('{0}:', r)
                logger.indent = 8
                try:
                    dist = SDist(self._download(build, r))
                except ConfigParser.MissingSectionHeaderError:
                    continue
                try:
                    logger.info('Looking for {0} dependencies', r)
                    logger.indent += 8
                    for requirement in dist.file('requires.txt'):
                        if requirement not in already_downloaded:
                            logger.info('Found: {0}', requirement)
                            reqs.append(Requirement(requirement))
                except KeyError:
                    logger.debug('debug: requires.txt not found for {0}', dist)
                try:
                    as_req = dist.as_req
                except KeyError:
                    as_req = str(r)
                already_downloaded.add(as_req)
            logger.indent = 0
            logger.success('Finished processing dependencies')

            ## Step 3: we remove all files in the build directory, so we make sure
            ## that when we collect packages we collect only dirs
            #####
            self._clean(build)

            ## Step 4: we collect the downloaded packages and bundle all together
            ## in a single file (zipped)
            #####
            logger.info('Adding packages to the bundle')
            bundle = zipfile.ZipFile(tmp_bundle, mode='w')
            _add_to_archive(bundle, build)

            ## Step 4: add the manifest file
            if include_manifest:
                logger.info('Adding the manifest file')
                bundle.writestr('pyg-manifest.txt', Bundler.MANIFEST.format('\n'.join(self.bundled)))

            # Additional files to add
            for path in additional_files:
                try:
                    _add_to_archive(bundle, path)
                except (IOError, OSError):
                    logger.debug('debug: Error while adding an additional file: {0}', path)
            bundle.close()

            ## Last step: move the bundle to the current working directory
            dest = os.path.join(os.getcwd(), self.bundle_name)
            if os.path.exists(dest):
                logger.debug('debug: dest already exists, removing it')
                os.remove(dest)
            shutil.move(tmp_bundle, os.getcwd())
Beispiel #14
0
    def find_files(self):
        _un_re = re.compile(r'{0}(-(\d\.?)+(\-py\d\.\d)?\.(egg|egg\-info))?$'.format(self.name), re.I)
        _un2_re = re.compile(r'{0}(?:(\.py|\.pyc))'.format(self.name), re.I)
        _un3_re = re.compile(r'{0}.*\.so'.format(self.name), re.I)
        _uninstall_re = [_un_re, _un2_re, _un3_re]

        to_del = set()
        try:
            dist = pkg_resources.get_distribution(self.name)
        except pkg_resources.DistributionNotFound:
            logger.debug('debug: Distribution not found: {0}', self.name)

            ## Create a fake distribution
            ## In Python2.6 we can only use site.USER_SITE
            class FakeDist(object):
                def __init__(self, o):
                    self._orig_o = o
                def __getattr__(self, a):
                    if a == 'location':
                        return USER_SITE
                    elif a == 'egg_name':
                        return (lambda *a: self._orig_o.name + '.egg')
                    return (lambda *a: False)
            dist = FakeDist(self)

        pkg_loc = dist.location

        glob_folder = False

        if pkg_loc in ALL_SITE_PACKAGES:

            # try to detect the real package location
            if dist.has_metadata('top_level.txt'):
                pkg_loc = os.path.join( pkg_loc,
                    dist.get_metadata_lines('top_level.txt').next())
            else:
                glob_folder = True

        # detect egg-info location
        _base_name = dist.egg_name().split('-')
        for n in range(len(_base_name) + 1):
            egg_info_dir = os.path.join(
                dist.location,
                '-'.join(_base_name[:-n if n else None]) + '.egg-info'
            )
            if os.path.exists(egg_info_dir):
                for file in os.listdir(egg_info_dir):
                    if any(u_re.match(file) for u_re in _uninstall_re):
                        to_del.add(os.path.join(egg_info_dir, file))
                to_del.add(egg_info_dir)
                break

        if glob_folder:
            # track individual files inside that folder

            for file in os.listdir(pkg_loc):
                if any(u_re.match(file) for u_re in _uninstall_re):
                    to_del.add(os.path.join(pkg_loc, file))
        else: # specific folder (non site-packages)
            if os.path.isdir(pkg_loc):
                to_del.add(pkg_loc)

            # finding package's files into that folder
            if os.path.isdir(pkg_loc):
                for file in os.listdir(pkg_loc):
                    if any(u_re.match(file) for u_re in _uninstall_re):
                        to_del.add(os.path.join(pkg_loc, file))
            else:
                # single file installation
                for ext in '.py .pyc .pyo'.split():
                    _p = pkg_loc + ext
                    if os.path.exists(_p):
                        to_del.add(_p)

        ## Checking for package's scripts...
        if dist.has_metadata('scripts') and dist.metadata_isdir('scripts'):
            for script in dist.metadata_listdir('scripts'):
                to_del.add(os.path.join(BIN, script))

                ## If we are on Windows we have to remove *.bat files too
                if is_windows():
                    to_del.add(os.path.join(BIN, script) + '.bat')

        ## Very important!
        ## We want to remove console scripts too.
        if dist.has_metadata('entry_points.txt'):
            config = ConfigParser.ConfigParser()
            config.readfp(File(dist.get_metadata_lines('entry_points.txt')))
            win32 = sys.platform == 'win32'
            if config.has_section('console_scripts'):
                for name, value in config.items('console_scripts'):
                    n = os.path.join(BIN, name)

                    ## Searches in the local path
                    if not os.path.exists(n) and n.startswith('/usr/bin'):
                        n = os.path.join('/usr/local/bin', name)

                    ## Check existance before adding to `to-del` set.
                    if os.path.exists(n):
                        to_del.add(n)
                    elif win32 and os.path.exists(n + '.exe'):
                        to_del.add(n + '.exe')
                        to_del.add(n + '.exe.manifest')
                        to_del.add(n + '-script.py')
        return to_del
Beispiel #15
0
Datei: inst.py Projekt: Malex/pyg
    def bundle(self, dest=None, include_manifest=True, build_dir=True, additional_files=[], add_func=None):
        '''
        Create a bundle of the specified package:

            1. Download all required packages (included dependencies)
            2. Clean the build directory
            3. Collect the packages in a single zip file (bundle)
            4. Add the manifest file
            5. Move the bundle from the build dir to the destination
        '''

        destination = dest or os.getcwd()
        with TempDir() as tempdir:
            with TempDir() as bundle_dir:
                ## Step 0: we create the `build` directory
                ## If you choose to create a bundle without the build directory,
                ## be aware that your bundle will not be compatible with Pip.
                #####
                if build_dir:
                    build = os.path.join(tempdir, 'build')
                    os.mkdir(build)
                else:
                    build = tempdir
                tmp_bundle = os.path.join(bundle_dir, self.bundle_name)

                ## Step 1: we *recursively* download all required packages
                #####
                self._download_all(build)

                ## Step 2: we remove all files in the build directory, so we make sure
                ## that when we collect packages we collect only dirs
                #####
                self._clean(build)

                ## Step 3: we collect the downloaded packages and bundle all together
                ## in a single file (zipped)
                #####
                logger.info('Adding packages to the bundle')
                bundle = zipfile.ZipFile(tmp_bundle, mode='w')
                self._add_to_archive(bundle, build, len(tempdir))

                ## Step 4: add the manifest file
                if include_manifest:
                    logger.info('Adding the manifest file')
                    bundle.writestr('pyg-manifest.txt', Bundler.MANIFEST.format('\n'.join(self.bundled)))

                # Additional files to add
                for path in additional_files:
                    try:
                        self._add_to_archive(bundle, path, len(tempdir))
                    except (IOError, OSError):
                        logger.debug('debug: Error while adding an additional file: {0}', path)
                if add_func is not None:
                    self._add_to_archive(bundle, add_func(), len(tempdir))
                bundle.close()

                ## Last step: move the bundle to the current working directory
                dest = os.path.join(destination, self.bundle_name)
                if os.path.exists(dest):
                    logger.debug('debug: dest already exists, removing it')
                    os.remove(dest)
                shutil.move(tmp_bundle, destination)