Example #1
0
def read_pkg_requires(path):
    """Read a package requires.txt
    """
    try:
        data = open(os.path.join(path, 'requires.txt'), 'r')
    except IOError:
        return Requirements(), {}
    lines = []
    requires = []
    extras = {}
    current = None
    for line in data.readlines():
        line = line.strip()
        if not line or line[0] in '#;':
            continue
        if line[0] == '[' and line[-1] == ']':
            # New extra
            if current is None:
                requires = lines
            else:
                extras[current] = Requirements.parse(lines)
            lines = []
            current = line[1:-1]
        else:
            lines.append(line)
    # Store last extra
    if current is None:
        requires = lines
    else:
        extras[current] = Requirements.parse(lines)
    return Requirements.parse(requires), extras
Example #2
0
    def test_add_fail(self):
        """Test adding two incompatible set of requirements together
        """
        reqs = Requirements.parse(
            ['zeam.form.base',
             'zeam.test>=2.1'])
        other_reqs = Requirements.parse(
            ['zeam.form.ztk[test] >=1.0b1',
             'zeam.test <=1.0'])

        self.assertRaises(IncompatibleVersion, operator.add, reqs, other_reqs)
Example #3
0
    def load(self):
        egginfo = self.configuration['egginfo']
        self.distribution.name = egginfo['name'].as_text()
        self.distribution.version = Version.parse(egginfo['version'].as_text())
        self.distribution.summary = egginfo.get('summary', '').as_text()
        self.distribution.author = egginfo.get('author', '').as_text()
        self.distribution.author_email = egginfo.get(
            'author_email', '').as_text()
        self.distribution.license = egginfo.get('license', '').as_text()
        self.distribution.classifiers = egginfo.get('classifier', '').as_list()
        self.distribution.format = None
        self.distribution.pyversion = None
        self.distribution.platform = None
        self.distribution.requirements = Requirements.parse(
            egginfo.get('requires', '').as_list())
        self.distribution.extras = {}

        # Source path of the extension
        path = os.path.join(self.path, egginfo.get('source', '.').as_text())
        if not os.path.isdir(path):
            raise PackageError(path, 'Invalid source path "%s"' % path)
        self.distribution.path = os.path.abspath(path)
        self.distribution.package_path = self.path

        # Entry points
        self.distribution.entry_points = {}
        entry_points = egginfo.get('entry_points', None)
        if entry_points is not None:
            for category_name in entry_points.as_list():
                info = self.configuration['entry_points:' + category_name]
                self.distribution.entry_points[category_name] = info.as_dict()

        return self.distribution
Example #4
0
 def __init__(self, options, installed_options=None):
     self.versions = dict(map(lambda v: (v.key, v),
                              map(KnownVersion, options.values())))
     self.name = options.name.split(':', 1)[1]
     self.used = set()
     self.missing = Requirements()
     self.picked = {}
     self.options = options
     self.installed_options = installed_options
     self._uptodate = None
     self._activated = False
Example #5
0
 def add_recipe_packages(self, names):
     """Install a list of packages required for the parts to
     execute. Packages are enabled in the current Python
     environment so that the parts can use them directly.
     """
     # This must be used only to install recipe and recipe dependency.
     requirements = Requirements.parse(names)
     install_set = self._installer(
         requirements,
         strategy=self.strategy)
     for requirement in requirements:
         install_set.get(requirement.key).activate()
Example #6
0
 def __init__(self, options, working_set, directory=None):
     __status__ = u"Configuring package installer."
     self.interpretor = working_set.interpretor
     self.working_set = working_set
     self.kgs = options.utilities.kgs.get(options)
     self.sources = options.utilities.sources
     self.query = None
     self._to_install = Requirements()
     self._verify_being_installed = Requirements()
     self._being_installed = Requirements()
     self._lock = threading.RLock()
     self._wait = threading.Condition(threading.RLock())
     self._worker_count = options.get_with_default(
         'install_workers', 'setup', '5').as_int()
     self._options = options
     self._first_done = False
     self._error = None
     if directory is None:
         directory = options.get_with_default(
             'lib_directory', 'setup').as_text()
     self._directory = directory
    def load(self, distribution, interpretor):
        distribution.package_path = self.path

        # Read extracted configuration
        config = Configuration.read_lines(self.source.splitlines, self.path)
        setuptool_config = config['setuptools']
        distribution.version = Version.parse(
            setuptool_config['version'].as_text())

        # Look for requirements
        if 'install_requires' in setuptool_config:
            distribution.requirements = Requirements.parse(
                setuptool_config['install_requires'].as_list())
        if 'extras_require' in setuptool_config:
            extra_config = config[setuptool_config['extras_require'].as_text()]
            for extra, extra_requirements in extra_config.items():
                distribution.extras[extra] = Requirements.parse(
                    extra_requirements.as_list())

        # Look for source directory
        if 'package_dir' in setuptool_config:
            package_config = config[
                setuptool_config['package_dir'].as_text()]
            if '_' in package_config:
                prefix = package_config['_'].as_text()
                distribution.path = os.path.join(self.path, prefix)
        if 'description' in setuptool_config:
            distribution.description = setuptool_config['description'].as_text()
        if 'license' in setuptool_config:
            distribution.license = setuptool_config['license'].as_text()
        if 'author' in setuptool_config:
            distribution.author = setuptool_config['author'].as_text()
        if 'autor_email' in setuptool_config:
            distribution.author_email = \
                setuptool_config['author_email'].as_text()
        if 'ext_modules' in setuptool_config:
            libraries = self.extensions(prefix, setuptool_config)
            create_autotools(distribution, prefix, libraries)
            distribution.extensions = libraries
        return distribution
Example #8
0
    def test_add(self):
        """Test adding two sets of requirements together
        """
        reqs = Requirements.parse(
            ['zeam.form.base',
             'zeam.test>=2.1',
             'zope.testing<=3.7dev'])
        self.assertRaises(ValueError, operator.add, reqs, 42)
        self.assertRaises(ValueError, operator.add, reqs, "zeam >= 1.1")

        other_reqs = Requirements.parse(
            ['zeam.form.ztk[test] >=1.0b1',
             'zeam.test <=4.0, !=3.0'])

        result_reqs = reqs + other_reqs

        self.assertEqual(
            str(result_reqs).split('\n'),
            ['zeam.form.base',
             'zeam.form.ztk[test]>=1.0b1',
             'zeam.test>=2.1,!=3.0,<=4.0',
             'zope.testing<=3.7dev'])
Example #9
0
    def test_parse(self):
        """Test requirements parsing and printing
        """
        reqs = Requirements.parse([])
        self.assertEqual(len(reqs), 0)
        self.assertEqual(len(reqs.requirements), 0)

        reqs = Requirements.parse('test.software')
        self.assertEqual(len(reqs), 1)
        self.assertEqual(len(reqs.requirements), 1)
        self.assertEqual(str(reqs), 'test.software')

        reqs = Requirements.parse(
            ['zeam.form.base',
             'zeam.test>=2.1',
             'zope.testing<=3.7dev'])
        self.assertEqual(len(reqs), 3)
        self.assertEqual(len(reqs.requirements), 3)
        self.assertEqual(
            str(reqs).split('\n'),
            ['zeam.form.base',
             'zeam.test>=2.1',
             'zope.testing<=3.7dev'])
Example #10
0
    def __init__(self, options, status):
        super(Package, self).__init__(options, status)
        self.isolation = options.get(
            'isolation', 'on').as_bool()
        self.directory = options.get(
            'lib_directory',
            '${setup:lib_directory}').as_directory()
        self.bin_directory = options.get(
            'bin_directory',
            '${setup:bin_directory}').as_directory()

        requirements = []
        self.extra_sets = []
        if 'packages' in options:
            for requirement in options['packages'].as_list():
                match = INSTALLED_SET.match(requirement)
                if match:
                    self.extra_sets.append(match.group('name'))
                else:
                    requirements.append(requirement)
        else:
            requirements = [get_package_name(options).as_text()]
        self.requirements = Requirements.parse(requirements)
        if self.extra_sets:
            status.depends.update(self.extra_sets)
        if self.requirements:
            # Run the recipe is installer settings changed.
            status.enable(is_installer_changed(options))

        self.wanted_scripts = None
        if 'scripts' in options:
            self.wanted_scripts = options['scripts'].as_list()
        self.extra_args = []
        if 'extra_args' in options:
            self.extra_args = options['extra_args'].as_list()
        self.extra_paths = []
        if 'extra_paths' in options:
            self.extra_paths = options['extra_paths'].as_list()

        self.working_set = None
Example #11
0
class PackageInstaller(object):
    """Package installer: install new package in a working set from
    sources.
    """

    def __init__(self, options, working_set, directory=None):
        __status__ = u"Configuring package installer."
        self.interpretor = working_set.interpretor
        self.working_set = working_set
        self.kgs = options.utilities.kgs.get(options)
        self.sources = options.utilities.sources
        self.query = None
        self._to_install = Requirements()
        self._verify_being_installed = Requirements()
        self._being_installed = Requirements()
        self._lock = threading.RLock()
        self._wait = threading.Condition(threading.RLock())
        self._worker_count = options.get_with_default(
            'install_workers', 'setup', '5').as_int()
        self._options = options
        self._first_done = False
        self._error = None
        if directory is None:
            directory = options.get_with_default(
                'lib_directory', 'setup').as_text()
        self._directory = directory

    def _wakeup_workers(self):
        # Wake up some workers to work.
        count_to_wake_up = max(
            self._worker_count - 1, len(self._to_install))
        self._wait.acquire()
        for count in range(count_to_wake_up):
            self._wait.notify()
        self._wait.release()

    def _verify_extra_install(self, requirement):
        # Verify is some extra need installation
        release = self.working_set[requirement]
        for extra in requirement.extras:
            if extra not in release.extras:
                raise PackageError(
                    u'Require missing extra requirements "%s" in "%s"' % (
                        extra, release))
            self._register_install(release.extras[extra])

    def _register_install(self, requirements):
        # Mark requirements to be installed.
        for requirement in requirements:
            if self.kgs is not None:
                requirement = self.kgs.upgrade(requirement)
            if requirement in self.working_set:
                if requirement.extras:
                    self._verify_extra_install(requirement)
                logger.debug(
                    u'Skip already installed dependency %s', requirement)
                continue
            if requirement in self._being_installed:
                if requirement.extras:
                    self._verify_being_installed.append(requirement)
                logger.debug(
                    u'Skip already being installed dependency %s',
                    requirement)
                continue
            logger.debug(
                u'Need to install dependency %s', requirement)
            self._to_install.append(requirement)

    def wait_for_requirements(self):
        """Called by a worker to wait for requirement to arrive.
        """
        logger.debug(u'Wait for dependencies')
        self._wait.acquire()
        self._wait.wait()
        self._wait.release()

    def mark_failed(self, error):
        """Called by a worked to report an error.
        """
        self._lock.acquire()
        try:
            logger.debug(u'Failure')
            self._error = error
            logs.report(fatal=False)
            self._wait.acquire()
            self._wait.notifyAll()
            self._wait.release()
        finally:
            self._lock.release()

    def get_requirement(self):
        """Called by a worker to get a new requirement to install.
        """
        self._lock.acquire()
        try:
            if self._error is not None:
                return INSTALLATION_DONE
            if not self._to_install:
                if not self._being_installed:
                    if not self._first_done:
                        # Wake up other waiting worker
                        self._wait.acquire()
                        self._wait.notifyAll()
                        self._wait.release()
                        self._first_done = True
                    return INSTALLATION_DONE
                return None
            requirement = self._to_install.pop()
            assert requirement not in self._being_installed
            self._being_installed.append(requirement)
            logger.info(u'Installing %s', requirement)
            return requirement
        finally:
            self._lock.release()

    def mark_installed(self, requirement, package):
        """Called by a worker to mark that the given requirement have
        been installed, using the given package.
        """
        self._lock.acquire()
        try:
            self.working_set.add(package)
            if self.kgs is not None:
                self.kgs.report_picked(requirement, package.version)
            if requirement in self._verify_being_installed:
                extra_requirement = self._verify_being_installed[requirement]
                logger.debug(
                    u'Verify pending extra for %s', extra_requirement)
                worker_need_wake_up = len(self._to_install) == 0
                self._verify_extra_install(extra_requirement)
                if worker_need_wake_up:
                    self._wakeup_workers()
                self._verify_being_installed.remove(extra_requirement)
            self._being_installed.remove(requirement)
            logger.debug(u'Mark %s as installed', requirement)
        finally:
            self._lock.release()

    def install_dependencies(self, requirements):
        self._lock.acquire()
        try:
            worker_need_wake_up = len(self._to_install) == 0
            self._register_install(requirements)
            if worker_need_wake_up:
                self._wakeup_workers()
        finally:
            self._lock.release()

    def __call__(self, requirements, strategy=STRATEGY_UPDATE, directory=None):
        """Called by the user to trigger the installation of the given
        list of requirements.
        """
        __status__ = u"Installing %r." % (requirements)
        if directory is None:
            directory = self._directory
        self._error = None
        self._first_done = False
        self._register_install(requirements)
        if self._to_install:
            self.query = self.sources(
                self.interpretor,
                os.path.abspath(directory))
            workers = []
            for count in range(self._worker_count):
                worker = PackageInstallerWorker(self, count, strategy)
                worker.start()
                workers.append(worker)
            for worker in workers:
                worker.join()
            if self._error is not None:
                raise self._error
        return self.working_set
Example #12
0
class KnownGoodVersionSet(object):
    """Represent a Known Good Version set.
    """

    def __init__(self, options, installed_options=None):
        self.versions = dict(map(lambda v: (v.key, v),
                                 map(KnownVersion, options.values())))
        self.name = options.name.split(':', 1)[1]
        self.used = set()
        self.missing = Requirements()
        self.picked = {}
        self.options = options
        self.installed_options = installed_options
        self._uptodate = None
        self._activated = False

    def get(self, requirement, default=None):
        if self._activated:
            __status__ = u"Looking version for %s in Known Good Set %s." % (
                str(requirement), self.name)
            if requirement.key in self.versions:
                self.used.add(requirement.key)
                return Version.parse(self.versions[requirement.key].version)
        return default

    def is_activated(self):
        return self._activated

    def is_uptodate(self):
        if self._uptodate is None:
            if self.installed_options is None:
                self._uptodate = True
            else:
                self._uptodate = (self.options == self.installed_options)
        return self._uptodate

    def activate(self):
        # This means the KGS is activated in order to be used.
        self._activated = True

    def report_missing(self, requirement):
        self.missing.append(requirement)

    def report_picked(self, requirement, version):
        picked = self.picked.setdefault(requirement, [])
        picked.append(version)

    def log_usage(self):
        if self.missing:
            for requirement in self.missing:
                msg = u"Missing requirement for '%s' in '%s'"
                args = (requirement, self.name)
                picked = self.picked.get(requirement)
                if picked:
                    msg += u", picked '%s'"
                    args += (','.join(map(str, picked)),)
                msg += u"."
                logger.warn(msg, *args)

        unused = sorted(set(self.versions.keys()) - self.used)
        if unused:
            for name in unused:
                logger.info(
                    u"Unused requirement '%s' in '%s'.",
                    self.versions[name].name, self.name)
Example #13
0
 def __init__(self, *args):
     __status__ = u"Initializing fake source."
     super(FakeSource, self).__init__(*args)
     self.requirements = Requirements.parse(
         self.options.get('packages', '').as_list())