Ejemplo n.º 1
0
    def __init__(self, index_root=None, urls=[], verbose=False, prefix=None,
                 platform=None):
        self.plat = platform or custom_plat
        self.prefix = prefix or sys.prefix
        self.verbose = verbose
        self.index = []   # list of dicts of product metadata
        self.history = History(self.prefix)
        self.enst = Enstaller(Chain(verbose=verbose), [self.prefix])
        self.product_list_path = 'products'

        if index_root:
            self.load_index(index_root)

        # Cache attributes
        self._installed_cnames = None
        self._status = None
        self._installed = None
Ejemplo n.º 2
0
    def __init__(self, index_root=None, urls=[], verbose=False, prefix=None, platform=None):
        self.plat = platform or custom_plat
        self.prefix = prefix
        self.verbose = verbose
        self.index = []
        self.history = History(prefix)
        self.enst = Enstaller(Chain(verbose=verbose), [prefix or sys.prefix])
        self.product_index_path = "products"
        self.authenticate = True

        for url in urls:
            self.add_product(url)

        if index_root:
            self.load_index(index_root)

        # Cache attributes
        self._installed_cnames = None
        self._status = None
        self._installed = None
Ejemplo n.º 3
0
class Resources(object):

    # FIXME: class and methods need docstrings.

    def __init__(self, index_root=None, urls=[], verbose=False, prefix=None,
                 platform=None):
        self.plat = platform or custom_plat
        self.prefix = prefix or sys.prefix
        self.verbose = verbose
        self.index = []   # list of dicts of product metadata
        self.history = History(self.prefix)
        self.enst = Enstaller(Chain(verbose=verbose), [self.prefix])
        self.product_list_path = 'products'

        if index_root:
            self.load_index(index_root)

        # Cache attributes
        self._installed_cnames = None
        self._status = None
        self._installed = None

    def clear_cache(self):
        self._installed_cnames = None
        self._status = None
        self._installed = None

    def load_index(self, url_root):
        """ Append to self.index, the metadata for all products found
        at url_root
        """
        url_root = url_root.rstrip('/')
        self._product_cache = ResourceCache(join(self.prefix, 'cache'),
                                            url_root)

        try:
            product_list = self._product_cache.get(self.product_list_path + '/',
                                                   None)
        except EnstallerResourceIndexError as e:
            if e.data:
                product_list = e.data
            else:
                raise

        filtered_product_list = {}
        for product in product_list:
            slug = product['repo_slug']
            if slug not in filtered_product_list or (
                    (not filtered_product_list[slug]['subscribed'])
                    and product['subscribed']):
                filtered_product_list[slug] = product

        for product_metadata in [pm for pm in product_list if
                                 filtered_product_list[pm['repo_slug']] == pm]:
            self._add_product(product_metadata)

    def _add_product(self, product_metadata):
        """ Append a dict of product metadata to self.index after
        filling in the full product metadata
        """
        if self.verbose:
            print "Adding product:", product_metadata['product']

        self._read_full_product_metadata(product_metadata)

        if ('platform' in product_metadata and
            product_metadata['platform'] != self.plat):
            raise Exception('Product metadata file for {}, but running {}'
                            .format(product_metadata['platform'], self.plat))

        if 'eggs' in product_metadata:
            self._add_egg_repos(product_metadata['url'], product_metadata)
        else:
            product_metadata['eggs'] = {}

        self.index.append(product_metadata)

    def _read_full_product_metadata(self, product_metadata):
        """ Given partial product metadata, fill in full product metatadata
        which includes a list of the resources (e.g. eggs) which it includes.

        Fills in the product_metadata dict in place.
        """
        product_path = '{}/{}'.format(self.product_list_path,
                                      product_metadata['product'])
        product_metadata['url'] = self._product_cache.url_for(product_path)
        parts = urlsplit(product_metadata['url'])
        if product_metadata['product'] == 'EPDFree':
            path = 'account/register/'
        else:
            path = 'products/getepd/'
        product_metadata['buy_url'] = urlunsplit((parts.scheme, parts.netloc,
                                                  path, '', ''))

        if product_metadata.get('platform_independent', False):
            index_filename = 'index.json'
        else:
            index_filename = 'index-{}.json'.format(self.plat)
        product_index_path = '{}/{}'.format(product_path, index_filename)
        last_update = datetime.strptime(product_metadata['last_update'],
                                        '%Y-%m-%d %H:%M:%S')

        product_info = self._product_cache.get(product_index_path, last_update)
        product_metadata.update(product_info)

    def _add_egg_repos(self, url, product_metadata):
        if 'egg_repos' in product_metadata:
            repos = ['{}/{}/'.format(url, path)
                     for path in product_metadata['egg_repos']]
        else:
            repos = [url]
        self.enst.chain.repos.extend(repos)

        if not product_metadata['subscribed']:
            for repo in repos:
                self.enst.chain.unsubscribed_repos[repo] = product_metadata

        for cname, project in product_metadata['eggs'].iteritems():
            for distname, data in project['files'].iteritems():
                name, version, build = dist_naming.split_eggname(distname)
                spec = dict(metadata_version='1.1',
                            name=name, version=version, build=build,
                            python=data.get('python', '2.7'),
                            packages=data.get('depends', []),
                            size=data.get('size'))
                add_Reqs_to_spec(spec)
                assert spec['cname'] == cname, distname
                dist = repos[data.get('repo', 0)] + distname
                self.enst.chain.index[dist] = spec
                self.enst.chain.groups[cname].append(dist)

    def get_installed_cnames(self):
        if not self._installed_cnames:
            self._installed_cnames = self.enst.get_installed_cnames()
        return self._installed_cnames

    def get_status(self):
        if not self._status:
            # the result is a dict mapping cname to ...
            res = {}
            for cname in self.get_installed_cnames():
                d = defaultdict(str)
                info = self.enst.get_installed_info(cname)[0][1]
                if info is None:
                    continue
                d.update(info)
                res[cname] = d

            for cname in self.enst.chain.groups.iterkeys():
                dist = self.enst.chain.get_dist(Req(cname),
                                                allow_unsubscribed=True)
                if dist is None:
                    continue
                repo, fn = dist_naming.split_dist(dist)
                n, v, b = dist_naming.split_eggname(fn)
                if cname not in res:
                    d = defaultdict(str)
                    d['name'] = d.get('name', cname)
                    res[cname] = d
                res[cname]['a-egg'] = fn
                res[cname]['a-ver'] = '%s-%d' % (v, b)

            def vb_egg(fn):
                try:
                    n, v, b = dist_naming.split_eggname(fn)
                    return comparable_version(v), b
                except IrrationalVersionError:
                    return None
                except AssertionError:
                    return None

            for d in res.itervalues():
                if d['egg_name']:                    # installed
                    if d['a-egg']:
                        if vb_egg(d['egg_name']) >= vb_egg(d['a-egg']):
                            d['status'] = 'up-to-date'
                        else:
                            d['status'] = 'updateable'
                    else:
                        d['status'] = 'installed'
                else:                                # not installed
                    if d['a-egg']:
                        d['status'] = 'installable'
            self._status = res
        return self._status

    def get_installed(self):
        if not self._installed:
            self._installed = set([pkg['egg_name']
                                   for pkg in self.get_status().values()
                                   if pkg['status'] != 'installable'])
        return self._installed

    def search(self, text):
        """ Search for eggs with name or description containing the given text.

        Returns a list of canonical names for the matching eggs.
        """
        regex = re.compile(re.escape(text), re.IGNORECASE)
        results = []
        for product_metadata in self.index:
            for cname, metadata in product_metadata.get('eggs', {}).iteritems():
                name = metadata.get('name', '')
                description = metadata.get('description', '')
                if regex.search(name) or regex.search(description):
                    results.append(cname)
        return results

    def _req_list(self, reqs):
        """ Take a single req or a list of reqs and return a list of
        Req instances
        """
        if not isinstance(reqs, list):
            reqs = [reqs]

        # Convert cnames to Req instances
        for i, req in enumerate(reqs):
            if not isinstance(req, Req):
                reqs[i] = Req(req)
        return reqs

    def install(self, reqs, overall_progress_cb=None):
        reqs = self._req_list(reqs)

        full_reqs = self.enst.full_install_sequence(reqs)
        total_count = len(full_reqs)
        with self.history:
            installed_count = 0
            for req in full_reqs:
                installed_count += self.enst.install(req)
                if overall_progress_cb:
                    overall_progress_cb(installed_count, total_count)

        # Clear the cache, since the status of several packages could now be
        # invalid
        self.clear_cache()

        return installed_count

    def uninstall(self, reqs, overall_progress_cb=None):
        reqs = self._req_list(reqs)

        total_count = len(reqs)
        with self.history:
            removed_count = 0
            for req in reqs:
                self.enst.remove(req)
                removed_count += 1
                if overall_progress_cb:
                    overall_progress_cb(removed_count, total_count)

        self.clear_cache()
        return 1

    def revert(self, revert_to):
        revert(self.enst, str(revert_to), quiet=True)
        self.clear_cache()
Ejemplo n.º 4
0
class Resources(object):
    def __init__(self, index_root=None, urls=[], verbose=False, prefix=None, platform=None):
        self.plat = platform or custom_plat
        self.prefix = prefix
        self.verbose = verbose
        self.index = []
        self.history = History(prefix)
        self.enst = Enstaller(Chain(verbose=verbose), [prefix or sys.prefix])
        self.product_index_path = "products"
        self.authenticate = True

        for url in urls:
            self.add_product(url)

        if index_root:
            self.load_index(index_root)

        # Cache attributes
        self._installed_cnames = None
        self._status = None
        self._installed = None

    def clear_cache(self):
        self._installed_cnames = None
        self._status = None
        self._installed = None

    def _http_auth(self):
        username, password = config.get_auth()
        if username and password and self.authenticate:
            return (username + ":" + password).encode("base64")
        else:
            return None

    def _read_json_from_url(self, url):
        logger.debug("Reading JSON from URL: %s" % url)
        req = Request(url)
        auth = self._http_auth()
        if auth:
            req.add_header("Authorization", auth)
        return json.load(urlopen(req))

    def load_index(self, url):
        url = url.rstrip("/")
        index_url = "%s/%s" % (url, self.product_index_path)
        try:
            index = self._read_json_from_url(index_url)
        except HTTPError as e:
            logger.exception("Error getting products file %s" % index_url)
            return

        for product in index:
            product_url = "%s/products/%s" % (url, product["product"])
            try:
                product["base_url"] = url
                product["url"] = product_url.rstrip("/")
                self.add_product(product)
            except HTTPError:
                logger.exception("Error getting index file %s" % product_url)

    def _read_product_index(self, product_url):
        """ Get the product index.

        Try the platform-independent one first, then try the
        platform-specific one if that one doesn't exist. Does both
        HTTP requests simultaneously.

        """
        independent = urlsplit("%s/index.json" % (product_url))
        specific = urlsplit("%s/index-%s.json" % (product_url, self.plat))
        logger.debug("Trying for JSON from URLs: %s, %s" % (independent.geturl(), specific.geturl()))
        conn1 = HTTPConnection(independent.netloc)
        conn2 = HTTPConnection(specific.netloc)
        auth = self._http_auth()
        if auth:
            headers = {"Authorization": auth}
        else:
            headers = {}
        conn1.request("GET", independent.path, headers=headers)
        conn2.request("GET", specific.path, headers=headers)

        try:
            res = conn1.getresponse()
            if res.status == 200:
                data = res.read()
                return independent, json.loads(data)
            res = conn2.getresponse()
            if res.status == 200:
                data = res.read()
                return specific, json.loads(data)
            else:
                raise HTTPError(specific, res.code, res.reason, res.msg)
        except ValueError:
            logger.exception("Error parsing index for %s" % product_url)
            logger.error('Invalid index file: """%s"""' % data)
            return None, None
        except HTTPError:
            logger.exception("Error reading index for %s" % product_url)
            return None, None
        finally:
            conn1.close()
            conn2.close()

    def add_product(self, index):

        if self.verbose:
            print "Adding product:", index["url"]

        index_url, product_index = self._read_product_index(index["url"])
        if product_index is None:
            return

        index["index_url"] = index_url
        index.update(product_index)

        if "platform" in index and index["platform"] != self.plat:
            raise Exception("index file for platform %s, but running %s" % (index["platform"], self.plat))

        if "eggs" in index:
            self._add_egg_repos(index["url"], index)

        self.index.append(index)
        return index

    def _add_egg_repos(self, url, index):
        if "egg_repos" in index:
            repos = [url + "/" + path + "/" for path in index["egg_repos"]]
        else:
            repos = [url]
        self.enst.chain.repos.extend(repos)

        for cname, project in index["eggs"].iteritems():
            for distname, data in project["files"].iteritems():
                name, version, build = dist_naming.split_eggname(distname)
                spec = dict(
                    metadata_version="1.1",
                    name=name,
                    version=version,
                    build=build,
                    python=data.get("python", "2.7"),
                    packages=data.get("depends", []),
                )
                add_Reqs_to_spec(spec)
                assert spec["cname"] == cname, distname
                dist = repos[data.get("repo", 0)] + distname
                self.enst.chain.index[dist] = spec
                self.enst.chain.groups[cname].append(dist)

    def get_installed_cnames(self):
        if not self._installed_cnames:
            self._installed_cnames = self.enst.get_installed_cnames()
        return self._installed_cnames

    def get_status(self):
        if not self._status:
            # the result is a dict mapping cname to ...
            res = {}
            for cname in self.get_installed_cnames():
                d = defaultdict(str)
                info = self.enst.get_installed_info(cname)[0][1]
                if info is None:
                    continue
                d.update(info)
                res[cname] = d

                for cname in self.enst.chain.groups.iterkeys():
                    dist = self.enst.chain.get_dist(Req(cname))
                    if dist is None:
                        continue
                    repo, fn = dist_naming.split_dist(dist)
                    n, v, b = dist_naming.split_eggname(fn)
                    if cname not in res:
                        d = defaultdict(str)
                        d["name"] = d.get("name", cname)
                        res[cname] = d
                    res[cname]["a-egg"] = fn
                    res[cname]["a-ver"] = "%s-%d" % (v, b)

            def vb_egg(fn):
                try:
                    n, v, b = dist_naming.split_eggname(fn)
                    return comparable_version(v), b
                except IrrationalVersionError:
                    return None
                except AssertionError:
                    return None

            for d in res.itervalues():
                if d["egg_name"]:  # installed
                    if d["a-egg"]:
                        if vb_egg(d["egg_name"]) >= vb_egg(d["a-egg"]):
                            d["status"] = "up-to-date"
                        else:
                            d["status"] = "updateable"
                    else:
                        d["status"] = "installed"
                else:  # not installed
                    if d["a-egg"]:
                        d["status"] = "installable"
            self._status = res
        return self._status

    def get_installed(self):
        if not self._installed:
            self._installed = set(
                [pkg["egg_name"] for pkg in self.get_status().values() if pkg["status"] != "installable"]
            )
        return self._installed

    def search(self, text):
        """ Search for eggs with name or description containing the given text.

        Returns a list of canonical names for the matching eggs.
        """
        regex = re.compile(re.escape(text), re.IGNORECASE)
        results = []
        for product in self.index:
            for cname, metadata in product.get("eggs", {}).iteritems():
                name = metadata.get("name", "")
                description = metadata.get("description", "")
                if regex.search(name) or regex.search(description):
                    results.append(cname)
        return results

    def _req_list(self, reqs):
        """ Take a single req or a list of reqs and return a list of
        Req instances
        """
        if not isinstance(reqs, list):
            reqs = [reqs]

        # Convert cnames to Req instances
        for i, req in enumerate(reqs):
            if not isinstance(req, Req):
                reqs[i] = Req(req)
        return reqs

    def install(self, reqs):
        reqs = self._req_list(reqs)

        with self.history:
            installed_count = 0
            for req in reqs:
                installed_count += self.enst.install(req)

        # Clear the cache, since the status of several packages could now be
        # invalid
        self.clear_cache()

        return installed_count

    def uninstall(self, reqs):
        reqs = self._req_list(reqs)

        with self.history:
            for req in reqs:
                self.enst.remove(req)

        self.clear_cache()
        return 1