def __init__(self, index_root=None, urls=[], verbose=False, prefix=None,
platform=None):
self.plat = platform or custom_plat
self.prefix = prefix or sys.prefix
self.verbose = verbose
self.index = [] # list of dicts of product metadata
self.history = History(self.prefix)
self.enst = Enstaller(Chain(verbose=verbose), [self.prefix])
self.product_list_path = 'products'
if index_root:
self.load_index(index_root)
# Cache attributes
self._installed_cnames = None
self._status = None
self._installed = None
def __init__(self, index_root=None, urls=[], verbose=False, prefix=None, platform=None):
self.plat = platform or custom_plat
self.prefix = prefix
self.verbose = verbose
self.index = []
self.history = History(prefix)
self.enst = Enstaller(Chain(verbose=verbose), [prefix or sys.prefix])
self.product_index_path = "products"
self.authenticate = True
for url in urls:
self.add_product(url)
if index_root:
self.load_index(index_root)
# Cache attributes
self._installed_cnames = None
self._status = None
self._installed = None
class Resources(object):
# FIXME: class and methods need docstrings.
def __init__(self, index_root=None, urls=[], verbose=False, prefix=None,
platform=None):
self.plat = platform or custom_plat
self.prefix = prefix or sys.prefix
self.verbose = verbose
self.index = [] # list of dicts of product metadata
self.history = History(self.prefix)
self.enst = Enstaller(Chain(verbose=verbose), [self.prefix])
self.product_list_path = 'products'
if index_root:
self.load_index(index_root)
# Cache attributes
self._installed_cnames = None
self._status = None
self._installed = None
def clear_cache(self):
self._installed_cnames = None
self._status = None
self._installed = None
def load_index(self, url_root):
""" Append to self.index, the metadata for all products found
at url_root
"""
url_root = url_root.rstrip('/')
self._product_cache = ResourceCache(join(self.prefix, 'cache'),
url_root)
try:
product_list = self._product_cache.get(self.product_list_path + '/',
None)
except EnstallerResourceIndexError as e:
if e.data:
product_list = e.data
else:
raise
filtered_product_list = {}
for product in product_list:
slug = product['repo_slug']
if slug not in filtered_product_list or (
(not filtered_product_list[slug]['subscribed'])
and product['subscribed']):
filtered_product_list[slug] = product
for product_metadata in [pm for pm in product_list if
filtered_product_list[pm['repo_slug']] == pm]:
self._add_product(product_metadata)
def _add_product(self, product_metadata):
""" Append a dict of product metadata to self.index after
filling in the full product metadata
"""
if self.verbose:
print "Adding product:", product_metadata['product']
self._read_full_product_metadata(product_metadata)
if ('platform' in product_metadata and
product_metadata['platform'] != self.plat):
raise Exception('Product metadata file for {}, but running {}'
.format(product_metadata['platform'], self.plat))
if 'eggs' in product_metadata:
self._add_egg_repos(product_metadata['url'], product_metadata)
else:
product_metadata['eggs'] = {}
self.index.append(product_metadata)
def _read_full_product_metadata(self, product_metadata):
""" Given partial product metadata, fill in full product metatadata
which includes a list of the resources (e.g. eggs) which it includes.
Fills in the product_metadata dict in place.
"""
product_path = '{}/{}'.format(self.product_list_path,
product_metadata['product'])
product_metadata['url'] = self._product_cache.url_for(product_path)
parts = urlsplit(product_metadata['url'])
if product_metadata['product'] == 'EPDFree':
path = 'account/register/'
else:
path = 'products/getepd/'
product_metadata['buy_url'] = urlunsplit((parts.scheme, parts.netloc,
path, '', ''))
if product_metadata.get('platform_independent', False):
index_filename = 'index.json'
else:
index_filename = 'index-{}.json'.format(self.plat)
product_index_path = '{}/{}'.format(product_path, index_filename)
last_update = datetime.strptime(product_metadata['last_update'],
'%Y-%m-%d %H:%M:%S')
product_info = self._product_cache.get(product_index_path, last_update)
product_metadata.update(product_info)
def _add_egg_repos(self, url, product_metadata):
if 'egg_repos' in product_metadata:
repos = ['{}/{}/'.format(url, path)
for path in product_metadata['egg_repos']]
else:
repos = [url]
self.enst.chain.repos.extend(repos)
if not product_metadata['subscribed']:
for repo in repos:
self.enst.chain.unsubscribed_repos[repo] = product_metadata
for cname, project in product_metadata['eggs'].iteritems():
for distname, data in project['files'].iteritems():
name, version, build = dist_naming.split_eggname(distname)
spec = dict(metadata_version='1.1',
name=name, version=version, build=build,
python=data.get('python', '2.7'),
packages=data.get('depends', []),
size=data.get('size'))
add_Reqs_to_spec(spec)
assert spec['cname'] == cname, distname
dist = repos[data.get('repo', 0)] + distname
self.enst.chain.index[dist] = spec
self.enst.chain.groups[cname].append(dist)
def get_installed_cnames(self):
if not self._installed_cnames:
self._installed_cnames = self.enst.get_installed_cnames()
return self._installed_cnames
def get_status(self):
if not self._status:
# the result is a dict mapping cname to ...
res = {}
for cname in self.get_installed_cnames():
d = defaultdict(str)
info = self.enst.get_installed_info(cname)[0][1]
if info is None:
continue
d.update(info)
res[cname] = d
for cname in self.enst.chain.groups.iterkeys():
dist = self.enst.chain.get_dist(Req(cname),
allow_unsubscribed=True)
if dist is None:
continue
repo, fn = dist_naming.split_dist(dist)
n, v, b = dist_naming.split_eggname(fn)
if cname not in res:
d = defaultdict(str)
d['name'] = d.get('name', cname)
res[cname] = d
res[cname]['a-egg'] = fn
res[cname]['a-ver'] = '%s-%d' % (v, b)
def vb_egg(fn):
try:
n, v, b = dist_naming.split_eggname(fn)
return comparable_version(v), b
except IrrationalVersionError:
return None
except AssertionError:
return None
for d in res.itervalues():
if d['egg_name']: # installed
if d['a-egg']:
if vb_egg(d['egg_name']) >= vb_egg(d['a-egg']):
d['status'] = 'up-to-date'
else:
d['status'] = 'updateable'
else:
d['status'] = 'installed'
else: # not installed
if d['a-egg']:
d['status'] = 'installable'
self._status = res
return self._status
def get_installed(self):
if not self._installed:
self._installed = set([pkg['egg_name']
for pkg in self.get_status().values()
if pkg['status'] != 'installable'])
return self._installed
def search(self, text):
""" Search for eggs with name or description containing the given text.
Returns a list of canonical names for the matching eggs.
"""
regex = re.compile(re.escape(text), re.IGNORECASE)
results = []
for product_metadata in self.index:
for cname, metadata in product_metadata.get('eggs', {}).iteritems():
name = metadata.get('name', '')
description = metadata.get('description', '')
if regex.search(name) or regex.search(description):
results.append(cname)
return results
def _req_list(self, reqs):
""" Take a single req or a list of reqs and return a list of
Req instances
"""
if not isinstance(reqs, list):
reqs = [reqs]
# Convert cnames to Req instances
for i, req in enumerate(reqs):
if not isinstance(req, Req):
reqs[i] = Req(req)
return reqs
def install(self, reqs, overall_progress_cb=None):
reqs = self._req_list(reqs)
full_reqs = self.enst.full_install_sequence(reqs)
total_count = len(full_reqs)
with self.history:
installed_count = 0
for req in full_reqs:
installed_count += self.enst.install(req)
if overall_progress_cb:
overall_progress_cb(installed_count, total_count)
# Clear the cache, since the status of several packages could now be
# invalid
self.clear_cache()
return installed_count
def uninstall(self, reqs, overall_progress_cb=None):
reqs = self._req_list(reqs)
total_count = len(reqs)
with self.history:
removed_count = 0
for req in reqs:
self.enst.remove(req)
removed_count += 1
if overall_progress_cb:
overall_progress_cb(removed_count, total_count)
self.clear_cache()
return 1
def revert(self, revert_to):
revert(self.enst, str(revert_to), quiet=True)
self.clear_cache()
class Resources(object):
def __init__(self, index_root=None, urls=[], verbose=False, prefix=None, platform=None):
self.plat = platform or custom_plat
self.prefix = prefix
self.verbose = verbose
self.index = []
self.history = History(prefix)
self.enst = Enstaller(Chain(verbose=verbose), [prefix or sys.prefix])
self.product_index_path = "products"
self.authenticate = True
for url in urls:
self.add_product(url)
if index_root:
self.load_index(index_root)
# Cache attributes
self._installed_cnames = None
self._status = None
self._installed = None
def clear_cache(self):
self._installed_cnames = None
self._status = None
self._installed = None
def _http_auth(self):
username, password = config.get_auth()
if username and password and self.authenticate:
return (username + ":" + password).encode("base64")
else:
return None
def _read_json_from_url(self, url):
logger.debug("Reading JSON from URL: %s" % url)
req = Request(url)
auth = self._http_auth()
if auth:
req.add_header("Authorization", auth)
return json.load(urlopen(req))
def load_index(self, url):
url = url.rstrip("/")
index_url = "%s/%s" % (url, self.product_index_path)
try:
index = self._read_json_from_url(index_url)
except HTTPError as e:
logger.exception("Error getting products file %s" % index_url)
return
for product in index:
product_url = "%s/products/%s" % (url, product["product"])
try:
product["base_url"] = url
product["url"] = product_url.rstrip("/")
self.add_product(product)
except HTTPError:
logger.exception("Error getting index file %s" % product_url)
def _read_product_index(self, product_url):
""" Get the product index.
Try the platform-independent one first, then try the
platform-specific one if that one doesn't exist. Does both
HTTP requests simultaneously.
"""
independent = urlsplit("%s/index.json" % (product_url))
specific = urlsplit("%s/index-%s.json" % (product_url, self.plat))
logger.debug("Trying for JSON from URLs: %s, %s" % (independent.geturl(), specific.geturl()))
conn1 = HTTPConnection(independent.netloc)
conn2 = HTTPConnection(specific.netloc)
auth = self._http_auth()
if auth:
headers = {"Authorization": auth}
else:
headers = {}
conn1.request("GET", independent.path, headers=headers)
conn2.request("GET", specific.path, headers=headers)
try:
res = conn1.getresponse()
if res.status == 200:
data = res.read()
return independent, json.loads(data)
res = conn2.getresponse()
if res.status == 200:
data = res.read()
return specific, json.loads(data)
else:
raise HTTPError(specific, res.code, res.reason, res.msg)
except ValueError:
logger.exception("Error parsing index for %s" % product_url)
logger.error('Invalid index file: """%s"""' % data)
return None, None
except HTTPError:
logger.exception("Error reading index for %s" % product_url)
return None, None
finally:
conn1.close()
conn2.close()
def add_product(self, index):
if self.verbose:
print "Adding product:", index["url"]
index_url, product_index = self._read_product_index(index["url"])
if product_index is None:
return
index["index_url"] = index_url
index.update(product_index)
if "platform" in index and index["platform"] != self.plat:
raise Exception("index file for platform %s, but running %s" % (index["platform"], self.plat))
if "eggs" in index:
self._add_egg_repos(index["url"], index)
self.index.append(index)
return index
def _add_egg_repos(self, url, index):
if "egg_repos" in index:
repos = [url + "/" + path + "/" for path in index["egg_repos"]]
else:
repos = [url]
self.enst.chain.repos.extend(repos)
for cname, project in index["eggs"].iteritems():
for distname, data in project["files"].iteritems():
name, version, build = dist_naming.split_eggname(distname)
spec = dict(
metadata_version="1.1",
name=name,
version=version,
build=build,
python=data.get("python", "2.7"),
packages=data.get("depends", []),
)
add_Reqs_to_spec(spec)
assert spec["cname"] == cname, distname
dist = repos[data.get("repo", 0)] + distname
self.enst.chain.index[dist] = spec
self.enst.chain.groups[cname].append(dist)
def get_installed_cnames(self):
if not self._installed_cnames:
self._installed_cnames = self.enst.get_installed_cnames()
return self._installed_cnames
def get_status(self):
if not self._status:
# the result is a dict mapping cname to ...
res = {}
for cname in self.get_installed_cnames():
d = defaultdict(str)
info = self.enst.get_installed_info(cname)[0][1]
if info is None:
continue
d.update(info)
res[cname] = d
for cname in self.enst.chain.groups.iterkeys():
dist = self.enst.chain.get_dist(Req(cname))
if dist is None:
continue
repo, fn = dist_naming.split_dist(dist)
n, v, b = dist_naming.split_eggname(fn)
if cname not in res:
d = defaultdict(str)
d["name"] = d.get("name", cname)
res[cname] = d
res[cname]["a-egg"] = fn
res[cname]["a-ver"] = "%s-%d" % (v, b)
def vb_egg(fn):
try:
n, v, b = dist_naming.split_eggname(fn)
return comparable_version(v), b
except IrrationalVersionError:
return None
except AssertionError:
return None
for d in res.itervalues():
if d["egg_name"]: # installed
if d["a-egg"]:
if vb_egg(d["egg_name"]) >= vb_egg(d["a-egg"]):
d["status"] = "up-to-date"
else:
d["status"] = "updateable"
else:
d["status"] = "installed"
else: # not installed
if d["a-egg"]:
d["status"] = "installable"
self._status = res
return self._status
def get_installed(self):
if not self._installed:
self._installed = set(
[pkg["egg_name"] for pkg in self.get_status().values() if pkg["status"] != "installable"]
)
return self._installed
def search(self, text):
""" Search for eggs with name or description containing the given text.
Returns a list of canonical names for the matching eggs.
"""
regex = re.compile(re.escape(text), re.IGNORECASE)
results = []
for product in self.index:
for cname, metadata in product.get("eggs", {}).iteritems():
name = metadata.get("name", "")
description = metadata.get("description", "")
if regex.search(name) or regex.search(description):
results.append(cname)
return results
def _req_list(self, reqs):
""" Take a single req or a list of reqs and return a list of
Req instances
"""
if not isinstance(reqs, list):
reqs = [reqs]
# Convert cnames to Req instances
for i, req in enumerate(reqs):
if not isinstance(req, Req):
reqs[i] = Req(req)
return reqs
def install(self, reqs):
reqs = self._req_list(reqs)
with self.history:
installed_count = 0
for req in reqs:
installed_count += self.enst.install(req)
# Clear the cache, since the status of several packages could now be
# invalid
self.clear_cache()
return installed_count
def uninstall(self, reqs):
reqs = self._req_list(reqs)
with self.history:
for req in reqs:
self.enst.remove(req)
self.clear_cache()
return 1