def _get_dependency_tree_for_package(package, parent=None, sync=False, jobs=1):
if package.name not in _TREE_DICT:
logger.info("Building dependency tree for package: %s..." % package)
tree = TreeNode(package, parent=parent)
dependencies = []
with parallel.no_daemon_pool(processes=jobs) as pool:
dependencies = pool.map(partial(_create_package, **{"sync": sync}),
_INFO_DICT[package.name]["dependencies"])
with parallel.no_daemon_pool(processes=jobs) as pool:
children = pool.map(
partial(_get_dependency_tree_for_package, **{"parent": tree}),
dependencies)
if children:
tree.add_children(*children)
_TREE_DICT[package.name] = tree
else:
logger.info("Using cached dependency tree for package: %s." % package)
tree = _TREE_DICT[package.name]
tree.parent = parent
return tree
def __init__(self, package, sync=False, pip_exec=None):
logger.info("Initializing Package %s of type %s..." %
(package, type(package)))
self.current_version = None
if isinstance(package, (_pip.Distribution, _pip.DistInfoDistribution,
_pip.EggInfoDistribution)):
self.name = package.project_name
self.current_version = package.version
elif isinstance(package, _pip.InstallRequirement):
self.name = package.name
if hasattr(package, "req"):
if hasattr(package.req, "specifier"):
self.current_version = str(package.req.specifier)
else:
self.current_version = package.installed_version
elif isinstance(package, dict):
self.name = package["name"]
self.current_version = package["version"]
self.latest_version = package.get("latest_version")
elif isinstance(package, str):
self.name = package
if pip_exec:
self.current_version = _get_package_version(package,
pip_exec=pip_exec)
res = None
try:
logger.info("Fetching package %s information from DB..." %
self.name)
res = _db.query("""
SELECT *
FROM `tabPackage`
WHERE name = '%s'
""" % self.name)
except db.OperationalError as e:
logger.warn("Unable to fetch package name. %s" % e)
if not res or sync:
logger.info("Fetching PyPI info for package %s..." % self)
_pypi_info = _get_pypi_info(self.name, raise_err=False) or {}
if not hasattr(self, "latest_version"):
self.latest_version = _pypi_info.get("version")
self.home_page = _pypi_info.get("home_page")
if not res:
try:
logger.info("Attempting to INSERT package %s into database." %
self)
_db.query("""
INSERT INTO `tabPackage`
(name, latest_version, home_page, _created_at)
VALUES
('%s', '%s', '%s', '%s')
""" % (self.name, self.latest_version, self.home_page, datetime.now()))
except (db.IntegrityError, db.OperationalError) as e:
logger.warn("Unable to save package name. %s" % e)
else:
if sync:
logger.info(
"Attempting to UPDATE package %s within database." % self)
try:
_db.query("""
UPDATE `tabPackage`
SET latest_version = '%s', home_page = '%s', _updated_at = '%s'
WHERE
name = '%s'
""" % (self.latest_version, self.home_page, datetime.now(), self.name))
except db.OperationalError as e:
logger.warn("Unable to update package name. %s" % e)
else:
logger.info("Using cached info for package %s." % self)
self.latest_version = res["latest_version"]
self.home_page = res["home_page"]
self.dependency_tree = TreeNode(self)
try:
self.difference = semver.difference(self.current_version,
self.latest_version)
except (TypeError, ValueError):
self.difference = None
def test_node():
tree1 = Node("foo")
assert tree1.empty == True
assert tree1 == Node("foo")
assert str(tree1) == "<Node 'foo'>"
assert tree1.render() == \
"""\
foo
"""
tree2 = Node("foo", children=["bar", "baz"])
assert tree2.parent == None
assert tree2 != Node("foo", children=["bar", "baz", "boo"])
assert Node("foo", children = ["bar", "baz"]) \
== Node("foo", children = ["bar", "baz"])
assert not Node("foo", children = ["bar", "baz"]) \
== Node("foo", children = ["baz", "boo"])
assert tree2.render() == \
"""\
foo
bar
baz
"""
assert tree2.render(indent = 4) == \
"""\
foo
bar
baz
"""
assert tree2.render(indent = 4, formatter = lambda x: "* %s" % x) == \
"""\
* foo
* bar
* baz
"""
assert tree2.find(lambda x: x.obj == "foo")
assert tree2.find(lambda x: x.obj == "bar")
assert not tree2.find(lambda x: x.obj == "foobaz")
tree3 = Node("foo")
tree3.add_children(["bar", "baz"])
tree4 = Node("foo")
tree4.children = ["bar", "baz"]
with pytest.raises(TypeError):
tree4.children = "bar"
"""
foo
-> bar
-> baz
-> boo
-> boo
-> bar
-> foo
"""
tree5 = Node("foo")
node1 = Node("bar", ["baz", "boo"])
node2 = Node("boo", ["bar", "foo"])
tree5.add_children(node1, node2)
assert tree5.find(node2) == node2
tree5.to_json() == dict(foo=dict(
bar=dict(baz=None, boo=None),
boo=dict(bar=None, foo=None),
))
assert hash(tree5) == id(tree5)
tree6 = Node("foo")
children = [Node("bar"), Node("baz")]
tree6.add_children(*children)
tree6.children = children
assert tree6.children == children
tree7 = Node("foo")
tree8 = Node("bar")
tree8.parent = tree7
assert tree8.parent == tree7
tree8.parent = tree7
assert tree8.parent == tree7
class Package(object):
def __init__(self, package, sync=False, pip_exec=None):
logger.info("Initializing Package %s of type %s..." %
(package, type(package)))
self.current_version = None
if isinstance(package, (_pip.Distribution, _pip.DistInfoDistribution,
_pip.EggInfoDistribution)):
self.name = package.project_name
self.current_version = package.version
elif isinstance(package, _pip.InstallRequirement):
self.name = package.name
if hasattr(package, "req"):
if hasattr(package.req, "specifier"):
self.current_version = str(package.req.specifier)
else:
self.current_version = package.installed_version
elif isinstance(package, dict):
self.name = package["name"]
self.current_version = package["version"]
self.latest_version = package.get("latest_version")
elif isinstance(package, str):
self.name = package
if pip_exec:
self.current_version = _get_package_version(package,
pip_exec=pip_exec)
res = None
try:
logger.info("Fetching package %s information from DB..." %
self.name)
res = _db.query("""
SELECT *
FROM `tabPackage`
WHERE name = '%s'
""" % self.name)
except db.OperationalError as e:
logger.warn("Unable to fetch package name. %s" % e)
if res:
cache_timeout = settings.get("cache_timeout")
if res["_updated_at"]:
time_difference = res["_updated_at"] + timedelta(
seconds=cache_timeout)
if datetime.now() > time_difference:
sync = True
else:
sync = True
if not res or sync:
logger.info("Fetching PyPI info for package %s..." % self)
_pypi_info = _get_pypi_info(self.name, raise_err=False) or {}
if not getattr(self, "latest_version", None) or sync:
self.latest_version = _pypi_info.get("version")
self.home_page = _pypi_info.get("home_page")
if not res:
try:
values = (self.name, self.latest_version
or "NULL", self.home_page, datetime.now(),
datetime.now())
logger.info(
"Attempting to INSERT package %s into database with values: %s."
% (self, values))
_db.query("""
INSERT INTO `tabPackage`
(name, latest_version, home_page, _created_at, _updated_at)
VALUES
('%s', '%s', '%s', '%s', '%s')
""" % values)
except (db.IntegrityError, db.OperationalError) as e:
logger.warn("Unable to save package name. %s" % e)
else:
if sync:
logger.info(
"Attempting to UPDATE package %s within database." % self)
try:
_db.query("""
UPDATE `tabPackage`
SET latest_version = '%s', home_page = '%s', _updated_at = '%s'
WHERE
name = '%s'
""" % (self.latest_version, self.home_page, datetime.now(), self.name))
except db.OperationalError as e:
logger.warn("Unable to update package name. %s" % e)
else:
logger.info("Using cached info for package %s." % self)
self.latest_version = res["latest_version"]
self.home_page = res["home_page"]
self.dependency_tree = TreeNode(self)
@property
def difference(self):
difference = None
try:
difference = semver.difference(self.current_version,
self.latest_version)
except (TypeError, ValueError):
pass
return difference
def __repr__(self):
repr_ = "<Package %s%s>" % (self.name, " (%s)" % self.current_version
if self.current_version else "")
return repr_
def to_dict(self):
dependencies = self.dependency_tree.to_dict(repr_=lambda x: x.name)
dict_ = dict({
"name": self.name,
"current_version": self.current_version,
"latest_version": self.latest_version,
"home_page": self.home_page,
"dependencies": dependencies[self.name]
})
return dict_
def test_node():
tree1 = Node("foo")
assert tree1.empty == True
assert tree1 == Node("foo")
assert str(tree1) == "<Node 'foo'>"
assert tree1.render() == \
"""\
foo
"""
tree2 = Node("foo", children=["bar", "baz"])
assert tree2.parent == None
assert tree2 != Node("foo", children=["bar", "baz", "boo"])
assert tree2.render() == \
"""\
foo
bar
baz
"""
assert tree2.render(indent = 4) == \
"""\
foo
bar
baz
"""
assert tree2.render(indent = 4, formatter = lambda x: "* %s" % x) == \
"""\
* foo
* bar
* baz
"""
assert tree2.find(lambda x: x.obj == "foo")
assert tree2.find(lambda x: x.obj == "bar")
assert not tree2.find(lambda x: x.obj == "foobaz")
def test_node():
tree1 = Node("foo")
assert tree1.empty == True
assert tree1 == Node("foo")
assert str(tree1) == "<Node 'foo'>"
assert tree1.render() == \
"""\
foo
"""
tree2 = Node("foo", children=["bar", "baz"])
assert tree2.parent == None
assert tree2 != Node("foo", children=["bar", "baz", "boo"])
assert Node("foo", children = ["bar", "baz"]) \
== Node("foo", children = ["bar", "baz"])
assert not Node("foo", children = ["bar", "baz"]) \
== Node("foo", children = ["baz", "boo"])
assert tree2.render() == \
"""\
foo
bar
baz
"""
assert tree2.render(indent = 4) == \
"""\
foo
bar
baz
"""
assert tree2.render(indent = 4, formatter = lambda x: "* %s" % x) == \
"""\
* foo
* bar
* baz
"""
assert tree2.find(lambda x: x.obj == "foo")
assert tree2.find(lambda x: x.obj == "bar")
assert not tree2.find(lambda x: x.obj == "foobaz")
tree3 = Node("foo")
tree3.add_children(["bar", "baz"])
tree4 = Node("foo")
tree4.children = ["bar", "baz"]
with pytest.raises(TypeError):
tree4.children = "bar"
def __init__(self, package, sync=False, pip_exec=None):
logger.info("Initializing Package %s of type %s..." %
(package, type(package)))
self.current_version = None
# self.dependencies = [ ]
if isinstance(package, (_pip.Distribution, _pip.DistInfoDistribution,
_pip.EggInfoDistribution)):
self.name = package.project_name
self.current_version = package.version
elif isinstance(package, _pip.InstallRequirement):
self.name = package.name
if hasattr(package, "req"):
if hasattr(package.req, "specifier"):
self.current_version = str(package.req.specifier)
else:
self.current_version = package.installed_version
elif isinstance(package, dict):
self.name = package["name"]
self.current_version = package["version"]
self.latest_version = package.get("latest_version")
elif isinstance(package, str):
self.name = package
if pip_exec:
info = _get_pip_info(self.name)
self.current_version = info["version"]
# self.dependencies = info["requires"]
# if pip_exec and not self.dependencies:
# self.dependencies = _get_pip_info(self.name)
res = None
try:
logger.info("Fetching package %s information from DB..." %
self.name)
res = _db.query("""
SELECT *
FROM `tabPackage`
WHERE name = '%s'
""" % self.name)
except db.OperationalError as e:
logger.warn("Unable to fetch package name. %s" % e)
if res:
cache_timeout = settings.get("cache_timeout")
if res["_updated_at"]:
time_difference = to_datetime(
res["_updated_at"]) + timedelta(seconds=cache_timeout)
if datetime.now() > time_difference:
sync = True
else:
sync = True
if not res or sync:
logger.info("Fetching PyPI info for package %s..." % self)
_pypi_info = _get_pypi_info(self.name, raise_err=False) or {}
if not getattr(self, "latest_version", None) or sync:
self.latest_version = _pypi_info.get("version")
self.home_page = _pypi_info.get("home_page")
self.releases = [
version for version in iterkeys(_pypi_info.get("releases"))
]
if not res:
try:
values = (self.name, self.latest_version
or "NULL", self.home_page, ",".join(self.releases),
datetime.now(), datetime.now())
logger.info(
"Attempting to INSERT package %s into database with values: %s."
% (self, values))
_db.query("""
INSERT INTO `tabPackage`
(name, latest_version, home_page, releases, _created_at, _updated_at)
VALUES
('%s', '%s', '%s', '%s', '%s', '%s')
""" % values)
except (db.IntegrityError, db.OperationalError) as e:
logger.warn("Unable to save package name. %s" % e)
else:
if sync:
logger.info(
"Attempting to UPDATE package %s within database." % self)
try:
_db.query("""
UPDATE `tabPackage`
SET latest_version = '%s', home_page = '%s', releases = '%s', _updated_at = '%s'
WHERE
name = '%s'
""" % (self.latest_version, self.home_page, ",".join(
self.releases), datetime.now(), self.name))
except db.OperationalError as e:
logger.warn("Unable to update package name. %s" % e)
else:
logger.info("Using cached info for package %s." % self)
self.latest_version = res["latest_version"]
self.home_page = res["home_page"]
self.releases = res["releases"].split(",")
self.dependency_tree = TreeNode(self)