def draw_networkx_with_pydot(nx_graph, include_third_party=False, outfile=None):
""" Draws a networkx graph using PyDot.
"""
log.info("")
import pydot
dot_graph = pydot.Dot(graph_type='digraph', suppress_disconnected=False)
dot_nodes = {}
for nx_node in nx_graph.nodes_iter():
if not include_third_party and not is_inhouse_package(nx_node.project_name):
continue
dot_node = get_dot_node(nx_node)
dot_nodes[nx_node] = dot_node
dot_graph.add_node(dot_node)
for edge in nx_graph.edges_iter():
if edge[0] not in dot_nodes or edge[1] not in dot_nodes:
continue
dot_graph.add_edge(get_dot_edge(edge, dot_nodes))
if not outfile:
tmpdir = tempfile.mkdtemp()
outfile = os.path.abspath(os.path.join(tmpdir, 'graph.png'))
# TODO: make neato work for nicer layouts
#dot_graph.write_png(outfile, prog='neato')
dot_graph.write_png(outfile)
webbrowser.open('file://%s' % outfile)
time.sleep(5)
if not outfile:
shutil.rmtree(tmpdir)
def pre(self, command, output_dir, vars):
if not is_inhouse_package(vars['package']):
sys.exit("Error: Package name doesn't start with an company prefix.\n" \
"Prefixes are:\n" \
"%s" % '\n'.join(CONFIG.namespaces))
prefix, package = vars['package'].split('.', 1)
vars['package'] = package
vars['namespace_package'] = prefix
vars['project_nodot'] = vars['project'].replace('.', '')
vars['project_dir'] = vars['project'].replace('.', '/')
def get_dot_node(req):
""" Get a pydot node object from a networkx node """
import pydot
fill_colour = "#cccccc"
shape = "box"
if is_inhouse_package(req.project_name):
fill_colour = "green"
return pydot.Node(' '.join((str(req.project_name), req._chosen_dist.version)),
style="filled",
fillcolor=fill_colour,
shape=shape)
def draw_networkx_with_pydot(nx_graph, include_third_party=False,
outfile=None, show_reqs=False):
""" Draws a networkx graph using PyDot.
Parameters
----------
nx_graph: `networkx.DiGraph`
The requirements graph
include_third_party: `bool`
Show third-party packages on the graph
outfile: `path`
If set, will save the graph to this path
show_reqs: `bool`
If set, will show the graph of requirements instead of the related
distributions - this more closely mirrors the underlying data.
"""
import pydot
dot_graph = pydot.Dot(graph_type='digraph', suppress_disconnected=False)
dot_nodes = {}
fn_node = get_dot_dist_node
fn_edge = get_dot_dist_edge
if show_reqs:
fn_node = get_dot_req_node
fn_edge = get_dot_req_edge
for nx_node in nx_graph.nodes_iter():
if (not include_third_party and
not is_inhouse_package(nx_node.project_name)):
continue
# Skip requirements that aren't yet 'installed' - this is only the
# case mid-way through dependency resolution
if hasattr(nx_node, '_chosen_dist'):
dot_node = fn_node(nx_node)
dot_nodes[nx_node] = dot_node
dot_graph.add_node(dot_node)
for edge in nx_graph.edges_iter():
if edge[0] not in dot_nodes or edge[1] not in dot_nodes:
continue
dot_graph.add_edge(fn_edge(edge, dot_nodes))
if not outfile:
tmpdir = tempfile.mkdtemp()
outfile = os.path.abspath(os.path.join(tmpdir, 'graph.png'))
# TODO: make neato work for nicer layouts
#dot_graph.write_png(outfile, prog='neato')
log.info("Rendering graph to {0}".format(outfile))
dot_graph.write_png(outfile)
webbrowser.open('file://%s' % outfile)
time.sleep(5)
if not outfile:
shutil.rmtree(tmpdir)
def pre(self, command, output_dir, pkg_vars):
if not is_inhouse_package(pkg_vars['project']):
msg = "Package name doesn't start with an company prefix.\n" \
"Prefixes are:\n" \
"%s" % '\n'.join(CONFIG.namespaces)
sys.exit(msg)
nss = pkg_vars['namespace_separator']
prefix, package = pkg_vars['project'].split(
pkg_vars['namespace_separator'], 1)
pkg_vars['package'] = package
pkg_vars['namespace_package'] = prefix
pkg_vars['project_nodot'] = pkg_vars['project'].replace(nss, '')
pkg_vars['project_dotted'] = pkg_vars['project'].replace(nss, '.')
pkg_vars['project_dir'] = pkg_vars['project'].replace(nss, '/')
def get_dot_req_node(req):
""" Get a pydot node object from a networkx node that represents the
requirement and its installed dist
"""
import pydot
fill_colour = "#cccccc"
shape = "box"
if is_inhouse_package(req.project_name):
fill_colour = "green"
if hasattr(req, '_chosen_dist'):
dist = req._chosen_dist.version
else:
dist = ''
return pydot.Node('{0} ({1})'.format(req, dist),
style="filled",
fillcolor=fill_colour,
shape=shape)
def components_command(plat, ns):
platforms = plat.get_platform_packages()
if ns.package not in platforms:
raise PlatError("Package %s is not a platform package" % ns.package)
pkg_info = plat.get_installed_version(ns.package)
if not pkg_info:
raise PlatError("Package %s is not installed" % ns.package)
meta = plat.get_package_metadata(ns.package, pkg_info.version)
platform = "%s (%s): %s" % (ns.package, pkg_info.version,
meta.get("summary", ""))
statusmsg(platform)
components = plat.get_package_dependencies(ns.package).itervalues()
components = ["%s (%s)" % (pkg.name, pkg.version) for pkg in components
if pkg and manage.is_inhouse_package(pkg.name)]
components.sort()
for component in components:
statusmsg(" " + component)
def write_all_revisions(self):
""" Create ``allrevisions.txt`` file containing subversion revision
of every project upon which we depend. This won't have the
dependencies in it if we've not yet been set-up with
'setup.py develop' or similar.
"""
my_dist = dependency.get_dist(self.distribution.metadata.name)
revisions = [] # list of (name,version,url,rev) tuples
revisions.append((self.distribution.metadata.get_name(),
self.distribution.metadata.get_version(),
self.full_url,
self.revision))
# get all our requirements
all_requires = []
if my_dist:
my_require = my_dist.as_requirement()
try:
all_requires = pkg_resources.working_set.resolve([my_require])
except (pkg_resources.DistributionNotFound,
pkg_resources.VersionConflict):
# not installed yet -- will probably be OK when we're
# called after the build has taken place.
pass
for dist in all_requires:
if dist == my_dist or not is_inhouse_package(dist.project_name):
continue
rev_data = self.read_all_revisions(dist) # (name,version,url,rev)
if rev_data:
revisions.append(rev_data)
data = ['# These are the VCS revision numbers used for this particular',
'# build. This file can be used by release tools to tag a',
'# working build.']
for rev_data in revisions:
data.append(','.join([str(e) for e in rev_data]))
self.write_file("all revisions", self.all_revisions_file,
'\n'.join(data))
def draw_networkx_with_d3(nx_graph, include_third_party=False, outfile=None):
""" Draws a networkx graph using d3.
"""
from path import path
tmpl = string.Template((path(__file__).parent / 'd3' / 'fbl.html').text())
nodes = {}
edges = []
i = 0
for req in nx_graph.nodes_iter():
if (not include_third_party and
not is_inhouse_package(req.project_name)):
continue
#print "Node %d: %s" % (i, req.project_name)
nodes[req.key] = (i, req.project_name)
i += 1
for edge in nx_graph.edges_iter():
if edge[0].key not in nodes or edge[1].key not in nodes:
continue
#print "edge: %s -> %s" % (edge[0].key, edge[1].key)
#print '{source : %d, target: %d, weight: 1}' % (nodes[edge[0].key][0], nodes[edge[1].key][0])
edges.append('{source : %d, target: %d, weight: 0.5}' % (nodes[edge[0].key][0], nodes[edge[1].key][0]))
nodes = ',\n'.join(['{label: "%s"}' % i[1] for i in sorted(nodes.values(), key=operator.itemgetter(0))])
edges = ',\n'.join(edges)
if not outfile:
tmpdir = path(tempfile.mkdtemp())
outfile = tmpdir / 'graph.html'
else:
outfile = path(outfile)
outfile.write_text(tmpl.safe_substitute(nodes=nodes, links=edges))
webbrowser.open('file://%s' % outfile)
time.sleep(5)
if not outfile:
shutil.rmtree(tmpdir)
def get_version_comparitor(self, requirement):
""" Here we pick between 'dev' or 'final' versions.
We want to use different logic depending on if this is a
third-party or in-house package:
In-house-packages: we usually want the latest dev version as
keeping on head revisions is sensible to stop
code going stale
Third-party: we usually want the latest final version to protect
ourselves from OSS developers exposing their
latest untested code to the internet at large.
To override this logic the packager needs to specify an explicit
version pin in install_requires or similar for third-party
packages, or use the prefer-final setup flag for in-house packages.
"""
if manage.is_inhouse_package(requirement.project_name):
if self._prefer_final:
log.debug(' in-house package, prefer-final')
return easy_install._final_version
else:
log.debug(' in-house package, prefer-dev')
return self.is_dev_version
else:
log.debug(' third-party package, always prefer-final')
return easy_install._final_version
def test_is_inhouse_package():
with mock.patch.object(manage, 'CONFIG', TEST_CONFIG):
assert manage.is_inhouse_package('acme.foo')
assert not manage.is_inhouse_package('foo')
def _obtain(self, requirement, source=None):
""" Copy in and override the installer's obtain method to modify the
following behavior:
- allow us to differentiate between third-party and in-house
packages when applying the prefer-final / prefer-dev logic
"""
# Look at zc.buildout source for comments on this logic
index = self._index
if index.obtain(requirement) is None:
return None
dists = [dist for dist in index[requirement.project_name] if (
dist in requirement and (
dist.location not in self._site_packages or
self.allow_site_package_egg(dist.project_name))
and (
(not source) or
(dist.precedence == pkg_resources.SOURCE_DIST))
)
]
# Here we pick between 'dev' or 'final' versions.
# We want to use different logic depending on if this is a third-party
# or in-house package:
# In-house-packages: we usually want the latest dev version as keeping
# on head revisions is sensible to stop code
# going stale
# Third-party: we usually want the latest final version to protect
# ourselves from OSS developers exposing their
# latest untested code to the intenet at large.
# To overridde this logic the packager needs to specify an explicit
# version pin in install_requires or similar for third-party packages,
# or use the prefer-final setup flag for in-house packages.
# If we prefer final dists, filter for final and use the
# result if it is non empty.
if manage.is_inhouse_package(requirement.project_name):
if self._prefer_final:
log.debug(' in-house package, prefer-final')
version_comparitor = easy_install._final_version
else:
log.debug(' in-house package, prefer-dev')
version_comparitor = self.is_dev_version
else:
log.debug(' third-party package, always prefer-final')
version_comparitor = easy_install._final_version
filtered_dists = [dist for dist in dists
if version_comparitor(dist.parsed_version)]
if filtered_dists:
log.debug(' filtered to {0}'.format(filtered_dists))
dists = filtered_dists
# The rest of this logic is as-is from buildout
best = []
bestv = ()
for dist in dists:
distv = dist.parsed_version
if distv > bestv:
best = [dist]
bestv = distv
elif distv == bestv:
best.append(dist)
log.debug(' best picks are {0}'.format(best))
if not best:
return None
if len(best) == 1:
return best[0]
if self._download_cache:
for dist in best:
if (easy_install.realpath(os.path.dirname(dist.location)) ==
self._download_cache):
return dist
best.sort()
return best[-1]
def test_is_inhouse_package():
with mock.patch.object(manage, 'CONFIG', config.OrganisationConfig(namespaces=['acme'])):
assert manage.is_inhouse_package('acme.foo')
assert not manage.is_inhouse_package('foo')