def load_adjlist(filename, directed=True):
edgelist = []
names = UniqueIdGenerator()
for line in open(filename):
parts = line.strip().split()
u = names[parts.pop(0)]
edgelist.extend([(u, names[v]) for v in parts])
logging.debug("Edgelist for line %s : %s" % (parts, edgelist))
g = Graph(edgelist, directed=directed)
g.vs["name"] = names.values()
return g
def _construct(self):
"""Construct the graph after it was populated."""
self.g = None
self.idgen = UniqueIdGenerator()
self._linkInstances()
edgelist = [(self.idgen[s], self.idgen[d]) for s, d in self.edges]
self.g = Graph(edgelist)
del edgelist
self.g.es["weight"] = list((self.weights[e] for e in self.edges))
del self.edges
self.g.vs["name"] = self.idgen.values()
self.g.vs["type"] = list((self.vertices[n] for n in self.g.vs["name"]))
del self.vertices
def reindexMembership(self, cl):
if hasattr(cl, "membership"):
cl = cl.membership
idgen = UniqueIdGenerator()
return [idgen[i] for i in cl]
tags = row.getAttribute("Tags")
taglist = tags.encode('utf-8').strip("><").split("><")
del tags
for i, source in enumerate(taglist):
for target in taglist[i + 1:]:
e = (source, target)
if e[0] > e[1]:
e = (target, source)
# cant add an edge if one of the verices are not made
g = Graph()
# print g
g.es["weight"] = 1.0
gen = UniqueIdGenerator()
for e in adjList:
# print e[0], e[1]
if e[0] not in gen:
sourceVertex = {}
# sourceVertex["uid"] = gen[e[0]]
sourceVertex = e[0]
g.add_vertex(sourceVertex)
# print g
if e[1] not in gen:
targetVertex = {}
# targetVertex["uid"] = gen[e[1]]
targetVertex = e[1]
g.add_vertex(targetVertex)
# print adjList[e]
class CommonGraph(object):
"""A graph representation of document accesses by userland apps."""
# Styling dictionaries.
cd = {"file": "blue", "app": "pink", "appstate": "red"}
sd = {"file": "circle", "app": "triangle-up", "appstate": "diamond"}
def __init__(self, outputDir: str = None):
"""Construct a CommonGraph."""
super(CommonGraph, self).__init__()
self.g = None
self.clusters = None
self.editCount = None
self.outputDir = outputDir or '/tmp/'
self.printClusterInstances = False
self.vertices = dict()
self.edges = set()
self.weights = dict()
self.instances = dict()
def _addFileNode(self, f: File):
"""Add a File vertex to the graph."""
# Add a vertex for the file.
self.vertices[str(f.inode)] = "file"
def _addAppNode(self, app: Application):
"""Add an Application vertex to the graph."""
raise NotImplementedError
# self.weights[(app.desktopid, app.uid())] = 0
def _addAccess(self, f: File, acc: FileAccess):
"""Add a FileAccess edge to the graph."""
raise NotImplementedError
def populate(self, policy: Policy = None, quiet: bool = False):
"""Populate the AccessGraph, filtering it based on a Policy."""
appStore = ApplicationStore.get()
fileStore = FileStore.get()
fileFactory = FileFactory.get()
userConf = UserConfigLoader.get()
# Add all user apps.
if not quiet:
tprnt("\t\tAdding apps...")
for app in appStore:
if app.isUserlandApp():
self._addAppNode(app)
def _allowed(policy, f, acc):
return acc.actor.isUserlandApp() and \
(acc.isByDesignation() or not policy or
policy.allowedByPolicy(f, acc.actor))
# Add all user documents.
if not quiet:
tprnt("\t\tAdding user documents...")
self.docCount = 0
for f in fileStore:
if not f.isUserDocument(userHome=userConf.getHomeDir(),
allowHiddenFiles=True):
continue
if f.isFolder():
continue
# Provided they have userland apps accessing them.
hasUserlandAccesses = False
for acc in f.getAccesses():
if _allowed(policy, f, acc):
hasUserlandAccesses = True
break
# And then add such userland apps to user document accesses.
if hasUserlandAccesses:
self.docCount += 1
self._addFileNode(f)
for acc in f.getAccesses():
if _allowed(policy, f, acc):
self._addAccess(f, acc)
if not quiet:
tprnt("\t\tAdding file links...")
links = fileFactory.getFileLinks()
for (pred, follow) in links.items():
source = str(pred.inode)
dest = str(follow)
if source in self.vertices and dest in self.vertices:
tprnt("Info: adding link from File %s to File %s in graph "
"as there is a file move/copy event between those." %
(source, dest))
edge = (source, dest) if source <= dest else (dest, source)
self.edges.add(edge)
self.weights[edge] = 999999999
if not quiet:
tprnt("\t\tConstructing graph...")
self._construct()
def _linkInstances(self):
"""Link application instance vertices together."""
raise NotImplementedError
def _construct(self):
"""Construct the graph after it was populated."""
self.g = None
self.idgen = UniqueIdGenerator()
self._linkInstances()
edgelist = [(self.idgen[s], self.idgen[d]) for s, d in self.edges]
self.g = Graph(edgelist)
del edgelist
self.g.es["weight"] = list((self.weights[e] for e in self.edges))
del self.edges
self.g.vs["name"] = self.idgen.values()
self.g.vs["type"] = list((self.vertices[n] for n in self.g.vs["name"]))
del self.vertices
def computeClusters(self):
"""Compute the clusters for this graph."""
comm = self.g.community_fastgreedy(weights=self.g.es["weight"])
self.clusters = comm.as_clustering()
def plot(self, output: str = None):
"""Plot the graph and its communities to an output file."""
# Get style options set for the base graph plot.
vs = {}
vs["vertex_size"] = 5
vs["vertex_color"] = [CommonGraph.cd[t] for t in self.g.vs["type"]]
vs["vertex_shape"] = [CommonGraph.sd[t] for t in self.g.vs["type"]]
labels = list(self.g.vs["name"])
for (idx, label) in enumerate(labels):
if self.g.vs["type"][idx] not in ("file", "appstate"):
labels[idx] = None
vs["vertex_label"] = labels
vs["edge_width"] = [.5 * (1 + int(c)) for c in self.g.es["weight"]]
vs["layout"] = self.g.layout("fr")
vs["bbox"] = (2400, 1600)
vs["margin"] = 20
# Plot the base graph.
try:
if output:
path = self.outputDir + "/" + output + ".graph.svg"
plot(self.g, path, **vs)
else:
plot(self.g, **vs)
except (OSError) as e:
print("Error while plotting to %s: %s " %
(self.outputDir + "/" + output + ".graph.svg", e))
except (MemoryError) as e:
print("Error (MemoryError) while plotting to %s: %s " %
(self.outputDir + "/" + output + ".graph.svg", e))
def plotClusters(self, output: str = None):
# Plot the base graph with colours based on the communities.
vs = {}
vs["vertex_size"] = 5
vs["vertex_shape"] = [CommonGraph.sd[t] for t in self.g.vs["type"]]
vs["layout"] = self.g.layout("fr")
vs["bbox"] = (2400, 1600)
vs["margin"] = 20
vs["vertex_color"] = self.clusters.membership
edge_widths = []
for (s, d) in self.g.get_edgelist():
if self.clusters.membership[s] == self.clusters.membership[d]:
edge_widths.append(1)
else:
edge_widths.append(3)
vs["edge_width"] = edge_widths
# Only keep labels for community-bridging vertices.
minimal_labels = list(self.g.vs["name"])
for (idx, label) in enumerate(minimal_labels):
if self.g.vs["type"][idx] not in ("file", "appstate") and not \
self.printClusterInstances:
minimal_labels[idx] = None
continue
for neighbour in self.g.neighbors(label):
if self.clusters.membership[neighbour] != \
self.clusters.membership[idx]:
break
else:
minimal_labels[idx] = None
vs["vertex_label"] = minimal_labels
try:
if output:
path = self.outputDir + "/" + output + ".clusters.svg"
plot(self.clusters, path, **vs)
else:
plot(self.clusters, **vs)
except (OSError) as e:
print("Error while plotting to %s: %s " %
(self.outputDir + "/" + output + ".clusters.svg", e))
except (MemoryError) as e:
print("Error (MemoryError) while plotting to %s: %s " %
(self.outputDir + "/" + output + ".graph.svg", e))
def calculateCosts(self,
output: str = None,
quiet: bool = False,
policy: Policy = None):
"""Model the usability costs needed to reach found communities."""
if not self.clusters:
raise ValueError("Clusters for a graph must be computed "
"before calculating its cost.")
msg = ""
appStore = ApplicationStore.get()
crossing = self.clusters.crossing()
grantingCost = 0
isolationCost = 0
splittingCost = 0
for (index, x) in enumerate(crossing):
if not x:
continue
edge = self.g.es[index]
source = self.g.vs[edge.source]
target = self.g.vs[edge.target]
sourceType = source.attributes()['type']
targetType = target.attributes()['type']
sourceName = source.attributes()['name']
targetName = target.attributes()['name']
# Case where a file-file node was removed. Should normally not
# happen so we will not write support for it yet.
if sourceType == "file":
if targetType == "app":
grantingCost += 1
if policy:
app = appStore.lookupUid(targetName)
policy.incrementScore('graphGrantingCost', None, app)
else:
# Check if an app co-accessed the files. If so, increase the
# cost of splitting that app instance into two.
sAccessors = []
for n in source.neighbors():
if n.attributes()['type'] == 'app':
sAccessors.append(n)
tAccessors = []
for n in target.neighbors():
if n.attributes()['type'] == 'app':
tAccessors.append(n)
inter = intersection(sAccessors, tAccessors)
for i in inter:
splittingCost += 1
if policy:
app = appStore.lookupUid(sourceName)
policy.incrementScore('graphSplittingCost', None,
app)
if not inter:
print(
"Warning: file-file node removed by graph "
"community finding algorithm. Not supported.",
file=sys.stderr)
print(source, target)
raise NotImplementedError
elif targetType == "file": # sourceType in "app", "appstate"
grantingCost += 1
if sourceType == "app" and policy:
app = appStore.lookupUid(sourceName)
policy.incrementScore('graphGrantingCost', None, app)
elif policy:
policy.incrementScore('graphGranting', None, None)
else:
# app-app links are just noise in the UnifiedGraph
if sourceType != "app" and targetType == "app":
isolationCost += 1
if policy:
app = appStore.lookupUid(targetName)
policy.incrementScore('graphIsolationCost', None, app)
elif sourceType == "app" and targetType != "app":
isolationCost += 1
if policy:
app = appStore.lookupUid(sourceName)
policy.incrementScore('graphIsolationCost', None, app)
editCount = grantingCost + isolationCost + splittingCost
msg += ("%d edits performed: %d apps isolated, %d apps split and "
"%d accesses revoked.\n" %
(editCount, isolationCost, splittingCost, grantingCost))
if not quiet:
tprnt(msg)
if output:
path = self.outputDir + "/" + output + ".graphstats.txt"
os.makedirs(File.getParentNameFromName(path), exist_ok=True)
with open(path, "w") as f:
print(msg, file=f)
self.editCount = editCount
def calculateReachability(self,
output: str = None,
quiet: bool = False,
nodeCount: int = 0):
"""Model the reachability improvement of community finding."""
if self.clusters is None:
raise ValueError("Clusters for a graph must be computed "
"before modelling how community isolation "
"decreases its average reachability.")
if self.editCount is None:
raise ValueError("Costs for a graph must be calculated "
"before modelling how community isolation "
"decreases its average reachability.")
msg = ""
def _print(clusters, header, tag):
msg = "\nGraph statistics %s:\n" % header
if len(clusters) == 0:
msg += "no clusters for this graph."
return (msg, 0, 1)
sizes = [
x for x in sorted(list((len(x) for x in clusters))) if x != 0
]
vertexSum = sum(sizes)
isolatedNC = nodeCount - self.docCount
msg += ("* %s-size distribution: %s\n" % (tag, sizes.__str__()))
msg += ("* %s-cluster count: %d\n" % (tag, len(sizes)))
msg += ("* %s-isolated nodes: %d\n" % (tag, isolatedNC))
msg += ("* %s-smallest cluster: %d\n" % (tag, min(sizes)))
msg += ("* %s-largest cluster: %d\n" % (tag, max(sizes)))
avgSize = vertexSum / len(sizes)
msg += ("* %s-average size: %f\n" % (tag, avgSize))
reach = sum([i**2 for i in sizes]) / vertexSum
msg += ("* %s-average reachability: %f\n" % (tag, reach))
reach = (sum([i ** 2 for i in sizes]) + isolatedNC) / \
(vertexSum + isolatedNC)
msg += ("* %s-adjusted reachability: %f\n" % (tag, reach))
return (msg, avgSize, reach)
def _printAndSum(g, editCount, tagPrefix=None):
msg = "\n"
preTag = tagPrefix + "-pre" if tagPrefix else "pre"
_m, avgPreSize, preReach = _print(g.g.clusters(),
"pre community finding", preTag)
msg += _m
postTag = tagPrefix + "-post" if tagPrefix else "post"
_m, avgPostSize, postReach = _print(g.clusters,
"post community finding",
postTag)
msg += _m
if avgPreSize:
deltaSize = 1 - (avgPostSize / avgPreSize)
sizeEfficiency = deltaSize / editCount if editCount else 1
msg += "\nEvol. of avg. cluster size: {:.2%}\n".format(
deltaSize)
msg += ("Efficiency of edits wrt. average size: %f\n" %
sizeEfficiency)
else:
msg += "\nEvol. of avg. cluster size: N/A\n"
if preReach:
deltaReach = 1 - (postReach / preReach)
reachEfficiency = deltaReach / editCount if editCount else 1
msg += "\nEvol. of reachability: {:.2%}\n".format(deltaReach)
msg += ("Efficiency of edits wrt. adj. reachability: %f\n" %
reachEfficiency)
else:
msg += "\nEvol. of adj. reachability: N/A\n"
return msg
if not quiet:
tprnt("\t\tPrinting statistics on whole graph...")
msg += _printAndSum(self, self.editCount)
if not quiet:
tprnt("\t\tBuilding flat file graph...")
fg = FlatGraph(parent=self, quiet=quiet)
if not plottingDisabled():
if not quiet:
tprnt("\t\tPlotting flat file graph...")
fg.plot(output=output)
if not quiet:
tprnt("\t\tPrinting statistics on flat file graph...")
msg += _printAndSum(fg, self.editCount, tagPrefix="flat")
if not quiet:
tprnt(msg)
if output:
path = self.outputDir + "/" + output + ".graphstats.txt"
os.makedirs(File.getParentNameFromName(path), exist_ok=True)
with open(path, "a") as f:
print(msg, file=f)
def __init__(self, parent: CommonGraph, quiet: bool = False):
"""Construct a FlatGraph."""
super(FlatGraph, self).__init__()
if not isinstance(parent, CommonGraph):
raise TypeError("FlatGraph constructor needs a CommonGraph "
"parent, received a %s." %
parent.__class__.__name__)
self.g = None
self.clusters = None
self.outputDir = parent.outputDir
self.vertices = dict()
self.edges = set()
self.weights = dict()
# Step 1. make a copy of the graph without file-file nodes, to
# find paths between files that go through apps.
if not quiet:
tprnt("\t\t\tStep 1: copy graph, excluding file-file nodes...")
tprnt("\t\t\t\tCopy graph...")
copy = parent.g.copy() # type: Graph
types = parent.g.vs['type']
names = parent.g.vs['name']
toBeRemoved = []
namesRemoved = []
if not quiet:
tprnt("\t\t\t\tFind edges to delete...")
for edge in copy.es:
if types[edge.source] == "file" and \
types[edge.target] == "file":
toBeRemoved.append(edge)
namesRemoved.append((names[edge.source], names[edge.target]))
if not quiet:
tprnt("\t\t\t\tDelete edges...")
copy.delete_edges(toBeRemoved)
# Step 2. run an all-pairs shortest path algorithm.
# Step 2. pick out file-file paths with no intermediary files.
# Step 2. save this info in the form of an edge list.
if not quiet:
tprnt("\t\t\tStep 2: run an all-pairs shortest path "
"algorithm, remove file-file paths with intermediary "
"files and gather final file-file edges...")
tprnt("\t\t\t\tCopy file nodes...")
fileNodes = list(
(copy.vs[i] for i, t in enumerate(types) if t == "file"))
edges = set()
# weights = dict()
self.idgen = UniqueIdGenerator()
fileNodeCount = len(fileNodes)
if not quiet:
tprnt("\t\t\t\tGet shortest paths for each of %d file nodes..." %
fileNodeCount)
threshold = fileNodeCount / 100
nodeI = 0
lastNodePct = 0
nodePct = 0
for v in fileNodes:
nodeI += 1
if nodeI >= (threshold * nodePct):
nodePct = int(nodeI / threshold)
if nodePct >= lastNodePct + 5:
print("\t\t\t\t\t... (%d%% done)" % nodePct)
lastNodePct = nodePct
# Get shortest paths.
vPaths = copy.get_shortest_paths(v, to=fileNodes)
# Remove unnecessary bits.
delSet = set()
for (idx, p) in enumerate(vPaths):
if len(p) < 1:
continue
# Ignore paths with intermediary files.
for node in p[1:-1]:
if types[node] == "file":
delSet.add(idx)
# Remove unsuitable paths.
for i in sorted(list(delSet), reverse=True):
del vPaths[i]
del delSet
# Save the shortest paths remaining as edges.
for p in vPaths:
if len(p) <= 1:
continue
key = (self.idgen[names[p[0]]], self.idgen[names[p[-1]]])
edges.add(key)
# weights[key] = 1 / (len(p) - 1)
# Add edges for removed names
if not quiet:
tprnt("\t\t\t\tRe-add file-file direct nodes into graph...")
for (src, dest) in namesRemoved:
edges.add((self.idgen[src], self.idgen[dest]))
# Step 3. construct a graph with only file nodes.
if not quiet:
tprnt("\t\t\tStep 3: construct a graph with only file nodes...")
edges = list(edges)
self.g = Graph(edges)
del edges
# self.g.es["weight"] = list((weights[e] for e in edges))
self.g.vs["name"] = self.idgen.values()
# Steph 4. apply community information to the nodes.
if not quiet:
tprnt("\t\t\tStep 4: apply communities to flat graph...")
applyCommunities(self, parent.clusters.membership, names)
class FlatGraph(object):
"""An internal class for graph flattening."""
def __init__(self, parent: CommonGraph, quiet: bool = False):
"""Construct a FlatGraph."""
super(FlatGraph, self).__init__()
if not isinstance(parent, CommonGraph):
raise TypeError("FlatGraph constructor needs a CommonGraph "
"parent, received a %s." %
parent.__class__.__name__)
self.g = None
self.clusters = None
self.outputDir = parent.outputDir
self.vertices = dict()
self.edges = set()
self.weights = dict()
# Step 1. make a copy of the graph without file-file nodes, to
# find paths between files that go through apps.
if not quiet:
tprnt("\t\t\tStep 1: copy graph, excluding file-file nodes...")
tprnt("\t\t\t\tCopy graph...")
copy = parent.g.copy() # type: Graph
types = parent.g.vs['type']
names = parent.g.vs['name']
toBeRemoved = []
namesRemoved = []
if not quiet:
tprnt("\t\t\t\tFind edges to delete...")
for edge in copy.es:
if types[edge.source] == "file" and \
types[edge.target] == "file":
toBeRemoved.append(edge)
namesRemoved.append((names[edge.source], names[edge.target]))
if not quiet:
tprnt("\t\t\t\tDelete edges...")
copy.delete_edges(toBeRemoved)
# Step 2. run an all-pairs shortest path algorithm.
# Step 2. pick out file-file paths with no intermediary files.
# Step 2. save this info in the form of an edge list.
if not quiet:
tprnt("\t\t\tStep 2: run an all-pairs shortest path "
"algorithm, remove file-file paths with intermediary "
"files and gather final file-file edges...")
tprnt("\t\t\t\tCopy file nodes...")
fileNodes = list(
(copy.vs[i] for i, t in enumerate(types) if t == "file"))
edges = set()
# weights = dict()
self.idgen = UniqueIdGenerator()
fileNodeCount = len(fileNodes)
if not quiet:
tprnt("\t\t\t\tGet shortest paths for each of %d file nodes..." %
fileNodeCount)
threshold = fileNodeCount / 100
nodeI = 0
lastNodePct = 0
nodePct = 0
for v in fileNodes:
nodeI += 1
if nodeI >= (threshold * nodePct):
nodePct = int(nodeI / threshold)
if nodePct >= lastNodePct + 5:
print("\t\t\t\t\t... (%d%% done)" % nodePct)
lastNodePct = nodePct
# Get shortest paths.
vPaths = copy.get_shortest_paths(v, to=fileNodes)
# Remove unnecessary bits.
delSet = set()
for (idx, p) in enumerate(vPaths):
if len(p) < 1:
continue
# Ignore paths with intermediary files.
for node in p[1:-1]:
if types[node] == "file":
delSet.add(idx)
# Remove unsuitable paths.
for i in sorted(list(delSet), reverse=True):
del vPaths[i]
del delSet
# Save the shortest paths remaining as edges.
for p in vPaths:
if len(p) <= 1:
continue
key = (self.idgen[names[p[0]]], self.idgen[names[p[-1]]])
edges.add(key)
# weights[key] = 1 / (len(p) - 1)
# Add edges for removed names
if not quiet:
tprnt("\t\t\t\tRe-add file-file direct nodes into graph...")
for (src, dest) in namesRemoved:
edges.add((self.idgen[src], self.idgen[dest]))
# Step 3. construct a graph with only file nodes.
if not quiet:
tprnt("\t\t\tStep 3: construct a graph with only file nodes...")
edges = list(edges)
self.g = Graph(edges)
del edges
# self.g.es["weight"] = list((weights[e] for e in edges))
self.g.vs["name"] = self.idgen.values()
# Steph 4. apply community information to the nodes.
if not quiet:
tprnt("\t\t\tStep 4: apply communities to flat graph...")
applyCommunities(self, parent.clusters.membership, names)
def plot(self, output: str = None):
"""Plot the graph and its communities to an output file."""
# Get style options set for the base graph plot.
vs = {}
vs["vertex_size"] = 5
vs["vertex_shape"] = "circle"
vs["layout"] = self.g.layout("fr")
vs["bbox"] = (2400, 1600)
vs["margin"] = 20
# Plot the base graph with colours based on the communities.
vs["vertex_color"] = self.membership
edge_widths = []
for (s, d) in self.g.get_edgelist():
if self.membership[s] == self.membership[d]:
edge_widths.append(1)
else:
edge_widths.append(3)
vs["edge_width"] = edge_widths
# Only keep labels for community-bridging vertices.
minimal_labels = list(self.g.vs["name"])
for (idx, label) in enumerate(minimal_labels):
for neighbour in self.g.neighbors(label):
if self.membership[neighbour] != self.membership[idx]:
break
else:
minimal_labels[idx] = None
vs["vertex_label"] = minimal_labels
try:
if output:
path = self.outputDir + "/" + output + ".flat.svg"
plot(self.clusters, path, **vs)
else:
plot(self.clusters, **vs)
except (OSError) as e:
print("Error while plotting to %s: %s " %
(self.outputDir + "/" + output + ".flat.svg", e))