def __init__(self, requests, dependencies_lens):
"""Creates a request dependency graph.
Args:
requests: ([Request]) a list of requests.
dependencies_lens: (RequestDependencyLens)
"""
self._requests = requests
deps = dependencies_lens.GetRequestDependencies()
self._nodes_by_id = {
r.request_id: _RequestNode(r)
for r in self._requests
}
edges = []
for (parent_request, child_request, reason) in deps:
if (parent_request.request_id not in self._nodes_by_id
or child_request.request_id not in self._nodes_by_id):
continue
parent_node = self._nodes_by_id[parent_request.request_id]
child_node = self._nodes_by_id[child_request.request_id]
edges.append(_Edge(parent_node, child_node, reason))
self._first_request_node = self._nodes_by_id[
self._requests[0].request_id]
self._deps_graph = graph.DirectedGraph(self._nodes_by_id.values(),
edges)
self._HandleTimingDependencies()
def testIgnoresUnknownEdges(self):
nodes = [_IndexedNode(i) for i in xrange(7)]
edges = [graph.Edge(nodes[from_index], nodes[to_index])
for (from_index, to_index) in [
(0, 1), (0, 2), (1, 3), (3, 4), (5, 6)]]
edges.append(graph.Edge(nodes[4], _IndexedNode(42)))
edges.append(graph.Edge(_IndexedNode(42), nodes[5]))
g = graph.DirectedGraph(nodes, edges)
self.assertListEqual(range(7), sorted(self._NodesIndices(g)))
self.assertEqual(5, len(g.Edges()))
def MakeGraph(cls, count, edge_tuples):
"""Makes a graph from a list of edges.
Args:
count: Number of nodes.
edge_tuples: (from_index, to_index). Both indices must be in [0, count),
and uniquely identify a node.
"""
nodes = [_IndexedNode(i) for i in xrange(count)]
edges = [
graph.Edge(nodes[from_index], nodes[to_index])
for (from_index, to_index) in edge_tuples
]
return (nodes, edges, graph.DirectedGraph(nodes, edges))
def ConsumeGraph(self, request_graph):
"""Add a graph and process.
Args:
graph: (RequestDependencyGraph) the graph to add.
"""
assert graph not in self._graph_info
cost = request_graph.Cost()
self._graph_info[request_graph] = self._GraphInfo(cost=cost)
for n in request_graph.graph.Nodes():
self.AddNode(request_graph, n)
# TODO(mattcary): this is inefficient but our current API doesn't require an
# explicit graph creation from the client.
self._graph = graph.DirectedGraph(self.bags, self._edges.itervalues())
def __init__(self, clArgs=None, argDB=None, loadArgDB=1, tmpDir=None):
import graph
import nargs
if argDB is None:
import RDict
argDB = RDict.RDict(load=loadArgDB)
# Storage for intermediate test results
self.tmpDir = tmpDir
script.LanguageProcessor.__init__(self, clArgs, argDB)
config.base.Configure.__init__(self, self)
self.childGraph = graph.DirectedGraph()
self.substRE = re.compile(r'@(?P<name>[^@]+)@')
self.substFiles = {}
self.logName = 'configure.log'
self.header = 'matt_config.h'
self.makeMacroHeader = ''
self.makeRuleHeader = ''
self.cHeader = 'matt_fix.h'
self.headerPrefix = ''
self.substPrefix = ''
self.warningRE = re.compile('warning', re.I)
if not nargs.Arg.findArgument('debugSections', self.clArgs):
self.argDB['debugSections'] = ['screen']
# Perhaps these initializations should just be local temporary arguments
self.argDB['CPPFLAGS'] = ''
if not 'LDFLAGS' in self.argDB:
self.argDB['LDFLAGS'] = ''
self.batchSetup = []
self.batchIncludes = []
self.batchBodies = []
self.batchCleanup = []
self.batchIncludeDirs = []
self.batchLibs = []
self.dependencies = {}
self.configureParent = None
# List of packages actually found
self.packages = []
self.createChildren()
# Create argDB for user specified options only
self.clArgDB = dict([(nargs.Arg.parseArgument(arg)[0], arg)
for arg in self.clArgs])
return
def MakeGraph(cls, count, edge_tuples, serialize=False):
"""Makes a graph from a list of edges.
Args:
count: Number of nodes.
edge_tuples: (from_index, to_index). Both indices must be in [0, count),
and uniquely identify a node. Must be sorted
lexicographically by node indices.
"""
nodes = [_IndexedNode(i) for i in xrange(count)]
edges = [graph.Edge(nodes[from_index], nodes[to_index])
for (from_index, to_index) in edge_tuples]
g = graph.DirectedGraph(nodes, edges)
if serialize:
g = graph.DirectedGraph.FromJsonDict(
g.ToJsonDict(), _IndexedNode, graph.Edge)
nodes = sorted(g.Nodes(), key=operator.attrgetter('index'))
edges = sorted(g.Edges(), key=operator.attrgetter(
'from_node.index', 'to_node.index'))
return (nodes, edges, g)
def __init__(self,
requests,
dependencies_lens,
node_class=RequestNode,
edge_class=Edge):
"""Creates a request dependency graph.
Args:
requests: ([Request]) a list of requests.
dependencies_lens: (RequestDependencyLens)
node_class: (subclass of RequestNode)
edge_class: (subclass of Edge)
"""
self._requests = None
self._first_request_node = None
self._deps_graph = None
self._nodes_by_id = None
if requests is None: # Deserialization.
return
assert issubclass(node_class, RequestNode)
assert issubclass(edge_class, Edge)
self._requests = requests
deps = dependencies_lens.GetRequestDependencies()
self._nodes_by_id = {
r.request_id: node_class(r)
for r in self._requests
}
edges = []
for (parent_request, child_request, reason) in deps:
if (parent_request.request_id not in self._nodes_by_id
or child_request.request_id not in self._nodes_by_id):
continue
parent_node = self._nodes_by_id[parent_request.request_id]
child_node = self._nodes_by_id[child_request.request_id]
edges.append(edge_class(parent_node, child_node, reason))
self._first_request_node = self._nodes_by_id[
self._requests[0].request_id]
self._deps_graph = graph.DirectedGraph(self._nodes_by_id.values(),
edges)
self._HandleTimingDependencies()
def recursively_download_pages(config: Config, io_manager: "IOManager",
start_page: str) -> Results:
pages_to_download = {start_page}
downloaded_page_names: Set[str] = set()
downloaded_pages: List[Page] = []
link_graph = graph.DirectedGraph({})
pbar = progressbar.ProgressBar(fd=io_manager.output_stream)
i = 0
while True:
# Stop if we run out of pages to download
if len(pages_to_download) == 0:
break
# Stop if we hit the page download limit
if not config.should_download_more_pages(len(downloaded_pages)):
pbar.finish()
assert config.max_num_pages is not None
io_manager.output_stream.write(
"Reached limit of max number of pages to download ({})\n".
format(config.max_num_pages))
break
page_name = pages_to_download.pop()
try:
page = download_page(config, io_manager, page_name)
except Exception as e:
pbar.finish()
io_manager.error_stream.write(str(e))
io_manager.error_stream.write(
'Error occured when attempting to download page "{}", so no further pages will be downloaded.\n'
.format(page_name))
break
for outgoing_link in page.outgoing_links:
link_graph.add_edge(page.name, outgoing_link)
downloaded_pages.append(page)
downloaded_page_names.add(page_name)
# Account for cases where the page is redirected
downloaded_page_names.add(page.name)
pages_to_download.discard(page.name)
# TODO: keep track of and resolve redirects
for outgoing_link in page.outgoing_links:
if outgoing_link not in downloaded_page_names:
pages_to_download.add(outgoing_link)
i += 1
pbar.update(i)
return Results(
pages=downloaded_pages,
link_graph=link_graph,
)
def _PrunedGraph(self):
roots = self.graph.graph.RootNodes()
nodes = self.graph.graph.ReachableNodes(
roots, should_stop=lambda n: not n.before)
return graph.DirectedGraph(nodes, self.graph.graph.Edges())
import sys
sys.path.append('../')
import random
import graph
import bananagram as bg
w = open('../../data/sowpods.txt', 'r').read()
G = graph.DirectedGraph()
G.parselex(w)
graph.trie_to_dawg(G)
print("DAWG Finished!")
# Bananagrams tile frequency
freq = [
13, 3, 3, 6, 18, 3, 4, 3, 12, 2, 2, 5, 3, 8, 11, 3, 2, 9, 6, 9, 6, 3, 3, 2,
3, 2
]
chars = sorted(G.top.children.keys())
tiles = reduce(lambda x, y: x + y,
[[chars[i] for j in range(f)] for i, f in enumerate(freq)])
B = bg.Bananagrams(G)
random.seed(9208914850)
rack = random.sample(tiles, 21)