def test_GetExpressionSets_wiki_with_commutativity(
wiki: programl_pb2.ProgramGraph,
wiki_with_commutativity: programl_pb2.ProgramGraph,
):
"""Test that commutative operands yield the same expression sets."""
assert subexpressions.GetExpressionSets(
programl.ProgramGraphToNetworkX(
wiki)) == subexpressions.GetExpressionSets(
programl.ProgramGraphToNetworkX(wiki_with_commutativity))
def CreateRandomGraphTuple(ir_id: int) -> graph_tuple_database.GraphTuple: """Generate a random graph tuple.""" g = random_cdfg_generator.FastCreateRandom() proto = networkx_to_protos.NetworkXGraphToProgramGraphProto(g) g = programl.ProgramGraphToNetworkX(proto) gt = graph_tuple.GraphTuple.CreateFromNetworkX(g) return graph_tuple_database.GraphTuple.CreateFromGraphTuple(gt, ir_id=ir_id)
def MakeAnnotated(
self,
n: Optional[int] = None
) -> data_flow_graphs.NetworkxDataFlowGraphs:
g = programl.ProgramGraphToNetworkX(self.unlabelled_graph)
return data_flow_graphs.NetworkxDataFlowGraphs(
[g for _ in range(n or 1)])
def test_fuzz_proto_networkx_equivalence(
node_x_dimensionality: int,
node_y_dimensionality: int,
graph_x_dimensionality: int,
graph_y_dimensionality: int,
node_count: int,
):
"""Fuzz proto -> networkx -> proto on random generated graphs."""
proto_in = random_programl_generator.CreateRandomProto(
node_x_dimensionality=node_x_dimensionality,
node_y_dimensionality=node_y_dimensionality,
graph_x_dimensionality=graph_x_dimensionality,
graph_y_dimensionality=graph_y_dimensionality,
node_count=node_count,
)
# proto -> networkx
g = programl.ProgramGraphToNetworkX(proto_in)
assert g.number_of_nodes() == len(proto_in.node)
assert g.number_of_edges() == len(proto_in.edge)
# Check that the functions match up.
functions_in_graph = set([
function for _, function in g.nodes(data="function")
if function is not None
])
functions_in_proto = [function.name for function in proto_in.function]
assert sorted(functions_in_proto) == sorted(functions_in_graph)
# networkx -> proto
proto_out = programl.NetworkXToProgramGraph(g)
assert proto_out.function == proto_in.function
assert proto_out.node == proto_in.node
# Randomly generated graphs don't have a stable edge order.
assert len(proto_out.edge) == len(proto_in.edge)
def BuildProgramGraphNetworkX(
module: str,
timeout: int = 120,
graph: Optional[programl_pb2.ProgramGraph] = None,
) -> nx.MultiDiGraph:
"""Construct a NetworkX program graph for the given LLVM IR.
Args:
module: The LLVM IR string for a module.
timeout: The maximum number of seconds to allow graph construction to run
for.
graph: An existing graph message to write the result to. If not provided,
a new graph message is constructed.
Returns:
A NetworkX MultiDiGraph instance.
Raises:
TimeoutError: If graph construction fails to complete within timeout
seconds.
ValueError: If graph construction fails.
"""
# NOTE: A more direct approach to generating a networkx graph instance would
# be to add a --stdout_fmt=json option to
# //deeplearning/ml4pl/graphs/llvm2graph which would produce output in the
# format expected by networkx. See:
# https://networkx.github.io/documentation/networkx-1.10/reference/generated/networkx.convert.to_dict_of_dicts.html#networkx.convert.to_dict_of_dicts
return programl.ProgramGraphToNetworkX(
BuildProgramGraphProto(module=module, timeout=timeout, graph=graph))
def CreateRandomGraph(
node_x_dimensionality: int = 1,
node_y_dimensionality: int = 0,
graph_x_dimensionality: int = 0,
graph_y_dimensionality: int = 0,
node_count: int = None,
) -> nx.MultiDiGraph:
"""Generate a random graph.
This generates a random graph which has sensible values for fields, but does
not have meaningful semantics, e.g. there may be data flow edges between
identifiers, etc. For speed, this generator guarantees only that:
1. There is a 'root' node with outgoing call edges.
2. Nodes are either statements, identifiers, or immediates.
3. Nodes have text, preprocessed_text, and a single node_x value.
4. Edges are either control, data, or call.
5. Edges have positions.
6. The graph is strongly connected.
"""
proto = random_programl_generator.CreateRandomProto(
node_x_dimensionality=node_x_dimensionality,
node_y_dimensionality=node_y_dimensionality,
graph_x_dimensionality=graph_x_dimensionality,
graph_y_dimensionality=graph_y_dimensionality,
node_count=node_count,
)
return programl.ProgramGraphToNetworkX(proto)
def Main():
"""Main entry point."""
proto = programl.ReadStdin()
g = programl.ProgramGraphToNetworkX(proto)
encoder = GraphNodeEncoder()
ir = fs.Read(FLAGS.ir) if FLAGS.ir else None
encoder.EncodeNodes(g, ir=ir)
programl.WriteStdout(programl.NetworkXToProgramGraph(g))
def test_GetExpressionSets_wiki(wiki: programl_pb2.ProgramGraph):
expressions = subexpressions.GetExpressionSets(
programl.ProgramGraphToNetworkX(wiki))
assert sorted(expressions) == [
{8, 9},
{10},
{11},
]
def AnnotateGraphMetas(
ir_db: ir_database.Database,
proto_db: unlabelled_graph_database.Database,
df: pd.DataFrame,
ctx: progress.ProgressContext = progress.NullContext,
) -> Iterable[graph_tuple_database.GraphTuple]:
"""Add features and labels to graph metas in database."""
with ir_db.Session() as ir_session, proto_db.Session() as proto_session:
for _, row in df.iterrows():
relpath = row["relpath"]
with ctx.Profile(
2,
f"Processed graph {row['relpath']}:{row['data:dataset_name']}"
):
# Select the corresponding IR.
ir_id = (ir_session.query(
ir_database.IntermediateRepresentation.id).filter(
ir_database.IntermediateRepresentation.source ==
"pact17_opencl_devmap",
ir_database.IntermediateRepresentation.relpath ==
relpath,
).scalar())
# Check that we have an exact 1:1 mapping from the opencl devmap dataset
# to IR.
if ir_id is None:
raise ValueError(f"Expected one IR with relpath {relpath}")
# Load the program graph.
proto_row = (proto_session.query(
unlabelled_graph_database.ProgramGraph).filter(
unlabelled_graph_database.ProgramGraph.ir_id ==
ir_id).options(
sql.orm.joinedload(unlabelled_graph_database.
ProgramGraph.data)).scalar())
if proto_row is None:
raise ValueError(
f"Expected one proto for relpath {relpath} with ID {ir_id}"
)
proto: programl_pb2.ProgramGraph = proto_row.proto
# Add the null "selector vector" value.
for node in proto.node:
node.x.append(0)
# Add the graph-level features.
proto.x[:] = [row["wgsize"], row["transfer"]]
# Add 'y' graph feature as target.
proto.y[:] = row["y"].tolist()
# Create the graph tuple. Note the jumping through hoops with converting
# proto -> nx -> graph_tuple, because there is currently no direct
# proto -> graph_tuple conversion.
graph_tuple = graph_tuple_database.GraphTuple.CreateFromGraphTuple(
graph_tuple=graph_tuples.GraphTuple.CreateFromNetworkX(
programl.ProgramGraphToNetworkX(proto)),
ir_id=ir_id,
)
yield graph_tuple
def __init__(self, *args, **kwargs):
super(NetworkXDataFlowGraphAnnotator, self).__init__(*args, **kwargs)
self.g = programl.ProgramGraphToNetworkX(self.unlabelled_graph)
self.root_nodes = [
node for node, data in self.g.nodes(data=True)
if self.IsValidRootNode(node, data)
]
self.i = -1
def MakeAnnotated(
self,
n: Optional[int] = None
) -> data_flow_graphs.NetworkxDataFlowGraphs:
if random.random() < 0.2:
raise OSError("something went wrong!")
g = programl.ProgramGraphToNetworkX(self.unlabelled_graph)
return data_flow_graphs.NetworkxDataFlowGraphs(
[g for _ in range(n or 1)])
def test_GetExpressionSets_wiki_without_subexpressions(
wiki_without_subexpressions: programl_pb2.ProgramGraph, ):
expressions = subexpressions.GetExpressionSets(
programl.ProgramGraphToNetworkX(wiki_without_subexpressions))
assert sorted(expressions) == [
{8},
{9},
{10},
{11},
]
def test_proto_networkx_equivalence(
random_100_proto: programl_pb2.ProgramGraph, ):
"""Test proto -> networkx -> proto on 100 "real" graphs."""
# proto -> networkx
g = programl.ProgramGraphToNetworkX(random_100_proto)
assert g.number_of_nodes() == len(random_100_proto.node)
assert g.number_of_edges() == len(random_100_proto.edge)
# networkx -> proto
proto_out = programl.NetworkXToProgramGraph(g)
assert proto_out.function == random_100_proto.function
assert proto_out.node == random_100_proto.node
assert proto_out.edge == random_100_proto.edge
def test_proto_networkx_equivalence(
llvm_program_graph: programl_pb2.ProgramGraph,
):
"""Test proto -> networkx -> proto equivalence."""
# proto -> networkx
g = programl.ProgramGraphToNetworkX(llvm_program_graph)
assert g.number_of_nodes() == len(llvm_program_graph.node)
assert g.number_of_edges() == len(llvm_program_graph.edge)
# networkx -> proto
proto_out = programl.NetworkXToProgramGraph(g)
assert set(fn.name for fn in proto_out.function) == set(
fn.name for fn in llvm_program_graph.function
)
assert len(llvm_program_graph.node) == len(proto_out.node)
assert len(llvm_program_graph.edge) == len(proto_out.edge)
def test_proto_networkx_equivalence_with_preallocated_proto(
random_100_proto: programl_pb2.ProgramGraph, ):
"""Test proto -> networkx -> proto on 100 "real" graphs using the same
proto instance."""
# proto -> networkx
g = programl.ProgramGraphToNetworkX(random_100_proto)
assert g.number_of_nodes() == len(random_100_proto.node)
assert g.number_of_edges() == len(random_100_proto.edge)
# networkx -> proto
# Allocate the proto ahead of time:
proto_out = programl_pb2.ProgramGraph()
programl.NetworkXToProgramGraph(g, proto=proto_out)
assert proto_out.function == random_100_proto.function
assert proto_out.node == random_100_proto.node
assert proto_out.edge == random_100_proto.edge
def test_proto_networkx_equivalence_with_preallocated_proto(
llvm_program_graph: programl_pb2.ProgramGraph,
):
"""Test proto -> networkx -> proto equivalent using pre-allocated protos."""
# proto -> networkx
g = programl.ProgramGraphToNetworkX(llvm_program_graph)
assert g.number_of_nodes() == len(llvm_program_graph.node)
assert g.number_of_edges() == len(llvm_program_graph.edge)
# networkx -> proto
# Allocate the proto ahead of time:
proto_out = programl_pb2.ProgramGraph()
programl.NetworkXToProgramGraph(g, proto=proto_out)
assert set(fn.name for fn in proto_out.function) == set(
fn.name for fn in llvm_program_graph.function
)
assert len(llvm_program_graph.node) == len(proto_out.node)
assert len(llvm_program_graph.edge) == len(proto_out.edge)
def test_profiled_ep_A():
"""Graph with profiling information."""
proto_in = llvm2graph.BuildProgramGraphProto(
fs.Read(REGRESSION_TESTS / "profiled_ep.A.ll"))
# Check that profiling data has been lifted.
assert any(f.HasField("llvm_entry_count") for f in proto_in.function)
assert any(n.HasField("llvm_profile_true_weight") for n in proto_in.node)
assert any(n.HasField("llvm_profile_false_weight") for n in proto_in.node)
assert any(n.HasField("llvm_profile_total_weight") for n in proto_in.node)
# Check that profiling data is preserved after conversion to networkx.
g = programl.ProgramGraphToNetworkX(proto_in)
assert g.graph["llvm_function_entry_count"]
assert any("llvm_profile_true_weight" in d for _, d in g.nodes(data=True))
assert any("llvm_profile_false_weight" in d for _, d in g.nodes(data=True))
assert any("llvm_profile_total_weight" in d for _, d in g.nodes(data=True))
# Check that profiling data is preserved after conversion back to proto.
proto_out = programl.NetworkXToProgramGraph(g)
assert any(f.HasField("llvm_entry_count") for f in proto_out.function)
assert any(n.HasField("llvm_profile_true_weight") for n in proto_out.node)
assert any(n.HasField("llvm_profile_false_weight") for n in proto_out.node)
assert any(n.HasField("llvm_profile_total_weight") for n in proto_out.node)
def CreateFromProgramGraph(cls, program_graph: programl_pb2.ProgramGraph):
# TODO(github.com/ChrisCummins/ProGraML/issues/31): Perform the conversion
# directly.
return cls.CreateFromNetworkX(
programl.ProgramGraphToNetworkX(program_graph))
def benchmark_networkx(
benchmark_proto: List[programl_pb2.ProgramGraph],
) -> List[nx.MultiDiGraph]:
"""A fixture which returns 10 graphs for benchmarking."""
return [programl.ProgramGraphToNetworkX(p) for p in benchmark_proto]
def EnumerateTestSet(n: Optional[int] = None) -> Iterable[nx.MultiDiGraph]:
"""Enumerate a test set of "real" graphs."""
for proto in random_programl_generator.EnumerateTestSet(n=n):
yield programl.ProgramGraphToNetworkX(proto)
