def test_multiple_inputs():
proto = pbutil.FromFile(runfiles_path("tests/data/a.hlo.pb"), xla_pb2.HloProto())
graphs = list(pg.from_xla_hlo_proto([proto] * 10))
assert len(graphs) == 10
for graph in graphs:
assert isinstance(graph, pg.ProgramGraph)
import subprocess
import time
import numpy as np
from absl import app, flags
from programl.util.py.runfiles_path import runfiles_path
flags.DEFINE_float(
"min_benchmark_time",
30,
"The minimum number of seconds to run the benchmark loop.",
)
FLAGS = flags.FLAGS
LLVM_IR = runfiles_path("programl/tests/data/llvm_ir")
LLVM2GRAPH = runfiles_path("programl/bin/llvm2graph")
def SummarizeFloats(floats, nplaces: int = 2, unit: str = "") -> str:
"""Summarize a sequence of floats."""
arr = np.array(list(floats), dtype=np.float32)
percs = " ".join([
f"{p}%={np.percentile(arr, p):.{nplaces}f}{unit}"
for p in [0, 50, 95, 99, 100]
])
return (
f"n={len(arr)}, mean={arr.mean():.{nplaces}f}{unit}, stdev={arr.std():.{nplaces}f}{unit}, "
f"percentiles=[{percs}]")
def test_non_empty_proto():
"""Build a graph proto from an example proto."""
proto = pbutil.FromFile(runfiles_path("tests/data/a.hlo.pb"), xla_pb2.HloProto())
graph = pg.from_xla_hlo_proto(proto)
assert len(graph.node) == 155
assert len(graph.function) == 5
# Copyright 2019-2020 the ProGraML authors.
#
# Contact Chris Cummins <*****@*****.**>.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from programl.util.py.runfiles_path import runfiles_path
# Directory containing ProgramGraphFeatures protocol messages for reachability
# analysis.
LLVM_REACHABILITY_FEATURES = runfiles_path(
"programl/tests/data/llvm_ir_reachability")
"""Benchmark for analyze command."""
import subprocess
import sys
import time
import numpy as np
from absl import app, flags
from programl.util.py.runfiles_path import runfiles_path
flags.DEFINE_integer(
"graph_count", 100, "If > 0, limit the number of graphs to benchmark."
)
FLAGS = flags.FLAGS
LLVM_IR_GRAPHS = runfiles_path("programl/tests/data/llvm_ir_graphs")
ANALYZE = runfiles_path("programl/bin/analyze")
ANALYSES = [
"reachability",
"dominance",
"datadep",
"liveness",
"subexpressions",
]
def SummarizeFloats(floats, nplaces: int = 2, unit: str = "") -> str:
"""Summarize a sequence of floats."""
arr = np.array(list(floats), dtype=np.float32)
percs = " ".join(
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Iterable, Tuple
import pytest
from programl.proto import program_graph_pb2
from programl.util.py import pbutil
from programl.util.py.runfiles_path import runfiles_path
LLVM_IR_GRAPHS = runfiles_path("programl/tests/data/llvm_ir_graphs")
def EnumerateLlvmProgramGraphs(
) -> Iterable[Tuple[str, program_graph_pb2.ProgramGraph]]:
"""Enumerate a test set of LLVM IR file paths."""
for path in LLVM_IR_GRAPHS.iterdir():
yield path.name, pbutil.FromFile(path,
program_graph_pb2.ProgramGraph())
_PROGRAM_GRAPHS = list(EnumerateLlvmProgramGraphs())
@pytest.fixture(
scope="session",
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for //deeplearning/ml4pl/graphs/xla2graph/py:xla2graph."""
import pytest
from absl import flags
from programl.ir.xla.py import xla
from programl.util.py import pbutil
from programl.util.py.runfiles_path import runfiles_path
from tests.test_main import main
from third_party.tensorflow import xla_pb2
FLAGS = flags.FLAGS
TEST_PROTO = runfiles_path("programl/tests/data/a.hlo.pb")
def test_empty_proto():
"""Build from an empty proto."""
proto = xla_pb2.HloProto()
with pytest.raises(ValueError) as e_ctx:
xla.BuildProgramGraphProto(proto)
assert "Failed to locate entry computation" in str(e_ctx.value)
def test_non_empty_proto():
"""Build a graph proto from an example proto."""
proto = pbutil.FromFile(TEST_PROTO, xla_pb2.HloProto())
graph = xla.BuildProgramGraphProto(proto)
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""End-to-end test that --strict flag on llvm2graph rejects invalid graphs."""
import subprocess
import pytest
from programl.util.py.runfiles_path import runfiles_path
from tests.test_main import main
LLVM2GRAPH = runfiles_path("programl/bin/llvm2graph-10")
# This IR file contains unreachable instructions, which will be rejected if
# --strict mode is enabled.
INVALID_MODULE = runfiles_path("tests/data/module_with_unreachable_instructions.ll")
def test_invalid_module():
subprocess.check_call([str(LLVM2GRAPH), str(INVALID_MODULE)])
def test_invalid_module_strict():
with pytest.raises(subprocess.CalledProcessError):
subprocess.check_call([str(LLVM2GRAPH), str(INVALID_MODULE), "--strict"])
import multiprocessing
import pathlib
import pickle
import random
import time
from typing import Any, Iterable, List, Optional, Tuple
import numpy as np
from programl.proto import node_pb2, program_graph_pb2
from programl.util.py import decorators, pbutil, progress
from programl.util.py.runfiles_path import runfiles_path
from third_party.py.ncc.inst2vec import inst2vec_preprocess
DICTIONARY = runfiles_path(
"programl/programl/ir/llvm/internal/inst2vec_augmented_dictionary.pickle"
)
AUGMENTED_INST2VEC_EMBEDDINGS = runfiles_path(
"programl/programl/ir/llvm/internal/inst2vec_augmented_embeddings.pickle"
)
def NodeFullText(node: node_pb2.Node) -> str:
"""Get the full text of a node, or an empty string if not set."""
if len(node.features.feature["full_text"].bytes_list.value):
return node.features.feature["full_text"].bytes_list.value[0].decode("utf-8")
return ""
class Inst2vecEncoder(object):
"""An encoder for LLVM program graphs using inst2vec."""
code or compiler intermediate representations (IRs).
"""
import subprocess
from typing import Iterable, List, Optional, Union
import google.protobuf.message
from programl.exceptions import GraphCreationError, UnsupportedCompiler
from programl.proto import ProgramGraph
from programl.third_party.tensorflow.xla_pb2 import HloProto
from programl.util.py.cc_system_includes import get_system_includes
from programl.util.py.executor import ExecutorLike, execute
from programl.util.py.runfiles_path import runfiles_path
LLVM2GRAPH_BINARIES = {
"10": str(runfiles_path("programl/bin/llvm2graph-10")),
"6": str(runfiles_path("programl/bin/llvm2graph-6")),
"3.8": str(runfiles_path("programl/bin/llvm2graph-3.8")),
}
LLVM_VERSIONS = list(LLVM2GRAPH_BINARIES.keys())
CLANG2GRAPH_BINARIES = {
"10": str(runfiles_path("programl/bin/clang2graph-10")),
}
CLANG_VERSIONS = list(CLANG2GRAPH_BINARIES.keys())
XLA2GRAPH = str(runfiles_path("programl/bin/xla2graph"))
load_graphs,
save_graphs,
to_bytes,
to_string,
)
from programl.transform_ops import to_dgl, to_dot, to_json, to_networkx
from programl.util.py.runfiles_path import runfiles_path
from programl.version import PROGRAML_VERSION
__version__ = PROGRAML_VERSION
__author__ = "Chris Cummins"
__email__ = "*****@*****.**"
__copyright__ = "Copyright 2019-2020 the ProGraML authors"
__license__ = "Apache License, Version 2.0"
binaries_path: Path = runfiles_path("programl/bin")
__all__ = [
"binaries_path",
"CLANG_VERSIONS",
"from_bytes",
"from_clang",
"from_cpp",
"from_llvm_ir",
"from_string",
"from_xla_hlo_proto",
"GraphCreationError",
"GraphTransformError",
"LLVM_VERSIONS",
"load_graphs",
"ProgramGraph",
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""This module defines convertors from ProgramGraph to CDFG."""
import subprocess
from typing import Optional
import google.protobuf.message
from programl.proto import program_graph_pb2
from programl.util.py.runfiles_path import runfiles_path
GRAPH2CDFG = runfiles_path("programl/bin/graph2cdfg")
def FromProgramGraphFile(path) -> Optional[program_graph_pb2.ProgramGraph]:
"""Convert a binary ProgramGraph message file to a CDFG.
Args:
path: The path of a ProgramGraph protocol buffer.
Returns:
A ProgramGraph instance, or None if graph conversion failed.
Raises:
ValueError: If the graph cannot be converted.
"""
graph = program_graph_pb2.ProgramGraph()
from programl.util.py.runfiles_path import runfiles_path
from tasks.dataflow.dataset import pathflag
from tasks.dataflow.dataset.encode_inst2vec import Inst2vecEncodeGraphs
flags.DEFINE_string(
"classifyapp",
str(pathlib.Path("~/programl/classifyapp").expanduser()),
"Path of the classifyapp database.",
)
flags.DEFINE_string("host", None, "The database to export from.")
flags.DEFINE_string("user", None, "The database to export from.")
flags.DEFINE_string("pwd", None, "The database to export from.")
flags.DEFINE_string("db", None, "The database to export from.")
FLAGS = flags.FLAGS
CREATE_LABELS = runfiles_path("programl/tasks/dataflow/dataset/create_labels")
CREATE_VOCAB = runfiles_path("programl/tasks/dataflow/dataset/create_vocab")
UNPACK_IR_LISTS = runfiles_path(
"programl/tasks/dataflow/dataset/unpack_ir_lists")
def _ProcessRow(output_directory, row, file_id) -> None:
source, src_lang, ir_type, binary_ir = row
# Decode database row.
source = source.decode("utf-8")
src_lang = {
"C": "c",
"CPP": "cc",
"OPENCL": "cl",
"SWIFT": "swift",
"""
import json
import subprocess
from typing import Any, Dict, Iterable, Optional, Union
import dgl
import networkx as nx
from dgl.heterograph import DGLHeteroGraph
from networkx.readwrite import json_graph as nx_json
from programl.exceptions import GraphTransformError
from programl.proto import ProgramGraph
from programl.util.py.executor import ExecutorLike, execute
from programl.util.py.runfiles_path import runfiles_path
GRAPH2DOT = str(runfiles_path("programl/bin/graph2dot"))
GRAPH2JSON = str(runfiles_path("programl/bin/graph2json"))
JsonDict = Dict[str, Any]
def _run_graph_transform_binary(
binary: str,
graph: ProgramGraph,
timeout: int = 300,
) -> Iterable[bytes]:
process = subprocess.Popen(
[binary, "--stdin_fmt=pb"],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Python interface for building program graphs from LLVM-IR."""
import subprocess
import tempfile
from programl.proto import program_graph_options_pb2, program_graph_pb2
from programl.util.py.runfiles_path import runfiles_path
GRAPH_BUILDER_BIN = runfiles_path(
"programl/programl/ir/llvm/py/graph_builder_bin")
DefaultOptions = program_graph_options_pb2.ProgramGraphOptions()
def BuildProgramGraph(
ir: str,
options: program_graph_options_pb2.ProgramGraphOptions = DefaultOptions,
timeout: int = 60,
) -> program_graph_pb2.ProgramGraph:
"""Construct a program graph from an LLVM-IR.
Args:
ir: The text of an LLVM-IR Module.
options: The graph construction options.
timeout: The number of seconds to permit before timing out.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import Iterable, Tuple
import pytest
from programl.util.py.runfiles_path import runfiles_path
LLVM_IR = runfiles_path("programl/tests/data/llvm_ir")
def EnumerateLlvmIrs() -> Iterable[Tuple[str, str]]:
"""Enumerate a test set of LLVM IR file paths."""
for path in LLVM_IR.iterdir():
with open(str(path), "r") as f:
yield path.name, f.read()
_LLVM_IRS = list(EnumerateLlvmIrs())
@pytest.fixture(scope="session",
params=_LLVM_IRS,
ids=[s[0] for s in _LLVM_IRS])
