def test_multiple_inputs(): proto = pbutil.FromFile(runfiles_path("tests/data/a.hlo.pb"), xla_pb2.HloProto()) graphs = list(pg.from_xla_hlo_proto([proto] * 10)) assert len(graphs) == 10 for graph in graphs: assert isinstance(graph, pg.ProgramGraph)
import subprocess import time import numpy as np from absl import app, flags from programl.util.py.runfiles_path import runfiles_path flags.DEFINE_float( "min_benchmark_time", 30, "The minimum number of seconds to run the benchmark loop.", ) FLAGS = flags.FLAGS LLVM_IR = runfiles_path("programl/tests/data/llvm_ir") LLVM2GRAPH = runfiles_path("programl/bin/llvm2graph") def SummarizeFloats(floats, nplaces: int = 2, unit: str = "") -> str: """Summarize a sequence of floats.""" arr = np.array(list(floats), dtype=np.float32) percs = " ".join([ f"{p}%={np.percentile(arr, p):.{nplaces}f}{unit}" for p in [0, 50, 95, 99, 100] ]) return ( f"n={len(arr)}, mean={arr.mean():.{nplaces}f}{unit}, stdev={arr.std():.{nplaces}f}{unit}, " f"percentiles=[{percs}]")
def test_non_empty_proto(): """Build a graph proto from an example proto.""" proto = pbutil.FromFile(runfiles_path("tests/data/a.hlo.pb"), xla_pb2.HloProto()) graph = pg.from_xla_hlo_proto(proto) assert len(graph.node) == 155 assert len(graph.function) == 5
# Copyright 2019-2020 the ProGraML authors. # # Contact Chris Cummins <*****@*****.**>. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from programl.util.py.runfiles_path import runfiles_path # Directory containing ProgramGraphFeatures protocol messages for reachability # analysis. LLVM_REACHABILITY_FEATURES = runfiles_path( "programl/tests/data/llvm_ir_reachability")
"""Benchmark for analyze command.""" import subprocess import sys import time import numpy as np from absl import app, flags from programl.util.py.runfiles_path import runfiles_path flags.DEFINE_integer( "graph_count", 100, "If > 0, limit the number of graphs to benchmark." ) FLAGS = flags.FLAGS LLVM_IR_GRAPHS = runfiles_path("programl/tests/data/llvm_ir_graphs") ANALYZE = runfiles_path("programl/bin/analyze") ANALYSES = [ "reachability", "dominance", "datadep", "liveness", "subexpressions", ] def SummarizeFloats(floats, nplaces: int = 2, unit: str = "") -> str: """Summarize a sequence of floats.""" arr = np.array(list(floats), dtype=np.float32) percs = " ".join(
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Iterable, Tuple import pytest from programl.proto import program_graph_pb2 from programl.util.py import pbutil from programl.util.py.runfiles_path import runfiles_path LLVM_IR_GRAPHS = runfiles_path("programl/tests/data/llvm_ir_graphs") def EnumerateLlvmProgramGraphs( ) -> Iterable[Tuple[str, program_graph_pb2.ProgramGraph]]: """Enumerate a test set of LLVM IR file paths.""" for path in LLVM_IR_GRAPHS.iterdir(): yield path.name, pbutil.FromFile(path, program_graph_pb2.ProgramGraph()) _PROGRAM_GRAPHS = list(EnumerateLlvmProgramGraphs()) @pytest.fixture( scope="session",
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Unit tests for //deeplearning/ml4pl/graphs/xla2graph/py:xla2graph.""" import pytest from absl import flags from programl.ir.xla.py import xla from programl.util.py import pbutil from programl.util.py.runfiles_path import runfiles_path from tests.test_main import main from third_party.tensorflow import xla_pb2 FLAGS = flags.FLAGS TEST_PROTO = runfiles_path("programl/tests/data/a.hlo.pb") def test_empty_proto(): """Build from an empty proto.""" proto = xla_pb2.HloProto() with pytest.raises(ValueError) as e_ctx: xla.BuildProgramGraphProto(proto) assert "Failed to locate entry computation" in str(e_ctx.value) def test_non_empty_proto(): """Build a graph proto from an example proto.""" proto = pbutil.FromFile(TEST_PROTO, xla_pb2.HloProto()) graph = xla.BuildProgramGraphProto(proto)
# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """End-to-end test that --strict flag on llvm2graph rejects invalid graphs.""" import subprocess import pytest from programl.util.py.runfiles_path import runfiles_path from tests.test_main import main LLVM2GRAPH = runfiles_path("programl/bin/llvm2graph-10") # This IR file contains unreachable instructions, which will be rejected if # --strict mode is enabled. INVALID_MODULE = runfiles_path("tests/data/module_with_unreachable_instructions.ll") def test_invalid_module(): subprocess.check_call([str(LLVM2GRAPH), str(INVALID_MODULE)]) def test_invalid_module_strict(): with pytest.raises(subprocess.CalledProcessError): subprocess.check_call([str(LLVM2GRAPH), str(INVALID_MODULE), "--strict"])
import multiprocessing import pathlib import pickle import random import time from typing import Any, Iterable, List, Optional, Tuple import numpy as np from programl.proto import node_pb2, program_graph_pb2 from programl.util.py import decorators, pbutil, progress from programl.util.py.runfiles_path import runfiles_path from third_party.py.ncc.inst2vec import inst2vec_preprocess DICTIONARY = runfiles_path( "programl/programl/ir/llvm/internal/inst2vec_augmented_dictionary.pickle" ) AUGMENTED_INST2VEC_EMBEDDINGS = runfiles_path( "programl/programl/ir/llvm/internal/inst2vec_augmented_embeddings.pickle" ) def NodeFullText(node: node_pb2.Node) -> str: """Get the full text of a node, or an empty string if not set.""" if len(node.features.feature["full_text"].bytes_list.value): return node.features.feature["full_text"].bytes_list.value[0].decode("utf-8") return "" class Inst2vecEncoder(object): """An encoder for LLVM program graphs using inst2vec."""
code or compiler intermediate representations (IRs). """ import subprocess from typing import Iterable, List, Optional, Union import google.protobuf.message from programl.exceptions import GraphCreationError, UnsupportedCompiler from programl.proto import ProgramGraph from programl.third_party.tensorflow.xla_pb2 import HloProto from programl.util.py.cc_system_includes import get_system_includes from programl.util.py.executor import ExecutorLike, execute from programl.util.py.runfiles_path import runfiles_path LLVM2GRAPH_BINARIES = { "10": str(runfiles_path("programl/bin/llvm2graph-10")), "6": str(runfiles_path("programl/bin/llvm2graph-6")), "3.8": str(runfiles_path("programl/bin/llvm2graph-3.8")), } LLVM_VERSIONS = list(LLVM2GRAPH_BINARIES.keys()) CLANG2GRAPH_BINARIES = { "10": str(runfiles_path("programl/bin/clang2graph-10")), } CLANG_VERSIONS = list(CLANG2GRAPH_BINARIES.keys()) XLA2GRAPH = str(runfiles_path("programl/bin/xla2graph"))
load_graphs, save_graphs, to_bytes, to_string, ) from programl.transform_ops import to_dgl, to_dot, to_json, to_networkx from programl.util.py.runfiles_path import runfiles_path from programl.version import PROGRAML_VERSION __version__ = PROGRAML_VERSION __author__ = "Chris Cummins" __email__ = "*****@*****.**" __copyright__ = "Copyright 2019-2020 the ProGraML authors" __license__ = "Apache License, Version 2.0" binaries_path: Path = runfiles_path("programl/bin") __all__ = [ "binaries_path", "CLANG_VERSIONS", "from_bytes", "from_clang", "from_cpp", "from_llvm_ir", "from_string", "from_xla_hlo_proto", "GraphCreationError", "GraphTransformError", "LLVM_VERSIONS", "load_graphs", "ProgramGraph",
# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """This module defines convertors from ProgramGraph to CDFG.""" import subprocess from typing import Optional import google.protobuf.message from programl.proto import program_graph_pb2 from programl.util.py.runfiles_path import runfiles_path GRAPH2CDFG = runfiles_path("programl/bin/graph2cdfg") def FromProgramGraphFile(path) -> Optional[program_graph_pb2.ProgramGraph]: """Convert a binary ProgramGraph message file to a CDFG. Args: path: The path of a ProgramGraph protocol buffer. Returns: A ProgramGraph instance, or None if graph conversion failed. Raises: ValueError: If the graph cannot be converted. """ graph = program_graph_pb2.ProgramGraph()
from programl.util.py.runfiles_path import runfiles_path from tasks.dataflow.dataset import pathflag from tasks.dataflow.dataset.encode_inst2vec import Inst2vecEncodeGraphs flags.DEFINE_string( "classifyapp", str(pathlib.Path("~/programl/classifyapp").expanduser()), "Path of the classifyapp database.", ) flags.DEFINE_string("host", None, "The database to export from.") flags.DEFINE_string("user", None, "The database to export from.") flags.DEFINE_string("pwd", None, "The database to export from.") flags.DEFINE_string("db", None, "The database to export from.") FLAGS = flags.FLAGS CREATE_LABELS = runfiles_path("programl/tasks/dataflow/dataset/create_labels") CREATE_VOCAB = runfiles_path("programl/tasks/dataflow/dataset/create_vocab") UNPACK_IR_LISTS = runfiles_path( "programl/tasks/dataflow/dataset/unpack_ir_lists") def _ProcessRow(output_directory, row, file_id) -> None: source, src_lang, ir_type, binary_ir = row # Decode database row. source = source.decode("utf-8") src_lang = { "C": "c", "CPP": "cc", "OPENCL": "cl", "SWIFT": "swift",
""" import json import subprocess from typing import Any, Dict, Iterable, Optional, Union import dgl import networkx as nx from dgl.heterograph import DGLHeteroGraph from networkx.readwrite import json_graph as nx_json from programl.exceptions import GraphTransformError from programl.proto import ProgramGraph from programl.util.py.executor import ExecutorLike, execute from programl.util.py.runfiles_path import runfiles_path GRAPH2DOT = str(runfiles_path("programl/bin/graph2dot")) GRAPH2JSON = str(runfiles_path("programl/bin/graph2json")) JsonDict = Dict[str, Any] def _run_graph_transform_binary( binary: str, graph: ProgramGraph, timeout: int = 300, ) -> Iterable[bytes]: process = subprocess.Popen( [binary, "--stdin_fmt=pb"], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
# # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Python interface for building program graphs from LLVM-IR.""" import subprocess import tempfile from programl.proto import program_graph_options_pb2, program_graph_pb2 from programl.util.py.runfiles_path import runfiles_path GRAPH_BUILDER_BIN = runfiles_path( "programl/programl/ir/llvm/py/graph_builder_bin") DefaultOptions = program_graph_options_pb2.ProgramGraphOptions() def BuildProgramGraph( ir: str, options: program_graph_options_pb2.ProgramGraphOptions = DefaultOptions, timeout: int = 60, ) -> program_graph_pb2.ProgramGraph: """Construct a program graph from an LLVM-IR. Args: ir: The text of an LLVM-IR Module. options: The graph construction options. timeout: The number of seconds to permit before timing out.
# You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from typing import Iterable, Tuple import pytest from programl.util.py.runfiles_path import runfiles_path LLVM_IR = runfiles_path("programl/tests/data/llvm_ir") def EnumerateLlvmIrs() -> Iterable[Tuple[str, str]]: """Enumerate a test set of LLVM IR file paths.""" for path in LLVM_IR.iterdir(): with open(str(path), "r") as f: yield path.name, f.read() _LLVM_IRS = list(EnumerateLlvmIrs()) @pytest.fixture(scope="session", params=_LLVM_IRS, ids=[s[0] for s in _LLVM_IRS])