def test_DataPath_different_working_dir(): """Test that DataPath is not affected by current working dir.""" p = bazelutil.DataPath('phd/lib/labm8/data/test/hello_world') with fs.chdir('/tmp'): assert bazelutil.DataPath('phd/lib/labm8/data/test/hello_world') == p with tempfile.TemporaryDirectory() as d: with fs.chdir(d): assert bazelutil.DataPath( 'phd/lib/labm8/data/test/hello_world') == p
def test_DataPath_path_not_found(): """Test that FileNotFoundError is raised if the file is not found.""" with pytest.raises(FileNotFoundError) as e_info: bazelutil.DataPath('') assert f"No such file or directory: ''" in str(e_info) with pytest.raises(FileNotFoundError) as e_info: bazelutil.DataPath('/not/a/real/path') assert f"No such file or directory: '/not/a/real/path'" in str(e_info)
def test_config_is_valid(): """Test that config proto is valid.""" with tempfile.TemporaryDirectory() as d: config = pbutil.FromFile( bazelutil.DataPath( 'phd/deeplearning/clgen/tests/data/tiny/config.pbtxt'), clgen_pb2.Instance()) # Change the working directory and corpus path to our bazel run dir. config.working_dir = d config.model.corpus.local_directory = str(bazelutil.DataPath( 'phd/deeplearning/clgen/tests/data/tiny/corpus.tar.bz2')) clgen.Instance(config)
def test_DataPath_missing_data_dep(): """FileNotFoundError is raised if the file exists is not in target data.""" # The file //lib/labm8/data/test/diabetes.csv exists, but is not a data # dependency of this test target, so is not found. with pytest.raises(FileNotFoundError) as e_info: bazelutil.DataPath('phd/lib/labm8/data/test/diabetes.csv') assert ("No such file or directory: " "'phd/lib/labm8/data/test/diabetes.csv'") in str(e_info)
def EnumerateLanguageInstanceConfigs( language: typing.Dict[str, typing.List[str]] ) -> typing.List[clgen_pb2.Instance]: """Enumerate the options for a language.""" configs = [] for corpus, model, sampler in itertools.product(language['corpuses'], EnumerateModels(), language['samplers']): instance_config = clgen_pb2.Instance() instance_config.working_dir = FLAGS.working_dir instance_config.model.CopyFrom(model) instance_config.model.corpus.CopyFrom( pbutil.FromFile( bazelutil.DataPath( f'phd/experimental/deeplearning/polyglot/corpuses/{corpus}.pbtxt' ), corpus_pb2.Corpus())) instance_config.sampler.CopyFrom( pbutil.FromFile( bazelutil.DataPath( f'phd/experimental/deeplearning/polyglot/samplers/{sampler}.pbtxt' ), sampler_pb2.Sampler())) configs.append(instance_config) return configs
def ExpandConfigPath(path: str, path_prefix: str = None) -> pathlib.Path: """Resolve an absolute path from a config proto string field. This performs shell-style expansion of $VARS, and prefixes the --clgen_local_path_prefix flag value, if it is set. Args: path: The string value as it appears in the proto. path_prefix: An optional string to prepend to the resolved path. Returns: An absolute path. """ # Set a useful variable for expansion. if 'HOME' not in os.environ: os.environ['HOME'] = str(pathlib.Path('~').expanduser()) os.environ['BAZEL_RUNFILES'] = str(bazelutil.DataPath('.')) return pathlib.Path(os.path.expandvars((path_prefix or '') + path)).expanduser().absolute()
def DirContainsProtos(data_path: str, proto_class) -> None: """Assert that contains protos of the given class.""" for path in bazelutil.DataPath(data_path).iterdir(): assert pbutil.ProtoIsReadable( bazelutil.DataPath(data_path) / path, proto_class())
"""Preprocessor passes for the OpenCL programming language.""" import typing from absl import flags from deeplearning.clgen.preprocessors import clang from deeplearning.clgen.preprocessors import normalizer from deeplearning.clgen.preprocessors import public from phd.lib.labm8 import bazelutil FLAGS = flags.FLAGS LIBCLC = bazelutil.DataPath('phd/third_party/libclc/generic/include') OPENCL_H = bazelutil.DataPath('phd/deeplearning/clgen/data/include/opencl.h') SHIMFILE = bazelutil.DataPath( 'phd/deeplearning/clgen/data/include/opencl-shim.h') def GetClangArgs(use_shim: bool) -> typing.List[str]: """Get the arguments to pass to clang for handling OpenCL. Args: use_shim: If true, inject the shim OpenCL header. error_limit: The number of errors to print before arboting Returns: A list of command line arguments to pass to Popen(). """ args = [ '-I' + str(LIBCLC), '-include', str(OPENCL_H), '-target', 'nvptx64-nvidia-nvcl', f'-ferror-limit=1',
from deeplearning.deepsmith import datastore from deeplearning.deepsmith import db from deeplearning.deepsmith import result from deeplearning.deepsmith import testbed from deeplearning.deepsmith import testcase from phd.lib.labm8 import bazelutil from phd.lib.labm8 import fs from phd.lib.labm8 import labtypes from phd.lib.labm8 import pbutil FLAGS = flags.FLAGS flags.DEFINE_string( 'datastore', str( bazelutil.DataPath( 'phd/docs/2018_07_issta/artifact_evaluation/data/datastore.pbtxt') ), 'Path to datastore configuration file.') flags.DEFINE_list('input_directories', [ str( bazelutil.DataPath( 'phd/docs/2018_07_issta/artifact_evaluation/data/our_results')), '/tmp/phd/docs/2018_07_issta/artifact_evaluation/results', ], 'Directories to read results from.') flags.DEFINE_string( 'output_directory', '/tmp/phd/docs/2018_07_issta/artifact_evaluation/difftest_classifications', 'Directory to write classified results to.') def GetResultRuntimeMs(r: result.Result) -> int: for event in r.profiling_events:
from absl import flags from absl import logging from phd.lib.labm8 import bazelutil from phd.lib.labm8 import system from compilers.llvm import llvm FLAGS = flags.FLAGS flags.DEFINE_integer('llvm_link_timeout_seconds', 60, 'The maximum number of seconds to allow process to run.') _LLVM_REPO = 'llvm_linux' if system.is_linux() else 'llvm_mac' # Path to llvm-link binary. LLVM_LINK = bazelutil.DataPath(f'{_LLVM_REPO}/bin/llvm-link') def Exec(args: typing.List[str], timeout_seconds: int = 60) -> subprocess.Popen: """Run LLVM's bitcode linker. Args: args: A list of arguments to pass to binary. timeout_seconds: The number of seconds to allow clang-format to run for. Returns: A Popen instance with stdout and stderr set to strings. """ cmd = ['timeout', '-s9', str(timeout_seconds), str(LLVM_LINK)] + args logging.debug('$ %s', ' '.join(cmd))
from absl import app from absl import flags from absl import logging from phd.lib.labm8 import bazelutil from phd.lib.labm8 import crypto from phd.lib.labm8 import pbutil from deeplearning.deepsmith.generators import clgen from deeplearning.deepsmith.proto import generator_pb2 FLAGS = flags.FLAGS flags.DEFINE_string( 'generator', str( bazelutil.DataPath('phd/docs/2018_07_issta/artifact_evaluation/' 'data/clgen.pbtxt')), 'The path of the generator config proto.') flags.DEFINE_integer('num_testcases', 1024, 'The number of testcases to generate.') flags.DEFINE_string( 'output_directory', '/tmp/phd/docs/2018_07_issta/artifact_evaluation', 'The directory to write generated programs and testcases to.') def GenerateTestcases(generator_config: generator_pb2.ClgenGenerator, output_directory: pathlib.Path, num_testcases: int) -> None: logging.info('Writing output to %s', output_directory) (output_directory / 'generated_kernels').mkdir(parents=True, exist_ok=True) (output_directory / 'generated_testcases').mkdir(parents=True, exist_ok=True)
flags.DEFINE_integer( 'batch_size', 128, 'The number of test cases to generate and execute in a single batch.') flags.DEFINE_string( 'rerun_result', None, 'If --rerun_result points to the path of a Result proto, the result ' 'testcase is executed under the specified --dut, and the new Result is ' 'printed to stdout.') flags.DEFINE_string( 'unpack_result', None, 'If --unpack_result points to the path of a Result proto, the result ' 'proto is unpacked into the testcase OpenCL kernel, and C source code. ' 'Files are written to the directory containing the result.') # The path of the CLSmith cl_launcher C program source. CL_LAUNCHER_SRC = bazelutil.DataPath('CLSmith/src/CLSmith/cl_launcher.c') def RunBatch(generator: base_generator.GeneratorServiceBase, dut_harness: base_harness.HarnessBase, gs_harness: base_harness.HarnessBase, filters: difftests.FiltersBase, batch_size: int) -> typing.List[deepsmith_pb2.Result]: """Run one batch of testing. A batch of testing involves generating a set of testcases, executing them on the device under test, then determining for each whether the result is interesting. The interesting-ness test may involve executing testcases on the gold-standard device and comparing the outputs. Args:
def test_DataPath_read_file(): """Test that DataPath is correct for a known data file.""" with open(bazelutil.DataPath('phd/lib/labm8/data/test/hello_world')) as f: assert f.read() == 'Hello, world!\n'
import re import sys from absl import flags from phd.lib.labm8 import bazelutil from compilers.llvm import clang as clanglib from compilers.llvm import llvm from deeplearning.clgen import errors from deeplearning.clgen.preprocessors import clang from deeplearning.clgen.preprocessors import normalizer from deeplearning.clgen.preprocessors import public FLAGS = flags.FLAGS _UNAME = 'mac' if sys.platform == 'darwin' else 'linux' LIBCXX_HEADERS = bazelutil.DataPath(f'libcxx_{_UNAME}/include/c++/v1') CLANG_HEADERS = bazelutil.DataPath(f'libcxx_{_UNAME}/lib/clang/6.0.0/include') C_COMMENT_RE = re.compile( r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE) # Flags to compile C++ files with. I've replicated the default search path, # but substituted the sandboxed header locations in place of the defaults. # bazel-phd/bazel-out/*-py3-opt/bin/deeplearning/clgen/preprocessors/\ # cxx_test.runfiles/llvm_mac/bin/clang -xc++ -E - -v CLANG_ARGS = [ '-xc++', '-isystem', str(LIBCXX_HEADERS), '-isystem', '/usr/local/include', '-isystem', str(CLANG_HEADERS), '-isystem', '/usr/include', '-Wno-ignored-pragmas', '-ferror-limit=1', '-Wno-implicit-function-declaration',
def test_datafile_read(): """Test reading a data file.""" with open(bazelutil.DataPath('phd/learn/docker/bazel/datafile.txt')) as f: assert f.read() == 'Hello, Docker!\n'
import platform import subprocess import sys import typing from phd.lib.labm8 import bazelutil from phd.lib.labm8 import pbutil from typing import Iterator from gpu.clinfo.proto import clinfo_pb2 from gpu.oclgrind import oclgrind CLINFO = bazelutil.DataPath('phd/gpu/clinfo/clinfo') class OpenCLEnvironment(object): def __init__(self, device: clinfo_pb2.OpenClDevice): self.name = device.name self.platform_name = device.platform_name self.device_name = device.device_name self.driver_version = device.driver_version self.opencl_version = device.opencl_version self.device_type = device.device_type self.platform_id = device.platform_id self.device_id = device.device_id self.opencl_opt = device.opencl_opt def ids(self) -> typing.Tuple[int, int]: """Return platform and device ID numbers. The ID numbers can be used to index into the list of platforms and devices. Note that the stability of these IDs is *not* guaranteed
"""Python entry point to the clang_rewriter binary.""" import os import subprocess import tempfile import typing from absl import flags from absl import logging from deeplearning.clgen import errors from phd.lib.labm8 import bazelutil FLAGS = flags.FLAGS CLGEN_REWRITER = bazelutil.DataPath( 'phd/deeplearning/clgen/preprocessors/clang_rewriter') assert CLGEN_REWRITER.is_file() # On Linux we must preload the LLVM sharded libraries. CLGEN_REWRITER_ENV = os.environ.copy() if bazelutil.DataPath('llvm_linux', must_exist=False).is_dir(): libclang = bazelutil.DataPath('llvm_linux/lib/libclang.so') liblto = bazelutil.DataPath('llvm_linux/lib/libLTO.so') CLGEN_REWRITER_ENV['LD_PRELOAD'] = f'{libclang}:{liblto}' def NormalizeIdentifiers(text: str, suffix: str, cflags: typing.List[str], timeout_seconds: int = 60) -> str: """Normalize identifiers in source code.
from deeplearning.deepsmith.harnesses import cldrive from deeplearning.deepsmith.proto import deepsmith_pb2 from deeplearning.deepsmith.proto import harness_pb2 from gpu.cldrive import env from phd.lib.labm8 import bazelutil from phd.lib.labm8 import crypto from phd.lib.labm8 import fs from phd.lib.labm8 import labtypes from phd.lib.labm8 import pbutil FLAGS = flags.FLAGS flags.DEFINE_list( 'input_directories', [ str(bazelutil.DataPath( 'phd/docs/2018_07_issta/artifact_evaluation/data/testcases')), '/tmp/phd/docs/2018_07_issta/artifact_evaluation/generated_testcases', ], 'Directories to read testcases from.') flags.DEFINE_string( 'output_directory', '/tmp/phd/docs/2018_07_issta/artifact_evaluation/results', 'Directory to write results to.') flags.DEFINE_bool( 'opencl_opt', True, 'If --noopencl_opt is set, disable OpenCL optimizations.') def main(argv): if len(argv) > 1: unknown_args = ', '.join(argv[1:])
import re import sys from collections import OrderedDict from io import StringIO, open from subprocess import PIPE, Popen from typing import List, TextIO import numpy as np from config import getconfig from deeplearning.clgen import errors from phd.lib.labm8 import bazelutil from phd.lib.labm8 import labmath CLGEN_FEATURES = bazelutil.DataPath( 'phd/deeplearning/clgen/native/clgen-features') SHIMFILE = bazelutil.DataPath( 'phd/deeplearning/clgen/data/include/opencl-shim.h') # On Linux we must preload the LLVM sharded libraries. CLGEN_FEATURES_ENV = os.environ.copy() _LIBCLANG_SO = bazelutil.DataPath( f'llvm_linux/lib/libclang.so', must_exist=False) _LIBLTO_SO = bazelutil.DataPath( f'llvm_linux/lib/libLTO.so', must_exist=False) if _LIBCLANG_SO.is_file() and _LIBLTO_SO.is_file(): CLGEN_FEATURES_ENV['LD_PRELOAD'] = f'{_LIBCLANG_SO}:{_LIBLTO_SO}' class FeatureExtractionError(errors.CLgenError): """ Thrown in case feature extraction fails """
from phd.lib.labm8 import bazelutil from phd.lib.labm8 import system from compilers.llvm import llvm FLAGS = flags.FLAGS flags.DEFINE_string('clang_format_file_suffix', '.c', 'The file name suffix to assume for files.') flags.DEFINE_integer('clang_format_timeout_seconds', 60, 'The maximum number of seconds to allow process to run.') _LLVM_REPO = 'llvm_linux' if system.is_linux() else 'llvm_mac' # Path to clang-format binary. CLANG_FORMAT = bazelutil.DataPath(f'{_LLVM_REPO}/bin/clang-format') class ClangFormatException(llvm.LlvmError): """An error from clang-format.""" pass def Exec(text: str, suffix: str, args: typing.List[str], timeout_seconds: int = 60) -> str: """Run clang-format on a source. Args: text: The source code to run through clang-format.
import pathlib import skimage import typing import vizdoom from absl import app from absl import flags from absl import logging from phd.lib.labm8 import bazelutil FLAGS = flags.FLAGS flags.DEFINE_string( 'doom_config', str( bazelutil.DataPath( 'phd/learn/deep_reinforcement_learning_course/data/doom_config.cfg' )), 'Path to Doom config file.') flags.DEFINE_string( 'doom_scenario', str( bazelutil.DataPath( 'phd/learn/deep_reinforcement_learning_course/data/doom_scenario.wad' )), 'Path to Doom scenario file.') def CreateEnvironment( config_path: typing.Optional[pathlib.Path] = None, scenario_path: typing.Optional[pathlib.Path] = None ) -> typing.Tuple[None, typing.List[typing.List[int]]]: """Create the Doom game environment.
import sys import tempfile import typing from absl import flags from absl import logging from phd.lib.labm8 import bazelutil from compilers.llvm import clang_format from compilers.llvm import llvm from deeplearning.clgen import errors FLAGS = flags.FLAGS _LLVM_REPO = 'llvm_mac' if sys.platform == 'darwin' else 'llvm_linux' # Path to clang binary. CLANG = bazelutil.DataPath(f'{_LLVM_REPO}/bin/clang') # The marker used to mark stdin from clang pre-processor output. CLANG_STDIN_MARKER = re.compile(r'# \d+ "<stdin>" 2') # Options to pass to clang-format. # See: http://clang.llvm.org/docs/ClangFormatStyleOptions.html CLANG_FORMAT_CONFIG = { 'BasedOnStyle': 'Google', 'ColumnLimit': 5000, 'IndentWidth': 2, 'AllowShortBlocksOnASingleLine': False, 'AllowShortCaseLabelsOnASingleLine': False, 'AllowShortFunctionsOnASingleLine': False, 'AllowShortLoopsOnASingleLine': False, 'AllowShortIfStatementsOnASingleLine': False, 'DerivePointerAlignment': False, 'PointerAlignment': 'Left',
'stdlib.h', 'stdnoreturn.h', 'string.h', 'tgmath.h', 'threads.h', 'time.h', 'uchar.h', 'wchar.h', 'wctype.h', } # The set of headers in the C++ standard library. _UNAME = 'mac' if sys.platform == 'darwin' else 'linux' CXX_HEADERS = set( public.GetAllFilesRelativePaths( bazelutil.DataPath(f'libcxx_{_UNAME}/include/c++/v1'), follow_symlinks=True) + public.GetAllFilesRelativePaths( bazelutil.DataPath(f'libcxx_{_UNAME}/lib/clang/6.0.0/include'), follow_symlinks=True)) @public.dataset_preprocessor def CxxHeaders(import_root: pathlib.Path, file_relpath: str, text: str, all_file_relpaths: typing.List[str]) -> str: """Inline C++ includes. Searches for occurrences of '#include <$file>' and attempts to resolve $file to a path within import_root. If successful, the include directive is replaced. Args:
import subprocess import sys import pytest from absl import app from absl import flags from absl import logging import deeplearning.clgen from deeplearning.clgen import errors from deeplearning.clgen.preprocessors import opencl from phd.lib.labm8 import bazelutil FLAGS = flags.FLAGS SHIMFILE = bazelutil.DataPath( 'phd/deeplearning/clgen/data/include/opencl-shim.h') class MockProcess(): """Mock class for subprocess.Popen() return.""" def __init__(self, returncode): self.returncode = returncode def communicate(self, *args): return '', '' # GetClangArgs() tests. def test_GetClangArgs_no_shim():
from absl import flags from absl import logging from phd.lib.labm8 import bazelutil from phd.lib.labm8 import system from compilers.llvm import llvm FLAGS = flags.FLAGS flags.DEFINE_integer('opt_timeout_seconds', 60, 'The maximum number of seconds to allow process to run.') _LLVM_REPO = 'llvm_linux' if system.is_linux() else 'llvm_mac' # Path to opt binary. OPT = bazelutil.DataPath(f'{_LLVM_REPO}/bin/opt') class OptException(llvm.LlvmError): """An error from opt.""" pass def Exec(args: typing.List[str], timeout_seconds: int = 60) -> subprocess.Popen: """Run LLVM's optimizer. Args: args: A list of arguments to pass to binary. timeout_seconds: The number of seconds to allow clang-format to run for.
import pathlib import subprocess import typing from absl import flags from absl import logging from phd.lib.labm8 import bazelutil from phd.lib.labm8 import pbutil from datasets.github.scrape_repos.preprocessors import public from datasets.github.scrape_repos.proto import scrape_repos_pb2 FLAGS = flags.FLAGS # The path to the methods extractor binary. JAVA_METHODS_EXTRACTOR = bazelutil.DataPath( 'phd/datasets/github/scrape_repos/preprocessors/JavaMethodsExtractor') # The environments to run the method extractor under. STATIC_ONLY_ENV = os.environ.copy() STATIC_ONLY_ENV['JAVA_METHOD_EXTRACTOR_STATIC_ONLY'] = '1' def ExtractJavaMethods(text: str, static_only: bool = True) -> typing.List[str]: """Extract Java methods from a file. Args: text: The text of the target file. static_only: If true, only static methods are returned. Returns: A list of method implementations.
""" import subprocess import sys import typing from absl import app from absl import flags from gpu.clinfo.proto import clinfo_pb2 from phd.lib.labm8 import bazelutil from phd.lib.labm8 import system FLAGS = flags.FLAGS _OCLGRIND_PKG = 'oclgrind_linux' if system.is_linux() else 'oclgrind_mac' # The path to the oclgrind binary. OCLGRIND_PATH = bazelutil.DataPath(f'{_OCLGRIND_PKG}/bin/oclgrind') # The clinfo description of the local Oclgrind binary. CLINFO_DESCRIPTION = clinfo_pb2.OpenClDevice( name='Emulator|Oclgrind|Oclgrind_Simulator|Oclgrind_18.3|1.2', platform_name='Oclgrind', device_name='Oclgrind Simulator', driver_version='Oclgrind 18.3', opencl_version='1.2', device_type='Emulator', platform_id=0, device_id=0, ) def Exec(argv: typing.List[str], env: typing.Dict[str, str] = None) -> subprocess.Popen:
Usage: bazel run //compilers/clsmith [-- -- <args>] """ import subprocess import sys from absl import app from absl import flags from phd.lib.labm8 import bazelutil from phd.lib.labm8 import fs FLAGS = flags.FLAGS CLSMITH = bazelutil.DataPath('CLSmith/CLSmith') class CLSmithError(EnvironmentError): """Error thrown in case CLSmith fails.""" def __init__(self, msg: str, returncode: int): self.msg = msg self.returncode = returncode def __repr__(self): return str(self.msg) def Exec(*opts) -> str: """Generate and return a CLSmith program.
from deeplearning.deepsmith.proto import harness_pb2_grpc from deeplearning.deepsmith.proto import service_pb2 from gpu.cldrive import cgen from gpu.cldrive import data from gpu.cldrive import driver from gpu.cldrive import env from phd.lib.labm8 import bazelutil from phd.lib.labm8 import fs from phd.lib.labm8 import labdate from phd.lib.labm8 import system FLAGS = flags.FLAGS _UNAME = 'linux' if system.is_linux() else 'mac' # Path to clang binary. CLANG_PATH = bazelutil.DataPath(f'llvm_{_UNAME}/bin/clang') # Flags for compiling with libcxx. LIBCXX_LIB_DIR = bazelutil.DataPath(f'llvm_{_UNAME}/lib') # Path to OpenCL headers. OPENCL_HEADERS_DIR = bazelutil.DataPath('opencl_120_headers') if system.is_linux(): LIBOPENCL_DIR = bazelutil.DataPath('libopencl') class DriverCompilationError(OSError): """Exception raised in case driver compilation fails.""" pass class CldriveHarness(harness.HarnessBase, harness_pb2_grpc.HarnessServiceServicer):
def test_DataPath_directory(): """Test that DataPath returns the path to a directory.""" assert str(bazelutil.DataPath('phd/lib/labm8/data/test')).endswith( 'phd/lib/labm8/data/test')