def test_DataPath_different_working_dir():
"""Test that DataPath is not affected by current working dir."""
p = bazelutil.DataPath('phd/lib/labm8/data/test/hello_world')
with fs.chdir('/tmp'):
assert bazelutil.DataPath('phd/lib/labm8/data/test/hello_world') == p
with tempfile.TemporaryDirectory() as d:
with fs.chdir(d):
assert bazelutil.DataPath(
'phd/lib/labm8/data/test/hello_world') == p
def test_DataPath_path_not_found():
"""Test that FileNotFoundError is raised if the file is not found."""
with pytest.raises(FileNotFoundError) as e_info:
bazelutil.DataPath('')
assert f"No such file or directory: ''" in str(e_info)
with pytest.raises(FileNotFoundError) as e_info:
bazelutil.DataPath('/not/a/real/path')
assert f"No such file or directory: '/not/a/real/path'" in str(e_info)
def test_config_is_valid():
"""Test that config proto is valid."""
with tempfile.TemporaryDirectory() as d:
config = pbutil.FromFile(
bazelutil.DataPath(
'phd/deeplearning/clgen/tests/data/tiny/config.pbtxt'),
clgen_pb2.Instance())
# Change the working directory and corpus path to our bazel run dir.
config.working_dir = d
config.model.corpus.local_directory = str(bazelutil.DataPath(
'phd/deeplearning/clgen/tests/data/tiny/corpus.tar.bz2'))
clgen.Instance(config)
def test_DataPath_missing_data_dep():
"""FileNotFoundError is raised if the file exists is not in target data."""
# The file //lib/labm8/data/test/diabetes.csv exists, but is not a data
# dependency of this test target, so is not found.
with pytest.raises(FileNotFoundError) as e_info:
bazelutil.DataPath('phd/lib/labm8/data/test/diabetes.csv')
assert ("No such file or directory: "
"'phd/lib/labm8/data/test/diabetes.csv'") in str(e_info)
def EnumerateLanguageInstanceConfigs(
language: typing.Dict[str, typing.List[str]]
) -> typing.List[clgen_pb2.Instance]:
"""Enumerate the options for a language."""
configs = []
for corpus, model, sampler in itertools.product(language['corpuses'],
EnumerateModels(),
language['samplers']):
instance_config = clgen_pb2.Instance()
instance_config.working_dir = FLAGS.working_dir
instance_config.model.CopyFrom(model)
instance_config.model.corpus.CopyFrom(
pbutil.FromFile(
bazelutil.DataPath(
f'phd/experimental/deeplearning/polyglot/corpuses/{corpus}.pbtxt'
), corpus_pb2.Corpus()))
instance_config.sampler.CopyFrom(
pbutil.FromFile(
bazelutil.DataPath(
f'phd/experimental/deeplearning/polyglot/samplers/{sampler}.pbtxt'
), sampler_pb2.Sampler()))
configs.append(instance_config)
return configs
def ExpandConfigPath(path: str, path_prefix: str = None) -> pathlib.Path:
"""Resolve an absolute path from a config proto string field.
This performs shell-style expansion of $VARS, and prefixes the
--clgen_local_path_prefix flag value, if it is set.
Args:
path: The string value as it appears in the proto.
path_prefix: An optional string to prepend to the resolved path.
Returns:
An absolute path.
"""
# Set a useful variable for expansion.
if 'HOME' not in os.environ:
os.environ['HOME'] = str(pathlib.Path('~').expanduser())
os.environ['BAZEL_RUNFILES'] = str(bazelutil.DataPath('.'))
return pathlib.Path(os.path.expandvars((path_prefix or '') +
path)).expanduser().absolute()
def DirContainsProtos(data_path: str, proto_class) -> None:
"""Assert that contains protos of the given class."""
for path in bazelutil.DataPath(data_path).iterdir():
assert pbutil.ProtoIsReadable(
bazelutil.DataPath(data_path) / path, proto_class())
"""Preprocessor passes for the OpenCL programming language."""
import typing
from absl import flags
from deeplearning.clgen.preprocessors import clang
from deeplearning.clgen.preprocessors import normalizer
from deeplearning.clgen.preprocessors import public
from phd.lib.labm8 import bazelutil
FLAGS = flags.FLAGS
LIBCLC = bazelutil.DataPath('phd/third_party/libclc/generic/include')
OPENCL_H = bazelutil.DataPath('phd/deeplearning/clgen/data/include/opencl.h')
SHIMFILE = bazelutil.DataPath(
'phd/deeplearning/clgen/data/include/opencl-shim.h')
def GetClangArgs(use_shim: bool) -> typing.List[str]:
"""Get the arguments to pass to clang for handling OpenCL.
Args:
use_shim: If true, inject the shim OpenCL header.
error_limit: The number of errors to print before arboting
Returns:
A list of command line arguments to pass to Popen().
"""
args = [
'-I' + str(LIBCLC), '-include',
str(OPENCL_H), '-target', 'nvptx64-nvidia-nvcl', f'-ferror-limit=1',
from deeplearning.deepsmith import datastore
from deeplearning.deepsmith import db
from deeplearning.deepsmith import result
from deeplearning.deepsmith import testbed
from deeplearning.deepsmith import testcase
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import fs
from phd.lib.labm8 import labtypes
from phd.lib.labm8 import pbutil
FLAGS = flags.FLAGS
flags.DEFINE_string(
'datastore',
str(
bazelutil.DataPath(
'phd/docs/2018_07_issta/artifact_evaluation/data/datastore.pbtxt')
), 'Path to datastore configuration file.')
flags.DEFINE_list('input_directories', [
str(
bazelutil.DataPath(
'phd/docs/2018_07_issta/artifact_evaluation/data/our_results')),
'/tmp/phd/docs/2018_07_issta/artifact_evaluation/results',
], 'Directories to read results from.')
flags.DEFINE_string(
'output_directory',
'/tmp/phd/docs/2018_07_issta/artifact_evaluation/difftest_classifications',
'Directory to write classified results to.')
def GetResultRuntimeMs(r: result.Result) -> int:
for event in r.profiling_events:
from absl import flags
from absl import logging
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import system
from compilers.llvm import llvm
FLAGS = flags.FLAGS
flags.DEFINE_integer('llvm_link_timeout_seconds', 60,
'The maximum number of seconds to allow process to run.')
_LLVM_REPO = 'llvm_linux' if system.is_linux() else 'llvm_mac'
# Path to llvm-link binary.
LLVM_LINK = bazelutil.DataPath(f'{_LLVM_REPO}/bin/llvm-link')
def Exec(args: typing.List[str],
timeout_seconds: int = 60) -> subprocess.Popen:
"""Run LLVM's bitcode linker.
Args:
args: A list of arguments to pass to binary.
timeout_seconds: The number of seconds to allow clang-format to run for.
Returns:
A Popen instance with stdout and stderr set to strings.
"""
cmd = ['timeout', '-s9', str(timeout_seconds), str(LLVM_LINK)] + args
logging.debug('$ %s', ' '.join(cmd))
from absl import app
from absl import flags
from absl import logging
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import crypto
from phd.lib.labm8 import pbutil
from deeplearning.deepsmith.generators import clgen
from deeplearning.deepsmith.proto import generator_pb2
FLAGS = flags.FLAGS
flags.DEFINE_string(
'generator',
str(
bazelutil.DataPath('phd/docs/2018_07_issta/artifact_evaluation/'
'data/clgen.pbtxt')),
'The path of the generator config proto.')
flags.DEFINE_integer('num_testcases', 1024,
'The number of testcases to generate.')
flags.DEFINE_string(
'output_directory', '/tmp/phd/docs/2018_07_issta/artifact_evaluation',
'The directory to write generated programs and testcases to.')
def GenerateTestcases(generator_config: generator_pb2.ClgenGenerator,
output_directory: pathlib.Path,
num_testcases: int) -> None:
logging.info('Writing output to %s', output_directory)
(output_directory / 'generated_kernels').mkdir(parents=True, exist_ok=True)
(output_directory / 'generated_testcases').mkdir(parents=True,
exist_ok=True)
flags.DEFINE_integer(
'batch_size', 128,
'The number of test cases to generate and execute in a single batch.')
flags.DEFINE_string(
'rerun_result', None,
'If --rerun_result points to the path of a Result proto, the result '
'testcase is executed under the specified --dut, and the new Result is '
'printed to stdout.')
flags.DEFINE_string(
'unpack_result', None,
'If --unpack_result points to the path of a Result proto, the result '
'proto is unpacked into the testcase OpenCL kernel, and C source code. '
'Files are written to the directory containing the result.')
# The path of the CLSmith cl_launcher C program source.
CL_LAUNCHER_SRC = bazelutil.DataPath('CLSmith/src/CLSmith/cl_launcher.c')
def RunBatch(generator: base_generator.GeneratorServiceBase,
dut_harness: base_harness.HarnessBase,
gs_harness: base_harness.HarnessBase,
filters: difftests.FiltersBase,
batch_size: int) -> typing.List[deepsmith_pb2.Result]:
"""Run one batch of testing.
A batch of testing involves generating a set of testcases, executing them on
the device under test, then determining for each whether the result is
interesting. The interesting-ness test may involve executing testcases on the
gold-standard device and comparing the outputs.
Args:
def test_DataPath_read_file():
"""Test that DataPath is correct for a known data file."""
with open(bazelutil.DataPath('phd/lib/labm8/data/test/hello_world')) as f:
assert f.read() == 'Hello, world!\n'
import re
import sys
from absl import flags
from phd.lib.labm8 import bazelutil
from compilers.llvm import clang as clanglib
from compilers.llvm import llvm
from deeplearning.clgen import errors
from deeplearning.clgen.preprocessors import clang
from deeplearning.clgen.preprocessors import normalizer
from deeplearning.clgen.preprocessors import public
FLAGS = flags.FLAGS
_UNAME = 'mac' if sys.platform == 'darwin' else 'linux'
LIBCXX_HEADERS = bazelutil.DataPath(f'libcxx_{_UNAME}/include/c++/v1')
CLANG_HEADERS = bazelutil.DataPath(f'libcxx_{_UNAME}/lib/clang/6.0.0/include')
C_COMMENT_RE = re.compile(
r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
re.DOTALL | re.MULTILINE)
# Flags to compile C++ files with. I've replicated the default search path,
# but substituted the sandboxed header locations in place of the defaults.
# bazel-phd/bazel-out/*-py3-opt/bin/deeplearning/clgen/preprocessors/\
# cxx_test.runfiles/llvm_mac/bin/clang -xc++ -E - -v
CLANG_ARGS = [
'-xc++', '-isystem',
str(LIBCXX_HEADERS), '-isystem', '/usr/local/include', '-isystem',
str(CLANG_HEADERS), '-isystem', '/usr/include', '-Wno-ignored-pragmas',
'-ferror-limit=1', '-Wno-implicit-function-declaration',
def test_datafile_read():
"""Test reading a data file."""
with open(bazelutil.DataPath('phd/learn/docker/bazel/datafile.txt')) as f:
assert f.read() == 'Hello, Docker!\n'
import platform
import subprocess
import sys
import typing
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import pbutil
from typing import Iterator
from gpu.clinfo.proto import clinfo_pb2
from gpu.oclgrind import oclgrind
CLINFO = bazelutil.DataPath('phd/gpu/clinfo/clinfo')
class OpenCLEnvironment(object):
def __init__(self, device: clinfo_pb2.OpenClDevice):
self.name = device.name
self.platform_name = device.platform_name
self.device_name = device.device_name
self.driver_version = device.driver_version
self.opencl_version = device.opencl_version
self.device_type = device.device_type
self.platform_id = device.platform_id
self.device_id = device.device_id
self.opencl_opt = device.opencl_opt
def ids(self) -> typing.Tuple[int, int]:
"""Return platform and device ID numbers.
The ID numbers can be used to index into the list of platforms and
devices. Note that the stability of these IDs is *not* guaranteed
"""Python entry point to the clang_rewriter binary."""
import os
import subprocess
import tempfile
import typing
from absl import flags
from absl import logging
from deeplearning.clgen import errors
from phd.lib.labm8 import bazelutil
FLAGS = flags.FLAGS
CLGEN_REWRITER = bazelutil.DataPath(
'phd/deeplearning/clgen/preprocessors/clang_rewriter')
assert CLGEN_REWRITER.is_file()
# On Linux we must preload the LLVM sharded libraries.
CLGEN_REWRITER_ENV = os.environ.copy()
if bazelutil.DataPath('llvm_linux', must_exist=False).is_dir():
libclang = bazelutil.DataPath('llvm_linux/lib/libclang.so')
liblto = bazelutil.DataPath('llvm_linux/lib/libLTO.so')
CLGEN_REWRITER_ENV['LD_PRELOAD'] = f'{libclang}:{liblto}'
def NormalizeIdentifiers(text: str,
suffix: str,
cflags: typing.List[str],
timeout_seconds: int = 60) -> str:
"""Normalize identifiers in source code.
from deeplearning.deepsmith.harnesses import cldrive
from deeplearning.deepsmith.proto import deepsmith_pb2
from deeplearning.deepsmith.proto import harness_pb2
from gpu.cldrive import env
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import crypto
from phd.lib.labm8 import fs
from phd.lib.labm8 import labtypes
from phd.lib.labm8 import pbutil
FLAGS = flags.FLAGS
flags.DEFINE_list(
'input_directories', [
str(bazelutil.DataPath(
'phd/docs/2018_07_issta/artifact_evaluation/data/testcases')),
'/tmp/phd/docs/2018_07_issta/artifact_evaluation/generated_testcases',
],
'Directories to read testcases from.')
flags.DEFINE_string(
'output_directory',
'/tmp/phd/docs/2018_07_issta/artifact_evaluation/results',
'Directory to write results to.')
flags.DEFINE_bool(
'opencl_opt', True,
'If --noopencl_opt is set, disable OpenCL optimizations.')
def main(argv):
if len(argv) > 1:
unknown_args = ', '.join(argv[1:])
import re
import sys
from collections import OrderedDict
from io import StringIO, open
from subprocess import PIPE, Popen
from typing import List, TextIO
import numpy as np
from config import getconfig
from deeplearning.clgen import errors
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import labmath
CLGEN_FEATURES = bazelutil.DataPath(
'phd/deeplearning/clgen/native/clgen-features')
SHIMFILE = bazelutil.DataPath(
'phd/deeplearning/clgen/data/include/opencl-shim.h')
# On Linux we must preload the LLVM sharded libraries.
CLGEN_FEATURES_ENV = os.environ.copy()
_LIBCLANG_SO = bazelutil.DataPath(
f'llvm_linux/lib/libclang.so', must_exist=False)
_LIBLTO_SO = bazelutil.DataPath(
f'llvm_linux/lib/libLTO.so', must_exist=False)
if _LIBCLANG_SO.is_file() and _LIBLTO_SO.is_file():
CLGEN_FEATURES_ENV['LD_PRELOAD'] = f'{_LIBCLANG_SO}:{_LIBLTO_SO}'
class FeatureExtractionError(errors.CLgenError):
""" Thrown in case feature extraction fails """
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import system
from compilers.llvm import llvm
FLAGS = flags.FLAGS
flags.DEFINE_string('clang_format_file_suffix', '.c',
'The file name suffix to assume for files.')
flags.DEFINE_integer('clang_format_timeout_seconds', 60,
'The maximum number of seconds to allow process to run.')
_LLVM_REPO = 'llvm_linux' if system.is_linux() else 'llvm_mac'
# Path to clang-format binary.
CLANG_FORMAT = bazelutil.DataPath(f'{_LLVM_REPO}/bin/clang-format')
class ClangFormatException(llvm.LlvmError):
"""An error from clang-format."""
pass
def Exec(text: str,
suffix: str,
args: typing.List[str],
timeout_seconds: int = 60) -> str:
"""Run clang-format on a source.
Args:
text: The source code to run through clang-format.
import pathlib
import skimage
import typing
import vizdoom
from absl import app
from absl import flags
from absl import logging
from phd.lib.labm8 import bazelutil
FLAGS = flags.FLAGS
flags.DEFINE_string(
'doom_config',
str(
bazelutil.DataPath(
'phd/learn/deep_reinforcement_learning_course/data/doom_config.cfg'
)), 'Path to Doom config file.')
flags.DEFINE_string(
'doom_scenario',
str(
bazelutil.DataPath(
'phd/learn/deep_reinforcement_learning_course/data/doom_scenario.wad'
)), 'Path to Doom scenario file.')
def CreateEnvironment(
config_path: typing.Optional[pathlib.Path] = None,
scenario_path: typing.Optional[pathlib.Path] = None
) -> typing.Tuple[None, typing.List[typing.List[int]]]:
"""Create the Doom game environment.
import sys
import tempfile
import typing
from absl import flags
from absl import logging
from phd.lib.labm8 import bazelutil
from compilers.llvm import clang_format
from compilers.llvm import llvm
from deeplearning.clgen import errors
FLAGS = flags.FLAGS
_LLVM_REPO = 'llvm_mac' if sys.platform == 'darwin' else 'llvm_linux'
# Path to clang binary.
CLANG = bazelutil.DataPath(f'{_LLVM_REPO}/bin/clang')
# The marker used to mark stdin from clang pre-processor output.
CLANG_STDIN_MARKER = re.compile(r'# \d+ "<stdin>" 2')
# Options to pass to clang-format.
# See: http://clang.llvm.org/docs/ClangFormatStyleOptions.html
CLANG_FORMAT_CONFIG = {
'BasedOnStyle': 'Google',
'ColumnLimit': 5000,
'IndentWidth': 2,
'AllowShortBlocksOnASingleLine': False,
'AllowShortCaseLabelsOnASingleLine': False,
'AllowShortFunctionsOnASingleLine': False,
'AllowShortLoopsOnASingleLine': False,
'AllowShortIfStatementsOnASingleLine': False,
'DerivePointerAlignment': False,
'PointerAlignment': 'Left',
'stdlib.h',
'stdnoreturn.h',
'string.h',
'tgmath.h',
'threads.h',
'time.h',
'uchar.h',
'wchar.h',
'wctype.h',
}
# The set of headers in the C++ standard library.
_UNAME = 'mac' if sys.platform == 'darwin' else 'linux'
CXX_HEADERS = set(
public.GetAllFilesRelativePaths(
bazelutil.DataPath(f'libcxx_{_UNAME}/include/c++/v1'),
follow_symlinks=True) + public.GetAllFilesRelativePaths(
bazelutil.DataPath(f'libcxx_{_UNAME}/lib/clang/6.0.0/include'),
follow_symlinks=True))
@public.dataset_preprocessor
def CxxHeaders(import_root: pathlib.Path, file_relpath: str, text: str,
all_file_relpaths: typing.List[str]) -> str:
"""Inline C++ includes.
Searches for occurrences of '#include <$file>' and attempts to resolve $file
to a path within import_root. If successful, the include directive is
replaced.
Args:
import subprocess
import sys
import pytest
from absl import app
from absl import flags
from absl import logging
import deeplearning.clgen
from deeplearning.clgen import errors
from deeplearning.clgen.preprocessors import opencl
from phd.lib.labm8 import bazelutil
FLAGS = flags.FLAGS
SHIMFILE = bazelutil.DataPath(
'phd/deeplearning/clgen/data/include/opencl-shim.h')
class MockProcess():
"""Mock class for subprocess.Popen() return."""
def __init__(self, returncode):
self.returncode = returncode
def communicate(self, *args):
return '', ''
# GetClangArgs() tests.
def test_GetClangArgs_no_shim():
from absl import flags
from absl import logging
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import system
from compilers.llvm import llvm
FLAGS = flags.FLAGS
flags.DEFINE_integer('opt_timeout_seconds', 60,
'The maximum number of seconds to allow process to run.')
_LLVM_REPO = 'llvm_linux' if system.is_linux() else 'llvm_mac'
# Path to opt binary.
OPT = bazelutil.DataPath(f'{_LLVM_REPO}/bin/opt')
class OptException(llvm.LlvmError):
"""An error from opt."""
pass
def Exec(args: typing.List[str],
timeout_seconds: int = 60) -> subprocess.Popen:
"""Run LLVM's optimizer.
Args:
args: A list of arguments to pass to binary.
timeout_seconds: The number of seconds to allow clang-format to run for.
import pathlib
import subprocess
import typing
from absl import flags
from absl import logging
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import pbutil
from datasets.github.scrape_repos.preprocessors import public
from datasets.github.scrape_repos.proto import scrape_repos_pb2
FLAGS = flags.FLAGS
# The path to the methods extractor binary.
JAVA_METHODS_EXTRACTOR = bazelutil.DataPath(
'phd/datasets/github/scrape_repos/preprocessors/JavaMethodsExtractor')
# The environments to run the method extractor under.
STATIC_ONLY_ENV = os.environ.copy()
STATIC_ONLY_ENV['JAVA_METHOD_EXTRACTOR_STATIC_ONLY'] = '1'
def ExtractJavaMethods(text: str, static_only: bool = True) -> typing.List[str]:
"""Extract Java methods from a file.
Args:
text: The text of the target file.
static_only: If true, only static methods are returned.
Returns:
A list of method implementations.
"""
import subprocess
import sys
import typing
from absl import app
from absl import flags
from gpu.clinfo.proto import clinfo_pb2
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import system
FLAGS = flags.FLAGS
_OCLGRIND_PKG = 'oclgrind_linux' if system.is_linux() else 'oclgrind_mac'
# The path to the oclgrind binary.
OCLGRIND_PATH = bazelutil.DataPath(f'{_OCLGRIND_PKG}/bin/oclgrind')
# The clinfo description of the local Oclgrind binary.
CLINFO_DESCRIPTION = clinfo_pb2.OpenClDevice(
name='Emulator|Oclgrind|Oclgrind_Simulator|Oclgrind_18.3|1.2',
platform_name='Oclgrind',
device_name='Oclgrind Simulator',
driver_version='Oclgrind 18.3',
opencl_version='1.2',
device_type='Emulator',
platform_id=0,
device_id=0,
)
def Exec(argv: typing.List[str],
env: typing.Dict[str, str] = None) -> subprocess.Popen:
Usage:
bazel run //compilers/clsmith [-- -- <args>]
"""
import subprocess
import sys
from absl import app
from absl import flags
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import fs
FLAGS = flags.FLAGS
CLSMITH = bazelutil.DataPath('CLSmith/CLSmith')
class CLSmithError(EnvironmentError):
"""Error thrown in case CLSmith fails."""
def __init__(self, msg: str, returncode: int):
self.msg = msg
self.returncode = returncode
def __repr__(self):
return str(self.msg)
def Exec(*opts) -> str:
"""Generate and return a CLSmith program.
from deeplearning.deepsmith.proto import harness_pb2_grpc
from deeplearning.deepsmith.proto import service_pb2
from gpu.cldrive import cgen
from gpu.cldrive import data
from gpu.cldrive import driver
from gpu.cldrive import env
from phd.lib.labm8 import bazelutil
from phd.lib.labm8 import fs
from phd.lib.labm8 import labdate
from phd.lib.labm8 import system
FLAGS = flags.FLAGS
_UNAME = 'linux' if system.is_linux() else 'mac'
# Path to clang binary.
CLANG_PATH = bazelutil.DataPath(f'llvm_{_UNAME}/bin/clang')
# Flags for compiling with libcxx.
LIBCXX_LIB_DIR = bazelutil.DataPath(f'llvm_{_UNAME}/lib')
# Path to OpenCL headers.
OPENCL_HEADERS_DIR = bazelutil.DataPath('opencl_120_headers')
if system.is_linux():
LIBOPENCL_DIR = bazelutil.DataPath('libopencl')
class DriverCompilationError(OSError):
"""Exception raised in case driver compilation fails."""
pass
class CldriveHarness(harness.HarnessBase,
harness_pb2_grpc.HarnessServiceServicer):
def test_DataPath_directory():
"""Test that DataPath returns the path to a directory."""
assert str(bazelutil.DataPath('phd/lib/labm8/data/test')).endswith(
'phd/lib/labm8/data/test')
