import typing
from deeplearning.clgen import errors
from deeplearning.clgen import sample_observers as sample_observers_lib
from deeplearning.clgen import samplers
from deeplearning.clgen.models import models
from deeplearning.clgen.models import pretrained
from deeplearning.clgen.proto import clgen_pb2
from deeplearning.clgen.proto import model_pb2
from labm8 import app
from labm8 import pbutil
from labm8 import prof
FLAGS = app.FLAGS
app.DEFINE_string('config', '/clgen/config.pbtxt',
'Path to a clgen.Instance proto file.')
app.DEFINE_integer(
'min_samples', 0,
'The minimum number of samples to make. If <= 0, sampling continues '
'indefinitely and never terminates.')
app.DEFINE_boolean('print_samples', True,
'If set, print the generated samples.')
app.DEFINE_boolean('cache_samples', False,
'If set, cache the generated sample protobufs.')
app.DEFINE_string('sample_text_dir', None,
'A directory to write plain text samples to.')
app.DEFINE_string('stop_after', None,
'Stop CLgen early. Valid options are: "corpus", or "train".')
app.DEFINE_string(
'print_cache_path', None,
'Print the directory of a cache and exit. Valid options are: "corpus", '
from tools.source_tree import phd_workspace
FLAGS = app.FLAGS
app.DEFINE_list('targets', [], 'The bazel target(s) to export.')
app.DEFINE_list('excluded_targets', [],
'A list of bazel targets to exclude from export.')
app.DEFINE_list(
'extra_files', [], 'A list of additional files to export. Each element in '
'the list is a relative path to export. E.g. `bar/baz.txt`.')
app.DEFINE_list(
'move_file_mapping', [],
'Each element in the list is a mapping of relative paths in the form '
'<src>:<dst>. E.g. `foo.py:bar/baz.txt` will move file `foo.py` to '
'destination `bar/baz.txt`.')
app.DEFINE_string('github_repo', None, 'Name of a GitHub repo to export to.')
app.DEFINE_boolean('github_create_repo', False,
'Whether to create the repo if it does not exist.')
app.DEFINE_boolean('github_repo_create_private', True,
'Whether to create new GitHub repos as private.')
app.DEFINE_boolean('export_source_tree_print_files', False,
'Print the files that will be exported and terminate.')
app.DEFINE_boolean(
'ignore_last_export', False,
'If true, run through the entire git history. Otherwise, '
'continue from the last commit exported. Use this flag if '
'the set of exported files changes.')
def GetOrCreateRepoOrDie(github: github_lib.Github,
repo_name: str) -> github_lib.Repository:
"""This file is the entry point for creating chunks."""
import pathlib
from labm8 import app
from util.photolib.shutterbug import shutterbug
FLAGS = app.FLAGS
app.DEFINE_string(
'src_dir', None,
'The directory to create chunks from. All files in this directory are '
'packed into chunks.')
app.DEFINE_string(
'chunks_dir', None,
'The root directory of the chunks. Each chunk is a directory containing '
'files and a manifest.')
app.DEFINE_integer(
'size_mb', 4695,
'The smaximum size of each chunk in megabytes. This excludes the MANIFEST '
'and README files which are generated.')
app.DEFINE_string('chunk_prefix', 'chunk_',
'The name to prepend to generated chunks.')
app.DEFINE_boolean(
'random_ordering', True,
'Whether to randomize the ordering of files across and within chunks. If '
'--norandom_ordering is used, the files are arranged in chunks in the order '
'in which they are found in --src_dir. This is not recommended, as it means '
'the loss of a chunk causes a loss in a contiguous block of files.')
app.DEFINE_integer(
'random_ordering_seed', 0,
'The number used to seed the random number generator. Not used if '
import pathlib
import typing
from deeplearning.clgen import clgen
from deeplearning.clgen import sample_observers
from deeplearning.clgen.proto import clgen_pb2
from deeplearning.clgen.proto import corpus_pb2
from deeplearning.clgen.proto import model_pb2
from deeplearning.clgen.proto import sampler_pb2
from labm8 import app
from labm8 import pbutil
FLAGS = app.FLAGS
app.DEFINE_string(
'clgen_instance', None,
'Path to a clgen.Instance proto file containing a full '
'CLgen configuration.')
app.DEFINE_string('clgen_working_dir',
str(pathlib.Path('~/.cache/clgen').expanduser()),
'The directory for CLgen working files.')
# Corpus options.
app.DEFINE_string('clgen_corpus_dir',
"/mnt/cc/data/datasets/github/corpuses/opencl",
"Directory where the corpus is stored.")
app.DEFINE_boolean('clgen_multichar_tokenizer', False,
'If true, use multichar OpenCL token.')
# Model options.
app.DEFINE_integer('clgen_layer_size', 512, 'Size of LSTM model layers.')
bazel run //compilers/llvm:clang_format \
[-- <script_args> [-- <clang_format_args>]]
"""
import fileinput
import subprocess
import sys
import typing
from compilers.llvm import llvm
from labm8 import app
from labm8 import bazelutil
from labm8 import system
FLAGS = app.FLAGS
app.DEFINE_string('clang_format_file_suffix', '.c',
'The file name suffix to assume for files.')
app.DEFINE_integer('clang_format_timeout_seconds', 60,
'The maximum number of seconds to allow process to run.')
_LLVM_REPO = 'llvm_linux' if system.is_linux() else 'llvm_mac'
# Path to clang-format binary.
CLANG_FORMAT = bazelutil.DataPath(f'{_LLVM_REPO}/bin/clang-format')
class ClangFormatException(llvm.LlvmError):
"""An error from clang-format."""
pass
def Exec(text: str,
from labm8 import app
from labm8 import bazelutil
from labm8 import crypto
from labm8 import hashcache
from labm8 import humanize
from labm8 import lockfile
from labm8 import pbutil
from labm8 import prof
from labm8 import sqlutil
FLAGS = app.FLAGS
app.DEFINE_string(
'clgen_local_path_prefix', None,
'An optional prefix to use when resolving the path to a local directory '
'or archive. For example, given a corpus which is configured for a '
'local_directory with value "foo/bar" and a --clgen_local_path_prefix of '
'"/tmp/", the absolute path of the corpus will resolve to "/tmp/foo/bar". '
'If the --clgen_local_path_prefix is a directory, the trailing slash must '
'not be omitted.')
def AssertConfigIsValid(config: corpus_pb2.Corpus) -> corpus_pb2.Corpus:
"""Assert that config proto is valid.
Args:
config: A Corpus proto.
Returns:
The Corpus proto.
Raises:
"""This file is the entry point for unpacking chunks."""
import pathlib
from labm8 import app
from util.photolib.shutterbug import shutterbug
FLAGS = app.FLAGS
app.DEFINE_string(
'chunks_dir', None,
'The root directory of the chunks. Each chunk is a directory containing '
'files and a manifest.')
app.DEFINE_string(
'out_dir', None,
'The directory to write the unpacked chunks to. Each chunk contains files '
'which are unpacked to a path relative to this directory.')
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError('Unknown flags "{}".'.format(', '.join(argv[1:])))
chunks_dir = pathlib.Path(FLAGS.chunks_dir)
if not chunks_dir.is_dir():
raise app.UsageError('--chunks_dir not found')
out_dir = pathlib.Path(FLAGS.out_dir)
if not out_dir.is_dir():
raise app.UsageError('--out_dir not found')
app.DEFINE_integer(
'test_maxfail', 1,
'The maximum number of tests that can fail before execution terminates. '
'If --test_maxfail=0, all tests will execute.')
app.DEFINE_boolean('test_capture_output', True,
'Capture stdout and stderr during test execution.')
app.DEFINE_boolean(
'test_print_durations', True,
'Print the duration of the slowest tests at the end of execution. Use '
'--test_durations to set the number of tests to print the durations of.')
app.DEFINE_integer(
'test_durations', 3,
'The number of slowest tests to print the durations of after execution. '
'If --test_durations=0, the duration of all tests is printed.')
app.DEFINE_string(
'test_coverage_data_dir', None,
'Run tests with statement coverage and write coverage.py data files to '
'this directory. The directory is created. Existing files are untouched.')
def AbsolutePathToModule(file_path: str) -> str:
"""Determine module name from an absolute path."""
match = re.match(r'.+\.runfiles/phd/(.+)', file_path)
if match:
# Strip everything up to the root of the project from the path.
module = match.group(1)
# Strip the .py suffix.
module = module[:-len('.py')]
# Replace path sep with module sep.
module = module.replace('/', '.')
return module
else:
