def main(argv: typing.List[str]):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments: '{}'.".format(' '.join(argv[1:])))
if not FLAGS.targets:
raise app.UsageError('--targets must be one-or-more bazel targets')
targets = list(sorted(set(FLAGS.targets)))
excluded_targets = set(FLAGS.excluded_targets)
def _GetFileMapping(f: str):
if len(f.split(':')) == 2:
return f.split(':')
else:
return f, f
extra_files = list(sorted(set(FLAGS.extra_files)))
move_file_tuples = [
_GetFileMapping(f) for f in list(sorted(set(FLAGS.mv_files)))
]
move_file_mapping = {x[0]: x[1] for x in move_file_tuples}
with DestinationDirectoryFromFlags() as destination:
connection = api.GetGithubConectionFromFlagsOrDie()
repo = GetOrCreateRepoOrDie(connection, FLAGS.github_repo)
api.CloneRepoToDestination(repo, destination)
destination_repo = git.Repo(destination)
source_workspace.ExportToRepo(destination_repo, targets, excluded_targets,
extra_files, move_file_mapping)
destination_repo.git.push('origin')
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError('Unknown flags "{}".'.format(', '.join(argv[1:])))
src_dir = pathlib.Path(FLAGS.src_dir)
if not src_dir.is_dir():
raise app.UsageError('--src_dir not found')
chunks_dir = pathlib.Path(FLAGS.chunks_dir)
chunks_dir.mkdir(parents=True, exist_ok=True)
size_in_bytes = FLAGS.size_mb * 1000**2
if FLAGS.size_mb < 1:
raise app.UsageError('--size_mb must be greater than or equal to one.')
chunk_prefix = FLAGS.chunk_prefix
if '/' in chunk_prefix:
raise app.UsageError("--chunk_prefix cannot contain '/' character.")
shutterbug.MakeChunks([src_dir],
chunks_dir,
size_in_bytes,
prefix=chunk_prefix,
shuffle=FLAGS.random_ordering,
seed=FLAGS.random_ordering_seed,
gzip=FLAGS.gzip_files)
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError('Unknown flags "{}".'.format(', '.join(argv[1:])))
chunks_dir = pathlib.Path(FLAGS.chunks_dir)
if not chunks_dir.is_dir():
raise app.UsageError('--chunks_dir not found')
out_dir = pathlib.Path(FLAGS.out_dir)
if not out_dir.is_dir():
raise app.UsageError('--out_dir not found')
shutterbug.unchunk(chunks_dir, out_dir)
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unrecognized command line options: '{}'".format(
' '.join(argv[1:])))
RunWithErrorHandling(DoFlagsAction)
def ConfigFromFlags() -> clgen_pb2.Instance:
config_path = pathlib.Path(FLAGS.config)
if not config_path.is_file():
raise app.UsageError(f"CLgen --config file not found: '{config_path}'")
config = pbutil.FromFile(config_path, clgen_pb2.Instance())
os.environ['PWD'] = str(config_path.parent)
return config
def main():
if not FLAGS.targets:
raise app.UsageError('--targets must be one-or-more bazel targets')
targets = list(sorted(set(FLAGS.targets)))
excluded_targets = set(FLAGS.excluded_targets)
def _GetFileMapping(f: str):
if len(f.split(':')) == 2:
return f.split(':')
else:
return f, f
extra_files = list(sorted(set(FLAGS.extra_files)))
move_file_tuples = [
_GetFileMapping(f) for f in list(sorted(set(FLAGS.move_file_mapping)))
]
move_file_mapping = {x[0]: x[1] for x in move_file_tuples}
workspace_root = pathlib.Path(getconfig.GetGlobalConfig().paths.repo_root)
Export(workspace_root=workspace_root,
github_repo=FLAGS.github_repo,
targets=targets,
excluded_targets=excluded_targets,
extra_files=extra_files,
move_file_mapping=move_file_mapping,
resume_export=not FLAGS.ignore_last_export,
run_handler=lambda x: x())
def RunPytestOnFileAndExit(file_path: str, argv: typing.List[str]):
"""Run pytest on a file and exit.
This is invoked by absl.app.RunWithArgs(), and has access to absl flags.
This function does not return.
Args:
file_path: The path of the file to test.
argv: Positional arguments not parsed by absl. No additional arguments are
supported.
"""
if len(argv) > 1:
raise app.UsageError("Unknown arguments: '{}'.".format(' '.join(
argv[1:])))
# Test files must end with _test.py suffix. This is a code style choice, not
# a hard requirement.
if not file_path.endswith('_test.py'):
app.Fatal('File `%s` does not end in suffix _test.py', file_path)
# Assemble the arguments to run pytest with. Note that the //:conftest file
# performs some additional configuration not captured here.
pytest_args = [
file_path,
# Run pytest verbosely.
'-vv',
'-p',
'no:cacheprovider',
]
if FLAGS.test_color:
pytest_args.append('--color=yes')
if FLAGS.test_maxfail != 0:
pytest_args.append(f'--maxfail={FLAGS.test_maxfail}')
# Print the slowest test durations at the end of execution.
if FLAGS.test_print_durations:
pytest_args.append(f'--durations={FLAGS.test_durations}')
# Capture stdout and stderr by default.
if not FLAGS.test_capture_output:
pytest_args.append('-s')
with CoverageContext(file_path, pytest_args) as pytest_args:
app.Log(1, 'Running pytest with arguments: %s', pytest_args)
ret = pytest.main(pytest_args)
sys.exit(ret)
def DoFlagsAction():
"""Do the action requested by the command line flags."""
config = ConfigFromFlags()
if FLAGS.clgen_profiling:
prof.enable()
instance = Instance(config)
with instance.Session():
if FLAGS.print_cache_path == 'corpus':
print(instance.model.corpus.cache.path)
return
elif FLAGS.print_cache_path == 'model':
print(instance.model.cache.path)
return
elif FLAGS.print_cache_path == 'sampler':
print(instance.model.SamplerCache(instance.sampler))
return
elif FLAGS.print_cache_path:
raise app.UsageError(
f"Invalid --print_cache_path argument: '{FLAGS.print_cache_path}'")
# The default action is to sample the model.
if FLAGS.stop_after == 'corpus':
instance.model.corpus.Create()
elif FLAGS.stop_after == 'train':
instance.m.Train()
app.Log(1, 'Model: %s', instance.model.cache.path)
elif FLAGS.stop_after:
raise app.UsageError(
f"Invalid --stop_after argument: '{FLAGS.stop_after}'")
elif FLAGS.export_model:
instance.ExportPretrainedModel(pathlib.Path(FLAGS.export_model))
else:
sample_observers = SampleObserversFromFlags()
instance.Sample(sample_observers)
def main(argv): # pylint: disable=missing-docstring
paths_to_lint = [pathlib.Path(arg) for arg in argv[1:]]
if not paths_to_lint:
raise app.UsageError("Usage: photolint <directory...>")
for path in paths_to_lint:
if not path.exists():
app.FatalWithoutStackTrace(
f"File or directory not found: '{path}'")
# Linting is on a per-directory level, not per-file.
directories_to_lint = {
path if path.is_dir() else path.parent
for path in paths_to_lint
}
for directory in sorted(directories_to_lint):
directory = directory.absolute()
app.Log(2, 'Linting directory `%s`', directory)
workspace_ = workspace.Workspace.FindWorkspace(directory)
linters.Lint(workspace_, directory)
# Print the carriage return once we've done updating the counts line.
if FLAGS.counts:
if linters.ERROR_COUNTS:
print("", file=sys.stderr)
else:
linters.PrintErrorCounts(end="\n")
# Print the profiling timers once we're done.
if FLAGS.profile:
total_time = linters.TIMERS.total_seconds
linting_time = linters.TIMERS.linting_seconds
cached_time = linters.TIMERS.cached_seconds
overhead = total_time - linting_time - cached_time
print(
f'timings: linting={humanize.Duration(linting_time)} '
f'({linting_time / total_time:.1%}), '
f'cached={humanize.Duration(cached_time)} '
f'({cached_time / total_time:.1%}), '
f'overhead={humanize.Duration(overhead)} '
f'({overhead / total_time:.1%}), '
f'total={humanize.Duration(total_time)}.',
file=sys.stderr)
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments: '{}'.".format(" ".join(argv[1:])))
PopulateBytecodeTable(FLAGS.db(), FLAGS.dataset)
