def test_Model_metafile(clgen_cache_dir, abc_model_config):
"""A newly instantiated model's cache has a metafile."""
del clgen_cache_dir
m = models.Model(abc_model_config)
assert (m.cache.path / 'META.pbtxt').is_file()
assert pbutil.ProtoIsReadable(m.cache.path / 'META.pbtxt',
internal_pb2.ModelMeta())
def GeneratorFromFlag(config_class,
generator_class) -> base_generator.GeneratorServiceBase:
"""Instantiate a generator from the --generator_config flag."""
if not pbutil.ProtoIsReadable(FLAGS.generator_config, config_class()):
raise app.UsageError(
f'--generator_config is not a {config_class.__name__} proto')
config = pbutil.FromFile(pathlib.Path(FLAGS.generator_config),
config_class())
return generator_class(config)
def InitManifest(package_dir: pathlib.Path,
contents: typing.List[pathlib.Path], update: bool) -> None:
"""Write the MANIFEST.pbtxt file for a package."""
manifest = CreatePackageManifest(package_dir, contents)
manifest_path = package_dir / 'MANIFEST.pbtxt'
if update and pbutil.ProtoIsReadable(manifest_path,
dpack_pb2.DataPackage()):
old = pbutil.FromFile(manifest_path, dpack_pb2.DataPackage())
MergeManifests(manifest, old)
elif manifest_path.is_file():
raise OSError('Refusing to overwrite MANIFEST.pbtxt file.')
pbutil.ToFile(manifest, manifest_path)
logging.info('Wrote %s', manifest_path.absolute())
def main(argv):
"""Main entry point."""
if len(argv) > 1:
raise app.UsageError("Unknown arguments: '{}'.".format(' '.join(
argv[1:])))
config = pathlib.Path(FLAGS.generator)
if not pbutil.ProtoIsReadable(config, generator_pb2.ClgenGenerator()):
raise app.UsageError(
'--generator is not a deepsmith.ClgenGenerator proto')
generator_config = pbutil.FromFile(config, generator_pb2.ClgenGenerator())
output_directory = pathlib.Path(FLAGS.output_directory)
GenerateTestcases(generator_config, output_directory, FLAGS.num_testcases)
def LsModels(cache_root: pathlib.Path) -> None:
for model_dir in (cache_root / 'model').iterdir():
meta_file = model_dir / 'META.pbtxt'
if pbutil.ProtoIsReadable(meta_file, internal_pb2.ModelMeta()):
model = models.Model(
pbutil.FromFile(meta_file, internal_pb2.ModelMeta()).config)
telemetry = list(model.TrainingTelemetry())
num_epochs = model.config.training.num_epochs
n = len(telemetry)
print(f'{model_dir} {n} / {num_epochs} epochs')
elif meta_file.is_file():
logging.warning('Meta file %s cannot be read.', meta_file)
else:
logging.warning('Meta file %s not found.', meta_file)
def ShouldImportRepo(session: orm.session.Session,
metafile: pathlib.Path) -> bool:
"""Determine if the repository described by a metafile should be imported.
A repository should be imported iff:
* The metafile is a valid GitHubRepoMetadata proto.
* The clone directory specified in the metafile appears to be a github repo.
* The repo does not exist in the contentfiles database.
"""
if not (metafile.is_file() and pbutil.ProtoIsReadable(
metafile, scrape_repos_pb2.GitHubRepoMetadata())):
return False
meta = pbutil.FromFile(metafile, scrape_repos_pb2.GitHubRepoMetadata())
clone_dir = metafile.parent / f'{meta.owner}_{meta.name}'
if not (clone_dir / '.git').is_dir():
return False
return not contentfiles.GitHubRepository.IsInDatabase(session, meta)
def MakeRepositoryMetas(self,
repos: typing.List[Repository.Repository]) -> None:
"""Make meta files for a list of repositories.
Args:
repos: A list of GitHub Repository instances.
"""
logging.debug('Scraping %s repositories',
humanize.intcomma(len(repos)))
for repo in repos:
self.i += 1
concat_name = '_'.join([repo.owner.login, repo.name])
clone_dir = self.destination_directory / concat_name
meta_path = pathlib.Path(str(clone_dir) + '.pbtxt')
if not pbutil.ProtoIsReadable(
meta_path, scrape_repos_pb2.GitHubRepoMetadata()):
meta = GetRepositoryMetadata(repo)
logging.debug('%s', meta)
pbutil.ToFile(meta, meta_path)
def ResultProtoFromFlag(flag: typing.Optional[str]) -> deepsmith_pb2.Result:
"""Read a result proto from a --flag path.
Args:
flag: The value of the flag which points to a result proto.
Returns:
The Result proto.
Raises:
UsageError: If the flag is not set or the flag does not point to a Result
proto.
"""
if not flag:
raise app.UsageError('Path is not set.')
path = pathlib.Path(flag)
if not path.is_file():
raise app.UsageError(f"File not found: '{path}'.")
if not pbutil.ProtoIsReadable(path, deepsmith_pb2.Result()):
raise app.UsageError(f"Cannot read Result proto: '{path}'.")
return pbutil.FromFile(path, deepsmith_pb2.Result())
def IsRepoMetaFile(f: str):
"""Determine if a path is a GitHubRepoMetadata message."""
return (fs.isfile(f) and pbutil.ProtoIsReadable(f,
scrape_repos_pb2.GitHubRepoMetadata()))
from absl import flags
from absl import logging
from labm8 import pbutil
from system.machines.proto import data_tiers_pb2
FLAGS = flags.FLAGS
flags.DEFINE_string('data_tiers', None,
'The path of the directory to package.')
flags.DEFINE_bool('summary', False, 'TODO')
flags.register_validator(
'data_tiers',
lambda path: pbutil.ProtoIsReadable(path, data_tiers_pb2.DataTiers()),
message='--data_tiers must be a DataTiers message.')
def _SetDirectorySize(tier: data_tiers_pb2.Directory):
path = pathlib.Path(tier.path).expanduser()
if not path.is_dir():
logging.warning("Directory '%s' not found", path)
return
os.chdir(path)
excludes = ['--exclude={}'.format(pathlib.Path(e).expanduser())
for e in tier.exclude]
cmd = ['du', '-b', '-s', '.'] + excludes
logging.info('$ cd %s && %s', path, ' '.join(cmd))
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, universal_newlines=True)
def _IsManifest(path: pathlib.Path) -> bool:
"""Check if a path contains a DataPackafe file."""
return pbutil.ProtoIsReadable(path, dpack_pb2.DataPackage())
def DirContainsProtos(data_path: str, proto_class) -> None:
"""Assert that contains protos of the given class."""
for path in bazelutil.DataPath(data_path).iterdir():
assert pbutil.ProtoIsReadable(
bazelutil.DataPath(data_path) / path, proto_class())