def test_KerasBackend_is_trained(clgen_cache_dir, abc_keras_model_config): """Test that is_trained changes to True when model is trained.""" del clgen_cache_dir m = models.Model(abc_keras_model_config) assert not m.is_trained m.Train() assert m.is_trained
def test_benchmark_TensorFlowModel_Train_already_trained( clgen_cache_dir, abc_tensorflow_model_config, benchmark): """Benchmark the Train() method on an already-trained model.""" del clgen_cache_dir m = models.Model(abc_tensorflow_model_config) m.Train() # "Offline" training from cold. benchmark(m.Train)
def test_KerasBackend_Sample_exact_multiple_of_batch_size( clgen_cache_dir, abc_keras_model_config): """Test that min_num_samples are returned when a multiple of batch_size.""" del clgen_cache_dir m = models.Model(abc_keras_model_config) assert len(m.Sample(MockSampler(batch_size=2), 2)) == 2 assert len(m.Sample(MockSampler(batch_size=2), 4)) == 4
def __init__(self, config: clgen_pb2.Instance, dashboard_opts={}): """Instantiate an instance. Args: config: An Instance proto. Raises: UserError: If the instance proto contains invalid values, is missing a model or sampler fields. """ try: pbutil.AssertFieldIsSet(config, "model_specification") pbutil.AssertFieldIsSet(config, "sampler") except pbutil.ProtoValueError as e: raise errors.UserError(e) self.config = config self.working_dir = None if config.HasField("working_dir"): self.working_dir: pathlib.Path = pathlib.Path( os.path.expandvars( config.working_dir)).expanduser().absolute() # Enter a session so that the cache paths are set relative to any requested # working directory. with self.Session(): if config.HasField("model"): self.model: models.Model = models.Model(config.model) else: self.model: pretrained.PreTrainedModel = pretrained.PreTrainedModel( pathlib.Path(config.pretrained_model)) self.sampler: samplers.Sampler = samplers.Sampler(config.sampler) self.dashboard = dashboard.Launch(**dashboard_opts)
def test_TensorFlowBackend_Train_GetShortSummary_before_create( clgen_cache_dir, abc_tensorflow_model_config): """Test that model training produced telemetry files.""" del clgen_cache_dir m = models.Model(abc_tensorflow_model_config) with test.Raises(ValueError): m.GetShortSummary()
def test_TensorFlowBackend_Train_missing_intermediate_checkpoints( clgen_cache_dir, abc_tensorflow_model_config): """Test that a missing intermediate checkpoint does not affect training.""" del clgen_cache_dir abc_tensorflow_model_config.training.num_epochs = 2 m = models.Model(abc_tensorflow_model_config) m.Train() assert 2 == len(m.backend.epoch_checkpoints) checkpoints_dir = m.cache.path / "checkpoints" for path in checkpoints_dir.iterdir(): # Remove all files which are not either the checkpoints list, or the most # recent checkpoint. if not path.name == "checkpoint" and not path.name.startswith( "checkpoint-2"): path.unlink() f1a = checksumdir.dirhash(checkpoints_dir) assert 1 == len(m.backend.epoch_checkpoints) assert 2 in m.backend.epoch_checkpoints # Run Train() again to check that nothing is changed. m.Train() assert 1 == len(m.backend.epoch_checkpoints) assert 2 in m.backend.epoch_checkpoints f1b = checksumdir.dirhash(checkpoints_dir) assert f1a == f1b
def test_TensorFlowBackend_Sample_return_value_matches_cached_sample( clgen_cache_dir, abc_tensorflow_model_config): """Test that Sample() returns Sample protos.""" del clgen_cache_dir abc_tensorflow_model_config.training.batch_size = 1 m = models.Model(abc_tensorflow_model_config) sample_observer = sample_observers.InMemorySampleSaver() m.Sample( MockSampler(hash="hash"), [ sample_observers.MaxSampleCountObserver(1), sample_observer, sample_observers.LegacySampleCacheObserver(), ], ) samples = sample_observer.samples # Samples are produced in batches of sampler.batch_size elements. assert len(samples) == 1 assert len(list((m.cache.path / "samples" / "hash").iterdir())) == 1 cached_sample_path = (m.cache.path / "samples" / "hash" / list( (m.cache.path / "samples" / "hash").iterdir())[0]) assert cached_sample_path.is_file() cached_sample = pbutil.FromFile(cached_sample_path, model_pb2.Sample()) assert samples[0].text == cached_sample.text assert samples[0].sample_time_ms == cached_sample.sample_time_ms assert (samples[0].sample_start_epoch_ms_utc == cached_sample.sample_start_epoch_ms_utc)
def test_Model_config_sequence_length_not_set(clgen_cache_dir, abc_model_config): """Test that an error is raised if sequence_length is < 1.""" del clgen_cache_dir abc_model_config.training.sequence_length = -1 with test.Raises(errors.UserError): models.Model(abc_model_config)
def test_Model_metafile(clgen_cache_dir, abc_model_config): """A newly instantiated model's cache has a metafile.""" del clgen_cache_dir m = models.Model(abc_model_config) assert (m.cache.path / "META.pbtxt").is_file() assert pbutil.ProtoIsReadable(m.cache.path / "META.pbtxt", internal_pb2.ModelMeta())
def test_benchmark_KerasBackend_Train_already_trained( clgen_cache_dir, abc_keras_model_config, benchmark): """Benchmark the Train() method on an already-trained model.""" del clgen_cache_dir m = models.Model(abc_keras_model_config) m.Train() # "Offline" training from cold. benchmark(m.Train)
def test_KerasBackend_GetInferenceModel_predict_output_shape( clgen_cache_dir, abc_keras_model_config): """Test that predict() on inference model is one-hot encoded.""" del clgen_cache_dir m = models.Model(abc_keras_model_config) im, batch_size = m.backend.GetInferenceModel() probabilities = im.predict(np.array([[0]]) * batch_size) assert (batch_size, 1, m.corpus.vocab_size) == probabilities.shape
def test_TensorFlowBackend_Train_GetShortSummary(clgen_cache_dir, abc_tensorflow_model_config): """Test that model training produced telemetry files.""" del clgen_cache_dir m = models.Model(abc_tensorflow_model_config) m.Create() assert (m.GetShortSummary() == "4×1 LSTM network, 59 token corpus with 25-element vocabulary")
def test_TensorFlowBackend_Sample_exact_multiple_of_batch_size( clgen_cache_dir, abc_tensorflow_model_config): """Test that min_num_samples are returned when a multiple of batch_size.""" del clgen_cache_dir abc_tensorflow_model_config.training.batch_size = 2 m = models.Model(abc_tensorflow_model_config) assert len(m.Sample(MockSampler(), 2)) == 2 assert len(m.Sample(MockSampler(), 4)) == 4
def test_KerasBackend_Sample_implicit_train(clgen_cache_dir, abc_keras_model_config): """Test that Sample() implicitly trains the model.""" del clgen_cache_dir m = models.Model(abc_keras_model_config) assert not m.is_trained m.Sample(MockSampler(), 1) assert m.is_trained
def test_TensorFlowBackend_Train_is_trained(clgen_cache_dir, abc_tensorflow_model_config): """Test that is_trained is initially false until trained.""" del clgen_cache_dir m = models.Model(abc_tensorflow_model_config) assert not m.is_trained m.Train() assert m.is_trained
def test_TensorFlowBackend_Sample_implicit_train(clgen_cache_dir, abc_tensorflow_model_config): """Test that Sample() implicitly trains the model.""" del clgen_cache_dir m = models.Model(abc_tensorflow_model_config) assert not m.is_trained m.Sample(MockSampler(), [sample_observers.MaxSampleCountObserver(1)]) assert m.is_trained
def test_Model_atomizer_symlink(clgen_cache_dir, abc_model_config): """Test path of symlink to atomizer.""" del clgen_cache_dir m = models.Model(abc_model_config) assert (m.cache.path / 'atomizer').is_symlink() path = str((m.cache.path / 'atomizer').resolve()) # We can't do a literal comparison because of bazel sandboxing. assert path.endswith(str(m.corpus.atomizer_path))
def test_Model_corpus_symlink(clgen_cache_dir, abc_model_config): """Test path of symlink to corpus files.""" del clgen_cache_dir m = models.Model(abc_model_config) assert (m.cache.path / 'corpus').is_symlink() path = str((m.cache.path / 'corpus').resolve()) # We can't do a literal comparison because of bazel sandboxing. assert path.endswith(str(m.corpus.encoded.database_path.parent))
def test_TensorFlowBackend_Train_with_sample_callback( clgen_cache_dir, abc_tensorflow_model_config): """Test that sampling during training does not blow up.""" del clgen_cache_dir abc_tensorflow_model_config.training.num_epochs = 2 sampler = MockSampler() m = models.Model(abc_tensorflow_model_config) m.Train(test_sampler=sampler) assert m.is_trained
def test_Model_directories(clgen_cache_dir, abc_model_config): """A newly instantiated model's cache has checkpoint and sample dirs.""" del clgen_cache_dir m = models.Model(abc_model_config) assert (m.cache.path / 'checkpoints').is_dir() assert (m.cache.path / 'samples').is_dir() # There should be nothing in these directories yet. assert not list((m.cache.path / 'checkpoints').iterdir()) assert not list((m.cache.path / 'samples').iterdir())
def test_Model_corpus_symlink(clgen_cache_dir, abc_model_config): """Test path of symlink to corpus files.""" del clgen_cache_dir m = models.Model(abc_model_config) assert (m.cache.path / "corpus").is_symlink() path = str((m.cache.path / "corpus").resolve()) # We can't do a literal comparison because of bazel sandboxing. assert path.endswith( str(pathlib.Path(m.corpus.encoded.url[len("sqlite:///"):]).parent))
def test_KerasBackend_Train_telemetry(clgen_cache_dir, abc_keras_model_config): """Test that model training produced telemetry files.""" del clgen_cache_dir abc_keras_model_config.training.num_epochs = 2 m = models.Model(abc_keras_model_config) assert len(m.TrainingTelemetry()) == 0 m.Train() assert len(m.TrainingTelemetry()) == 2 for telemetry in m.TrainingTelemetry(): assert isinstance(telemetry, telemetry_pb2.ModelEpochTelemetry)
def test_KerasBackend_Train_epoch_checkpoints(clgen_cache_dir, abc_keras_model_config): """Test that a trained model generates weight checkpoints.""" del clgen_cache_dir abc_keras_model_config.training.num_epochs = 2 m = models.Model(abc_keras_model_config) m.Train() assert len(m.backend.epoch_checkpoints) == 2 for path in m.backend.epoch_checkpoints: assert path.is_file()
def test_TensorFlowBackend_Train_epoch_checkpoints(clgen_cache_dir, abc_tensorflow_model_config): """Test that epoch_checkpoints returns a <int, str> dict.""" del clgen_cache_dir abc_tensorflow_model_config.training.num_epochs = 2 m = models.Model(abc_tensorflow_model_config) assert not m.backend.epoch_checkpoints m.Train() epoch_checkpoints = m.backend.epoch_checkpoints assert 2 == len(epoch_checkpoints) assert 1 in epoch_checkpoints assert 2 in epoch_checkpoints
def test_KerasBackend_Sample_exact_multiple_of_batch_size( clgen_cache_dir, abc_keras_model_config): """Test that min_num_samples are returned when a multiple of batch_size.""" del clgen_cache_dir m = models.Model(abc_keras_model_config) sample_observer = sample_observers.InMemorySampleSaver() m.Sample(MockSampler(batch_size=2), [sample_observers.MaxSampleCountObserver(2), sample_observer]) assert len(sample_observer.samples) == 2 sample_observer = sample_observers.InMemorySampleSaver() m.Sample(MockSampler(batch_size=2), [sample_observers.MaxSampleCountObserver(4), sample_observer]) assert len(sample_observer.samples) == 4
def test_KerasBackend_Train_twice(clgen_cache_dir, abc_keras_model_config): """Test that TensorFlow checkpoint does not change after training twice.""" del clgen_cache_dir abc_keras_model_config.training.num_epochs = 1 m = models.Model(abc_keras_model_config) m.Train() f1a = checksumdir.dirhash(m.cache.path / "checkpoints") f1b = crypto.md5_file(m.cache.path / "META.pbtxt") m.Train() f2a = checksumdir.dirhash(m.cache.path / "checkpoints") f2b = crypto.md5_file(m.cache.path / "META.pbtxt") assert f1a == f2a assert f1b == f2b
def LsModels(cache_root: pathlib.Path) -> None: for model_dir in (cache_root / "model").iterdir(): meta_file = model_dir / "META.pbtxt" if pbutil.ProtoIsReadable(meta_file, internal_pb2.ModelMeta()): model = models.Model( pbutil.FromFile(meta_file, internal_pb2.ModelMeta()).config) telemetry = list(model.TrainingTelemetry()) num_epochs = model.config.training.num_epochs n = len(telemetry) print(f"{model_dir} {n} / {num_epochs} epochs") elif meta_file.is_file(): app.Warning("Meta file %s cannot be read.", meta_file) else: app.Warning("Meta file %s not found.", meta_file)
def test_TensorFlowBackend_Sample_inexact_multiple_of_batch_size( clgen_cache_dir, abc_tensorflow_model_config): """Test that min_num_samples are returned when a multiple of batch_size.""" del clgen_cache_dir m = models.Model(abc_tensorflow_model_config) sampler = MockSampler() sampler.batch_size = 3 # 3 = 1 * sizeof(batch). saver = sample_observers.InMemorySampleSaver() m.Sample(sampler, [sample_observers.MaxSampleCountObserver(2), saver]) assert len(saver.samples) == 3 # 6 = 2 * sizeof(batch). saver = sample_observers.InMemorySampleSaver() m.Sample(sampler, [sample_observers.MaxSampleCountObserver(4), saver]) assert len(saver.samples) == 6
def test_KerasBackend_Sample_return_value_matches_cached_sample( clgen_cache_dir, abc_keras_model_config): """Test that Sample() returns Sample protos.""" del clgen_cache_dir m = models.Model(abc_keras_model_config) samples = m.Sample(MockSampler(hash='hash'), 1) assert len(samples) == 1 assert len(list((m.cache.path / 'samples' / 'hash').iterdir())) == 1 cached_sample_path = (m.cache.path / 'samples' / 'hash' / list( (m.cache.path / 'samples' / 'hash').iterdir())[0]) assert cached_sample_path.is_file() cached_sample = pbutil.FromFile(cached_sample_path, model_pb2.Sample()) assert samples[0].text == cached_sample.text assert samples[0].sample_time_ms == cached_sample.sample_time_ms assert samples[ 0].sample_start_epoch_ms_utc == cached_sample.sample_start_epoch_ms_utc
def test_KerasBackend_Sample_return_value_matches_cached_sample( clgen_cache_dir, abc_keras_model_config): """Test that Sample() returns Sample protos.""" del clgen_cache_dir m = models.Model(abc_keras_model_config) sample_observer = sample_observers.InMemorySampleSaver() m.Sample( MockSampler(hash="hash"), [sample_observers.MaxSampleCountObserver(1), sample_observer], ) samples = sample_observer.samples assert len(samples) == 1 assert len(list((m.cache.path / "samples" / "hash").iterdir())) == 1 cached_sample_path = (m.cache.path / "samples" / "hash" / list( (m.cache.path / "samples" / "hash").iterdir())[0]) assert cached_sample_path.is_file() cached_sample = pbutil.FromFile(cached_sample_path, model_pb2.Sample()) assert samples[0].text == cached_sample.text assert samples[0].sample_time_ms == cached_sample.sample_time_ms assert (samples[0].sample_start_epoch_ms_utc == cached_sample.sample_start_epoch_ms_utc)