def ResolvePreprocessedId(content_id: str, config: corpus_pb2.Corpus) -> str:
"""Compute the hash of a corpus of preprocessed contentfiles.
The hash is computed from the ID of the input files and the serialized
representation of the preprocessor pipeline.
"""
return crypto.sha1_list(content_id, *config.preprocessor)
def ResolveEncodedId(content_id: str, config: corpus_pb2.Corpus) -> str:
"""Compute the hash of a corpus of preprocessed and encoded contentfiles.
The hash is computed from the ID of the input files and the serialized
representation of the config proto.
"""
config_without_contentfiles = corpus_pb2.Corpus()
config_without_contentfiles.CopyFrom(config)
# Clear the contentfiles field, since we use the content_id to uniquely
# identify the input files. This means that corpuses with the same content
# files delivered through different means (e.g. two separate but identical
# directories) have the same hash.
config_without_contentfiles.ClearField('contentfiles')
return crypto.sha1_list(content_id,
config_without_contentfiles.SerializeToString())
def _ComputeHash(corpus_: corpuses.Corpus, config: model_pb2.Model) -> str:
"""Compute model hash.
The hash is computed from the ID of the corpus and the serialized
representation of the config proto. The number of epochs that the model is
trained for does not affect the hash, since we can share checkpoints
between different models if the only variable is the epoch count. E.g.
we have a model trained for 10 epochs, we can use the checkpoint as the
starting point for a training a model for 20 epochs.
Args:
corpus: A corpus instance.
config: A Model config proto.
Returns:
The unique model ID.
"""
config_to_hash = model_pb2.Model()
config_to_hash.CopyFrom(config)
config_to_hash.ClearField('corpus')
config_to_hash.training.ClearField('num_epochs')
return crypto.sha1_list(corpus_.hash,
config_to_hash.SerializeToString())
def test_sha1_list():
assert ("06bf71070d31b2ebe4bdae828fc76a70e4b56f00" == crypto.sha1_list(
["hello", "world"]))
def test_sha1_empty_list():
assert ("da39a3ee5e6b4b0d3255bfef95601890afd80709" == crypto.sha1_list())
assert ("97d170e1550eee4afc0af065b78cda302a97674c" == crypto.sha1_list([]))
