def test_no_update_if_time_is_same(self, mock_getmtime):
mock_getmtime.return_value = 100
file = tempfile.NamedTemporaryFile('wb', delete=False)
try:
tokens.write_csv(
database.load_token_database(
io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS)), file)
file.close()
detok = detokenize.AutoUpdatingDetokenizer(file,
min_poll_period_s=0)
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# Empty the database, but keep the mock modified time the same.
with open(file.name, 'wb'):
pass
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# Move back time so the now-empty file is reloaded.
mock_getmtime.return_value = 50
self.assertFalse(detok.detokenize(JELLO_WORLD_TOKEN).ok())
finally:
os.unlink(file.name)
def _handle_create(databases, database, force, output_type, include, exclude,
replace):
"""Creates a token database file from one or more ELF files."""
if database == '-':
# Must write bytes to stdout; use sys.stdout.buffer.
fd = sys.stdout.buffer
elif not force and os.path.exists(database):
raise FileExistsError(
f'The file {database} already exists! Use --force to overwrite.')
else:
fd = open(database, 'wb')
database = tokens.Database.merged(*databases)
database.filter(include, exclude, replace)
with fd:
if output_type == 'csv':
tokens.write_csv(database, fd)
elif output_type == 'binary':
tokens.write_binary(database, fd)
else:
raise ValueError(f'Unknown database type "{output_type}"')
_LOG.info('Wrote database with %d entries to %s as %s', len(database),
fd.name, output_type)
def test_no_update_if_time_is_same(self, mock_getmtime):
mock_getmtime.return_value = 100
with tempfile.NamedTemporaryFile('wb', delete=True) as fd:
tokens.write_csv(
database.load_token_database(
io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS)), fd)
fd.flush()
detok = detokenize.AutoUpdatingDetokenizer(fd, min_poll_period_s=0)
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# Empty the database, but keep the modified time the same.
fd.truncate(0)
fd.flush()
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# Move back time so the now-empty file is reloaded.
mock_getmtime.return_value = 50
self.assertFalse(detok.detokenize(JELLO_WORLD_TOKEN).ok())