def test_update(self, mock_getmtime):
"""Tests the update command."""
db = database.load_token_database(
io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS))
self.assertEqual(len(db), TOKENS_IN_ELF)
the_time = [100]
def move_back_time_if_file_exists(path):
if os.path.exists(path):
the_time[0] -= 1
return the_time[0]
raise FileNotFoundError
mock_getmtime.side_effect = move_back_time_if_file_exists
file = tempfile.NamedTemporaryFile('wb', delete=False)
try:
file.close()
detok = detokenize.AutoUpdatingDetokenizer(file.name,
min_poll_period_s=0)
self.assertFalse(detok.detokenize(JELLO_WORLD_TOKEN).ok())
with open(file.name, 'wb') as fd:
tokens.write_binary(db, fd)
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
finally:
os.unlink(file.name)
# The database stays around if the file is deleted.
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
def _handle_create(databases, database, force, output_type, include, exclude,
replace):
"""Creates a token database file from one or more ELF files."""
if database == '-':
# Must write bytes to stdout; use sys.stdout.buffer.
fd = sys.stdout.buffer
elif not force and os.path.exists(database):
raise FileExistsError(
f'The file {database} already exists! Use --force to overwrite.')
else:
fd = open(database, 'wb')
database = tokens.Database.merged(*databases)
database.filter(include, exclude, replace)
with fd:
if output_type == 'csv':
tokens.write_csv(database, fd)
elif output_type == 'binary':
tokens.write_binary(database, fd)
else:
raise ValueError(f'Unknown database type "{output_type}"')
_LOG.info('Wrote database with %d entries to %s as %s', len(database),
fd.name, output_type)
def test_update(self, mock_getmtime):
db = database.load_token_database(
io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS))
self.assertEqual(len(db), 16)
the_time = [100]
def move_back_time_if_file_exists(path):
if os.path.exists(path):
the_time[0] -= 1
return the_time[0]
raise FileNotFoundError
mock_getmtime.side_effect = move_back_time_if_file_exists
with tempfile.NamedTemporaryFile('wb', delete=True) as fd:
detok = detokenize.AutoUpdatingDetokenizer(fd.name,
min_poll_period_s=0)
self.assertFalse(detok.detokenize(JELLO_WORLD_TOKEN).ok())
tokens.write_binary(db, fd)
fd.flush()
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# The database stays around if the file is deleted.
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
def test_binary_format_write(self):
db = read_db_from_csv(CSV_DATABASE)
with io.BytesIO() as fd:
tokens.write_binary(db, fd)
binary_db = fd.getvalue()
self.assertEqual(BINARY_DATABASE, binary_db)
