def test_no_update_if_time_is_same(self, mock_getmtime):
mock_getmtime.return_value = 100
file = tempfile.NamedTemporaryFile('wb', delete=False)
try:
tokens.write_csv(
database.load_token_database(
io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS)), file)
file.close()
detok = detokenize.AutoUpdatingDetokenizer(file,
min_poll_period_s=0)
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# Empty the database, but keep the mock modified time the same.
with open(file.name, 'wb'):
pass
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# Move back time so the now-empty file is reloaded.
mock_getmtime.return_value = 50
self.assertFalse(detok.detokenize(JELLO_WORLD_TOKEN).ok())
finally:
os.unlink(file.name)
def test_update(self, mock_getmtime):
"""Tests the update command."""
db = database.load_token_database(
io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS))
self.assertEqual(len(db), TOKENS_IN_ELF)
the_time = [100]
def move_back_time_if_file_exists(path):
if os.path.exists(path):
the_time[0] -= 1
return the_time[0]
raise FileNotFoundError
mock_getmtime.side_effect = move_back_time_if_file_exists
file = tempfile.NamedTemporaryFile('wb', delete=False)
try:
file.close()
detok = detokenize.AutoUpdatingDetokenizer(file.name,
min_poll_period_s=0)
self.assertFalse(detok.detokenize(JELLO_WORLD_TOKEN).ok())
with open(file.name, 'wb') as fd:
tokens.write_binary(db, fd)
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
finally:
os.unlink(file.name)
# The database stays around if the file is deleted.
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
def test_update(self, mock_getmtime):
db = database.load_token_database(
io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS))
self.assertEqual(len(db), 16)
the_time = [100]
def move_back_time_if_file_exists(path):
if os.path.exists(path):
the_time[0] -= 1
return the_time[0]
raise FileNotFoundError
mock_getmtime.side_effect = move_back_time_if_file_exists
with tempfile.NamedTemporaryFile('wb', delete=True) as fd:
detok = detokenize.AutoUpdatingDetokenizer(fd.name,
min_poll_period_s=0)
self.assertFalse(detok.detokenize(JELLO_WORLD_TOKEN).ok())
tokens.write_binary(db, fd)
fd.flush()
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# The database stays around if the file is deleted.
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
def test_no_update_if_time_is_same(self, mock_getmtime):
mock_getmtime.return_value = 100
with tempfile.NamedTemporaryFile('wb', delete=True) as fd:
tokens.write_csv(
database.load_token_database(
io.BytesIO(ELF_WITH_TOKENIZER_SECTIONS)), fd)
fd.flush()
detok = detokenize.AutoUpdatingDetokenizer(fd, min_poll_period_s=0)
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# Empty the database, but keep the modified time the same.
fd.truncate(0)
fd.flush()
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
self.assertTrue(detok.detokenize(JELLO_WORLD_TOKEN).ok())
# Move back time so the now-empty file is reloaded.
mock_getmtime.return_value = 50
self.assertFalse(detok.detokenize(JELLO_WORLD_TOKEN).ok())