def test_extract(self):
config, _ = self._make_config('config_extract.ini')
def _count():
self.es_client.refresh()
return self.es_client.count({'match_all': {}})['count']
start, end = word2daterange('today')
extract(config, start, end)
count = _count()
self.assertEqual(count, 102)
# A second attempt should not write more logs.
extract(config, start, end, force=True)
self.assertEqual(count, _count())
# unless we force it
extract(config, start, end, force=True)
# overwrite has generated the same entries with new ids, so
# we end up with double the entries
self.assertEqual(count, _count())
# forcing only the load phase
extract(config, start, end, sequence='load', force=True)
# loading the same data (ids) won't generate any more entries
self.assertEqual(count, _count())
def test_retry(self):
config, _ = self._make_config('config_retry.ini')
# retrying 3 times before failing in the load phase.
start, end = word2daterange('today')
extract(config, start, end)
self.assertEqual(len(_res), 102)
