def test_extract(self): config, _ = self._make_config('config_extract.ini') def _count(): self.es_client.refresh() return self.es_client.count({'match_all': {}})['count'] start, end = word2daterange('today') extract(config, start, end) count = _count() self.assertEqual(count, 102) # A second attempt should not write more logs. extract(config, start, end, force=True) self.assertEqual(count, _count()) # unless we force it extract(config, start, end, force=True) # overwrite has generated the same entries with new ids, so # we end up with double the entries self.assertEqual(count, _count()) # forcing only the load phase extract(config, start, end, sequence='load', force=True) # loading the same data (ids) won't generate any more entries self.assertEqual(count, _count())
def test_retry(self): config, _ = self._make_config('config_retry.ini') # retrying 3 times before failing in the load phase. start, end = word2daterange('today') extract(config, start, end) self.assertEqual(len(_res), 102)