def save_stubhub_putpost_requests(event, context): ''' reindex entries to another index for longer storage ''' prefix = PREFIX_DEFAULT_VALUE if event: if PREFIX_KEY in event: prefix = event[PREFIX_KEY] ilo = filter_yesterday(prefix) if ilo.working_list(): yesterday = datetime.today() - timedelta(1) dateformat = yesterday.strftime('%Y-%m-%d') index_name = ilo.working_list()[0] from_date = dateformat + "T00:00:00" to_date = dateformat + "T23:59:59" logger.info( f'reindexing entries from index {index_name}, filtering from {from_date} to {to_date}' ) index_suffix = datetime.today().strftime('%Y-%m') index = f'infinity-{index_suffix}' request_body = { "source": { "index": index_name, "query": { "bool": { "must": [{ "query_string": { "query": "(appname:mpputtickets OR appname:shsyncx) AND (message:\"POST https://api.stubhub.com\" OR message:\"PUT https://api.stubhub.com\")", "analyze_wildcard": True, "default_field": "*" } }, { "range": { "@timestamp": { "gte": from_date, "lte": to_date } } }], "filter": [], "should": [], "must_not": [] } } }, "dest": { "index": index } } reindex = curator.Reindex(ilo, request_body=request_body, wait_for_completion=False) reindex.do_action()
def test_replace_index_list(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.indices.get_settings.return_value = testvars.settings_four client.cluster.state.return_value = testvars.clu_state_four client.indices.stats.return_value = testvars.stats_four ilo = curator.IndexList(client) ro = curator.Reindex(ilo, testvars.reindex_replace) self.assertEqual(ro.index_list.indices, ro.body['source']['index'])
def test_do_dry_run(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.indices.get_settings.return_value = testvars.settings_four client.cluster.state.return_value = testvars.clu_state_four client.indices.stats.return_value = testvars.stats_four ilo = curator.IndexList(client) ro = curator.Reindex(ilo, testvars.reindex_basic) self.assertIsNone(ro.do_dry_run())
def test_init(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.indices.get_settings.return_value = testvars.settings_one client.cluster.state.return_value = testvars.clu_state_one client.indices.stats.return_value = testvars.stats_one ilo = curator.IndexList(client) ro = curator.Reindex(ilo, testvars.reindex_basic) self.assertEqual(ilo, ro.index_list) self.assertEqual(client, ro.client)
def test_init_raise_empty_source_list(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'}} client.indices.get_settings.return_value = testvars.settings_one client.cluster.state.return_value = testvars.clu_state_one client.indices.stats.return_value = testvars.stats_one ilo = curator.IndexList(client) badval = {'source': {'index': []}, 'dest': {'index': 'other_index'}} ro = curator.Reindex(ilo, badval) self.assertRaises(curator.FailedExecution, ro.do_action)
def test_reindex_timedout(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.indices.get_settings.return_value = testvars.settings_four client.cluster.state.return_value = testvars.clu_state_four client.indices.stats.return_value = testvars.stats_four client.reindex.return_value = testvars.generic_task client.tasks.get.return_value = testvars.incomplete_task ilo = curator.IndexList(client) ro = curator.Reindex(ilo, testvars.reindex_basic, max_wait=1, wait_interval=1) self.assertRaises(curator.FailedExecution, ro.do_action)
def test_reindex_without_wait(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.indices.get_settings.return_value = testvars.settings_four client.cluster.state.return_value = testvars.clu_state_four client.indices.stats.return_value = testvars.stats_four client.reindex.return_value = testvars.generic_task client.tasks.get.return_value = testvars.completed_task ilo = curator.IndexList(client) ro = curator.Reindex(ilo, testvars.reindex_basic, wait_for_completion=False) self.assertIsNone(ro.do_action())
def test_reindex_with_wait(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.indices.get_settings.return_value = testvars.settings_four client.cluster.state.return_value = testvars.clu_state_four client.indices.stats.return_value = testvars.stats_four client.reindex.return_value = testvars.generic_task client.tasks.get.return_value = testvars.completed_task ilo = curator.IndexList(client) # After building ilo, we need a different return value client.indices.get_settings.return_value = {'other_index':{}} ro = curator.Reindex(ilo, testvars.reindex_basic) self.assertIsNone(ro.do_action())
def test_reindex_with_wait_zero_total_fail(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'} } client.indices.get_settings.return_value = testvars.settings_four client.cluster.state.return_value = testvars.clu_state_four client.indices.stats.return_value = testvars.stats_four client.reindex.return_value = testvars.generic_task client.tasks.get.side_effect = testvars.fake_fail ilo = curator.IndexList(client) # After building ilo, we need a different return value client.indices.get_settings.return_value = {'other_index':{}} ro = curator.Reindex(ilo, testvars.reindex_basic) self.assertRaises(curator.CuratorException, ro.do_action)