def save_stubhub_putpost_requests(event, context):
''' reindex entries to another index for longer storage '''
prefix = PREFIX_DEFAULT_VALUE
if event:
if PREFIX_KEY in event:
prefix = event[PREFIX_KEY]
ilo = filter_yesterday(prefix)
if ilo.working_list():
yesterday = datetime.today() - timedelta(1)
dateformat = yesterday.strftime('%Y-%m-%d')
index_name = ilo.working_list()[0]
from_date = dateformat + "T00:00:00"
to_date = dateformat + "T23:59:59"
logger.info(
f'reindexing entries from index {index_name}, filtering from {from_date} to {to_date}'
)
index_suffix = datetime.today().strftime('%Y-%m')
index = f'infinity-{index_suffix}'
request_body = {
"source": {
"index": index_name,
"query": {
"bool": {
"must": [{
"query_string": {
"query":
"(appname:mpputtickets OR appname:shsyncx) AND (message:\"POST https://api.stubhub.com\" OR message:\"PUT https://api.stubhub.com\")",
"analyze_wildcard": True,
"default_field": "*"
}
}, {
"range": {
"@timestamp": {
"gte": from_date,
"lte": to_date
}
}
}],
"filter": [],
"should": [],
"must_not": []
}
}
},
"dest": {
"index": index
}
}
reindex = curator.Reindex(ilo,
request_body=request_body,
wait_for_completion=False)
reindex.do_action()
def test_replace_index_list(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_four
client.cluster.state.return_value = testvars.clu_state_four
client.indices.stats.return_value = testvars.stats_four
ilo = curator.IndexList(client)
ro = curator.Reindex(ilo, testvars.reindex_replace)
self.assertEqual(ro.index_list.indices, ro.body['source']['index'])
def test_do_dry_run(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_four
client.cluster.state.return_value = testvars.clu_state_four
client.indices.stats.return_value = testvars.stats_four
ilo = curator.IndexList(client)
ro = curator.Reindex(ilo, testvars.reindex_basic)
self.assertIsNone(ro.do_dry_run())
def test_init(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
ilo = curator.IndexList(client)
ro = curator.Reindex(ilo, testvars.reindex_basic)
self.assertEqual(ilo, ro.index_list)
self.assertEqual(client, ro.client)
def test_init_raise_empty_source_list(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'}}
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
ilo = curator.IndexList(client)
badval = {'source': {'index': []}, 'dest': {'index': 'other_index'}}
ro = curator.Reindex(ilo, badval)
self.assertRaises(curator.FailedExecution, ro.do_action)
def test_reindex_timedout(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_four
client.cluster.state.return_value = testvars.clu_state_four
client.indices.stats.return_value = testvars.stats_four
client.reindex.return_value = testvars.generic_task
client.tasks.get.return_value = testvars.incomplete_task
ilo = curator.IndexList(client)
ro = curator.Reindex(ilo, testvars.reindex_basic,
max_wait=1, wait_interval=1)
self.assertRaises(curator.FailedExecution, ro.do_action)
def test_reindex_without_wait(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_four
client.cluster.state.return_value = testvars.clu_state_four
client.indices.stats.return_value = testvars.stats_four
client.reindex.return_value = testvars.generic_task
client.tasks.get.return_value = testvars.completed_task
ilo = curator.IndexList(client)
ro = curator.Reindex(ilo, testvars.reindex_basic,
wait_for_completion=False)
self.assertIsNone(ro.do_action())
def test_reindex_with_wait(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_four
client.cluster.state.return_value = testvars.clu_state_four
client.indices.stats.return_value = testvars.stats_four
client.reindex.return_value = testvars.generic_task
client.tasks.get.return_value = testvars.completed_task
ilo = curator.IndexList(client)
# After building ilo, we need a different return value
client.indices.get_settings.return_value = {'other_index':{}}
ro = curator.Reindex(ilo, testvars.reindex_basic)
self.assertIsNone(ro.do_action())
def test_reindex_with_wait_zero_total_fail(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_four
client.cluster.state.return_value = testvars.clu_state_four
client.indices.stats.return_value = testvars.stats_four
client.reindex.return_value = testvars.generic_task
client.tasks.get.side_effect = testvars.fake_fail
ilo = curator.IndexList(client)
# After building ilo, we need a different return value
client.indices.get_settings.return_value = {'other_index':{}}
ro = curator.Reindex(ilo, testvars.reindex_basic)
self.assertRaises(curator.CuratorException, ro.do_action)
