def test_init(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.indices.segments.return_value = testvars.shards
ilo = curator.IndexList(client)
fmo = curator.ForceMerge(ilo, max_num_segments=2)
self.assertEqual(ilo, fmo.index_list)
self.assertEqual(client, fmo.client)
def test_do_action_with_delay(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.indices.segments.return_value = testvars.shards
client.info.return_value = {'version': {'number': '5.0.0'}}
client.indices.forcemerge.return_value = None
ilo = curator.IndexList(client)
fmo = curator.ForceMerge(ilo, max_num_segments=2, delay=0.050)
self.assertIsNone(fmo.do_action())
def test_do_action_pre5(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.indices.segments.return_value = testvars.shards
client.info.return_value = {'version': {'number': '2.3.2'}}
client.indices.optimize.return_value = None
ilo = curator.IndexList(client)
fmo = curator.ForceMerge(ilo, max_num_segments=2)
self.assertIsNone(fmo.do_action())
def test_do_dry_run(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.indices.segments.return_value = testvars.shards
client.indices.forcemerge.return_value = None
client.indices.optimize.return_value = None
ilo = curator.IndexList(client)
fmo = curator.ForceMerge(ilo, max_num_segments=2)
self.assertIsNone(fmo.do_dry_run())
def test_do_action_raises_exception(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.indices.segments.return_value = testvars.shards
client.indices.forcemerge.return_value = None
client.indices.optimize.return_value = None
client.indices.forcemerge.side_effect = testvars.fake_fail
client.indices.optimize.side_effect = testvars.fake_fail
ilo = curator.IndexList(client)
fmo = curator.ForceMerge(ilo, max_num_segments=2)
self.assertRaises(curator.FailedExecution, fmo.do_action)
ilo = curator.IndexList(client)
ilo.filter_by_regex(kind="prefix", value=prefix)
ilo.filter_by_age(source="name",
direction="older",
timestring="%Y.%m.%d",
unit="days",
unit_count=days)
except:
print("No indices found with prefix:", prefix)
sys.exit(0)
if not ilo.indices:
print("No indices older than", days, "days with prefix:", prefix)
sys.exit(0)
print('Running curator prefix:', prefix, "days:", days, "action:", action)
print('\nAffected indices')
for index in ilo.indices:
print(index)
forcemerge = curator.ForceMerge(ilo, max_num_segments=1, delay=10)
if action == "MERGE":
try:
forcemerge.do_action()
print("\nForcemerge completed")
except:
print("\nAll affected indices are already merged")
else:
forcemerge.do_dry_run()
def lambda_handler(event, dummy_contest):
"""
this methode is called by lambda.
event must contain:
* es_endpoint: elasticsearch endpoint
* delete_older_days: delete indices older than number (in days)
* snapshot_older_days: snapshot indices older than number (in days)
* repository: name of the repository
* backup_bucket: name of the bucket to store backup in
* backup_path: path in the bucket to store backup
this function will exit when elasticsearch take more than 60 sec,
to avoid having that running for long.
The next run will continue, but this has to be run multiple time a day
"""
logging.warning(event)
logging.warning("connecting to elasticsearch")
client = elasticsearch.Elasticsearch([event['es_endpoint']], timeout=120)
curator.utils.override_timeout(client, 120)
logging.warning("Checking for repositories")
repo_dict = curator.utils.get_repository(
client,
repository=event["repository"]
)
if not repo_dict:
repo_setting = {
"repository": event["repository"],
"client": client,
"repo_type": "s3",
"chunk_size": "100m",
"bucket": event["backup_bucket"],
"base_path": event["backup_path"]
}
if curator.utils.create_repository(**repo_setting):
return "Snapshot's repository setup Failed"
logging.warning("Prepare merging indices")
# optimize/force merge
ilo = curator.IndexList(client)
ilo.filter_by_age(
source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=1
)
fm_indices = curator.ForceMerge(ilo, max_num_segments=1)
try:
logging.warning("Merging indices %s", ilo.working_list())
fm_indices.do_action()
except elasticsearch.ConnectionTimeout:
# exit on timeout make sure it's not running for nothing, next run will
# continue
logging.warning("Merge timeout, exiting")
return "Wait for Merge"
logging.warning("Prepare snapshoting indices")
# snapshot
ils = curator.IndexList(client)
ils.filter_by_regex(kind='prefix', value='logs-')
ils.filter_by_age(
source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=int(event["snapshot_older_days"])
)
if len(ils.indices) > 0:
snap_indices = curator.Snapshot(
ils,
name="logs_%Y-%m-%d_%H:%M:%S",
repository=event["repository"],
wait_for_completion=True
)
try:
logging.warning("Snapshoting indices %s", ils.working_list())
snap_indices.do_action()
except curator.SnapshotInProgress:
# exit on timeout make sure it's not running for nothing, next run
# will continue
logging.warning("Snapshot in progress")
return "Wait for Snapshots"
except elasticsearch.ConnectionTimeout:
# exit on timeout make sure it's not running for nothing, next run
# will continue
logging.warning("Snapshot timeout, exiting")
return "Wait for Snapshots"
logging.warning("Prepare deleting indices")
free_space = client.cluster.stats()["nodes"]["fs"]["free_in_bytes"] / 1024
size_unit = {
'M': 1024.0,
'G': 1048576.0,
'T': 1073741824.0
}
current_unit = size_unit[event['delete_when_free_space_remaining'][-1]]
free_space /= current_unit
size_needed = float(event['delete_when_free_space_remaining'][:-1])
if free_space > size_needed:
logging.warning("Enough space remaining, no need to delete indices")
return
extra_space = free_space - size_needed
ild = curator.IndexList(client)
ild.filter_by_regex(kind='prefix', value='logs-')
ild.filter_by_age(
source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=2
)
def atoi(text):
""" transformt str to int"""
return int(text) if text.isdigit() else text
def natural_keys(text):
""" transformt str to int and remove non int"""
return [atoi(c) for c in re.split(r'(\d+)', text)]
sorted_indices = sorted(ild.indices, key=natural_keys, reverse=True)
stats = client.indices.stats(index=','.join(sorted_indices))
size_indices = {}
for indice in sorted_indices:
size = stats["indices"][indice]["total"]["store"]["size_in_bytes"]
size_indices[indice] = size / (current_unit * 1024)
while extra_space < 0:
indice_to_remove = sorted_indices.pop()
try:
logging.warning("Deleting indices %s", indice_to_remove)
client.indices.delete(
index=indice_to_remove,
params={"timeout": "5s"}
)
except elasticsearch.ConnectionTimeout:
# exit on timeout make sure it's not running for nothing, next
# run will continue
logging.warning("Delete timeout, exiting")
return "Wait for Delete"
extra_space += size_indices[indice_to_remove]
