def test_do_action_raise_snap_in_progress(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.create.return_value = None
client.snapshot.status.return_value = testvars.snap_running
client.snapshot.verify_repository.return_value = testvars.verified_nodes
ilo = curator.IndexList(client)
so = curator.Snapshot(ilo,
repository=testvars.repo_name,
name=testvars.snap_name)
self.assertRaises(curator.SnapshotInProgress, so.do_action)
def test_do_dry_run(self):
client = Mock()
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.create.return_value = None
client.snapshot.status.return_value = testvars.nosnap_running
client.snapshot.verify_repository.return_value = testvars.verified_nodes
ilo = curator.IndexList(client)
so = curator.Snapshot(ilo,
repository=testvars.repo_name,
name=testvars.snap_name)
self.assertIsNone(so.do_dry_run())
def test_do_action_raise_on_failure(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.create.return_value = None
client.snapshot.create.side_effect = testvars.fake_fail
client.snapshot.status.return_value = testvars.nosnap_running
client.snapshot.verify_repository.return_value = testvars.verified_nodes
ilo = curator.IndexList(client)
so = curator.Snapshot(ilo, repository=testvars.repo_name,
name=testvars.snap_name)
self.assertRaises(curator.FailedExecution, so.do_action)
def test_do_action_success(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'} }
client.indices.get_settings.return_value = testvars.settings_one
client.cluster.state.return_value = testvars.clu_state_one
client.indices.stats.return_value = testvars.stats_one
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.create.return_value = testvars.generic_task
client.tasks.get.return_value = testvars.completed_task
client.snapshot.status.return_value = testvars.nosnap_running
client.snapshot.verify_repository.return_value = testvars.verified_nodes
ilo = curator.IndexList(client)
so = curator.Snapshot(ilo, repository=testvars.repo_name,
name=testvars.snap_name)
self.assertIsNone(so.do_action())
def test_deletesnapshot(self):
def add_docs(idx):
for i in ["1", "2", "3"]:
self.client.create(
index=idx, doc_type='log',
body={"doc" + i :'TEST DOCUMENT'},
)
# This should force each doc to be in its own segment.
self.client.indices.flush(index=idx, force=True)
### Create snapshots to delete and verify them
self.create_repository()
timestamps = []
for i in range(1,4):
add_docs('my_index{0}'.format(i))
ilo = curator.IndexList(self.client)
snap = curator.Snapshot(ilo, repository=self.args['repository'],
name='curator-%Y%m%d%H%M%S'
)
snap.do_action()
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
self.assertEqual(i, len(snapshot['snapshots']))
time.sleep(1.0)
timestamps.append(int(time.time()))
time.sleep(1.0)
### Setup the actual delete
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.delete_snap_proto.format(
self.args['repository'], 'age', 'creation_date', 'older', ' ',
'seconds', '0', timestamps[0]
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
self.assertEqual(2, len(snapshot['snapshots']))
def lambda_handler(event, context):
""" Prior to running snapshots, we're checking if we can connect,
and if for the current status of the cluster and the repository.
"""
try:
health = es.cat.health().split(" ")[3]
repo = es.cat.repositories()
if health == 'red':
slackpost('danger', 'Cluster status is RED! Investigate ASAP!! ')
exit
if repo is None:
slackpost(
'warning',
'Missing repository for snapshot.'
'Trying to recreate the repository...')
try:
es.snapshot.create_repository(
repository='s3', body=repository_body)
es.snapshot.get_repository('s3')
slackpost('good', 'Successfully re-created the repository.')
except exceptions.NotFoundError as e:
slackpost(
'danger', 'Failed to create the repository. Please check!')
# Create the snapshot
ilo = curator.IndexList(es)
create_snapshot = curator.Snapshot(ilo=ilo,
repository='s3',
name=snapshot_name)
create_snapshot.do_action()
create_snapshot_status = create_snapshot.get_state()
if create_snapshot_status == 'SUCCESS':
msg = "Successfully created new snapshot:\n" + snapshot_name
color = 'good'
slackpost(color, msg)
else:
msg = "Failed to create new snapshot"
color = 'danger'
slackpost(color, msg)
# Deleting old indices
ilo.filter_by_age(source='creation_date', direction='older',
unit='months', unit_count=1)
curator.DeleteIndices(ilo)
except exceptions.ConnectionError as e:
slackpost('danger', e)
def test_deletesnapshot(self):
### Create snapshots to delete and verify them
self.create_repository()
timestamps = []
for i in range(1,4):
self.add_docs('my_index{0}'.format(i))
ilo = curator.IndexList(self.client)
snap = curator.Snapshot(ilo, repository=self.args['repository'],
name='curator-%Y%m%d%H%M%S', wait_interval=0.5
)
snap.do_action()
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
self.assertEqual(i, len(snapshot['snapshots']))
time.sleep(1.0)
timestamps.append(int(time.time()))
time.sleep(1.0)
### Setup the actual delete
self.write_config(
self.args['configfile'], testvars.client_config.format(host, port))
self.write_config(self.args['actionfile'],
testvars.delete_snap_proto.format(
self.args['repository'], 'age', 'creation_date', 'older', ' ',
'seconds', '0', timestamps[0]
)
)
test = clicktest.CliRunner()
result = test.invoke(
curator.cli,
[
'--config', self.args['configfile'],
self.args['actionfile']
],
)
snapshot = curator.get_snapshot(
self.client, self.args['repository'], '_all'
)
self.assertEqual(2, len(snapshot['snapshots']))
def test_deletesnapshot(self):
### Create snapshots to delete and verify them
self.create_repository()
timestamps = []
for i in range(1, 4):
self.add_docs('my_index{0}'.format(i))
ilo = curator.IndexList(self.client)
snap = curator.Snapshot(ilo,
repository=self.args['repository'],
name='curator-%Y%m%d%H%M%S',
wait_interval=0.5)
snap.do_action()
snapshot = curator.get_snapshot(self.client,
self.args['repository'], '_all')
self.assertEqual(i, len(snapshot['snapshots']))
time.sleep(1.0)
timestamps.append(int(time.time()))
time.sleep(1.0)
### Setup the actual delete
args = self.get_runner_args()
args += [
'--config',
self.args['configfile'],
'delete-snapshots',
'--repository',
self.args['repository'],
'--filter_list',
'{"filtertype":"age","source":"creation_date","direction":"older","unit":"seconds","unit_count":0,"epoch":'
+ str(timestamps[0]) + '}',
]
self.assertEqual(
0,
self.run_subprocess(
args, logname='TestCLIDeleteSnapshots.test_deletesnapshot'))
snapshot = curator.get_snapshot(self.client, self.args['repository'],
'_all')
self.assertEqual(2, len(snapshot['snapshots']))
def lambda_handler(event, context):
# Build the Elasticsearch client.
es = Elasticsearch(hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection)
#print(es.info())
index_list = curator.IndexList(es)
# Do index deletion first
index_list.filter_by_age(source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=index_rentention_days)
if index_list.indices:
print("Found %s indices to delete " % len(index_list.indices))
print(index_list.indices)
try:
curator.DeleteIndices(index_list).do_action()
except (curator.exceptions.FailedExecution) as e:
print(e)
# Snapshots next
# QA first
# Filters by age, anything with a time stamp older than delete_index_day days in the index name.
index_list = curator.IndexList(es)
index_list.filter_by_regex(kind='prefix', value='qa', exclude=False)
if index_list.indices:
index_list.filter_by_age(source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=snapshot_index_days)
if index_list.indices:
print("Found %s indices to snapshot " % len(index_list.indices))
print(index_list.indices)
try:
curator.Snapshot(index_list,
repository='my-es-snapshot-repo',
name='qa-%Y%m%d%H%M%S',
ignore_unavailable=True,
include_global_state=False,
partial=True,
wait_for_completion=False,
wait_interval=10,
max_wait=-1,
skip_repo_fs_check=True).do_action()
except (curator.exceptions.SnapshotInProgress,
curator.exceptions.FailedExecution) as e:
print(e)
# Do STAGE now
index_list = curator.IndexList(es)
index_list.filter_by_regex(kind='prefix', value='stage', exclude=False)
if index_list.indices:
index_list.filter_by_age(source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=snapshot_index_days)
if index_list.indices:
print("Found %s indices to snapshot " % len(index_list.indices))
print(index_list.indices)
try:
curator.Snapshot(index_list,
repository='my-es-snapshot-repo',
name='stage-%Y%m%d%H%M%S',
ignore_unavailable=True,
include_global_state=False,
partial=True,
wait_for_completion=False,
wait_interval=10,
max_wait=-1,
skip_repo_fs_check=True).do_action()
except (curator.exceptions.SnapshotInProgress,
curator.exceptions.FailedExecution) as e:
print(e)
# Do PROD now
index_list = curator.IndexList(es)
index_list.filter_by_regex(kind='prefix', value='prod', exclude=False)
if index_list.indices:
index_list.filter_by_age(source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=snapshot_index_days)
if index_list.indices:
print("Found %s indices to snapshot " % len(index_list.indices))
print(index_list.indices)
try:
curator.Snapshot(index_list,
repository='my-es-snapshot-repo',
name='prod-%Y%m%d%H%M%S',
ignore_unavailable=True,
include_global_state=False,
partial=True,
wait_for_completion=False,
wait_interval=10,
max_wait=-1,
skip_repo_fs_check=True).do_action()
except (curator.exceptions.SnapshotInProgress,
curator.exceptions.FailedExecution) as e:
print(e)
# Now delete old snapshots
snapshot_list = curator.SnapshotList(es, repository='my-es-snapshot-repo')
if snapshot_list.snapshots:
snapshot_list.filter_by_age(source='creation_date',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=snapshot_rentention_days)
if snapshot_list.snapshots:
try:
curator.DeleteSnapshots(snapshot_list,
retry_interval=10,
retry_count=3).do_action()
except (curator.exceptions.SnapshotInProgress,
curator.exceptions.NoSnapshots,
curator.exceptions.FailedExecution) as e:
print(e)
def lambda_handler(event, dummy_contest):
"""
this methode is called by lambda.
event must contain:
* es_endpoint: elasticsearch endpoint
* delete_older_days: delete indices older than number (in days)
* snapshot_older_days: snapshot indices older than number (in days)
* repository: name of the repository
* backup_bucket: name of the bucket to store backup in
* backup_path: path in the bucket to store backup
this function will exit when elasticsearch take more than 60 sec,
to avoid having that running for long.
The next run will continue, but this has to be run multiple time a day
"""
logging.warning(event)
logging.warning("connecting to elasticsearch")
client = elasticsearch.Elasticsearch([event['es_endpoint']], timeout=120)
curator.utils.override_timeout(client, 120)
logging.warning("Checking for repositories")
repo_dict = curator.utils.get_repository(
client,
repository=event["repository"]
)
if not repo_dict:
repo_setting = {
"repository": event["repository"],
"client": client,
"repo_type": "s3",
"chunk_size": "100m",
"bucket": event["backup_bucket"],
"base_path": event["backup_path"]
}
if curator.utils.create_repository(**repo_setting):
return "Snapshot's repository setup Failed"
logging.warning("Prepare merging indices")
# optimize/force merge
ilo = curator.IndexList(client)
ilo.filter_by_age(
source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=1
)
fm_indices = curator.ForceMerge(ilo, max_num_segments=1)
try:
logging.warning("Merging indices %s", ilo.working_list())
fm_indices.do_action()
except elasticsearch.ConnectionTimeout:
# exit on timeout make sure it's not running for nothing, next run will
# continue
logging.warning("Merge timeout, exiting")
return "Wait for Merge"
logging.warning("Prepare snapshoting indices")
# snapshot
ils = curator.IndexList(client)
ils.filter_by_regex(kind='prefix', value='logs-')
ils.filter_by_age(
source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=int(event["snapshot_older_days"])
)
if len(ils.indices) > 0:
snap_indices = curator.Snapshot(
ils,
name="logs_%Y-%m-%d_%H:%M:%S",
repository=event["repository"],
wait_for_completion=True
)
try:
logging.warning("Snapshoting indices %s", ils.working_list())
snap_indices.do_action()
except curator.SnapshotInProgress:
# exit on timeout make sure it's not running for nothing, next run
# will continue
logging.warning("Snapshot in progress")
return "Wait for Snapshots"
except elasticsearch.ConnectionTimeout:
# exit on timeout make sure it's not running for nothing, next run
# will continue
logging.warning("Snapshot timeout, exiting")
return "Wait for Snapshots"
logging.warning("Prepare deleting indices")
free_space = client.cluster.stats()["nodes"]["fs"]["free_in_bytes"] / 1024
size_unit = {
'M': 1024.0,
'G': 1048576.0,
'T': 1073741824.0
}
current_unit = size_unit[event['delete_when_free_space_remaining'][-1]]
free_space /= current_unit
size_needed = float(event['delete_when_free_space_remaining'][:-1])
if free_space > size_needed:
logging.warning("Enough space remaining, no need to delete indices")
return
extra_space = free_space - size_needed
ild = curator.IndexList(client)
ild.filter_by_regex(kind='prefix', value='logs-')
ild.filter_by_age(
source='name',
direction='older',
timestring='%Y.%m.%d',
unit='days',
unit_count=2
)
def atoi(text):
""" transformt str to int"""
return int(text) if text.isdigit() else text
def natural_keys(text):
""" transformt str to int and remove non int"""
return [atoi(c) for c in re.split(r'(\d+)', text)]
sorted_indices = sorted(ild.indices, key=natural_keys, reverse=True)
stats = client.indices.stats(index=','.join(sorted_indices))
size_indices = {}
for indice in sorted_indices:
size = stats["indices"][indice]["total"]["store"]["size_in_bytes"]
size_indices[indice] = size / (current_unit * 1024)
while extra_space < 0:
indice_to_remove = sorted_indices.pop()
try:
logging.warning("Deleting indices %s", indice_to_remove)
client.indices.delete(
index=indice_to_remove,
params={"timeout": "5s"}
)
except elasticsearch.ConnectionTimeout:
# exit on timeout make sure it's not running for nothing, next
# run will continue
logging.warning("Delete timeout, exiting")
return "Wait for Delete"
extra_space += size_indices[indice_to_remove]
def lambda_handler(event, context):
now = datetime.now()
snapshot_prefix = 'automatic-'
snapshot_name = snapshot_prefix + now.strftime("%Y%m%d%H%M%S")
# Build the Elasticsearch client.
es = Elasticsearch(
hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection,
# Deleting snapshots can take a while, so keep the connection open for long enough to get a response.
timeout=120)
# REGISTER
try:
payload = {
"type": "s3",
"settings": {
"bucket": bucket,
"region": region,
"role_arn": role_arn
}
}
es.snapshot.create_repository(repository_name, json.dumps(payload))
except (ElasticsearchException) as e:
print(e)
raise
# DELETE
try:
snapshot_list = curator.SnapshotList(es, repository=repository_name)
snapshot_list.filter_by_regex(kind='prefix', value=snapshot_prefix)
snapshot_list.filter_by_age(source='creation_date',
direction='older',
unit='days',
unit_count=int(retention))
# Delete the old snapshots.
curator.DeleteSnapshots(snapshot_list,
retry_interval=30,
retry_count=3).do_action()
except (curator.exceptions.NoSnapshots) as e:
# This is fine
print(e)
except (curator.exceptions.SnapshotInProgress,
curator.exceptions.FailedExecution) as e:
print(e)
raise
# CREATE
try:
index_list = curator.IndexList(es)
# Take a new snapshot. This operation can take a while, so we don't want to wait for it to complete.
curator.Snapshot(index_list,
repository=repository_name,
name=snapshot_name,
wait_for_completion=False).do_action()
except (curator.exceptions.SnapshotInProgress,
curator.exceptions.FailedExecution) as e:
print(e)
raise
