def test_report_state_all(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'}}
client.snapshot.get.return_value = testvars.snapshot
client.snapshot.get_repository.return_value = testvars.test_repo
client.indices.get_settings.return_value = testvars.settings_named
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo)
self.assertIsNone(ro.report_state())
def test_filter_by_age__name_older_than_now(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
sl.filter_by_age(source='name', direction='older',
timestring='%Y.%m.%d', unit='days', unit_count=1
)
self.assertEqual(['snapshot-2015.03.01'], sl.snapshots)
def test_invalid_filtertype(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
slo = curator.SnapshotList(client, repository=testvars.repo_name)
config = yaml.load(testvars.invalid_ft,
Loader=yaml.FullLoader)['actions'][1]
self.assertRaises(curator.ConfigurationError, slo.iterate_filters,
config)
def test_filter_by_regex_prefix_exclude(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
self.assertEqual([u'snap_name', u'snapshot-2015.03.01'],
sorted(sl.snapshots))
sl.filter_by_regex(kind='prefix', value='snap_', exclude=True)
self.assertEqual([u'snapshot-2015.03.01'], sl.snapshots)
def test_init(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
self.assertEqual(testvars.snapshots['snapshots'],sl.all_snapshots)
self.assertEqual(
['snap_name','snapshot-2015.03.01'], sorted(sl.snapshots)
)
def test_filter_by_age__name_no_timestring(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
self.assertRaises(curator.MissingArgument,
sl.filter_by_age,
source='name', unit='days', unit_count=1, direction='older'
)
def test_none_filtertype(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
slo = curator.SnapshotList(client, repository=testvars.repo_name)
config = yaml.load(testvars.snap_none_ft)['actions'][1]
slo.iterate_filters(config)
self.assertEqual(
['snap_name', 'snapshot-2015.03.01'], sorted(slo.snapshots))
def test_do_dry_run_with_renames(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo,
rename_pattern='(.+)',
rename_replacement='new_$1')
self.assertIsNone(ro.do_dry_run())
def test_get_name_based_ages_match(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
sl._get_name_based_ages('%Y.%m.%d')
self.assertEqual(1425168000,
sl.snapshot_info['snapshot-2015.03.01']['age_by_name']
)
def test_filter_by_age_missing_direction(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
self.assertRaises(curator.MissingArgument,
sl.filter_by_age,
unit='days',
unit_count=1)
def test_no_filters(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
slo = curator.SnapshotList(client, repository=testvars.repo_name)
slo.iterate_filters({})
self.assertEqual(
['snap_name', 'snapshot-2015.03.01'], sorted(slo.snapshots)
)
def test_do_action(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
client.tasks.get.return_value = testvars.no_snap_tasks
client.snapshot.delete.return_value = None
slo = curator.SnapshotList(client, repository=testvars.repo_name)
do = curator.DeleteSnapshots(slo)
self.assertIsNone(do.do_action())
def test_filter_by_age__creation_date_younger_than_now(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
sl.filter_by_age(direction='younger',
timestring='%Y.%m.%d', unit='days', unit_count=1
)
self.assertEqual([], sl.snapshots)
def test_do_action_raises_exception(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.delete.return_value = None
client.tasks.get.return_value = testvars.no_snap_tasks
client.snapshot.delete.side_effect = testvars.fake_fail
slo = curator.SnapshotList(client, repository=testvars.repo_name)
do = curator.DeleteSnapshots(slo)
self.assertRaises(curator.FailedExecution, do.do_action)
def test_sort_by_age(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
slo = curator.SnapshotList(client, repository=testvars.repo_name)
slo._calculate_ages()
slo.age_keyfield = 'invalid'
snaps = slo.snapshots
slo._sort_by_age(snaps)
self.assertEqual(['snapshot-2015.03.01'], slo.snapshots)
def test_with_age_reversed(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
slo = curator.SnapshotList(client, repository=testvars.repo_name)
slo.filter_by_count(count=1,
source='creation_date',
use_age=True,
reverse=False)
self.assertEqual(['snapshot-2015.03.01'], slo.snapshots)
def test_filter_by_age_bad_direction(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
self.assertRaises(ValueError,
sl.filter_by_age,
unit='days',
unit_count=1,
direction="invalid")
def test_report_state_not_all(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
client.indices.get_settings.return_value = testvars.settings_one
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo,
rename_pattern='(.+)',
rename_replacement='new_$1')
self.assertIsNone(ro.report_state())
def test_filter_by_age__creation_date_older_than_past_date(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
sl.filter_by_age(direction='older',
timestring='%Y.%m.%d', unit='seconds', unit_count=0,
epoch=1425168001
)
self.assertEqual(['snap_name'], sl.snapshots)
def test_filter_by_age__name_younger_than_past_date(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
sl.filter_by_age(source='name', direction='younger',
timestring='%Y.%m.%d', unit='seconds', unit_count=0,
epoch=1422748800
)
self.assertEqual(['snapshot-2015.03.01'], sl.snapshots)
def test_get_expected_output(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo,
rename_pattern='(.+)',
rename_replacement='new_$1')
self.assertEqual(ro.expected_output,
['new_index-2015.01.01', 'new_index-2015.02.01'])
def test_do_action_snap_in_progress(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.status.return_value = testvars.snap_running
client.snapshot.verify_repository.return_value = testvars.verified_nodes
client.indices.get_settings.return_value = testvars.settings_named
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo)
self.assertRaises(curator.SnapshotInProgress, ro.do_action)
def test_do_action_success_no_wfc(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.status.return_value = testvars.nosnap_running
client.snapshot.verify_repository.return_value = testvars.verified_nodes
client.indices.get_settings.return_value = testvars.settings_named
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo, wait_for_completion=False)
self.assertIsNone(ro.do_action())
def test_success_inclusive(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
sl.filter_by_state(state='SUCCESS')
self.assertEqual(
[u'snap_name', u'snapshot-2015.03.01'],
sorted(sl.snapshots)
)
def test_report_state_not_all(self):
client = Mock()
client.info.return_value = {'version': {'number': '5.0.0'}}
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
client.indices.get_settings.return_value = testvars.settings_one
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo,
rename_pattern='(.+)',
rename_replacement='new_$1')
self.assertRaises(curator.exceptions.FailedRestore, ro.report_state)
def test_do_action_report_on_failure(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.status.return_value = testvars.nosnap_running
client.snapshot.verify_repository.return_value = testvars.verified_nodes
client.indices.get_settings.return_value = testvars.settings_named
client.snapshot.restore.side_effect = testvars.fake_fail
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo)
self.assertRaises(curator.FailedExecution, ro.do_action)
def test_do_action_success(self):
client = Mock()
client.info.return_value = {'version': {'number': '2.4.1'}}
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
client.snapshot.status.return_value = testvars.nosnap_running
client.snapshot.verify_repository.return_value = testvars.verified_nodes
client.indices.get_settings.return_value = testvars.settings_named
slo = curator.SnapshotList(client, repository=testvars.repo_name)
ro = curator.Restore(slo)
self.assertIsNone(ro.do_action())
def test_filter_by_regex_bad_kind(self):
client = Mock()
client.snapshot.get.return_value = testvars.snapshots
client.snapshot.get_repository.return_value = testvars.test_repo
sl = curator.SnapshotList(client, repository=testvars.repo_name)
self.assertEqual(
[u'snap_name', u'snapshot-2015.03.01'],
sorted(sl.snapshots)
)
self.assertRaises(
ValueError, sl.filter_by_regex, kind='invalid', value=None)
def fetch(self, act_on, on_nofilters_showall=False):
"""
Forwarder method to indices/snapshots selector.
"""
if act_on not in ['indices', 'snapshots', 'cluster']:
raise ValueError('invalid argument: ' + act_on)
if act_on == 'indices':
return curator.IndexList(self.client)
elif act_on == 'snapshots':
return curator.SnapshotList(self.client)
else:
return []
def lambda_handler(event, context):
# Build the Elasticsearch client.
es = Elasticsearch(
hosts=[{
'host': host,
'port': 443
}],
http_auth=awsauth,
use_ssl=True,
verify_certs=True,
connection_class=RequestsHttpConnection,
timeout=
120 # Deleting snapshots can take a while, so keep the connection open for long enough to get a response.
)
try:
# Get all snapshots in the repository.
snapshot_list = curator.SnapshotList(es, repository=repository_name)
# Filter by age, any snapshot older than two weeks.
snapshot_list.filter_by_age(source='creation_date',
direction='older',
unit='weeks',
unit_count=2)
# Delete the old snapshots.
curator.DeleteSnapshots(snapshot_list,
retry_interval=30,
retry_count=3).do_action()
except (curator.exceptions.SnapshotInProgress,
curator.exceptions.NoSnapshots,
curator.exceptions.FailedExecution) as e:
print(e)
# Split into two try blocks. We still want to try and take a snapshot if deletion failed.
try:
# Get the list of indices.
# You can filter this list if you didn't want to snapshot all indices.
index_list = curator.IndexList(es)
# Take a new snapshot. This operation can take a while, so we don't want to wait for it to complete.
curator.Snapshot(index_list,
repository=repository_name,
name=snapshot_name,
wait_for_completion=False).do_action()
except (curator.exceptions.SnapshotInProgress,
curator.exceptions.FailedExecution) as e:
print(e)
