def test_report_state_all(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'}} client.snapshot.get.return_value = testvars.snapshot client.snapshot.get_repository.return_value = testvars.test_repo client.indices.get_settings.return_value = testvars.settings_named slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo) self.assertIsNone(ro.report_state())
def test_filter_by_age__name_older_than_now(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(source='name', direction='older', timestring='%Y.%m.%d', unit='days', unit_count=1 ) self.assertEqual(['snapshot-2015.03.01'], sl.snapshots)
def test_invalid_filtertype(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = curator.SnapshotList(client, repository=testvars.repo_name) config = yaml.load(testvars.invalid_ft, Loader=yaml.FullLoader)['actions'][1] self.assertRaises(curator.ConfigurationError, slo.iterate_filters, config)
def test_filter_by_regex_prefix_exclude(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) self.assertEqual([u'snap_name', u'snapshot-2015.03.01'], sorted(sl.snapshots)) sl.filter_by_regex(kind='prefix', value='snap_', exclude=True) self.assertEqual([u'snapshot-2015.03.01'], sl.snapshots)
def test_init(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) self.assertEqual(testvars.snapshots['snapshots'],sl.all_snapshots) self.assertEqual( ['snap_name','snapshot-2015.03.01'], sorted(sl.snapshots) )
def test_filter_by_age__name_no_timestring(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) self.assertRaises(curator.MissingArgument, sl.filter_by_age, source='name', unit='days', unit_count=1, direction='older' )
def test_none_filtertype(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = curator.SnapshotList(client, repository=testvars.repo_name) config = yaml.load(testvars.snap_none_ft)['actions'][1] slo.iterate_filters(config) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(slo.snapshots))
def test_do_dry_run_with_renames(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo, rename_pattern='(.+)', rename_replacement='new_$1') self.assertIsNone(ro.do_dry_run())
def test_get_name_based_ages_match(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) sl._get_name_based_ages('%Y.%m.%d') self.assertEqual(1425168000, sl.snapshot_info['snapshot-2015.03.01']['age_by_name'] )
def test_filter_by_age_missing_direction(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) self.assertRaises(curator.MissingArgument, sl.filter_by_age, unit='days', unit_count=1)
def test_no_filters(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = curator.SnapshotList(client, repository=testvars.repo_name) slo.iterate_filters({}) self.assertEqual( ['snap_name', 'snapshot-2015.03.01'], sorted(slo.snapshots) )
def test_do_action(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.tasks.get.return_value = testvars.no_snap_tasks client.snapshot.delete.return_value = None slo = curator.SnapshotList(client, repository=testvars.repo_name) do = curator.DeleteSnapshots(slo) self.assertIsNone(do.do_action())
def test_filter_by_age__creation_date_younger_than_now(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(direction='younger', timestring='%Y.%m.%d', unit='days', unit_count=1 ) self.assertEqual([], sl.snapshots)
def test_do_action_raises_exception(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.delete.return_value = None client.tasks.get.return_value = testvars.no_snap_tasks client.snapshot.delete.side_effect = testvars.fake_fail slo = curator.SnapshotList(client, repository=testvars.repo_name) do = curator.DeleteSnapshots(slo) self.assertRaises(curator.FailedExecution, do.do_action)
def test_sort_by_age(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = curator.SnapshotList(client, repository=testvars.repo_name) slo._calculate_ages() slo.age_keyfield = 'invalid' snaps = slo.snapshots slo._sort_by_age(snaps) self.assertEqual(['snapshot-2015.03.01'], slo.snapshots)
def test_with_age_reversed(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = curator.SnapshotList(client, repository=testvars.repo_name) slo.filter_by_count(count=1, source='creation_date', use_age=True, reverse=False) self.assertEqual(['snapshot-2015.03.01'], slo.snapshots)
def test_filter_by_age_bad_direction(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) self.assertRaises(ValueError, sl.filter_by_age, unit='days', unit_count=1, direction="invalid")
def test_report_state_not_all(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.indices.get_settings.return_value = testvars.settings_one slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo, rename_pattern='(.+)', rename_replacement='new_$1') self.assertIsNone(ro.report_state())
def test_filter_by_age__creation_date_older_than_past_date(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(direction='older', timestring='%Y.%m.%d', unit='seconds', unit_count=0, epoch=1425168001 ) self.assertEqual(['snap_name'], sl.snapshots)
def test_filter_by_age__name_younger_than_past_date(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) sl.filter_by_age(source='name', direction='younger', timestring='%Y.%m.%d', unit='seconds', unit_count=0, epoch=1422748800 ) self.assertEqual(['snapshot-2015.03.01'], sl.snapshots)
def test_get_expected_output(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo, rename_pattern='(.+)', rename_replacement='new_$1') self.assertEqual(ro.expected_output, ['new_index-2015.01.01', 'new_index-2015.02.01'])
def test_do_action_snap_in_progress(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.snap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes client.indices.get_settings.return_value = testvars.settings_named slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo) self.assertRaises(curator.SnapshotInProgress, ro.do_action)
def test_do_action_success_no_wfc(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.nosnap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes client.indices.get_settings.return_value = testvars.settings_named slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo, wait_for_completion=False) self.assertIsNone(ro.do_action())
def test_success_inclusive(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) sl.filter_by_state(state='SUCCESS') self.assertEqual( [u'snap_name', u'snapshot-2015.03.01'], sorted(sl.snapshots) )
def test_report_state_not_all(self): client = Mock() client.info.return_value = {'version': {'number': '5.0.0'}} client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.indices.get_settings.return_value = testvars.settings_one slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo, rename_pattern='(.+)', rename_replacement='new_$1') self.assertRaises(curator.exceptions.FailedRestore, ro.report_state)
def test_do_action_report_on_failure(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.nosnap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes client.indices.get_settings.return_value = testvars.settings_named client.snapshot.restore.side_effect = testvars.fake_fail slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo) self.assertRaises(curator.FailedExecution, ro.do_action)
def test_do_action_success(self): client = Mock() client.info.return_value = {'version': {'number': '2.4.1'}} client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo client.snapshot.status.return_value = testvars.nosnap_running client.snapshot.verify_repository.return_value = testvars.verified_nodes client.indices.get_settings.return_value = testvars.settings_named slo = curator.SnapshotList(client, repository=testvars.repo_name) ro = curator.Restore(slo) self.assertIsNone(ro.do_action())
def test_filter_by_regex_bad_kind(self): client = Mock() client.snapshot.get.return_value = testvars.snapshots client.snapshot.get_repository.return_value = testvars.test_repo sl = curator.SnapshotList(client, repository=testvars.repo_name) self.assertEqual( [u'snap_name', u'snapshot-2015.03.01'], sorted(sl.snapshots) ) self.assertRaises( ValueError, sl.filter_by_regex, kind='invalid', value=None)
def fetch(self, act_on, on_nofilters_showall=False): """ Forwarder method to indices/snapshots selector. """ if act_on not in ['indices', 'snapshots', 'cluster']: raise ValueError('invalid argument: ' + act_on) if act_on == 'indices': return curator.IndexList(self.client) elif act_on == 'snapshots': return curator.SnapshotList(self.client) else: return []
def lambda_handler(event, context): # Build the Elasticsearch client. es = Elasticsearch( hosts=[{ 'host': host, 'port': 443 }], http_auth=awsauth, use_ssl=True, verify_certs=True, connection_class=RequestsHttpConnection, timeout= 120 # Deleting snapshots can take a while, so keep the connection open for long enough to get a response. ) try: # Get all snapshots in the repository. snapshot_list = curator.SnapshotList(es, repository=repository_name) # Filter by age, any snapshot older than two weeks. snapshot_list.filter_by_age(source='creation_date', direction='older', unit='weeks', unit_count=2) # Delete the old snapshots. curator.DeleteSnapshots(snapshot_list, retry_interval=30, retry_count=3).do_action() except (curator.exceptions.SnapshotInProgress, curator.exceptions.NoSnapshots, curator.exceptions.FailedExecution) as e: print(e) # Split into two try blocks. We still want to try and take a snapshot if deletion failed. try: # Get the list of indices. # You can filter this list if you didn't want to snapshot all indices. index_list = curator.IndexList(es) # Take a new snapshot. This operation can take a while, so we don't want to wait for it to complete. curator.Snapshot(index_list, repository=repository_name, name=snapshot_name, wait_for_completion=False).do_action() except (curator.exceptions.SnapshotInProgress, curator.exceptions.FailedExecution) as e: print(e)