Пример #1
0
 def test_restore_with_rename(self):
     indices = []
     for i in range(1, 4):
         self.add_docs('my_index{0}'.format(i))
         indices.append('my_index{0}'.format(i))
     snap_name = 'snapshot1'
     self.create_snapshot(snap_name, ','.join(indices))
     snapshot = curator.get_snapshot(self.client, self.args['repository'],
                                     '_all')
     self.assertEqual(1, len(snapshot['snapshots']))
     self.client.indices.delete(','.join(indices))
     self.assertEqual([], curator.get_indices(self.client))
     self.write_config(self.args['configfile'],
                       testvars.client_config.format(host, port))
     self.write_config(
         self.args['actionfile'],
         testvars.restore_snapshot_proto.format(self.args['repository'],
                                                snap_name, indices, False,
                                                False, True, False,
                                                'my_index(.+)',
                                                'new_index$1', ' ', True,
                                                False, 301))
     test = clicktest.CliRunner()
     result = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     restored_indices = sorted(curator.get_indices(self.client))
     self.assertEqual(['new_index1', 'new_index2', 'new_index3'],
                      restored_indices)
Пример #2
0
 def test_restore(self):
     indices = []
     for i in range(1,4):
         self.add_docs('my_index{0}'.format(i))
         indices.append('my_index{0}'.format(i))
     snap_name = 'snapshot1'
     self.create_snapshot(snap_name, ','.join(indices))
     snapshot = curator.get_snapshot(
                 self.client, self.args['repository'], '_all'
                )
     self.assertEqual(1, len(snapshot['snapshots']))
     self.client.indices.delete(','.join(indices))
     self.assertEqual([], curator.get_indices(self.client))
     args = self.get_runner_args()
     args += [
         '--config', self.args['configfile'],
         'restore',
         '--repository', self.args['repository'],
         '--name', snap_name,
         '--index', indices[0],
         '--index', indices[1],
         '--index', indices[2],
         '--wait_interval', '1',
         '--max_wait', '3',
         '--filter_list', '{"filtertype":"none"}',
     ]
     self.assertEqual(0, self.run_subprocess(args, logname='TestCLIRestore.test_restore'))
     restored_indices = sorted(curator.get_indices(self.client))
     self.assertEqual(indices, restored_indices)
     # The test runs so fast that it tries to execute the cleanup step
     # and delete the repository before Elasticsearch is actually ready
     time.sleep(0.5)
Пример #3
0
 def test_restore(self):
     indices = []
     for i in range(1, 4):
         self.add_docs('my_index{0}'.format(i))
         indices.append('my_index{0}'.format(i))
     snap_name = 'snapshot1'
     self.create_snapshot(snap_name, ','.join(indices))
     snapshot = curator.get_snapshot(self.client, self.args['repository'],
                                     '_all')
     self.assertEqual(1, len(snapshot['snapshots']))
     self.client.indices.delete(','.join(indices))
     self.assertEqual([], curator.get_indices(self.client))
     self.write_config(self.args['configfile'],
                       testvars.client_config.format(host, port))
     self.write_config(
         self.args['actionfile'],
         testvars.restore_snapshot_proto.format(self.args['repository'],
                                                snap_name, indices, False,
                                                False, True, False, ' ',
                                                ' ', ' ', True, False, 301,
                                                1, 3))
     test = clicktest.CliRunner()
     result = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     restored_indices = sorted(curator.get_indices(self.client))
     self.assertEqual(indices, restored_indices)
     # The test runs so fast that it tries to execute the cleanup step
     # and delete the repository before Elasticsearch is actually ready
     time.sleep(0.5)
Пример #4
0
 def test_plain(self):
     self.write_config(self.args['configfile'],
                       testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
                       testvars.create_index.format('testing'))
     self.assertEqual([], curator.get_indices(self.client))
     test = clicktest.CliRunner()
     result = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     self.assertEqual(['testing'], curator.get_indices(self.client))
Пример #5
0
 def test_restore_wildcard(self):
     indices = []
     my_indices = []
     wildcard = ['my_*']
     for i in range(1,4):
         for prefix in ['my_', 'not_my_']:
             self.add_docs('{0}index{1}'.format(prefix, i))
             indices.append('{0}index{1}'.format(prefix, i))
             if prefix == 'my_':
                 my_indices.append('{0}index{1}'.format(prefix, i))
     snap_name = 'snapshot1'
     self.create_snapshot(snap_name, ','.join(indices))
     snapshot = curator.get_snapshot(
                 self.client, self.args['repository'], '_all'
                )
     self.assertEqual(1, len(snapshot['snapshots']))
     self.client.indices.delete(','.join(indices))
     self.assertEqual([], curator.get_indices(self.client))
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.restore_snapshot_proto.format(
             self.args['repository'],
             snap_name,
             wildcard,
             False,
             False,
             True,
             False,
             ' ',
             ' ',
             ' ',
             True,
             False,
             301,
             1,
             3
         )
     )
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     restored_indices = sorted(curator.get_indices(self.client))
     self.assertEqual(my_indices, restored_indices)
     # The test runs so fast that it tries to execute the cleanup step
     # and delete the repository before Elasticsearch is actually ready
     time.sleep(0.5)
Пример #6
0
 def test_restore_with_rename(self):
     indices = []
     for i in range(1,4):
         self.add_docs('my_index{0}'.format(i))
         indices.append('my_index{0}'.format(i))
     snap_name = 'snapshot1'
     self.create_snapshot(snap_name, ','.join(indices))
     snapshot = curator.get_snapshot(
                 self.client, self.args['repository'], '_all'
                )
     time.sleep(1)
     self.assertEqual(1, len(snapshot['snapshots']))
     self.client.indices.delete(','.join(indices))
     self.assertEqual([], curator.get_indices(self.client))
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.restore_snapshot_proto.format(
             self.args['repository'],
             snap_name,
             indices,
             False,
             False,
             True,
             False,
             'my_index(.+)',
             'new_index$1',
             ' ',
             True,
             False,
             301,
             1,
             3,
         )
     )
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     time.sleep(1)
     restored_indices = sorted(curator.get_indices(self.client))
     self.assertEqual(
         ['new_index1', 'new_index2', 'new_index3'],
         restored_indices
     )
     # The test runs so fast that it tries to execute the cleanup step
     # and delete the repository before Elasticsearch is actually ready
     time.sleep(1)
Пример #7
0
 def test_with_date_math(self):
     self.write_config(self.args['configfile'],
                       testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
                       testvars.create_index.format('<testing-{now/d}>'))
     self.assertEqual([], curator.get_indices(self.client))
     name = curator.parse_date_pattern('testing-%Y.%m.%d')
     test = clicktest.CliRunner()
     result = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     self.assertEqual([name], curator.get_indices(self.client))
Пример #8
0
 def test_plain(self):
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.create_index.format('testing'))
     self.assertEqual([], curator.get_indices(self.client))
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     self.assertEqual(['testing'], curator.get_indices(self.client))
Пример #9
0
 def test_shrink_with_extras(self):
     suffix = '-shrunken'
     allocation_type = 'exclude'
     key = '_name'
     value = 'not_this_node'
     self.builder(
         with_extra_settings.format(
             'DETERMINISTIC',
             'permit_masters',
             'True',
             1,
             0,
             '',
             suffix,
             'False',
             'index.codec',
             'best_compression',
             dict(),
             allocation_type,
             key,
             value
         )
     )
     indices = curator.get_indices(self.client)
     self.assertEqual(2, len(indices)) # Should only have `my_index-shrunken`
     settings = self.client.indices.get_settings()
     self.assertTrue(settings[self.target]['settings']['index']['routing']['allocation'][allocation_type][key] == value)
     self.assertTrue(settings[self.idx]['settings']['index']['routing']['allocation']['require']['_name'] == '')
     self.assertEqual(settings[self.target]['settings']['index']['codec'], 'best_compression')
     self.assertTrue(self.client.indices.exists_alias(index=self.target, name='my_alias'))
Пример #10
0
    def test_shrink_explicit_shard_filter(self):
        self.create_index('my_invalid_shrink_index', shards=3)
        self.create_index('my_valid_shrink_index', shards=5)

        suffix = '-shrunken'
        self.builder(
            shrink_filter_by_shards.format(
                'DETERMINISTIC',
                'permit_masters',
                'True',
                1,
                0,
                '',
                suffix,
                'True',
                'False',
                5,
                'greater_than_or_equal'
            )
        )

        indices = curator.get_indices(self.client)
        self.assertEqual(3, len(indices)) 
        self.assertTrue('my_invalid_shrink_index-shrunken' not in indices)
        self.assertTrue('my_valid_shrink_index-shrunken' in indices)
        self.assertTrue('my_index-shrunken' not in indices)
Пример #11
0
 def test_shrink_with_extras(self):
     suffix = '-shrunken'
     allocation_type = 'exclude'
     key = '_name'
     value = 'not_this_node'
     self.builder(
         with_extra_settings.format(
             'DETERMINISTIC',
             'permit_masters',
             'True',
             1,
             0,
             '',
             suffix,
             'False',
             'index.codec',
             'best_compression',
             dict(),
             allocation_type,
             key,
             value
         )
     )
     indices = curator.get_indices(self.client)
     self.assertEqual(2, len(indices)) # Should only have `my_index-shrunken`
     settings = self.client.indices.get_settings()
     self.assertTrue(settings[self.target]['settings']['index']['routing']['allocation'][allocation_type][key] == value)
     self.assertTrue(settings[self.idx]['settings']['index']['routing']['allocation']['require']['_name'] == '')
     self.assertEqual(settings[self.target]['settings']['index']['codec'], 'best_compression')
     self.assertTrue(self.client.indices.exists_alias(index=self.target, name='my_alias'))
Пример #12
0
 def test_issue_826(self):
     client = Mock()
     client.info.return_value = {'version': {'number': '2.4.2'}}
     client.indices.get_settings.return_value = testvars.settings_two
     client.indices.exists.return_value = True
     self.assertEqual(['.security', 'index-2016.03.03', 'index-2016.03.04'],
                      sorted(curator.get_indices(client)))
Пример #13
0
 def test_with_date(self):
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.create_index.format('testing-%Y.%m.%d'))
     self.assertEqual([], curator.get_indices(self.client))
     name = curator.parse_date_pattern('testing-%Y.%m.%d')
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     self.assertEqual([name], curator.get_indices(self.client))
Пример #14
0
 def test_retention_from_name_days_ignore_failed_match(self):
     # Test extraction of unit_count from index name
     # Create indices for 10 days with retention time of 5 days in index name
     # Create indices for 10 days with no retention time in index name
     # Expected: 5 oldest indices are deleted, 5 remain - 10 indices without retention time are ignored and remain
     self.args['prefix'] = 'logstash_5_'
     self.create_indices(10)
     self.args['prefix'] = 'logstash_'
     self.create_indices(10)
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
                       testvars.delete_pattern_proto.format(
                           'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 30, '_([0-9]+)_', ' ', ' ', ' '
                       )
                       )
     test = clicktest.CliRunner()
     result = test.invoke(
         curator.cli,
         [
             '--config', self.args['configfile'],
             self.args['actionfile']
         ],
     )
     self.assertEquals(15, len(curator.get_indices(self.client)))
Пример #15
0
 def test_delete_in_period(self):
     # filtertype: {0}
     # source: {1}
     # range_from: {2}
     # range_to: {3}
     # timestring: {4}
     # unit: {5}
     # field: {6}
     # stats_result: {7}
     # intersect: {8}
     # epoch: {9}
     # week_starts_on: {10}
     self.create_indices(10)
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.delete_period_proto.format(
             'period', 'name', '-5', '-1', "'%Y.%m.%d'", 'days',
             ' ', ' ', ' ', ' ', 'monday'
         )
     )
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     self.assertEqual(0, result.exit_code)
     self.assertEquals(5, len(curator.get_indices(self.client)))
Пример #16
0
    def test_retention_from_name_days_keep_exclude_false_after_failed_match(
            self):
        # Test extraction of unit_count from index name and confirm correct
        # behavior after a failed regex match with no fallback time - see gh issue 1206
        # Create indices for 30 days with retention time of 5 days in index name
        #
        # Create indices for 10 days with no retention time in index name
        # that alphabetically sort before the 10 with retention time
        #
        # Create indices for 10 days with no retention time in index name
        # that sort after the 10 with retention time

        # Expected: 45 oldest matching indices are deleted, 5 matching indices remain
        # 20 indices without retention time are ignored and remain
        # overall 25 indices should remain
        self.args['prefix'] = 'logstash-aanomatch-'
        self.create_indices(10)
        self.args['prefix'] = 'logstash-match-5-'
        self.create_indices(30)
        self.args['prefix'] = 'logstash-zznomatch-'
        self.create_indices(10)
        self.write_config(self.args['configfile'],
                          testvars.client_config.format(host, port))
        self.write_config(
            self.args['actionfile'],
            testvars.delete_pattern_proto.format(
                'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', -1,
                r'logstash-\w+-([0-9]+)-[0-9]{4}\.[0-9]{2}\.[0-9]{2}', ' ',
                ' ', ' '))
        test = clicktest.CliRunner()
        _ = test.invoke(
            curator.cli,
            ['--config', self.args['configfile'], self.args['actionfile']],
        )
        self.assertEquals(25, len(curator.get_indices(self.client)))
def main(argv):
    parser = OptionParser()
    parser.add_option("-p", "--prefix", dest="prefix", action="store", help="prefix to indices filter ")
    parser.add_option("-r", "--repository", dest="repository", action="store", help="repository name")

    (opts, args) = parser.parse_args()
    if not opts.repository:  # if repository is not given
        print 'repository not given'
        usage()
        sys.exit()
    if not opts.prefix:  # if repository is not given
        print 'prefix not given'
        usage()
        sys.exit()

    client = elasticsearch.Elasticsearch([
        {'host': 'localhost', 'port': 9200}
    ])

    indices_list = curator.get_indices(client)
    _filter = curator.build_filter(kindOf='prefix', value=opts.prefix)
    working_list = curator.apply_filter(indices_list, **_filter)


    res = curator.create_snapshot(client, indices=working_list, name=None, prefix='curator-',
                                  repository=opts.repository, ignore_unavailable=True, include_global_state=False,
                                  partial=False, wait_for_completion=True, request_timeout=21600,
                                  skip_repo_validation=False)

    print res
Пример #18
0
 def test_cli_snapshot_indices(self):
     self.create_indices(5)
     self.create_repository()
     snap_name = 'snapshot1'
     indices = curator.get_indices(self.client)
     expected = sorted(indices, reverse=True)[:4]
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--logfile', os.devnull,
                     '--host', host,
                     '--port', str(port),
                     'snapshot',
                     '--repository', self.args['repository'],
                     '--name', snap_name,
                     'indices',
                     '--all-indices',
                 ],
                 obj={"filters":[]})
     snapshot = curator.get_snapshot(
                 self.client, self.args['repository'], '_all'
                )
     self.assertEqual(1, len(snapshot['snapshots']))
     self.assertEqual(snap_name, snapshot['snapshots'][0]['snapshot'])
Пример #19
0
 def test_retention_from_name_days_ignore_failed_match(self):
     # Test extraction of unit_count from index name
     # Create indices for 10 days with retention time of 5 days in index name
     # Create indices for 10 days with no retention time in index name
     # Expected: 5 oldest indices are deleted, 5 remain - 10 indices without retention time are ignored and remain
     self.args['prefix'] = 'logstash_5_'
     self.create_indices(10)
     self.args['prefix'] = 'logstash_'
     self.create_indices(10)
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
                       testvars.delete_pattern_proto.format(
                           'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 30, '_([0-9]+)_', ' ', ' ', ' '
                       )
                       )
     test = clicktest.CliRunner()
     result = test.invoke(
         curator.cli,
         [
             '--config', self.args['configfile'],
             self.args['actionfile']
         ],
     )
     self.assertEquals(15, len(curator.get_indices(self.client)))
Пример #20
0
 def test_delete_indices_by_space_dry_run(self):
     for i in range(1, 10):
         self.client.create(
             index="index" + str(i),
             doc_type='log',
             body={'message': 'TEST DOCUMENT'},
         )
     test = clicktest.CliRunner()
     result = test.invoke(curator.cli, [
         '--dry-run',
         '--host',
         host,
         '--port',
         str(port),
         'delete',
         '--disk-space',
         '0.0000001',
         'indices',
         '--all-indices',
     ],
                          obj={"filters": []})
     l = curator.get_indices(self.client)
     self.assertEquals(9, len(l))
     output = sorted(result.output.splitlines(), reverse=True)
     # I tried doing a nested, double list comprehension here.
     # It works in the interpreter, but not here for some reason.
     output = [x.split(' ')[-1:] for x in output]
     output = [x[0] for x in output if x[0].startswith('index')]
     self.assertEqual(sorted(l, reverse=True), output)
Пример #21
0
 def test_positive(self):
     client = Mock()
     client.indices.get_settings.return_value = testvars.settings_two
     self.assertEqual(
         ['index-2016.03.03', 'index-2016.03.04'],
         sorted(curator.get_indices(client))
     )
Пример #22
0
 def test_cli_snapshot_indices(self):
     self.create_indices(5)
     self.create_repository()
     snap_name = 'snapshot1'
     indices = curator.get_indices(self.client)
     expected = sorted(indices, reverse=True)[:4]
     test = clicktest.CliRunner()
     result = test.invoke(curator.cli, [
         '--logfile',
         os.devnull,
         '--host',
         host,
         '--port',
         str(port),
         'snapshot',
         '--repository',
         self.args['repository'],
         '--name',
         snap_name,
         'indices',
         '--all-indices',
     ],
                          obj={"filters": []})
     snapshot = curator.get_snapshot(self.client, self.args['repository'],
                                     '_all')
     self.assertEqual(1, len(snapshot['snapshots']))
     self.assertEqual(snap_name, snapshot['snapshots'][0]['snapshot'])
Пример #23
0
 def test_delete_indices_by_space_dry_run_huge_list(self):
     for i in range(100, 150):
         self.client.create(
             index=
             "superlongindexnamebyanystandardyouchoosethisissillyhowbigcanthisgetbeforeitbreaks"
             + str(i),
             doc_type='log',
             body={'message': 'TEST DOCUMENT'},
         )
     l = curator.get_indices(self.client)
     self.assertEquals(50, len(l))
     test = clicktest.CliRunner()
     result = test.invoke(curator.cli, [
         '--dry-run',
         '--host',
         host,
         '--port',
         str(port),
         'delete',
         '--disk-space',
         '0.0000001',
         'indices',
         '--all-indices',
     ],
                          obj={"filters": []})
     output = sorted(result.output.splitlines(), reverse=True)
     # # I tried doing a nested, double list comprehension here.
     # # It works in the interpreter, but not here for some reason.
     output = [x.split(' ')[-1:] for x in output]
     output = [
         x[0] for x in output if x[0].startswith('superlongindexname')
     ]
     self.assertEqual(sorted(l, reverse=True), output)
Пример #24
0
 def test_delete_indices_by_space_dry_run_huge_list(self):
     for i in range(100,150):
         self.client.create(
             index="superlongindexnamebyanystandardyouchoosethisissillyhowbigcanthisgetbeforeitbreaks" + str(i), doc_type='log',
             body={'message':'TEST DOCUMENT'},
         )
     l = curator.get_indices(self.client)
     self.assertEquals(50, len(l))
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--dry-run',
                     '--host', host,
                     '--port', str(port),
                     'delete',
                     '--disk-space', '0.0000001',
                     'indices',
                     '--all-indices',
                 ],
                 obj={"filters":[]})
     output = sorted(result.output.splitlines(), reverse=True)
     # # I tried doing a nested, double list comprehension here.
     # # It works in the interpreter, but not here for some reason.
     output = [ x.split(' ')[-1:] for x in output ]
     output = [ x[0] for x in output if x[0].startswith('superlongindexname') ]
     self.assertEqual(sorted(l, reverse=True), output)
Пример #25
0
 def test_allow_ilm_indices_false(self):
     # ILM will not be added until 6.4
     if curator.get_version(self.client) < (6,4,0):
         self.assertTrue(True)
     else:
         self.create_indices(10)
         settings = {
             'index': {
                 'lifecycle': {
                     'name': 'mypolicy'
                 }
             }
         }
         self.client.indices.put_settings(index='_all', body=settings)
         self.write_config(
             self.args['configfile'], testvars.client_config.format(host, port))
         self.write_config(self.args['actionfile'],
             testvars.ilm_delete_proto.format(
                 'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' ', 'false'
             )
         )
         test = clicktest.CliRunner()
         _ = test.invoke(
             curator.cli,
             [ '--config', self.args['configfile'], self.args['actionfile'] ],
         )
         self.assertEquals(10, len(curator.get_indices(self.client)))
Пример #26
0
def show(client, **kwargs):
    """
    Show indices or snapshots matching supplied parameters and exit.
    
    :arg client: The Elasticsearch client connection
    :arg data_type: Either ``index`` or ``snapshot``
    :arg prefix: A string that comes before the datestamp in an index name.
        Can be empty. Wildcards acceptable.  Default is ``logstash-``.
    :arg suffix: A string that comes after the datestamp of an index name.
        Can be empty. Wildcards acceptable.  Default is empty, ``''``.
    :arg repository: The Elasticsearch snapshot repository to use (only with
        snapshots)
    :arg snapshot_prefix: Override the default with this value. Defaults to
        ``curator-``
    """
    if kwargs['show_indices']:
        for index_name in curator.get_indices(client,
                                              prefix=kwargs['prefix'],
                                              suffix=kwargs['suffix']):
            print('{0}'.format(index_name))
        sys.exit(0)
    elif kwargs['show_snapshots']:
        for snapshot in curator.get_snaplist(
                client,
                kwargs['repository'],
                snapshot_prefix=kwargs['snapshot_prefix']):
            print('{0}'.format(snapshot))
        sys.exit(0)
Пример #27
0
 def test_positive(self):
     client = Mock()
     client.indices.get_settings.return_value = testvars.settings_two
     self.assertEqual(
         ['index-2016.03.03', 'index-2016.03.04'],
         sorted(curator.get_indices(client))
     )
Пример #28
0
 def test_do_something_with_int_value(self):
     self.create_indices(10)
     evar = random_envvar(8)
     os.environ[evar] = "1234"
     dollar = '${' + evar + '}'
     self.write_config(
         self.args['configfile'],
         testvars.client_config_envvars.format(host, port, dollar)
     )
     cfg = curator.get_yaml(self.args['configfile'])
     self.assertEqual(
         cfg['client']['timeout'],
         os.environ.get(evar)
     )
     self.write_config(self.args['actionfile'],
         testvars.delete_proto.format(
             'age', 'name', 'older', '\'%Y.%m.%d\'', 'days', 5, ' ', ' ', ' '
         )
     )
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     self.assertEquals(5, len(curator.get_indices(self.client)))
     del os.environ[evar]
Пример #29
0
 def test_delete_in_period(self):
     # filtertype: {0}
     # source: {1}
     # range_from: {2}
     # range_to: {3}
     # timestring: {4}
     # unit: {5}
     # field: {6}
     # stats_result: {7}
     # intersect: {8}
     # epoch: {9}
     # week_starts_on: {10}
     self.create_indices(10)
     self.write_config(self.args['configfile'],
                       testvars.client_config.format(host, port))
     self.write_config(
         self.args['actionfile'],
         testvars.delete_period_proto.format('period', 'name', '-5', '-1',
                                             "'%Y.%m.%d'", 'days', ' ', ' ',
                                             ' ', ' ', 'monday'))
     test = clicktest.CliRunner()
     result = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     self.assertEqual(0, result.exit_code)
     self.assertEquals(5, len(curator.get_indices(self.client)))
Пример #30
0
 def test_delete_indices_by_space_dry_run(self):
     for i in range(1,10):
         self.client.create(
             index="index" + str(i), doc_type='log',
             body={'message':'TEST DOCUMENT'},
         )
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--dry-run',
                     '--host', host,
                     '--port', str(port),
                     'delete',
                     '--disk-space', '0.0000001',
                     'indices',
                     '--all-indices',
                 ],
                 obj={"filters":[]})
     l = curator.get_indices(self.client)
     self.assertEquals(9, len(l))
     output = sorted(result.output.splitlines(), reverse=True)
     # I tried doing a nested, double list comprehension here.
     # It works in the interpreter, but not here for some reason.
     output = [ x.split(' ')[-1:] for x in output ]
     output = [ x[0] for x in output if x[0].startswith('index') ]
     self.assertEqual(sorted(l, reverse=True), output)
Пример #31
0
 def test_positive(self):
     client = Mock()
     client.indices.get_settings.return_value = testvars.settings_two
     client.info.return_value = {'version': {'number': '5.0.0'} }
     self.assertEqual(
         ['index-2016.03.03', 'index-2016.03.04'],
         sorted(curator.get_indices(client))
     )
Пример #32
0
 def test_positive(self):
     client = Mock()
     client.indices.get_settings.return_value = testvars.settings_two
     client.info.return_value = {'version': {'number': '2.4.1'} }
     self.assertEqual(
         ['index-2016.03.03', 'index-2016.03.04'],
         sorted(curator.get_indices(client))
     )
Пример #33
0
 def test_restore_with_rename(self):
     indices = []
     for i in range(1,4):
         self.add_docs('my_index{0}'.format(i))
         indices.append('my_index{0}'.format(i))
     snap_name = 'snapshot1'
     self.create_snapshot(snap_name, ','.join(indices))
     snapshot = curator.get_snapshot(
                 self.client, self.args['repository'], '_all'
                )
     self.assertEqual(1, len(snapshot['snapshots']))
     self.client.indices.delete(','.join(indices))
     self.assertEqual([], curator.get_indices(self.client))
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.restore_snapshot_proto.format(
             self.args['repository'],
             snap_name,
             indices,
             False,
             False,
             True,
             False,
             'my_index(.+)',
             'new_index$1',
             ' ',
             True,
             False,
             301
         )
     )
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     restored_indices = sorted(curator.get_indices(self.client))
     self.assertEqual(
         ['new_index1', 'new_index2', 'new_index3'], 
         restored_indices
     )
Пример #34
0
 def test_shrink(self):
     suffix = '-shrunken'
     self.builder(
         shrink.format('DETERMINISTIC', 'permit_masters', 'True', 1, 0, '',
                       suffix, 'True'))
     indices = curator.get_indices(self.client)
     self.assertEqual(1,
                      len(indices))  # Should only have `my_index-shrunken`
     self.assertEqual(indices[0], self.target)
Пример #35
0
 def test_delete_indices_huge_list(self):
     self.create_indices(365)
     pre = curator.get_indices(self.client)
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--logfile', os.devnull,
                     '--host', host,
                     '--port', str(port),
                     'delete',
                     'indices',
                     '--all-indices',
                     '--exclude', pre[0],
                 ],
                 obj={"filters":[]})
     post = curator.get_indices(self.client)
     self.assertEquals(1, len(post))
Пример #36
0
 def test_delete_indices_huge_list(self):
     self.create_indices(365)
     pre = curator.get_indices(self.client)
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--logfile', os.devnull,
                     '--host', host,
                     '--port', str(port),
                     'delete',
                     'indices',
                     '--all-indices',
                     '--exclude', pre[0],
                 ],
                 obj={"filters":[]})
     post = curator.get_indices(self.client)
     self.assertEquals(1, len(post))
Пример #37
0
 def test_issue_826(self):
     client = Mock()
     client.info.return_value = {'version': {'number': '2.4.2'} }
     client.indices.get_settings.return_value = testvars.settings_two
     client.indices.exists.return_value = True
     self.assertEqual(
         ['.security', 'index-2016.03.03', 'index-2016.03.04'],
         sorted(curator.get_indices(client))
     )
Пример #38
0
 def test_name_older_than_now_cli(self):
     self.create_indices(10)
     args = self.get_runner_args()
     args += [
         '--config', self.args['configfile'],
         'delete_indices',
         '--filter_list', '{"filtertype":"age","source":"name","direction":"older","timestring":"%Y.%m.%d","unit":"days","unit_count":5}',
     ]
     self.assertEqual(0, self.run_subprocess(args, logname='TestCLIDeleteIndices.test_name_older_than_now_cli'))
     self.assertEquals(5, len(curator.get_indices(self.client)))
Пример #39
0
 def test_permit_masters_false(self):
     suffix = '-shrunken'
     self.builder(
         no_permit_masters.format('DETERMINISTIC', 1, 0, '', suffix,
                                  'True'))
     indices = curator.get_indices(self.client)
     self.assertEqual(1,
                      len(indices))  # Should only have `my_index-shrunken`
     self.assertEqual(indices[0], self.idx)
     self.assertEqual(self.result.exit_code, 1)
Пример #40
0
 def test_name_older_than_now_cli(self):
     self.create_indices(10)
     args = self.get_runner_args()
     args += [
         '--config', self.args['configfile'],
         'delete_indices',
         '--filter_list', '{"filtertype":"age","source":"name","direction":"older","timestring":"%Y.%m.%d","unit":"days","unit_count":5}',
     ]
     self.assertEqual(0, self.run_subprocess(args, logname='TestCLIDeleteIndices.test_name_older_than_now_cli'))
     self.assertEquals(5, len(curator.get_indices(self.client)))
Пример #41
0
 def test_filter_by_array_of_aliases(self):
     alias = 'testalias'
     self.write_config(self.args['configfile'],
                       testvars.client_config.format(host, port))
     self.write_config(
         self.args['actionfile'],
         testvars.filter_by_alias.format(' [ testalias, foo ]', False))
     self.create_index('my_index')
     self.create_index('dummy')
     self.client.indices.put_alias(index='dummy', name=alias)
     test = clicktest.CliRunner()
     _ = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     ver = curator.get_version(self.client)
     if ver >= (5, 5, 0):
         self.assertEquals(2, len(curator.get_indices(self.client)))
     else:
         self.assertEquals(1, len(curator.get_indices(self.client)))
Пример #42
0
 def test_shrink_with_copy_alias(self):
     suffix = '-shrunken'
     self.builder(
         copy_aliases.format('DETERMINISTIC', 'permit_masters', 'True', 1,
                             0, '', suffix, 'True', 'True'))
     indices = curator.get_indices(self.client)
     self.assertEqual(1, len(indices))  # Should have `my_index-shrunken`
     self.assertEqual(indices[0], self.target)
     self.assertTrue(
         self.client.indices.exists_alias(index=self.target,
                                          name=self.alias))
Пример #43
0
 def test_extra_option(self):
     self.write_config(self.args['configfile'],
                       testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
                       testvars.bad_option_proto_test.format('rollover'))
     test = clicktest.CliRunner()
     _ = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     self.assertEqual([], curator.get_indices(self.client))
     self.assertEqual(-1, _.exit_code)
Пример #44
0
 def test_filter_by_array_of_aliases(self):
     alias = 'testalias'
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.filter_by_alias.format(' [ testalias, foo ]', False))
     self.create_index('my_index')
     self.create_index('dummy')
     self.client.indices.put_alias(index='dummy', name=alias)
     test = clicktest.CliRunner()
     _ = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     ver = curator.get_version(self.client)
     if ver >= (5,5,0):
         self.assertEquals(2, len(curator.get_indices(self.client)))
     else:
         self.assertEquals(1, len(curator.get_indices(self.client)))
Пример #45
0
 def test_logging_with_debug_flag(self):
     self.create_indices(10)
     indices = curator.get_indices(self.client)
     expected = sorted(indices, reverse=True)[:4]
     test = clicktest.CliRunner()
     result = test.invoke(curator.cli, [
         '--debug', '--logfile', os.devnull, '--host', host, '--port',
         str(port), 'show', 'indices', '--newer-than', '5', '--timestring',
         '%Y.%m.%d', '--time-unit', 'days'
     ],
                          obj={"filters": []})
     output = sorted(result.output.splitlines(), reverse=True)[:4]
     self.assertEqual(expected, output)
def filter_indices(module, client):
    prefix = module.params.get('prefix')
    older_than = module.params.get('older_than')
    time_unit = module.params.get('time_unit')
    timestring = module.params.get('timestring')

    indices = curator.get_indices(client)

    _filter = curator.build_filter(kindOf='prefix', value=prefix)
    indices = curator.apply_filter(indices, **_filter)

    _filter = curator.build_filter(kindOf='older_than', value=int(older_than), time_unit=time_unit, timestring=timestring)
    return curator.apply_filter(indices, **_filter)
Пример #47
0
 def test_cli_show_indices_older_than_zero(self):
     self.create_indices(10)
     indices = curator.get_indices(self.client)
     expected = sorted(indices, reverse=True)
     test = clicktest.CliRunner()
     result = test.invoke(curator.cli, [
         '--logfile', os.devnull, '--host', host, '--port',
         str(port), 'show', 'indices', '--older-than', '0', '--timestring',
         '%Y.%m.%d', '--time-unit', 'days'
     ],
                          obj={"filters": []})
     output = sorted(result.output.splitlines(), reverse=True)
     self.assertEqual(expected, output)
Пример #48
0
def main():
    global logger
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", default="dopey.yaml", help="yaml config")
    parser.add_argument("-l", default="-", help="log file")
    parser.add_argument("--level", default="info")
    args = parser.parse_args()

    config = yaml.load(open(args.c))

    initlog(level=args.level, log=config["l"] if "log" in config else args.l)
    logger = logging.getLogger("dopey")

    eshosts = config.get("esclient")
    logger.debug(eshosts)
    if eshosts is not None:
        esclient = elasticsearch.Elasticsearch(eshosts, timeout=300)
    else:
        esclient = elasticsearch.Elasticsearch(timeout=300)

    all_indices = curator.get_indices(esclient)
    logger.debug("all_indices: {}".format(all_indices))
    if all_indices is False:
        raise Exception("could not get indices")

    process_threads = []
    for index_prefix, index_config in config.get("indices").items():
        t = Thread(target=process,
                   args=(esclient, all_indices, index_prefix, index_config))
        t.start()
        process_threads.append(t)

    for t in process_threads:
        t.join()

    not_dealt = list(set(all_indices).difference(_dealt))
    dopey_summary.add(
        json.dumps(
            {
                "not_dealt": not_dealt,
                "delete": _delete,
                "close": _close,
                "optimize": _optimize
            },
            indent=2))
    sumary_config = config.get("sumary")
    for action, kargs in sumary_config.items():
        if kargs:
            getattr(dopey_summary, action)(**kargs)
        else:
            getattr(dopey_summary, action)()
Пример #49
0
 def test_action_disabled(self):
     self.create_indices(10)
     self.write_config(self.args['configfile'],
                       testvars.client_config.format(host, port))
     self.write_config(
         self.args['actionfile'],
         testvars.disabled_proto.format('close', 'delete_indices'))
     test = clicktest.CliRunner()
     result = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     self.assertEquals(0, len(curator.get_indices(self.client)))
     self.assertEqual(0, result.exit_code)
Пример #50
0
    def test_show_indices_with_suffix_and_no_prefix(self):
        client = Mock()
        client.indices.get_settings.return_value = {
            '2014.01.03-suffix': True,
            '2014.01.02-suffix': True,
            '2014.01.01-suffix': True
        }
        indices = curator.get_indices(client, prefix='', suffix='-suffix')

        self.assertEquals([
            '2014.01.01-suffix',
            '2014.01.02-suffix',
            '2014.01.03-suffix',
        ], indices)
Пример #51
0
 def test_with_extra_settings(self):
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.create_index_with_extra_settings.format('testing'))
     self.assertEqual([], curator.get_indices(self.client))
     test = clicktest.CliRunner()
     _ = test.invoke(
         curator.cli,
         ['--config', self.args['configfile'], self.args['actionfile']],
     )
     ilo = curator.IndexList(self.client)
     self.assertEqual(['testing'], ilo.indices)
     self.assertEqual(ilo.index_info['testing']['number_of_shards'], '1')
     self.assertEqual(ilo.index_info['testing']['number_of_replicas'], '0')
Пример #52
0
 def test_extra_option(self):
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.bad_option_proto_test.format('rollover'))
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     self.assertEqual([], curator.get_indices(self.client))
     self.assertEqual(-1, result.exit_code)
Пример #53
0
 def test_action_disabled(self):
     self.create_indices(10)
     self.write_config(
         self.args['configfile'], testvars.client_config.format(host, port))
     self.write_config(self.args['actionfile'],
         testvars.disabled_proto.format('close', 'delete_indices'))
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--config', self.args['configfile'],
                     self.args['actionfile']
                 ],
                 )
     self.assertEquals(0, len(curator.get_indices(self.client)))
     self.assertEqual(0, result.exit_code)
Пример #54
0
 def test_permit_masters_false(self):
     suffix = '-shrunken'
     self.builder(
         no_permit_masters.format(
             'DETERMINISTIC',
             1,
             0,
             '',
             suffix,
             'True'
         )
     )
     indices = curator.get_indices(self.client)
     self.assertEqual(1, len(indices)) # Should only have `my_index-shrunken`
     self.assertEqual(indices[0], self.idx)
     self.assertEqual(self.result.exit_code, 1)
Пример #55
0
    def test_show_indices(self):
        client = Mock()
        client.indices.get_settings.return_value = {
            'prefix-2014.01.03': True,
            'prefix-2014.01.02': True,
            'prefix-2014.01.01': True
        }
        indices = curator.get_indices(client, prefix='prefix-')

        self.assertEquals([
                'prefix-2014.01.01',
                'prefix-2014.01.02',
                'prefix-2014.01.03',
            ],
            indices
        )
Пример #56
0
    def test_show_indices_with_suffix_and_no_prefix(self):
        client = Mock()
        client.indices.get_settings.return_value = {
            '2014.01.03-suffix': True,
            '2014.01.02-suffix': True,
            '2014.01.01-suffix': True
        }
        indices = curator.get_indices(client, prefix='', suffix='-suffix')

        self.assertEquals([
                '2014.01.01-suffix',
                '2014.01.02-suffix',
                '2014.01.03-suffix',
            ],
            indices
        )
Пример #57
0
    def test_reindex_empty_list(self):
        wait_interval = 1
        max_wait = 3
        source = 'my_source'
        dest = 'my_dest'
        expected = '.tasks'

        self.write_config(
            self.args['configfile'], testvars.client_config.format(host, port))
        self.write_config(self.args['actionfile'],
            testvars.reindex.format(wait_interval, max_wait, source, dest))
        test = clicktest.CliRunner()
        _ = test.invoke(
            curator.cli,
            ['--config', self.args['configfile'], self.args['actionfile']],
        )
        self.assertEqual(expected, curator.get_indices(self.client)[0])
Пример #58
0
 def test_index_selection_only_timestamp_filter(self):
     self.create_indices(10)
     indices = curator.get_indices(self.client)
     # expected = sorted(indices, reverse=True)[:4]
     test = clicktest.CliRunner()
     result = test.invoke(
                 curator.cli,
                 [
                     '--logfile', os.devnull,
                     '--host', host,
                     '--port', str(port),
                     'close',
                     'indices',
                     '--timestring', '%Y.%m.%d',
                 ],
                 obj={"filters":[]})
     self.assertEqual(0, result.exit_code)