def main(argv): parser = OptionParser() parser.add_option("-r", "--repository", dest="repository", action="store", help="repository name") parser.add_option("-t", "--timestring", dest="timestring", action="store", help="timestring of the datestamp in an index name") parser.add_option("-o", "--older_than", dest="older_than", action="store", help="older than ( days )") parser.add_option("-i", "--ignore_if_newer_not_exist", dest="ignore_if_newer_not_exist", action="store", help="don't perform deletion if there are no snapshots newer than this range") parser.add_option("-s", "--host", dest="host", action="store", help="host, default : localhost") parser.add_option("-p", "--port", dest="port", action="store", help="port, default : 9200") #parser.add_option("-m", "--timeout", dest="timeout", action="store", help="timeout (sec), default : 21600") host = 'localhost' port = 9200 timeout = 21600 (opts, args) = parser.parse_args() if not opts.repository: # if repository is not given print 'repository not given' usage() sys.exit() if not opts.older_than: # if repository is not given print 'older_then not given' usage() sys.exit() if not opts.timestring: # if repository is not given print 'timestring not given' usage() sys.exit() if not opts.ignore_if_newer_not_exist: # if repository is not given print 'ignore_if_newer_not_exist not given' usage() sys.exit() if opts.host: # if host is given host = opts.host if opts.port: # if port is given host = int(opts.port) #if opts.timeout: # if port is given # host = int(opts.timeout) client = elasticsearch.Elasticsearch([ {'host': host, 'port': port, 'timeout' : timeout} ]) snapshots_list = curator.get_snapshots(client, opts.repository) _filter = curator.build_filter(kindOf='newer_than', value=int(opts.ignore_if_newer_not_exist), time_unit='days', timestring=opts.timestring) snapshots_verification_list = curator.apply_filter(snapshots_list, **_filter) if not snapshots_verification_list: print 'no new snapshots available, existing' sys.exit() _filter = curator.build_filter(kindOf='older_than', value=int(opts.older_than), time_unit='days', timestring=opts.timestring) snapshots_to_delete_list = curator.apply_filter(snapshots_list, **_filter) print ', '.join(snapshots_to_delete_list) for snapshot in snapshots_to_delete_list: print 'deleting snapshot ' + snapshot + ' ...' res = curator.delete_snapshot(client,snapshot, opts.repository) if not res: print 'error! could not delete snapshot ' + snapshot
def filter_indices(module, client): prefix = module.params.get('prefix') older_than = module.params.get('older_than') time_unit = module.params.get('time_unit') timestring = module.params.get('timestring') indices = curator.get_indices(client) _filter = curator.build_filter(kindOf='prefix', value=prefix) indices = curator.apply_filter(indices, **_filter) _filter = curator.build_filter(kindOf='older_than', value=int(older_than), time_unit=time_unit, timestring=timestring) return curator.apply_filter(indices, **_filter)
def main(argv): parser = OptionParser() parser.add_option("-p", "--prefix", dest="prefix", action="store", help="prefix to indices filter ") parser.add_option("-r", "--repository", dest="repository", action="store", help="repository name") (opts, args) = parser.parse_args() if not opts.repository: # if repository is not given print 'repository not given' usage() sys.exit() if not opts.prefix: # if repository is not given print 'prefix not given' usage() sys.exit() client = elasticsearch.Elasticsearch([ {'host': 'localhost', 'port': 9200} ]) indices_list = curator.get_indices(client) _filter = curator.build_filter(kindOf='prefix', value=opts.prefix) working_list = curator.apply_filter(indices_list, **_filter) res = curator.create_snapshot(client, indices=working_list, name=None, prefix='curator-', repository=opts.repository, ignore_unavailable=True, include_global_state=False, partial=False, wait_for_completion=True, request_timeout=21600, skip_repo_validation=False) print res
def indices_date_filter(filter_type='older_than', filter_value=30, time_unit='days', timestring='%Y.%m.%d'): """ valid filter types are older_than and newer_than time unit could be days, months, or years. """ return curator.build_filter(kindOf=filter_type, value=filter_value, time_unit=time_unit, timestring=timestring)
def rotate_logs(self): """ We will delete indices older than configured maximum age. """ if not self.is_connected == CONNECTED: if not self.open(): self.next_logs_rotation = time.time() + 600 logger.info( "[elastic-logs] log rotation failed, next log rotation at %s " % time.asctime(time.localtime(self.next_logs_rotation)) ) return logger.info("[elastic-logs] rotating logs ...") now = time.time() today = date.today() today0000 = datetime(today.year, today.month, today.day, 0, 0, 0) today0005 = datetime(today.year, today.month, today.day, 0, 5, 0) oldest = today0000 - timedelta(days=self.max_logs_age) if now < time.mktime(today0005.timetuple()): next_rotation = today0005 else: next_rotation = today0005 + timedelta(days=1) try: indices = curator.get_indices(self.es) filter_list = [] filter_list.append(curator.build_filter(kindOf="prefix", value=self.index_prefix)) filter_list.append( curator.build_filter( kindOf="older_than", value=self.max_logs_age, time_unit="days", timestring="%Y.%m.%d" ) ) working_list = indices for filter in filter_list: working_list = curator.apply_filter(working_list, **filter) curator.delete(self.es, working_list) logger.info("[elastic-logs] removed %d logs older than %s days.", working_list, self.max_logs_age) except Exception, exp: logger.error("[elastic-logs] Exception while rotating indices: %s", str(exp))
def prune_es_indices(): """Prune ES indices older than the age defined in settings.PRUNE_OLDER_THAN.""" curation_params = [ {"prefix": "events_", "time_string": "%Y-%m-%d"}, {"prefix": "logstash-", "time_string": "%Y.%m.%d"}, {"prefix": "goldstone-", "time_string": "%Y.%m.%d"}, {"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"}, {"prefix": "api_stats-", "time_string": "%Y.%m.%d"}, {"prefix": "internal-", "time_string": "%Y.%m.%d"}, ] client = es_conn() all_indices = curator.get_indices(client) deleted_indices = [] working_list = all_indices # we'll whittle this down with filters for index_set in curation_params: # filter on our prefix name_filter = curator.build_filter( kindOf='prefix', value=index_set['prefix']) # filter on the datestring age_filter = curator.build_filter( kindOf='older_than', time_unit='days', timestring=index_set['time_string'], value=settings.PRUNE_OLDER_THAN) # apply the filters to get the final list of indices to delete working_list = curator.apply_filter(working_list, **name_filter) working_list = curator.apply_filter(working_list, **age_filter) if working_list is not None and len(working_list) > 0: try: curator.delete_indices(client, working_list) deleted_indices = deleted_indices + working_list except Exception: logger.exception("curator.delete_indices raised an exception") working_list = all_indices # reset for the next loop iteration return deleted_indices
def select_indices(client, prefix, timestring, time_unit, older_than_days=None, newer_than_days=None): """ time_unit: One of hours, days, weeks, months, NO years!!! older_than_days, newer_than_days: Value of older_than_days or newer_than_days should be > 0 If older_than_days and newer_than_days are both set, newer_than_days should be greater than older_than_days older_than_days = n will filters indices n days before today, newer_than_days = m will filters indices from m days before today to today """ import curator indices = curator.get_indices(client) filter_list = [] index_regex = prefix + curator.get_date_regex(timestring) filter_list.append(curator.build_filter(kindOf='regex', value=index_regex)) if older_than_days: filter_list.append( curator.build_filter(kindOf='older_than', value=older_than_days, time_unit=time_unit, timestring=timestring)) if newer_than_days: filter_list.append( curator.build_filter(kindOf='newer_than', value=newer_than_days, time_unit=time_unit, timestring=timestring)) working_list = indices for _filter in filter_list: working_list = curator.apply_filter(working_list, **_filter) return working_list
def main(argv): parser = OptionParser() parser.add_option("-p", "--prefix", dest="prefix", action="store", help="prefix to indices filter ") parser.add_option("-r", "--repository", dest="repository", action="store", help="repository name") (opts, args) = parser.parse_args() if not opts.repository: # if repository is not given print 'repository not given' usage() sys.exit() if not opts.prefix: # if repository is not given print 'prefix not given' usage() sys.exit() client = elasticsearch.Elasticsearch([{'host': 'localhost', 'port': 9200}]) indices_list = curator.get_indices(client) _filter = curator.build_filter(kindOf='prefix', value=opts.prefix) working_list = curator.apply_filter(indices_list, **_filter) res = curator.create_snapshot(client, indices=working_list, name=None, prefix='curator-', repository=opts.repository, ignore_unavailable=True, include_global_state=False, partial=False, wait_for_completion=True, request_timeout=21600, skip_repo_validation=False) print res
def indices_value_filter(filter_type='prefix', filter_value='.marvel'): """ valid filter types are prefix, suffix, exclude, and regex """ return curator.build_filter(kindOf=filter_type, value=filter_value)
def main(argv): parser = OptionParser() parser.add_option("--timestring", dest="timestring", action="store", help="timestring of the datestamp in an index name") parser.add_option("--range", dest="range", action="store", help="the time range of the alias ( days )") parser.add_option("--prefix", dest="prefix", action="store", help="prefix to filter indices") parser.add_option("--host", dest="host", action="store", help="host, default : localhost") parser.add_option("--port", dest="port", action="store", help="port, default : 9200") parser.add_option("--alias", dest="alias", action="store", help="alias name") host = 'localhost' port = 9200 timeout = 21600 (opts, args) = parser.parse_args() if not opts.range: # if repository is not given print 'range not given' usage() sys.exit() if not opts.timestring: # if repository is not given print 'timestring not given' usage() sys.exit() if not opts.alias: # if repository is not given print 'alias not given' usage() sys.exit() if not opts.prefix: # if repository is not given print 'prefix not given' usage() sys.exit() if opts.host: # if host is given host = opts.host if opts.port: # if port is given host = int(opts.port) #if opts.timeout: # if port is given # host = int(opts.timeout) client = elasticsearch.Elasticsearch([{ 'host': host, 'port': port, 'timeout': timeout }]) # first add the new indices to the alias indices = curator.get_indices(client) _filter = curator.build_filter(kindOf='prefix', value=opts.prefix) prefix_indices = curator.apply_filter(indices, **_filter) _filter = curator.build_filter(kindOf='newer_than', value=int(opts.range) - 1, time_unit='days', timestring=opts.timestring) new_prefix_indices = curator.apply_filter(prefix_indices, **_filter) for index in new_prefix_indices: print 'adding index ' + index + ' to alias ' + opts.alias curator.add_to_alias(client, index, opts.alias) alias_list = curator.get_alias(client, opts.alias) #if not alias_list: # sys.exit() # remove old indices from alias _filter = curator.build_filter(kindOf='older_than', value=int(opts.range), time_unit='days', timestring=opts.timestring) old_indices = curator.apply_filter(alias_list, **_filter) for index in old_indices: print 'removing index ' + index + ' from alias ' + opts.alias + ' ...' curator.remove_from_alias(client, index, opts.alias)
def main(argv): parser = OptionParser() parser.add_option("--timestring", dest="timestring", action="store", help="timestring of the datestamp in an index name") parser.add_option("--range", dest="range", action="store", help="the time range of the alias ( days )") parser.add_option("--prefix", dest="prefix", action="store", help="prefix to filter indices") parser.add_option("--host", dest="host", action="store", help="host, default : localhost") parser.add_option("--port", dest="port", action="store", help="port, default : 9200") parser.add_option("--alias", dest="alias", action="store", help="alias name") host = 'localhost' port = 9200 timeout = 21600 (opts, args) = parser.parse_args() if not opts.range: # if repository is not given print 'range not given' usage() sys.exit() if not opts.timestring: # if repository is not given print 'timestring not given' usage() sys.exit() if not opts.alias: # if repository is not given print 'alias not given' usage() sys.exit() if not opts.prefix: # if repository is not given print 'prefix not given' usage() sys.exit() if opts.host: # if host is given host = opts.host if opts.port: # if port is given host = int(opts.port) #if opts.timeout: # if port is given # host = int(opts.timeout) client = elasticsearch.Elasticsearch([ {'host': host, 'port': port, 'timeout' : timeout} ]) # first add the new indices to the alias indices = curator.get_indices(client) _filter = curator.build_filter(kindOf='prefix', value=opts.prefix) prefix_indices = curator.apply_filter(indices, **_filter) _filter = curator.build_filter(kindOf='newer_than', value=int(opts.range)-1, time_unit='days', timestring=opts.timestring) new_prefix_indices = curator.apply_filter(prefix_indices, **_filter) for index in new_prefix_indices: print 'adding index ' + index + ' to alias ' + opts.alias curator.add_to_alias(client, index, opts.alias) alias_list = curator.get_alias(client, opts.alias) #if not alias_list: # sys.exit() # remove old indices from alias _filter = curator.build_filter(kindOf='older_than', value=int(opts.range), time_unit='days', timestring=opts.timestring) old_indices = curator.apply_filter(alias_list, **_filter) for index in old_indices: print 'removing index ' + index + ' from alias ' + opts.alias + ' ...' curator.remove_from_alias(client, index, opts.alias)
def main(argv): parser = OptionParser() parser.add_option("-r", "--repository", dest="repository", action="store", help="repository name") parser.add_option("-t", "--timestring", dest="timestring", action="store", help="timestring of the datestamp in an index name") parser.add_option("-o", "--older_than", dest="older_than", action="store", help="older than ( days )") parser.add_option( "-i", "--ignore_if_newer_not_exist", dest="ignore_if_newer_not_exist", action="store", help= "don't perform deletion if there are no snapshots newer than this range" ) parser.add_option("-s", "--host", dest="host", action="store", help="host, default : localhost") parser.add_option("-p", "--port", dest="port", action="store", help="port, default : 9200") #parser.add_option("-m", "--timeout", dest="timeout", action="store", help="timeout (sec), default : 21600") host = 'localhost' port = 9200 timeout = 21600 (opts, args) = parser.parse_args() if not opts.repository: # if repository is not given print 'repository not given' usage() sys.exit() if not opts.older_than: # if repository is not given print 'older_then not given' usage() sys.exit() if not opts.timestring: # if repository is not given print 'timestring not given' usage() sys.exit() if not opts.ignore_if_newer_not_exist: # if repository is not given print 'ignore_if_newer_not_exist not given' usage() sys.exit() if opts.host: # if host is given host = opts.host if opts.port: # if port is given host = int(opts.port) #if opts.timeout: # if port is given # host = int(opts.timeout) client = elasticsearch.Elasticsearch([{ 'host': host, 'port': port, 'timeout': timeout }]) snapshots_list = curator.get_snapshots(client, opts.repository) _filter = curator.build_filter(kindOf='newer_than', value=int(opts.ignore_if_newer_not_exist), time_unit='days', timestring=opts.timestring) snapshots_verification_list = curator.apply_filter(snapshots_list, **_filter) if not snapshots_verification_list: print 'no new snapshots available, existing' sys.exit() _filter = curator.build_filter(kindOf='older_than', value=int(opts.older_than), time_unit='days', timestring=opts.timestring) snapshots_to_delete_list = curator.apply_filter(snapshots_list, **_filter) print ', '.join(snapshots_to_delete_list) for snapshot in snapshots_to_delete_list: print 'deleting snapshot ' + snapshot + ' ...' res = curator.delete_snapshot(client, snapshot, opts.repository) if not res: print 'error! could not delete snapshot ' + snapshot