def main(argv):
parser = OptionParser()
parser.add_option("-r", "--repository", dest="repository", action="store", help="repository name")
parser.add_option("-t", "--timestring", dest="timestring", action="store",
help="timestring of the datestamp in an index name")
parser.add_option("-o", "--older_than", dest="older_than", action="store", help="older than ( days )")
parser.add_option("-i", "--ignore_if_newer_not_exist", dest="ignore_if_newer_not_exist", action="store",
help="don't perform deletion if there are no snapshots newer than this range")
parser.add_option("-s", "--host", dest="host", action="store", help="host, default : localhost")
parser.add_option("-p", "--port", dest="port", action="store", help="port, default : 9200")
#parser.add_option("-m", "--timeout", dest="timeout", action="store", help="timeout (sec), default : 21600")
host = 'localhost'
port = 9200
timeout = 21600
(opts, args) = parser.parse_args()
if not opts.repository: # if repository is not given
print 'repository not given'
usage()
sys.exit()
if not opts.older_than: # if repository is not given
print 'older_then not given'
usage()
sys.exit()
if not opts.timestring: # if repository is not given
print 'timestring not given'
usage()
sys.exit()
if not opts.ignore_if_newer_not_exist: # if repository is not given
print 'ignore_if_newer_not_exist not given'
usage()
sys.exit()
if opts.host: # if host is given
host = opts.host
if opts.port: # if port is given
host = int(opts.port)
#if opts.timeout: # if port is given
# host = int(opts.timeout)
client = elasticsearch.Elasticsearch([
{'host': host, 'port': port, 'timeout' : timeout}
])
snapshots_list = curator.get_snapshots(client, opts.repository)
_filter = curator.build_filter(kindOf='newer_than', value=int(opts.ignore_if_newer_not_exist), time_unit='days',
timestring=opts.timestring)
snapshots_verification_list = curator.apply_filter(snapshots_list, **_filter)
if not snapshots_verification_list:
print 'no new snapshots available, existing'
sys.exit()
_filter = curator.build_filter(kindOf='older_than', value=int(opts.older_than), time_unit='days',
timestring=opts.timestring)
snapshots_to_delete_list = curator.apply_filter(snapshots_list, **_filter)
print ', '.join(snapshots_to_delete_list)
for snapshot in snapshots_to_delete_list:
print 'deleting snapshot ' + snapshot + ' ...'
res = curator.delete_snapshot(client,snapshot, opts.repository)
if not res:
print 'error! could not delete snapshot ' + snapshot
def filter_indices(module, client):
prefix = module.params.get('prefix')
older_than = module.params.get('older_than')
time_unit = module.params.get('time_unit')
timestring = module.params.get('timestring')
indices = curator.get_indices(client)
_filter = curator.build_filter(kindOf='prefix', value=prefix)
indices = curator.apply_filter(indices, **_filter)
_filter = curator.build_filter(kindOf='older_than', value=int(older_than), time_unit=time_unit, timestring=timestring)
return curator.apply_filter(indices, **_filter)
def main(argv):
parser = OptionParser()
parser.add_option("-p", "--prefix", dest="prefix", action="store", help="prefix to indices filter ")
parser.add_option("-r", "--repository", dest="repository", action="store", help="repository name")
(opts, args) = parser.parse_args()
if not opts.repository: # if repository is not given
print 'repository not given'
usage()
sys.exit()
if not opts.prefix: # if repository is not given
print 'prefix not given'
usage()
sys.exit()
client = elasticsearch.Elasticsearch([
{'host': 'localhost', 'port': 9200}
])
indices_list = curator.get_indices(client)
_filter = curator.build_filter(kindOf='prefix', value=opts.prefix)
working_list = curator.apply_filter(indices_list, **_filter)
res = curator.create_snapshot(client, indices=working_list, name=None, prefix='curator-',
repository=opts.repository, ignore_unavailable=True, include_global_state=False,
partial=False, wait_for_completion=True, request_timeout=21600,
skip_repo_validation=False)
print res
def filter_indices(module, client):
prefix = module.params.get('prefix')
older_than = module.params.get('older_than')
time_unit = module.params.get('time_unit')
timestring = module.params.get('timestring')
indices = curator.get_indices(client)
_filter = curator.build_filter(kindOf='prefix', value=prefix)
indices = curator.apply_filter(indices, **_filter)
_filter = curator.build_filter(kindOf='older_than',
value=int(older_than),
time_unit=time_unit,
timestring=timestring)
return curator.apply_filter(indices, **_filter)
def indices_date_filter(filter_type='older_than', filter_value=30,
time_unit='days', timestring='%Y.%m.%d'):
"""
valid filter types are older_than and newer_than
time unit could be days, months, or years.
"""
return curator.build_filter(kindOf=filter_type, value=filter_value,
time_unit=time_unit, timestring=timestring)
def rotate_logs(self):
"""
We will delete indices older than configured maximum age.
"""
if not self.is_connected == CONNECTED:
if not self.open():
self.next_logs_rotation = time.time() + 600
logger.info(
"[elastic-logs] log rotation failed, next log rotation at %s "
% time.asctime(time.localtime(self.next_logs_rotation))
)
return
logger.info("[elastic-logs] rotating logs ...")
now = time.time()
today = date.today()
today0000 = datetime(today.year, today.month, today.day, 0, 0, 0)
today0005 = datetime(today.year, today.month, today.day, 0, 5, 0)
oldest = today0000 - timedelta(days=self.max_logs_age)
if now < time.mktime(today0005.timetuple()):
next_rotation = today0005
else:
next_rotation = today0005 + timedelta(days=1)
try:
indices = curator.get_indices(self.es)
filter_list = []
filter_list.append(curator.build_filter(kindOf="prefix", value=self.index_prefix))
filter_list.append(
curator.build_filter(
kindOf="older_than", value=self.max_logs_age, time_unit="days", timestring="%Y.%m.%d"
)
)
working_list = indices
for filter in filter_list:
working_list = curator.apply_filter(working_list, **filter)
curator.delete(self.es, working_list)
logger.info("[elastic-logs] removed %d logs older than %s days.", working_list, self.max_logs_age)
except Exception, exp:
logger.error("[elastic-logs] Exception while rotating indices: %s", str(exp))
def prune_es_indices():
"""Prune ES indices older than the age defined in
settings.PRUNE_OLDER_THAN."""
curation_params = [
{"prefix": "events_", "time_string": "%Y-%m-%d"},
{"prefix": "logstash-", "time_string": "%Y.%m.%d"},
{"prefix": "goldstone-", "time_string": "%Y.%m.%d"},
{"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"},
{"prefix": "api_stats-", "time_string": "%Y.%m.%d"},
{"prefix": "internal-", "time_string": "%Y.%m.%d"},
]
client = es_conn()
all_indices = curator.get_indices(client)
deleted_indices = []
working_list = all_indices # we'll whittle this down with filters
for index_set in curation_params:
# filter on our prefix
name_filter = curator.build_filter(
kindOf='prefix', value=index_set['prefix'])
# filter on the datestring
age_filter = curator.build_filter(
kindOf='older_than', time_unit='days',
timestring=index_set['time_string'],
value=settings.PRUNE_OLDER_THAN)
# apply the filters to get the final list of indices to delete
working_list = curator.apply_filter(working_list, **name_filter)
working_list = curator.apply_filter(working_list, **age_filter)
if working_list is not None and len(working_list) > 0:
try:
curator.delete_indices(client, working_list)
deleted_indices = deleted_indices + working_list
except Exception:
logger.exception("curator.delete_indices raised an exception")
working_list = all_indices # reset for the next loop iteration
return deleted_indices
def prune_es_indices():
"""Prune ES indices older than the age defined in
settings.PRUNE_OLDER_THAN."""
curation_params = [
{"prefix": "events_", "time_string": "%Y-%m-%d"},
{"prefix": "logstash-", "time_string": "%Y.%m.%d"},
{"prefix": "goldstone-", "time_string": "%Y.%m.%d"},
{"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"},
{"prefix": "api_stats-", "time_string": "%Y.%m.%d"},
{"prefix": "internal-", "time_string": "%Y.%m.%d"},
]
client = es_conn()
all_indices = curator.get_indices(client)
deleted_indices = []
working_list = all_indices # we'll whittle this down with filters
for index_set in curation_params:
# filter on our prefix
name_filter = curator.build_filter(
kindOf='prefix', value=index_set['prefix'])
# filter on the datestring
age_filter = curator.build_filter(
kindOf='older_than', time_unit='days',
timestring=index_set['time_string'],
value=settings.PRUNE_OLDER_THAN)
# apply the filters to get the final list of indices to delete
working_list = curator.apply_filter(working_list, **name_filter)
working_list = curator.apply_filter(working_list, **age_filter)
if working_list is not None and len(working_list) > 0:
try:
curator.delete_indices(client, working_list)
deleted_indices = deleted_indices + working_list
except Exception:
logger.exception("curator.delete_indices raised an exception")
working_list = all_indices # reset for the next loop iteration
return deleted_indices
def select_indices(client,
prefix,
timestring,
time_unit,
older_than_days=None,
newer_than_days=None):
"""
time_unit: One of hours, days, weeks, months, NO years!!!
older_than_days, newer_than_days:
Value of older_than_days or newer_than_days should be > 0
If older_than_days and newer_than_days are both set, newer_than_days should be greater than older_than_days
older_than_days = n will filters indices n days before today,
newer_than_days = m will filters indices from m days before today to today
"""
import curator
indices = curator.get_indices(client)
filter_list = []
index_regex = prefix + curator.get_date_regex(timestring)
filter_list.append(curator.build_filter(kindOf='regex', value=index_regex))
if older_than_days:
filter_list.append(
curator.build_filter(kindOf='older_than',
value=older_than_days,
time_unit=time_unit,
timestring=timestring))
if newer_than_days:
filter_list.append(
curator.build_filter(kindOf='newer_than',
value=newer_than_days,
time_unit=time_unit,
timestring=timestring))
working_list = indices
for _filter in filter_list:
working_list = curator.apply_filter(working_list, **_filter)
return working_list
def main(argv):
parser = OptionParser()
parser.add_option("-p",
"--prefix",
dest="prefix",
action="store",
help="prefix to indices filter ")
parser.add_option("-r",
"--repository",
dest="repository",
action="store",
help="repository name")
(opts, args) = parser.parse_args()
if not opts.repository: # if repository is not given
print 'repository not given'
usage()
sys.exit()
if not opts.prefix: # if repository is not given
print 'prefix not given'
usage()
sys.exit()
client = elasticsearch.Elasticsearch([{'host': 'localhost', 'port': 9200}])
indices_list = curator.get_indices(client)
_filter = curator.build_filter(kindOf='prefix', value=opts.prefix)
working_list = curator.apply_filter(indices_list, **_filter)
res = curator.create_snapshot(client,
indices=working_list,
name=None,
prefix='curator-',
repository=opts.repository,
ignore_unavailable=True,
include_global_state=False,
partial=False,
wait_for_completion=True,
request_timeout=21600,
skip_repo_validation=False)
print res
def indices_value_filter(filter_type='prefix', filter_value='.marvel'):
"""
valid filter types are prefix, suffix, exclude, and regex
"""
return curator.build_filter(kindOf=filter_type, value=filter_value)
def main(argv):
parser = OptionParser()
parser.add_option("--timestring",
dest="timestring",
action="store",
help="timestring of the datestamp in an index name")
parser.add_option("--range",
dest="range",
action="store",
help="the time range of the alias ( days )")
parser.add_option("--prefix",
dest="prefix",
action="store",
help="prefix to filter indices")
parser.add_option("--host",
dest="host",
action="store",
help="host, default : localhost")
parser.add_option("--port",
dest="port",
action="store",
help="port, default : 9200")
parser.add_option("--alias",
dest="alias",
action="store",
help="alias name")
host = 'localhost'
port = 9200
timeout = 21600
(opts, args) = parser.parse_args()
if not opts.range: # if repository is not given
print 'range not given'
usage()
sys.exit()
if not opts.timestring: # if repository is not given
print 'timestring not given'
usage()
sys.exit()
if not opts.alias: # if repository is not given
print 'alias not given'
usage()
sys.exit()
if not opts.prefix: # if repository is not given
print 'prefix not given'
usage()
sys.exit()
if opts.host: # if host is given
host = opts.host
if opts.port: # if port is given
host = int(opts.port)
#if opts.timeout: # if port is given
# host = int(opts.timeout)
client = elasticsearch.Elasticsearch([{
'host': host,
'port': port,
'timeout': timeout
}])
# first add the new indices to the alias
indices = curator.get_indices(client)
_filter = curator.build_filter(kindOf='prefix', value=opts.prefix)
prefix_indices = curator.apply_filter(indices, **_filter)
_filter = curator.build_filter(kindOf='newer_than',
value=int(opts.range) - 1,
time_unit='days',
timestring=opts.timestring)
new_prefix_indices = curator.apply_filter(prefix_indices, **_filter)
for index in new_prefix_indices:
print 'adding index ' + index + ' to alias ' + opts.alias
curator.add_to_alias(client, index, opts.alias)
alias_list = curator.get_alias(client, opts.alias)
#if not alias_list:
# sys.exit()
# remove old indices from alias
_filter = curator.build_filter(kindOf='older_than',
value=int(opts.range),
time_unit='days',
timestring=opts.timestring)
old_indices = curator.apply_filter(alias_list, **_filter)
for index in old_indices:
print 'removing index ' + index + ' from alias ' + opts.alias + ' ...'
curator.remove_from_alias(client, index, opts.alias)
def main(argv):
parser = OptionParser()
parser.add_option("--timestring", dest="timestring", action="store",
help="timestring of the datestamp in an index name")
parser.add_option("--range", dest="range", action="store", help="the time range of the alias ( days )")
parser.add_option("--prefix", dest="prefix", action="store", help="prefix to filter indices")
parser.add_option("--host", dest="host", action="store", help="host, default : localhost")
parser.add_option("--port", dest="port", action="store", help="port, default : 9200")
parser.add_option("--alias", dest="alias", action="store", help="alias name")
host = 'localhost'
port = 9200
timeout = 21600
(opts, args) = parser.parse_args()
if not opts.range: # if repository is not given
print 'range not given'
usage()
sys.exit()
if not opts.timestring: # if repository is not given
print 'timestring not given'
usage()
sys.exit()
if not opts.alias: # if repository is not given
print 'alias not given'
usage()
sys.exit()
if not opts.prefix: # if repository is not given
print 'prefix not given'
usage()
sys.exit()
if opts.host: # if host is given
host = opts.host
if opts.port: # if port is given
host = int(opts.port)
#if opts.timeout: # if port is given
# host = int(opts.timeout)
client = elasticsearch.Elasticsearch([
{'host': host, 'port': port, 'timeout' : timeout}
])
# first add the new indices to the alias
indices = curator.get_indices(client)
_filter = curator.build_filter(kindOf='prefix', value=opts.prefix)
prefix_indices = curator.apply_filter(indices, **_filter)
_filter = curator.build_filter(kindOf='newer_than', value=int(opts.range)-1, time_unit='days',
timestring=opts.timestring)
new_prefix_indices = curator.apply_filter(prefix_indices, **_filter)
for index in new_prefix_indices:
print 'adding index ' + index + ' to alias ' + opts.alias
curator.add_to_alias(client, index, opts.alias)
alias_list = curator.get_alias(client, opts.alias)
#if not alias_list:
# sys.exit()
# remove old indices from alias
_filter = curator.build_filter(kindOf='older_than', value=int(opts.range), time_unit='days',
timestring=opts.timestring)
old_indices = curator.apply_filter(alias_list, **_filter)
for index in old_indices:
print 'removing index ' + index + ' from alias ' + opts.alias + ' ...'
curator.remove_from_alias(client, index, opts.alias)
def main(argv):
parser = OptionParser()
parser.add_option("-r",
"--repository",
dest="repository",
action="store",
help="repository name")
parser.add_option("-t",
"--timestring",
dest="timestring",
action="store",
help="timestring of the datestamp in an index name")
parser.add_option("-o",
"--older_than",
dest="older_than",
action="store",
help="older than ( days )")
parser.add_option(
"-i",
"--ignore_if_newer_not_exist",
dest="ignore_if_newer_not_exist",
action="store",
help=
"don't perform deletion if there are no snapshots newer than this range"
)
parser.add_option("-s",
"--host",
dest="host",
action="store",
help="host, default : localhost")
parser.add_option("-p",
"--port",
dest="port",
action="store",
help="port, default : 9200")
#parser.add_option("-m", "--timeout", dest="timeout", action="store", help="timeout (sec), default : 21600")
host = 'localhost'
port = 9200
timeout = 21600
(opts, args) = parser.parse_args()
if not opts.repository: # if repository is not given
print 'repository not given'
usage()
sys.exit()
if not opts.older_than: # if repository is not given
print 'older_then not given'
usage()
sys.exit()
if not opts.timestring: # if repository is not given
print 'timestring not given'
usage()
sys.exit()
if not opts.ignore_if_newer_not_exist: # if repository is not given
print 'ignore_if_newer_not_exist not given'
usage()
sys.exit()
if opts.host: # if host is given
host = opts.host
if opts.port: # if port is given
host = int(opts.port)
#if opts.timeout: # if port is given
# host = int(opts.timeout)
client = elasticsearch.Elasticsearch([{
'host': host,
'port': port,
'timeout': timeout
}])
snapshots_list = curator.get_snapshots(client, opts.repository)
_filter = curator.build_filter(kindOf='newer_than',
value=int(opts.ignore_if_newer_not_exist),
time_unit='days',
timestring=opts.timestring)
snapshots_verification_list = curator.apply_filter(snapshots_list,
**_filter)
if not snapshots_verification_list:
print 'no new snapshots available, existing'
sys.exit()
_filter = curator.build_filter(kindOf='older_than',
value=int(opts.older_than),
time_unit='days',
timestring=opts.timestring)
snapshots_to_delete_list = curator.apply_filter(snapshots_list, **_filter)
print ', '.join(snapshots_to_delete_list)
for snapshot in snapshots_to_delete_list:
print 'deleting snapshot ' + snapshot + ' ...'
res = curator.delete_snapshot(client, snapshot, opts.repository)
if not res:
print 'error! could not delete snapshot ' + snapshot
