def main(): argument_spec = ec2_argument_spec() argument_spec.update(dict( host = dict(required=True), port = dict(default=80), prefix = dict(default="logstash"), older_than = dict(default=7), time_unit = dict(default="days", choices=["hours", "days", "weeks", "months"]), timestring = dict(default="%Y.%m.%d"), )) module = AnsibleModule( argument_spec=argument_spec, ) if not HAS_DEPENDENCIES: module.fail_json(msg='requests_aws4auth, logging, elasticsearch and elasticsearch-curator are required for this module, install via pip or your package manager') try: client = get_elasticsearch_client(module) indices = filter_indices(module, client) if indices: curator.delete_indices(client, indices) module.exit_json(changed=True, msg="The following indices have been removed: " + ", ".join(indices)) else: module.exit_json(msg="No indices matching the filters were found. No indices removed") except (StandardError), e: module.fail_json(msg=str(e))
def delete_indices(esclient, indices): if not indices: return indices = indices.keys() logger.debug("try to delete %s" % ','.join(indices)) for index in indices: if curator.delete_indices(esclient, [index]): logger.info('%s deleted' % index)
def delete_indices(esclient, indices): if not indices: return indices = indices.keys() logger.debug("try to delete %s" % ','.join(indices)) for index in indices: if curator.delete_indices(esclient, [index]): logger.info('%s deleted' % index)
def prune_es_indices(): """Prune ES indices older than the age defined in settings.PRUNE_OLDER_THAN.""" curation_params = [ {"prefix": "events_", "time_string": "%Y-%m-%d"}, {"prefix": "logstash-", "time_string": "%Y.%m.%d"}, {"prefix": "goldstone-", "time_string": "%Y.%m.%d"}, {"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"}, {"prefix": "api_stats-", "time_string": "%Y.%m.%d"}, {"prefix": "internal-", "time_string": "%Y.%m.%d"}, ] client = es_conn() all_indices = curator.get_indices(client) deleted_indices = [] working_list = all_indices # we'll whittle this down with filters for index_set in curation_params: # filter on our prefix name_filter = curator.build_filter( kindOf='prefix', value=index_set['prefix']) # filter on the datestring age_filter = curator.build_filter( kindOf='older_than', time_unit='days', timestring=index_set['time_string'], value=settings.PRUNE_OLDER_THAN) # apply the filters to get the final list of indices to delete working_list = curator.apply_filter(working_list, **name_filter) working_list = curator.apply_filter(working_list, **age_filter) if working_list is not None and len(working_list) > 0: try: curator.delete_indices(client, working_list) deleted_indices = deleted_indices + working_list except Exception: logger.exception("curator.delete_indices raised an exception") working_list = all_indices # reset for the next loop iteration return deleted_indices
def prune_es_indices(): """Prune ES indices older than the age defined in settings.PRUNE_OLDER_THAN.""" curation_params = [ {"prefix": "events_", "time_string": "%Y-%m-%d"}, {"prefix": "logstash-", "time_string": "%Y.%m.%d"}, {"prefix": "goldstone-", "time_string": "%Y.%m.%d"}, {"prefix": "goldstone_metrics-", "time_string": "%Y.%m.%d"}, {"prefix": "api_stats-", "time_string": "%Y.%m.%d"}, {"prefix": "internal-", "time_string": "%Y.%m.%d"}, ] client = es_conn() all_indices = curator.get_indices(client) deleted_indices = [] working_list = all_indices # we'll whittle this down with filters for index_set in curation_params: # filter on our prefix name_filter = curator.build_filter( kindOf='prefix', value=index_set['prefix']) # filter on the datestring age_filter = curator.build_filter( kindOf='older_than', time_unit='days', timestring=index_set['time_string'], value=settings.PRUNE_OLDER_THAN) # apply the filters to get the final list of indices to delete working_list = curator.apply_filter(working_list, **name_filter) working_list = curator.apply_filter(working_list, **age_filter) if working_list is not None and len(working_list) > 0: try: curator.delete_indices(client, working_list) deleted_indices = deleted_indices + working_list except Exception: logger.exception("curator.delete_indices raised an exception") working_list = all_indices # reset for the next loop iteration return deleted_indices
def main(): argument_spec = ec2_argument_spec() argument_spec.update( dict( host=dict(required=True), port=dict(default=80), prefix=dict(default="logstash"), older_than=dict(default=7), time_unit=dict(default="days", choices=["hours", "days", "weeks", "months"]), timestring=dict(default="%Y.%m.%d"), )) module = AnsibleModule(argument_spec=argument_spec, ) if not HAS_DEPENDENCIES: module.fail_json( msg= 'requests_aws4auth, logging, elasticsearch and elasticsearch-curator are required for this module, install via pip or your package manager' ) try: client = get_elasticsearch_client(module) indices = filter_indices(module, client) if indices: curator.delete_indices(client, indices) module.exit_json(changed=True, msg="The following indices have been removed: " + ", ".join(indices)) else: module.exit_json( msg= "No indices matching the filters were found. No indices removed" ) except (StandardError), e: module.fail_json(msg=str(e))
def delete_indices(esclient, indices, settings): """ :type esclient: elasticsearch.Elasticsearch :type indices: list of (indexname,index_settings) :type settings: dict, not used :rtype: None """ if not indices: return indices = [e[0] for e in indices] _delete.extend(indices) logger.debug("try to delete %s" % ",".join(indices)) global lock with lock: for index in indices: if curator.delete_indices(esclient, [index], master_timeout="300s"): logger.info("%s deleted" % index) dopey_summary.add(u"%s 己删除" % index) else: logger.warn("%s deleted failed" % index) dopey_summary.add(u"%s 删除失败" % index)
def delete_indices(esclient, indices, settings): """ :type esclient: elasticsearch.Elasticsearch :type indices: list of (indexname,index_settings) :type settings: dict, not used :rtype: None """ if not indices: return indices = [e[0] for e in indices] _delete.extend(indices) logger.debug("try to delete %s" % ','.join(indices)) global lock with lock: for index in indices: if curator.delete_indices(esclient, [index], master_timeout='300s'): logger.info('%s deleted' % index) dopey_summary.add(u'%s 己删除' % index) else: logger.warn('%s deleted failed' % index) dopey_summary.add(u'%s 删除失败' % index)
def delete(self): """return number of indices closed """ indices = select_indices(self.es, self.model.index_name_prefix, self.model.index_timestring, self.model.index_timestring_interval, self.model.delete.exec_offset + 1) indices_deleted = 0 for index in indices: try: ret = curator.delete_indices(self.es, index) if ret is True: indices_deleted += 1 # elif ret is False: index doesn't exist except elasticsearch.exceptions.ConnectionTimeout as e: raise CanNotDeleteIndex(str(e)) return indices_deleted
def delete(self): """return number of indices closed """ indices = select_indices( self.es, self.model.index_name_prefix, self.model.index_timestring, self.model.index_timestring_interval, self.model.delete.exec_offset + 1 ) indices_deleted = 0 for index in indices: try: ret = curator.delete_indices(self.es, index) if ret is True: indices_deleted += 1 # elif ret is False: index doesn't exist except elasticsearch.exceptions.ConnectionTimeout as e: raise CanNotDeleteIndex(str(e)) return indices_deleted