def delete_old_indices(client_config, index, index_retention_policies):
"""Deletes indices past retention policy
Args:
client_config (dict): Client configuration
index (str): Index name
index_retention_policies (dict): Retention policy
"""
elastic_connection = es.build_es_connection(client_config)
newest_record = ""
newest_record = es.get_newest_document_date_in_index(
client_config, index, elastic_connection)
# make sure newest record is not empty
if newest_record != "":
# Get the index specific retention policy
policy = es.check_index_retention_policy(index,
index_retention_policies)
# Get policy retention days from specific policy
policy_days = index_retention_policies[policy]
# Get current datetime
current_date = datetime.utcnow()
# Figure out how many days since current_date vs. newest_record
days_ago = (current_date - newest_record).days
# Check if days_ago is greater than or equal to policy date
# If greater than or equal to policy date, delete index
if days_ago >= policy_days:
# Delete old index
es.delete_index(client_config, index)
elastic_connection.close()
def main(args):
"""Entry point."""
print(args)
delete_index(args.index)
create_index_aminer_v1(args.index)
update_settings(args.index)
inputs = sorted(args.inputs)
for ipath in tqdm(inputs, total=len(inputs), unit='files'):
index_worker(args.index, ipath)
time.sleep(120) # make sure that elasticsearch have time to refresh
def on_post(self, req, resp):
cmd = req.get_param('cmd')
result = {}
if cmd == 'add':
book = req.get_param('book')
file_path = save_file(book)
task_data = {'path': file_path}
try:
add_book_task.delay(task_data)
result = {'msg': 'file putted in queue'}
except Exception as e:
result = {'error': str(e)}
delete_file(file_path)
elif cmd == 'create':
result = create_index()
elif cmd == 'delete':
result = delete_index()
elif cmd == 'count':
result = count_items()
elif cmd == 'search':
q = req.get_param('q')
result = search(q)
elif cmd == 'search_advanced':
q = req.get_param('q')
result = search_advanced(q)
resp.body = json.dumps(result)
resp.status = falcon.HTTP_200
import es
if __name__ == '__main__':
es_conn = es.connect_elasticsearch()
output = es.delete_index(es_conn, 'matches')
es.close_connection(es_conn)
print(output)
import es
import json
import glob
import time
es.delete_index("messages")
es.create_index("messages")
nb = 0
for filename in glob.iglob('data/**/*.json', recursive=True):
nb += 1
if nb % 500 == 0:
time.sleep(1)
with open(filename, encoding="utf8") as f:
item = json.load(f)
es.index("messages", "message", item)