def lifecycle():
global db
queries = db.get_all_queries()
for query_raw in queries:
query = Query(json=query_raw)
logger.debug(f'checking query {query.id}')
logger.debug("performing search...")
results = search(query.search, num_results=10)
logger.info(f"query {query.id}: hits: {len(results)}")
new_items = False
for result_raw in results:
#logger.debug(f'result_raw: {result_raw[0]}\t {result_raw[1]}')
result = Result(id=str(uuid.uuid4()),
query=query.id,
timestamp=str(get_now_as_timestamp()),
title=result_raw[0],
content=result_raw[1])
logger.debug(
"result: " +
json.dumps(result.as_dict(), indent=4, sort_keys=True))
if not db.result_exists(result):
logger.debug("result has not previously been recorded")
new_items = True
db.save_result(result)
if new_items:
feed_string = generate_feed_for_query(query)
write_to_file_or_s3(uri=settings.RESULTS_FOLDER,
query=query,
body=feed_string)
