def fetch_mongo_data_for_fetch_range(collection, offset, fetch_limit,query_filters):
_mongo_fetched_data_dict = []
data_cursor = collection.find(query_filters, {'_id': 0}).skip(offset).limit(fetch_limit)
for document in data_cursor:
try:
if not 'category' in document:
continue
_mongo_fetched_data_dict.append(document)
except Exception, e:
LOGGER.error(
"For document {0} encountered error {1} ".format(doc, e))
def dump_data_dict_to_es_readable_file(mongo_fetched_data_dict):
put = {"index": {"_index": es_conf[_ES_INDEX], "_type": "wevent"}}
events_dump_file = open(_ES_READ_FILE, 'w')
try:
for document in mongo_fetched_data_dict:
put['index']['_type'] = document['category']
json.dump(put,events_dump_file)
events_dump_file.write('\n')
json.dump(document,events_dump_file)
events_dump_file.write('\n')
except Exception, e:
LOGGER.error(
"For document {0} encountered dumping data error {1} ".format(document, e))
