def fetch_mongo_data_for_fetch_range(collection, offset, fetch_limit,query_filters): _mongo_fetched_data_dict = [] data_cursor = collection.find(query_filters, {'_id': 0}).skip(offset).limit(fetch_limit) for document in data_cursor: try: if not 'category' in document: continue _mongo_fetched_data_dict.append(document) except Exception, e: LOGGER.error( "For document {0} encountered error {1} ".format(doc, e))
def dump_data_dict_to_es_readable_file(mongo_fetched_data_dict): put = {"index": {"_index": es_conf[_ES_INDEX], "_type": "wevent"}} events_dump_file = open(_ES_READ_FILE, 'w') try: for document in mongo_fetched_data_dict: put['index']['_type'] = document['category'] json.dump(put,events_dump_file) events_dump_file.write('\n') json.dump(document,events_dump_file) events_dump_file.write('\n') except Exception, e: LOGGER.error( "For document {0} encountered dumping data error {1} ".format(document, e))