def main(argv):
parser = define_arguments()
args = parser.parse_args(argv[1:])
for profile_path in args.profile_path:
print "Creating ingestion document for profile %s" % profile_path
ingestion_doc_id = create_ingestion_document.main([None, profile_path])
if ingestion_doc_id is None:
print "Error creating ingestion document"
continue
resp = fetch_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error fetching records"
continue
resp = enrich_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error enriching records"
continue
resp = save_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error saving records"
continue
resp = remove_deleted_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error deleting records"
continue
resp = check_ingestion_counts.main([None, ingestion_doc_id])
if not resp == 0:
print "Error checking counts"
continue
resp = dashboard_cleanup.main([None, ingestion_doc_id])
if not resp == 0:
print "Error cleaning up dashboard"
continue
# Bulk data being updated via elasticdump.
#
# resp = upload_bulk_data.main([None, ingestion_doc_id])
# if not resp == 0:
# print "Error uploading bulk download data"
# continue
print "Ingestion complete!"
def main(argv):
parser = define_arguments()
args = parser.parse_args(argv[1:])
for profile_path in args.profile_path:
print "Creating ingestion document for profile %s" % profile_path
ingestion_doc_id = create_ingestion_document.main([None, profile_path])
if ingestion_doc_id is None:
print "Error creating ingestion document"
continue
resp = fetch_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error fetching records"
continue
resp = enrich_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error enriching records"
continue
resp = save_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error saving records"
continue
resp = remove_deleted_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error deleting records"
continue
resp = dashboard_cleanup.main([None, ingestion_doc_id])
if not resp == 0:
print "Error cleaning up dashboard"
continue
print "Ingestion complete!"
kwargs = {
"poll_storage_process/status": "complete",
"poll_storage_process/end_time": datetime.now().isoformat(),
"poll_storage_process/total_enriched": total_enriched,
"poll_storage_process/total_items": enriched_items,
"poll_storage_process/total_collections": enriched_colls,
"poll_storage_process/missing_id": missing_id,
"poll_storage_process/missing_source_resource": missing_source_resource,
"poll_storage_process/error": errors,
"save_process/status": "complete"
}
try:
couch.update_ingestion_doc(ingestion_doc, **kwargs)
except:
print "Error updating ingestion document " + ingestion_doc["_id"]
return
# Run remove_deleted_records
resp = remove_deleted_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error deleting records"
return
# Run dashboard cleanup
resp = dashboard_cleanup.main([None, ingestion_doc_id])
if not resp == 0:
print "Error cleaning up dashboard"
if __name__ == "__main__":
main(sys.argv)
"poll_storage_process/error": enrich_errors,
"save_process/total_items": saved_items,
"save_process/total_collections": saved_colls,
"save_process/error": save_errors,
"save_process/status": "complete"
}
try:
couch.update_ingestion_doc(ingestion_doc, **kwargs)
except:
print "Error updating ingestion document " + ingestion_doc["_id"]
return
# Run remove_deleted_records
resp = remove_deleted_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error deleting records"
return
# Run check counts
resp = check_ingestion_counts.main([None, ingestion_doc_id])
if not resp == 0:
print "Error checking ingestion counts"
# Run dashboard cleanup
resp = dashboard_cleanup.main([None, ingestion_doc_id])
if not resp == 0:
print "Error cleaning up dashboard"
if __name__ == "__main__":
main(sys.argv)
