def main(argv):
parser = define_arguments()
args = parser.parse_args(argv[1:])
for profile_path in args.profile_path:
print "Creating ingestion document for profile %s" % profile_path
ingestion_doc_id = create_ingestion_document.main([None, profile_path])
if ingestion_doc_id is None:
print "Error creating ingestion document"
continue
resp = fetch_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error fetching records"
continue
resp = enrich_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error enriching records"
continue
resp = save_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error saving records"
continue
resp = remove_deleted_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error saving records..."
continue
print "Ingestion complete!"
def main(argv):
parser = define_arguments()
args = parser.parse_args(argv[1:])
for profile_path in args.profile_path:
print "Creating ingestion document for profile %s" % profile_path
ingestion_doc_id = create_ingestion_document.main([None, profile_path])
if ingestion_doc_id is None:
print "Error creating ingestion document"
continue
resp = fetch_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error fetching records"
continue
resp = enrich_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error enriching records"
continue
resp = save_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error saving records"
continue
resp = remove_deleted_records.main([None, ingestion_doc_id])
if not resp == 0:
print "Error deleting records"
continue
resp = check_ingestion_counts.main([None, ingestion_doc_id])
if not resp == 0:
print "Error checking counts"
continue
resp = dashboard_cleanup.main([None, ingestion_doc_id])
if not resp == 0:
print "Error cleaning up dashboard"
continue
# Bulk data being updated via elasticdump.
#
# resp = upload_bulk_data.main([None, ingestion_doc_id])
# if not resp == 0:
# print "Error uploading bulk download data"
# continue
print "Ingestion complete!"
try:
profile = json.load(f)
except Exception, e:
print "Error reading profile: %s" % e
return False
if args.pipeline in profile:
pipeline = ",".join(profile[args.pipeline])
else:
pipeline = args.pipeline
provider = profile.get(u"name")
contributor = profile.get(u"contributor", {})
# Create ingestion document
couch = Couch()
ingestion_doc_id = create_ingestion_document.main([None,
args.profile_path])
ingestion_doc = couch.dashboard_db[ingestion_doc_id]
# Update ingestion document
kwargs = {
"poll_storage_process/status": "running",
"poll_storage_process/start_time": datetime.now().isoformat(),
"poll_storage_process/end_time": None,
}
try:
couch.update_ingestion_doc(ingestion_doc, **kwargs)
except:
print "Error updating ingestion document " + ingestion_doc["_id"]
return
# Back up current data
try:
profile = json.load(f)
except Exception, e:
print "Error reading profile: %s" % e
return False
if args.pipeline in profile:
pipeline = ",".join(profile[args.pipeline])
else:
pipeline = args.pipeline
provider = profile.get(u"name")
contributor = profile.get(u"contributor", {})
# Create ingestion document
couch = Couch()
ingestion_doc_id = create_ingestion_document.main([None,
args.profile_path])
ingestion_doc = couch.dashboard_db[ingestion_doc_id]
# Update ingestion document
kwargs = {
"poll_storage_process/status": "running",
"poll_storage_process/start_time": iso_utc_with_tz(),
"poll_storage_process/end_time": None,
}
try:
couch.update_ingestion_doc(ingestion_doc, **kwargs)
except:
print "Error updating ingestion document " + ingestion_doc["_id"]
return
# Back up current data
