def main(argv): parser = define_arguments() args = parser.parse_args(argv[1:]) for profile_path in args.profile_path: print "Creating ingestion document for profile %s" % profile_path ingestion_doc_id = create_ingestion_document.main([None, profile_path]) if ingestion_doc_id is None: print "Error creating ingestion document" continue resp = fetch_records.main([None, ingestion_doc_id]) if not resp == 0: print "Error fetching records" continue resp = enrich_records.main([None, ingestion_doc_id]) if not resp == 0: print "Error enriching records" continue resp = save_records.main([None, ingestion_doc_id]) if not resp == 0: print "Error saving records" continue resp = remove_deleted_records.main([None, ingestion_doc_id]) if not resp == 0: print "Error saving records..." continue print "Ingestion complete!"
def main(argv): parser = define_arguments() args = parser.parse_args(argv[1:]) for profile_path in args.profile_path: print "Creating ingestion document for profile %s" % profile_path ingestion_doc_id = create_ingestion_document.main([None, profile_path]) if ingestion_doc_id is None: print "Error creating ingestion document" continue resp = fetch_records.main([None, ingestion_doc_id]) if not resp == 0: print "Error fetching records" continue resp = enrich_records.main([None, ingestion_doc_id]) if not resp == 0: print "Error enriching records" continue resp = save_records.main([None, ingestion_doc_id]) if not resp == 0: print "Error saving records" continue resp = remove_deleted_records.main([None, ingestion_doc_id]) if not resp == 0: print "Error deleting records" continue resp = check_ingestion_counts.main([None, ingestion_doc_id]) if not resp == 0: print "Error checking counts" continue resp = dashboard_cleanup.main([None, ingestion_doc_id]) if not resp == 0: print "Error cleaning up dashboard" continue # Bulk data being updated via elasticdump. # # resp = upload_bulk_data.main([None, ingestion_doc_id]) # if not resp == 0: # print "Error uploading bulk download data" # continue print "Ingestion complete!"
try: profile = json.load(f) except Exception, e: print "Error reading profile: %s" % e return False if args.pipeline in profile: pipeline = ",".join(profile[args.pipeline]) else: pipeline = args.pipeline provider = profile.get(u"name") contributor = profile.get(u"contributor", {}) # Create ingestion document couch = Couch() ingestion_doc_id = create_ingestion_document.main([None, args.profile_path]) ingestion_doc = couch.dashboard_db[ingestion_doc_id] # Update ingestion document kwargs = { "poll_storage_process/status": "running", "poll_storage_process/start_time": datetime.now().isoformat(), "poll_storage_process/end_time": None, } try: couch.update_ingestion_doc(ingestion_doc, **kwargs) except: print "Error updating ingestion document " + ingestion_doc["_id"] return # Back up current data
try: profile = json.load(f) except Exception, e: print "Error reading profile: %s" % e return False if args.pipeline in profile: pipeline = ",".join(profile[args.pipeline]) else: pipeline = args.pipeline provider = profile.get(u"name") contributor = profile.get(u"contributor", {}) # Create ingestion document couch = Couch() ingestion_doc_id = create_ingestion_document.main([None, args.profile_path]) ingestion_doc = couch.dashboard_db[ingestion_doc_id] # Update ingestion document kwargs = { "poll_storage_process/status": "running", "poll_storage_process/start_time": iso_utc_with_tz(), "poll_storage_process/end_time": None, } try: couch.update_ingestion_doc(ingestion_doc, **kwargs) except: print "Error updating ingestion document " + ingestion_doc["_id"] return # Back up current data