Exemplo n.º 1
0
def main(argv):
    parser = define_arguments()
    args = parser.parse_args(argv[1:])

    for profile_path in args.profile_path:
        print "Creating ingestion document for profile %s" % profile_path
        ingestion_doc_id = create_ingestion_document.main([None, profile_path])
        if ingestion_doc_id is None:
            print "Error creating ingestion document"
            continue

        resp = fetch_records.main([None, ingestion_doc_id])
        if not resp == 0:
            print "Error fetching records"
            continue

        resp = enrich_records.main([None, ingestion_doc_id])
        if not resp == 0:
            print "Error enriching records"
            continue

        resp = save_records.main([None, ingestion_doc_id])
        if not resp == 0:
            print "Error saving records"
            continue

        resp = remove_deleted_records.main([None, ingestion_doc_id]) 
        if not resp == 0:
            print "Error saving records..."
            continue

        print "Ingestion complete!"
def main(argv):
    parser = define_arguments()
    args = parser.parse_args(argv[1:])

    for profile_path in args.profile_path:
        print "Creating ingestion document for profile %s" % profile_path
        ingestion_doc_id = create_ingestion_document.main([None, profile_path])
        if ingestion_doc_id is None:
            print "Error creating ingestion document"
            continue

        resp = fetch_records.main([None, ingestion_doc_id])
        if not resp == 0:
            print "Error fetching records"
            continue

        resp = enrich_records.main([None, ingestion_doc_id])
        if not resp == 0:
            print "Error enriching records"
            continue

        resp = save_records.main([None, ingestion_doc_id])
        if not resp == 0:
            print "Error saving records"
            continue

        resp = remove_deleted_records.main([None, ingestion_doc_id]) 
        if not resp == 0:
            print "Error deleting records"
            continue

        resp = check_ingestion_counts.main([None, ingestion_doc_id])
        if not resp == 0:
            print "Error checking counts"
            continue

        resp = dashboard_cleanup.main([None, ingestion_doc_id])
        if not resp == 0:
            print "Error cleaning up dashboard"
            continue

        # Bulk data being updated via elasticdump. 
        #
        # resp = upload_bulk_data.main([None, ingestion_doc_id])
        # if not resp == 0:
        #     print "Error uploading bulk download data"
        #     continue

        print "Ingestion complete!"