def delete_ingested_source(): done_dir = listdir(str(AppConstants.DONE_SOURCE_PATH)) for d_dir in done_dir: path = str(AppConstants.DONE_SOURCE_PATH) + "/" + str(d_dir) if isdir(path): if check_delete_dates(path, done_dir): rmtree(path) db_handler(AppConstants.CLEANING, None, d_dir) write_log("sourceHandler.py:\tDeleted " + str(path), "[DELETE]")
def create_delete_date(path, uuid): delete_date = (datetime.now() + timedelta(days=30)).strftime('%Y-%m-%d %H:%M:%S') db_handler(AppConstants.TRANSFER, AppConstants.UPDATE_DELETE_DATE, delete_date, uuid) write_log("sourceHandler.py:\t" + str(delete_date), "[DEBUG]") delete_name = "delete_" + str(path).split("/")[-1] + "_" + str(delete_date) write_log("sourceHandler.py:\t" + str(delete_name), "[DEBUG]") subprocess.call( ['touch', (str(AppConstants.DONE_SOURCE_PATH) + "/" + delete_name)]) return
def restart_transfer_api_db(t_uuid): failed_transfer = db_handler(AppConstants.TRANSFER, AppConstants.ONE_TRANSFER_UUID, t_uuid) failed_counter = db_handler(AppConstants.TYPE_TRANSFER, AppConstants.COUNT_FAILED_TRANSFER, failed_transfer[1]) if failed_counter <= 2: start_transfer_api(failed_transfer) else: failed_source = db_handler(AppConstants.SOURCE, AppConstants.ONE_SOURCE_ID, failed_transfer[1]) path_of_failed_source = str(AppConstants.SOURCE_LIST[failed_source[2]]) + "/" + str(failed_source[1]) if exists(path_of_failed_source): write_log("Ingest " + t_uuid + " failed too often. Please check source!", "[FAILED]") move(path_of_failed_source, str(AppConstants.FAILED_SOURCE_PATH)) return
def get_transfer_api(uuid): t_status = json.loads(status_transfer(uuid).text) if is_ingest(t_status): if db_handler(AppConstants.TRANSFER, AppConstants.UPDATE_SIP_UUID_TRANSFER, uuid, t_status["sip_uuid"]): return json.loads(status_ingest(t_status["sip_uuid"]).text) else: return t_status
def refresh_source_db(list_new_source): for key in list_new_source: for value in list_new_source[key]: success = db_handler(AppConstants.SOURCE, AppConstants.INSERT, value, key) if success: write_log("Insert in DB from " + str(key) + "/" + str(value) + " was successful", "[INFO]") else: write_log(str(key) + "/" + str(value) + " already exist in DB", "[DEBUG]") return
def refresh_transfer_list_db(): db_list = get_active_transfers_db() if len(db_list) > 0: for item in db_list: item_uuid = item[5] # item[5] = t_uuid in transfer table item_status = item[7] # item[7] = status in transfer table transfer_item = get_transfer_api(item_uuid) transfer_status = transfer_item["status"] transfer_type = transfer_item["type"] if item_status is not transfer_status: db_handler(AppConstants.TRANSFER, AppConstants.UPDATE_STATUS_TRANSFER, transfer_status, item_uuid) if transfer_status is AppConstants.FAILED: restart_transfer_api_db(item_uuid) if (transfer_status == str(AppConstants.COMPLETE)) and (not is_ingest(transfer_item)) and (transfer_type == str(AppConstants.TYPE_TRANSFER)): db_handler(AppConstants.TRANSFER, AppConstants.UPDATE_STATUS_TRANSFER, AppConstants.FAILED, item_uuid) restart_transfer_api_db(item_uuid) elif (transfer_status == AppConstants.COMPLETE) and transfer_type == str(AppConstants.TYPE_INGEST): source = get_source_db(item[1]) # item[1] = source in transfer table move_source_to_done(str(AppConstants.SOURCE_DICT[source[2]]) + "/" + str(source[1]), item[6]) # else: # write_log("No Transfer in DB.", "[INFO]") return
def get_unstarted_source_from_db(): return db_handler(AppConstants.SOURCE, AppConstants.GET_UNSTARTED)
def get_sources_from_db(): return db_handler(AppConstants.SOURCE, AppConstants.GET_ALL)
def get_source_db(_id): return db_handler(AppConstants.SOURCE, AppConstants.ONE_SOURCE_ID, _id)
def insert_transfer_db(s_id, s_name, acnumber, uuid, status, conf): return db_handler(AppConstants.TRANSFER, AppConstants.INSERT, s_id, s_name, acnumber, uuid, status, conf)
def insert_sources_db(source_list): for item in source_list: db_handler(AppConstants.SOURCE, AppConstants.INSERT, item) return
def get_all_transfers_ingests_db(): t_list = db_handler(AppConstants.TRANSFER, AppConstants.GET_ALL) return t_list
def update_source(id): return db_handler(AppConstants.SOURCE, AppConstants.UPDATE_STATUS_SOURCE, id)
def get_active_transfers_db(): return db_handler(AppConstants.TRANSFER, AppConstants.GET_ACTIVE)
def get_transfers_db(): return db_handler(AppConstants.TRANSFER, AppConstants.GET_ALL)