def delete_ingested_source():
done_dir = listdir(str(AppConstants.DONE_SOURCE_PATH))
for d_dir in done_dir:
path = str(AppConstants.DONE_SOURCE_PATH) + "/" + str(d_dir)
if isdir(path):
if check_delete_dates(path, done_dir):
rmtree(path)
db_handler(AppConstants.CLEANING, None, d_dir)
write_log("sourceHandler.py:\tDeleted " + str(path),
"[DELETE]")
def create_delete_date(path, uuid):
delete_date = (datetime.now() +
timedelta(days=30)).strftime('%Y-%m-%d %H:%M:%S')
db_handler(AppConstants.TRANSFER, AppConstants.UPDATE_DELETE_DATE,
delete_date, uuid)
write_log("sourceHandler.py:\t" + str(delete_date), "[DEBUG]")
delete_name = "delete_" + str(path).split("/")[-1] + "_" + str(delete_date)
write_log("sourceHandler.py:\t" + str(delete_name), "[DEBUG]")
subprocess.call(
['touch', (str(AppConstants.DONE_SOURCE_PATH) + "/" + delete_name)])
return
def restart_transfer_api_db(t_uuid):
failed_transfer = db_handler(AppConstants.TRANSFER, AppConstants.ONE_TRANSFER_UUID, t_uuid)
failed_counter = db_handler(AppConstants.TYPE_TRANSFER, AppConstants.COUNT_FAILED_TRANSFER, failed_transfer[1])
if failed_counter <= 2:
start_transfer_api(failed_transfer)
else:
failed_source = db_handler(AppConstants.SOURCE, AppConstants.ONE_SOURCE_ID, failed_transfer[1])
path_of_failed_source = str(AppConstants.SOURCE_LIST[failed_source[2]]) + "/" + str(failed_source[1])
if exists(path_of_failed_source):
write_log("Ingest " + t_uuid + " failed too often. Please check source!", "[FAILED]")
move(path_of_failed_source, str(AppConstants.FAILED_SOURCE_PATH))
return
def get_transfer_api(uuid):
t_status = json.loads(status_transfer(uuid).text)
if is_ingest(t_status):
if db_handler(AppConstants.TRANSFER, AppConstants.UPDATE_SIP_UUID_TRANSFER, uuid, t_status["sip_uuid"]):
return json.loads(status_ingest(t_status["sip_uuid"]).text)
else:
return t_status
def refresh_source_db(list_new_source):
for key in list_new_source:
for value in list_new_source[key]:
success = db_handler(AppConstants.SOURCE, AppConstants.INSERT, value, key)
if success:
write_log("Insert in DB from " + str(key) + "/" + str(value) + " was successful", "[INFO]")
else:
write_log(str(key) + "/" + str(value) + " already exist in DB", "[DEBUG]")
return
def refresh_transfer_list_db():
db_list = get_active_transfers_db()
if len(db_list) > 0:
for item in db_list:
item_uuid = item[5] # item[5] = t_uuid in transfer table
item_status = item[7] # item[7] = status in transfer table
transfer_item = get_transfer_api(item_uuid)
transfer_status = transfer_item["status"]
transfer_type = transfer_item["type"]
if item_status is not transfer_status:
db_handler(AppConstants.TRANSFER, AppConstants.UPDATE_STATUS_TRANSFER, transfer_status, item_uuid)
if transfer_status is AppConstants.FAILED:
restart_transfer_api_db(item_uuid)
if (transfer_status == str(AppConstants.COMPLETE)) and (not is_ingest(transfer_item)) and (transfer_type == str(AppConstants.TYPE_TRANSFER)):
db_handler(AppConstants.TRANSFER, AppConstants.UPDATE_STATUS_TRANSFER, AppConstants.FAILED, item_uuid)
restart_transfer_api_db(item_uuid)
elif (transfer_status == AppConstants.COMPLETE) and transfer_type == str(AppConstants.TYPE_INGEST):
source = get_source_db(item[1]) # item[1] = source in transfer table
move_source_to_done(str(AppConstants.SOURCE_DICT[source[2]]) + "/" + str(source[1]), item[6])
# else:
# write_log("No Transfer in DB.", "[INFO]")
return
def get_unstarted_source_from_db():
return db_handler(AppConstants.SOURCE, AppConstants.GET_UNSTARTED)
def get_sources_from_db():
return db_handler(AppConstants.SOURCE, AppConstants.GET_ALL)
def get_source_db(_id):
return db_handler(AppConstants.SOURCE, AppConstants.ONE_SOURCE_ID, _id)
def insert_transfer_db(s_id, s_name, acnumber, uuid, status, conf):
return db_handler(AppConstants.TRANSFER, AppConstants.INSERT, s_id, s_name, acnumber, uuid, status, conf)
def insert_sources_db(source_list):
for item in source_list:
db_handler(AppConstants.SOURCE, AppConstants.INSERT, item)
return
def get_all_transfers_ingests_db():
t_list = db_handler(AppConstants.TRANSFER, AppConstants.GET_ALL)
return t_list
def update_source(id):
return db_handler(AppConstants.SOURCE, AppConstants.UPDATE_STATUS_SOURCE, id)
def get_active_transfers_db():
return db_handler(AppConstants.TRANSFER, AppConstants.GET_ACTIVE)
def get_transfers_db():
return db_handler(AppConstants.TRANSFER, AppConstants.GET_ALL)