def check_request_actual_status(index_in_db, request, request_db):
req_id = int(request[REQUEST_ID_FIELD])
res = rc.get_status(req_id)
if res['status'].lower() == 'error':
logger.info("Request file_id: {} has failed".format(req_id))
request_db.loc[index_in_db,
REQUEST_STATUS_FIELD] = RequestStatus.FAILED.value
elif res['status'] == 'ok':
request_status = res['result']['status']
logger.info("Status of request {0} is {1}".format(
req_id, request_status))
if request_status == RequestStatus.ERROR.value:
request_db.loc[index_in_db,
REQUEST_STATUS_FIELD] = RequestStatus.ERROR.value
print(res)
if request_status == RequestStatus.COMPLETED.value:
request_db.loc[
index_in_db,
REQUEST_STATUS_FIELD] = RequestStatus.COMPLETED.value
else:
logger.error("Unhandled request status: {0} for request {1}".format(
res['status'], req_id))
request_db.to_csv(REQ_ID_PATH)
def download_request(req_id: str, target_dir):
logger.info("Downloading files from request {0} into {1}".format(
req_id, target_dir))
try:
download(req_id, target_dir)
except Exception as e:
logger.error("Downloading failed", exc_info=True)
raise e
def schedule_processing():
try:
job = schedule.every(15).minutes.do(
lambda: process_netCDF_files_to_csv())
job.run()
while True:
schedule.run_pending()
time.sleep(60)
except Exception as e:
logger.error(e, exc_info=True)
def scheduler(purge_done=False, tidy=False):
try:
job = schedule.every(60).minutes.do(
lambda: processor(purge_done, tidy))
job.run()
while True:
schedule.run_pending()
time.sleep(60)
except Exception as e:
logger.error(e, exc_info=True)
def prepare_and_start_processor(**kwargs):
if kwargs['send_only'] is False:
prepare_requests(**kwargs)
job = {}
try:
logger.info("Scheduling sender job.")
job = schedule.every(60).minutes.do(send_prepared_requests, kwargs)
except Exception as e:
logger.error(e, exc_info=True)
job.run()
while True:
schedule.run_pending()
time.sleep(60)
def download_completed_request(index_in_db, request, request_db):
req_id, req_type = str(int(request[REQUEST_ID_FIELD])), str(
request[REQUEST_TYPE_FIELD])
nlat, elon = str(request[NLAT_FIELD]), str(request[ELON_FIELD])
param, level = str(request[PARAM_FIELD]), str(request[LEVEL_FIELD])
download_target_path = get_download_target_path_tar(
req_id, req_type, nlat, elon, param, level)
os.makedirs(download_target_path, exist_ok=True)
try:
download_request(req_id, download_target_path)
request_db.loc[index_in_db,
REQUEST_STATUS_FIELD] = RequestStatus.DOWNLOADED.value
request_db.to_csv(REQ_ID_PATH)
except Exception as e:
logger.error(e, exc_info=True)