def check_request_actual_status(index_in_db, request, request_db): req_id = int(request[REQUEST_ID_FIELD]) res = rc.get_status(req_id) if res['status'].lower() == 'error': logger.info("Request file_id: {} has failed".format(req_id)) request_db.loc[index_in_db, REQUEST_STATUS_FIELD] = RequestStatus.FAILED.value elif res['status'] == 'ok': request_status = res['result']['status'] logger.info("Status of request {0} is {1}".format( req_id, request_status)) if request_status == RequestStatus.ERROR.value: request_db.loc[index_in_db, REQUEST_STATUS_FIELD] = RequestStatus.ERROR.value print(res) if request_status == RequestStatus.COMPLETED.value: request_db.loc[ index_in_db, REQUEST_STATUS_FIELD] = RequestStatus.COMPLETED.value else: logger.error("Unhandled request status: {0} for request {1}".format( res['status'], req_id)) request_db.to_csv(REQ_ID_PATH)
def download_request(req_id: str, target_dir): logger.info("Downloading files from request {0} into {1}".format( req_id, target_dir)) try: download(req_id, target_dir) except Exception as e: logger.error("Downloading failed", exc_info=True) raise e
def schedule_processing(): try: job = schedule.every(15).minutes.do( lambda: process_netCDF_files_to_csv()) job.run() while True: schedule.run_pending() time.sleep(60) except Exception as e: logger.error(e, exc_info=True)
def scheduler(purge_done=False, tidy=False): try: job = schedule.every(60).minutes.do( lambda: processor(purge_done, tidy)) job.run() while True: schedule.run_pending() time.sleep(60) except Exception as e: logger.error(e, exc_info=True)
def prepare_and_start_processor(**kwargs): if kwargs['send_only'] is False: prepare_requests(**kwargs) job = {} try: logger.info("Scheduling sender job.") job = schedule.every(60).minutes.do(send_prepared_requests, kwargs) except Exception as e: logger.error(e, exc_info=True) job.run() while True: schedule.run_pending() time.sleep(60)
def download_completed_request(index_in_db, request, request_db): req_id, req_type = str(int(request[REQUEST_ID_FIELD])), str( request[REQUEST_TYPE_FIELD]) nlat, elon = str(request[NLAT_FIELD]), str(request[ELON_FIELD]) param, level = str(request[PARAM_FIELD]), str(request[LEVEL_FIELD]) download_target_path = get_download_target_path_tar( req_id, req_type, nlat, elon, param, level) os.makedirs(download_target_path, exist_ok=True) try: download_request(req_id, download_target_path) request_db.loc[index_in_db, REQUEST_STATUS_FIELD] = RequestStatus.DOWNLOADED.value request_db.to_csv(REQ_ID_PATH) except Exception as e: logger.error(e, exc_info=True)