def do_avhrr_calibration(process_config, msg, timestamp): if not process_config['process_avhrr']: LOG.debug("Skipping avhrr processing due to config or missing data.") return True accepted_return_codes_avhrcl = [0] return_value = True LOG.debug("Do the avhrr calibration") # This function relays on beeing in a working directory current_dir = os.getcwd( ) # Store the dir to change back to after function complete os.chdir(process_config['aapp_processes'][process_config.process_name] ['working_dir']) # calibration_location = "-c -l" try: cmd = "avhrcl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format( process_config['calibration_location'], process_config['platform_name'], timestamp, int(process_config['orbit_number']), process_config['aapp_static_configuration']['decommutation_files'] ['avhrr_file']) except Exception as err: LOG.error("Failed to build avhrcl command: {}".format(err)) try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Command {} failed.".format(cmd)) else: if returncode in accepted_return_codes_avhrcl: LOG.info("Command {} complete.".format(cmd)) else: LOG.error("Command {} failed with return code {}.".format( cmd, returncode)) return_value = False # Change back after this is done os.chdir(current_dir) LOG.info("do_avhrr_calibration complete!") return return_value
def do_tle_satpos(config, timestamp, satellite): return_status = True LOG.info("satpos files is stored under the dir_navigation/satpos") satpos_dir = os.path.join(os.environ['DIR_NAVIGATION'], "satpos") file_satpos = os.path.join( satpos_dir, "satpos_{}_{:%Y%m%d}.txt".format(satellite, timestamp)) if (not os.path.exists(file_satpos) or os.stat(file_satpos).st_size == 0) and return_status: """Usage is: satpostle [ -o] [-s satellite] [-S station] [-d start date] [-n number of days] [-i increment in seconds] [-c search criteria] -o -s -S -d -n -i –c are optional. If no parameter is specified as an option, defaults are : noaa14, Lannion, today 0h, 1.0, 120.0, n (n= nearest, p = preceding). The option -o specifies that the data will be stored in the file satpos_noaxx_yyyymmdd.txt. Output default is the standard output.. """ cmd = "satpostle -o -s {} -d {:%d/%m/%y} -n 1.2".format( satellite, timestamp) try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Failed to run command: {}".format(cmd)) return_status = False else: if returncode != 0: LOG.error("cmd: {} failed with returncode: {}".format( cmd, returncode)) return_status = False elif not os.path.exists(file_satpos): LOG.error( "file: {} does not exists after satpostle run.".format( file_satpos)) return_status = False else: LOG.info("satpos file already there. Use this") return return_status
def do_ana_correction(process_config, msg, timestamp): """The software ANA is 3party. User will have to get it themself. Further the ana binaries and scripts must be in the PATH, preferable in the primary AAPP bin directory Also it relays on reference_landmarks directory under ENV(DIR_NAVIGATION)/ana dir This needs to be copied/installed here by the user TODO: Replace the ana_lmk_loc script with code here to better be able to control the directories used in the processing """ if not process_config['process_avhrr']: LOG.debug( "Skipping ANA as AVHRR is not processed de to config or missing data." ) return True try: if 'do_ana_correction' in process_config['aapp_processes'][ process_config.process_name]: if not process_config['aapp_processes'][ process_config.process_name]['do_ana_correction']: return True except Exception as err: LOG.error("Failed with: {}".format(err)) return_status = True # This function relays on beeing in a working directory current_dir = os.getcwd( ) # Store the dir to change back to after function complete os.chdir(process_config['aapp_processes'][process_config.process_name] ['working_dir']) # Must check of the ana dir exists ana_dir = os.path.join(os.getenv('DIR_NAVIGATION'), 'ana') if not os.path.exists(ana_dir): try: os.makedirs(ana_dir) os.environ['DIR_ANA'] = ana_dir except OSError in e: LOG.error( "Failed to create directory: {}. This is needed to run the ANA software." ) return_status = False # And the ana dir must contain a reference_landmarks directory to work! if not os.path.exists(os.path.join(ana_dir, "reference_landmarks")): LOG.error( "Can not run ANA because reference_landmarks under the ana dir is missing." ) return_status = False if return_status: # Find all matching landmarks cmd = "ana_lmk_loc -D {}".format( process_config['aapp_static_configuration']['decommutation_files'] ['avhrr_file']) try: status = False status, returncode, std, err = run_shell_command( cmd, stdout_logfile="ana_lmk_loc.log", stderr_logfile="ana_lmk_loc.err") except: import sys LOG.error("Command {} failed with {}.".format( cmd, sys.exc_info()[0])) if not status: return_status = False else: if not status or returncode != 0: LOG.error("Command {} failed with {}".format(cmd, returncode)) _ana_file = open('ana_lmk_loc.err', "w") _ana_file.write(std) _ana_file.write(err) _ana_file.close() return_status = False if return_status: # Check of ana landmark location file with correct time stamp exist. # If not try to find correct timestamp and copy to this. expected_ana_loc_file = "lmkloc_{}_{:%Y%m%d_%H%M}_{:05d}.txt".format( process_config['platform_name'], timestamp, process_config['orbit_number']) if not os.path.exists( os.path.join(ana_dir, "{:%Y-%m}".format(timestamp), expected_ana_loc_file)): # Need to run this AAPP command to detect the start of the hrpt.l1b(avhrr data) file # Because this might differ from the time stamp in file name cmd = "l1bidf.exe" try: status, returncode, std, err = run_shell_command( cmd, stdin="{}\n".format( process_config['aapp_static_configuration'] ['decommutation_files']['avhrr_file'])) except: import sys LOG.error("Command {} failed with {}.".format( cmd, sys.exc_info()[0])) if not status: return_status = False else: if not status or returncode != 0: LOG.error("Command {} failed with {}".format( cmd, returncode)) _ana_file = open('ana_lmk_loc.err', "w") _ana_file.write(std) _ana_file.write(err) _ana_file.close() return_status = False else: l1bidf_list = std.split() l1bidf_aapp_datetime = datetime.strptime( l1bidf_list[1] + l1bidf_list[2], "%Y%m%d%H%M") LOG.debug("l1b idf aapp datetime: {}".format( l1bidf_aapp_datetime)) # estatt_file_name = lmkloc_metop02_20160617_0952_50132.txt ana_loc_file = "lmkloc_{}_{:%Y%m%d_%H%M}_{:05d}.txt".format( l1bidf_list[0], l1bidf_aapp_datetime, int(l1bidf_list[3])) if os.path.exists( os.path.join( ana_dir, "{:%Y-%m}".format(l1bidf_aapp_datetime), ana_loc_file)): LOG.debug( "timestamp {} l1bidf_aapp_datetime: {}".format( timestamp, l1bidf_aapp_datetime)) from shutil import copy2 copy2( os.path.join(ana_dir, "{:%Y-%m}".format(timestamp), ana_loc_file), os.path.join( ana_dir, "{:%Y-%m}".format(l1bidf_aapp_datetime), expected_ana_loc_file)) else: LOG.warning("Could not find ana loc file: {}".format( ana_loc_file)) else: LOG.debug("File ok: {}".format( os.path.join(ana_dir, "{:%Y-%m}".format(timestamp), expected_ana_loc_file))) import hashlib if return_status: # LOG.debug("sha256 of aapp input avhrr_file: {}".format(hashlib.sha256(open( # process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest())) # Calculate correction from landmarks and update avhrr_file with these new attitude coefisients. cmd = "ana_estatt -s {0} -d {1:%Y%m%d} -h {1:%H%M} -n {2:05d}".format( process_config['platform_name'], timestamp, process_config['orbit_number']) try: status, returncode, std, err = run_shell_command(cmd) except: import sys LOG.error("Command {} failed with {}.".format( cmd, sys.exc_info()[0])) if not status: return_status = False else: if not status or returncode != 0: LOG.error("Command {} failed with {}".format(cmd, returncode)) _ana_file = open('ana_lmk_loc.err', "w") _ana_file.write(std) _ana_file.write(err) _ana_file.close() return_status = False # LOG.debug("sha256 of aapp input avhrr_file: {}".format(hashlib.sha256(open( # process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest())) sha256_before_correction = hashlib.sha256( open( process_config['aapp_static_configuration'] ['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest() if return_status: # Recalculate the location in the avhhr data file with the new correction attitude coefisients. from do_avhrr_calibration import do_avhrr_calibration LOG.info( "Need to recalculate the avhrcl with new attitude coefisients from ANA." ) if not do_avhrr_calibration(process_config, msg, timestamp): LOG.warning( "The avhrr location with ana correction failed for some reason." "It might be that the processing can continue") LOG.warning( "Please check the previous log carefully to see if this is an error you can accept." ) return_status = False if return_status: sha256_after_correction = hashlib.sha256( open( process_config['aapp_static_configuration'] ['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest() LOG.debug("sha256 of aapp input avhrr_file BEFORE ana: {}".format( sha256_before_correction)) LOG.debug("sha256 of aapp input avhrr_file AFTER ana: {}".format( sha256_after_correction)) if (sha256_before_correction == sha256_after_correction): LOG.warning( "The correction of the avhrr location with data from ANA did not take place for some reason," "or the correction was 0 for all pitch yaw and roll.") else: LOG.info( "The correction of AVHRR location with data from ANA was performed." ) # Change back after this is done os.chdir(current_dir) LOG.info("do_ana_correction complete!") return return_status
def do_hirs_calibration(process_config, msg, timestamp): return_status = True if not process_config['process_hirs']: return True # A list of accepted return codes for the various scripts/binaries run in this function accepted_return_codes_hirs_historic_file_manage = [0] # This function relays on beeing in a working directory current_dir = os.getcwd( ) # Store the dir to change back to after function complete os.chdir(process_config['aapp_processes'][process_config.process_name] ['working_dir']) hirs_version_use = None hirs_version = os.getenv('HIRSCL_VERSION', 0) hirs_version_list = hirs_version.split() hirs_sats = os.getenv('HIRSCL_SAT', 'default') hirs_sat_list = hirs_sats.split() index = 0 for sat in hirs_sat_list: if sat in process_config['platform_name']: hirs_version_use = hirs_version_list[index] else: hirs_version_def = hirs_version_list[index] index += 1 if hirs_version_use is None: hirs_version_use = hirs_version_def hirs_script = "hirscl" hirs_err_file = "hirscl.err" calibration_location = process_config['calibration_location'] LOG.debug("hirs_version_use {}".format(hirs_version_use)) # pdb.set_trace() if int(hirs_version_use) > 1: # no calibration, just navigation calibration_location = "-l" elif int(hirs_version_use) == 0 or "".join( process_config['a_tovs']) == 'TOVS': calibration_location = "-c -l" elif int(hirs_version_use) == 1: file_historic = os.path.join(os.getenv('PAR_CALIBRATION_MONITOR'), process_config['platform_name'], "hirs_historic.txt") if os.path.exists(file_historic): cmd = "hirs_historic_file_manage -m {} -r {} -n {} {}".format( os.getenv('HIST_SIZE_HIGH'), os.getenv('HIST_SIZE_LOW'), os.getenv('HIST_NMAX'), file_historic) try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Command {} failed.".format(cmd)) return_status = False else: if returncode in accepted_return_codes_hirs_historic_file_manage: LOG.debug("Command complete.") else: LOG.error("Command {} failed with returncode {}".format( cmd, returncode)) LOG.error("stdout was: {}".format(std)) LOG.error("stderr was: {}".format(err)) return_status = False if return_status: cmd = "hcalcb1_algoV4 -s {0} -y {1:%Y} -m {1:%m} -d {1:%d} -h {1:%H} -n {1:%M}".format( process_config['platform_name'], timestamp) try: status, returncode, std, err = run_shell_command(cmd) except: import sys LOG.error("Command {} failed with {}.".format( cmd, sys.exc_info()[0])) else: if returncode != 0: LOG.error("Command {} failed with {}".format( cmd, returncode)) _hirs_file = open(hirs_err_file, "w") _hirs_file.write(std) _hirs_file.write(err) _hirs_file.close() return_status = False hirs_script = "hirscl_algoV4" hirs_err_file = "hirscl_algoV4.err" calibration_location = "-c -l" else: LOG.error( "Can not figure out which hirs calibration algo version to use.") return_status = False if return_status: try: cmd = "{} {} -s {} -d {:%Y%m%d} -h {:%H%M} -n {:05d} {}".format( hirs_script, calibration_location, process_config['platform_name'], timestamp, timestamp, int(process_config['orbit_number']), process_config['aapp_static_configuration'] ['decommutation_files']['hirs_file']) except KeyError as ke: LOG.error( "Building command string failed with key error: {}".format(ke)) try: status, returncode, out, err = run_shell_command( cmd, stdout_logfile="{}.log".format(hirs_script), stderr_logfile="{}".format(hirs_err_file)) except: import sys LOG.error("Command {} failed {}.".format(cmd, sys.exc_info()[0])) else: if (returncode != 0): LOG.error("Command {} failed with {}".format(cmd, returncode)) _hirs_file = open(hirs_err_file, "w") _hirs_file.write(out) _hirs_file.write(err) _hirs_file.close() return_status = False # Change back after this is done os.chdir(current_dir) LOG.info("do_hirs_calibration complete!") return return_status
def setup_aapp_processing(config): """ Setup various env variables needed for the aapp processing and set eg working dir """ if (not 'working_dir' in config['aapp_processes'][config.process_name] and 'use_dyn_work_dir' in config['aapp_processes'][config.process_name] and config['aapp_processes'][config.process_name]['use_dyn_work_dir']): try: config['aapp_processes'][ config.process_name]['working_dir'] = tempfile.mkdtemp( dir=config['aapp_processes'][ config.process_name]['aapp_workdir']) LOG.debug("working dir set based on aapp_workdir and tmp " + str( config['aapp_processes'][config.process_name]['working_dir'])) except OSError: config['aapp_processes'][ config.process_name]['working_dir'] = tempfile.mkdtemp() except KeyError as ke: LOG.error(": {}".format(ke)) raise finally: LOG.info("Create new working dir...") elif not 'working_dir' in config['aapp_processes'][config.process_name]: try: config['aapp_processes'][ config.process_name]['working_dir'] = config['aapp_processes'][ config.process_name]['aapp_workdir'] except KeyError as ke: LOG.error( "working_dir nor aapp_workdir is given in config. Dont know where to store what I do." ) raise LOG.info("Working dir = " + str(config['aapp_processes'][config.process_name]['working_dir'])) os.environ["AAPP_PREFIX"] = config['aapp_processes'][ config.process_name]['aapp_prefix'] aapp_atovs_conf = os.path.join( os.environ["AAPP_PREFIX"], config['aapp_processes'][config.process_name]['aapp_environment_file']) status, returncode, out, err = run_shell_command( "bash -c \"source {}\";env".format(aapp_atovs_conf)) if not status: LOG.error("Failed to run the bash source env command for " + str(aapp_atovs_conf)) return False else: for line in out.splitlines(): if line: (key, _, value) = line.partition("=") os.environ[key] = value # Default AAPP config for PAR_NAVIGATION_DEFAULT_LISTESAT Metop platform is M01, M02, M04 # but needed names are metop01 etc. Replace this inside the processing # from now on. aapp_satellite_list = os.getenv('PAR_NAVIGATION_DEFAULT_LISTESAT').split() if config['platform_name'] not in aapp_satellite_list: LOG.warning( "Can not find this platform in AAPP config variable PAR_NAVIGATION_DEFAULT_LISTESAT. Will try to find matches. But it can be a good idea to change this variable in the ATOVS_ENV7 file." ) LOG.warning("Platform {} not in list: {}".format( config['platform_name'], aapp_satellite_list)) if 'metop' in config['platform_name'] and ( ('M01' or 'M02' or 'M03' or 'M04') in aapp_satellite_list): LOG.debug("Replace in this processing") PAR_NAVIGATION_DEFAULT_LISTESAT = os.getenv( 'PAR_NAVIGATION_DEFAULT_LISTESAT') PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace( 'M01', 'metop01') PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace( 'M02', 'metop02') PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace( 'M03', 'metop03') PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace( 'M04', 'metop04') os.environ[ 'PAR_NAVIGATION_DEFAULT_LISTESAT'] = PAR_NAVIGATION_DEFAULT_LISTESAT LOG.debug("New LISTESAT: {}".format( os.getenv('PAR_NAVIGATION_DEFAULT_LISTESAT'))) list_of_needed_programs = [ 'tleing.exe', 'satpostle', 'decommutation.exe', 'chk1btime.exe', 'decom-amsua-metop', 'decom-mhs-metop', 'decom-hirs-metop', 'decom-avhrr-metop', 'hirs_historic_file_manage', 'hcalcb1_algoV4', 'msucl', 'amsuacl', 'amsubcl', 'mhscl', 'avhrcl', 'atovin', 'atovpp', 'l1didf' ] for program in list_of_needed_programs: if not which(program): LOG.error( "Can not find needed AAPP program '{}' in environment. Please check." .format(program)) return False return True
def download_tle(config, timestamp, dir_data_tle): user = os.getenv("PAR_NAVIGATION_TLE_USER", "xxxxxx") passwd = os.getenv("PAR_NAVIGATION_TLE_PASSWD", "xxxxxx") url = os.getenv("PAR_NAVIGATION_TLE_URL_DOWNLOAD") timeout = 60 catalogue = "25338,26536,27453,28654,33591,37849,29499,38771,27431,32958,37214,25994,27424" tle_infile = "" tle_dict = {} tle_dict['timestamp'] = timestamp tle_cnf = [] try: tle_cnf = config['aapp_processes'][config.process_name]['tle_download'] except KeyError as ke: LOG.info("No tle_download config found. Using the default.") pass finally: tle_cnf.append({'url': url, 'user': user, 'passwd': passwd, 'timeout': timeout, 'catalogue': catalogue}) try: tle_infile = compose(config['aapp_processes'][config.process_name]['tle_infile_format'], tle_dict) except KeyError as ke: if 'tle_infile_format' in ke: tle_infile = compose('tle_{timestamp:%Y%m%d_%H%M}.txt', tle_dict) LOG.warning("Using default TLE file name format: %s", tle_infile) pass else: LOG.error("Key error: {}".format(ke)) LOG.error("Valid keys :") for key in tle_dict.keys(): LOG.error("{}".format(key)) raise except: raise tle_file_list = [] for cnf in tle_cnf: for cnf_url in cnf['url'].split(): LOG.debug("Will try to download TLE from {}.".format(cnf_url)) if "space-track" in cnf_url: # Do the special space-track login status = False returncode = 0 stdout = "" stderr = "" cmd = "wget -T {} --post-data=\"identity={}&password={}\" --cookies=on ", "--keep-session-cookies --save-cookies=cookies_spacetrack \"{}/ajaxauth/login\" -olog".format( cnf['timeout'], cnf['user'], cnf['passwd'], cnf_url) try: status, returncode, stdout, stderr = run_shell_command(cmd) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) return_status = False else: if returncode != 0: LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) else: cmd = "wget -T {} --keep-session-cookies --load-cookies=cookies_spacetrack -O weather.txt ", "\"{}/basicspacedata/query/class/tle_latest/ORDINAL/1/NORAD_CAT_ID/{}/orderby/TLE_LINE1\"".format( cnf['timeout'], cnf_url, cnf['catalogue']) try: status, returncode, stdout, stderr = run_shell_command(cmd) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) else: if returncode != 0: LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) else: LOG.debug("TLE download ok") if os.path.exists("weather.txt"): try: tle_file = open("weather.txt", 'r') tle_string = tle_file.read() tle_file.close() tle_file_out = os.path.join(dir_data_tle, tle_infile) with open(tle_file_out, "a") as tle_file: tle_file.write(tle_string) if tle_file_out not in tle_file_list: tle_file_list.append(tle_file_out) except Exception as ex: LOG.debug("Failed rename tle download file: {}".format(ex)) raise else: import urllib2 f = urllib2.urlopen(cnf_url) tle_string = f.read() f.close() tle_file_out = os.path.join(dir_data_tle, tle_infile) with open(tle_file_out, "a") as tle_file: tle_file.write(tle_string) if tle_file_out not in tle_file_list: tle_file_list.append(tle_file_out) return tle_file_list
def do_tleing(config, timestamp, satellite): """Get the tle-file and copy them to the AAPP data structure and run the AAPP tleing script and executable""" return_status = True # This function relays on beeing in a working directory try: current_dir = os.getcwd() # Store the dir to change back to after function complete except OSError as ose: LOG.error("Failed to get current working dir: {}".format(ose)) raise os.chdir(config['aapp_processes'][config.process_name]['working_dir']) tle_match_tests = (('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2})(\d{2}).*', _do_6_matches), ('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2}).*', _do_5_matches), ('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2}).*', _do_4_matches), ('.*(\d{4})(\d{2})(\d{2}).*', _do_3_matches), ('.*(\d{2})(\d{2})(\d{2}).*', _do_3_matchesYY)) if '7' in os.getenv('AAPP_PREFIX'): aapp_env = 'AAPP_ENV7' elif '8' in os.getenv('AAPP_PREFIX'): aapp_env = 'AAPP_ENV8' else: aapp_env = 'AAPP_ENV' if 'dir_navigation' in config['aapp_processes'][config.process_name]: LOG.warning("Override the env variable set in {} DIR_NAVIGATION from {} to {}.".format( aapp_env, os.environ['DIR_NAVIGATION'], config['aapp_processes'][config.process_name]['dir_navigation'])) os.environ['DIR_NAVIGATION'] = config['aapp_processes'][config.process_name]['dir_navigation'] # Need to update DIR_DATA_TLE dir to be sure this is corect if not tle_indir is given os.environ['DIR_DATA_TLE'] = os.path.join(os.getenv('DIR_NAVIGATION'), 'tle_db') if 'tle_indir' in config['aapp_processes'][config.process_name]: tle_indir = config['aapp_processes'][config.process_name]['tle_indir'] LOG.warning("Override the env variable set in {} DIR_DATA_TLE from {} to {}.".format( aapp_env, os.environ['DIR_DATA_TLE'], tle_indir)) os.environ['DIR_DATA_TLE'] = tle_indir # variables for the TLE HOME directory DIR_DATA_TLE = os.getenv('DIR_DATA_TLE', os.path.join(os.getenv('DIR_NAVIGATION'), 'orb_elem')) # This is needed by AAPP tleing. Try other if not existing if not os.path.exists(DIR_DATA_TLE): LOG.warning("Dir " + DIR_DATA_TLE + " does not exist. Create ") try: os.makedirs(DIR_DATA_TLE) except: LOG.error("Failed to create %s. Can not handle TLEs without this", DIR_DATA_TLE) raise if 'tle_file_to_data_diff_limit_days' in config['aapp_processes'][config.process_name]: select_closest_tle_file_to_data = True min_closest_tle_file = int( config['aapp_processes'][config.process_name]['tle_file_to_data_diff_limit_days']) * 24 * 60 * 60 else: select_closest_tle_file_to_data = False TLE_INDEX = os.path.join(DIR_DATA_TLE, "tle_{}.index".format(satellite)) tle_search_dir = DIR_DATA_TLE tle_file_list = [] # dict to hold needed tle keys tle_dict = {} if not select_closest_tle_file_to_data: if os.path.exists(TLE_INDEX): # Loop over all tle files, and only do tle tle_files = [s for s in glob(os.path.join(DIR_DATA_TLE, 'tle*txt')) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))] tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE, s))) tle_index_mtime = os.path.getmtime(TLE_INDEX) for s in tle_files: if os.path.getmtime(os.path.join(DIR_DATA_TLE, s)) > tle_index_mtime: tle_file_list.append(s) if len(tle_file_list) == 0: import time LOG.warning(("No newer tle files than last update of the index file. " + "Last update of index file is {:d}s. If more than a few days you should check.".format( int(time.time() - tle_index_mtime)))) else: LOG.info("Will use tle files {}".format(tle_file_list)) else: LOG.warning("index file does not exist. If this is the first run of AAPP tleing.exe it is ok," " otherwise it is a bit suspisiuos.") try: tle_files = [s for s in os.listdir(DIR_DATA_TLE) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))] tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE, s))) tle_file_list = tle_files except OSError: LOG.warning("Found to tle files .... ") else: # dict to hold needed tle keys tle_dict['timestamp'] = timestamp try: infile = compose(config['aapp_processes'][config.process_name]['tle_infile_format'], tle_dict) except KeyError as ke: LOG.error("Key error: {}".format(ke)) LOG.error("Valid keys :") for key in tle_dict.keys(): LOG.error("{}".format(key)) raise except: raise LOG.debug("tle file name: {}".format(infile)) # Check if I can read the tle file. first_search = True # FIXME: In AAPP default get_tle script direcory timestamp is TLE_MONTH=`date +%Y-\%m` for tle_search_dir in [compose(os.path.join(DIR_DATA_TLE, "{timestamp:%Y_%m}"), tle_dict), DIR_DATA_TLE]: if not os.path.exists(tle_search_dir): LOG.debug("tle_search_dir {} does not exists.".format(tle_search_dir)) continue LOG.debug("tle_search_dir {}".format(tle_search_dir)) try: with open(os.path.join(tle_search_dir, infile)) as tle_file: del tle_file_list[:] tle_file_list.append(os.path.join(tle_search_dir, infile)) min_closest_tle_file = 0 pass except IOError as e: LOG.warning("Could not find tle file: {}. Try find closest ... ".format(infile)) tle_file_list = glob(os.path.join(tle_search_dir, '*')) LOG.debug("tle file list: {}".format(tle_file_list)) LOG.debug(tle_file_list) infile_closest = "" for tle_file_name in tle_file_list: for regex, test in tle_match_tests: m = re.match(regex, tle_file_name) if m: try: LOG.debug("{} {}".format(tle_file_name, test(m))) delta = timestamp - test(m) if (abs(delta.total_seconds()) < min_closest_tle_file): min_closest_tle_file = abs(delta.total_seconds()) # infile_closest = os.path.basename(tle_file_name) infile_closest = tle_file_name LOG.debug("Closest tle infile so far: {}".format(infile_closest)) except ValueError: pass if infile_closest: del tle_file_list[:] tle_file_list.append(infile_closest) break else: if not first_search: LOG.error("Could not find tle file close enough to timestamp {} with limit {}".format( timestamp, min_closest_tle_file)) LOG.error("Update your TLE files or adjust the limit(Not recomended!).") first_search = False else: break # DIR_DATA_TLE = tle_search_dir if tle_file_list: LOG.debug("Use this: {} offset {}s".format(tle_file_list, min_closest_tle_file)) if not tle_file_list and config['aapp_processes'][config.process_name]['download_tle_files']: LOG.warning("Found no tle files. Try to download ... ") tle_file_list = download_tle(config, timestamp, DIR_DATA_TLE) for tle_file in tle_file_list: archive = False # SATellite IDentification mandatory # so take care of default values os.environ['SATID_FILE'] = os.getenv('SATID_FILE', 'satid.txt') """Don't use the tle_indir because this is handeled by the tleing script""" if (DIR_DATA_TLE != tle_search_dir): tle_filename = compose(os.path.join("{timestamp:%Y_%m}", os.path.basename(tle_file)), tle_dict) else: tle_filename = os.path.basename(tle_file) status = False returncode = 0 stdout = "" stderr = "" cmd = "tleing.exe" stdin = "{}\n{}\n{}\n{}\n".format(DIR_DATA_TLE, tle_filename, satellite, TLE_INDEX) LOG.debug('stdin arguments to command: ' + str(stdin)) try: status, returncode, stdout, stderr = run_shell_command(cmd, stdin=stdin) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) return_status = False else: if returncode != 0: LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) elif not os.path.exists(TLE_INDEX): LOG.error("index file: {} does not exist after tleing. Something is wrong.".format(TLE_INDEX)) LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) else: LOG.debug("Running command: {} with return code: {}".format(cmd, returncode)) LOG.debug("stdout: {}".format(stdout)) LOG.debug("stderr: {}".format(stderr)) LOG.debug("DIR_DATA_TLE : {}".format(DIR_DATA_TLE)) LOG.debug("tle_file : {}".format(os.path.basename(tle_file))) LOG.debug("satellite : {}".format(satellite)) LOG.debug("TLE_INDEX : {}".format(TLE_INDEX)) # When a index file is generated above one line is added for each tle file. # If several tle files contains equal TLEs each of these TLEs generate one line in the index file # To avoid this, sort the index file keeping only unique lines(skipping the tle filename at the end # The sort options +0b -3b is guessed to be sort from column 0 to 3, but this is not documented # Could cause problems with future version of sort. # See eg. http://search.cpan.org/~sdague/ppt-0.12/bin/sort # cmd="sort -u -o {} +0b -3b {}".format(os.path.join(DIR_DATA_TLE, "{}.sort".format(TLE_INDEX)), # os.path.join(DIR_DATA_TLE, TLE_INDEX)) if os.path.exists(TLE_INDEX): cmd = "sort -u +0b -3b {}".format(TLE_INDEX) try: status, returncode, stdout, stderr = run_shell_command( cmd, stdout_logfile="{}.sort1".format(TLE_INDEX)) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) return_status = False else: if returncode == 0 and os.path.exists("{}.sort1".format(TLE_INDEX)): cmd = "grep -v NaN {}.sort1".format(TLE_INDEX) try: status, returncode, stdout, stderr = run_shell_command( cmd, stdout_logfile="{}.sort".format(TLE_INDEX)) except: LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode)) LOG.error("stdout: {}".format(stdout)) LOG.error("stderr: {}".format(stderr)) return_status = False else: try: os.remove(TLE_INDEX) os.remove("{}.sort1".format(TLE_INDEX)) except OSError as e: LOG.error("Failed to remove unsorted and duplicated index file: {} with {}" .format(TLE_INDEX, e)) else: try: os.rename("{}.sort".format(TLE_INDEX), TLE_INDEX) archive = True except: LOG.error("Failed to rename sorted index file to original name.") else: LOG.error("Returncode other than 0: {} or tle index sort file does exists.".format( returncode, "{}.sort".format(TLE_INDEX))) else: LOG.error("tle index file: {} does not exists after tleing before sort. This can not happen.") # If a new tle is used and archive dir is given in config, copy TLEs to archive if archive and ('tle_archive_dir' in config['aapp_processes'][config.process_name]): archive_dict = {} archive_dict['tle_indir'] = config['aapp_processes'][config.process_name]['tle_indir'] for tle_file_name in tle_file_list: for regex, test in tle_match_tests: m = re.match(regex, tle_file_name) if m: try: archive_dict['timestamp'] = test(m) tle_archive_dir = compose( config['aapp_processes'][config.process_name]['tle_archive_dir'], archive_dict) if not os.path.exists(tle_archive_dir): try: os.makedirs(tle_archive_dir) except: LOG.error("Failed to make archive dir: {}".format(tle_archive_dir)) except ValueError: LOG.exception('Failed in archive step...') pass try: copy(tle_file_name, tle_archive_dir) LOG.debug("Copied {} to {}.".format(tle_file_name, tle_archive_dir)) archive = False except IOError as ioe: LOG.error("Failed to copy TLE file: {} to archive: {} because {}".format( tle_file_name, tle_archive_dir, ioe)) LOG.error("CWD: {}".format(os.getcwd())) # Change back after this is done os.chdir(current_dir) return return_status
def do_atovpp_and_avh2hirs_processing(process_config, timestamp): if (not process_config['process_amsua'] and not process_config['process_amsub'] and not process_config['process_hirs']): LOG.debug("Skipping atovpp and avh2hirs processing.") return True LOG.debug("Do preprocessing atovpp and avh2hirs ... ") return_status = True # This function relays on beeing in a working directory current_dir = os.getcwd( ) # Store the dir to change back to after function complete os.chdir(process_config['aapp_processes'][process_config.process_name] ['working_dir']) instruments = "AMSU-A AMSU-B HIRS" grids = "AMSU-A AMSU-B HIRS" if not process_config['process_hirs']: instruments = "AMSU-A AMSU-B" grids = "AMSU-A AMSU-B" if "".join(process_config['a_tovs']) == 'TOVS': instruments = "MSU HIRS" grids = "MSU HIRS" if 'do_atovpp' in process_config['aapp_processes'][ process_config.process_name]: process_config['do_atovpp'] = process_config['aapp_processes'][ process_config.process_name]['do_atovpp'] else: process_config['do_atovpp'] = False if 'do_avh2hirs' in process_config['aapp_processes'][ process_config.process_name]: process_config['do_avh2hirs'] = process_config['aapp_processes'][ process_config.process_name]['do_avh2hirs'] else: process_config['do_avh2hirs'] = False if process_config['do_atovpp']: cmd = "atovin {}".format(instruments) try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Command {} failed.".format(cmd)) else: if returncode != 0: LOG.error("Command {} failed with return code {}.".format( cmd, returncode)) LOG.error(std) LOG.error(err) return_status = False else: LOG.info("Command {} complete.".format(cmd)) LOG.debug(std) LOG.debug(err) if return_status: cmd = "atovpp -i \"{}\" -g \"{}\"".format(instruments, grids) try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Command {} failed.".format(cmd)) else: if returncode != 0: LOG.error("Command {} failed with return code {}.".format( cmd, returncode)) LOG.error(std) LOG.error(err) return_status = False else: LOG.info("Command {} complete.".format(cmd)) if return_status and process_config['do_avh2hirs'] and process_config[ 'process_hirs']: if os.path.exists( "./{}".format(process_config['aapp_static_configuration'] ['decommutation_files']['avhrr_file'])): os.symlink( "./{}".format(process_config['aapp_static_configuration'] ['decommutation_files']['avhrr_file']), "{}11".format(os.environ["FORT"])) else: LOG.error("Could not find file: ./{}".format( process_config['aapp_static_configuration'] ['decommutation_files']['avhrr_file'])) return_status = False if os.path.exists("./hirs.l1d"): os.symlink("./hirs.l1d", "{}12".format(os.environ["FORT"])) else: LOG.error("Could not find file: {}".format("./hirs.l1d")) return_status = False if return_status: cmd = "l1didf -i hirs.l1d" try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Command {} failed.".format(cmd)) else: if returncode != 0: LOG.error("Command {} failed with return code {}.".format( cmd, returncode)) LOG.error(std) LOG.error(err) return_status = False else: LOG.info("Command {} complete.".format(cmd)) satimg, yyyymmdd, hhmn, orbit, instr, loc1, loc2 = std.split( ) if return_status: if "noaa" in satimg: satnb = int(satimg[4:6]) else: satnb = int(satimg[5:7]) LOG.debug("Using satnb: {}".format(satnb)) unit = satnb + 50 mmc = int(yyyymmdd[4:6]) LOG.debug("Using unit: {} and mmc: {}".format(unit, mmc)) if os.path.exists( os.path.join(os.environ["DIR_PREPROC"], "cor_{}.dat".format(satimg))): os.symlink( os.path.join(os.environ["DIR_PREPROC"], "cor_{}.dat".format(satimg)), "{}{}".format(os.environ["FORT"], unit)) else: LOG.error("Failed to find {}".format( os.path.join(os.environ["DIR_PREPROC"], "cor_{}.dat".format(satimg)))) return_status = False if return_status: os.environ["SATIMG"] = satimg os.environ["YYYYMMDD"] = yyyymmdd os.environ["HHMN"] = hhmn os.environ["DIR_MAIA2_ATLAS"] = os.path.join( os.environ["DIR_PREPROC"], "atlas") os.environ["DIR_MAIA2_THRESHOLDS"] = os.path.join( os.environ["DIR_PREPROC"], "thresholds") if "".join(process_config['a_tovs']) == 'TOVS': cmd = "maia2_env;maia2_environment;avh2hirs.exe 2>&1" else: cmd = "bash -c \"source liblog.sh;source maia2_env;maia2_environment\";avh2hirs_atovs.exe 2>&1" LOG.debug("Before running command: {}".format(cmd)) try: status, returncode, std, err = run_shell_command( cmd, stdout_logfile="avh2hirs.log", use_shlex=False, use_shell=True) except: LOG.error("Command {} failed.".format(cmd)) else: if returncode != 0: LOG.error("Command {} failed with return code {}.".format( cmd, returncode)) LOG.error(std) LOG.error(err) return_status = False else: LOG.info("Command {} complete.".format(cmd)) LOG.debug(std) LOG.debug(err) if return_status: import glob for fortfile in glob.glob("./fort*"): os.remove(fortfile) os.remove("./albedo") os.remove("./sst") os.remove("./wv") # Change back after this is done os.chdir(current_dir) LOG.info("atovpp and avh2hirs complete!") return return_status
def do_atovs_calibration(process_config, timestamp): if (not process_config['process_amsua'] and not process_config['process_amsub'] and not process_config['process_msu']): LOG.debug("Skipping atovs processing") return True accepted_return_codes_msucl = [0] accepted_return_codes_amsuacl = [0] accepted_return_codes_amsubcl = [0] return_value = True # This function relays on beeing in a working directory current_dir = os.getcwd( ) # Store the dir to change back to after function complete os.chdir(process_config['aapp_processes'][process_config.process_name] ['working_dir']) # calibration_location = "-c -l" if "".join(process_config['a_tovs']) == 'TOVS': cmd = "msucl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format( process_config['calibration_location'], process_config['platform_name'], timestamp, int(process_config['orbit_number']), process_config['aapp_static_configuration']['decommutation_files'] ['msun_file']) try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Command {} failed.".format(cmd)) else: if returncode in accepted_return_codes_msucl: LOG.info("Command {} complete.".format(cmd)) else: LOG.error("Command {} failed with return code {}.".format( cmd, returncode)) return_value = False elif "".join(process_config['a_tovs']) == 'ATOVS': if process_config['process_amsua']: cmd = "amsuacl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format( process_config['calibration_location'], process_config['platform_name'], timestamp, int(process_config['orbit_number']), process_config['aapp_static_configuration'] ['decommutation_files']['amsua_file']) try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Command {} failed.".format(cmd)) else: if returncode in accepted_return_codes_amsuacl: LOG.info("Command {} complete.".format(cmd)) else: LOG.error("Command {} failed with return code {}.".format( cmd, returncode)) return_value = False if process_config['process_amsub'] and return_value: amsub_script = "mhscl" try: if 'noaa' in process_config['platform_name'] and int( process_config['platform_name'][-2:]) <= 17: amsu_script = "amsubcl" process_config['process_mhs'] = False else: process_config['process_amsub'] = False except ValueError as ve: LOG.warning( "Could not exctract satellite number from the last two characters in the platform name: " "{} and convert it to a number.".format( process_config['platform_name'])) cmd = "{0} {1} -s {2} -d {3:%Y%m%d} -h {3:%H%M} -n {4:05d} {5}".format( amsub_script, process_config['calibration_location'], process_config['platform_name'], timestamp, int(process_config['orbit_number']), process_config['aapp_static_configuration'] ['decommutation_files']['amsub_file']) try: status, returncode, std, err = run_shell_command(cmd) except: LOG.error("Command {} failed.".format(cmd)) else: if returncode in accepted_return_codes_amsubcl: LOG.info("Command {} complete.".format(cmd)) else: LOG.error("Command {} failed with return code {}.".format( cmd, returncode)) return_value = False else: LOG.error("Unknown A|TOVS key string: {}".format("".join( process_config['a_tovs']))) return_value = False # Change back after this is done os.chdir(current_dir) LOG.info("do_atovs_calibration complete!") return return_value