def do_avhrr_calibration(process_config, msg, timestamp):
if not process_config['process_avhrr']:
LOG.debug("Skipping avhrr processing due to config or missing data.")
return True
accepted_return_codes_avhrcl = [0]
return_value = True
LOG.debug("Do the avhrr calibration")
# This function relays on beeing in a working directory
current_dir = os.getcwd(
) # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]
['working_dir'])
# calibration_location = "-c -l"
try:
cmd = "avhrcl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(
process_config['calibration_location'],
process_config['platform_name'], timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']['decommutation_files']
['avhrr_file'])
except Exception as err:
LOG.error("Failed to build avhrcl command: {}".format(err))
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_avhrcl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
return_value = False
# Change back after this is done
os.chdir(current_dir)
LOG.info("do_avhrr_calibration complete!")
return return_value
def do_tle_satpos(config, timestamp, satellite):
return_status = True
LOG.info("satpos files is stored under the dir_navigation/satpos")
satpos_dir = os.path.join(os.environ['DIR_NAVIGATION'], "satpos")
file_satpos = os.path.join(
satpos_dir, "satpos_{}_{:%Y%m%d}.txt".format(satellite, timestamp))
if (not os.path.exists(file_satpos)
or os.stat(file_satpos).st_size == 0) and return_status:
"""Usage is: satpostle [ -o] [-s satellite] [-S station] [-d start date] [-n number of days]
[-i increment in seconds] [-c search criteria]
-o -s -S -d -n -i –c are optional.
If no parameter is specified as an option, defaults are : noaa14, Lannion, today 0h, 1.0, 120.0,
n (n= nearest, p = preceding).
The option -o specifies that the data will be stored in the file satpos_noaxx_yyyymmdd.txt.
Output default is the standard output..
"""
cmd = "satpostle -o -s {} -d {:%d/%m/%y} -n 1.2".format(
satellite, timestamp)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Failed to run command: {}".format(cmd))
return_status = False
else:
if returncode != 0:
LOG.error("cmd: {} failed with returncode: {}".format(
cmd, returncode))
return_status = False
elif not os.path.exists(file_satpos):
LOG.error(
"file: {} does not exists after satpostle run.".format(
file_satpos))
return_status = False
else:
LOG.info("satpos file already there. Use this")
return return_status
def do_ana_correction(process_config, msg, timestamp):
"""The software ANA is 3party. User will have to get it themself.
Further the ana binaries and scripts must be in the PATH,
preferable in the primary AAPP bin directory
Also it relays on reference_landmarks directory under ENV(DIR_NAVIGATION)/ana dir
This needs to be copied/installed here by the user
TODO: Replace the ana_lmk_loc script with code here to better be able to control
the directories used in the processing
"""
if not process_config['process_avhrr']:
LOG.debug(
"Skipping ANA as AVHRR is not processed de to config or missing data."
)
return True
try:
if 'do_ana_correction' in process_config['aapp_processes'][
process_config.process_name]:
if not process_config['aapp_processes'][
process_config.process_name]['do_ana_correction']:
return True
except Exception as err:
LOG.error("Failed with: {}".format(err))
return_status = True
# This function relays on beeing in a working directory
current_dir = os.getcwd(
) # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]
['working_dir'])
# Must check of the ana dir exists
ana_dir = os.path.join(os.getenv('DIR_NAVIGATION'), 'ana')
if not os.path.exists(ana_dir):
try:
os.makedirs(ana_dir)
os.environ['DIR_ANA'] = ana_dir
except OSError in e:
LOG.error(
"Failed to create directory: {}. This is needed to run the ANA software."
)
return_status = False
# And the ana dir must contain a reference_landmarks directory to work!
if not os.path.exists(os.path.join(ana_dir, "reference_landmarks")):
LOG.error(
"Can not run ANA because reference_landmarks under the ana dir is missing."
)
return_status = False
if return_status:
# Find all matching landmarks
cmd = "ana_lmk_loc -D {}".format(
process_config['aapp_static_configuration']['decommutation_files']
['avhrr_file'])
try:
status = False
status, returncode, std, err = run_shell_command(
cmd,
stdout_logfile="ana_lmk_loc.log",
stderr_logfile="ana_lmk_loc.err")
except:
import sys
LOG.error("Command {} failed with {}.".format(
cmd,
sys.exc_info()[0]))
if not status:
return_status = False
else:
if not status or returncode != 0:
LOG.error("Command {} failed with {}".format(cmd, returncode))
_ana_file = open('ana_lmk_loc.err', "w")
_ana_file.write(std)
_ana_file.write(err)
_ana_file.close()
return_status = False
if return_status:
# Check of ana landmark location file with correct time stamp exist.
# If not try to find correct timestamp and copy to this.
expected_ana_loc_file = "lmkloc_{}_{:%Y%m%d_%H%M}_{:05d}.txt".format(
process_config['platform_name'], timestamp,
process_config['orbit_number'])
if not os.path.exists(
os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
expected_ana_loc_file)):
# Need to run this AAPP command to detect the start of the hrpt.l1b(avhrr data) file
# Because this might differ from the time stamp in file name
cmd = "l1bidf.exe"
try:
status, returncode, std, err = run_shell_command(
cmd,
stdin="{}\n".format(
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file']))
except:
import sys
LOG.error("Command {} failed with {}.".format(
cmd,
sys.exc_info()[0]))
if not status:
return_status = False
else:
if not status or returncode != 0:
LOG.error("Command {} failed with {}".format(
cmd, returncode))
_ana_file = open('ana_lmk_loc.err', "w")
_ana_file.write(std)
_ana_file.write(err)
_ana_file.close()
return_status = False
else:
l1bidf_list = std.split()
l1bidf_aapp_datetime = datetime.strptime(
l1bidf_list[1] + l1bidf_list[2], "%Y%m%d%H%M")
LOG.debug("l1b idf aapp datetime: {}".format(
l1bidf_aapp_datetime))
# estatt_file_name = lmkloc_metop02_20160617_0952_50132.txt
ana_loc_file = "lmkloc_{}_{:%Y%m%d_%H%M}_{:05d}.txt".format(
l1bidf_list[0], l1bidf_aapp_datetime,
int(l1bidf_list[3]))
if os.path.exists(
os.path.join(
ana_dir,
"{:%Y-%m}".format(l1bidf_aapp_datetime),
ana_loc_file)):
LOG.debug(
"timestamp {} l1bidf_aapp_datetime: {}".format(
timestamp, l1bidf_aapp_datetime))
from shutil import copy2
copy2(
os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
ana_loc_file),
os.path.join(
ana_dir,
"{:%Y-%m}".format(l1bidf_aapp_datetime),
expected_ana_loc_file))
else:
LOG.warning("Could not find ana loc file: {}".format(
ana_loc_file))
else:
LOG.debug("File ok: {}".format(
os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
expected_ana_loc_file)))
import hashlib
if return_status:
# LOG.debug("sha256 of aapp input avhrr_file: {}".format(hashlib.sha256(open(
# process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest()))
# Calculate correction from landmarks and update avhrr_file with these new attitude coefisients.
cmd = "ana_estatt -s {0} -d {1:%Y%m%d} -h {1:%H%M} -n {2:05d}".format(
process_config['platform_name'], timestamp,
process_config['orbit_number'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
import sys
LOG.error("Command {} failed with {}.".format(
cmd,
sys.exc_info()[0]))
if not status:
return_status = False
else:
if not status or returncode != 0:
LOG.error("Command {} failed with {}".format(cmd, returncode))
_ana_file = open('ana_lmk_loc.err', "w")
_ana_file.write(std)
_ana_file.write(err)
_ana_file.close()
return_status = False
# LOG.debug("sha256 of aapp input avhrr_file: {}".format(hashlib.sha256(open(
# process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest()))
sha256_before_correction = hashlib.sha256(
open(
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file'],
'rb').read()).hexdigest()
if return_status:
# Recalculate the location in the avhhr data file with the new correction attitude coefisients.
from do_avhrr_calibration import do_avhrr_calibration
LOG.info(
"Need to recalculate the avhrcl with new attitude coefisients from ANA."
)
if not do_avhrr_calibration(process_config, msg, timestamp):
LOG.warning(
"The avhrr location with ana correction failed for some reason."
"It might be that the processing can continue")
LOG.warning(
"Please check the previous log carefully to see if this is an error you can accept."
)
return_status = False
if return_status:
sha256_after_correction = hashlib.sha256(
open(
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file'],
'rb').read()).hexdigest()
LOG.debug("sha256 of aapp input avhrr_file BEFORE ana: {}".format(
sha256_before_correction))
LOG.debug("sha256 of aapp input avhrr_file AFTER ana: {}".format(
sha256_after_correction))
if (sha256_before_correction == sha256_after_correction):
LOG.warning(
"The correction of the avhrr location with data from ANA did not take place for some reason,"
"or the correction was 0 for all pitch yaw and roll.")
else:
LOG.info(
"The correction of AVHRR location with data from ANA was performed."
)
# Change back after this is done
os.chdir(current_dir)
LOG.info("do_ana_correction complete!")
return return_status
def do_hirs_calibration(process_config, msg, timestamp):
return_status = True
if not process_config['process_hirs']:
return True
# A list of accepted return codes for the various scripts/binaries run in this function
accepted_return_codes_hirs_historic_file_manage = [0]
# This function relays on beeing in a working directory
current_dir = os.getcwd(
) # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]
['working_dir'])
hirs_version_use = None
hirs_version = os.getenv('HIRSCL_VERSION', 0)
hirs_version_list = hirs_version.split()
hirs_sats = os.getenv('HIRSCL_SAT', 'default')
hirs_sat_list = hirs_sats.split()
index = 0
for sat in hirs_sat_list:
if sat in process_config['platform_name']:
hirs_version_use = hirs_version_list[index]
else:
hirs_version_def = hirs_version_list[index]
index += 1
if hirs_version_use is None:
hirs_version_use = hirs_version_def
hirs_script = "hirscl"
hirs_err_file = "hirscl.err"
calibration_location = process_config['calibration_location']
LOG.debug("hirs_version_use {}".format(hirs_version_use))
# pdb.set_trace()
if int(hirs_version_use) > 1: # no calibration, just navigation
calibration_location = "-l"
elif int(hirs_version_use) == 0 or "".join(
process_config['a_tovs']) == 'TOVS':
calibration_location = "-c -l"
elif int(hirs_version_use) == 1:
file_historic = os.path.join(os.getenv('PAR_CALIBRATION_MONITOR'),
process_config['platform_name'],
"hirs_historic.txt")
if os.path.exists(file_historic):
cmd = "hirs_historic_file_manage -m {} -r {} -n {} {}".format(
os.getenv('HIST_SIZE_HIGH'), os.getenv('HIST_SIZE_LOW'),
os.getenv('HIST_NMAX'), file_historic)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
return_status = False
else:
if returncode in accepted_return_codes_hirs_historic_file_manage:
LOG.debug("Command complete.")
else:
LOG.error("Command {} failed with returncode {}".format(
cmd, returncode))
LOG.error("stdout was: {}".format(std))
LOG.error("stderr was: {}".format(err))
return_status = False
if return_status:
cmd = "hcalcb1_algoV4 -s {0} -y {1:%Y} -m {1:%m} -d {1:%d} -h {1:%H} -n {1:%M}".format(
process_config['platform_name'], timestamp)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
import sys
LOG.error("Command {} failed with {}.".format(
cmd,
sys.exc_info()[0]))
else:
if returncode != 0:
LOG.error("Command {} failed with {}".format(
cmd, returncode))
_hirs_file = open(hirs_err_file, "w")
_hirs_file.write(std)
_hirs_file.write(err)
_hirs_file.close()
return_status = False
hirs_script = "hirscl_algoV4"
hirs_err_file = "hirscl_algoV4.err"
calibration_location = "-c -l"
else:
LOG.error(
"Can not figure out which hirs calibration algo version to use.")
return_status = False
if return_status:
try:
cmd = "{} {} -s {} -d {:%Y%m%d} -h {:%H%M} -n {:05d} {}".format(
hirs_script, calibration_location,
process_config['platform_name'], timestamp, timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']
['decommutation_files']['hirs_file'])
except KeyError as ke:
LOG.error(
"Building command string failed with key error: {}".format(ke))
try:
status, returncode, out, err = run_shell_command(
cmd,
stdout_logfile="{}.log".format(hirs_script),
stderr_logfile="{}".format(hirs_err_file))
except:
import sys
LOG.error("Command {} failed {}.".format(cmd, sys.exc_info()[0]))
else:
if (returncode != 0):
LOG.error("Command {} failed with {}".format(cmd, returncode))
_hirs_file = open(hirs_err_file, "w")
_hirs_file.write(out)
_hirs_file.write(err)
_hirs_file.close()
return_status = False
# Change back after this is done
os.chdir(current_dir)
LOG.info("do_hirs_calibration complete!")
return return_status
def setup_aapp_processing(config):
"""
Setup various env variables needed for the aapp processing
and set eg working dir
"""
if (not 'working_dir' in config['aapp_processes'][config.process_name] and
'use_dyn_work_dir' in config['aapp_processes'][config.process_name]
and
config['aapp_processes'][config.process_name]['use_dyn_work_dir']):
try:
config['aapp_processes'][
config.process_name]['working_dir'] = tempfile.mkdtemp(
dir=config['aapp_processes'][
config.process_name]['aapp_workdir'])
LOG.debug("working dir set based on aapp_workdir and tmp " + str(
config['aapp_processes'][config.process_name]['working_dir']))
except OSError:
config['aapp_processes'][
config.process_name]['working_dir'] = tempfile.mkdtemp()
except KeyError as ke:
LOG.error(": {}".format(ke))
raise
finally:
LOG.info("Create new working dir...")
elif not 'working_dir' in config['aapp_processes'][config.process_name]:
try:
config['aapp_processes'][
config.process_name]['working_dir'] = config['aapp_processes'][
config.process_name]['aapp_workdir']
except KeyError as ke:
LOG.error(
"working_dir nor aapp_workdir is given in config. Dont know where to store what I do."
)
raise
LOG.info("Working dir = " +
str(config['aapp_processes'][config.process_name]['working_dir']))
os.environ["AAPP_PREFIX"] = config['aapp_processes'][
config.process_name]['aapp_prefix']
aapp_atovs_conf = os.path.join(
os.environ["AAPP_PREFIX"],
config['aapp_processes'][config.process_name]['aapp_environment_file'])
status, returncode, out, err = run_shell_command(
"bash -c \"source {}\";env".format(aapp_atovs_conf))
if not status:
LOG.error("Failed to run the bash source env command for " +
str(aapp_atovs_conf))
return False
else:
for line in out.splitlines():
if line:
(key, _, value) = line.partition("=")
os.environ[key] = value
# Default AAPP config for PAR_NAVIGATION_DEFAULT_LISTESAT Metop platform is M01, M02, M04
# but needed names are metop01 etc. Replace this inside the processing
# from now on.
aapp_satellite_list = os.getenv('PAR_NAVIGATION_DEFAULT_LISTESAT').split()
if config['platform_name'] not in aapp_satellite_list:
LOG.warning(
"Can not find this platform in AAPP config variable PAR_NAVIGATION_DEFAULT_LISTESAT. Will try to find matches. But it can be a good idea to change this variable in the ATOVS_ENV7 file."
)
LOG.warning("Platform {} not in list: {}".format(
config['platform_name'], aapp_satellite_list))
if 'metop' in config['platform_name'] and (
('M01' or 'M02' or 'M03' or 'M04') in aapp_satellite_list):
LOG.debug("Replace in this processing")
PAR_NAVIGATION_DEFAULT_LISTESAT = os.getenv(
'PAR_NAVIGATION_DEFAULT_LISTESAT')
PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
'M01', 'metop01')
PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
'M02', 'metop02')
PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
'M03', 'metop03')
PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
'M04', 'metop04')
os.environ[
'PAR_NAVIGATION_DEFAULT_LISTESAT'] = PAR_NAVIGATION_DEFAULT_LISTESAT
LOG.debug("New LISTESAT: {}".format(
os.getenv('PAR_NAVIGATION_DEFAULT_LISTESAT')))
list_of_needed_programs = [
'tleing.exe', 'satpostle', 'decommutation.exe', 'chk1btime.exe',
'decom-amsua-metop', 'decom-mhs-metop', 'decom-hirs-metop',
'decom-avhrr-metop', 'hirs_historic_file_manage', 'hcalcb1_algoV4',
'msucl', 'amsuacl', 'amsubcl', 'mhscl', 'avhrcl', 'atovin', 'atovpp',
'l1didf'
]
for program in list_of_needed_programs:
if not which(program):
LOG.error(
"Can not find needed AAPP program '{}' in environment. Please check."
.format(program))
return False
return True
def download_tle(config, timestamp, dir_data_tle):
user = os.getenv("PAR_NAVIGATION_TLE_USER", "xxxxxx")
passwd = os.getenv("PAR_NAVIGATION_TLE_PASSWD", "xxxxxx")
url = os.getenv("PAR_NAVIGATION_TLE_URL_DOWNLOAD")
timeout = 60
catalogue = "25338,26536,27453,28654,33591,37849,29499,38771,27431,32958,37214,25994,27424"
tle_infile = ""
tle_dict = {}
tle_dict['timestamp'] = timestamp
tle_cnf = []
try:
tle_cnf = config['aapp_processes'][config.process_name]['tle_download']
except KeyError as ke:
LOG.info("No tle_download config found. Using the default.")
pass
finally:
tle_cnf.append({'url': url, 'user': user, 'passwd': passwd, 'timeout': timeout, 'catalogue': catalogue})
try:
tle_infile = compose(config['aapp_processes'][config.process_name]['tle_infile_format'], tle_dict)
except KeyError as ke:
if 'tle_infile_format' in ke:
tle_infile = compose('tle_{timestamp:%Y%m%d_%H%M}.txt', tle_dict)
LOG.warning("Using default TLE file name format: %s", tle_infile)
pass
else:
LOG.error("Key error: {}".format(ke))
LOG.error("Valid keys :")
for key in tle_dict.keys():
LOG.error("{}".format(key))
raise
except:
raise
tle_file_list = []
for cnf in tle_cnf:
for cnf_url in cnf['url'].split():
LOG.debug("Will try to download TLE from {}.".format(cnf_url))
if "space-track" in cnf_url:
# Do the special space-track login
status = False
returncode = 0
stdout = ""
stderr = ""
cmd = "wget -T {} --post-data=\"identity={}&password={}\" --cookies=on ",
"--keep-session-cookies --save-cookies=cookies_spacetrack \"{}/ajaxauth/login\" -olog".format(
cnf['timeout'], cnf['user'], cnf['passwd'], cnf_url)
try:
status, returncode, stdout, stderr = run_shell_command(cmd)
except:
LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode))
LOG.error("stdout: {}".format(stdout))
LOG.error("stderr: {}".format(stderr))
return_status = False
else:
if returncode != 0:
LOG.debug("Running command: {} with return code: {}".format(cmd, returncode))
LOG.debug("stdout: {}".format(stdout))
LOG.debug("stderr: {}".format(stderr))
else:
cmd = "wget -T {} --keep-session-cookies --load-cookies=cookies_spacetrack -O weather.txt ",
"\"{}/basicspacedata/query/class/tle_latest/ORDINAL/1/NORAD_CAT_ID/{}/orderby/TLE_LINE1\"".format(
cnf['timeout'], cnf_url, cnf['catalogue'])
try:
status, returncode, stdout, stderr = run_shell_command(cmd)
except:
LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode))
LOG.error("stdout: {}".format(stdout))
LOG.error("stderr: {}".format(stderr))
else:
if returncode != 0:
LOG.debug("Running command: {} with return code: {}".format(cmd, returncode))
LOG.debug("stdout: {}".format(stdout))
LOG.debug("stderr: {}".format(stderr))
else:
LOG.debug("TLE download ok")
if os.path.exists("weather.txt"):
try:
tle_file = open("weather.txt", 'r')
tle_string = tle_file.read()
tle_file.close()
tle_file_out = os.path.join(dir_data_tle, tle_infile)
with open(tle_file_out, "a") as tle_file:
tle_file.write(tle_string)
if tle_file_out not in tle_file_list:
tle_file_list.append(tle_file_out)
except Exception as ex:
LOG.debug("Failed rename tle download file: {}".format(ex))
raise
else:
import urllib2
f = urllib2.urlopen(cnf_url)
tle_string = f.read()
f.close()
tle_file_out = os.path.join(dir_data_tle, tle_infile)
with open(tle_file_out, "a") as tle_file:
tle_file.write(tle_string)
if tle_file_out not in tle_file_list:
tle_file_list.append(tle_file_out)
return tle_file_list
def do_tleing(config, timestamp, satellite):
"""Get the tle-file and copy them to the AAPP data structure
and run the AAPP tleing script and executable"""
return_status = True
# This function relays on beeing in a working directory
try:
current_dir = os.getcwd() # Store the dir to change back to after function complete
except OSError as ose:
LOG.error("Failed to get current working dir: {}".format(ose))
raise
os.chdir(config['aapp_processes'][config.process_name]['working_dir'])
tle_match_tests = (('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2})(\d{2}).*', _do_6_matches),
('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2}).*', _do_5_matches),
('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2}).*', _do_4_matches),
('.*(\d{4})(\d{2})(\d{2}).*', _do_3_matches),
('.*(\d{2})(\d{2})(\d{2}).*', _do_3_matchesYY))
if '7' in os.getenv('AAPP_PREFIX'):
aapp_env = 'AAPP_ENV7'
elif '8' in os.getenv('AAPP_PREFIX'):
aapp_env = 'AAPP_ENV8'
else:
aapp_env = 'AAPP_ENV'
if 'dir_navigation' in config['aapp_processes'][config.process_name]:
LOG.warning("Override the env variable set in {} DIR_NAVIGATION from {} to {}.".format(
aapp_env,
os.environ['DIR_NAVIGATION'], config['aapp_processes'][config.process_name]['dir_navigation']))
os.environ['DIR_NAVIGATION'] = config['aapp_processes'][config.process_name]['dir_navigation']
# Need to update DIR_DATA_TLE dir to be sure this is corect if not tle_indir is given
os.environ['DIR_DATA_TLE'] = os.path.join(os.getenv('DIR_NAVIGATION'), 'tle_db')
if 'tle_indir' in config['aapp_processes'][config.process_name]:
tle_indir = config['aapp_processes'][config.process_name]['tle_indir']
LOG.warning("Override the env variable set in {} DIR_DATA_TLE from {} to {}.".format(
aapp_env,
os.environ['DIR_DATA_TLE'], tle_indir))
os.environ['DIR_DATA_TLE'] = tle_indir
# variables for the TLE HOME directory
DIR_DATA_TLE = os.getenv('DIR_DATA_TLE', os.path.join(os.getenv('DIR_NAVIGATION'), 'orb_elem'))
# This is needed by AAPP tleing. Try other if not existing
if not os.path.exists(DIR_DATA_TLE):
LOG.warning("Dir " + DIR_DATA_TLE + " does not exist. Create ")
try:
os.makedirs(DIR_DATA_TLE)
except:
LOG.error("Failed to create %s. Can not handle TLEs without this", DIR_DATA_TLE)
raise
if 'tle_file_to_data_diff_limit_days' in config['aapp_processes'][config.process_name]:
select_closest_tle_file_to_data = True
min_closest_tle_file = int(
config['aapp_processes'][config.process_name]['tle_file_to_data_diff_limit_days']) * 24 * 60 * 60
else:
select_closest_tle_file_to_data = False
TLE_INDEX = os.path.join(DIR_DATA_TLE, "tle_{}.index".format(satellite))
tle_search_dir = DIR_DATA_TLE
tle_file_list = []
# dict to hold needed tle keys
tle_dict = {}
if not select_closest_tle_file_to_data:
if os.path.exists(TLE_INDEX):
# Loop over all tle files, and only do tle
tle_files = [s for s in glob(os.path.join(DIR_DATA_TLE, 'tle*txt'))
if os.path.isfile(os.path.join(DIR_DATA_TLE, s))]
tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE, s)))
tle_index_mtime = os.path.getmtime(TLE_INDEX)
for s in tle_files:
if os.path.getmtime(os.path.join(DIR_DATA_TLE, s)) > tle_index_mtime:
tle_file_list.append(s)
if len(tle_file_list) == 0:
import time
LOG.warning(("No newer tle files than last update of the index file. " +
"Last update of index file is {:d}s. If more than a few days you should check.".format(
int(time.time() - tle_index_mtime))))
else:
LOG.info("Will use tle files {}".format(tle_file_list))
else:
LOG.warning("index file does not exist. If this is the first run of AAPP tleing.exe it is ok,"
" otherwise it is a bit suspisiuos.")
try:
tle_files = [s for s in os.listdir(DIR_DATA_TLE) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))]
tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE, s)))
tle_file_list = tle_files
except OSError:
LOG.warning("Found to tle files .... ")
else:
# dict to hold needed tle keys
tle_dict['timestamp'] = timestamp
try:
infile = compose(config['aapp_processes'][config.process_name]['tle_infile_format'], tle_dict)
except KeyError as ke:
LOG.error("Key error: {}".format(ke))
LOG.error("Valid keys :")
for key in tle_dict.keys():
LOG.error("{}".format(key))
raise
except:
raise
LOG.debug("tle file name: {}".format(infile))
# Check if I can read the tle file.
first_search = True
# FIXME: In AAPP default get_tle script direcory timestamp is TLE_MONTH=`date +%Y-\%m`
for tle_search_dir in [compose(os.path.join(DIR_DATA_TLE, "{timestamp:%Y_%m}"), tle_dict), DIR_DATA_TLE]:
if not os.path.exists(tle_search_dir):
LOG.debug("tle_search_dir {} does not exists.".format(tle_search_dir))
continue
LOG.debug("tle_search_dir {}".format(tle_search_dir))
try:
with open(os.path.join(tle_search_dir, infile)) as tle_file:
del tle_file_list[:]
tle_file_list.append(os.path.join(tle_search_dir, infile))
min_closest_tle_file = 0
pass
except IOError as e:
LOG.warning("Could not find tle file: {}. Try find closest ... ".format(infile))
tle_file_list = glob(os.path.join(tle_search_dir, '*'))
LOG.debug("tle file list: {}".format(tle_file_list))
LOG.debug(tle_file_list)
infile_closest = ""
for tle_file_name in tle_file_list:
for regex, test in tle_match_tests:
m = re.match(regex, tle_file_name)
if m:
try:
LOG.debug("{} {}".format(tle_file_name, test(m)))
delta = timestamp - test(m)
if (abs(delta.total_seconds()) < min_closest_tle_file):
min_closest_tle_file = abs(delta.total_seconds())
# infile_closest = os.path.basename(tle_file_name)
infile_closest = tle_file_name
LOG.debug("Closest tle infile so far: {}".format(infile_closest))
except ValueError:
pass
if infile_closest:
del tle_file_list[:]
tle_file_list.append(infile_closest)
break
else:
if not first_search:
LOG.error("Could not find tle file close enough to timestamp {} with limit {}".format(
timestamp, min_closest_tle_file))
LOG.error("Update your TLE files or adjust the limit(Not recomended!).")
first_search = False
else:
break
# DIR_DATA_TLE = tle_search_dir
if tle_file_list:
LOG.debug("Use this: {} offset {}s".format(tle_file_list, min_closest_tle_file))
if not tle_file_list and config['aapp_processes'][config.process_name]['download_tle_files']:
LOG.warning("Found no tle files. Try to download ... ")
tle_file_list = download_tle(config, timestamp, DIR_DATA_TLE)
for tle_file in tle_file_list:
archive = False
# SATellite IDentification mandatory
# so take care of default values
os.environ['SATID_FILE'] = os.getenv('SATID_FILE', 'satid.txt')
"""Don't use the tle_indir because this is handeled by the tleing script"""
if (DIR_DATA_TLE != tle_search_dir):
tle_filename = compose(os.path.join("{timestamp:%Y_%m}", os.path.basename(tle_file)), tle_dict)
else:
tle_filename = os.path.basename(tle_file)
status = False
returncode = 0
stdout = ""
stderr = ""
cmd = "tleing.exe"
stdin = "{}\n{}\n{}\n{}\n".format(DIR_DATA_TLE, tle_filename, satellite, TLE_INDEX)
LOG.debug('stdin arguments to command: ' + str(stdin))
try:
status, returncode, stdout, stderr = run_shell_command(cmd, stdin=stdin)
except:
LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode))
LOG.error("stdout: {}".format(stdout))
LOG.error("stderr: {}".format(stderr))
return_status = False
else:
if returncode != 0:
LOG.debug("Running command: {} with return code: {}".format(cmd, returncode))
LOG.debug("stdout: {}".format(stdout))
LOG.debug("stderr: {}".format(stderr))
elif not os.path.exists(TLE_INDEX):
LOG.error("index file: {} does not exist after tleing. Something is wrong.".format(TLE_INDEX))
LOG.debug("Running command: {} with return code: {}".format(cmd, returncode))
LOG.debug("stdout: {}".format(stdout))
LOG.debug("stderr: {}".format(stderr))
else:
LOG.debug("Running command: {} with return code: {}".format(cmd, returncode))
LOG.debug("stdout: {}".format(stdout))
LOG.debug("stderr: {}".format(stderr))
LOG.debug("DIR_DATA_TLE : {}".format(DIR_DATA_TLE))
LOG.debug("tle_file : {}".format(os.path.basename(tle_file)))
LOG.debug("satellite : {}".format(satellite))
LOG.debug("TLE_INDEX : {}".format(TLE_INDEX))
# When a index file is generated above one line is added for each tle file.
# If several tle files contains equal TLEs each of these TLEs generate one line in the index file
# To avoid this, sort the index file keeping only unique lines(skipping the tle filename at the end
# The sort options +0b -3b is guessed to be sort from column 0 to 3, but this is not documented
# Could cause problems with future version of sort.
# See eg. http://search.cpan.org/~sdague/ppt-0.12/bin/sort
# cmd="sort -u -o {} +0b -3b {}".format(os.path.join(DIR_DATA_TLE, "{}.sort".format(TLE_INDEX)),
# os.path.join(DIR_DATA_TLE, TLE_INDEX))
if os.path.exists(TLE_INDEX):
cmd = "sort -u +0b -3b {}".format(TLE_INDEX)
try:
status, returncode, stdout, stderr = run_shell_command(
cmd, stdout_logfile="{}.sort1".format(TLE_INDEX))
except:
LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode))
LOG.error("stdout: {}".format(stdout))
LOG.error("stderr: {}".format(stderr))
return_status = False
else:
if returncode == 0 and os.path.exists("{}.sort1".format(TLE_INDEX)):
cmd = "grep -v NaN {}.sort1".format(TLE_INDEX)
try:
status, returncode, stdout, stderr = run_shell_command(
cmd, stdout_logfile="{}.sort".format(TLE_INDEX))
except:
LOG.error("Failed running command: {} with return code: {}".format(cmd, returncode))
LOG.error("stdout: {}".format(stdout))
LOG.error("stderr: {}".format(stderr))
return_status = False
else:
try:
os.remove(TLE_INDEX)
os.remove("{}.sort1".format(TLE_INDEX))
except OSError as e:
LOG.error("Failed to remove unsorted and duplicated index file: {} with {}"
.format(TLE_INDEX, e))
else:
try:
os.rename("{}.sort".format(TLE_INDEX), TLE_INDEX)
archive = True
except:
LOG.error("Failed to rename sorted index file to original name.")
else:
LOG.error("Returncode other than 0: {} or tle index sort file does exists.".format(
returncode, "{}.sort".format(TLE_INDEX)))
else:
LOG.error("tle index file: {} does not exists after tleing before sort. This can not happen.")
# If a new tle is used and archive dir is given in config, copy TLEs to archive
if archive and ('tle_archive_dir' in config['aapp_processes'][config.process_name]):
archive_dict = {}
archive_dict['tle_indir'] = config['aapp_processes'][config.process_name]['tle_indir']
for tle_file_name in tle_file_list:
for regex, test in tle_match_tests:
m = re.match(regex, tle_file_name)
if m:
try:
archive_dict['timestamp'] = test(m)
tle_archive_dir = compose(
config['aapp_processes'][config.process_name]['tle_archive_dir'], archive_dict)
if not os.path.exists(tle_archive_dir):
try:
os.makedirs(tle_archive_dir)
except:
LOG.error("Failed to make archive dir: {}".format(tle_archive_dir))
except ValueError:
LOG.exception('Failed in archive step...')
pass
try:
copy(tle_file_name, tle_archive_dir)
LOG.debug("Copied {} to {}.".format(tle_file_name, tle_archive_dir))
archive = False
except IOError as ioe:
LOG.error("Failed to copy TLE file: {} to archive: {} because {}".format(
tle_file_name, tle_archive_dir, ioe))
LOG.error("CWD: {}".format(os.getcwd()))
# Change back after this is done
os.chdir(current_dir)
return return_status
def do_atovpp_and_avh2hirs_processing(process_config, timestamp):
if (not process_config['process_amsua']
and not process_config['process_amsub']
and not process_config['process_hirs']):
LOG.debug("Skipping atovpp and avh2hirs processing.")
return True
LOG.debug("Do preprocessing atovpp and avh2hirs ... ")
return_status = True
# This function relays on beeing in a working directory
current_dir = os.getcwd(
) # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]
['working_dir'])
instruments = "AMSU-A AMSU-B HIRS"
grids = "AMSU-A AMSU-B HIRS"
if not process_config['process_hirs']:
instruments = "AMSU-A AMSU-B"
grids = "AMSU-A AMSU-B"
if "".join(process_config['a_tovs']) == 'TOVS':
instruments = "MSU HIRS"
grids = "MSU HIRS"
if 'do_atovpp' in process_config['aapp_processes'][
process_config.process_name]:
process_config['do_atovpp'] = process_config['aapp_processes'][
process_config.process_name]['do_atovpp']
else:
process_config['do_atovpp'] = False
if 'do_avh2hirs' in process_config['aapp_processes'][
process_config.process_name]:
process_config['do_avh2hirs'] = process_config['aapp_processes'][
process_config.process_name]['do_avh2hirs']
else:
process_config['do_avh2hirs'] = False
if process_config['do_atovpp']:
cmd = "atovin {}".format(instruments)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
LOG.debug(std)
LOG.debug(err)
if return_status:
cmd = "atovpp -i \"{}\" -g \"{}\"".format(instruments, grids)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
if return_status and process_config['do_avh2hirs'] and process_config[
'process_hirs']:
if os.path.exists(
"./{}".format(process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file'])):
os.symlink(
"./{}".format(process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file']),
"{}11".format(os.environ["FORT"]))
else:
LOG.error("Could not find file: ./{}".format(
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file']))
return_status = False
if os.path.exists("./hirs.l1d"):
os.symlink("./hirs.l1d", "{}12".format(os.environ["FORT"]))
else:
LOG.error("Could not find file: {}".format("./hirs.l1d"))
return_status = False
if return_status:
cmd = "l1didf -i hirs.l1d"
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
satimg, yyyymmdd, hhmn, orbit, instr, loc1, loc2 = std.split(
)
if return_status:
if "noaa" in satimg:
satnb = int(satimg[4:6])
else:
satnb = int(satimg[5:7])
LOG.debug("Using satnb: {}".format(satnb))
unit = satnb + 50
mmc = int(yyyymmdd[4:6])
LOG.debug("Using unit: {} and mmc: {}".format(unit, mmc))
if os.path.exists(
os.path.join(os.environ["DIR_PREPROC"],
"cor_{}.dat".format(satimg))):
os.symlink(
os.path.join(os.environ["DIR_PREPROC"],
"cor_{}.dat".format(satimg)),
"{}{}".format(os.environ["FORT"], unit))
else:
LOG.error("Failed to find {}".format(
os.path.join(os.environ["DIR_PREPROC"],
"cor_{}.dat".format(satimg))))
return_status = False
if return_status:
os.environ["SATIMG"] = satimg
os.environ["YYYYMMDD"] = yyyymmdd
os.environ["HHMN"] = hhmn
os.environ["DIR_MAIA2_ATLAS"] = os.path.join(
os.environ["DIR_PREPROC"], "atlas")
os.environ["DIR_MAIA2_THRESHOLDS"] = os.path.join(
os.environ["DIR_PREPROC"], "thresholds")
if "".join(process_config['a_tovs']) == 'TOVS':
cmd = "maia2_env;maia2_environment;avh2hirs.exe 2>&1"
else:
cmd = "bash -c \"source liblog.sh;source maia2_env;maia2_environment\";avh2hirs_atovs.exe 2>&1"
LOG.debug("Before running command: {}".format(cmd))
try:
status, returncode, std, err = run_shell_command(
cmd,
stdout_logfile="avh2hirs.log",
use_shlex=False,
use_shell=True)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
LOG.debug(std)
LOG.debug(err)
if return_status:
import glob
for fortfile in glob.glob("./fort*"):
os.remove(fortfile)
os.remove("./albedo")
os.remove("./sst")
os.remove("./wv")
# Change back after this is done
os.chdir(current_dir)
LOG.info("atovpp and avh2hirs complete!")
return return_status
def do_atovs_calibration(process_config, timestamp):
if (not process_config['process_amsua']
and not process_config['process_amsub']
and not process_config['process_msu']):
LOG.debug("Skipping atovs processing")
return True
accepted_return_codes_msucl = [0]
accepted_return_codes_amsuacl = [0]
accepted_return_codes_amsubcl = [0]
return_value = True
# This function relays on beeing in a working directory
current_dir = os.getcwd(
) # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]
['working_dir'])
# calibration_location = "-c -l"
if "".join(process_config['a_tovs']) == 'TOVS':
cmd = "msucl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(
process_config['calibration_location'],
process_config['platform_name'], timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']['decommutation_files']
['msun_file'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_msucl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
return_value = False
elif "".join(process_config['a_tovs']) == 'ATOVS':
if process_config['process_amsua']:
cmd = "amsuacl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(
process_config['calibration_location'],
process_config['platform_name'], timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']
['decommutation_files']['amsua_file'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_amsuacl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
return_value = False
if process_config['process_amsub'] and return_value:
amsub_script = "mhscl"
try:
if 'noaa' in process_config['platform_name'] and int(
process_config['platform_name'][-2:]) <= 17:
amsu_script = "amsubcl"
process_config['process_mhs'] = False
else:
process_config['process_amsub'] = False
except ValueError as ve:
LOG.warning(
"Could not exctract satellite number from the last two characters in the platform name: "
"{} and convert it to a number.".format(
process_config['platform_name']))
cmd = "{0} {1} -s {2} -d {3:%Y%m%d} -h {3:%H%M} -n {4:05d} {5}".format(
amsub_script, process_config['calibration_location'],
process_config['platform_name'], timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']
['decommutation_files']['amsub_file'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_amsubcl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
return_value = False
else:
LOG.error("Unknown A|TOVS key string: {}".format("".join(
process_config['a_tovs'])))
return_value = False
# Change back after this is done
os.chdir(current_dir)
LOG.info("do_atovs_calibration complete!")
return return_value