def _convert_to_yuv_and_delete(output_directory, file_name, pattern):
"""Converts a PNG file to a YUV file and deletes the PNG file.
Args:
output_directory(string): The output directory for the YUV file.
file_name(string): The PNG file name.
pattern(string): The file pattern of the PNG/YUV file. The PNG/YUV files are
named patternxx..x.png/yuv, where xx..x are digits starting from 00..0.
Return:
(bool): True upon successful conversion, false otherwise.
"""
# Pattern should be in file name
if not pattern in file_name:
return False
pattern_position = file_name.rfind(pattern)
# Strip the path to the PNG file and replace the png extension with yuv
yuv_file_name = file_name[pattern_position:-3] + 'yuv'
yuv_file_name = os.path.join(output_directory, yuv_file_name)
command = [
'ffmpeg', '-i',
'%s' % (file_name), '-pix_fmt', 'yuv420p',
'%s' % (yuv_file_name)
]
try:
helper_functions.run_shell_command(
command,
fail_msg=('Error during PNG to YUV conversion of %s' % file_name))
os.remove(file_name)
except helper_functions.HelperError as err:
print >> sys.stderr, err
return False
return True
def _convert_to_yuv_and_delete(output_directory, file_name, pattern):
"""Converts a PNG file to a YUV file and deletes the PNG file.
Args:
output_directory(string): The output directory for the YUV file.
file_name(string): The PNG file name.
pattern(string): The file pattern of the PNG/YUV file. The PNG/YUV files are
named patternxx..x.png/yuv, where xx..x are digits starting from 00..0.
Return:
(bool): True upon successful conversion, false otherwise.
"""
# Pattern should be in file name
if not pattern in file_name:
return False
pattern_position = file_name.rfind(pattern)
# Strip the path to the PNG file and replace the png extension with yuv
yuv_file_name = file_name[pattern_position:-3] + 'yuv'
yuv_file_name = os.path.join(output_directory, yuv_file_name)
command = ['ffmpeg', '-i', '%s' % (file_name), '-pix_fmt', 'yuv420p',
'%s' % (yuv_file_name)]
try:
helper_functions.run_shell_command(
command, fail_msg=('Error during PNG to YUV conversion of %s' %
file_name))
os.remove(file_name)
except helper_functions.HelperError as err:
print >> sys.stderr, err
return False
return True
def convert_yuv_to_png_files(yuv_file_name, yuv_frame_width, yuv_frame_height,
output_directory = '.'):
"""Converts a YUV video file into PNG frames.
The function uses ffmpeg to convert the YUV file. The output of ffmpeg is in
the form frame_xxxx.png, where xxxx is the frame number, starting from 0001.
Args:
yuv_file_name(string): The name of the YUV file.
yuv_frame_width(int): The width of one YUV frame.
yuv_frame_height(int): The height of one YUV frame.
output_directory(string): The output directory where the PNG frames will be
stored.
Return:
(bool): True if the conversion was OK.
"""
size_string = str(yuv_frame_width) + 'x' + str(yuv_frame_height)
output_files_pattern = os.path.join(output_directory, 'frame_%04d.png')
command = ['ffmpeg', '-s', '%s' % size_string, '-i', '%s'
% yuv_file_name, '-f', 'image2', '-vcodec', 'png',
'%s' % output_files_pattern]
try:
helper_functions.run_shell_command(
command, msg='Error during YUV to PNG conversion')
except helper_functions.HelperError, err:
print err
return False
def convert_yuv_to_png_files(yuv_file_name, yuv_frame_width, yuv_frame_height,
output_directory, ffmpeg_dir=None):
"""Converts a YUV video file into PNG frames.
The function uses ffmpeg to convert the YUV file. The output of ffmpeg is in
the form frame_xxxx.png, where xxxx is the frame number, starting from 0001.
Args:
yuv_file_name(string): The name of the YUV file.
yuv_frame_width(int): The width of one YUV frame.
yuv_frame_height(int): The height of one YUV frame.
output_directory(string): The output directory where the PNG frames will be
stored.
ffmpeg_dir(string): The directory containing the ffmpeg executable. If
omitted, the PATH will be searched for it.
Return:
(bool): True if the conversion was OK.
"""
size_string = str(yuv_frame_width) + 'x' + str(yuv_frame_height)
output_files_pattern = os.path.join(output_directory, 'frame_%04d.png')
ffmpeg_executable = 'ffmpeg.exe' if sys.platform == 'win32' else 'ffmpeg'
if ffmpeg_dir:
ffmpeg_executable = os.path.join(ffmpeg_dir, ffmpeg_executable)
command = [ffmpeg_executable, '-s', '%s' % size_string, '-i', '%s'
% yuv_file_name, '-f', 'image2', '-vcodec', 'png',
'%s' % output_files_pattern]
try:
print 'Converting YUV file to PNG images (may take a while)...'
print ' '.join(command)
helper_functions.run_shell_command(
command, fail_msg='Error during YUV to PNG conversion')
except helper_functions.HelperError, err:
print 'Error executing command: %s. Error: %s' % (command, err)
return False
def convert_yuv_to_png_files(yuv_file_name,
yuv_frame_width,
yuv_frame_height,
output_directory='.'):
"""Converts a YUV video file into PNG frames.
The function uses ffmpeg to convert the YUV file. The output of ffmpeg is in
the form frame_xxxx.png, where xxxx is the frame number, starting from 0001.
Args:
yuv_file_name(string): The name of the YUV file.
yuv_frame_width(int): The width of one YUV frame.
yuv_frame_height(int): The height of one YUV frame.
output_directory(string): The output directory where the PNG frames will be
stored.
Return:
(bool): True if the conversion was OK.
"""
size_string = str(yuv_frame_width) + 'x' + str(yuv_frame_height)
output_files_pattern = os.path.join(output_directory, 'frame_%04d.png')
command = [
'ffmpeg', '-s',
'%s' % size_string, '-i',
'%s' % yuv_file_name, '-f', 'image2', '-vcodec', 'png',
'%s' % output_files_pattern
]
try:
helper_functions.run_shell_command(
command, msg='Error during YUV to PNG conversion')
except helper_functions.HelperError, err:
print err
return False
def generate_upca_barcodes(number_of_barcodes,
barcode_width,
barcode_height,
output_directory='.',
path_to_zxing='zxing-read-only'):
"""Generates UPC-A barcodes.
This function generates a number_of_barcodes UPC-A barcodes. The function
calls an example Java encoder from the Zxing library. The barcodes are
generated as PNG images. The width of the barcodes shouldn't be less than 102
pixels because otherwise Zxing can't properly generate the barcodes.
Args:
number_of_barcodes(int): The number of barcodes to generate.
barcode_width(int): Width of barcode in pixels.
barcode_height(int): Height of barcode in pixels.
output_directory(string): Output directory where to store generated
barcodes.
path_to_zxing(string): The path to Zxing.
Return:
(bool): True if the conversion is successful.
"""
base_file_name = os.path.join(output_directory, "barcode_")
jars = _form_jars_string(path_to_zxing)
command_line_encoder = 'com.google.zxing.client.j2se.CommandLineEncoder'
barcode_width = str(barcode_width)
barcode_height = str(barcode_height)
errors = False
for i in range(number_of_barcodes):
suffix = helper_functions.zero_pad(i)
# Barcodes starting from 0
content = helper_functions.zero_pad(i, 11)
output_file_name = base_file_name + suffix + ".png"
command = [
"java", "-cp", jars, command_line_encoder,
"--barcode_format=UPC_A",
"--height=%s" % barcode_height,
"--width=%s" % barcode_width,
"--output=%s" % (output_file_name),
"%s" % (content)
]
try:
helper_functions.run_shell_command(
command,
fail_msg=('Error during barcode %s generation' % content))
except helper_functions.HelperError as err:
print >> sys.stderr, err
errors = True
return not errors
def _decode_barcode_in_file(file_name, barcode_width, barcode_height, jars,
command_line_decoder):
"""Decodes the barcode in the upper left corner of a PNG file.
Args:
file_name(string): File name of the PNG file.
barcode_width(int): Width of the barcode (in pixels).
barcode_height(int): Height of the barcode (in pixels)
jars(string): The Zxing core and javase string.
command_line_decoder(string): The ZXing command-line decoding tool.
Return:
(bool): True upon success, False otherwise.
"""
command = [
'java', '-Djava.awt.headless=true', '-cp',
'%s' % jars,
'%s' % command_line_decoder, '--products_only', '--dump_results',
'--brief',
'--crop=%d,%d,%d,%d' % (0, 0, barcode_width, barcode_height),
'%s' % file_name
]
try:
out = helper_functions.run_shell_command(
command, msg='Error during decoding of %s' % file_name)
if not 'Success' in out:
sys.stderr.write('Barcode in %s cannot be decoded\n' % file_name)
return False
except helper_functions.HelperError, err:
print err
return False
def _decode_barcode_in_file(file_name, barcode_width, barcode_height, jars,
command_line_decoder):
"""Decodes the barcode in the upper left corner of a PNG file.
Args:
file_name(string): File name of the PNG file.
barcode_width(int): Width of the barcode (in pixels).
barcode_height(int): Height of the barcode (in pixels)
jars(string): The Zxing core and javase string.
command_line_decoder(string): The ZXing command-line decoding tool.
Return:
(bool): True upon success, False otherwise.
"""
command = ['java', '-Djava.awt.headless=true', '-cp', '%s' % jars,
'%s' % command_line_decoder, '--products_only',
'--dump_results', '--brief', '--crop=%d,%d,%d,%d' %
(0, 0, barcode_width, barcode_height),
'%s' % file_name]
try:
out = helper_functions.run_shell_command(
command, msg='Error during decoding of %s' % file_name)
if not 'Success' in out:
sys.stderr.write('Barcode in %s cannot be decoded\n' % file_name)
return False
except helper_functions.HelperError, err:
print err
return False
def do_avhrr_calibration(process_config, timestamp):
accepted_return_codes_avhrcl = [0]
return_value = True
LOG.debug("Do the avhrr calibration")
#This function relays on beeing in a working directory
current_dir = os.getcwd() #Store the dir to change back to after function complete
os.chdir(process_config['working_directory'])
#calibration_location = "-c -l"
cmd = "avhrcl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(process_config['calibration_location'],
process_config['platform'],
timestamp,
process_config['orbit_number'],
process_config['avhrr_file'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_avhrcl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_value = False
#Change back after this is done
os.chdir(current_dir)
LOG.info("do_avhrr_calibration complete!")
return return_value
def generate_upca_barcodes(number_of_barcodes, barcode_width, barcode_height,
output_directory='.',
path_to_zxing='zxing-read-only'):
"""Generates UPC-A barcodes.
This function generates a number_of_barcodes UPC-A barcodes. The function
calls an example Java encoder from the Zxing library. The barcodes are
generated as PNG images. The width of the barcodes shouldn't be less than 102
pixels because otherwise Zxing can't properly generate the barcodes.
Args:
number_of_barcodes(int): The number of barcodes to generate.
barcode_width(int): Width of barcode in pixels.
barcode_height(int): Height of barcode in pixels.
output_directory(string): Output directory where to store generated
barcodes.
path_to_zxing(string): The path to Zxing.
Return:
(bool): True if the conversion is successful.
"""
base_file_name = os.path.join(output_directory, "barcode_")
jars = _form_jars_string(path_to_zxing)
command_line_encoder = 'com.google.zxing.client.j2se.CommandLineEncoder'
barcode_width = str(barcode_width)
barcode_height = str(barcode_height)
errors = False
for i in range(number_of_barcodes):
suffix = helper_functions.zero_pad(i)
# Barcodes starting from 0
content = helper_functions.zero_pad(i, 11)
output_file_name = base_file_name + suffix + ".png"
command = ["java", "-cp", jars, command_line_encoder,
"--barcode_format=UPC_A", "--height=%s" % barcode_height,
"--width=%s" % barcode_width,
"--output=%s" % (output_file_name), "%s" % (content)]
try:
helper_functions.run_shell_command(
command, fail_msg=('Error during barcode %s generation' % content))
except helper_functions.HelperError as err:
print >> sys.stderr, err
errors = True
return not errors
def convert_yuv_to_png_files(yuv_file_name,
yuv_frame_width,
yuv_frame_height,
output_directory='.'):
"""Converts a YUV video file into PNG frames.
The function uses ffmpeg to convert the YUV file. The output of ffmpeg is in
the form frame_xxxx.png, where xxxx is the frame number, starting from 0001.
Args:
yuv_file_name(string): The name of the YUV file.
yuv_frame_width(int): The width of one YUV frame.
yuv_frame_height(int): The height of one YUV frame.
output_directory(string): The output directory where the PNG frames will be
stored.
Return:
(bool): True if the conversion was OK.
"""
size_string = str(yuv_frame_width) + 'x' + str(yuv_frame_height)
output_files_pattern = os.path.join(output_directory, 'frame_%04d.png')
ffmpeg_executable = 'ffmpeg'
if sys.platform == 'win32':
if os.getenv('FFMPEG_HOME'):
ffmpeg_executable = os.path.join(os.getenv('FFMPEG_HOME'), 'bin',
'ffmpeg.exe')
else:
ffmpeg_executable = 'ffmpeg.exe'
command = [
ffmpeg_executable, '-s',
'%s' % size_string, '-i',
'%s' % yuv_file_name, '-f', 'image2', '-vcodec', 'png',
'%s' % output_files_pattern
]
try:
helper_functions.run_shell_command(
command, msg='Error during YUV to PNG conversion')
except helper_functions.HelperError, err:
print >> sys.stderr, 'Error executing command: %s. Error: %s' % (
command, err)
return False
def do_tle_satpos(timestamp, satellite, satpos_dir=None):
return_status = True
LOG.info("satpos files is stored under the given directory/satpos")
if satpos_dir == None:
satpos_dir = os.path.join(os.environ['DIR_NAVIGATION'], "satpos" )
else:
satpos_dir = os.path.join(satpos_dir, "satpos" )
if not os.path.exists(satpos_dir):
try:
os.makedirs(satpos_dir)
except:
LOG.error("Could not create satpos directory: {}".format(satpos_dir))
return_status = False
file_satpos = os.path.join(satpos_dir, "satpos_{}_{:%Y%m%d}.txt".format(satellite,timestamp))
if (not os.path.exists(file_satpos) or os.stat(file_satpos).st_size == 0) and return_status:
"""Usage is: satpostle [ -o] [-s satellite] [-S station] [-d start date] [-n number of days] [-i increment in seconds] [-c search criteria]
-o -s -S -d -n -i –c are optional.
If no parameter is specified as an option, defaults are : noaa14, Lannion, today 0h, 1.0, 120.0, n (n= nearest, p = preceding).
The option -o specifies that the data will be stored in the file satpos_noaxx_yyyymmdd.txt.
Output default is the standard output..
"""
cmd="satpostle -o -s {} -d {:%d/%m/%y} -n 1.2".format(satellite,timestamp)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Failed to run command: {}".format(cmd))
return_status = False
else:
if returncode != 0:
LOG.error("cmd: {} failed with returncode: {}".format(cmd,returncode))
return_status = False
elif not os.path.exists(file_satpos):
LOG.error("file: {} does not exists after satpostle run.".format(file_satpos))
return_status = False
else:
LOG.info("satpos file already there. Use this")
return return_status
def do_avhrr_calibration(process_config, msg, timestamp):
if not process_config['process_avhrr']:
LOG.debug("Skipping avhrr processing due to config or missing data.")
return True
accepted_return_codes_avhrcl = [0]
return_value = True
LOG.debug("Do the avhrr calibration")
# This function relays on beeing in a working directory
current_dir = os.getcwd(
) # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]
['working_dir'])
# calibration_location = "-c -l"
try:
cmd = "avhrcl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(
process_config['calibration_location'],
process_config['platform_name'], timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']['decommutation_files']
['avhrr_file'])
except Exception as err:
LOG.error("Failed to build avhrcl command: {}".format(err))
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_avhrcl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
return_value = False
# Change back after this is done
os.chdir(current_dir)
LOG.info("do_avhrr_calibration complete!")
return return_value
def _decode_barcode_in_file(file_name, command_line_decoder):
"""Decodes the barcode in the upper left corner of a PNG file.
Args:
file_name(string): File name of the PNG file.
command_line_decoder(string): The ZXing command-line decoding tool.
Return:
(bool): True upon success, False otherwise.
"""
command = [command_line_decoder, '--try-harder', '--dump-raw', file_name]
try:
out = helper_functions.run_shell_command(
command, fail_msg='Error during decoding of %s' % file_name)
text_file = open('%s.txt' % file_name[:-4], 'w')
text_file.write(out)
text_file.close()
except helper_functions.HelperError, err:
print 'Barcode in %s cannot be decoded.' % file_name
print err
return False
def _decode_barcode_in_file(file_name, command_line_decoder):
"""Decodes the barcode in the upper left corner of a PNG file.
Args:
file_name(string): File name of the PNG file.
command_line_decoder(string): The ZXing command-line decoding tool.
Return:
(bool): True upon success, False otherwise.
"""
command = [command_line_decoder, '--try-harder', '--dump-raw', file_name]
try:
out = helper_functions.run_shell_command(
command, fail_msg='Error during decoding of %s' % file_name)
text_file = open('%s.txt' % file_name[:-4], 'w')
text_file.write(out)
text_file.close()
except helper_functions.HelperError, err:
print 'Barcode in %s cannot be decoded.' % file_name
print err
return False
def _decode_barcode_in_file(file_name, barcode_width, barcode_height, jars,
command_line_decoder):
"""Decodes the barcode in the upper left corner of a PNG file.
Args:
file_name(string): File name of the PNG file.
barcode_width(int): Width of the barcode (in pixels).
barcode_height(int): Height of the barcode (in pixels)
jars(string): The Zxing core and javase string.
command_line_decoder(string): The ZXing command-line decoding tool.
Return:
(bool): True upon success, False otherwise.
"""
java_executable = 'java'
if sys.platform == 'win32':
if os.getenv('JAVA_HOME'):
java_executable = os.path.join(os.getenv('JAVA_HOME'), 'bin',
'java.exe')
else:
java_executable = 'java.exe'
command = [
java_executable, '-Djava.awt.headless=true', '-cp',
'%s' % jars,
'%s' % command_line_decoder, '--products_only', '--dump_results',
'--brief',
'--crop=%d,%d,%d,%d' % (0, 0, barcode_width, barcode_height),
'%s' % file_name
]
try:
out = helper_functions.run_shell_command(
command, msg='Error during decoding of %s' % file_name)
if not 'Success' in out:
print >> sys.stderr, 'Barcode in %s cannot be decoded\n' % file_name
return False
except helper_functions.HelperError, err:
print >> sys.stderr, err
return False
def do_atovpp_and_avh2hirs_processing(process_config, timestamp):
LOG.debug("Do preprocessing atovpp and avh2hirs ... ")
return_status = True
#This function relays on beeing in a working directory
current_dir = os.getcwd() #Store the dir to change back to after function complete
os.chdir(process_config['working_directory'])
instruments = "AMSU-A AMSU-B HIRS"
if "".join(process_config['a_tovs']) == 'TOVS':
instruments = "MSU HIRS"
cmd = "atovin {}".format(instruments)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
LOG.debug(std)
LOG.debug(err)
if return_status:
cmd = "atovpp -i \"{}\" -g HIRS ".format(instruments)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
#THis will not work since the avh2hirs script sources the ATOVS_CONF at start
#and relay on WRK dir, which is a static dir
#
#Skip for now
if 0:
cmd = "avh2hirs {}".format("".join(process_config['a_tovs']))
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.warning("avh2hirs is not implemented in the processing.")
#Change back after this is done
os.chdir(current_dir)
LOG.info("atovpp and avh2hirs complete!")
return return_status
def do_atovs_calibration(process_config, timestamp):
if(not process_config['process_amsua'] and
not process_config['process_amsub'] and
not process_config['process_msu']):
LOG.debug("Skipping atovs processing")
return True
accepted_return_codes_msucl = [0]
accepted_return_codes_amsuacl = [0]
accepted_return_codes_amsubcl = [0]
return_value = True
# This function relays on beeing in a working directory
current_dir = os.getcwd() # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]['working_dir'])
# calibration_location = "-c -l"
if "".join(process_config['a_tovs']) == 'TOVS':
cmd = "msucl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(process_config['calibration_location'],
process_config['platform_name'],
timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']['decommutation_files']['msun_file'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_msucl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_value = False
elif "".join(process_config['a_tovs']) == 'ATOVS':
if process_config['process_amsua']:
cmd = "amsuacl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(process_config['calibration_location'],
process_config['platform_name'],
timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']['decommutation_files']['amsua_file'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_amsuacl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_value = False
if process_config['process_amsub'] and return_value:
amsub_script = "mhscl"
try:
if 'noaa' in process_config['platform_name'] and int(process_config['platform_name'][-2:]) <= 17:
amsu_script = "amsubcl"
process_config['process_mhs'] = False
else:
process_config['process_amsub'] = False
except ValueError as ve:
LOG.warning("Could not exctract satellite number from the last two characters in the platform name: "
"{} and convert it to a number.".format(process_config['platform_name']))
cmd = "{0} {1} -s {2} -d {3:%Y%m%d} -h {3:%H%M} -n {4:05d} {5}".format(amsub_script,
process_config['calibration_location'],
process_config['platform_name'],
timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']['decommutation_files']['amsub_file'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_amsubcl:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_value = False
else:
LOG.error("Unknown A|TOVS key string: {}".format("".join(process_config['a_tovs'])))
return_value = False
# Change back after this is done
os.chdir(current_dir)
LOG.info("do_atovs_calibration complete!")
return return_value
def do_atovpp_and_avh2hirs_processing(process_config, timestamp):
if (not process_config['process_amsua']
and not process_config['process_amsub']
and not process_config['process_hirs']):
LOG.debug("Skipping atovpp and avh2hirs processing.")
return True
LOG.debug("Do preprocessing atovpp and avh2hirs ... ")
return_status = True
# This function relays on beeing in a working directory
current_dir = os.getcwd(
) # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]
['working_dir'])
instruments = "AMSU-A AMSU-B HIRS"
grids = "AMSU-A AMSU-B HIRS"
if not process_config['process_hirs']:
instruments = "AMSU-A AMSU-B"
grids = "AMSU-A AMSU-B"
if "".join(process_config['a_tovs']) == 'TOVS':
instruments = "MSU HIRS"
grids = "MSU HIRS"
if 'do_atovpp' in process_config['aapp_processes'][
process_config.process_name]:
process_config['do_atovpp'] = process_config['aapp_processes'][
process_config.process_name]['do_atovpp']
else:
process_config['do_atovpp'] = False
if 'do_avh2hirs' in process_config['aapp_processes'][
process_config.process_name]:
process_config['do_avh2hirs'] = process_config['aapp_processes'][
process_config.process_name]['do_avh2hirs']
else:
process_config['do_avh2hirs'] = False
if process_config['do_atovpp']:
cmd = "atovin {}".format(instruments)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
LOG.debug(std)
LOG.debug(err)
if return_status:
cmd = "atovpp -i \"{}\" -g \"{}\"".format(instruments, grids)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
if return_status and process_config['do_avh2hirs'] and process_config[
'process_hirs']:
if os.path.exists(
"./{}".format(process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file'])):
os.symlink(
"./{}".format(process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file']),
"{}11".format(os.environ["FORT"]))
else:
LOG.error("Could not find file: ./{}".format(
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file']))
return_status = False
if os.path.exists("./hirs.l1d"):
os.symlink("./hirs.l1d", "{}12".format(os.environ["FORT"]))
else:
LOG.error("Could not find file: {}".format("./hirs.l1d"))
return_status = False
if return_status:
cmd = "l1didf -i hirs.l1d"
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
satimg, yyyymmdd, hhmn, orbit, instr, loc1, loc2 = std.split(
)
if return_status:
if "noaa" in satimg:
satnb = int(satimg[4:6])
else:
satnb = int(satimg[5:7])
LOG.debug("Using satnb: {}".format(satnb))
unit = satnb + 50
mmc = int(yyyymmdd[4:6])
LOG.debug("Using unit: {} and mmc: {}".format(unit, mmc))
if os.path.exists(
os.path.join(os.environ["DIR_PREPROC"],
"cor_{}.dat".format(satimg))):
os.symlink(
os.path.join(os.environ["DIR_PREPROC"],
"cor_{}.dat".format(satimg)),
"{}{}".format(os.environ["FORT"], unit))
else:
LOG.error("Failed to find {}".format(
os.path.join(os.environ["DIR_PREPROC"],
"cor_{}.dat".format(satimg))))
return_status = False
if return_status:
os.environ["SATIMG"] = satimg
os.environ["YYYYMMDD"] = yyyymmdd
os.environ["HHMN"] = hhmn
os.environ["DIR_MAIA2_ATLAS"] = os.path.join(
os.environ["DIR_PREPROC"], "atlas")
os.environ["DIR_MAIA2_THRESHOLDS"] = os.path.join(
os.environ["DIR_PREPROC"], "thresholds")
if "".join(process_config['a_tovs']) == 'TOVS':
cmd = "maia2_env;maia2_environment;avh2hirs.exe 2>&1"
else:
cmd = "bash -c \"source liblog.sh;source maia2_env;maia2_environment\";avh2hirs_atovs.exe 2>&1"
LOG.debug("Before running command: {}".format(cmd))
try:
status, returncode, std, err = run_shell_command(
cmd,
stdout_logfile="avh2hirs.log",
use_shlex=False,
use_shell=True)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode != 0:
LOG.error("Command {} failed with return code {}.".format(
cmd, returncode))
LOG.error(std)
LOG.error(err)
return_status = False
else:
LOG.info("Command {} complete.".format(cmd))
LOG.debug(std)
LOG.debug(err)
if return_status:
import glob
for fortfile in glob.glob("./fort*"):
os.remove(fortfile)
os.remove("./albedo")
os.remove("./sst")
os.remove("./wv")
# Change back after this is done
os.chdir(current_dir)
LOG.info("atovpp and avh2hirs complete!")
return return_status
def do_decommutation(process_config, sensors, timestamp, hrpt_file):
"""
decommutation ${A_TOVS} decommutation.par ${FILE}
The 3 arguments are obligatory.
${A_TOVS} = TOVS for satellite number < or = 14
${A_TOVS} = ATOVS for satellite number > 14
In the decommutation.par file, options are written in this order:
$1,$2,$3,$4,$5, $6, $7, $8,$9,$10,$11,$12 !OPTION NUMBERS
$lu1,$lu2,$lu3,$lu4,$lu5,$lu6,$lu7,$lu8,$lu9,$lu10,$lu11,$lu12 !STREAM NO.S
${YEAR} ! year of the data
0 ! operational mode
${NNNNN},${NNNNN} ! start and end orbit numbers
$1 = 0 or 1 for level of error logging
$2 = 0 or 1 for HIRS/3 or HIRS/4 (1 indicates extract HIRS/3 or HIRS/4 data)
$3 = 0 or 1 for AMSU-A1 (1 indicates extract AMSU-A1 data)
$4 = 0 or 1 for AMSU-A2 (1 indicates extract AMSU-A2 data)
$5 = 0 or 1 for AMSU-B/MHS (1 indicates extract AMSU-B/MHS data)
$6 = 0 or 1 for HIRS/2 (1 indicates extract HIRS/2 data)
$7 = 0 or 1 for MSU (1 indicates extract MSU data)
$8 = 0 or 1 for DCS (1 indicates extract DCS data)
$9 = 0 or 1 for SEM (1 indicates extract SEM data)
$10= 0 or 1 for SBUV (1 indicates extract SBUV data)
$11= 0 or 1 for SAR (1 indicates extract SAR data)
$12= 0 or 1 for AVHRR (1 indicates extract AVHRR data)
$lu1 is the logical unit of the log file
$lu2 is the logical unit of the HIRS/3 or HIRS/4.l1a output file
$lu3 is the logical unit of the AMSU-A1.l1a output file
$lu4 is the logical unit of the AMSU-A2.l1a output file
$lu5 is the logical unit of the AMSU-B.l1a output file
$lu6 is the logical unit of the HIRS/2.l1a output file
$lu7 is the logical unit of the MSU.l1a output file
$lu8 is the logical unit of the DCS.l1a output file
$lu9 is the logical unit of the SEM.l1a output file
$lu10 is the logical unit of the SBUV.l1a output file
$lu11 is the logical unit of the SAR.l1a output file
$lu12 is the logical unit of the HRPT.l1a output file
"""
#A list of accepted return codes for the various scripts/binaries run in this function
accepted_return_codes_decom_hrpt = [0]
accepted_return_codes_decom_amsua_metop = [0]
accepted_return_codes_decom_amsub_metop = [0]
accepted_return_codes_decom_hirs_metop = [0]
accepted_return_codes_decom_avhrr_metop = [0]
return_status = True
#This function relays on beeing in a working directory
current_dir = os.getcwd() #Store the dir to change back to after function complete
os.chdir(process_config['working_directory'])
print "decom file: ", hrpt_file
for sensor in sensors:
if str(sensor) in "amsu-a":
process_config['process_amsua'] = True
elif str(sensor) in ("amsu-b", "mhs"):
process_config['process_amsub'] = True
elif str(sensor) in ("hirs/3","hirs/4"):
process_config['process_hirs'] = True
elif str(sensor) in "avhrr/3":
process_config['process_avhrr'] = True
elif str(sensor) in "msu":
process_config['process_msu'] = True
elif str(sensor) in "dcs":
process_config['process_dcs'] = True
else:
LOG.warning("Instrument/sensor {} not recognised.".format(sensor))
if 'noaa' in process_config['platform']:
print "Do the commutaion for NOAA"
#a_tovs = "ATOVS"
#PROCESS IN THIS ORDER
#ama = 0 #AMSU-A
#amb = 0 #AMSU-B
#hrs = 0 #HIRS
#avh = 0 #avhrr
#msu = 0
#dcs = 0
decom_file = "decommutation.par"
#Needs to find platform number for A/TOVS
decom = open(decom_file, 'w')
if int(process_config['platform'][4:6]) <= 14:
del process_config['a_tovs'][0]
decom.write("1,0,0,0,0,{},{},{},0,0,0,{} ! OPTION NUMBERS\n".format(1 if process_config['process_hirs'] else 0,
1 if process_config['process_msu'] else 0,
1 if process_config['process_dcs'] else 0,
1 if process_config['process_avhrr'] else 0))
decom.write("10,0,0,0,0,11,12,13,0,0,0,14 ! STREAM NO.S\n")
else:
decom.write("1,{},{},{},{},0,0,{},0,0,0,{} ! OPTION NUMBERS\n".format(1 if process_config['process_hirs'] else 0,
1 if process_config['process_amsua'] else 0,
1 if process_config['process_amsua'] else 0,
1 if process_config['process_amsub'] else 0,
1 if process_config['process_dcs'] else 0,
1 if process_config['process_avhrr'] else 0))
decom.write("10,11,15,15,16,0,0,13,0,0,0,14 ! STREAM NO.S\n")
decom.write("{:%Y} ! year of the data\n".format(timestamp))
decom.write("0 ! operational mode\n")
decom.write("{0:05d},{0:05d} ! start and end orbit numbers\n".format(process_config['orbit_number']))
decom.close()
cmd="decommutation {} {} {}".format("".join(process_config['a_tovs']),decom_file, hrpt_file)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_hrpt:
LOG.info("Command {} complete.".format(cmd))
if not os.path.exists(process_config['avhrr_file']):
LOG.warning("Decom gave OK status, but no {} data is produced. Something is wrong".format(process_config['avhrr_file']))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
elif 'metop' in process_config['platform']:
LOG.info("Do the metop decommutation")
if process_config['process_amsua']:
cmd="decom-amsua-metop {} {} ".format(process_config['input_amsua_file'],process_config['amsua_file'])
try:
status, returncode, std, err = run_shell_command(cmd,stdout_logfile="decom-amsua-metop.log")
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_amsua_metop:
LOG.info("Command {} complete.".format(cmd))
if not os.path.exists(process_config['amsua_file']):
LOG.warning("Decom gave OK status, but no data is produced. Something is wrong")
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
if process_config['process_amsub'] and return_status:
cmd="decom-mhs-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
process_config['input_amsub_file'],
process_config['amsub_file'])
try:
status, returncode, std, err = run_shell_command(cmd,stdout_logfile="decom-mhs-metop.log")
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_amsub_metop:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
if process_config['process_hirs'] and return_status:
cmd="decom-hirs-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
process_config['input_hirs_file'],
process_config['hirs_file'])
try:
status, returncode, std, err = run_shell_command(cmd,stdout_logfile="decom-hirs-metop.log")
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_hirs_metop:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
if process_config['process_avhrr'] and return_status:
cmd="decom-avhrr-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
process_config['input_avhrr_file'],
process_config['avhrr_file'])
try:
status, returncode, std, err = run_shell_command(cmd,stdout_logfile="decom-avhrr-metop.log")
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_avhrr_metop:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
else:
print "Unknown platform: {}".format(process_config['platform'])
return_status = False
print os.listdir("./")
#Change back after this is done
os.chdir(current_dir)
LOG.info("Decommutation complete.")
return return_status
def do_ana_correction(process_config, msg, timestamp):
"""The software ANA is 3party. User will have to get it themself.
Further the ana binaries and scripts must be in the PATH,
preferable in the primary AAPP bin directory
Also it relays on reference_landmarks directory under ENV(DIR_NAVIGATION)/ana dir
This needs to be copied/installed here by the user
TODO: Replace the ana_lmk_loc script with code here to better be able to control
the directories used in the processing
"""
if not process_config['process_avhrr']:
LOG.debug(
"Skipping ANA as AVHRR is not processed de to config or missing data."
)
return True
try:
if 'do_ana_correction' in process_config['aapp_processes'][
process_config.process_name]:
if not process_config['aapp_processes'][
process_config.process_name]['do_ana_correction']:
return True
except Exception as err:
LOG.error("Failed with: {}".format(err))
return_status = True
# This function relays on beeing in a working directory
current_dir = os.getcwd(
) # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]
['working_dir'])
# Must check of the ana dir exists
ana_dir = os.path.join(os.getenv('DIR_NAVIGATION'), 'ana')
if not os.path.exists(ana_dir):
try:
os.makedirs(ana_dir)
os.environ['DIR_ANA'] = ana_dir
except OSError in e:
LOG.error(
"Failed to create directory: {}. This is needed to run the ANA software."
)
return_status = False
# And the ana dir must contain a reference_landmarks directory to work!
if not os.path.exists(os.path.join(ana_dir, "reference_landmarks")):
LOG.error(
"Can not run ANA because reference_landmarks under the ana dir is missing."
)
return_status = False
if return_status:
# Find all matching landmarks
cmd = "ana_lmk_loc -D {}".format(
process_config['aapp_static_configuration']['decommutation_files']
['avhrr_file'])
try:
status = False
status, returncode, std, err = run_shell_command(
cmd,
stdout_logfile="ana_lmk_loc.log",
stderr_logfile="ana_lmk_loc.err")
except:
import sys
LOG.error("Command {} failed with {}.".format(
cmd,
sys.exc_info()[0]))
if not status:
return_status = False
else:
if not status or returncode != 0:
LOG.error("Command {} failed with {}".format(cmd, returncode))
_ana_file = open('ana_lmk_loc.err', "w")
_ana_file.write(std)
_ana_file.write(err)
_ana_file.close()
return_status = False
if return_status:
# Check of ana landmark location file with correct time stamp exist.
# If not try to find correct timestamp and copy to this.
expected_ana_loc_file = "lmkloc_{}_{:%Y%m%d_%H%M}_{:05d}.txt".format(
process_config['platform_name'], timestamp,
process_config['orbit_number'])
if not os.path.exists(
os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
expected_ana_loc_file)):
# Need to run this AAPP command to detect the start of the hrpt.l1b(avhrr data) file
# Because this might differ from the time stamp in file name
cmd = "l1bidf.exe"
try:
status, returncode, std, err = run_shell_command(
cmd,
stdin="{}\n".format(
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file']))
except:
import sys
LOG.error("Command {} failed with {}.".format(
cmd,
sys.exc_info()[0]))
if not status:
return_status = False
else:
if not status or returncode != 0:
LOG.error("Command {} failed with {}".format(
cmd, returncode))
_ana_file = open('ana_lmk_loc.err', "w")
_ana_file.write(std)
_ana_file.write(err)
_ana_file.close()
return_status = False
else:
l1bidf_list = std.split()
l1bidf_aapp_datetime = datetime.strptime(
l1bidf_list[1] + l1bidf_list[2], "%Y%m%d%H%M")
LOG.debug("l1b idf aapp datetime: {}".format(
l1bidf_aapp_datetime))
# estatt_file_name = lmkloc_metop02_20160617_0952_50132.txt
ana_loc_file = "lmkloc_{}_{:%Y%m%d_%H%M}_{:05d}.txt".format(
l1bidf_list[0], l1bidf_aapp_datetime,
int(l1bidf_list[3]))
if os.path.exists(
os.path.join(
ana_dir,
"{:%Y-%m}".format(l1bidf_aapp_datetime),
ana_loc_file)):
LOG.debug(
"timestamp {} l1bidf_aapp_datetime: {}".format(
timestamp, l1bidf_aapp_datetime))
from shutil import copy2
copy2(
os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
ana_loc_file),
os.path.join(
ana_dir,
"{:%Y-%m}".format(l1bidf_aapp_datetime),
expected_ana_loc_file))
else:
LOG.warning("Could not find ana loc file: {}".format(
ana_loc_file))
else:
LOG.debug("File ok: {}".format(
os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
expected_ana_loc_file)))
import hashlib
if return_status:
# LOG.debug("sha256 of aapp input avhrr_file: {}".format(hashlib.sha256(open(
# process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest()))
# Calculate correction from landmarks and update avhrr_file with these new attitude coefisients.
cmd = "ana_estatt -s {0} -d {1:%Y%m%d} -h {1:%H%M} -n {2:05d}".format(
process_config['platform_name'], timestamp,
process_config['orbit_number'])
try:
status, returncode, std, err = run_shell_command(cmd)
except:
import sys
LOG.error("Command {} failed with {}.".format(
cmd,
sys.exc_info()[0]))
if not status:
return_status = False
else:
if not status or returncode != 0:
LOG.error("Command {} failed with {}".format(cmd, returncode))
_ana_file = open('ana_lmk_loc.err', "w")
_ana_file.write(std)
_ana_file.write(err)
_ana_file.close()
return_status = False
# LOG.debug("sha256 of aapp input avhrr_file: {}".format(hashlib.sha256(open(
# process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest()))
sha256_before_correction = hashlib.sha256(
open(
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file'],
'rb').read()).hexdigest()
if return_status:
# Recalculate the location in the avhhr data file with the new correction attitude coefisients.
from do_avhrr_calibration import do_avhrr_calibration
LOG.info(
"Need to recalculate the avhrcl with new attitude coefisients from ANA."
)
if not do_avhrr_calibration(process_config, msg, timestamp):
LOG.warning(
"The avhrr location with ana correction failed for some reason."
"It might be that the processing can continue")
LOG.warning(
"Please check the previous log carefully to see if this is an error you can accept."
)
return_status = False
if return_status:
sha256_after_correction = hashlib.sha256(
open(
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file'],
'rb').read()).hexdigest()
LOG.debug("sha256 of aapp input avhrr_file BEFORE ana: {}".format(
sha256_before_correction))
LOG.debug("sha256 of aapp input avhrr_file AFTER ana: {}".format(
sha256_after_correction))
if (sha256_before_correction == sha256_after_correction):
LOG.warning(
"The correction of the avhrr location with data from ANA did not take place for some reason,"
"or the correction was 0 for all pitch yaw and roll.")
else:
LOG.info(
"The correction of AVHRR location with data from ANA was performed."
)
# Change back after this is done
os.chdir(current_dir)
LOG.info("do_ana_correction complete!")
return return_status
def do_tleing(timestamp, satellite, workdir, tle_indir=None, select_closest_tle_file_to_data=False):
"""Get the tle-file and copy them to the AAPP data structure
and run the AAPP tleing script and executable"""
return_status = True
#This function relays on beeing in a working directory
current_dir = os.getcwd() #Store the dir to change back to after function complete
os.chdir(workdir)
# SATellite IDentification mandatory
# so take care of default values
SATID_FILE=os.getenv('SATID_FILE', 'satid.txt')
if tle_indir == None:
"""This is the default directory for the tles"""
tle_indir = os.getenv('DIR_NAVIGATION')
else:
LOG.warning("Override the env variable set in AAPP_ENV7 DIR_NAVIGATION from {} to {}.".format(os.environ['DIR_NAVIGATION'], tle_indir))
os.environ['DIR_NAVIGATION'] = tle_indir
# variables for the TLE HOME directory
DIR_DATA_TLE=os.path.join(os.getenv('DIR_NAVIGATION'),'tle_db')
#This is needed by AAPP tleing. Try other of not existing
if not os.path.exists(DIR_DATA_TLE):
DIR_DATA_TLE=os.path.join(os.getenv('DIR_NAVIGATION'),'orb_elem')
FIC_WRK=os.path.join(os.getenv('DIR_DATA_TLE'),'tmp_tle_',"{}".format(os.getpid()))
#LISTESAT=os.getenv('LISTESAT',os.getenv('PAR_NAVIGATION_DEFAULT_LISTESAT_INGEST_TLE'))
#print LISTESAT
TLE_INDEX=os.path.join(DIR_DATA_TLE,"tle_{}.index".format(satellite))
tle_file_list = []
if not select_closest_tle_file_to_data:
if os.path.exists(TLE_INDEX):
#tle_files = [s for s in os.listdir(DIR_DATA_TLE) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))]
#_tle_file_list = glob(os.path.join(DIR_DATA_TLE,'tle*txt'))
tle_files = [s for s in glob(os.path.join(DIR_DATA_TLE,'tle*txt')) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))]
tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE,s)))
tle_index_mtime = os.path.getmtime(TLE_INDEX)
for s in tle_files:
if os.path.getmtime(os.path.join(DIR_DATA_TLE,s)) > tle_index_mtime:
tle_file_list.append(s)
if len(tle_file_list) == 0:
import time
LOG.warning("No newer tle files than last update of the index file. Last update of index file is {:d}s. If more than a few days you should check.".format(int(time.time()-tle_index_mtime)))
else:
LOG.info("Will use tle files {}".format(tle_file_list))
else:
LOG.warning("index file does not exist. If this is the first run of AAPP tleing.exe it is ok, elsewise it is a bit suspisiuos.")
tle_files = [s for s in os.listdir(DIR_DATA_TLE) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))]
tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE,s)))
tle_file_list = tle_files
else:
#FIXME add as config
tle_file_timestamp_format = "%Y%m%dT%H%M"
infile = "tle-{0:{1}}.txt".format(timestamp,tle_file_timestamp_format)
LOG.debug("tle file name: {}".format(infile))
min_closest_tle_file = 3*24*60*60
#Check if I can read the tle file.
try:
with open(os.path.join(tle_indir, infile)) as tle_file:
pass
except IOError as e:
LOG.warning("Could not find tle file: {}. Try find closest ... ".format(infile))
infile = None
#tle_file_list = glob(os.path.join(tle_indir,'tle*'))
#print "tle file list: {}".format(tle_file_list)
import re
tle_match_tests = (('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2})(\d{2}).*',_do_6_matches),
('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2}).*',_do_5_matches),
('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2}).*',_do_4_matches),
('.*(\d{4})(\d{2})(\d{2}).*',_do_3_matches))
#print tle_file_list
for tle_file_name in tle_files:
for regex, test in tle_match_tests:
m = re.match(regex, tle_file_name)
if m:
delta = timestamp - test(m)
if ( abs(delta.total_seconds()) < min_closest_tle_file):
min_closest_tle_file = abs(delta.total_seconds())
infile = os.path.basename(tle_file_name)
break
tle_file_list.append(infile)
LOG.debug("Use this: {} {}".format(tle_file_list,min_closest_tle_file))
#print "{}".format(tle_file_list)
#infile = "tle-{0:{1}}.txt".format(timestamp,tle_file_timestamp_format)
for tle_file in tle_file_list:
if not os.path.exists(os.path.join(tle_indir,'tle_db',tle_file)):
print "Could not find the tle file: {}".format(tle_indir + "/" + tle_file)
return_status = False
else:
"""Dont use the tle_indir because this is handeled by the tleing script"""
#tle_cmd = open("tle_commands", 'w')
#tle_cmd.write("{}\n".format(DIR_DATA_TLE))
#tle_cmd.write("{}\n".format(tle_file))
#tle_cmd.write("{}\n".format(satellite))
#tle_cmd.write("{}\n".format(TLE_INDEX))
#tle_cmd.close()
#LOG.info("TLE file ok. Do the calc for {} ... ".format(satellite))
status = False
returncode = 0
stdout = ""
stderr = ""
cmd="tleing.exe"
try:
status, returncode, stdout, stderr = run_shell_command(cmd,stdin="{}\n{}\n{}\n{}\n".format(DIR_DATA_TLE,
os.path.basename(tle_file),
satellite,
TLE_INDEX))
except:
LOG.error("Failed running command: {} with return code: {}".format(cmd,returncode))
LOG.error("stdout: {}".format(stdout))
LOG.error("stderr: {}".format(stderr))
return_status = False
else:
if returncode != 0:
LOG.debug("Running command: {} with return code: {}".format(cmd,returncode))
LOG.debug("stdout: {}".format(stdout))
LOG.debug("stderr: {}".format(stderr))
elif not os.path.exists(TLE_INDEX):
LOG.error("index file: {} does not exist after tleing. Something is wrong.".format(TLE_INDEX))
LOG.debug("Running command: {} with return code: {}".format(cmd,returncode))
LOG.debug("stdout: {}".format(stdout))
LOG.debug("stderr: {}".format(stderr))
else:
LOG.debug("Running command: {} with return code: {}".format(cmd,returncode))
LOG.debug("stdout: {}".format(stdout))
LOG.debug("stderr: {}".format(stderr))
#When a index file is generated above one line is added for each tle file.
#If several tle files contains equal TLEs each of these TLEs generate one line in the index file
#To avoid this, sort the index file keeping only unique lines(skipping the tle filename at the end
#The sort options +0b -3b is guessed to be sort from column 0 to 3, but this is not documented
#Could cause problems with future version of sort. See eg. http://search.cpan.org/~sdague/ppt-0.12/bin/sort
#cmd="sort -u -o {} +0b -3b {}".format(os.path.join(DIR_DATA_TLE, "{}.sort".format(TLE_INDEX)),os.path.join(DIR_DATA_TLE, TLE_INDEX))
if os.path.exists(TLE_INDEX):
cmd="sort -u -o {} +0b -3b {}".format("{}.sort".format(TLE_INDEX), TLE_INDEX)
try:
status, returncode, stdout, stderr = run_shell_command(cmd)
except:
LOG.error("Failed running command: {} with return code: {}".format(cmd,returncode))
LOG.error("stdout: {}".format(stdout))
LOG.error("stderr: {}".format(stderr))
return_status = False
else:
if returncode == 0 and os.path.exists("{}.sort".format(TLE_INDEX)):
try:
#os.remove(os.path.join(DIR_DATA_TLE, TLE_INDEX))
os.remove(TLE_INDEX)
except OSError as e:
#LOG.error("Failed to remove unsorted and duplicated index file: {}".format(os.path.join(DIR_DATA_TLE, TLE_INDEX)))
LOG.error("Failed to remove unsorted and duplicated index file: {}".format(TLE_INDEX))
else:
try:
#os.rename(os.path.join(DIR_DATA_TLE, "{}.sort".fromat(TLE_INDEX)),os.path.join(DIR_DATA_TLE, TLE_INDEX))
os.rename("{}.sort".format(TLE_INDEX),TLE_INDEX)
except:
LOG.error("Failed to rename sorted index file to original name.")
else:
LOG.error("Returncode other than 0: {} or tle index sort file does exists.".format(returncode, "{}.sort".format(TLE_INDEX)))
else:
LOG.error("tle index file: {} does not exists after tleing before sort. This can not happen.")
#Change back after this is done
os.chdir(current_dir)
return return_status
def do_decommutation(process_config, msg, timestamp):
"""
decommutation ${A_TOVS} decommutation.par ${FILE}
The 3 arguments are obligatory.
${A_TOVS} = TOVS for satellite number < or = 14
${A_TOVS} = ATOVS for satellite number > 14
In the decommutation.par file, options are written in this order:
$1,$2,$3,$4,$5, $6, $7, $8,$9,$10,$11,$12 !OPTION NUMBERS
$lu1,$lu2,$lu3,$lu4,$lu5,$lu6,$lu7,$lu8,$lu9,$lu10,$lu11,$lu12 !STREAM NO.S
${YEAR} ! year of the data
0 ! operational mode
${NNNNN},${NNNNN} ! start and end orbit numbers
$1 = 0 or 1 for level of error logging
$2 = 0 or 1 for HIRS/3 or HIRS/4 (1 indicates extract HIRS/3 or HIRS/4 data)
$3 = 0 or 1 for AMSU-A1 (1 indicates extract AMSU-A1 data)
$4 = 0 or 1 for AMSU-A2 (1 indicates extract AMSU-A2 data)
$5 = 0 or 1 for AMSU-B/MHS (1 indicates extract AMSU-B/MHS data)
$6 = 0 or 1 for HIRS/2 (1 indicates extract HIRS/2 data)
$7 = 0 or 1 for MSU (1 indicates extract MSU data)
$8 = 0 or 1 for DCS (1 indicates extract DCS data)
$9 = 0 or 1 for SEM (1 indicates extract SEM data)
$10= 0 or 1 for SBUV (1 indicates extract SBUV data)
$11= 0 or 1 for SAR (1 indicates extract SAR data)
$12= 0 or 1 for AVHRR (1 indicates extract AVHRR data)
$lu1 is the logical unit of the log file
$lu2 is the logical unit of the HIRS/3 or HIRS/4.l1a output file
$lu3 is the logical unit of the AMSU-A1.l1a output file
$lu4 is the logical unit of the AMSU-A2.l1a output file
$lu5 is the logical unit of the AMSU-B.l1a output file
$lu6 is the logical unit of the HIRS/2.l1a output file
$lu7 is the logical unit of the MSU.l1a output file
$lu8 is the logical unit of the DCS.l1a output file
$lu9 is the logical unit of the SEM.l1a output file
$lu10 is the logical unit of the SBUV.l1a output file
$lu11 is the logical unit of the SAR.l1a output file
$lu12 is the logical unit of the HRPT.l1a output file
"""
# A list of accepted return codes for the various scripts/binaries run in this function
accepted_return_codes_decom_hrpt = [0]
accepted_return_codes_chk1btime = [0]
accepted_return_codes_decom_amsua_metop = [0]
accepted_return_codes_decom_amsub_metop = [0]
accepted_return_codes_decom_hirs_metop = [0]
accepted_return_codes_decom_avhrr_metop = [0]
return_status = True
# This function relays on beeing in a working directory
current_dir = os.getcwd() # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]['working_dir'])
# for sensor in sensors:
# if str(sensor) in "amsu-a":
# process_config['process_amsua'] = True
# elif str(sensor) in ("amsu-b", "mhs"):
# process_config['process_amsub'] = True
# elif str(sensor) in ("hirs/3","hirs/4"):
# process_config['process_hirs'] = True
# elif str(sensor) in "avhrr/3":
# process_config['process_avhrr'] = True
# elif str(sensor) in "msu":
# process_config['process_msu'] = True
# elif str(sensor) in "dcs":
# process_config['process_dcs'] = True
# else:
# LOG.warning("Instrument/sensor {} not recognised.".format(sensor))
if 'noaa' in process_config['platform_name']:
LOG.debug("Do the commutation for NOAA")
# a_tovs = "ATOVS"
# PROCESS IN THIS ORDER
# ama = 0 # AMSU-A
# amb = 0 # AMSU-B
# hrs = 0 # HIRS
# avh = 0 # avhrr
# msu = 0
# dcs = 0
try:
decom_file = "decommutation.par"
# Needs to find platform number for A/TOVS
decom = open(decom_file, 'w')
if int(process_config['platform_name'][4:6]) <= 14:
del process_config['a_tovs'][0]
decom.write("1,0,0,0,0,{},{},{},0,0,0,{} ! OPTION NUMBERS\n".format(1 if process_config['process_hirs'] else 0,
1 if process_config['process_msu'] else 0,
1 if process_config['process_dcs'] else 0,
1 if process_config['process_avhrr'] else 0))
decom.write("10,0,0,0,0,11,12,13,0,0,0,14 ! STREAM NO.S\n")
else:
decom.write("1,{},{},{},{},0,0,{},0,0,0,{} ! OPTION NUMBERS\n".format(1 if process_config['process_hirs'] else 0,
1 if process_config['process_amsua'] else 0,
1 if process_config['process_amsua'] else 0,
1 if process_config['process_amsub'] else 0,
1 if process_config['process_dcs'] else 0,
1 if process_config['process_avhrr'] else 0))
decom.write("10,11,15,15,16,0,0,13,0,0,0,14 ! STREAM NO.S\n")
decom.write("{:%Y} ! year of the data\n".format(timestamp))
decom.write("0 ! operational mode\n")
decom.write("{0:05d},{0:05d} ! start and end orbit numbers\n".format(int(process_config['orbit_number'])))
decom.close()
except Exception as err:
LOG.error("Building decommutation command string failed: {}".format(err))
return False
# Read decommitation input into variable
decom = open(decom_file, 'r')
decom_input = decom.read()
decom.close()
os.environ['FILE_COEF'] = os.path.join(os.environ.get('PAR_CALIBRATION_COEF'), 'amsua', 'amsua_clparams.dat')
os.symlink(os.environ['FILE_COEF'], "{}50".format(os.environ['FORT']))
cmd = "decommutation.exe" # .format("".join(process_config['a_tovs']),decom_file, process_config['input_hrpt_file'])
try:
status, returncode, std, err = run_shell_command(cmd, stdin="{}\n{}{}\n".format(process_config['input_hrpt_file'], decom_input, os.getenv('STATION_ID', 'ST')),
stdout_logfile='decommutation.log',
stderr_logfile='decommutation.log')
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_hrpt:
LOG.info("Decommutation command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
# Need to check the decommutation output and rename fort files
if return_status:
# hrpt 14
if os.path.exists("{}14".format(os.environ['FORT'])):
shutil.move("{}14".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'])
cmd = "chk1btime.exe"
try:
status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['avhrr_file']))
except:
LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
else:
if returncode in accepted_return_codes_chk1btime:
LOG.debug("chk1btime command {} ok.".format(cmd))
LOG.debug("std: {}".format(std))
else:
LOG.error("This means that the start of the data are bad,"
" and that the processing for this data later will fail.")
LOG.debug("Return code: {}".format(returncode))
LOG.debug("std: {}".format(std))
LOG.debug("err: {}".format(err))
LOG.debug("status: {}".format(status))
process_config['process_avhrr'] = False
elif process_config['process_avhrr']:
LOG.warning("Fort file for avhrr does not exist after decommutation. Skip processing this.")
process_config['process_avhrr'] = False
# hrsn 11
if os.path.exists("{}11".format(os.environ['FORT'])):
shutil.move("{}11".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['hirs_file'])
cmd = "chk1btime.exe"
try:
status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['hirs_file']))
except:
LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
else:
if returncode in accepted_return_codes_chk1btime:
LOG.debug("chk1btime command {} ok.".format(cmd))
else:
LOG.error("This means that the start of the data are bad,"
" and that the processing for this data later will fail.")
LOG.debug("Return code: {}".format(returncode))
LOG.debug("std: {}".format(std))
LOG.debug("err: {}".format(err))
LOG.debug("status: {}".format(status))
process_config['process_hirs'] = False
elif process_config['process_hirs']:
LOG.warning("Fort file for hirs does not exist after decommutation. Skip processing this.")
process_config['process_hirs'] = False
# msun 12
if os.path.exists("{}12".format(os.environ['FORT'])):
shutil.move("{}12".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['msu_file'])
cmd = "chk1btime.exe"
try:
status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['msu_file']))
except:
LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
else:
if returncode in accepted_return_codes_chk1btime:
LOG.debug("chk1btime command {} ok.".format(cmd))
else:
LOG.error("This means that the start of the data are bad, "
" and that the processing for this data later will fail.")
LOG.debug("Return code: {}".format(returncode))
LOG.debug("std: {}".format(std))
LOG.debug("err: {}".format(err))
LOG.debug("status: {}".format(status))
process_config['process_msu'] = False
elif process_config['process_msu']:
LOG.warning("Fort file for msu does not exist after decommutation. Skip processing this.")
process_config['process_msu'] = False
# dcsn 13
if os.path.exists("{}13".format(os.environ['FORT'])):
shutil.move("{}13".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['dcs_file'])
cmd = "chk1btime.exe"
try:
status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['dcs_file']))
except:
LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
else:
if returncode in accepted_return_codes_chk1btime:
LOG.debug("chk1btime command {} ok.".format(cmd))
else:
LOG.error("This means that the start of the data are bad,"
" and that the processing for this data later will fail.")
LOG.debug("Return code: {}".format(returncode))
LOG.debug("std: {}".format(std))
LOG.debug("err: {}".format(err))
LOG.debug("status: {}".format(status))
process_config['process_dcs'] = False
elif process_config['process_dcs']:
LOG.warning("Fort file for dcs does not exist after decommutation. Skip processing this.")
process_config['process_dcs'] = False
# aman 15
if os.path.exists("{}15".format(os.environ['FORT'])):
shutil.move("{}15".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['amsua_file'])
cmd = "chk1btime.exe"
try:
status, returncode, std, err = run_shell_command(cmd,
stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['amsua_file']))
except:
LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
else:
if returncode in accepted_return_codes_chk1btime:
LOG.debug("chk1btime command {} ok.".format(cmd))
else:
LOG.error("This means that the start of the data are bad, "
"and that the processing for this data later will fail?")
LOG.debug("Return code: {}".format(returncode))
LOG.debug("std: {}".format(std))
LOG.debug("err: {}".format(err))
LOG.debug("status: {}".format(status))
process_config['process_amsua'] = False
elif process_config['process_amsua']:
LOG.warning("Fort file for amsu-a does not exist after decommutation. Skip processing this.")
process_config['process_amsua'] = False
# ambn 16
if os.path.exists("{}16".format(os.environ['FORT'])):
shutil.move("{}16".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['amsub_file'])
cmd = "chk1btime.exe"
try:
status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['amsub_file']))
except:
LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
else:
if returncode in accepted_return_codes_chk1btime:
LOG.debug("chk1btime command {} ok.".format(cmd))
else:
LOG.error("This means that the start of the data are bad, "
"and that the processing for this data later will fail.")
LOG.debug("Return code: {}".format(returncode))
LOG.debug("std: {}".format(std))
LOG.debug("err: {}".format(err))
LOG.debug("status: {}".format(status))
process_config['process_amsub'] = False
elif process_config['process_amsub']:
LOG.warning("Fort file for amsu-b does not exist after decommutation. Skip processing this.")
process_config['process_amsub'] = False
if os.path.exists("decommutation.log"):
dd = None
tt = None
with open("decommutation.log") as decomlog:
while True:
line = decomlog.readline()
if not line:
break
p_date = re.compile(".*avhrr\send\sdata\sday\s([0-9]{2})/([0-9]{2})/([0-9]{2})")
p_time = re.compile(".*avhrr\send\sdata\stime\s([0-9]{2}):([0-9]{2}):([0-9]{2})\.([0-9]{0,3})")
se = p_date.search(line)
if se:
dd = datetime.date(year=(int(se.group(3)) + 2000), month=int(se.group(2)), day=int(se.group(1)))
te = p_time.search(line)
if te:
tt = datetime.time(hour=int(te.group(1)),
minute=int(te.group(2)),
second=int(te.group(3)),
microsecond=int(te.group(4)) * 1000)
if tt and dd:
et = datetime.datetime.combine(dd, tt)
if abs((et - process_config['endtime']).total_seconds()) < 20 * 60:
LOG.debug("Adjusting the endtime with data from decommutaion.log.")
LOG.debug("Before {}".format(process_config['endtime']))
process_config['endtime'] = et
LOG.debug("After {}".format(process_config['endtime']))
break
elif 'METOP' in process_config['platform_name'].upper():
LOG.info("Do the metop decommutation")
if process_config['process_amsua']:
cmd = "decom-amsua-metop {} {} ".format(process_config['input_amsua_file'],
process_config['aapp_static_configuration']['decommutation_files']['amsua_file'])
try:
status, returncode, std, err = run_shell_command(cmd, stdout_logfile="decom-amsua-metop.log")
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_amsua_metop:
LOG.info("Command {} complete.".format(cmd))
if not os.path.exists(process_config['aapp_static_configuration']['decommutation_files']['amsua_file']):
LOG.warning("Decom gave OK status, but no data is produced. Something is wrong")
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
if process_config['process_mhs'] and return_status:
cmd = "decom-mhs-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
process_config['input_mhs_file'],
process_config['aapp_static_configuration']['decommutation_files']['mhs_file'])
try:
status, returncode, std, err = run_shell_command(cmd, stdout_logfile="decom-mhs-metop.log")
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_amsub_metop:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
if process_config['process_hirs'] and return_status:
cmd = "decom-hirs-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
process_config['input_hirs_file'],
process_config['aapp_static_configuration']['decommutation_files']['hirs_file'])
try:
status, returncode, std, err = run_shell_command(cmd, stdout_logfile="decom-hirs-metop.log")
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_hirs_metop:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
if process_config['process_avhrr'] and return_status:
cmd = "decom-avhrr-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
process_config['input_avhrr_file'],
process_config['aapp_static_configuration']
['decommutation_files']['avhrr_file'])
try:
status, returncode, std, err = run_shell_command(cmd, stdout_logfile="decom-avhrr-metop.log")
except:
LOG.error("Command {} failed.".format(cmd))
else:
if returncode in accepted_return_codes_decom_avhrr_metop:
LOG.info("Command {} complete.".format(cmd))
else:
LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
return_status = False
else:
LOG.error("Unknown platform: {}".format(process_config['platform_name']))
return_status = False
# print os.listdir("./")
# Change back after this is done
os.chdir(current_dir)
LOG.info("All decommutations complete.")
return return_status
def do_hirs_calibration(process_config, msg, timestamp):
return_status = True
if not process_config['process_hirs']:
return True
# A list of accepted return codes for the various scripts/binaries run in this function
accepted_return_codes_hirs_historic_file_manage = [0]
# This function relays on beeing in a working directory
current_dir = os.getcwd() # Store the dir to change back to after function complete
os.chdir(process_config['aapp_processes'][process_config.process_name]['working_dir'])
hirs_version_use = None
hirs_version = os.getenv('HIRSCL_VERSION', 0)
hirs_version_list = hirs_version.split()
hirs_sats = os.getenv('HIRSCL_SAT', 'default')
hirs_sat_list = hirs_sats.split()
index = 0
for sat in hirs_sat_list:
if sat in process_config['platform_name']:
hirs_version_use = hirs_version_list[index]
else:
hirs_version_def = hirs_version_list[index]
index += 1
if hirs_version_use is None:
hirs_version_use = hirs_version_def
hirs_script = "hirscl"
hirs_err_file = "hirscl.err"
calibration_location = process_config['calibration_location']
LOG.debug("hirs_version_use {}".format(hirs_version_use))
# pdb.set_trace()
if int(hirs_version_use) > 1: # no calibration, just navigation
calibration_location = "-l"
elif int(hirs_version_use) == 0 or "".join(process_config['a_tovs']) == 'TOVS':
calibration_location = "-c -l"
elif int(hirs_version_use) == 1:
file_historic = os.path.join(os.getenv('PAR_CALIBRATION_MONITOR'),
process_config['platform_name'], "hirs_historic.txt")
if os.path.exists(file_historic):
cmd = "hirs_historic_file_manage -m {} -r {} -n {} {}".format(os.getenv('HIST_SIZE_HIGH'),
os.getenv('HIST_SIZE_LOW'),
os.getenv('HIST_NMAX'),
file_historic)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
return_status = False
else:
if returncode in accepted_return_codes_hirs_historic_file_manage:
LOG.debug("Command complete.")
else:
LOG.error("Command {} failed with returncode {}".format(cmd, returncode))
LOG.error("stdout was: {}".format(std))
LOG.error("stderr was: {}".format(err))
return_status = False
if return_status:
cmd = "hcalcb1_algoV4 -s {0} -y {1:%Y} -m {1:%m} -d {1:%d} -h {1:%H} -n {1:%M}".format(process_config['platform_name'], timestamp)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
import sys
LOG.error("Command {} failed with {}.".format(cmd, sys.exc_info()[0]))
else:
if returncode != 0:
LOG.error("Command {} failed with {}".format(cmd, returncode))
_hirs_file = open(hirs_err_file, "w")
_hirs_file.write(std)
_hirs_file.write(err)
_hirs_file.close()
return_status = False
hirs_script = "hirscl_algoV4"
hirs_err_file = "hirscl_algoV4.err"
calibration_location = "-c -l"
else:
LOG.error("Can not figure out which hirs calibration algo version to use.")
return_status = False
if return_status:
try:
cmd = "{} {} -s {} -d {:%Y%m%d} -h {:%H%M} -n {:05d} {}".format(hirs_script,
calibration_location,
process_config['platform_name'],
timestamp,
timestamp,
int(process_config['orbit_number']),
process_config['aapp_static_configuration']
['decommutation_files']['hirs_file'])
except KeyError as ke:
LOG.error("Building command string failed with key error: {}".format(ke))
try:
status, returncode, out, err = run_shell_command(cmd, stdout_logfile="{}.log".format(hirs_script),
stderr_logfile="{}".format(hirs_err_file))
except:
import sys
LOG.error("Command {} failed {}.".format(cmd, sys.exc_info()[0]))
else:
if (returncode != 0):
LOG.error("Command {} failed with {}".format(cmd, returncode))
_hirs_file = open(hirs_err_file, "w")
_hirs_file.write(out)
_hirs_file.write(err)
_hirs_file.close()
return_status = False
# Change back after this is done
os.chdir(current_dir)
LOG.info("do_hirs_calibration complete!")
return return_status
def do_hirs_calibration(process_config, timestamp):
return_status = True
#A list of accepted return codes for the various scripts/binaries run in this function
accepted_return_codes_hirs_historic_file_manage = [0]
#This function relays on beeing in a working directory
current_dir = os.getcwd(
) #Store the dir to change back to after function complete
os.chdir(process_config['working_directory'])
hirs_version_use = None
hirs_version = os.getenv('HIRSCL_VERSION', 0)
hirs_version_list = hirs_version.split()
hirs_sats = os.getenv('HIRSCL_SAT', 'default')
hirs_sat_list = hirs_sats.split()
index = 0
for sat in hirs_sat_list:
if sat in process_config['platform']:
hirs_version_use = hirs_version_list[index]
else:
hirs_version_def = hirs_version_list[index]
index += 1
if hirs_version_use == None:
hirs_version_use = hirs_version_def
hirs_script = "hirscl"
hirs_err_file = "hirscl.err"
calibration_location = process_config['calibration_location']
print "hirs_version_use {}".format(hirs_version_use)
#pdb.set_trace()
if int(hirs_version_use) > 1: # no calibration, just navigation
calibration_location = "-l"
elif int(hirs_version_use) == 0 or "".join(
process_config['a_tovs']) == 'TOVS':
calibration_location = "-c -l"
elif int(hirs_version_use) == 1:
file_historic = os.path.join(os.getenv('PAR_CALIBRATION_MONITOR'),
process_config['platform'],
"hirs_historic.txt")
if os.path.exists(file_historic):
cmd = "hirs_historic_file_manage -m {} -r {} -n {} {}".format(
os.getenv('HIST_SIZE_HIGH'), os.getenv('HIST_SIZE_LOW'),
os.getenv('HIST_NMAX'), file_historic)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
LOG.error("Command {} failed.".format(cmd))
return_status = False
else:
if returncode in accepted_return_codes_hirs_historic_file_manage:
LOG.debug("Command complete.")
else:
LOG.error("Command {} failed with returncode {}".format(
cmd, returncode))
LOG.error("stdout was: {}".format(std))
LOG.error("stderr was: {}".format(err))
return_status = False
if return_status:
cmd = "hcalcb1_algoV4 -s {0} -y {1:%Y} -m {1:%m} -d {1:%d} -h {1:%H} -n {1:%M}".format(
process_config['platform'], timestamp)
try:
status, returncode, std, err = run_shell_command(cmd)
except:
import sys
LOG.error("Command {} failed with {}.".format(
cmd,
sys.exc_info()[0]))
else:
if returncode != 0:
LOG.error("Command {} failed with {}".format(
cmd, returncode))
_hirs_file = open(hirs_err_file, "w")
_hirs_file.write(std)
_hirs_file.write(err)
_hirs_file.close()
return_status = False
hirs_script = "hirscl_algoV4"
hirs_err_file = "hirscl_algoV4.err"
calibration_location = "-c -l"
else:
LOG.error(
"Can not figure out which hirs calibration algo version to use.")
return_status = False
if return_status:
#Default AAPP config for PAR_NAVIGATION_DEFAULT_LISTESAT Metop platform is M01, M02, M04
#but needed names are metop01 etc. Replace this inside the processing from now on.
aapp_satellite_list = os.getenv(
'PAR_NAVIGATION_DEFAULT_LISTESAT').split()
if process_config['platform'] not in aapp_satellite_list:
LOG.warning(
"Can not find this platform in AAPP config variable PAR_NAVIGATION_DEFAULT_LISTESAT. Will try to find matches. But it can be a good idea to change this variable in the ATOVS_ENV7 file."
)
LOG.warning("Platform {} not in list: {}".format(
process_config['platform'], aapp_satellite_list))
if 'metop' in process_config['platform'] and (
('M01' or 'M02' or 'M03' or 'M04') in aapp_satellite_list):
LOG.info("Replace in this processing")
PAR_NAVIGATION_DEFAULT_LISTESAT = os.getenv(
'PAR_NAVIGATION_DEFAULT_LISTESAT')
PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
'M01', 'metop01')
PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
'M02', 'metop02')
PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
'M03', 'metop03')
PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
'M04', 'metop04')
os.environ[
'PAR_NAVIGATION_DEFAULT_LISTESAT'] = PAR_NAVIGATION_DEFAULT_LISTESAT
cmd = "{} {} -s {} -d {:%Y%m%d} -h {:%H%M} -n {:05d} {}".format(
hirs_script, calibration_location, process_config['platform'],
timestamp, timestamp, process_config['orbit_number'],
process_config['hirs_file'])
try:
status, returncode, out, err = run_shell_command(
cmd,
stdout_logfile="{}.log".format(hirs_script),
stderr_logfile="{}".format(hirs_err_file))
except:
import sys
LOG.error("Command {} failed {}.".format(cmd, sys.exc_info()[0]))
else:
if (returncode != 0):
LOG.error("Command {} failed with {}".format(cmd, returncode))
_hirs_file = open(hirs_err_file, "w")
_hirs_file.write(out)
_hirs_file.write(err)
_hirs_file.close()
return_status = False
#Change back after this is done
os.chdir(current_dir)
LOG.info("do_hirs_calibration complete!")
return return_status