コード例 #1
0
def _convert_to_yuv_and_delete(output_directory, file_name, pattern):
    """Converts a PNG file to a YUV file and deletes the PNG file.

  Args:
    output_directory(string): The output directory for the YUV file.
    file_name(string): The PNG file name.
    pattern(string): The file pattern of the PNG/YUV file. The PNG/YUV files are
      named patternxx..x.png/yuv, where xx..x are digits starting from 00..0.
  Return:
    (bool): True upon successful conversion, false otherwise.
  """
    # Pattern should be in file name
    if not pattern in file_name:
        return False
    pattern_position = file_name.rfind(pattern)

    # Strip the path to the PNG file and replace the png extension with yuv
    yuv_file_name = file_name[pattern_position:-3] + 'yuv'
    yuv_file_name = os.path.join(output_directory, yuv_file_name)

    command = [
        'ffmpeg', '-i',
        '%s' % (file_name), '-pix_fmt', 'yuv420p',
        '%s' % (yuv_file_name)
    ]
    try:
        helper_functions.run_shell_command(
            command,
            fail_msg=('Error during PNG to YUV conversion of %s' % file_name))
        os.remove(file_name)
    except helper_functions.HelperError as err:
        print >> sys.stderr, err
        return False
    return True
コード例 #2
0
def _convert_to_yuv_and_delete(output_directory, file_name, pattern):
  """Converts a PNG file to a YUV file and deletes the PNG file.

  Args:
    output_directory(string): The output directory for the YUV file.
    file_name(string): The PNG file name.
    pattern(string): The file pattern of the PNG/YUV file. The PNG/YUV files are
      named patternxx..x.png/yuv, where xx..x are digits starting from 00..0.
  Return:
    (bool): True upon successful conversion, false otherwise.
  """
  # Pattern should be in file name
  if not pattern in file_name:
    return False
  pattern_position = file_name.rfind(pattern)

  # Strip the path to the PNG file and replace the png extension with yuv
  yuv_file_name = file_name[pattern_position:-3] + 'yuv'
  yuv_file_name = os.path.join(output_directory, yuv_file_name)

  command = ['ffmpeg', '-i', '%s' % (file_name), '-pix_fmt', 'yuv420p',
             '%s' % (yuv_file_name)]
  try:
    helper_functions.run_shell_command(
        command, fail_msg=('Error during PNG to YUV conversion of %s' %
                           file_name))
    os.remove(file_name)
  except helper_functions.HelperError as err:
    print >> sys.stderr, err
    return False
  return True
コード例 #3
0
def convert_yuv_to_png_files(yuv_file_name, yuv_frame_width, yuv_frame_height,
                             output_directory = '.'):
  """Converts a YUV video file into PNG frames.

  The function uses ffmpeg to convert the YUV file. The output of ffmpeg is in
  the form frame_xxxx.png, where xxxx is the frame number, starting from 0001.

  Args:
    yuv_file_name(string): The name of the YUV file.
    yuv_frame_width(int): The width of one YUV frame.
    yuv_frame_height(int): The height of one YUV frame.
    output_directory(string): The output directory where the PNG frames will be
      stored.

  Return:
    (bool): True if the conversion was OK.
  """
  size_string = str(yuv_frame_width) + 'x' + str(yuv_frame_height)
  output_files_pattern = os.path.join(output_directory, 'frame_%04d.png')
  command = ['ffmpeg', '-s', '%s' % size_string, '-i', '%s'
             % yuv_file_name, '-f', 'image2', '-vcodec', 'png',
             '%s' % output_files_pattern]
  try:
    helper_functions.run_shell_command(
        command, msg='Error during YUV to PNG conversion')
  except helper_functions.HelperError, err:
    print err
    return False
コード例 #4
0
def convert_yuv_to_png_files(yuv_file_name, yuv_frame_width, yuv_frame_height,
                             output_directory, ffmpeg_dir=None):
  """Converts a YUV video file into PNG frames.

  The function uses ffmpeg to convert the YUV file. The output of ffmpeg is in
  the form frame_xxxx.png, where xxxx is the frame number, starting from 0001.

  Args:
    yuv_file_name(string): The name of the YUV file.
    yuv_frame_width(int): The width of one YUV frame.
    yuv_frame_height(int): The height of one YUV frame.
    output_directory(string): The output directory where the PNG frames will be
      stored.
    ffmpeg_dir(string): The directory containing the ffmpeg executable. If
      omitted, the PATH will be searched for it.

  Return:
    (bool): True if the conversion was OK.
  """
  size_string = str(yuv_frame_width) + 'x' + str(yuv_frame_height)
  output_files_pattern = os.path.join(output_directory, 'frame_%04d.png')
  ffmpeg_executable = 'ffmpeg.exe' if sys.platform == 'win32' else 'ffmpeg'
  if ffmpeg_dir:
    ffmpeg_executable = os.path.join(ffmpeg_dir, ffmpeg_executable)
  command = [ffmpeg_executable, '-s', '%s' % size_string, '-i', '%s'
             % yuv_file_name, '-f', 'image2', '-vcodec', 'png',
             '%s' % output_files_pattern]
  try:
    print 'Converting YUV file to PNG images (may take a while)...'
    print ' '.join(command)
    helper_functions.run_shell_command(
        command, fail_msg='Error during YUV to PNG conversion')
  except helper_functions.HelperError, err:
    print 'Error executing command: %s. Error: %s' % (command, err)
    return False
コード例 #5
0
def convert_yuv_to_png_files(yuv_file_name,
                             yuv_frame_width,
                             yuv_frame_height,
                             output_directory='.'):
    """Converts a YUV video file into PNG frames.

  The function uses ffmpeg to convert the YUV file. The output of ffmpeg is in
  the form frame_xxxx.png, where xxxx is the frame number, starting from 0001.

  Args:
    yuv_file_name(string): The name of the YUV file.
    yuv_frame_width(int): The width of one YUV frame.
    yuv_frame_height(int): The height of one YUV frame.
    output_directory(string): The output directory where the PNG frames will be
      stored.

  Return:
    (bool): True if the conversion was OK.
  """
    size_string = str(yuv_frame_width) + 'x' + str(yuv_frame_height)
    output_files_pattern = os.path.join(output_directory, 'frame_%04d.png')
    command = [
        'ffmpeg', '-s',
        '%s' % size_string, '-i',
        '%s' % yuv_file_name, '-f', 'image2', '-vcodec', 'png',
        '%s' % output_files_pattern
    ]
    try:
        helper_functions.run_shell_command(
            command, msg='Error during YUV to PNG conversion')
    except helper_functions.HelperError, err:
        print err
        return False
コード例 #6
0
def generate_upca_barcodes(number_of_barcodes,
                           barcode_width,
                           barcode_height,
                           output_directory='.',
                           path_to_zxing='zxing-read-only'):
    """Generates UPC-A barcodes.

  This function generates a number_of_barcodes UPC-A barcodes. The function
  calls an example Java encoder from the Zxing library. The barcodes are
  generated as PNG images. The width of the barcodes shouldn't be less than 102
  pixels because otherwise Zxing can't properly generate the barcodes.

  Args:
    number_of_barcodes(int): The number of barcodes to generate.
    barcode_width(int): Width of barcode in pixels.
    barcode_height(int): Height of barcode in pixels.
    output_directory(string): Output directory where to store generated
      barcodes.
    path_to_zxing(string): The path to Zxing.

  Return:
    (bool): True if the conversion is successful.
  """
    base_file_name = os.path.join(output_directory, "barcode_")
    jars = _form_jars_string(path_to_zxing)
    command_line_encoder = 'com.google.zxing.client.j2se.CommandLineEncoder'
    barcode_width = str(barcode_width)
    barcode_height = str(barcode_height)

    errors = False
    for i in range(number_of_barcodes):
        suffix = helper_functions.zero_pad(i)
        # Barcodes starting from 0
        content = helper_functions.zero_pad(i, 11)
        output_file_name = base_file_name + suffix + ".png"

        command = [
            "java", "-cp", jars, command_line_encoder,
            "--barcode_format=UPC_A",
            "--height=%s" % barcode_height,
            "--width=%s" % barcode_width,
            "--output=%s" % (output_file_name),
            "%s" % (content)
        ]
        try:
            helper_functions.run_shell_command(
                command,
                fail_msg=('Error during barcode %s generation' % content))
        except helper_functions.HelperError as err:
            print >> sys.stderr, err
            errors = True
    return not errors
コード例 #7
0
def _decode_barcode_in_file(file_name, barcode_width, barcode_height, jars,
                            command_line_decoder):
    """Decodes the barcode in the upper left corner of a PNG file.

  Args:
    file_name(string): File name of the PNG file.
    barcode_width(int): Width of the barcode (in pixels).
    barcode_height(int): Height of the barcode (in pixels)
    jars(string): The Zxing core and javase string.
    command_line_decoder(string): The ZXing command-line decoding tool.

  Return:
    (bool): True upon success, False otherwise.
  """
    command = [
        'java', '-Djava.awt.headless=true', '-cp',
        '%s' % jars,
        '%s' % command_line_decoder, '--products_only', '--dump_results',
        '--brief',
        '--crop=%d,%d,%d,%d' % (0, 0, barcode_width, barcode_height),
        '%s' % file_name
    ]
    try:
        out = helper_functions.run_shell_command(
            command, msg='Error during decoding of %s' % file_name)
        if not 'Success' in out:
            sys.stderr.write('Barcode in %s cannot be decoded\n' % file_name)
            return False
    except helper_functions.HelperError, err:
        print err
        return False
コード例 #8
0
def _decode_barcode_in_file(file_name, barcode_width, barcode_height, jars,
                            command_line_decoder):
  """Decodes the barcode in the upper left corner of a PNG file.

  Args:
    file_name(string): File name of the PNG file.
    barcode_width(int): Width of the barcode (in pixels).
    barcode_height(int): Height of the barcode (in pixels)
    jars(string): The Zxing core and javase string.
    command_line_decoder(string): The ZXing command-line decoding tool.

  Return:
    (bool): True upon success, False otherwise.
  """
  command = ['java', '-Djava.awt.headless=true', '-cp', '%s' % jars,
             '%s' % command_line_decoder, '--products_only',
             '--dump_results', '--brief', '--crop=%d,%d,%d,%d' %
             (0, 0, barcode_width, barcode_height),
             '%s' % file_name]
  try:
    out = helper_functions.run_shell_command(
        command, msg='Error during decoding of %s' % file_name)
    if not 'Success' in out:
      sys.stderr.write('Barcode in %s cannot be decoded\n' % file_name)
      return False
  except helper_functions.HelperError, err:
    print err
    return False
コード例 #9
0
def do_avhrr_calibration(process_config, timestamp):

    accepted_return_codes_avhrcl = [0]

    return_value = True
    LOG.debug("Do the avhrr calibration")        

    #This function relays on beeing in a working directory
    current_dir = os.getcwd() #Store the dir to change back to after function complete
    os.chdir(process_config['working_directory'])

    #calibration_location = "-c -l"

    cmd = "avhrcl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(process_config['calibration_location'],
                                                                              process_config['platform'],
                                                                              timestamp,
                                                                              process_config['orbit_number'],
                                                                              process_config['avhrr_file'])
    try:
        status, returncode, std, err = run_shell_command(cmd)
    except:
        LOG.error("Command {} failed.".format(cmd))
    else:
        if returncode in accepted_return_codes_avhrcl:
            LOG.info("Command {} complete.".format(cmd))
        else:
            LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
            return_value = False


    #Change back after this is done
    os.chdir(current_dir)

    LOG.info("do_avhrr_calibration complete!")
    return return_value
コード例 #10
0
def generate_upca_barcodes(number_of_barcodes, barcode_width, barcode_height,
                           output_directory='.',
                           path_to_zxing='zxing-read-only'):
  """Generates UPC-A barcodes.

  This function generates a number_of_barcodes UPC-A barcodes. The function
  calls an example Java encoder from the Zxing library. The barcodes are
  generated as PNG images. The width of the barcodes shouldn't be less than 102
  pixels because otherwise Zxing can't properly generate the barcodes.

  Args:
    number_of_barcodes(int): The number of barcodes to generate.
    barcode_width(int): Width of barcode in pixels.
    barcode_height(int): Height of barcode in pixels.
    output_directory(string): Output directory where to store generated
      barcodes.
    path_to_zxing(string): The path to Zxing.

  Return:
    (bool): True if the conversion is successful.
  """
  base_file_name = os.path.join(output_directory, "barcode_")
  jars = _form_jars_string(path_to_zxing)
  command_line_encoder = 'com.google.zxing.client.j2se.CommandLineEncoder'
  barcode_width = str(barcode_width)
  barcode_height = str(barcode_height)

  errors = False
  for i in range(number_of_barcodes):
    suffix = helper_functions.zero_pad(i)
    # Barcodes starting from 0
    content = helper_functions.zero_pad(i, 11)
    output_file_name = base_file_name + suffix + ".png"

    command = ["java", "-cp", jars, command_line_encoder,
               "--barcode_format=UPC_A", "--height=%s" % barcode_height,
               "--width=%s" % barcode_width,
               "--output=%s" % (output_file_name), "%s" % (content)]
    try:
      helper_functions.run_shell_command(
          command, fail_msg=('Error during barcode %s generation' % content))
    except helper_functions.HelperError as err:
      print >> sys.stderr, err
      errors = True
  return not errors
コード例 #11
0
def convert_yuv_to_png_files(yuv_file_name,
                             yuv_frame_width,
                             yuv_frame_height,
                             output_directory='.'):
    """Converts a YUV video file into PNG frames.

  The function uses ffmpeg to convert the YUV file. The output of ffmpeg is in
  the form frame_xxxx.png, where xxxx is the frame number, starting from 0001.

  Args:
    yuv_file_name(string): The name of the YUV file.
    yuv_frame_width(int): The width of one YUV frame.
    yuv_frame_height(int): The height of one YUV frame.
    output_directory(string): The output directory where the PNG frames will be
      stored.

  Return:
    (bool): True if the conversion was OK.
  """
    size_string = str(yuv_frame_width) + 'x' + str(yuv_frame_height)
    output_files_pattern = os.path.join(output_directory, 'frame_%04d.png')
    ffmpeg_executable = 'ffmpeg'
    if sys.platform == 'win32':
        if os.getenv('FFMPEG_HOME'):
            ffmpeg_executable = os.path.join(os.getenv('FFMPEG_HOME'), 'bin',
                                             'ffmpeg.exe')
        else:
            ffmpeg_executable = 'ffmpeg.exe'
    command = [
        ffmpeg_executable, '-s',
        '%s' % size_string, '-i',
        '%s' % yuv_file_name, '-f', 'image2', '-vcodec', 'png',
        '%s' % output_files_pattern
    ]
    try:
        helper_functions.run_shell_command(
            command, msg='Error during YUV to PNG conversion')
    except helper_functions.HelperError, err:
        print >> sys.stderr, 'Error executing command: %s. Error: %s' % (
            command, err)
        return False
コード例 #12
0
def do_tle_satpos(timestamp, satellite, satpos_dir=None):
    
    return_status = True
    
    LOG.info("satpos files is stored under the given directory/satpos")
    if satpos_dir == None:
        satpos_dir = os.path.join(os.environ['DIR_NAVIGATION'], "satpos" )
    else:
        satpos_dir = os.path.join(satpos_dir, "satpos" )
        
    if not os.path.exists(satpos_dir):
        try:
            os.makedirs(satpos_dir)
        except:
            LOG.error("Could not create satpos directory: {}".format(satpos_dir))
            return_status = False
        
    file_satpos = os.path.join(satpos_dir, "satpos_{}_{:%Y%m%d}.txt".format(satellite,timestamp))
    
    if (not os.path.exists(file_satpos) or os.stat(file_satpos).st_size == 0) and return_status:
        """Usage is: satpostle  [ -o] [-s satellite] [-S station] [-d start date] [-n number of days] [-i increment in seconds] [-c search criteria] 

        -o -s -S -d -n -i –c are optional. 

        If no parameter is specified as an option, defaults are : noaa14, Lannion, today 0h, 1.0, 120.0, n (n= nearest, p = preceding). 

        The option -o specifies that the data will be stored in the file satpos_noaxx_yyyymmdd.txt.
         
        Output default is the standard output.. 
        """
        cmd="satpostle -o -s {} -d {:%d/%m/%y} -n 1.2".format(satellite,timestamp)
        try:
            status, returncode, std, err = run_shell_command(cmd)
        except:
            LOG.error("Failed to run command: {}".format(cmd))
            return_status = False
        else:
            if returncode != 0:
                LOG.error("cmd: {} failed with returncode: {}".format(cmd,returncode))
                return_status = False
            elif not os.path.exists(file_satpos):
                LOG.error("file: {} does not exists after satpostle run.".format(file_satpos))
                return_status = False
    else:
        LOG.info("satpos file already there. Use this")
        
    return return_status
コード例 #13
0
def do_avhrr_calibration(process_config, msg, timestamp):

    if not process_config['process_avhrr']:
        LOG.debug("Skipping avhrr processing due to config or missing data.")
        return True

    accepted_return_codes_avhrcl = [0]

    return_value = True
    LOG.debug("Do the avhrr calibration")

    # This function relays on beeing in a working directory
    current_dir = os.getcwd(
    )  # Store the dir to change back to after function complete
    os.chdir(process_config['aapp_processes'][process_config.process_name]
             ['working_dir'])

    # calibration_location = "-c -l"

    try:
        cmd = "avhrcl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(
            process_config['calibration_location'],
            process_config['platform_name'], timestamp,
            int(process_config['orbit_number']),
            process_config['aapp_static_configuration']['decommutation_files']
            ['avhrr_file'])
    except Exception as err:
        LOG.error("Failed to build avhrcl command: {}".format(err))

    try:
        status, returncode, std, err = run_shell_command(cmd)
    except:
        LOG.error("Command {} failed.".format(cmd))
    else:
        if returncode in accepted_return_codes_avhrcl:
            LOG.info("Command {} complete.".format(cmd))
        else:
            LOG.error("Command {} failed with return code {}.".format(
                cmd, returncode))
            return_value = False

    # Change back after this is done
    os.chdir(current_dir)

    LOG.info("do_avhrr_calibration complete!")
    return return_value
コード例 #14
0
def _decode_barcode_in_file(file_name, command_line_decoder):
    """Decodes the barcode in the upper left corner of a PNG file.

  Args:
    file_name(string): File name of the PNG file.
    command_line_decoder(string): The ZXing command-line decoding tool.

  Return:
    (bool): True upon success, False otherwise.
  """
    command = [command_line_decoder, '--try-harder', '--dump-raw', file_name]
    try:
        out = helper_functions.run_shell_command(
            command, fail_msg='Error during decoding of %s' % file_name)
        text_file = open('%s.txt' % file_name[:-4], 'w')
        text_file.write(out)
        text_file.close()
    except helper_functions.HelperError, err:
        print 'Barcode in %s cannot be decoded.' % file_name
        print err
        return False
コード例 #15
0
ファイル: barcode_decoder.py プロジェクト: 0x0B501E7E/webrtc
def _decode_barcode_in_file(file_name, command_line_decoder):
  """Decodes the barcode in the upper left corner of a PNG file.

  Args:
    file_name(string): File name of the PNG file.
    command_line_decoder(string): The ZXing command-line decoding tool.

  Return:
    (bool): True upon success, False otherwise.
  """
  command = [command_line_decoder, '--try-harder', '--dump-raw', file_name]
  try:
    out = helper_functions.run_shell_command(
        command, fail_msg='Error during decoding of %s' % file_name)
    text_file = open('%s.txt' % file_name[:-4], 'w')
    text_file.write(out)
    text_file.close()
  except helper_functions.HelperError, err:
    print 'Barcode in %s cannot be decoded.' % file_name
    print err
    return False
コード例 #16
0
def _decode_barcode_in_file(file_name, barcode_width, barcode_height, jars,
                            command_line_decoder):
    """Decodes the barcode in the upper left corner of a PNG file.

  Args:
    file_name(string): File name of the PNG file.
    barcode_width(int): Width of the barcode (in pixels).
    barcode_height(int): Height of the barcode (in pixels)
    jars(string): The Zxing core and javase string.
    command_line_decoder(string): The ZXing command-line decoding tool.

  Return:
    (bool): True upon success, False otherwise.
  """
    java_executable = 'java'
    if sys.platform == 'win32':
        if os.getenv('JAVA_HOME'):
            java_executable = os.path.join(os.getenv('JAVA_HOME'), 'bin',
                                           'java.exe')
        else:
            java_executable = 'java.exe'
    command = [
        java_executable, '-Djava.awt.headless=true', '-cp',
        '%s' % jars,
        '%s' % command_line_decoder, '--products_only', '--dump_results',
        '--brief',
        '--crop=%d,%d,%d,%d' % (0, 0, barcode_width, barcode_height),
        '%s' % file_name
    ]
    try:
        out = helper_functions.run_shell_command(
            command, msg='Error during decoding of %s' % file_name)
        if not 'Success' in out:
            print >> sys.stderr, 'Barcode in %s cannot be decoded\n' % file_name
            return False
    except helper_functions.HelperError, err:
        print >> sys.stderr, err
        return False
コード例 #17
0
def do_atovpp_and_avh2hirs_processing(process_config, timestamp):
    LOG.debug("Do preprocessing atovpp and avh2hirs ... ")

    return_status = True
    
    #This function relays on beeing in a working directory
    current_dir = os.getcwd() #Store the dir to change back to after function complete
    os.chdir(process_config['working_directory'])

    instruments = "AMSU-A AMSU-B HIRS"
    if "".join(process_config['a_tovs']) == 'TOVS':
        instruments = "MSU HIRS"
    
    cmd = "atovin {}".format(instruments)
    try:
        status, returncode, std, err = run_shell_command(cmd)
    except:
        LOG.error("Command {} failed.".format(cmd))
    else:
        if returncode != 0:
            LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
            LOG.error(std)
            LOG.error(err)
            return_status = False
        else:
            LOG.info("Command {} complete.".format(cmd))
            LOG.debug(std)
            LOG.debug(err)

    if return_status:
        cmd = "atovpp -i \"{}\" -g HIRS ".format(instruments)
        try:
            status, returncode, std, err = run_shell_command(cmd)
        except:
            LOG.error("Command {} failed.".format(cmd))
        else:
            if returncode != 0:
                LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                LOG.error(std)
                LOG.error(err)
                return_status = False
            else:
                LOG.info("Command {} complete.".format(cmd))

    #THis will not work since the avh2hirs script sources the ATOVS_CONF at start
    #and relay on WRK dir, which is a static dir
    #
    #Skip for now
    if 0:
        cmd = "avh2hirs {}".format("".join(process_config['a_tovs']))
        try:
            status, returncode, std, err = run_shell_command(cmd)
        except:
            LOG.error("Command {} failed.".format(cmd))
        else:
            if returncode != 0:
                LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                LOG.error(std)
                LOG.error(err)
                return_status = False
            else:
                LOG.info("Command {} complete.".format(cmd))
    else:
        LOG.warning("avh2hirs is not implemented in the processing.")
        
    #Change back after this is done
    os.chdir(current_dir)

    LOG.info("atovpp and avh2hirs complete!")
    return return_status
コード例 #18
0
def do_atovs_calibration(process_config, timestamp):

    if(not process_config['process_amsua'] and
       not process_config['process_amsub'] and
       not process_config['process_msu']):
        LOG.debug("Skipping atovs processing")
        return True

    accepted_return_codes_msucl = [0]
    accepted_return_codes_amsuacl = [0]
    accepted_return_codes_amsubcl = [0]

    return_value = True

    # This function relays on beeing in a working directory
    current_dir = os.getcwd()  # Store the dir to change back to after function complete
    os.chdir(process_config['aapp_processes'][process_config.process_name]['working_dir'])

    # calibration_location = "-c -l"
    if "".join(process_config['a_tovs']) == 'TOVS':
        cmd = "msucl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(process_config['calibration_location'],
                                                                                 process_config['platform_name'],
                                                                                 timestamp,
                                                                                 int(process_config['orbit_number']),
                                                                                 process_config['aapp_static_configuration']['decommutation_files']['msun_file'])
        try:
            status, returncode, std, err = run_shell_command(cmd)
        except:
            LOG.error("Command {} failed.".format(cmd))
        else:
            if returncode in accepted_return_codes_msucl:
                LOG.info("Command {} complete.".format(cmd))
            else:
                LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                return_value = False

    elif "".join(process_config['a_tovs']) == 'ATOVS':
        if process_config['process_amsua']:
            cmd = "amsuacl {0} -s {1} -d {2:%Y%m%d} -h {2:%H%M} -n {3:05d} {4}".format(process_config['calibration_location'],
                                                                                       process_config['platform_name'],
                                                                                       timestamp,
                                                                                       int(process_config['orbit_number']),
                                                                                       process_config['aapp_static_configuration']['decommutation_files']['amsua_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd)
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_amsuacl:
                    LOG.info("Command {} complete.".format(cmd))
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_value = False

        if process_config['process_amsub'] and return_value:
            amsub_script = "mhscl"
            try:
                if 'noaa' in process_config['platform_name'] and int(process_config['platform_name'][-2:]) <= 17:
                    amsu_script = "amsubcl"
                    process_config['process_mhs'] = False
                else:
                    process_config['process_amsub'] = False

            except ValueError as ve:
                LOG.warning("Could not exctract satellite number from the last two characters in the platform name: "
                            "{} and convert it to a number.".format(process_config['platform_name']))

            cmd = "{0} {1} -s {2} -d {3:%Y%m%d} -h {3:%H%M} -n {4:05d} {5}".format(amsub_script,
                                                                                   process_config['calibration_location'],
                                                                                   process_config['platform_name'],
                                                                                   timestamp,
                                                                                   int(process_config['orbit_number']),
                                                                                   process_config['aapp_static_configuration']['decommutation_files']['amsub_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd)
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_amsubcl:
                    LOG.info("Command {} complete.".format(cmd))
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_value = False

    else:
        LOG.error("Unknown A|TOVS key string: {}".format("".join(process_config['a_tovs'])))
        return_value = False

    # Change back after this is done
    os.chdir(current_dir)

    LOG.info("do_atovs_calibration complete!")
    return return_value
コード例 #19
0
def do_atovpp_and_avh2hirs_processing(process_config, timestamp):
    if (not process_config['process_amsua']
            and not process_config['process_amsub']
            and not process_config['process_hirs']):
        LOG.debug("Skipping atovpp and avh2hirs processing.")
        return True

    LOG.debug("Do preprocessing atovpp and avh2hirs ... ")

    return_status = True

    # This function relays on beeing in a working directory
    current_dir = os.getcwd(
    )  # Store the dir to change back to after function complete
    os.chdir(process_config['aapp_processes'][process_config.process_name]
             ['working_dir'])

    instruments = "AMSU-A AMSU-B HIRS"
    grids = "AMSU-A AMSU-B HIRS"
    if not process_config['process_hirs']:
        instruments = "AMSU-A AMSU-B"
        grids = "AMSU-A AMSU-B"
    if "".join(process_config['a_tovs']) == 'TOVS':
        instruments = "MSU HIRS"
        grids = "MSU HIRS"

    if 'do_atovpp' in process_config['aapp_processes'][
            process_config.process_name]:
        process_config['do_atovpp'] = process_config['aapp_processes'][
            process_config.process_name]['do_atovpp']
    else:
        process_config['do_atovpp'] = False

    if 'do_avh2hirs' in process_config['aapp_processes'][
            process_config.process_name]:
        process_config['do_avh2hirs'] = process_config['aapp_processes'][
            process_config.process_name]['do_avh2hirs']
    else:
        process_config['do_avh2hirs'] = False

    if process_config['do_atovpp']:
        cmd = "atovin {}".format(instruments)
        try:
            status, returncode, std, err = run_shell_command(cmd)
        except:
            LOG.error("Command {} failed.".format(cmd))
        else:
            if returncode != 0:
                LOG.error("Command {} failed with return code {}.".format(
                    cmd, returncode))
                LOG.error(std)
                LOG.error(err)
                return_status = False
            else:
                LOG.info("Command {} complete.".format(cmd))
                LOG.debug(std)
                LOG.debug(err)

        if return_status:
            cmd = "atovpp -i \"{}\" -g \"{}\"".format(instruments, grids)
            try:
                status, returncode, std, err = run_shell_command(cmd)
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode != 0:
                    LOG.error("Command {} failed with return code {}.".format(
                        cmd, returncode))
                    LOG.error(std)
                    LOG.error(err)
                    return_status = False
                else:
                    LOG.info("Command {} complete.".format(cmd))

    if return_status and process_config['do_avh2hirs'] and process_config[
            'process_hirs']:
        if os.path.exists(
                "./{}".format(process_config['aapp_static_configuration']
                              ['decommutation_files']['avhrr_file'])):
            os.symlink(
                "./{}".format(process_config['aapp_static_configuration']
                              ['decommutation_files']['avhrr_file']),
                "{}11".format(os.environ["FORT"]))
        else:
            LOG.error("Could not find file: ./{}".format(
                process_config['aapp_static_configuration']
                ['decommutation_files']['avhrr_file']))
            return_status = False

        if os.path.exists("./hirs.l1d"):
            os.symlink("./hirs.l1d", "{}12".format(os.environ["FORT"]))
        else:
            LOG.error("Could not find file: {}".format("./hirs.l1d"))
            return_status = False

        if return_status:
            cmd = "l1didf -i hirs.l1d"
            try:
                status, returncode, std, err = run_shell_command(cmd)
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode != 0:
                    LOG.error("Command {} failed with return code {}.".format(
                        cmd, returncode))
                    LOG.error(std)
                    LOG.error(err)
                    return_status = False
                else:
                    LOG.info("Command {} complete.".format(cmd))
                    satimg, yyyymmdd, hhmn, orbit, instr, loc1, loc2 = std.split(
                    )

        if return_status:
            if "noaa" in satimg:
                satnb = int(satimg[4:6])
            else:
                satnb = int(satimg[5:7])
            LOG.debug("Using satnb: {}".format(satnb))
            unit = satnb + 50
            mmc = int(yyyymmdd[4:6])
            LOG.debug("Using unit: {} and mmc: {}".format(unit, mmc))

            if os.path.exists(
                    os.path.join(os.environ["DIR_PREPROC"],
                                 "cor_{}.dat".format(satimg))):
                os.symlink(
                    os.path.join(os.environ["DIR_PREPROC"],
                                 "cor_{}.dat".format(satimg)),
                    "{}{}".format(os.environ["FORT"], unit))
            else:
                LOG.error("Failed to find {}".format(
                    os.path.join(os.environ["DIR_PREPROC"],
                                 "cor_{}.dat".format(satimg))))
                return_status = False

        if return_status:
            os.environ["SATIMG"] = satimg
            os.environ["YYYYMMDD"] = yyyymmdd
            os.environ["HHMN"] = hhmn
            os.environ["DIR_MAIA2_ATLAS"] = os.path.join(
                os.environ["DIR_PREPROC"], "atlas")
            os.environ["DIR_MAIA2_THRESHOLDS"] = os.path.join(
                os.environ["DIR_PREPROC"], "thresholds")
            if "".join(process_config['a_tovs']) == 'TOVS':
                cmd = "maia2_env;maia2_environment;avh2hirs.exe 2>&1"
            else:
                cmd = "bash -c \"source liblog.sh;source maia2_env;maia2_environment\";avh2hirs_atovs.exe 2>&1"

            LOG.debug("Before running command: {}".format(cmd))
            try:
                status, returncode, std, err = run_shell_command(
                    cmd,
                    stdout_logfile="avh2hirs.log",
                    use_shlex=False,
                    use_shell=True)
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode != 0:
                    LOG.error("Command {} failed with return code {}.".format(
                        cmd, returncode))
                    LOG.error(std)
                    LOG.error(err)
                    return_status = False
                else:
                    LOG.info("Command {} complete.".format(cmd))
                    LOG.debug(std)
                    LOG.debug(err)

        if return_status:
            import glob
            for fortfile in glob.glob("./fort*"):
                os.remove(fortfile)
            os.remove("./albedo")
            os.remove("./sst")
            os.remove("./wv")

    # Change back after this is done
    os.chdir(current_dir)

    LOG.info("atovpp and avh2hirs complete!")
    return return_status
コード例 #20
0
def do_decommutation(process_config, sensors, timestamp, hrpt_file):
    """    
    decommutation ${A_TOVS}  decommutation.par  ${FILE} 
    
    The 3 arguments are obligatory. 
  
    ${A_TOVS} = TOVS for satellite number < or = 14
    ${A_TOVS} = ATOVS for satellite number > 14
  
    In the decommutation.par file, options are written in this order: 
    $1,$2,$3,$4,$5, $6, $7, $8,$9,$10,$11,$12 !OPTION NUMBERS 
    $lu1,$lu2,$lu3,$lu4,$lu5,$lu6,$lu7,$lu8,$lu9,$lu10,$lu11,$lu12   !STREAM NO.S 
    ${YEAR} ! year of the data 
    0  ! operational mode 
    ${NNNNN},${NNNNN} ! start and end orbit numbers
    
    $1 = 0 or 1 for level of error logging 
    $2 = 0 or 1 for HIRS/3 or HIRS/4 (1 indicates extract HIRS/3 or HIRS/4 data) 
    $3 = 0 or 1 for AMSU-A1 (1 indicates extract AMSU-A1 data) 
    $4 = 0 or 1 for AMSU-A2 (1 indicates extract AMSU-A2 data) 
    $5 = 0 or 1 for AMSU-B/MHS (1 indicates extract AMSU-B/MHS data) 
    $6 = 0 or 1 for HIRS/2 (1 indicates extract HIRS/2 data) 
    $7 = 0 or 1 for MSU (1 indicates extract MSU data) 
    $8 = 0 or 1 for DCS (1 indicates extract DCS data) 
    $9 = 0 or 1 for SEM (1 indicates extract SEM data) 
    $10= 0 or 1 for SBUV (1 indicates extract SBUV data) 
    $11= 0 or 1 for SAR (1 indicates extract SAR data) 
    $12= 0 or 1 for AVHRR (1 indicates extract AVHRR data) 
    $lu1 is the logical unit of the log file 
    $lu2 is the logical unit of the HIRS/3 or HIRS/4.l1a output file 
    $lu3 is the logical unit of the AMSU-A1.l1a output file 
    $lu4 is the logical unit of the AMSU-A2.l1a output file 
    $lu5 is the logical unit of the AMSU-B.l1a output file 
    $lu6 is the logical unit of the HIRS/2.l1a output file 
    $lu7 is the logical unit of the MSU.l1a output file
    $lu8 is the logical unit of the DCS.l1a output file
    $lu9 is the logical unit of the SEM.l1a output file
    $lu10 is the logical unit of the SBUV.l1a output file 
    $lu11 is the logical unit of the SAR.l1a output file 
    $lu12 is the logical unit of the HRPT.l1a output file 
    """

    #A list of accepted return codes for the various scripts/binaries run in this function
    accepted_return_codes_decom_hrpt = [0]
    accepted_return_codes_decom_amsua_metop = [0]
    accepted_return_codes_decom_amsub_metop = [0]
    accepted_return_codes_decom_hirs_metop  = [0]
    accepted_return_codes_decom_avhrr_metop = [0]
    
    return_status = True
    
    #This function relays on beeing in a working directory
    current_dir = os.getcwd() #Store the dir to change back to after function complete
    os.chdir(process_config['working_directory'])
    
    print "decom file: ", hrpt_file
    for sensor in sensors:
        if str(sensor) in "amsu-a":
            process_config['process_amsua'] = True
        elif str(sensor) in ("amsu-b", "mhs"):
            process_config['process_amsub'] = True
        elif str(sensor) in ("hirs/3","hirs/4"):
            process_config['process_hirs'] = True
        elif str(sensor) in "avhrr/3":
            process_config['process_avhrr'] = True
        elif str(sensor) in "msu":
            process_config['process_msu'] = True
        elif str(sensor) in "dcs":
            process_config['process_dcs'] = True
        else:
            LOG.warning("Instrument/sensor {} not recognised.".format(sensor))

    if 'noaa' in process_config['platform']:
        print "Do the commutaion for NOAA"
        #a_tovs = "ATOVS"
        
        #PROCESS IN THIS ORDER
        #ama = 0 #AMSU-A
        #amb = 0 #AMSU-B
        #hrs = 0 #HIRS
        #avh = 0 #avhrr
        #msu = 0 
        #dcs = 0 
                 
        decom_file = "decommutation.par"
        #Needs to find platform number for A/TOVS
        decom = open(decom_file, 'w')

        if int(process_config['platform'][4:6]) <= 14:
            del process_config['a_tovs'][0]
        
            decom.write("1,0,0,0,0,{},{},{},0,0,0,{}               ! OPTION NUMBERS\n".format(1 if process_config['process_hirs'] else 0,
                                                                                              1 if process_config['process_msu'] else 0,
                                                                                              1 if process_config['process_dcs'] else 0,
                                                                                              1 if process_config['process_avhrr'] else 0))
            decom.write("10,0,0,0,0,11,12,13,0,0,0,14              ! STREAM NO.S\n")
        else:
            decom.write("1,{},{},{},{},0,0,{},0,0,0,{}               ! OPTION NUMBERS\n".format(1 if process_config['process_hirs'] else 0,
                                                                                                1 if process_config['process_amsua'] else 0,
                                                                                                1 if process_config['process_amsua'] else 0,
                                                                                                1 if process_config['process_amsub'] else 0,
                                                                                                1 if process_config['process_dcs'] else 0,
                                                                                                1 if process_config['process_avhrr'] else 0))
            decom.write("10,11,15,15,16,0,0,13,0,0,0,14              ! STREAM NO.S\n")
            
        decom.write("{:%Y}                                     ! year of the data\n".format(timestamp))
        decom.write("0                                         ! operational mode\n")
        decom.write("{0:05d},{0:05d}                           ! start and end orbit numbers\n".format(process_config['orbit_number']))
        decom.close()
            
        cmd="decommutation {} {} {}".format("".join(process_config['a_tovs']),decom_file, hrpt_file)
        try:
            status, returncode, std, err = run_shell_command(cmd)
        except:
            LOG.error("Command {} failed.".format(cmd))
        else:
            if returncode in accepted_return_codes_decom_hrpt:
                LOG.info("Command {} complete.".format(cmd))
                if not os.path.exists(process_config['avhrr_file']):
                    LOG.warning("Decom gave OK status, but no {} data is produced. Something is wrong".format(process_config['avhrr_file']))
            else:
                LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                return_status = False
        
    elif 'metop' in process_config['platform']:
        LOG.info("Do the metop decommutation")
        if process_config['process_amsua']:
            cmd="decom-amsua-metop {} {} ".format(process_config['input_amsua_file'],process_config['amsua_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd,stdout_logfile="decom-amsua-metop.log")
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_decom_amsua_metop:
                    LOG.info("Command {} complete.".format(cmd))
                    if not os.path.exists(process_config['amsua_file']):
                        LOG.warning("Decom gave OK status, but no data is produced. Something is wrong")
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_status = False
                        
        if process_config['process_amsub'] and return_status:
            cmd="decom-mhs-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr", 
                                                   process_config['input_amsub_file'],
                                                   process_config['amsub_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd,stdout_logfile="decom-mhs-metop.log")
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_decom_amsub_metop:
                    LOG.info("Command {} complete.".format(cmd))
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_status = False

        if process_config['process_hirs'] and return_status:
            cmd="decom-hirs-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr", 
                                                   process_config['input_hirs_file'],
                                                   process_config['hirs_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd,stdout_logfile="decom-hirs-metop.log")
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_decom_hirs_metop:
                    LOG.info("Command {} complete.".format(cmd))
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_status = False

        if process_config['process_avhrr'] and return_status:
            cmd="decom-avhrr-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr", 
                                                   process_config['input_avhrr_file'],
                                                   process_config['avhrr_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd,stdout_logfile="decom-avhrr-metop.log")
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_decom_avhrr_metop:
                    LOG.info("Command {} complete.".format(cmd))
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_status = False

    else:
        print "Unknown platform: {}".format(process_config['platform'])
        return_status = False
    
    print os.listdir("./")
    #Change back after this is done
    os.chdir(current_dir)

    LOG.info("Decommutation complete.")
    return return_status
コード例 #21
0
def do_ana_correction(process_config, msg, timestamp):
    """The software ANA is 3party. User will have to get it themself.
    Further the ana binaries and scripts must be in the PATH,
    preferable in the primary AAPP bin directory

    Also it relays on reference_landmarks directory under ENV(DIR_NAVIGATION)/ana dir
    This needs to be copied/installed here by the user

    TODO: Replace the ana_lmk_loc script with code here to better be able to control
    the directories used in the processing
    """

    if not process_config['process_avhrr']:
        LOG.debug(
            "Skipping ANA as AVHRR is not processed de to config or missing data."
        )
        return True

    try:
        if 'do_ana_correction' in process_config['aapp_processes'][
                process_config.process_name]:
            if not process_config['aapp_processes'][
                    process_config.process_name]['do_ana_correction']:
                return True
    except Exception as err:
        LOG.error("Failed with: {}".format(err))

    return_status = True

    # This function relays on beeing in a working directory
    current_dir = os.getcwd(
    )  # Store the dir to change back to after function complete
    os.chdir(process_config['aapp_processes'][process_config.process_name]
             ['working_dir'])

    # Must check of the ana dir exists
    ana_dir = os.path.join(os.getenv('DIR_NAVIGATION'), 'ana')
    if not os.path.exists(ana_dir):
        try:
            os.makedirs(ana_dir)
            os.environ['DIR_ANA'] = ana_dir
        except OSError in e:
            LOG.error(
                "Failed to create directory: {}. This is needed to run the ANA software."
            )
            return_status = False

    # And the ana dir must contain a reference_landmarks directory to work!
    if not os.path.exists(os.path.join(ana_dir, "reference_landmarks")):
        LOG.error(
            "Can not run ANA because reference_landmarks under the ana dir is missing."
        )
        return_status = False

    if return_status:
        # Find all matching landmarks
        cmd = "ana_lmk_loc -D {}".format(
            process_config['aapp_static_configuration']['decommutation_files']
            ['avhrr_file'])
        try:
            status = False
            status, returncode, std, err = run_shell_command(
                cmd,
                stdout_logfile="ana_lmk_loc.log",
                stderr_logfile="ana_lmk_loc.err")
        except:
            import sys
            LOG.error("Command {} failed with {}.".format(
                cmd,
                sys.exc_info()[0]))
            if not status:
                return_status = False
        else:
            if not status or returncode != 0:
                LOG.error("Command {} failed with {}".format(cmd, returncode))
                _ana_file = open('ana_lmk_loc.err', "w")
                _ana_file.write(std)
                _ana_file.write(err)
                _ana_file.close()
                return_status = False

    if return_status:
        # Check of ana landmark location file with correct time stamp exist.
        # If not try to find correct timestamp and copy to this.
        expected_ana_loc_file = "lmkloc_{}_{:%Y%m%d_%H%M}_{:05d}.txt".format(
            process_config['platform_name'], timestamp,
            process_config['orbit_number'])
        if not os.path.exists(
                os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
                             expected_ana_loc_file)):
            # Need to run this AAPP command to detect the start of the hrpt.l1b(avhrr data) file
            # Because this might differ from the time stamp in file name
            cmd = "l1bidf.exe"
            try:
                status, returncode, std, err = run_shell_command(
                    cmd,
                    stdin="{}\n".format(
                        process_config['aapp_static_configuration']
                        ['decommutation_files']['avhrr_file']))
            except:
                import sys
                LOG.error("Command {} failed with {}.".format(
                    cmd,
                    sys.exc_info()[0]))
                if not status:
                    return_status = False
            else:
                if not status or returncode != 0:
                    LOG.error("Command {} failed with {}".format(
                        cmd, returncode))
                    _ana_file = open('ana_lmk_loc.err', "w")
                    _ana_file.write(std)
                    _ana_file.write(err)
                    _ana_file.close()
                    return_status = False
                else:
                    l1bidf_list = std.split()
                    l1bidf_aapp_datetime = datetime.strptime(
                        l1bidf_list[1] + l1bidf_list[2], "%Y%m%d%H%M")
                    LOG.debug("l1b idf aapp datetime: {}".format(
                        l1bidf_aapp_datetime))
                    # estatt_file_name =     lmkloc_metop02_20160617_0952_50132.txt
                    ana_loc_file = "lmkloc_{}_{:%Y%m%d_%H%M}_{:05d}.txt".format(
                        l1bidf_list[0], l1bidf_aapp_datetime,
                        int(l1bidf_list[3]))
                    if os.path.exists(
                            os.path.join(
                                ana_dir,
                                "{:%Y-%m}".format(l1bidf_aapp_datetime),
                                ana_loc_file)):
                        LOG.debug(
                            "timestamp {} l1bidf_aapp_datetime: {}".format(
                                timestamp, l1bidf_aapp_datetime))
                        from shutil import copy2
                        copy2(
                            os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
                                         ana_loc_file),
                            os.path.join(
                                ana_dir,
                                "{:%Y-%m}".format(l1bidf_aapp_datetime),
                                expected_ana_loc_file))
                    else:
                        LOG.warning("Could not find ana loc file: {}".format(
                            ana_loc_file))
        else:
            LOG.debug("File ok: {}".format(
                os.path.join(ana_dir, "{:%Y-%m}".format(timestamp),
                             expected_ana_loc_file)))

    import hashlib
    if return_status:
        # LOG.debug("sha256 of aapp input avhrr_file: {}".format(hashlib.sha256(open(
        # process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest()))

        # Calculate correction from landmarks and update avhrr_file with these new attitude coefisients.
        cmd = "ana_estatt -s {0} -d {1:%Y%m%d} -h {1:%H%M} -n {2:05d}".format(
            process_config['platform_name'], timestamp,
            process_config['orbit_number'])
        try:
            status, returncode, std, err = run_shell_command(cmd)
        except:
            import sys
            LOG.error("Command {} failed with {}.".format(
                cmd,
                sys.exc_info()[0]))
            if not status:
                return_status = False
        else:
            if not status or returncode != 0:
                LOG.error("Command {} failed with {}".format(cmd, returncode))
                _ana_file = open('ana_lmk_loc.err', "w")
                _ana_file.write(std)
                _ana_file.write(err)
                _ana_file.close()
                return_status = False

        # LOG.debug("sha256 of aapp input avhrr_file: {}".format(hashlib.sha256(open(
        # process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'], 'rb').read()).hexdigest()))

        sha256_before_correction = hashlib.sha256(
            open(
                process_config['aapp_static_configuration']
                ['decommutation_files']['avhrr_file'],
                'rb').read()).hexdigest()

    if return_status:
        # Recalculate the location in the avhhr data file with the new correction attitude coefisients.
        from do_avhrr_calibration import do_avhrr_calibration
        LOG.info(
            "Need to recalculate the avhrcl with new attitude coefisients from ANA."
        )
        if not do_avhrr_calibration(process_config, msg, timestamp):
            LOG.warning(
                "The avhrr location with ana correction failed for some reason."
                "It might be that the processing can continue")
            LOG.warning(
                "Please check the previous log carefully to see if this is an error you can accept."
            )
            return_status = False

    if return_status:
        sha256_after_correction = hashlib.sha256(
            open(
                process_config['aapp_static_configuration']
                ['decommutation_files']['avhrr_file'],
                'rb').read()).hexdigest()
        LOG.debug("sha256 of aapp input avhrr_file BEFORE ana: {}".format(
            sha256_before_correction))
        LOG.debug("sha256 of aapp input avhrr_file AFTER  ana: {}".format(
            sha256_after_correction))
        if (sha256_before_correction == sha256_after_correction):
            LOG.warning(
                "The correction of the avhrr location with data from ANA did not take place for some reason,"
                "or the correction was 0 for all pitch yaw and roll.")
        else:
            LOG.info(
                "The correction of AVHRR location with data from ANA was performed."
            )

    # Change back after this is done
    os.chdir(current_dir)

    LOG.info("do_ana_correction complete!")

    return return_status
コード例 #22
0
def do_tleing(timestamp, satellite, workdir, tle_indir=None, select_closest_tle_file_to_data=False):
    """Get the tle-file and copy them to the AAPP data structure 
       and run the AAPP tleing script and executable"""
    
    return_status = True
    
    #This function relays on beeing in a working directory
    current_dir = os.getcwd() #Store the dir to change back to after function complete
    os.chdir(workdir)

    # SATellite IDentification mandatory
    # so take care of default values 
    SATID_FILE=os.getenv('SATID_FILE', 'satid.txt')

    if tle_indir == None:
        """This is the default directory for the tles"""
        tle_indir = os.getenv('DIR_NAVIGATION')
    else:
        LOG.warning("Override the env variable set in AAPP_ENV7 DIR_NAVIGATION from {} to {}.".format(os.environ['DIR_NAVIGATION'], tle_indir))
        os.environ['DIR_NAVIGATION'] = tle_indir

    # variables for the TLE HOME directory
    DIR_DATA_TLE=os.path.join(os.getenv('DIR_NAVIGATION'),'tle_db')

    #This is needed by AAPP tleing. Try other of not existing
    if not os.path.exists(DIR_DATA_TLE):
        DIR_DATA_TLE=os.path.join(os.getenv('DIR_NAVIGATION'),'orb_elem')
        
    FIC_WRK=os.path.join(os.getenv('DIR_DATA_TLE'),'tmp_tle_',"{}".format(os.getpid()))
    
    #LISTESAT=os.getenv('LISTESAT',os.getenv('PAR_NAVIGATION_DEFAULT_LISTESAT_INGEST_TLE'))

    #print LISTESAT

    TLE_INDEX=os.path.join(DIR_DATA_TLE,"tle_{}.index".format(satellite))

    tle_file_list = []
    if not select_closest_tle_file_to_data:
        if os.path.exists(TLE_INDEX):
            #tle_files = [s for s in os.listdir(DIR_DATA_TLE) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))]
            #_tle_file_list = glob(os.path.join(DIR_DATA_TLE,'tle*txt'))
            tle_files = [s for s in glob(os.path.join(DIR_DATA_TLE,'tle*txt')) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))]
            tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE,s)))
            
            tle_index_mtime = os.path.getmtime(TLE_INDEX)
            for s in tle_files:
                if os.path.getmtime(os.path.join(DIR_DATA_TLE,s)) > tle_index_mtime:
                    tle_file_list.append(s)
                    
            if len(tle_file_list) == 0:
                import time
                LOG.warning("No newer tle files than last update of the index file. Last update of index file is {:d}s. If more than a few days you should check.".format(int(time.time()-tle_index_mtime)))
            else:
                LOG.info("Will use tle files {}".format(tle_file_list))
        else:
            LOG.warning("index file does not exist. If this is the first run of AAPP tleing.exe it is ok, elsewise it is a bit suspisiuos.")
            tle_files = [s for s in os.listdir(DIR_DATA_TLE) if os.path.isfile(os.path.join(DIR_DATA_TLE, s))]
            tle_files.sort(key=lambda s: os.path.getctime(os.path.join(DIR_DATA_TLE,s)))
            tle_file_list = tle_files
            
    else:
        #FIXME add as config
        tle_file_timestamp_format = "%Y%m%dT%H%M"

        infile = "tle-{0:{1}}.txt".format(timestamp,tle_file_timestamp_format)
    
        LOG.debug("tle file name: {}".format(infile))
    
        min_closest_tle_file = 3*24*60*60
        #Check if I can read the tle file.
        try:
            with open(os.path.join(tle_indir, infile)) as tle_file:
                pass
        except IOError as e:
            LOG.warning("Could not find tle file: {}. Try find closest ... ".format(infile))
            infile = None
            #tle_file_list = glob(os.path.join(tle_indir,'tle*'))
            #print "tle file list: {}".format(tle_file_list)
            import re
            tle_match_tests = (('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2})(\d{2}).*',_do_6_matches),
                               ('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2})(\d{2}).*',_do_5_matches),
                               ('.*(\d{4})(\d{2})(\d{2})_?-?T?(\d{2}).*',_do_4_matches),
                               ('.*(\d{4})(\d{2})(\d{2}).*',_do_3_matches))
            #print tle_file_list
            for tle_file_name in tle_files:
                for regex, test in tle_match_tests:
                    m = re.match(regex, tle_file_name)
                    if m:
                        delta = timestamp - test(m)
                        if ( abs(delta.total_seconds()) < min_closest_tle_file):
                            min_closest_tle_file = abs(delta.total_seconds()) 
                            infile = os.path.basename(tle_file_name)
                        break
        

        tle_file_list.append(infile)                
        LOG.debug("Use this: {} {}".format(tle_file_list,min_closest_tle_file))

        #print "{}".format(tle_file_list)
        #infile = "tle-{0:{1}}.txt".format(timestamp,tle_file_timestamp_format)
        
    for tle_file in tle_file_list:
        if not os.path.exists(os.path.join(tle_indir,'tle_db',tle_file)):
            print "Could not find the tle file: {}".format(tle_indir + "/" + tle_file)
            return_status = False
        else:
            """Dont use the tle_indir because this is handeled by the tleing script"""
            #tle_cmd = open("tle_commands", 'w')
            #tle_cmd.write("{}\n".format(DIR_DATA_TLE))
            #tle_cmd.write("{}\n".format(tle_file))
            #tle_cmd.write("{}\n".format(satellite))
            #tle_cmd.write("{}\n".format(TLE_INDEX))
            #tle_cmd.close()
            #LOG.info("TLE file ok. Do the calc for {} ... ".format(satellite))
            status = False
            returncode = 0
            stdout = ""
            stderr = ""
            cmd="tleing.exe"
            try:
                status, returncode, stdout, stderr = run_shell_command(cmd,stdin="{}\n{}\n{}\n{}\n".format(DIR_DATA_TLE,
                                                                                                           os.path.basename(tle_file),
                                                                                                           satellite,
                                                                                                           TLE_INDEX))
            except:
                LOG.error("Failed running command: {} with return code: {}".format(cmd,returncode))
                LOG.error("stdout: {}".format(stdout))
                LOG.error("stderr: {}".format(stderr))
                return_status = False
            else:
                if returncode != 0:
                    LOG.debug("Running command: {} with return code: {}".format(cmd,returncode))
                    LOG.debug("stdout: {}".format(stdout))
                    LOG.debug("stderr: {}".format(stderr))
                elif not os.path.exists(TLE_INDEX):
                    LOG.error("index file: {} does not exist after tleing. Something is wrong.".format(TLE_INDEX))
                    LOG.debug("Running command: {} with return code: {}".format(cmd,returncode))
                    LOG.debug("stdout: {}".format(stdout))
                    LOG.debug("stderr: {}".format(stderr))
                else:
                    LOG.debug("Running command: {} with return code: {}".format(cmd,returncode))
                    LOG.debug("stdout: {}".format(stdout))
                    LOG.debug("stderr: {}".format(stderr))

                    #When a index file is generated above one line is added for each tle file.
                    #If several tle files contains equal TLEs each of these TLEs generate one line in the index file
                    #To avoid this, sort the index file keeping only unique lines(skipping the tle filename at the end
                    
                    #The sort options +0b -3b is guessed to be sort from column 0 to 3, but this is not documented
                    #Could cause problems with future version of sort. See eg. http://search.cpan.org/~sdague/ppt-0.12/bin/sort
                    #cmd="sort -u -o {} +0b -3b {}".format(os.path.join(DIR_DATA_TLE, "{}.sort".format(TLE_INDEX)),os.path.join(DIR_DATA_TLE, TLE_INDEX))
                    if os.path.exists(TLE_INDEX):
                        cmd="sort -u -o {} +0b -3b {}".format("{}.sort".format(TLE_INDEX), TLE_INDEX)
                        try:
                            status, returncode, stdout, stderr = run_shell_command(cmd)
                        except:
                            LOG.error("Failed running command: {} with return code: {}".format(cmd,returncode))
                            LOG.error("stdout: {}".format(stdout))
                            LOG.error("stderr: {}".format(stderr))
                            return_status = False
                        else:
                            if returncode == 0 and os.path.exists("{}.sort".format(TLE_INDEX)):
                                try:
                                    #os.remove(os.path.join(DIR_DATA_TLE, TLE_INDEX))
                                    os.remove(TLE_INDEX)
                                except OSError as e:
                                    #LOG.error("Failed to remove unsorted and duplicated index file: {}".format(os.path.join(DIR_DATA_TLE, TLE_INDEX)))
                                    LOG.error("Failed to remove unsorted and duplicated index file: {}".format(TLE_INDEX))
                                else:
                                    try:
                                        #os.rename(os.path.join(DIR_DATA_TLE, "{}.sort".fromat(TLE_INDEX)),os.path.join(DIR_DATA_TLE, TLE_INDEX))
                                        os.rename("{}.sort".format(TLE_INDEX),TLE_INDEX)
                                    except:
                                        LOG.error("Failed to rename sorted index file to original name.")
                            else:
                                LOG.error("Returncode other than 0: {} or tle index sort file does exists.".format(returncode, "{}.sort".format(TLE_INDEX)))
                    else:
                        LOG.error("tle index file: {} does not exists after tleing before sort. This can not happen.")
                                            
    #Change back after this is done
    os.chdir(current_dir)

    return return_status
コード例 #23
0
def do_decommutation(process_config, msg, timestamp):
    """
    decommutation ${A_TOVS}  decommutation.par  ${FILE}

    The 3 arguments are obligatory.

    ${A_TOVS} = TOVS for satellite number < or = 14
    ${A_TOVS} = ATOVS for satellite number > 14

    In the decommutation.par file, options are written in this order:
    $1,$2,$3,$4,$5, $6, $7, $8,$9,$10,$11,$12 !OPTION NUMBERS
    $lu1,$lu2,$lu3,$lu4,$lu5,$lu6,$lu7,$lu8,$lu9,$lu10,$lu11,$lu12   !STREAM NO.S
    ${YEAR} ! year of the data
    0  ! operational mode
    ${NNNNN},${NNNNN} ! start and end orbit numbers

    $1 = 0 or 1 for level of error logging
    $2 = 0 or 1 for HIRS/3 or HIRS/4 (1 indicates extract HIRS/3 or HIRS/4 data)
    $3 = 0 or 1 for AMSU-A1 (1 indicates extract AMSU-A1 data)
    $4 = 0 or 1 for AMSU-A2 (1 indicates extract AMSU-A2 data)
    $5 = 0 or 1 for AMSU-B/MHS (1 indicates extract AMSU-B/MHS data)
    $6 = 0 or 1 for HIRS/2 (1 indicates extract HIRS/2 data)
    $7 = 0 or 1 for MSU (1 indicates extract MSU data)
    $8 = 0 or 1 for DCS (1 indicates extract DCS data)
    $9 = 0 or 1 for SEM (1 indicates extract SEM data)
    $10= 0 or 1 for SBUV (1 indicates extract SBUV data)
    $11= 0 or 1 for SAR (1 indicates extract SAR data)
    $12= 0 or 1 for AVHRR (1 indicates extract AVHRR data)
    $lu1 is the logical unit of the log file
    $lu2 is the logical unit of the HIRS/3 or HIRS/4.l1a output file
    $lu3 is the logical unit of the AMSU-A1.l1a output file
    $lu4 is the logical unit of the AMSU-A2.l1a output file
    $lu5 is the logical unit of the AMSU-B.l1a output file
    $lu6 is the logical unit of the HIRS/2.l1a output file
    $lu7 is the logical unit of the MSU.l1a output file
    $lu8 is the logical unit of the DCS.l1a output file
    $lu9 is the logical unit of the SEM.l1a output file
    $lu10 is the logical unit of the SBUV.l1a output file
    $lu11 is the logical unit of the SAR.l1a output file
    $lu12 is the logical unit of the HRPT.l1a output file
    """

    # A list of accepted return codes for the various scripts/binaries run in this function
    accepted_return_codes_decom_hrpt = [0]
    accepted_return_codes_chk1btime = [0]
    accepted_return_codes_decom_amsua_metop = [0]
    accepted_return_codes_decom_amsub_metop = [0]
    accepted_return_codes_decom_hirs_metop = [0]
    accepted_return_codes_decom_avhrr_metop = [0]

    return_status = True

    # This function relays on beeing in a working directory
    current_dir = os.getcwd()  # Store the dir to change back to after function complete
    os.chdir(process_config['aapp_processes'][process_config.process_name]['working_dir'])

    # for sensor in sensors:
    #    if str(sensor) in "amsu-a":
    #        process_config['process_amsua'] = True
    #    elif str(sensor) in ("amsu-b", "mhs"):
    #        process_config['process_amsub'] = True
    #   elif str(sensor) in ("hirs/3","hirs/4"):
    #       process_config['process_hirs'] = True
    #    elif str(sensor) in "avhrr/3":
    #        process_config['process_avhrr'] = True
    #    elif str(sensor) in "msu":
    #        process_config['process_msu'] = True
    #    elif str(sensor) in "dcs":
    #        process_config['process_dcs'] = True
    #    else:
    #        LOG.warning("Instrument/sensor {} not recognised.".format(sensor))

    if 'noaa' in process_config['platform_name']:
        LOG.debug("Do the commutation for NOAA")
        # a_tovs = "ATOVS"

        # PROCESS IN THIS ORDER
        # ama = 0 # AMSU-A
        # amb = 0 # AMSU-B
        # hrs = 0 # HIRS
        # avh = 0 # avhrr
        # msu = 0
        # dcs = 0

        try:
            decom_file = "decommutation.par"
            # Needs to find platform number for A/TOVS
            decom = open(decom_file, 'w')

            if int(process_config['platform_name'][4:6]) <= 14:
                del process_config['a_tovs'][0]

                decom.write("1,0,0,0,0,{},{},{},0,0,0,{}               ! OPTION NUMBERS\n".format(1 if process_config['process_hirs'] else 0,
                                                                                                  1 if process_config['process_msu'] else 0,
                                                                                                  1 if process_config['process_dcs'] else 0,
                                                                                                  1 if process_config['process_avhrr'] else 0))
                decom.write("10,0,0,0,0,11,12,13,0,0,0,14              ! STREAM NO.S\n")
            else:
                decom.write("1,{},{},{},{},0,0,{},0,0,0,{}               ! OPTION NUMBERS\n".format(1 if process_config['process_hirs'] else 0,
                                                                                                    1 if process_config['process_amsua'] else 0,
                                                                                                    1 if process_config['process_amsua'] else 0,
                                                                                                    1 if process_config['process_amsub'] else 0,
                                                                                                    1 if process_config['process_dcs'] else 0,
                                                                                                    1 if process_config['process_avhrr'] else 0))
                decom.write("10,11,15,15,16,0,0,13,0,0,0,14              ! STREAM NO.S\n")

            decom.write("{:%Y}                                     ! year of the data\n".format(timestamp))
            decom.write("0                                         ! operational mode\n")
            decom.write("{0:05d},{0:05d}                           ! start and end orbit numbers\n".format(int(process_config['orbit_number'])))
            decom.close()
        except Exception as err:
            LOG.error("Building decommutation command string failed: {}".format(err))
            return False

        # Read decommitation input into variable
        decom = open(decom_file, 'r')
        decom_input = decom.read()
        decom.close()

        os.environ['FILE_COEF'] = os.path.join(os.environ.get('PAR_CALIBRATION_COEF'), 'amsua', 'amsua_clparams.dat')
        os.symlink(os.environ['FILE_COEF'], "{}50".format(os.environ['FORT']))

        cmd = "decommutation.exe"  # .format("".join(process_config['a_tovs']),decom_file, process_config['input_hrpt_file'])
        try:
            status, returncode, std, err = run_shell_command(cmd, stdin="{}\n{}{}\n".format(process_config['input_hrpt_file'], decom_input, os.getenv('STATION_ID', 'ST')),
                                                             stdout_logfile='decommutation.log',
                                                             stderr_logfile='decommutation.log')
        except:
            LOG.error("Command {} failed.".format(cmd))
        else:
            if returncode in accepted_return_codes_decom_hrpt:
                LOG.info("Decommutation command {} complete.".format(cmd))
            else:
                LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                return_status = False

        # Need to check the decommutation output and rename fort files
        if return_status:
            # hrpt 14
            if os.path.exists("{}14".format(os.environ['FORT'])):
                shutil.move("{}14".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['avhrr_file'])
                cmd = "chk1btime.exe"
                try:
                    status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['avhrr_file']))
                except:
                    LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
                else:
                    if returncode in accepted_return_codes_chk1btime:
                        LOG.debug("chk1btime command {} ok.".format(cmd))
                        LOG.debug("std: {}".format(std))
                    else:
                        LOG.error("This means that the start of the data are bad,"
                                  " and that the processing for this data later will fail.")
                        LOG.debug("Return code: {}".format(returncode))
                        LOG.debug("std: {}".format(std))
                        LOG.debug("err: {}".format(err))
                        LOG.debug("status: {}".format(status))
                        process_config['process_avhrr'] = False

            elif process_config['process_avhrr']:
                LOG.warning("Fort file for avhrr does not exist after decommutation. Skip processing this.")
                process_config['process_avhrr'] = False

            # hrsn 11
            if os.path.exists("{}11".format(os.environ['FORT'])):
                shutil.move("{}11".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['hirs_file'])
                cmd = "chk1btime.exe"
                try:
                    status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['hirs_file']))
                except:
                    LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
                else:
                    if returncode in accepted_return_codes_chk1btime:
                        LOG.debug("chk1btime command {} ok.".format(cmd))
                    else:
                        LOG.error("This means that the start of the data are bad,"
                                  " and that the processing for this data later will fail.")
                        LOG.debug("Return code: {}".format(returncode))
                        LOG.debug("std: {}".format(std))
                        LOG.debug("err: {}".format(err))
                        LOG.debug("status: {}".format(status))
                        process_config['process_hirs'] = False
            elif process_config['process_hirs']:
                LOG.warning("Fort file for hirs does not exist after decommutation. Skip processing this.")
                process_config['process_hirs'] = False

            # msun 12
            if os.path.exists("{}12".format(os.environ['FORT'])):
                shutil.move("{}12".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['msu_file'])
                cmd = "chk1btime.exe"
                try:
                    status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['msu_file']))
                except:
                    LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
                else:
                    if returncode in accepted_return_codes_chk1btime:
                        LOG.debug("chk1btime command {} ok.".format(cmd))
                    else:
                        LOG.error("This means that the start of the data are bad, "
                                  " and that the processing for this data later will fail.")
                        LOG.debug("Return code: {}".format(returncode))
                        LOG.debug("std: {}".format(std))
                        LOG.debug("err: {}".format(err))
                        LOG.debug("status: {}".format(status))
                        process_config['process_msu'] = False
            elif process_config['process_msu']:
                LOG.warning("Fort file for msu does not exist after decommutation. Skip processing this.")
                process_config['process_msu'] = False

            # dcsn 13
            if os.path.exists("{}13".format(os.environ['FORT'])):
                shutil.move("{}13".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['dcs_file'])
                cmd = "chk1btime.exe"
                try:
                    status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['dcs_file']))
                except:
                    LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
                else:
                    if returncode in accepted_return_codes_chk1btime:
                        LOG.debug("chk1btime command {} ok.".format(cmd))
                    else:
                        LOG.error("This means that the start of the data are bad,"
                                  " and that the processing for this data later will fail.")
                        LOG.debug("Return code: {}".format(returncode))
                        LOG.debug("std: {}".format(std))
                        LOG.debug("err: {}".format(err))
                        LOG.debug("status: {}".format(status))
                        process_config['process_dcs'] = False
            elif process_config['process_dcs']:
                LOG.warning("Fort file for dcs does not exist after decommutation. Skip processing this.")
                process_config['process_dcs'] = False

            # aman 15
            if os.path.exists("{}15".format(os.environ['FORT'])):
                shutil.move("{}15".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['amsua_file'])
                cmd = "chk1btime.exe"
                try:
                    status, returncode, std, err = run_shell_command(cmd,
                                                                     stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['amsua_file']))
                except:
                    LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
                else:
                    if returncode in accepted_return_codes_chk1btime:
                        LOG.debug("chk1btime command {} ok.".format(cmd))
                    else:
                        LOG.error("This means that the start of the data are bad, "
                                  "and that the processing for this data later will fail?")
                        LOG.debug("Return code: {}".format(returncode))
                        LOG.debug("std: {}".format(std))
                        LOG.debug("err: {}".format(err))
                        LOG.debug("status: {}".format(status))
                        process_config['process_amsua'] = False
            elif process_config['process_amsua']:
                LOG.warning("Fort file for amsu-a does not exist after decommutation. Skip processing this.")
                process_config['process_amsua'] = False

            # ambn 16
            if os.path.exists("{}16".format(os.environ['FORT'])):
                shutil.move("{}16".format(os.environ['FORT']), process_config['aapp_static_configuration']['decommutation_files']['amsub_file'])
                cmd = "chk1btime.exe"
                try:
                    status, returncode, std, err = run_shell_command(cmd, stdin="{}\n".format(process_config['aapp_static_configuration']['decommutation_files']['amsub_file']))
                except:
                    LOG.error("Failed to execute command {}. Something wrong with the command.".format(cmd))
                else:
                    if returncode in accepted_return_codes_chk1btime:
                        LOG.debug("chk1btime command {} ok.".format(cmd))
                    else:
                        LOG.error("This means that the start of the data are bad, "
                                  "and that the processing for this data later will fail.")
                        LOG.debug("Return code: {}".format(returncode))
                        LOG.debug("std: {}".format(std))
                        LOG.debug("err: {}".format(err))
                        LOG.debug("status: {}".format(status))
                        process_config['process_amsub'] = False
            elif process_config['process_amsub']:
                LOG.warning("Fort file for amsu-b does not exist after decommutation. Skip processing this.")
                process_config['process_amsub'] = False

        if os.path.exists("decommutation.log"):
            dd = None
            tt = None
            with open("decommutation.log") as decomlog:
                while True:
                    line = decomlog.readline()
                    if not line:
                        break
                    p_date = re.compile(".*avhrr\send\sdata\sday\s([0-9]{2})/([0-9]{2})/([0-9]{2})")
                    p_time = re.compile(".*avhrr\send\sdata\stime\s([0-9]{2}):([0-9]{2}):([0-9]{2})\.([0-9]{0,3})")
                    se = p_date.search(line)
                    if se:
                        dd = datetime.date(year=(int(se.group(3)) + 2000), month=int(se.group(2)), day=int(se.group(1)))
                    te = p_time.search(line)
                    if te:
                        tt = datetime.time(hour=int(te.group(1)),
                                           minute=int(te.group(2)),
                                           second=int(te.group(3)),
                                           microsecond=int(te.group(4)) * 1000)
                    if tt and dd:
                        et = datetime.datetime.combine(dd, tt)
                        if abs((et - process_config['endtime']).total_seconds()) < 20 * 60:
                            LOG.debug("Adjusting the endtime with data from decommutaion.log.")
                            LOG.debug("Before {}".format(process_config['endtime']))
                            process_config['endtime'] = et
                            LOG.debug("After  {}".format(process_config['endtime']))
                        break
    elif 'METOP' in process_config['platform_name'].upper():
        LOG.info("Do the metop decommutation")
        if process_config['process_amsua']:
            cmd = "decom-amsua-metop {} {} ".format(process_config['input_amsua_file'],
                                                    process_config['aapp_static_configuration']['decommutation_files']['amsua_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd, stdout_logfile="decom-amsua-metop.log")
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_decom_amsua_metop:
                    LOG.info("Command {} complete.".format(cmd))
                    if not os.path.exists(process_config['aapp_static_configuration']['decommutation_files']['amsua_file']):
                        LOG.warning("Decom gave OK status, but no data is produced. Something is wrong")
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_status = False

        if process_config['process_mhs'] and return_status:
            cmd = "decom-mhs-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
                                                     process_config['input_mhs_file'],
                                                     process_config['aapp_static_configuration']['decommutation_files']['mhs_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd, stdout_logfile="decom-mhs-metop.log")
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_decom_amsub_metop:
                    LOG.info("Command {} complete.".format(cmd))
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_status = False

        if process_config['process_hirs'] and return_status:
            cmd = "decom-hirs-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
                                                      process_config['input_hirs_file'],
                                                      process_config['aapp_static_configuration']['decommutation_files']['hirs_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd, stdout_logfile="decom-hirs-metop.log")
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_decom_hirs_metop:
                    LOG.info("Command {} complete.".format(cmd))
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_status = False

        if process_config['process_avhrr'] and return_status:
            cmd = "decom-avhrr-metop {} {} {} ".format("-ignore_degraded_inst_mdr -ignore_degraded_proc_mdr",
                                                       process_config['input_avhrr_file'],
                                                       process_config['aapp_static_configuration']
                                                       ['decommutation_files']['avhrr_file'])
            try:
                status, returncode, std, err = run_shell_command(cmd, stdout_logfile="decom-avhrr-metop.log")
            except:
                LOG.error("Command {} failed.".format(cmd))
            else:
                if returncode in accepted_return_codes_decom_avhrr_metop:
                    LOG.info("Command {} complete.".format(cmd))
                else:
                    LOG.error("Command {} failed with return code {}.".format(cmd, returncode))
                    return_status = False

    else:
        LOG.error("Unknown platform: {}".format(process_config['platform_name']))
        return_status = False

    # print os.listdir("./")
    # Change back after this is done
    os.chdir(current_dir)

    LOG.info("All decommutations complete.")
    return return_status
コード例 #24
0
def do_hirs_calibration(process_config, msg, timestamp):

    return_status = True

    if not process_config['process_hirs']:
        return True

    # A list of accepted return codes for the various scripts/binaries run in this function
    accepted_return_codes_hirs_historic_file_manage = [0]

    # This function relays on beeing in a working directory
    current_dir = os.getcwd()  # Store the dir to change back to after function complete
    os.chdir(process_config['aapp_processes'][process_config.process_name]['working_dir'])
    hirs_version_use = None
    hirs_version = os.getenv('HIRSCL_VERSION', 0)
    hirs_version_list = hirs_version.split()
    hirs_sats = os.getenv('HIRSCL_SAT', 'default')
    hirs_sat_list = hirs_sats.split()
    index = 0
    for sat in hirs_sat_list:
        if sat in process_config['platform_name']:
            hirs_version_use = hirs_version_list[index]
        else:
            hirs_version_def = hirs_version_list[index]

        index += 1

    if hirs_version_use is None:
        hirs_version_use = hirs_version_def

    hirs_script = "hirscl"
    hirs_err_file = "hirscl.err"
    calibration_location = process_config['calibration_location']

    LOG.debug("hirs_version_use {}".format(hirs_version_use))

    # pdb.set_trace()

    if int(hirs_version_use) > 1:  # no calibration, just navigation
        calibration_location = "-l"
    elif int(hirs_version_use) == 0 or "".join(process_config['a_tovs']) == 'TOVS':
        calibration_location = "-c -l"
    elif int(hirs_version_use) == 1:
        file_historic = os.path.join(os.getenv('PAR_CALIBRATION_MONITOR'),
                                     process_config['platform_name'], "hirs_historic.txt")
        if os.path.exists(file_historic):
            cmd = "hirs_historic_file_manage -m {} -r {} -n {} {}".format(os.getenv('HIST_SIZE_HIGH'),
                                                                          os.getenv('HIST_SIZE_LOW'),
                                                                          os.getenv('HIST_NMAX'),
                                                                          file_historic)
            try:
                status, returncode, std, err = run_shell_command(cmd)
            except:
                LOG.error("Command {} failed.".format(cmd))
                return_status = False
            else:
                if returncode in accepted_return_codes_hirs_historic_file_manage:
                    LOG.debug("Command complete.")
                else:
                    LOG.error("Command {} failed with returncode {}".format(cmd, returncode))
                    LOG.error("stdout was: {}".format(std))
                    LOG.error("stderr was: {}".format(err))
                    return_status = False

        if return_status:
            cmd = "hcalcb1_algoV4 -s {0} -y {1:%Y} -m {1:%m} -d {1:%d} -h {1:%H} -n {1:%M}".format(process_config['platform_name'], timestamp)
            try:
                status, returncode, std, err = run_shell_command(cmd)
            except:
                import sys
                LOG.error("Command {} failed with {}.".format(cmd, sys.exc_info()[0]))
            else:
                if returncode != 0:
                    LOG.error("Command {} failed with {}".format(cmd, returncode))
                    _hirs_file = open(hirs_err_file, "w")
                    _hirs_file.write(std)
                    _hirs_file.write(err)
                    _hirs_file.close()
                    return_status = False

            hirs_script = "hirscl_algoV4"
            hirs_err_file = "hirscl_algoV4.err"
            calibration_location = "-c -l"
    else:
        LOG.error("Can not figure out which hirs calibration algo version to use.")
        return_status = False

    if return_status:

        try:
            cmd = "{} {} -s {} -d {:%Y%m%d} -h {:%H%M} -n {:05d} {}".format(hirs_script,
                                                                            calibration_location,
                                                                            process_config['platform_name'],
                                                                            timestamp,
                                                                            timestamp,
                                                                            int(process_config['orbit_number']),
                                                                            process_config['aapp_static_configuration']
                                                                            ['decommutation_files']['hirs_file'])
        except KeyError as ke:
            LOG.error("Building command string failed with key error: {}".format(ke))

        try:
            status, returncode, out, err = run_shell_command(cmd, stdout_logfile="{}.log".format(hirs_script),
                                                             stderr_logfile="{}".format(hirs_err_file))
        except:
            import sys
            LOG.error("Command {} failed {}.".format(cmd, sys.exc_info()[0]))
        else:
            if (returncode != 0):
                LOG.error("Command {} failed with {}".format(cmd, returncode))
                _hirs_file = open(hirs_err_file, "w")
                _hirs_file.write(out)
                _hirs_file.write(err)
                _hirs_file.close()
                return_status = False

    # Change back after this is done
    os.chdir(current_dir)

    LOG.info("do_hirs_calibration complete!")

    return return_status
コード例 #25
0
def do_hirs_calibration(process_config, timestamp):

    return_status = True

    #A list of accepted return codes for the various scripts/binaries run in this function
    accepted_return_codes_hirs_historic_file_manage = [0]

    #This function relays on beeing in a working directory
    current_dir = os.getcwd(
    )  #Store the dir to change back to after function complete
    os.chdir(process_config['working_directory'])

    hirs_version_use = None
    hirs_version = os.getenv('HIRSCL_VERSION', 0)
    hirs_version_list = hirs_version.split()
    hirs_sats = os.getenv('HIRSCL_SAT', 'default')
    hirs_sat_list = hirs_sats.split()
    index = 0
    for sat in hirs_sat_list:
        if sat in process_config['platform']:
            hirs_version_use = hirs_version_list[index]
        else:
            hirs_version_def = hirs_version_list[index]

        index += 1

    if hirs_version_use == None:
        hirs_version_use = hirs_version_def

    hirs_script = "hirscl"
    hirs_err_file = "hirscl.err"
    calibration_location = process_config['calibration_location']

    print "hirs_version_use {}".format(hirs_version_use)

    #pdb.set_trace()

    if int(hirs_version_use) > 1:  # no calibration, just navigation
        calibration_location = "-l"
    elif int(hirs_version_use) == 0 or "".join(
            process_config['a_tovs']) == 'TOVS':
        calibration_location = "-c -l"
    elif int(hirs_version_use) == 1:
        file_historic = os.path.join(os.getenv('PAR_CALIBRATION_MONITOR'),
                                     process_config['platform'],
                                     "hirs_historic.txt")
        if os.path.exists(file_historic):
            cmd = "hirs_historic_file_manage -m {} -r {} -n {} {}".format(
                os.getenv('HIST_SIZE_HIGH'), os.getenv('HIST_SIZE_LOW'),
                os.getenv('HIST_NMAX'), file_historic)
            try:
                status, returncode, std, err = run_shell_command(cmd)
            except:
                LOG.error("Command {} failed.".format(cmd))
                return_status = False
            else:
                if returncode in accepted_return_codes_hirs_historic_file_manage:
                    LOG.debug("Command complete.")
                else:
                    LOG.error("Command {} failed with returncode {}".format(
                        cmd, returncode))
                    LOG.error("stdout was: {}".format(std))
                    LOG.error("stderr was: {}".format(err))
                    return_status = False

        if return_status:
            cmd = "hcalcb1_algoV4 -s {0} -y {1:%Y} -m {1:%m} -d {1:%d} -h {1:%H} -n {1:%M}".format(
                process_config['platform'], timestamp)
            try:
                status, returncode, std, err = run_shell_command(cmd)
            except:
                import sys
                LOG.error("Command {} failed with {}.".format(
                    cmd,
                    sys.exc_info()[0]))
            else:
                if returncode != 0:
                    LOG.error("Command {} failed with {}".format(
                        cmd, returncode))
                    _hirs_file = open(hirs_err_file, "w")
                    _hirs_file.write(std)
                    _hirs_file.write(err)
                    _hirs_file.close()
                    return_status = False

            hirs_script = "hirscl_algoV4"
            hirs_err_file = "hirscl_algoV4.err"
            calibration_location = "-c -l"
    else:
        LOG.error(
            "Can not figure out which hirs calibration algo version to use.")
        return_status = False

    if return_status:
        #Default AAPP config for PAR_NAVIGATION_DEFAULT_LISTESAT Metop platform is M01, M02, M04
        #but needed names are metop01 etc. Replace this inside the processing from now on.
        aapp_satellite_list = os.getenv(
            'PAR_NAVIGATION_DEFAULT_LISTESAT').split()
        if process_config['platform'] not in aapp_satellite_list:
            LOG.warning(
                "Can not find this platform in AAPP config variable PAR_NAVIGATION_DEFAULT_LISTESAT. Will try to find matches. But it can be a good idea to change this variable in the ATOVS_ENV7 file."
            )
            LOG.warning("Platform {} not in list: {}".format(
                process_config['platform'], aapp_satellite_list))
            if 'metop' in process_config['platform'] and (
                ('M01' or 'M02' or 'M03' or 'M04') in aapp_satellite_list):
                LOG.info("Replace in this processing")
                PAR_NAVIGATION_DEFAULT_LISTESAT = os.getenv(
                    'PAR_NAVIGATION_DEFAULT_LISTESAT')
                PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
                    'M01', 'metop01')
                PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
                    'M02', 'metop02')
                PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
                    'M03', 'metop03')
                PAR_NAVIGATION_DEFAULT_LISTESAT = PAR_NAVIGATION_DEFAULT_LISTESAT.replace(
                    'M04', 'metop04')
                os.environ[
                    'PAR_NAVIGATION_DEFAULT_LISTESAT'] = PAR_NAVIGATION_DEFAULT_LISTESAT

        cmd = "{} {} -s {} -d {:%Y%m%d} -h {:%H%M} -n {:05d} {}".format(
            hirs_script, calibration_location, process_config['platform'],
            timestamp, timestamp, process_config['orbit_number'],
            process_config['hirs_file'])
        try:
            status, returncode, out, err = run_shell_command(
                cmd,
                stdout_logfile="{}.log".format(hirs_script),
                stderr_logfile="{}".format(hirs_err_file))
        except:
            import sys
            LOG.error("Command {} failed {}.".format(cmd, sys.exc_info()[0]))
        else:
            if (returncode != 0):
                LOG.error("Command {} failed with {}".format(cmd, returncode))
                _hirs_file = open(hirs_err_file, "w")
                _hirs_file.write(out)
                _hirs_file.write(err)
                _hirs_file.close()
                return_status = False

    #Change back after this is done
    os.chdir(current_dir)

    LOG.info("do_hirs_calibration complete!")

    return return_status