Exemple #1
0
def read_and_compress_table(parser, table, debug):
    """Read data from a FITS file and save it into a FITS binary table

    The data are read from the FITS file specified in the "table"
    section of the "parser" object (an instance of
    ConfigurationParser). If "debug" is true, save additional
    information (useful for debugging) in the table."""

    input_file_name = os.path.normpath(parser.get(table, 'file'))
    cur_hdu = None
    with pyfits.open(input_file_name) as input_file:
        hdu, column = get_hdu_and_column_from_schema(parser, table,
                                                     input_file)
        compression = parser.get(table, 'compression')
        log.info('compressing file %s (HDU %s, column %s) '
                 'into table "%s", '
                 'compression is "%s"',
                 input_file_name, str(hdu), str(column),
                 table, compression)

        samples_format = input_file[hdu].columns.formats[column]
        samples = ppc.to_native_endianness(input_file[hdu].data.field(column))
        if parser.has_option(table, 'datatype'):
            samples = np.array(samples, dtype=parser.get(table, 'datatype'))

        cur_hdu, num_of_bytes, elapsed_time = compress_to_FITS_table(parser, table,
                                                                     samples_format,
                                                                     samples,
                                                                     debug)
        cur_hdu.name = table
        cur_hdu.header['PCNUMSA'] = (len(samples),
                                     'Number of uncompressed samples')
        cur_hdu.header['PCCOMPR'] = (compression,
                                     'Polycomp compression algorithm')
        cur_hdu.header['PCSRCTP'] = (str(samples.dtype),
                                     'Original NumPy type of the data')
        cur_hdu.header['PCUNCSZ'] = (samples.itemsize * samples.size,
                                     'Size of the uncompressed data [bytes]')
        cur_hdu.header['PCCOMSZ'] = (num_of_bytes,
                                     'Size of the compressed data [bytes]')
        cur_hdu.header['PCTIME'] = (elapsed_time,
                                    'Time used for compression [s]')
        cr = float(cur_hdu.header['PCUNCSZ']) / float(cur_hdu.header['PCCOMSZ'])
        cur_hdu.header['PCCR'] = (cr, 'Compression ratio')
        log.info('table "%s" compressed, %s compressed to %s (cr: %.4f)',
                 table, humanize_size(cur_hdu.header['PCUNCSZ']),
                 humanize_size(cur_hdu.header['PCCOMSZ']), cr)

    return cur_hdu
Exemple #2
0
def read_and_compress_table(parser, table, debug):
    """Read data from a FITS file and save it into a FITS binary table

    The data are read from the FITS file specified in the "table"
    section of the "parser" object (an instance of
    ConfigurationParser). If "debug" is true, save additional
    information (useful for debugging) in the table."""

    input_file_name = os.path.normpath(parser.get(table, 'file'))
    cur_hdu = None
    with pyfits.open(input_file_name) as input_file:
        hdu, column = get_hdu_and_column_from_schema(parser, table, input_file)
        compression = parser.get(table, 'compression')
        log.info(
            'compressing file %s (HDU %s, column %s) '
            'into table "%s", '
            'compression is "%s"', input_file_name, str(hdu), str(column),
            table, compression)

        samples_format = input_file[hdu].columns.formats[column]
        samples = ppc.to_native_endianness(input_file[hdu].data.field(column))
        if parser.has_option(table, 'datatype'):
            samples = np.array(samples, dtype=parser.get(table, 'datatype'))

        cur_hdu, num_of_bytes, elapsed_time = compress_to_FITS_table(
            parser, table, samples_format, samples, debug)
        cur_hdu.name = table
        cur_hdu.header['PCNUMSA'] = (len(samples),
                                     'Number of uncompressed samples')
        cur_hdu.header['PCCOMPR'] = (compression,
                                     'Polycomp compression algorithm')
        cur_hdu.header['PCSRCTP'] = (str(samples.dtype),
                                     'Original NumPy type of the data')
        cur_hdu.header['PCUNCSZ'] = (samples.itemsize * samples.size,
                                     'Size of the uncompressed data [bytes]')
        cur_hdu.header['PCCOMSZ'] = (num_of_bytes,
                                     'Size of the compressed data [bytes]')
        cur_hdu.header['PCTIME'] = (elapsed_time,
                                    'Time used for compression [s]')
        cr = float(cur_hdu.header['PCUNCSZ']) / float(
            cur_hdu.header['PCCOMSZ'])
        cur_hdu.header['PCCR'] = (cr, 'Compression ratio')
        log.info('table "%s" compressed, %s compressed to %s (cr: %.4f)',
                 table, humanize_size(cur_hdu.header['PCUNCSZ']),
                 humanize_size(cur_hdu.header['PCCOMSZ']), cr)

    return cur_hdu
Exemple #3
0
def decompress_quant(hdu):
    "Decompress an HDU containing quantized data"

    quant = ppc.QuantParams(element_size=hdu.header['PCELEMSZ'],
                            bits_per_sample=hdu.header['PCBITSPS'])
    quant.set_normalization(normalization=hdu.header['PCNORM'],
                            offset=hdu.header['PCOFS'])
    size_to_fits_fmt = {4: '1E', 8: '1D'}
    compr_samples = ppc.to_native_endianness(hdu.data.field(0))

    try:
        return (quant.decompress(compr_samples,
                                 hdu.header['PCNUMSA']),
                size_to_fits_fmt[quant.element_size()])
    except KeyError:
        log.error('unable to handle floating-point types which '
                  'are %d bytes wide, allowed sizes are %s',
                  quant.element_size(), str(list(size_to_fits_fmt.keys())))
        sys.exit(1)
Exemple #4
0
def decompress_quant(hdu):
    "Decompress an HDU containing quantized data"

    quant = ppc.QuantParams(element_size=hdu.header['PCELEMSZ'],
                            bits_per_sample=hdu.header['PCBITSPS'])
    quant.set_normalization(normalization=hdu.header['PCNORM'],
                            offset=hdu.header['PCOFS'])
    size_to_fits_fmt = {4: '1E', 8: '1D'}
    compr_samples = ppc.to_native_endianness(hdu.data.field(0))

    try:
        return (quant.decompress(compr_samples, hdu.header['PCNUMSA']),
                size_to_fits_fmt[quant.element_size()])
    except KeyError:
        log.error(
            'unable to handle floating-point types which '
            'are %d bytes wide, allowed sizes are %s', quant.element_size(),
            str(list(size_to_fits_fmt.keys())))
        sys.exit(1)