Example #1
0
def export_product(product, filename, file_format="netcdf"):
    """Export a HARP compliant product.

    Arguments:
    product     -- Product to export.
    filename    -- Filename of the exported product.
    file_format -- File format to use; one of 'netcdf', 'hdf4', or 'hdf5'.

    """
    if not isinstance(product, Product):
        raise TypeError("product must be Product, not %r" %
                        product.__class__.__name__)

    # Create C product.
    c_product_ptr = _ffi.new("harp_product **")
    if _lib.harp_product_new(c_product_ptr) != 0:
        raise CLibraryError()

    try:
        # Convert the Python product to its C representation.
        _export_product(product, c_product_ptr[0])

        # Export the C product to a file.
        if _lib.harp_export(_encode_path(filename),
                            _encode_string(file_format),
                            c_product_ptr[0]) != 0:
            raise CLibraryError()

    finally:
        _lib.harp_product_delete(c_product_ptr[0])
Example #2
0
def export_product(product, filename, file_format="netcdf"):
    """Export a HARP compliant product.

    Arguments:
    product     -- Product to export.
    filename    -- Filename of the exported product.
    file_format -- File format to use; one of 'netcdf', 'hdf4', or 'hdf5'.

    """
    if not isinstance(product, Product):
        raise TypeError("product must be Product, not %r" % product.__class__.__name__)

    # Create C product.
    c_product_ptr = _ffi.new("harp_product **")
    if _lib.harp_product_new(c_product_ptr) != 0:
        raise CLibraryError()

    try:
        # Convert the Python product to its C representation.
        _export_product(product, c_product_ptr[0])

        # Export the C product to a file.
        if _lib.harp_export(_encode_path(filename), _encode_string(file_format), c_product_ptr[0]) != 0:
            raise CLibraryError()

    finally:
        _lib.harp_product_delete(c_product_ptr[0])
Example #3
0
def import_product(filename, operations="", options=""):
    """Import a product from a file.

    This will first try to import the file as an HDF4, HDF5, or netCDF file that
    complies to the HARP Data Format. If the file is not stored using the HARP
    format then it will try to import it using one of the available ingestion
    modules.

    If the filename argument is a list of filenames or a globbing (glob.glob())
    pattern then the harp.import_product() function will be called on each
    individual file and the result of harp.concatenate() on the imported products
    will be returned.

    Arguments:
    filename -- Filename, list of filenames or file pattern of the product(s) to import
    operations -- Actions to apply as part of the import; should be specified as a
                  semi-colon separated string of operations.
    options -- Ingestion module specific options; should be specified as a semi-
               colon separated string of key=value pairs; only used if the file is not
               in HARP format.

    """
    if not (isinstance(filename, bytes) or isinstance(filename, str)):
        # Assume this is a list of filenames or patterns
        return concatenate(
            [import_product(file, operations, options) for file in filename])
    filename = _encode_path(filename)
    if '*' in filename or '?' in filename:
        # This is a globbing pattern
        filenames = sorted(glob.glob(filename))
        if len(filenames) == 0:
            raise Error("no files matching '%s'" % (filename))
        return concatenate(
            [import_product(file, operations, options) for file in filenames])

    c_product_ptr = _ffi.new("harp_product **")

    # Import the product as a C product.
    if _lib.harp_import(_encode_path(filename), _encode_string(operations),
                        _encode_string(options), c_product_ptr) != 0:
        raise CLibraryError()

    try:
        # Raise an exception if the imported C product contains no variables, or variables without data.
        if _lib.harp_product_is_empty(c_product_ptr[0]) == 1:
            raise NoDataError()

        # Convert the C product into its Python representation.
        return _import_product(c_product_ptr[0])

    finally:
        _lib.harp_product_delete(c_product_ptr[0])
Example #4
0
def export_product(product,
                   filename,
                   file_format="netcdf",
                   operations="",
                   hdf5_compression=0):
    """Export a HARP compliant product.

    Arguments:
    product          -- Product to export.
    filename         -- Filename of the exported product.
    file_format      -- File format to use; one of 'netcdf', 'hdf4', or 'hdf5'.
    operations       -- Actions to apply as part of the export; should be specified as a
                        semi-colon separated string of operations.
    hdf5_compression -- Compression level when exporting to hdf5 (0=disabled, 1=low, ..., 9=high).

    """
    if not isinstance(product, Product):
        raise TypeError("product must be Product, not %r" %
                        product.__class__.__name__)

    if operations:
        # Update history (but only if the export modifies the product)
        command = "harp.export_product('{0}', operations='{1}')".format(
            filename, operations)
        _update_history(product, command)

    # Create C product.
    c_product_ptr = _ffi.new("harp_product **")
    if _lib.harp_product_new(c_product_ptr) != 0:
        raise CLibraryError()

    try:
        # Convert the Python product to its C representation.
        _export_product(product, c_product_ptr[0])

        if operations:
            # Apply operations to the product before export
            _lib.harp_product_execute_operations(c_product_ptr[0],
                                                 _encode_string(operations))

        # Export the C product to a file.
        if file_format == 'hdf5':
            _lib.harp_set_option_hdf5_compression(int(hdf5_compression))
        if _lib.harp_export(_encode_path(filename),
                            _encode_string(file_format),
                            c_product_ptr[0]) != 0:
            raise CLibraryError()

    finally:
        _lib.harp_product_delete(c_product_ptr[0])
Example #5
0
def import_product(filename, operations=""):
    """Import a HARP compliant product.

    The file format (NetCDF/HDF4/HDF5) of the product will be auto-detected.

    If the filename argument is a list of filenames or a globbing (glob.glob()) pattern
    then the harp.import_product() function will be called on each individual file and
    the result of harp.concatenate() on the imported products will be returned.

    Arguments:
    filename -- Filename, list of filenames or file pattern of the product(s) to import
    operations  -- Actions to execute on the product after it has been imported; should
                be specified as a semi-colon separated string of operations.

    """
    if not (isinstance(filename, bytes) or isinstance(filename, str)):
        # Assume this is a list of filenames or patterns
        return concatenate([import_product(file, operations) for file in filename])
    filename = _encode_path(filename)
    if '*' in filename or '?' in filename:
        # This is a globbing pattern
        filenames = sorted(glob.glob(filename))
        if len(filenames) == 0:
            raise Error("no files matching '%s'" % (filename))
        return concatenate([import_product(file, operations) for file in filenames])

    c_product_ptr = _ffi.new("harp_product **")

    # Import the product as a C product.
    if _lib.harp_import(_encode_path(filename), c_product_ptr) != 0:
        raise CLibraryError()

    try:
        # Execute operation list expression on the C product.
        if operations and _lib.harp_product_execute_operations(c_product_ptr[0], _encode_string(operations)) != 0:
            raise CLibraryError()

        # Raise an exception if the imported C product contains no variables, or variables without data.
        if _lib.harp_product_is_empty(c_product_ptr[0]) == 1:
            raise NoDataError()

        # Convert the C product into its Python representation.
        return _import_product(c_product_ptr[0])

    finally:
        _lib.harp_product_delete(c_product_ptr[0])
Example #6
0
def ingest_product(filename, operations="", options=""):
    """Ingest a product of a type supported by HARP.

    If the filename argument is a list of filenames or a globbing (glob.glob()) pattern
    then the harp.ingest_product() function will be called on each individual file and
    the result of harp.concatenate() on the ingested products will be returned.

    Arguments:
    filename -- Filename, list of filenames or file pattern of the product(s) to ingest
    operations  -- Actions to execute as part of the ingestion; should be specified as a
                semi-colon separated string of operations.
    options  -- Ingestion module specific options; should be specified as a semi-
                colon separated string of key=value pairs.

    """
    if not (isinstance(filename, bytes) or isinstance(filename, str)):
        # Assume this is a list of filenames or patterns
        return concatenate([ingest_product(file, operations, options) for file in filename])
    filename = _encode_path(filename)
    if '*' in filename or '?' in filename:
        # This is a globbing pattern
        filenames = sorted(glob.glob(filename))
        if len(filenames) == 0:
            raise Error("no files matching '%s'" % (filename))
        return concatenate([ingest_product(file, operations, options) for file in filenames])

    c_product_ptr = _ffi.new("harp_product **")

    # Ingest the product as a C product.
    if _lib.harp_ingest(_encode_path(filename), _encode_string(operations), _encode_string(options), c_product_ptr) != 0:
        raise CLibraryError()

    try:
        # Raise an exception if the ingested C product contains no variables, or variables without data.
        if _lib.harp_product_is_empty(c_product_ptr[0]) == 1:
            raise NoDataError()

        # Convert the C product into its Python representation.
        return _import_product(c_product_ptr[0])

    finally:
        _lib.harp_product_delete(c_product_ptr[0])
Example #7
0
def _export_variable(name, variable, c_product):
    data = getattr(variable, "data", None)
    if data is None:
        raise Error("no data or data is None")

    # Check dimensions
    dimension = getattr(variable, "dimension", [])
    if not dimension and isinstance(data, numpy.ndarray) and data.size != 1:
        raise Error("dimensions missing or incomplete")

    if dimension and (not isinstance(data, numpy.ndarray)
                      or data.ndim != len(dimension)):
        raise Error("dimensions incorrect")

    # Determine C data type.
    c_data_type = _get_c_data_type(data)
    if len(dimension) == 0:
        # Allow valid_min/valid_max to influence data type as well
        try:
            min_data_type = _get_c_data_type(variable.valid_min)
            if min_data_type != c_data_type:
                c_data_type = min_data_type
        except:
            pass
        try:
            max_data_type = _get_c_data_type(variable.valid_max)
            if max_data_type != c_data_type:
                c_data_type = max_data_type
        except:
            pass

    # Encode variable name.
    c_name = _encode_string(name)

    # Determine C dimension types and lengths.
    c_num_dimensions = len(dimension)
    c_dimension_type = [
        _get_c_dimension_type(dimension_name) for dimension_name in dimension
    ]
    c_dimension = _ffi.NULL if not dimension else data.shape

    # Create C variable of the proper size.
    c_variable_ptr = _ffi.new("harp_variable **")
    if _lib.harp_variable_new(c_name, c_data_type, c_num_dimensions,
                              c_dimension_type, c_dimension,
                              c_variable_ptr) != 0:
        raise CLibraryError()

    # Add C variable to C product.
    if _lib.harp_product_add_variable(c_product, c_variable_ptr[0]) != 0:
        _lib.harp_variable_delete(c_variable_ptr[0])
        raise CLibraryError()

    # The C variable has been successfully added to the C product. Therefore, the memory management of the C variable is
    # tied to the life time of the C product. If an error occurs, the memory occupied by the C variable will be freed
    # along with the C product.
    c_variable = c_variable_ptr[0]

    # Copy data into the C variable.
    _export_array(data, c_variable)

    # Variable attributes.
    if c_data_type != _lib.harp_type_string:
        try:
            valid_min = variable.valid_min
        except AttributeError:
            pass
        else:
            if isinstance(valid_min, numpy.ndarray):
                if valid_min.size == 1:
                    valid_min = valid_min.flat[0]
                else:
                    raise Error("valid_min attribute should be scalar")

            c_data_type_valid_min = _get_c_data_type(valid_min)
            if _c_can_cast(c_data_type_valid_min, c_data_type):
                _export_scalar(valid_min, c_data_type, c_variable.valid_min)
            else:
                raise Error(
                    "type '%s' of valid_min attribute incompatible with type '%s' of data"
                    % (_get_c_data_type_name(c_data_type_valid_min),
                       _get_c_data_type_name(c_data_type)))

        try:
            valid_max = variable.valid_max
        except AttributeError:
            pass
        else:
            if isinstance(valid_max, numpy.ndarray):
                if valid_max.size == 1:
                    valid_max = valid_max.flat[0]
                else:
                    raise Error("valid_max attribute should be scalar")

            c_data_type_valid_max = _get_c_data_type(valid_max)
            if _c_can_cast(c_data_type_valid_max, c_data_type):
                _export_scalar(valid_max, c_data_type, c_variable.valid_max)
            else:
                raise Error(
                    "type '%s' of valid_max attribute incompatible with type '%s' of data"
                    % (_get_c_data_type_name(c_data_type_valid_max),
                       _get_c_data_type_name(c_data_type)))

    try:
        unit = variable.unit
    except AttributeError:
        pass
    else:
        if unit is not None and _lib.harp_variable_set_unit(
                c_variable, _encode_string(unit)) != 0:
            raise CLibraryError()

    try:
        description = variable.description
    except AttributeError:
        pass
    else:
        if description and _lib.harp_variable_set_description(
                c_variable, _encode_string(description)) != 0:
            raise CLibraryError()

    try:
        enum = variable.enum
    except AttributeError:
        pass
    else:
        if enum and _lib.harp_variable_set_enumeration_values(
                c_variable, len(enum),
            [_ffi.new("char[]", _encode_string(name)) for name in enum]) != 0:
            raise CLibraryError()
Example #8
0
def _export_variable(name, variable, c_product):
    data = getattr(variable, "data", None)
    if data is None:
        raise Error("no data or data is None")

    # Determine C data type.
    c_data_type = _get_c_data_type(data)

    # Check dimensions
    dimension = getattr(variable, "dimension", [])
    if not dimension and isinstance(data, numpy.ndarray) and data.size != 1:
        raise Error("dimensions missing or incomplete")

    if dimension and (not isinstance(data, numpy.ndarray) or data.ndim != len(dimension)):
        raise Error("dimensions incorrect")

    # Encode variable name.
    c_name = _encode_string(name)

    # Determine C dimension types and lengths.
    c_num_dimensions = len(dimension)
    c_dimension_type = [_get_c_dimension_type(dimension_name) for dimension_name in dimension]
    c_dimension = _ffi.NULL if not dimension else data.shape

    # Create C variable of the proper size.
    c_variable_ptr = _ffi.new("harp_variable **")
    if _lib.harp_variable_new(c_name, c_data_type, c_num_dimensions, c_dimension_type, c_dimension,
                              c_variable_ptr) != 0:
        raise CLibraryError()

    # Add C variable to C product.
    if _lib.harp_product_add_variable(c_product, c_variable_ptr[0]) != 0:
        _lib.harp_variable_delete(c_variable_ptr[0])
        raise CLibraryError()

    # The C variable has been successfully added to the C product. Therefore, the memory management of the C variable is
    # tied to the life time of the C product. If an error occurs, the memory occupied by the C variable will be freed
    # along with the C product.
    c_variable = c_variable_ptr[0]

    # Copy data into the C variable.
    _export_array(data, c_variable)

    # Variable attributes.
    if c_data_type != _lib.harp_type_string:
        try:
            valid_min = variable.valid_min
        except AttributeError:
            pass
        else:
            if isinstance(valid_min, numpy.ndarray):
                if valid_min.size == 1:
                    valid_min = valid_min.flat[0]
                else:
                    raise Error("valid_min attribute should be scalar")

            c_data_type_valid_min = _get_c_data_type(valid_min)
            if _c_can_cast(c_data_type_valid_min, c_data_type):
                _export_scalar(valid_min, c_data_type, c_variable.valid_min)
            else:
                raise Error("type '%s' of valid_min attribute incompatible with type '%s' of data"
                            % (_get_c_data_type_name(c_data_type_valid_min), _get_c_data_type_name(c_data_type)))

        try:
            valid_max = variable.valid_max
        except AttributeError:
            pass
        else:
            if isinstance(valid_max, numpy.ndarray):
                if valid_max.size == 1:
                    valid_max = valid_max.flat[0]
                else:
                    raise Error("valid_max attribute should be scalar")

            c_data_type_valid_max = _get_c_data_type(valid_max)
            if _c_can_cast(c_data_type_valid_max, c_data_type):
                _export_scalar(valid_max, c_data_type, c_variable.valid_max)
            else:
                raise Error("type '%s' of valid_max attribute incompatible with type '%s' of data"
                            % (_get_c_data_type_name(c_data_type_valid_max), _get_c_data_type_name(c_data_type)))

    try:
        unit = variable.unit
    except AttributeError:
        pass
    else:
        if unit is not None and _lib.harp_variable_set_unit(c_variable, _encode_string(unit)) != 0:
            raise CLibraryError()

    try:
        description = variable.description
    except AttributeError:
        pass
    else:
        if description and _lib.harp_variable_set_description(c_variable, _encode_string(description)) != 0:
            raise CLibraryError()