def _extract_field(self):
# FF table pointer initialisation based on FF LOOKUP table
# configuration.
lookup_table = self._ff_header.lookup_table
table_index, table_entry_depth, table_count = lookup_table
table_offset = (table_index - 1) * self._word_depth # in bytes
table_entry_depth = table_entry_depth * self._word_depth # in bytes
# Open the FF for processing.
with open(self._ff_header.ff_filename, 'rb') as ff_file:
ff_file_seek = ff_file.seek
is_boundary_packed = self._ff_header.dataset_type == 5
grid = self._ff_header.grid()
# Process each FF LOOKUP table entry.
while table_count:
table_count -= 1
# Move file pointer to the start of the current FF LOOKUP
# table entry.
ff_file_seek(table_offset, os.SEEK_SET)
# Read the current PP header entry from the FF LOOKUP table.
header_longs = np.fromfile(
ff_file, dtype='>i{0}'.format(self._word_depth),
count=pp.NUM_LONG_HEADERS)
# Check whether the current FF LOOKUP table entry is valid.
if header_longs[0] == _FF_LOOKUP_TABLE_TERMINATE:
# There are no more FF LOOKUP table entries to read.
break
header_floats = np.fromfile(
ff_file, dtype='>f{0}'.format(self._word_depth),
count=pp.NUM_FLOAT_HEADERS)
header = tuple(header_longs) + tuple(header_floats)
# Calculate next FF LOOKUP table entry.
table_offset += table_entry_depth
# Construct a PPField object and populate using the header_data
# read from the current FF LOOKUP table.
# (The PPField sub-class will depend on the header release
# number.)
# (Some Fields File fields are UM specific scratch spaces, with
# no header release number, these will throw an exception from
# the PP module and are skipped to enable reading of the file.
try:
field = pp.make_pp_field(header)
# Fast stash look-up.
stash_s = field.lbuser[3] // 1000
stash_i = field.lbuser[3] % 1000
stash = 'm{:02}s{:02}i{:03}'.format(field.lbuser[6],
stash_s, stash_i)
stash_entry = STASH_TRANS.get(stash, None)
if stash_entry is None:
subgrid = None
else:
subgrid = stash_entry.grid_code
if subgrid not in HANDLED_GRIDS:
warnings.warn('The stash code {} is on a grid {} '
'which has not been explicitly '
'handled by the fieldsfile loader.'
' Assuming the data is on a P grid'
'.'.format(stash, subgrid))
field.x, field.y = grid.vectors(subgrid)
# Use the per-file grid if no per-field metadata is
# available.
no_x = field.bzx in (0, field.bmdi) and field.x is None
no_y = field.bzy in (0, field.bmdi) and field.y is None
if no_x and no_y:
if subgrid is None:
msg = ('The STASH code {0} was not found in the '
'STASH to grid type mapping. Picking the P '
'position as the cell type'.format(stash))
warnings.warn(msg)
field.bzx, field.bdx = grid.regular_x(subgrid)
field.bzy, field.bdy = grid.regular_y(subgrid)
field.bplat = grid.pole_lat
field.bplon = grid.pole_lon
elif no_x or no_y:
warnings.warn(
'Partially missing X or Y coordinate values.')
# Check for LBC fields.
is_boundary_packed = self._ff_header.dataset_type == 5
if is_boundary_packed:
# Apply adjustments specific to LBC data.
self._adjust_field_for_lbc(field)
# Calculate start address of the associated PP header data.
data_offset = field.lbegin * self._word_depth
# Determine PP field payload depth and type.
data_depth, data_type = self._payload(field)
# Produce (yield) output fields.
if is_boundary_packed:
fields = self._fields_over_all_levels(field)
else:
fields = [field]
for result_field in fields:
# Add a field data element.
if self._read_data:
# Read the actual bytes. This can then be converted
# to a numpy array at a higher level.
ff_file_seek(data_offset, os.SEEK_SET)
result_field.data = pp.LoadedArrayBytes(
ff_file.read(data_depth), data_type)
else:
# Provide enough context to read the data bytes
# later.
result_field.data = (self._filename, data_offset,
data_depth, data_type)
data_offset += data_depth
yield result_field
except ValueError as valerr:
msg = ('Input field skipped as PPField creation failed :'
' error = {!r}')
warnings.warn(msg.format(str(valerr)))
def _extract_field(self):
# FF table pointer initialisation based on FF LOOKUP table
# configuration.
lookup_table = self._ff_header.lookup_table
table_index, table_entry_depth, table_count = lookup_table
table_offset = (table_index - 1) * self._word_depth # in bytes
table_entry_depth = table_entry_depth * self._word_depth # in bytes
# Open the FF for processing.
with open(self._ff_header.ff_filename, 'rb') as ff_file:
ff_file_seek = ff_file.seek
is_boundary_packed = self._ff_header.dataset_type == 5
grid = self._ff_header.grid()
# Process each FF LOOKUP table entry.
while table_count:
table_count -= 1
# Move file pointer to the start of the current FF LOOKUP
# table entry.
ff_file_seek(table_offset, os.SEEK_SET)
# Read the current PP header entry from the FF LOOKUP table.
header_longs = np.fromfile(
ff_file, dtype='>i{0}'.format(self._word_depth),
count=pp.NUM_LONG_HEADERS)
# Check whether the current FF LOOKUP table entry is valid.
if header_longs[0] == _FF_LOOKUP_TABLE_TERMINATE:
# There are no more FF LOOKUP table entries to read.
break
header_floats = np.fromfile(
ff_file, dtype='>f{0}'.format(self._word_depth),
count=pp.NUM_FLOAT_HEADERS)
header = tuple(header_longs) + tuple(header_floats)
# Calculate next FF LOOKUP table entry.
table_offset += table_entry_depth
# Construct a PPField object and populate using the header_data
# read from the current FF LOOKUP table.
# (The PPField sub-class will depend on the header release
# number.)
# (Some Fields File fields are UM specific scratch spaces, with
# no header release number, these will throw an exception from
# the PP module and are skipped to enable reading of the file.
try:
field = pp.make_pp_field(header)
# Fast stash look-up.
stash_s = field.lbuser[3] // 1000
stash_i = field.lbuser[3] % 1000
stash = 'm{:02}s{:02}i{:03}'.format(field.lbuser[6],
stash_s, stash_i)
stash_entry = STASH_TRANS.get(stash, None)
if stash_entry is None:
subgrid = None
else:
subgrid = stash_entry.grid_code
if subgrid not in HANDLED_GRIDS:
warnings.warn('The stash code {} is on a grid {} '
'which has not been explicitly '
'handled by the fieldsfile loader.'
' Assuming the data is on a P grid'
'.'.format(stash, subgrid))
field.x, field.y = grid.vectors(subgrid)
# Use the per-file grid if no per-field metadata is
# available.
no_x = field.bzx in (0, field.bmdi) and field.x is None
no_y = field.bzy in (0, field.bmdi) and field.y is None
if no_x and no_y:
if subgrid is None:
msg = ('The STASH code {0} was not found in the '
'STASH to grid type mapping. Picking the P '
'position as the cell type'.format(stash))
warnings.warn(msg)
field.bzx, field.bdx = grid.regular_x(subgrid)
field.bzy, field.bdy = grid.regular_y(subgrid)
field.bplat = grid.pole_lat
field.bplon = grid.pole_lon
elif no_x or no_y:
warnings.warn(
'Partially missing X or Y coordinate values.')
# Check for LBC fields.
is_boundary_packed = self._ff_header.dataset_type == 5
if is_boundary_packed:
# Apply adjustments specific to LBC data.
self._adjust_field_for_lbc(field)
# Calculate start address of the associated PP header data.
data_offset = field.lbegin * self._word_depth
# Determine PP field payload depth and type.
data_depth, data_type = self._payload(field)
# Produce (yield) output fields.
if is_boundary_packed:
fields = self._fields_over_all_levels(field)
else:
fields = [field]
for result_field in fields:
# Add a field data element.
if self._read_data:
# Read the actual bytes. This can then be converted
# to a numpy array at a higher level.
ff_file_seek(data_offset, os.SEEK_SET)
result_field.data = pp.LoadedArrayBytes(
ff_file.read(data_depth), data_type)
else:
# Provide enough context to read the data bytes
# later.
result_field.data = (self._filename, data_offset,
data_depth, data_type)
data_offset += data_depth
yield result_field
except ValueError as valerr:
msg = ('Input field skipped as PPField creation failed :'
' error = {!r}')
warnings.warn(msg.format(str(valerr)))
def _extract_field(self):
# FF table pointer initialisation based on FF LOOKUP table
# configuration.
lookup_table = self._ff_header.lookup_table
table_index, table_entry_depth, table_count = lookup_table
table_offset = (table_index - 1) * self._word_depth # in bytes
table_entry_depth = table_entry_depth * self._word_depth # in bytes
# Open the FF for processing.
ff_file = open(self._ff_header.ff_filename, 'rb')
ff_file_seek = ff_file.seek
# Check for an instantaneous dump.
if self._ff_header.dataset_type == 1:
table_count = self._ff_header.total_prognostic_fields
is_boundary_packed = self._ff_header.dataset_type == 5
grid = self._ff_header.grid()
# Process each FF LOOKUP table entry.
while table_count:
table_count -= 1
# Move file pointer to the start of the current FF LOOKUP
# table entry.
ff_file_seek(table_offset, os.SEEK_SET)
# Read the current PP header entry from the FF LOOKUP table.
header_longs = np.fromfile(
ff_file, dtype='>i{0}'.format(self._word_depth),
count=pp.NUM_LONG_HEADERS)
# Check whether the current FF LOOKUP table entry is valid.
if header_longs[0] == _FF_LOOKUP_TABLE_TERMINATE:
# There are no more FF LOOKUP table entries to read.
break
header_floats = np.fromfile(
ff_file, dtype='>f{0}'.format(self._word_depth),
count=pp.NUM_FLOAT_HEADERS)
header = tuple(header_longs) + tuple(header_floats)
# Calculate next FF LOOKUP table entry.
table_offset += table_entry_depth
# Construct a PPField object and populate using the header_data
# read from the current FF LOOKUP table.
# (The PPField sub-class will depend on the header release number.)
field = pp.make_pp_field(header)
# Calculate start address of the associated PP header data.
data_offset = field.lbegin * self._word_depth
# Determine PP field payload depth and type.
data_depth, data_type = self._payload(field)
# Fast stash look-up.
stash_s = field.lbuser[3] / 1000
stash_i = field.lbuser[3] % 1000
stash = 'm{:02}s{:02}i{:03}'.format(field.lbuser[6],
stash_s, stash_i)
stash_entry = STASH_TRANS.get(stash, None)
if stash_entry is None:
subgrid = None
warnings.warn('The STASH code {0} was not found in the '
'STASH to grid type mapping. Picking the P '
'position as the cell type'.format(stash))
else:
subgrid = stash_entry.grid_code
if subgrid not in HANDLED_GRIDS:
warnings.warn('The stash code {} is on a grid {} which '
'has not been explicitly handled by the '
'fieldsfile loader. Assuming the data is on '
'a P grid.'.format(stash, subgrid))
field.x, field.y = grid.vectors(subgrid)
# Use the per-file grid if no per-field metadata is available.
no_x = field.bzx in (0, field.bmdi) and field.x is None
no_y = field.bzy in (0, field.bmdi) and field.y is None
if no_x and no_y:
field.bzx, field.bdx = grid.regular_x(subgrid)
field.bzy, field.bdy = grid.regular_y(subgrid)
field.bplat = grid.pole_lat
field.bplon = grid.pole_lon
elif no_x or no_y:
warnings.warn('Partially missing X or Y coordinate values.')
if is_boundary_packed:
name_mapping = dict(rim_width=slice(4, 6), y_halo=slice(2, 4),
x_halo=slice(0, 2))
b_packing = pp.SplittableInt(field.lbuser[2], name_mapping)
field.lbpack.boundary_packing = b_packing
# Fix the lbrow and lbnpt to be the actual size of the data
# array, since the field is no longer a "boundary" fields file
# field.
# Note: The documentation states that lbrow (y) doesn't
# contain the halo rows, but no such comment exists at UM v8.5
# for lbnpt (x). Experimentation has shown that lbnpt also
# excludes the halo size.
field.lbrow += 2 * field.lbpack.boundary_packing.y_halo
field.lbnpt += 2 * field.lbpack.boundary_packing.x_halo
# Update the x and y coordinates for this field. Note: it may
# be that this needs to update x and y also, but that is yet
# to be confirmed.
if (field.bdx in (0, field.bmdi) or
field.bdy in (0, field.bmdi)):
field.x = self._det_border(field.x, b_packing.x_halo)
field.y = self._det_border(field.y, b_packing.y_halo)
else:
if field.bdy < 0:
warnings.warn('The LBC has a bdy less than 0. No '
'case has previously been seen of '
'this, and the decompression may be '
'erroneous.')
field.bzx -= field.bdx * b_packing.x_halo
field.bzy -= field.bdy * b_packing.y_halo
if self._read_data:
# Read the actual bytes. This can then be converted to a
# numpy array at a higher level.
ff_file_seek(data_offset, os.SEEK_SET)
field._data = pp.LoadedArrayBytes(ff_file.read(data_depth),
data_type)
else:
# Provide enough context to read the data bytes later on.
field._data = (self._filename, data_offset,
data_depth, data_type)
yield field
ff_file.close()
def _extract_field(self):
# FF table pointer initialisation based on FF LOOKUP table
# configuration.
lookup_table = self._ff_header.lookup_table
table_index, table_entry_depth, table_count = lookup_table
table_offset = (table_index - 1) * self._word_depth # in bytes
table_entry_depth = table_entry_depth * self._word_depth # in bytes
# Open the FF for processing.
ff_file = open(self._ff_header.ff_filename, 'rb')
ff_file_seek = ff_file.seek
# Check for an instantaneous dump.
if self._ff_header.dataset_type == 1:
table_count = self._ff_header.total_prognostic_fields
is_boundary_packed = self._ff_header.dataset_type == 5
grid = self._ff_header.grid()
# Process each FF LOOKUP table entry.
while table_count:
table_count -= 1
# Move file pointer to the start of the current FF LOOKUP
# table entry.
ff_file_seek(table_offset, os.SEEK_SET)
# Read the current PP header entry from the FF LOOKUP table.
header_longs = np.fromfile(ff_file,
dtype='>i{0}'.format(self._word_depth),
count=pp.NUM_LONG_HEADERS)
# Check whether the current FF LOOKUP table entry is valid.
if header_longs[0] == _FF_LOOKUP_TABLE_TERMINATE:
# There are no more FF LOOKUP table entries to read.
break
header_floats = np.fromfile(ff_file,
dtype='>f{0}'.format(self._word_depth),
count=pp.NUM_FLOAT_HEADERS)
header = tuple(header_longs) + tuple(header_floats)
# Calculate next FF LOOKUP table entry.
table_offset += table_entry_depth
# Construct a PPField object and populate using the header_data
# read from the current FF LOOKUP table.
# (The PPField sub-class will depend on the header release number.)
field = pp.make_pp_field(header)
# Calculate start address of the associated PP header data.
data_offset = field.lbegin * self._word_depth
# Determine PP field payload depth and type.
data_depth, data_type = self._payload(field)
# Fast stash look-up.
stash_s = field.lbuser[3] / 1000
stash_i = field.lbuser[3] % 1000
stash = 'm{:02}s{:02}i{:03}'.format(field.lbuser[6], stash_s,
stash_i)
stash_entry = STASH_TRANS.get(stash, None)
if stash_entry is None:
subgrid = None
warnings.warn('The STASH code {0} was not found in the '
'STASH to grid type mapping. Picking the P '
'position as the cell type'.format(stash))
else:
subgrid = stash_entry.grid_code
if subgrid not in HANDLED_GRIDS:
warnings.warn('The stash code {} is on a grid {} which '
'has not been explicitly handled by the '
'fieldsfile loader. Assuming the data is on '
'a P grid.'.format(stash, subgrid))
field.x, field.y = grid.vectors(subgrid)
# Use the per-file grid if no per-field metadata is available.
no_x = field.bzx in (0, field.bmdi) and field.x is None
no_y = field.bzy in (0, field.bmdi) and field.y is None
if no_x and no_y:
field.bzx, field.bdx = grid.regular_x(subgrid)
field.bzy, field.bdy = grid.regular_y(subgrid)
field.bplat = grid.pole_lat
field.bplon = grid.pole_lon
elif no_x or no_y:
warnings.warn('Partially missing X or Y coordinate values.')
if is_boundary_packed:
name_mapping = dict(rim_width=slice(4, 6),
y_halo=slice(2, 4),
x_halo=slice(0, 2))
b_packing = pp.SplittableInt(field.lbuser[2], name_mapping)
field.lbpack.boundary_packing = b_packing
# Fix the lbrow and lbnpt to be the actual size of the data
# array, since the field is no longer a "boundary" fields file
# field.
# Note: The documentation states that lbrow (y) doesn't
# contain the halo rows, but no such comment exists at UM v8.5
# for lbnpt (x). Experimentation has shown that lbnpt also
# excludes the halo size.
field.lbrow += 2 * field.lbpack.boundary_packing.y_halo
field.lbnpt += 2 * field.lbpack.boundary_packing.x_halo
# Update the x and y coordinates for this field. Note: it may
# be that this needs to update x and y also, but that is yet
# to be confirmed.
if (field.bdx in (0, field.bmdi)
or field.bdy in (0, field.bmdi)):
field.x = self._det_border(field.x, b_packing.x_halo)
field.y = self._det_border(field.y, b_packing.y_halo)
else:
if field.bdy < 0:
warnings.warn('The LBC has a bdy less than 0. No '
'case has previously been seen of '
'this, and the decompression may be '
'erroneous.')
field.bzx -= field.bdx * b_packing.x_halo
field.bzy -= field.bdy * b_packing.y_halo
if self._read_data:
# Read the actual bytes. This can then be converted to a
# numpy array at a higher level.
ff_file_seek(data_offset, os.SEEK_SET)
field._data = pp.LoadedArrayBytes(ff_file.read(data_depth),
data_type)
else:
# Provide enough context to read the data bytes later on.
field._data = (self._filename, data_offset, data_depth,
data_type)
yield field
ff_file.close()
