def test_generic_masked_bad_min_max_value(self):
_, tpath = tempfile.mkstemp(suffix='.nc', prefix='pyaxiom-test')
shutil.copy2(self.input_file, tpath)
with EnhancedDataset(tpath, 'a') as ncd:
v = ncd.variables['v_component_wind_true_direction_all_geometries']
v.valid_min = 0.1
v.valid_max = 0.1
r = generic_masked(v[:], attrs=ncd.vatts(v.name))
rflat = r.flatten()
assert rflat[~rflat.mask].size == 0
# Create a byte variable with a float valid_min and valid_max
# to make sure it doesn't error
b = ncd.createVariable('imabyte', 'b')
b.valid_min = 0
b.valid_max = 600 # this ss over a byte and thus invalid
b[:] = 3
r = generic_masked(b[:], attrs=ncd.vatts(b.name))
assert np.all(r.mask == False) # noqa
b.valid_min = 0
b.valid_max = 2
r = generic_masked(b[:], attrs=ncd.vatts(b.name))
assert np.all(r.mask == True) # noqa
c = ncd.createVariable('imanotherbyte', 'f4')
c.setncattr('valid_min', '0b')
c.setncattr('valid_max', '9b')
c[:] = 3
r = generic_masked(c[:], attrs=ncd.vatts(c.name))
assert np.all(r.mask == False) # noqa
c = ncd.createVariable('imarange', 'f4')
c.valid_range = [0.0, 2.0]
c[:] = 3.0
r = generic_masked(c[:], attrs=ncd.vatts(c.name))
assert np.all(r.mask == True) # noqa
c.valid_range = [0.0, 2.0]
c[:] = 1.0
r = generic_masked(c[:], attrs=ncd.vatts(c.name))
assert np.all(r.mask == False) # noqa
if os.path.exists(tpath):
os.remove(tpath)
def to_dataframe(self, clean_cols=True, clean_rows=True):
zvar = self.z_axes()[0]
zs = len(self.dimensions[zvar.dimensions[0]])
# Profiles
pvar = self.get_variables_by_attributes(cf_role='profile_id')[0]
try:
p = normalize_array(pvar)
except ValueError:
p = np.asarray(list(range(len(pvar))), dtype=np.integer)
ps = p.size
p = p.repeat(zs)
logger.debug(['profile data size: ', p.size])
# Z
z = generic_masked(zvar[:], attrs=self.vatts(zvar.name)).round(5)
try:
z = np.tile(z, ps)
except ValueError:
z = z.flatten()
logger.debug(['z data size: ', z.size])
# T
tvar = self.t_axes()[0]
t = nc4.num2date(tvar[:], tvar.units,
getattr(tvar, 'calendar', 'standard'))
if isinstance(t, datetime):
# Size one
t = np.array([t.isoformat()], dtype='datetime64')
t = t.repeat(zs)
logger.debug(['time data size: ', t.size])
# X
xvar = self.x_axes()[0]
x = generic_masked(xvar[:].repeat(zs),
attrs=self.vatts(xvar.name)).round(5)
logger.debug(['x data size: ', x.size])
# Y
yvar = self.y_axes()[0]
y = generic_masked(yvar[:].repeat(zs),
attrs=self.vatts(yvar.name)).round(5)
logger.debug(['y data size: ', y.size])
# Distance
d = np.ma.zeros(y.size, dtype=np.float64)
d[1:] = great_distance(start_latitude=y[0:-1],
end_latitude=y[1:],
start_longitude=x[0:-1],
end_longitude=x[1:])['distance']
d = generic_masked(np.cumsum(d), minv=0).round(2)
logger.debug(['distance data size: ', d.size])
df_data = {'t': t, 'x': x, 'y': y, 'z': z, 'profile': p, 'distance': d}
building_index_to_drop = np.ones(t.size, dtype=bool)
extract_vars = list(set(self.data_vars() + self.ancillary_vars()))
for i, dvar in enumerate(extract_vars):
vdata = np.ma.fix_invalid(
np.ma.MaskedArray(dvar[:].round(3).flatten()))
building_index_to_drop = (building_index_to_drop == True) & (
vdata.mask == True) # noqa
df_data[dvar.name] = vdata
df = pd.DataFrame(df_data)
# Drop all data columns with no data
if clean_cols:
df = df.dropna(axis=1, how='all')
# Drop all data rows with no data variable data
if clean_rows:
df = df.iloc[~building_index_to_drop]
return df
def to_dataframe(self, clean_cols=True, clean_rows=True):
# The index variable (trajectory_index) is identified by having an
# attribute with name of instance_dimension whose value is the instance
# dimension name (trajectory in this example). The index variable must
# have the profile dimension as its sole dimension, and must be type
# integer. Each value in the index variable is the zero-based trajectory
# index that the profile belongs to i.e. profile p belongs to trajectory
# i=trajectory_index(p), as in section H.2.5.
r_index_var = self.get_variables_by_attributes(
instance_dimension=lambda x: x is not None)[0]
p_dim = self.dimensions[r_index_var.dimensions[0]] # Profile dimension
r_dim = self.dimensions[
r_index_var.instance_dimension] # Trajectory dimension
# The count variable (row_size) contains the number of elements for
# each profile, which must be written contiguously. The count variable
# is identified by having an attribute with name sample_dimension whose
# value is the sample dimension (obs in this example) being counted. It
# must have the profile dimension as its sole dimension, and must be
# type integer
o_index_var = self.get_variables_by_attributes(
sample_dimension=lambda x: x is not None)[0]
o_dim = self.dimensions[
o_index_var.sample_dimension] # Sample dimension
try:
rvar = self.get_variables_by_attributes(cf_role='trajectory_id')[0]
traj_indexes = normalize_array(rvar)
assert traj_indexes.size == r_dim.size
except BaseException:
logger.warning(
'Could not pull trajectory values a variable with "cf_role=trajectory_id", using a computed range.'
)
traj_indexes = np.arange(r_dim.size)
try:
pvar = self.get_variables_by_attributes(cf_role='profile_id')[0]
profile_indexes = normalize_array(pvar)
assert profile_indexes.size == p_dim.size
except BaseException:
logger.warning(
'Could not pull profile values from a variable with "cf_role=profile_id", using a computed range.'
)
profile_indexes = np.arange(p_dim.size)
# Profile dimension
tvars = self.t_axes()
if len(tvars) > 1:
tvar = [
v for v in self.t_axes() if v.dimensions == (
p_dim.name, ) and getattr(v, 'axis', '').lower() == 't'
][0]
else:
tvar = tvars[0]
xvars = self.x_axes()
if len(xvars) > 1:
xvar = [
v for v in self.x_axes() if v.dimensions == (
p_dim.name, ) and getattr(v, 'axis', '').lower() == 'x'
][0]
else:
xvar = xvars[0]
yvars = self.y_axes()
if len(yvars) > 1:
yvar = [
v for v in self.y_axes() if v.dimensions == (
p_dim.name, ) and getattr(v, 'axis', '').lower() == 'y'
][0]
else:
yvar = yvars[0]
zvars = self.z_axes()
if len(zvars) > 1:
zvar = [
v for v in self.z_axes() if v.dimensions == (
o_dim.name, ) and getattr(v, 'axis', '').lower() == 'z'
][0]
else:
zvar = zvars[0]
p = np.ma.masked_all(o_dim.size, dtype=profile_indexes.dtype)
r = np.ma.masked_all(o_dim.size, dtype=traj_indexes.dtype)
t = np.ma.masked_all(o_dim.size, dtype=tvar.dtype)
x = np.ma.masked_all(o_dim.size, dtype=xvar.dtype)
y = np.ma.masked_all(o_dim.size, dtype=yvar.dtype)
si = 0
for i in np.arange(profile_indexes.size):
ei = si + o_index_var[i]
p[si:ei] = profile_indexes[i]
r[si:ei] = traj_indexes[r_index_var[i]]
t[si:ei] = tvar[i]
x[si:ei] = xvar[i]
y[si:ei] = yvar[i]
si = ei
t_mask = False
tfill = get_fill_value(tvar)
if tfill is not None:
t_mask = np.copy(np.ma.getmaskarray(t))
t[t_mask] = 1
t = np.ma.MaskedArray(
nc4.num2date(t, tvar.units, getattr(tvar, 'calendar', 'standard')))
# Patch the time variable back to its original mask, since num2date
# breaks any missing/fill values
t[t_mask] = np.ma.masked
# X and Y
x = generic_masked(x, minv=-180, maxv=180).round(5)
y = generic_masked(y, minv=-90, maxv=90).round(5)
# Distance
d = np.ma.zeros(o_dim.size, dtype=np.float64)
d[1:] = great_distance(start_latitude=y[0:-1],
end_latitude=y[1:],
start_longitude=x[0:-1],
end_longitude=x[1:])['distance']
d = generic_masked(np.cumsum(d), minv=0).round(2)
# Sample dimension
z = generic_masked(zvar[:].flatten(),
attrs=self.vatts(zvar.name)).round(5)
df_data = {
't': t,
'x': x,
'y': y,
'z': z,
'trajectory': r,
'profile': p,
'distance': d
}
building_index_to_drop = np.ones(o_dim.size, dtype=bool)
extract_vars = list(set(self.data_vars() + self.ancillary_vars()))
for i, dvar in enumerate(extract_vars):
# Profile dimensions
if dvar.dimensions == (p_dim.name, ):
vdata = np.ma.masked_all(o_dim.size, dtype=dvar.dtype)
si = 0
for j in np.arange(profile_indexes.size):
ei = si + o_index_var[j]
vdata[si:ei] = dvar[j]
si = ei
# Sample dimensions
elif dvar.dimensions == (o_dim.name, ):
vdata = generic_masked(dvar[:].flatten(),
attrs=self.vatts(dvar.name)).round(3)
else:
logger.warning(
"Skipping variable {}... it didn't seem like a data variable"
.format(dvar))
building_index_to_drop = (building_index_to_drop == True) & (
vdata.mask == True) # noqa
df_data[dvar.name] = vdata
df = pd.DataFrame(df_data)
# Drop all data columns with no data
if clean_cols:
df = df.dropna(axis=1, how='all')
# Drop all data rows with no data variable data
if clean_rows:
df = df.iloc[~building_index_to_drop]
return df
