def add(self, tspan, vspan, fspan=None, zspan=None, period=None):
assert isfixedlength(tspan, 2)
# If period is defined, tspan is a numeric
# if it isn't defined, its a parsable date
if period is not None:
tspan = span(*sorted(tspan))
else:
tspan = span(
*sorted([pd.Timestamp(tspan[0]),
pd.Timestamp(tspan[1])]))
assert isfixedlength(vspan, 2)
vspan = span(*sorted(vspan))
if fspan is not None:
assert isfixedlength(fspan, 2)
fspan = span(*sorted(fspan))
if zspan is not None:
assert isfixedlength(zspan, 2)
zspan = span(*sorted(zspan))
if period is not None:
# Test to make sure this is callable on a Timestamp
try:
getattr(pd.Timestamp.now(), period)
except AttributeError:
raise ValueError('The period "{period}" is not recognized')
self._members.append(self.mem(tspan, fspan, vspan, zspan, period))
def gross_range_test(
inp: Sequence[N],
fail_span: Tuple[N, N],
suspect_span: Tuple[N, N] = None) -> np.ma.core.MaskedArray:
"""Checks that values are within reasonable range bounds.
Given a 2-tuple of minimum/maximum values, flag data outside of the given
range as FAIL data. Optionally also flag data which falls outside of a user
defined range as SUSPECT. Missing and masked data is flagged as UNKNOWN.
Args:
inp: Input data as a numeric numpy array or a list of numbers.
fail_span: 2-tuple range which to flag outside data as FAIL.
suspect_span: 2-tuple range which to flag outside data as SUSPECT. [optional]
Returns:
A masked array of flag values equal in size to that of the input.
"""
assert isfixedlength(fail_span, 2)
sspan = span(*sorted(fail_span))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
inp = np.ma.masked_invalid(np.array(inp).astype(np.float64))
# Save original shape
original_shape = inp.shape
inp = inp.flatten()
# Start with everything as passing (1)
flag_arr = np.ma.ones(inp.size, dtype='uint8')
# If the value is masked set the flag to MISSING
flag_arr[inp.mask] = QartodFlags.MISSING
if suspect_span is not None:
assert isfixedlength(suspect_span, 2)
uspan = span(*sorted(suspect_span))
if uspan.minv < sspan.minv or uspan.maxv > sspan.maxv:
raise ValueError('Suspect {} must fall within the Fail {}'.format(
uspan, sspan))
# Flag suspect outside of user span
with np.errstate(invalid='ignore'):
flag_arr[(inp < uspan.minv) |
(inp > uspan.maxv)] = QartodFlags.SUSPECT
# Flag suspect outside of sensor span
with np.errstate(invalid='ignore'):
flag_arr[(inp < sspan.minv) | (inp > sspan.maxv)] = QartodFlags.FAIL
return flag_arr.reshape(original_shape)
def add(self,
tspan: Tuple[N, N],
vspan: Tuple[N, N],
zspan: Tuple[N, N] = None) -> None:
assert isfixedlength(tspan, 2)
tspan = mapdates(tspan)
tspan = span(*sorted(tspan))
assert isfixedlength(vspan, 2)
vspan = span(*sorted(vspan))
if zspan is not None:
assert isfixedlength(zspan, 2)
zspan = span(*sorted(zspan))
self._members.append(self.mem(tspan, vspan, zspan))
def attenuated_signal_test(inp : Sequence[N],
threshold : Tuple[N, N],
check_type : str = 'std'
) -> np.ma.MaskedArray:
"""Check for near-flat-line conditions using a range or standard deviation.
Missing and masked data is flagged as UNKNOWN.
Args:
inp: Input data as a numeric numpy array or a list of numbers.
threshold: 2-tuple representing the minimum thresholds to use for SUSPECT
and FAIL checks. The smaller of the two values is used in the SUSPECT
tests and the greater of the two values is used in the FAIL tests.
check_type: Either 'std' (default) or 'range', depending on the type of check
you wish to perform.
Returns:
A masked array of flag values equal in size to that of the input.
This array will always contain only a single unique value since all
input data is flagged together.
"""
assert isfixedlength(threshold, 2)
threshold = span(*reversed(sorted(threshold)))
with warnings.catch_warnings():
warnings.simplefilter("ignore")
inp = np.ma.masked_invalid(np.array(inp).astype(np.floating))
# Save original shape
original_shape = inp.shape
inp = inp.flatten()
if check_type == 'std':
check_val = np.std(inp)
elif check_type == 'range':
check_val = np.ptp(inp)
else:
raise ValueError('Check type "{}" is not defined'.format(check_type))
# Start with everything as passing (1)
flag_arr = np.ma.ones(inp.size, dtype='uint8')
if check_val < threshold.maxv:
flag_arr.fill(QartodFlags.FAIL)
elif check_val < threshold.minv:
flag_arr.fill(QartodFlags.SUSPECT)
# If the value is masked set the flag to MISSING
flag_arr[inp.mask] = QartodFlags.MISSING
return flag_arr.reshape(original_shape)
def location_test(lon: Sequence[N],
lat: Sequence[N],
bbox: Tuple[N, N, N, N] = (-180, -90, 180, 90),
range_max: N = None) -> np.ma.core.MaskedArray:
"""Checks that a location is within reasonable bounds.
Checks that longitude and latitude are within reasonable bounds defaulting
to lon = [-180, 180] and lat = [-90, 90]. Optionally, check for a maximum
range parameter in great circle distance defaulting to meters which can
also use a unit from the quantities library. Missing and masked data is
flagged as UNKNOWN.
Args:
lon: Longitudes as a numeric numpy array or a list of numbers.
lat: Latitudes as a numeric numpy array or a list of numbers.
bbox: A length 4 tuple expressed in (minx, miny, maxx, maxy) [optional].
range_max: Maximum allowed range expressed in geodesic curve distance (meters).
Returns:
A masked array of flag values equal in size to that of the input.
"""
bboxnt = namedtuple('BBOX', 'minx miny maxx maxy')
if bbox is not None:
assert isfixedlength(bbox, 4)
bbox = bboxnt(*bbox)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
lat = np.ma.masked_invalid(np.array(lat).astype(np.floating))
lon = np.ma.masked_invalid(np.array(lon).astype(np.floating))
if lon.shape != lat.shape:
raise ValueError(
'Lon ({0.shape}) and lat ({1.shape}) are different shapes'.format(
lon, lat))
# Save original shape
original_shape = lon.shape
lon = lon.flatten()
lat = lat.flatten()
# Start with everything as passing (1)
flag_arr = np.ma.ones(lon.size, dtype='uint8')
# If either lon or lat are masked we just set the flag to MISSING
mloc = lon.mask & lat.mask
flag_arr[mloc] = QartodFlags.MISSING
# If there is only one masked value fail the location test
mismatch = lon.mask != lat.mask
flag_arr[mismatch] = QartodFlags.FAIL
if range_max is not None and lon.size > 1:
# Calculating the great_distance between each point
# Flag suspect any distance over range_max
d = np.ma.zeros(lon.size, dtype=np.float64)
d[1:] = great_distance(start_latitude=lat[:-1],
end_latitude=lat[1:],
start_longitude=lon[:-1],
end_longitude=lon[1:])['distance']
flag_arr[d > range_max] = QartodFlags.SUSPECT
# Ignore warnings when comparing NaN values even though they are masked
# https://github.com/numpy/numpy/blob/master/doc/release/1.8.0-notes.rst#runtime-warnings-when-comparing-nan-numbers
with np.errstate(invalid='ignore'):
flag_arr[(lon < bbox.minx) | (lat < bbox.miny) | (lon > bbox.maxx) |
(lat > bbox.maxy)] = QartodFlags.FAIL
return flag_arr.reshape(original_shape)
def location_test(lon: Sequence[N],
lat: Sequence[N],
bbox: Tuple[N, N, N, N] = (-180, -90, 180, 90),
range_max: N = None,
target_lat: [N] = None,
target_lon: [N] = None,
target_range: N = None) -> np.ma.core.MaskedArray:
"""Checks that a location is within reasonable bounds.
Checks that longitude and latitude are within reasonable bounds defaulting
to lon = [-180, 180] and lat = [-90, 90]. Optionally, check for a maximum
range parameter in great circle distance defaulting to meters which can
also use a unit from the quantities library. Missing and masked data is
flagged as UNKNOWN.
Args:
lon: Longitudes as a numeric numpy array or a list of numbers.
lat: Latitudes as a numeric numpy array or a list of numbers.
bbox: A length 4 tuple expressed in (minx, miny, maxx, maxy) [optional].
range_max: Maximum allowed range expressed in geodesic curve distance (meters).
target_lat: Target Latitude as numeric numpy array or a list of numbers,
it can either same size as lat/lon or a unique values
target_lon: Target Longitude as numeric numpy array or a list of numbers,
it can either same size as lat/lon or a unique values
target_range: Maximum allowed range in geodesic curve distance (meters) away from target position.
Returns:
A masked array of flag values equal in size to that of the input.
"""
bboxnt = namedtuple('BBOX', 'minx miny maxx maxy')
if bbox is not None:
assert isfixedlength(bbox, 4)
bbox = bboxnt(*bbox)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
lat = np.ma.masked_invalid(np.array(lat).astype(np.float64))
lon = np.ma.masked_invalid(np.array(lon).astype(np.float64))
if lon.shape != lat.shape:
raise ValueError(
'Lon ({0.shape}) and lat ({1.shape}) are different shapes'.format(
lon, lat))
# Save original shape
original_shape = lon.shape
lon = lon.flatten()
lat = lat.flatten()
# Handle target inputs
# If any target inputs are provided
if target_lon is not None or target_lat is not None or target_range is not None:
# All target inputs should be there
if target_lon is not None and target_lat is not None and target_range is not None:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
target_lat = np.ma.masked_invalid(
np.array(target_lat).astype(np.float64))
target_lon = np.ma.masked_invalid(
np.array(target_lon).astype(np.float64))
if type(target_range) not in [int, float]:
raise ValueError(
'Bad target_range input. target_range should either float or int.'
)
elif target_lon is not None and target_lat is not None and target_range is None:
raise ValueError(
'Missing target_range input if target_lat and target_lon are provided'
)
else:
raise ValueError('Missing some target inputs')
if target_lon.shape != target_lat.shape:
raise ValueError(
'Target_lon ({0.shape}) and target_lat ({1.shape}) are different shapes'
.format(target_lon, target_lat))
# Flatten target_lon and target_lat
target_lon = target_lon.flatten()
target_lat = target_lat.flatten()
# Start with everything as passing (1)
flag_arr = np.ma.ones(lon.size, dtype='uint8')
# If either lon or lat are masked we just set the flag to MISSING
mloc = lon.mask & lat.mask
flag_arr[mloc] = QartodFlags.MISSING
# If there is only one masked value fail the location test
mismatch = lon.mask != lat.mask
flag_arr[mismatch] = QartodFlags.FAIL
if range_max is not None and lon.size > 1:
# Calculating the great_distance between each point
# Flag suspect any distance over range_max
d = great_circle_distance(lat, lon)
flag_arr[d > range_max] = QartodFlags.SUSPECT
# Distance From Target Test
if target_lat is not None and target_lon is not None and \
target_range is not None:
# If only one value is given assume to be constant for all positions
if target_lon.size == 1 and target_lat.size == 1:
(target_lon, target_lat) = (target_lon * np.ones(lat.size),
target_lat * np.ones(lat.size))
# Compute the range from the target location
d_from_target = distance_from_target(lat, lon, target_lat, target_lon)
# Flag as suspect distances greater than target_range
with np.errstate(invalid='ignore'):
flag_arr[d_from_target > target_range] = QartodFlags.SUSPECT
# Flag as missing target_location distance
flag_arr[(target_lat.mask | target_lon.mask)] = QartodFlags.MISSING
# Ignore warnings when comparing NaN values even though they are masked
# https://github.com/numpy/numpy/blob/master/doc/release/1.8.0-notes.rst#runtime-warnings-when-comparing-nan-numbers
with np.errstate(invalid='ignore'):
flag_arr[(lon < bbox.minx) | (lat < bbox.miny) | (lon > bbox.maxx) |
(lat > bbox.maxy)] = QartodFlags.FAIL
return flag_arr.reshape(original_shape)
