def add(self, tspan, vspan, fspan=None, zspan=None, period=None): assert isfixedlength(tspan, 2) # If period is defined, tspan is a numeric # if it isn't defined, its a parsable date if period is not None: tspan = span(*sorted(tspan)) else: tspan = span( *sorted([pd.Timestamp(tspan[0]), pd.Timestamp(tspan[1])])) assert isfixedlength(vspan, 2) vspan = span(*sorted(vspan)) if fspan is not None: assert isfixedlength(fspan, 2) fspan = span(*sorted(fspan)) if zspan is not None: assert isfixedlength(zspan, 2) zspan = span(*sorted(zspan)) if period is not None: # Test to make sure this is callable on a Timestamp try: getattr(pd.Timestamp.now(), period) except AttributeError: raise ValueError('The period "{period}" is not recognized') self._members.append(self.mem(tspan, fspan, vspan, zspan, period))
def gross_range_test( inp: Sequence[N], fail_span: Tuple[N, N], suspect_span: Tuple[N, N] = None) -> np.ma.core.MaskedArray: """Checks that values are within reasonable range bounds. Given a 2-tuple of minimum/maximum values, flag data outside of the given range as FAIL data. Optionally also flag data which falls outside of a user defined range as SUSPECT. Missing and masked data is flagged as UNKNOWN. Args: inp: Input data as a numeric numpy array or a list of numbers. fail_span: 2-tuple range which to flag outside data as FAIL. suspect_span: 2-tuple range which to flag outside data as SUSPECT. [optional] Returns: A masked array of flag values equal in size to that of the input. """ assert isfixedlength(fail_span, 2) sspan = span(*sorted(fail_span)) with warnings.catch_warnings(): warnings.simplefilter("ignore") inp = np.ma.masked_invalid(np.array(inp).astype(np.float64)) # Save original shape original_shape = inp.shape inp = inp.flatten() # Start with everything as passing (1) flag_arr = np.ma.ones(inp.size, dtype='uint8') # If the value is masked set the flag to MISSING flag_arr[inp.mask] = QartodFlags.MISSING if suspect_span is not None: assert isfixedlength(suspect_span, 2) uspan = span(*sorted(suspect_span)) if uspan.minv < sspan.minv or uspan.maxv > sspan.maxv: raise ValueError('Suspect {} must fall within the Fail {}'.format( uspan, sspan)) # Flag suspect outside of user span with np.errstate(invalid='ignore'): flag_arr[(inp < uspan.minv) | (inp > uspan.maxv)] = QartodFlags.SUSPECT # Flag suspect outside of sensor span with np.errstate(invalid='ignore'): flag_arr[(inp < sspan.minv) | (inp > sspan.maxv)] = QartodFlags.FAIL return flag_arr.reshape(original_shape)
def add(self, tspan: Tuple[N, N], vspan: Tuple[N, N], zspan: Tuple[N, N] = None) -> None: assert isfixedlength(tspan, 2) tspan = mapdates(tspan) tspan = span(*sorted(tspan)) assert isfixedlength(vspan, 2) vspan = span(*sorted(vspan)) if zspan is not None: assert isfixedlength(zspan, 2) zspan = span(*sorted(zspan)) self._members.append(self.mem(tspan, vspan, zspan))
def attenuated_signal_test(inp : Sequence[N], threshold : Tuple[N, N], check_type : str = 'std' ) -> np.ma.MaskedArray: """Check for near-flat-line conditions using a range or standard deviation. Missing and masked data is flagged as UNKNOWN. Args: inp: Input data as a numeric numpy array or a list of numbers. threshold: 2-tuple representing the minimum thresholds to use for SUSPECT and FAIL checks. The smaller of the two values is used in the SUSPECT tests and the greater of the two values is used in the FAIL tests. check_type: Either 'std' (default) or 'range', depending on the type of check you wish to perform. Returns: A masked array of flag values equal in size to that of the input. This array will always contain only a single unique value since all input data is flagged together. """ assert isfixedlength(threshold, 2) threshold = span(*reversed(sorted(threshold))) with warnings.catch_warnings(): warnings.simplefilter("ignore") inp = np.ma.masked_invalid(np.array(inp).astype(np.floating)) # Save original shape original_shape = inp.shape inp = inp.flatten() if check_type == 'std': check_val = np.std(inp) elif check_type == 'range': check_val = np.ptp(inp) else: raise ValueError('Check type "{}" is not defined'.format(check_type)) # Start with everything as passing (1) flag_arr = np.ma.ones(inp.size, dtype='uint8') if check_val < threshold.maxv: flag_arr.fill(QartodFlags.FAIL) elif check_val < threshold.minv: flag_arr.fill(QartodFlags.SUSPECT) # If the value is masked set the flag to MISSING flag_arr[inp.mask] = QartodFlags.MISSING return flag_arr.reshape(original_shape)
def location_test(lon: Sequence[N], lat: Sequence[N], bbox: Tuple[N, N, N, N] = (-180, -90, 180, 90), range_max: N = None) -> np.ma.core.MaskedArray: """Checks that a location is within reasonable bounds. Checks that longitude and latitude are within reasonable bounds defaulting to lon = [-180, 180] and lat = [-90, 90]. Optionally, check for a maximum range parameter in great circle distance defaulting to meters which can also use a unit from the quantities library. Missing and masked data is flagged as UNKNOWN. Args: lon: Longitudes as a numeric numpy array or a list of numbers. lat: Latitudes as a numeric numpy array or a list of numbers. bbox: A length 4 tuple expressed in (minx, miny, maxx, maxy) [optional]. range_max: Maximum allowed range expressed in geodesic curve distance (meters). Returns: A masked array of flag values equal in size to that of the input. """ bboxnt = namedtuple('BBOX', 'minx miny maxx maxy') if bbox is not None: assert isfixedlength(bbox, 4) bbox = bboxnt(*bbox) with warnings.catch_warnings(): warnings.simplefilter("ignore") lat = np.ma.masked_invalid(np.array(lat).astype(np.floating)) lon = np.ma.masked_invalid(np.array(lon).astype(np.floating)) if lon.shape != lat.shape: raise ValueError( 'Lon ({0.shape}) and lat ({1.shape}) are different shapes'.format( lon, lat)) # Save original shape original_shape = lon.shape lon = lon.flatten() lat = lat.flatten() # Start with everything as passing (1) flag_arr = np.ma.ones(lon.size, dtype='uint8') # If either lon or lat are masked we just set the flag to MISSING mloc = lon.mask & lat.mask flag_arr[mloc] = QartodFlags.MISSING # If there is only one masked value fail the location test mismatch = lon.mask != lat.mask flag_arr[mismatch] = QartodFlags.FAIL if range_max is not None and lon.size > 1: # Calculating the great_distance between each point # Flag suspect any distance over range_max d = np.ma.zeros(lon.size, dtype=np.float64) d[1:] = great_distance(start_latitude=lat[:-1], end_latitude=lat[1:], start_longitude=lon[:-1], end_longitude=lon[1:])['distance'] flag_arr[d > range_max] = QartodFlags.SUSPECT # Ignore warnings when comparing NaN values even though they are masked # https://github.com/numpy/numpy/blob/master/doc/release/1.8.0-notes.rst#runtime-warnings-when-comparing-nan-numbers with np.errstate(invalid='ignore'): flag_arr[(lon < bbox.minx) | (lat < bbox.miny) | (lon > bbox.maxx) | (lat > bbox.maxy)] = QartodFlags.FAIL return flag_arr.reshape(original_shape)
def location_test(lon: Sequence[N], lat: Sequence[N], bbox: Tuple[N, N, N, N] = (-180, -90, 180, 90), range_max: N = None, target_lat: [N] = None, target_lon: [N] = None, target_range: N = None) -> np.ma.core.MaskedArray: """Checks that a location is within reasonable bounds. Checks that longitude and latitude are within reasonable bounds defaulting to lon = [-180, 180] and lat = [-90, 90]. Optionally, check for a maximum range parameter in great circle distance defaulting to meters which can also use a unit from the quantities library. Missing and masked data is flagged as UNKNOWN. Args: lon: Longitudes as a numeric numpy array or a list of numbers. lat: Latitudes as a numeric numpy array or a list of numbers. bbox: A length 4 tuple expressed in (minx, miny, maxx, maxy) [optional]. range_max: Maximum allowed range expressed in geodesic curve distance (meters). target_lat: Target Latitude as numeric numpy array or a list of numbers, it can either same size as lat/lon or a unique values target_lon: Target Longitude as numeric numpy array or a list of numbers, it can either same size as lat/lon or a unique values target_range: Maximum allowed range in geodesic curve distance (meters) away from target position. Returns: A masked array of flag values equal in size to that of the input. """ bboxnt = namedtuple('BBOX', 'minx miny maxx maxy') if bbox is not None: assert isfixedlength(bbox, 4) bbox = bboxnt(*bbox) with warnings.catch_warnings(): warnings.simplefilter("ignore") lat = np.ma.masked_invalid(np.array(lat).astype(np.float64)) lon = np.ma.masked_invalid(np.array(lon).astype(np.float64)) if lon.shape != lat.shape: raise ValueError( 'Lon ({0.shape}) and lat ({1.shape}) are different shapes'.format( lon, lat)) # Save original shape original_shape = lon.shape lon = lon.flatten() lat = lat.flatten() # Handle target inputs # If any target inputs are provided if target_lon is not None or target_lat is not None or target_range is not None: # All target inputs should be there if target_lon is not None and target_lat is not None and target_range is not None: with warnings.catch_warnings(): warnings.simplefilter("ignore") target_lat = np.ma.masked_invalid( np.array(target_lat).astype(np.float64)) target_lon = np.ma.masked_invalid( np.array(target_lon).astype(np.float64)) if type(target_range) not in [int, float]: raise ValueError( 'Bad target_range input. target_range should either float or int.' ) elif target_lon is not None and target_lat is not None and target_range is None: raise ValueError( 'Missing target_range input if target_lat and target_lon are provided' ) else: raise ValueError('Missing some target inputs') if target_lon.shape != target_lat.shape: raise ValueError( 'Target_lon ({0.shape}) and target_lat ({1.shape}) are different shapes' .format(target_lon, target_lat)) # Flatten target_lon and target_lat target_lon = target_lon.flatten() target_lat = target_lat.flatten() # Start with everything as passing (1) flag_arr = np.ma.ones(lon.size, dtype='uint8') # If either lon or lat are masked we just set the flag to MISSING mloc = lon.mask & lat.mask flag_arr[mloc] = QartodFlags.MISSING # If there is only one masked value fail the location test mismatch = lon.mask != lat.mask flag_arr[mismatch] = QartodFlags.FAIL if range_max is not None and lon.size > 1: # Calculating the great_distance between each point # Flag suspect any distance over range_max d = great_circle_distance(lat, lon) flag_arr[d > range_max] = QartodFlags.SUSPECT # Distance From Target Test if target_lat is not None and target_lon is not None and \ target_range is not None: # If only one value is given assume to be constant for all positions if target_lon.size == 1 and target_lat.size == 1: (target_lon, target_lat) = (target_lon * np.ones(lat.size), target_lat * np.ones(lat.size)) # Compute the range from the target location d_from_target = distance_from_target(lat, lon, target_lat, target_lon) # Flag as suspect distances greater than target_range with np.errstate(invalid='ignore'): flag_arr[d_from_target > target_range] = QartodFlags.SUSPECT # Flag as missing target_location distance flag_arr[(target_lat.mask | target_lon.mask)] = QartodFlags.MISSING # Ignore warnings when comparing NaN values even though they are masked # https://github.com/numpy/numpy/blob/master/doc/release/1.8.0-notes.rst#runtime-warnings-when-comparing-nan-numbers with np.errstate(invalid='ignore'): flag_arr[(lon < bbox.minx) | (lat < bbox.miny) | (lon > bbox.maxx) | (lat > bbox.maxy)] = QartodFlags.FAIL return flag_arr.reshape(original_shape)