def run(self, variable_name: str) -> Optional[np.ndarray]:
valid_delta = self.ds[variable_name].attrs.get(ATTS.VALID_DELTA, None)
# If no valid_delta is available, then we just skip this definition
results_array = None
if valid_delta is not None:
# We need to get the dim to diff on from the parameters
# If dim is not specified, then we use the first dim for the variable
dim = self.params.get('dim', None)
if dim is None and len(self.ds[variable_name].dims) > 0:
dim = self.ds[variable_name].dims[0]
if dim is not None:
# If previous data exists, then we must add the last row of
# previous data as the first row of the variable's data array.
# This is so that the diff function can compare the first value
# of the file to make sure it is consistent with the previous file.
# convert to np array
variable_data = self.ds[variable_name].data
axis = self.ds[variable_name].get_axis_num(dim)
previous_row = None
# Load the previous row from the other dataset
if self.previous_data is not None:
previous_variable_data = self.previous_data.get(variable_name, None)
if previous_variable_data is not None:
# convert to np array
previous_variable_data = previous_variable_data.data
# Get the last value from the first axis
previous_row = previous_variable_data[-1]
# Insert that value as the first value of the first axis
variable_data = np.insert(variable_data, 0, previous_row, axis=axis)
# If the variable is a time variable, then we convert to nanoseconds before doing our check
if self.ds[variable_name].values.dtype.type == np.datetime64:
variable_data = DSUtil.datetime64_to_timestamp(variable_data)
# Compute the difference between each two numbers and check if it exceeds valid_delta
diff = np.absolute(np.diff(variable_data, axis=axis))
results_array = np.greater(diff, valid_delta)
if previous_row is None:
# This means our results array is missing one value for the first row, which is
# not included in the diff computation.
# We need to add False for the first row of results, since it won't fail
# the check.
first_row = np.zeros(results_array[0].size, dtype=bool)
results_array = np.insert(results_array, 0, first_row, axis=axis)
return results_array
def run(self, variable_name: str) -> Optional[np.ndarray]:
results_array = None
# We need to get the dim to diff on from the parameters
# If dim is not specified, then we use the first dim for the variable
dim = self.params.get("dim", None)
if dim is None and len(self.ds[variable_name].dims) > 0:
dim = self.ds[variable_name].dims[0]
if dim is not None:
# If previous data exists, then we must add the last row of
# previous data as the first row of the variable's data array.
# This is so that the diff function can compare the first value
# of the file to make sure it is consistent with the previous file.
# convert to np array
variable_data = self.ds[variable_name].data
axis = self.ds[variable_name].get_axis_num(dim)
previous_row = None
# Load the previous row from the other dataset
if self.previous_data is not None and dim == "time":
previous_variable_data = self.previous_data.get(
variable_name, None)
if previous_variable_data is not None:
# convert to np array
previous_variable_data = previous_variable_data.data
# Get the last value from the first axis
previous_row = previous_variable_data[-1]
# Insert that value as the first value of the first axis
variable_data = np.insert(variable_data,
0,
previous_row,
axis=axis)
# If the variable is a time variable, then we convert to nanoseconds before doing our check
if self.ds[variable_name].data.dtype.type == np.datetime64:
variable_data = DSUtil.datetime64_to_timestamp(variable_data)
# Compute the difference between each two numbers and check if they are either all
# increasing or all decreasing
diff = np.diff(variable_data, axis=axis)
is_monotonic = np.all(diff > 0) | np.all(
diff < 0) # this returns a scalar
# Create a results array, with all values set to the results of the is_monotonic check
results_array = np.full(variable_data.shape,
not is_monotonic,
dtype=bool)
return results_array