def img_stats(test: str, mast: str, diff_img: np.ndarray, dir_in: str, fn_in: str, dir_out: str, sds_ct: int=0) -> None:
"""
Log stats from array
:param test: name of test file
:param mast: name of master file
:param diff_img: image array
:param dir_in: directory where test data exists
:param fn_in: input filename (to identify csv entry)
:param dir_out: output directory
:param sds_ct: index of SDS (default=0)
:return:
"""
diff_img = np.ma.masked_where(diff_img == 0, diff_img)
fn_out = dir_out + os.sep + "stats.csv"
logging.info("Writing stats for {0} to {1}.".format(fn_in, fn_out))
file_exists = os.path.isfile(fn_out)
# Changed "ab" to "a", using python 3.6 was getting TypeError: a bytes-like object is required, not 'str'
with open(fn_out, "a") as f:
writer = csv.writer(f)
# write header if file didn't already exist
if not file_exists:
writer.writerow(("dir",
"test_file",
"master_file",
"mean",
"min",
"max",
"25_percentile",
"75_percentile",
"1_percentile",
"99_percentile",
"std_dev",
"median"))
writer.writerow((dir_in,
test + "_" + str(sds_ct),
mast + "_" + str(sds_ct),
np.mean(diff_img),
np.amin(diff_img),
np.amax(diff_img),
np.percentile(diff_img.compressed(), 25),
np.percentile(diff_img.compressed(), 75),
np.percentile(diff_img.compressed(), 1),
np.percentile(diff_img.compressed(), 99),
np.std(diff_img),
np.median(diff_img.compressed())))
return None
def _plot_relative_error(ax, error: ndarray, ax_values: tuple):
pl = ax.pcolorfast(*ax_values,
error[:-1, :-1].T,
cmap='RdBu',
vmin=-30,
vmax=30)
colorbar = _init_colorbar(pl, ax)
colorbar.set_label("%", fontsize=13)
median_error = ma.median(error.compressed())
median_error = "%.3f" % median_error
ax.set_title(f"Median relative error: {median_error} %", fontsize=14)
def __call__(self, dtimes: np.ndarray, array: np.ndarray, axis: int = -1):
"""
Apply filter to the input time series data.
If input data is masked array, masked values will not be considered.
Parameters
----------
dtimes: np.ndarray
Datetimes of input sequence.
array: np.ndarray or np.MaskedArray
Input sequence to filter.
axis: int
Axis
Returns
-------
out: np.ma.MaskedArray
Filtered array with masked outliers.
"""
dtimes = np.asarray(dtimes, dtype=datetime)
array = np.ma.asarray(array)
if dtimes.size != array.size:
raise ValueError('Length of input arrays should be equal')
size = dtimes.size
if self.window is None:
window = size
else:
window = self.window
if window > size:
raise ValueError('Window {} is bigger than whole data length {}'
''.format(window, size))
# Search for discontinuity
if array.mask.any():
dtimes = np.ma.array(dtimes, mask=array.mask)
if self.timeout is not None:
if isinstance(dtimes, np.ma.MaskedArray):
no_mask_indices = np.where(~dtimes.mask)[0][1:]
compressed_indices = \
np.where(np.diff(dtimes.compressed()) > self.timeout)[0]
timeout_indices = no_mask_indices[compressed_indices]
else:
timeout_indices = np.where(np.diff(dtimes) >= self.timeout)[0]
timeout_indices += 1 # np.diff shift indices
array_split = np.split(array, timeout_indices)
else:
array_split = [array]
# Apply window filtering
masked_arrays = []
for arr in array_split:
if window <= arr.size:
out_array = filter_outliers(arr,
self.filter,
window=window,
pad_value=self.pad_value,
padding=self.padding)
elif window > arr.size and self.below_window_mode == 'reduce':
out_array = filter_outliers(arr,
self.filter,
window=arr.size,
pad_value=self.pad_value,
padding=self.padding)
elif window > arr.size and self.below_window_mode == 'skip':
out_array = arr
else:
raise RuntimeError('Unexpected error when apply filtering')
masked_arrays.append(out_array)
return np.ma.concatenate(masked_arrays)
