def quality_fis(self,fis):
"""
Count the correct classifications of a given FIS on the check data.
:param fis: The Fuzzy Inference System to be tested
:type fis: :class:`rule.ClassifierSet`
:returns: A tuple containing the number of correct classifications and the total number of classifications
:rtype: (:class:`int`,:class:`int`)
"""
if fis.dimension() == self.training_data.shape[1]:
last_res = 0.0
count = 0
for i in range(self.check_data.shape[0]):
last_res = fis.evaluate(np.hstack((self.check_data[i],last_res)))
if abs(last_res - self.id) < 0.5:
count = count + 1
return (count,self.check_data.shape[0])
else:
rvec = fis.evaluates(self.check_data) - self.id
rvec = ma.masked_inside(rvec,-0.5,0.5)
return (ma.count_masked(rvec),self.check_data.shape[0])
if fis.dimension() == self.training_data.shape[1]:
dat = np.hstack((self.check_data,self.id*np.ones((self.check_data.shape[0],1))))
else:
dat = self.check_data
#if self.check_data.shape[1] == self.training_data.shape[1]:
# dat = self.check_data
#else:
# dat = np.hstack((self.check_data,np.zeros((self.check_data.shape[0],1))))
rvec = fis.evaluates(dat) - self.id
rvec = ma.masked_inside(rvec,-0.5,0.5)
return (ma.count_masked(rvec),self.check_data.shape[0])
def action(self, Data) :
params = self.params
# Keep track of how many pre existing flags there are for feedback
# purposes.
already_flagged = ma.count_masked(Data.data)
if params["rotate"]:
if (tuple(Data.field['CRVAL4']) == (1, 2, 3, 4)):
rotate_pol.rotate(Data, (-5,-7,-8,-6))
Data.add_history('Rotated to XX,XY,YX,YY')
# Few operations to be performed before flagging.
if params["perform_hanning"] :
hanning.hanning_smooth(Data)
Data.add_history('Hanning smoothed.')
if params["cal_scale"] :
cal_scale.scale_by_cal(Data, True, False, False)
Data.add_history('Converted to units of noise cal temperture.')
# Flag the data.
apply_cuts(Data, sigma_thres=params['sigma_thres'],
badness_thres=params['badness_thres'],
time_cut=params['time_cut'])
Data.add_history('Flagged Bad Data.', ('Sigma threshold: '
+ str(self.params['sigma_thres']), 'Badness threshold: '
+ str(self.params['badness_thres']), 'Time mask size: '
+ str(self.params['time_cut'])))
# Report the number of new flags.
new_flags = ma.count_masked(Data.data) - already_flagged
self.block_feedback = str(new_flags) + ', '
return Data
def test_masked_pts_lazy(self):
data = self.masked_pts_lazy
computed = data.compute()
self.assertTrue(ma.isMaskedArray(computed))
self.assertTrue(ma.count_masked(computed))
coord = AuxCoord(data)
self.assertTrue(coord.has_lazy_points())
self.assertTrue(ma.isMaskedArray(coord.points))
self.assertTrue(ma.count_masked(coord.points))
def test_masked_data_lazy(self):
data = self.masked_data_lazy
computed = data.compute()
self.assertTrue(ma.isMaskedArray(computed))
self.assertTrue(ma.count_masked(computed))
ancill_var = AncillaryVariable(data)
self.assertTrue(ancill_var.has_lazy_data())
self.assertTrue(ma.isMaskedArray(ancill_var.data))
self.assertTrue(ma.count_masked(ancill_var.data))
def test_no_masked_bds_lazy(self):
data = self.no_masked_bds_lazy
computed = data.compute()
self.assertTrue(ma.isMaskedArray(computed))
self.assertEqual(ma.count_masked(computed), 0)
coord = AuxCoord(self.pts_real, bounds=data)
self.assertTrue(coord.has_lazy_bounds())
self.assertTrue(ma.isMaskedArray(coord.bounds))
self.assertEqual(ma.count_masked(coord.bounds), 0)
def get_mask_for_unphysical(U, cutoffU=2000., fill_value=np.nan):
"""
Returns a mask for masking module. if absolute value of value is greater than cutoff, the value is masked.
Parameters
----------
U: array-like
cutoffU: float
if |value| > cutoff, this method considers those values unphysical.
fill_value:
Returns
-------
mask: multidimensional boolean array
"""
print 'number of invalid values (nan and inf) in the array: ' + str(
np.isnan(U).sum() + np.isinf(U).sum())
print 'number of nan values in U: ' + str(np.isnan(U).sum())
print 'number of inf values in U: ' + str(np.isinf(U).sum()) + '\n'
# a=ma.masked_invalid(U)
# print 'number of masked elements by masked_invalid: '+ str(ma.count_masked(a))
# Replace all nan and inf values with fill_value.
# fix_invalid still enforces a mask on elements with originally invalid values
U_fixed = ma.fix_invalid(U, fill_value=99999)
n_invalid = ma.count_masked(U_fixed)
print 'number of masked elements by masked_invalid: ' + str(n_invalid)
# Update the mask to False (no masking)
U_fixed.mask = False
# Mask unreasonable values of U_fixed
b = ma.masked_greater(U_fixed, cutoffU)
c = ma.masked_less(U_fixed, -cutoffU)
n_greater = ma.count_masked(b) - n_invalid
n_less = ma.count_masked(c)
print 'number of masked elements greater than cutoff: ' + str(n_greater)
print 'number of masked elements less than -cutoff: ' + str(n_less)
# Generate a mask for all nonsense values in the array U
mask = ~(~b.mask * ~c.mask)
d = ma.array(U_fixed, mask=mask)
n_total = ma.count_masked(d)
# U_filled = ma.filled(d, fill_value)
#Total number of elements in U
N = 1
for i in range(len(U.shape)):
N *= U.shape[i]
print 'total number of unphysical values: ' + str(
ma.count_masked(d)) + ' (' + str((float(n_total) / N * 100)) + '%)\n'
return mask
def process_file(self, file_ind) :
params = self.params
file_middle = params['file_middles'][file_ind]
input_fname = (params['input_root'] + file_middle +
params['input_end'])
sub_input_fname = (params['subtracted_input_root'] + file_middle
+ params['input_end'])
output_fname = (params['output_root']
+ file_middle + params['output_end'])
sub_output_fname = (params['subtracted_output_root']
+ file_middle + params['output_end'])
Writer = fitsGBT.Writer(feedback=self.feedback)
SubWriter = fitsGBT.Writer(feedback=self.feedback)
# Read in the data, and loop over data blocks.
Reader = fitsGBT.Reader(input_fname, feedback=self.feedback)
SubReader = fitsGBT.Reader(sub_input_fname, feedback=self.feedback)
if (sp.any(Reader.scan_set != SubReader.scan_set)
or sp.any(Reader.IF_set != SubReader.IF_set)) :
raise ce.DataError("IFs and scans don't match signal subtracted"
" data.")
# Get the number of scans if asked for all of them.
scan_inds = params['scans']
if len(scan_inds) == 0 or scan_inds is None :
scan_inds = range(len(Reader.scan_set))
if_inds = params['IFs']
if len(if_inds) == 0 or scan_inds is None :
if_inds = range(len(Reader.IF_set))
if self.feedback > 1 :
print "New flags each block:",
# Loop over scans and IFs
for thisscan in scan_inds :
for thisIF in if_inds :
Data = Reader.read(thisscan, thisIF)
SubData = SubReader.read(thisscan, thisIF)
n_flags = ma.count_masked(Data.data)
# Now do the flagging.
flag(Data, SubData, params['thres'])
Data.add_history("Reflaged for outliers.", ("Used file: "
+ utils.abbreviate_file_path(sub_input_fname),))
SubData.add_history("Reflaged for outliers.")
Writer.add_data(Data)
SubWriter.add_data(SubData)
# Report the numbe of new flags.
n_flags = ma.count_masked(Data.data) - n_flags
if self.feedback > 1 :
print n_flags,
if self.feedback > 1 :
print ''
# Finally write the data back to file.
utils.mkparents(output_fname)
utils.mkparents(sub_output_fname)
Writer.write(output_fname)
SubWriter.write(sub_output_fname)
def breakfinder(ts, br, w, N, per, lag=1, cond_thresh=1e6):
'''
Look around each break to identify weather it was real or artificial
ts is the time series we which to segment
br is the set of breaks found using r_window
w_step is a dictionary containing steps and the respective candidate windows
defined for the artifical break finder
N in the number of observations in the null likelihood ratio
per is the percentile of the null likelihood ratio distribution used as a threshold
'''
steps = np.unique(np.diff(w))
w_step = {}
w_step[steps[0]] = w[:np.arange(len(w) - 1)[np.diff(w) == steps[0]][-1] +
2]
for i in range(len(steps) - 1):
w_step[steps[
i +
1]] = w[np.arange(len(w) - 1)[np.diff(w) == steps[i]][-1] +
1:np.arange(len(w) - 1)[np.diff(w) == steps[i + 1]][-1] +
2]
for step in w_step.keys():
br_w = w_step[step]
min_w = np.min(br_w)
start = br - min_w
#this step is probably not needed since br>=min(window_sizes)
if start < 0:
start = 0
for i in range(len(br_w) - 1):
w1 = ts[start:br]
w2 = ts[start:br + step]
if ma.count_masked(w1, axis=0)[0] < ma.count_masked(w2, axis=0)[0]:
return br
else:
w1 = np.array(w1)
w2 = np.array(w2)
theta_1, eps1 = lvar_c.get_theta(w1, lag)
theta_2, eps2 = lvar_c.get_theta(w2, lag)
c1, A1, cov1 = lvar_c.decomposed_theta(theta_1)
c2, A2, cov2 = lvar_c.decomposed_theta(theta_2)
cond1 = np.linalg.cond(cov1)
cond2 = np.linalg.cond(cov2)
if cond1 > cond_thresh or cond2 > cond_thresh:
continue
else:
first_test = test(w1, w2, theta_1, theta_2, N, per, lag)
if first_test:
return br
else:
continue
start = br - br_w[i + 1]
return False
def action(self, Data):
'''Prepares Data and flags RFI.
Parameters
----------
Data : DataBlock
Contains information in a usable format direct from GBT.
Returns
-------
Data : DataBlock
The input `Data` with RFI flagged. Will also be cal scaled and
rotated to XX,YY... if so chosen.
'''
params = self.params
# Keep track of how many pre existing flags there are for feedback
# purposes.
already_flagged = ma.count_masked(Data.data)
if params["rotate"]:
if (tuple(Data.field['CRVAL4']) == (1, 2, 3, 4)):
rotate_pol.rotate(Data, (-5, -7, -8, -6))
Data.add_history('Rotated to XX,XY,YX,YY')
# Few operations to be performed before flagging.
if params["perform_hanning"]:
hanning.hanning_smooth(Data)
Data.add_history('Hanning smoothed.')
if params["cal_scale"] or params["cal_phase"]:
cal_scale.scale_by_cal(Data,
params['cal_scale'],
False,
False,
False,
rotate=params['cal_phase'])
Data.add_history('Converted to units of noise cal temperture.')
# Flag the data.
apply_cuts(Data,
sigma_thres=params['sigma_thres'],
badness_thres=params['badness_thres'],
time_cut=params['time_cut'])
Data.add_history(
'Flagged Bad Data.',
('Sigma threshold: ' + str(self.params['sigma_thres']),
'Badness threshold: ' + str(self.params['badness_thres']),
'Time mask size: ' + str(self.params['time_cut'])))
# Report the number of new flags.
new_flags = ma.count_masked(Data.data) - already_flagged
percent = float(new_flags) / Data.data.size * 100
self.block_feedback = '%d (%f%%), ' % (new_flags, percent)
return Data
def test_flags(self) :
self.Data.data[...] = 10
self.Data.data[4, 0, 0, 235] = 100
nf = self.Data.dims[-1]
self.DataSub.data[0::2, :, :, :] = sp.arange(1, nf+1)
self.DataSub.data[1::2, :, :, :] = 0
self.DataSub.data[3, 0, 0, 458] = 2*458
self.DataSub.data[3, 0, 0, 986] = 2*986
self.DataSub.data[8, 0, 1, 734] = 2*734
reflag.flag(self.Data, self.DataSub, thres=2.0)
self.assertTrue(self.Data.data[3, 0, 0, 458] is ma.masked)
self.assertTrue(self.Data.data[3, 0, 0, 986] is ma.masked)
self.assertTrue(self.Data.data[8, 0, 1, 734] is ma.masked)
self.assertEqual(ma.count_masked(self.Data.data), 3)
self.assertEqual(ma.count_masked(self.DataSub.data), 3)
def test_flags(self):
self.Data.data[...] = 10
self.Data.data[4, 0, 0, 235] = 100
nf = self.Data.dims[-1]
self.DataSub.data[0::2, :, :, :] = sp.arange(1, nf + 1)
self.DataSub.data[1::2, :, :, :] = 0
self.DataSub.data[3, 0, 0, 458] = 2 * 458
self.DataSub.data[3, 0, 0, 986] = 2 * 986
self.DataSub.data[8, 0, 1, 734] = 2 * 734
reflag.flag(self.Data, self.DataSub, thres=2.0)
self.assertTrue(self.Data.data[3, 0, 0, 458] is ma.masked)
self.assertTrue(self.Data.data[3, 0, 0, 986] is ma.masked)
self.assertTrue(self.Data.data[8, 0, 1, 734] is ma.masked)
self.assertEqual(ma.count_masked(self.Data.data), 3)
self.assertEqual(ma.count_masked(self.DataSub.data), 3)
def _sample_distant_same(img, a_lat, a_lon, neighborhood_radius,
distant_radius, tile_radius, timestep, size_even):
if neighborhood_radius is None:
return _sample_distant_diff(img, tile_radius, timestep, size_even)
_, _, img_h, img_w = img.shape
while True:
d_lat, d_lon = a_lat, a_lon
if distant_radius is None:
while (d_lat >= a_lat - neighborhood_radius) and (
d_lat <= a_lat + neighborhood_radius):
d_lat = np.random.randint(tile_radius, img_h - tile_radius)
while (d_lon >= a_lon - neighborhood_radius) and (
d_lon <= a_lon + neighborhood_radius):
d_lon = np.random.randint(tile_radius, img_w - tile_radius)
else:
while ((d_lat >= a_lat - neighborhood_radius) and (d_lat <= a_lat + neighborhood_radius)) \
or d_lat >= a_lat + distant_radius \
or d_lat <= a_lat - distant_radius:
d_lat = np.random.randint(tile_radius, img_h - tile_radius)
while ((d_lon >= a_lon - neighborhood_radius) and (d_lon <= a_lon + neighborhood_radius))\
or d_lon >= a_lon + distant_radius \
or d_lon <= a_lon - distant_radius:
d_lon = np.random.randint(tile_radius, img_w - tile_radius)
lat0, lat1, lon0, lon1 = _get_lat_lon_range(d_lat, d_lon, tile_radius,
size_even)
tile = img[:, timestep, lat0:lat1 + 1, lon0:lon1 + 1]
if ma.count_masked(tile) == 0:
break
return tile, d_lat, d_lon
def interpolate(self):
# N.B. The counter interpolate() fucntion also divides by
# the step intervals, so the resulting series is a set of
# observations of the true rate in MB/s.
masked_vals = np.where(self.Read.Values.mask == True)
self.Missing = np.zeros(len(self.Read.Values), dtype=np.int32)
if ma.count_masked(self.Read.Values) != 0:
self.Missing[masked_vals] = 1
n = self.Read.interpolate()
if n == 0:
handleError(self,
OSTNoDataError,
"OST.interpolate(): Warning - No Read data")
# not reached
n = self.Write.interpolate()
if n == 0:
handleError(self,
OSTNoDataError,
"OST.interpolate(): Warning - No Write data")
# not reached
n = self.OST.interpolate()
if n == 0:
handleError(self,
OSTNoDataError,
"OST.interpolate(): Warning - No combined data")
def generate_dims_for_counties(croptype):
yield_data = pd.read_csv('processed_data/crop_yield/{}_2000_2018.csv'.format(croptype))[[
'Year', 'State ANSI', 'County ANSI', 'Value']]
yield_data.columns = ['year', 'state', 'county', 'value']
ppt_fh = Dataset('experiment_data/spatial_temporal/nc_files/2014.nc', 'r')
v_ppt = ppt_fh.variables['ppt'][0, :, :]
if yield_data.value.dtype != float:
yield_data['value'] = yield_data['value'].str.replace(',', '')
yield_data = yield_data.astype({'year': int, 'state': int, 'county': int, 'value': float})
counties = pd.read_csv('processed_data/counties/lst/us_counties_cro_cvm_locations.csv')
county_dic = {}
for c in counties.itertuples():
state, county, lat, lon, lat0, lat1, lon0, lon1 = c.state, c.county, c.lat, c.lon, c.lat0, c.lat1, c.lon0, c.lon1
county_dic[(state, county)] = [lat, lon, lat0, lat1, lon0, lon1]
yield_dim_csv = []
for yd in yield_data.itertuples():
year, state, county, value = yd.year, yd.state, yd.county, yd.value
if (state, county) not in county_dic:
continue
lat, lon, lat0, lat1, lon0, lon1 = county_dic[(state, county)]
assert lat1 - lat0 == 49
assert lon1 - lon0 == 49
selected_ppt = v_ppt[lat0:lat1+1, lon0:lon1+1]
if ma.count_masked(selected_ppt) != 0:
continue
yield_dim_csv.append([state, county, year, value, lat, lon])
yield_dim_csv = pd.DataFrame(yield_dim_csv, columns=['state', 'county', 'year', 'value', 'lat', 'lon'])
yield_dim_csv.to_csv('experiment_data/spatial_temporal/counties/deep_gaussian_dim_y.csv')
def test_slicing(self):
cube = self._load_3d_cube()
# Test the slicing before deferred loading
full_slice = cube[3]
partial_slice = cube[0]
self.assertIsInstance(full_slice.data, np.ndarray)
self.assertIsInstance(partial_slice.data, ma.core.MaskedArray)
self.assertEqual(ma.count_masked(partial_slice.data), 25)
# Test the slicing is consistent after deferred loading
full_slice = cube[3]
partial_slice = cube[0]
self.assertIsInstance(full_slice.data, np.ndarray)
self.assertIsInstance(partial_slice.data, ma.core.MaskedArray)
self.assertEqual(ma.count_masked(partial_slice.data), 25)
def test_masked_bds_real(self):
data = self.masked_bds_real
self.assertTrue(ma.isMaskedArray(data))
self.assertTrue(ma.count_masked(data))
emsg = "bounds array must not be masked"
with self.assertRaisesRegex(TypeError, emsg):
DimCoord(self.pts_real, bounds=data)
def test_cube_slice_w_ignore_dead_traces_trilinear():
"""Get cube slice trilinear aka Auto4D input, with scrambled data with
dead traces to be ignored, various YFLIP cases."""
cube1 = Cube(XCUB2)
surf1 = RegularSurface()
surf1.from_cube(cube1, 1000.0)
cells = [(18, 12), (20, 2), (0, 4)]
surf1.slice_cube(cube1,
sampling="trilinear",
snapxy=True,
deadtraces=False)
plotfile = ojn(td, "slice_tri1.png")
title = "Cube with dead traces; trilinear; keep as is at dead traces"
surf1.quickplot(filename=plotfile, minmax=(-10000, 10000), title=title)
for cell in cells:
icell, jcell = cell
assert surf1.values[icell,
jcell] == pytest.approx(cube1.values[icell, jcell,
0],
abs=0.1)
assert ma.count_masked(surf1.values) == 0 # shall be no masked cells
def test_subcube(mask):
"""Cube class: testing sub-cube extraction methods"""
cube1 = generate_cube(data=np.arange(10 * 6 * 5).reshape(10, 6, 5),
wave=WaveCoord(crval=1), mask=mask)
# Extract a sub-cube whose images are centered close to pixel
# (2.3, 2.8) of the cube and have a width and height of 2 pixels.
# The center of a 2x2 pixel region is at the shared corner between
# these pixels, and the closest corner to the requested center
# of 2.3,2.8 is 2.5,2.5. Thus the sub-images should be from pixels
# 2,3 along both the X and Y axes.
cube2 = cube1.subcube(center=(2.3, 2.8), size=2, lbda=(5, 8),
unit_center=None, unit_size=None, unit_wave=None)
assert_allclose(cube1.data[5:9, 2:4, 2:4], cube2.data)
# Test when subcube is on the edges
cube2 = cube1.subcube(center=(0.3, 0.8), size=4,
unit_center=None, unit_size=None)
assert_allclose(cube1.data[:, :3, :3], cube2.data[:, 1:, 1:])
# pixels inside the selected region are not masked
assert np.all(~cube2.mask[:, 1:, 1:])
# pixels outside the selected region are masked
assert np.all(cube2.mask[:, :, 0])
assert np.all(cube2.mask[:, 0, :])
# The following should select the same image area as above, followed by
# masking pixels in this area outside a circle of radius 1.
cube2 = cube1.subcube_circle_aperture(center=(2.3, 2.8), radius=1,
unit_center=None, unit_radius=None)
# masking the subcube should not mask the original cube
assert ma.count_masked(cube1[0].data) == 0
if cube2.mask is not ma.nomask:
assert bool(cube2.mask[0, 0, 0]) is True
assert_array_equal(cube2.get_start(), (1, 2, 2))
assert_array_equal(cube2.shape, (10, 2, 2))
def data_stats(curr_delay, curr_dist, depth_crop, ir_crop, frame_count, window,
scale_factor, sw_gain, sw_offset, cam_handle):
# Mask Saturated/0 Pixels
depth_masked = ma.masked_equal(depth_crop, 0)
satCount = ma.count_masked(depth_masked)
if satCount == frame_count * window['X'] * window['Y']:
meas_depth = 0
else:
meas_depth = np.mean(depth_masked)
meas_depth_14b = meas_depth * scale_factor
expected_depth_14b = (curr_dist * scale_factor)
correction_offset_14b = ((
(curr_dist - sw_offset) / sw_gain) * 4) - meas_depth_14b
meas_depth_adj = (np.median(depth_crop) * sw_gain) + sw_offset
meas_ir = np.median(ir_crop)
#Get TAL Values
TAL_values = get_TAL_values(cam_handle)
depth_std = np.std(depth_crop) * sw_gain
depth_noise = 100 * depth_std / meas_depth_adj
ir_std = np.std(ir_crop)
ir_noise = 100 * ir_std / meas_ir
error = meas_depth_adj - curr_dist
return [curr_delay, curr_dist, meas_depth, meas_depth_adj, error, meas_ir, \
satCount, depth_std, depth_noise, ir_std, ir_noise, \
TAL_values[0], TAL_values[7], TAL_values[1], TAL_values[8], TAL_values[2], TAL_values[9], TAL_values[3], TAL_values[10], TAL_values[4], TAL_values[11], \
expected_depth_14b, meas_depth_14b, correction_offset_14b]
def test_masked_pts_real(self):
data = self.masked_pts_real
self.assertTrue(ma.isMaskedArray(data))
self.assertTrue(ma.count_masked(data))
emsg = 'points array must not be masked'
with six.assertRaisesRegex(self, TypeError, emsg):
DimCoord(data)
def register(self):
"""
We have all the sie and timestamp values in dictionaries, so just
build the time step sequence from them
"""
self.numSteps = len(self.StepDict)
self.Steps = ma.empty(self.numSteps, dtype=np.int32)
self.Steps.mask = True
self.TS_IDs = ma.empty(self.numSteps, dtype=np.int32)
self.TS_IDs.mask = True
self.Steps.soften_mask()
self.TS_IDs.soften_mask()
for sie,index in self.StepDict.iteritems():
self.Steps[index] = sie
self.TS_IDs[index] = self.TS_IDDict[sie]
self.__registered += 1
if ma.count_masked(self.Steps) != 0:
handleError(self,
TimeStepsRegisterError,
"TimeSteps.register(): Warning - registered all %d, but still have %d masked values" % (self.numSteps, len(np.nonzero(self.Steps.mask))))
# not reached
if self.Debug == True:
self.DebugMessages += "TimeSteps.register(): Registred all %d expected values, now to check them and take the Step differences" % self.numSteps
self.Diff = np.diff(self.Steps)
if (np.min(self.Diff) <= 0) and (self.Debug == True):
message = "TimeSteps.register(): Warning - negative or zero time differentials at\n"
message += "indices:" + str(np.where(self.Diff <= 0)) + "values:" + str(self.Diff[np.where(self.Diff <= 0)])
handleError(self,
TimeStepsRegisterError,
message)
# not reached
self.HaveData = True
def test_slicing(self):
cube = self._load_3d_cube()
# Test the slicing before deferred loading
full_slice = cube[3]
partial_slice = cube[0]
self.assertIsInstance(full_slice.data, np.ndarray)
self.assertIsInstance(partial_slice.data, ma.core.MaskedArray)
self.assertEqual(ma.count_masked(partial_slice.data), 25)
# Test the slicing is consistent after deferred loading
full_slice = cube[3]
partial_slice = cube[0]
self.assertIsInstance(full_slice.data, np.ndarray)
self.assertIsInstance(partial_slice.data, ma.core.MaskedArray)
self.assertEqual(ma.count_masked(partial_slice.data), 25)
def register(self):
"""
We have all the bin and timestamp values in dictionaries, so just
build the time bin sequence from them
"""
self.numBins = len(self.BinDict)
self.Bins = ma.empty(self.numBins, dtype=np.float64)
self.Bins.mask = True
self.Bins.soften_mask()
# the bins some times emerge in a random order
# so put them in order, and then preserve that
count = 0
for bin,index in sorted(self.BinDict.iteritems()):
self.Bins[count] = bin
count += 1
for count in range(self.numBins):
self.BinDict[self.Bins[count]] = count
self.__registered += 1
if ma.count_masked(self.Bins) != 0:
handleError(self,
HistBinsRegisterError,
"HistBins.register(): Warning - registered all %d, but still have %d masked values" % (self.numBins, len(np.nonzero(self.Bins.mask))))
# not reached
if self.Debug == True:
self.DebugMessages += "HistBins.register(): Registred all %d expected values, now to check them and take the Bin differences" % self.numBins
self.HaveData = True
def test_masked_bds_lazy(self):
data = self.masked_bds_lazy
computed = data.compute()
self.assertTrue(ma.isMaskedArray(computed))
self.assertTrue(ma.count_masked(computed))
emsg = 'bounds array must not be masked'
with six.assertRaisesRegex(self, TypeError, emsg):
DimCoord(self.pts_real, bounds=data)
def test_slicing(self):
cube = self._load_3d_cube()
# Test the slicing before deferred loading
full_slice = cube[3]
partial_slice = cube[0]
self.assertTrue(isinstance(full_slice.data, np.ndarray), "Expected a numpy array")
self.assertTrue(isinstance(partial_slice.data, ma.core.MaskedArray), "Expected a numpy.ma.core.MaskedArray")
self.assertEqual(ma.count_masked(partial_slice._data), 25)
# Test the slicing is consistent after deferred loading
cube.data
full_slice = cube[3]
partial_slice = cube[0]
self.assertTrue(isinstance(full_slice.data, np.ndarray), "Expected a numpy array")
self.assertTrue(isinstance(partial_slice.data, ma.core.MaskedArray), "Expected a numpy.ma.core.MaskedArray")
self.assertEqual(ma.count_masked(partial_slice._data), 25)
def check_for_all(self):
"""
Function to check if all the values are masked or all not masked
"""
masked_N = ma.count_masked(self.xs, axis=1)[np.newaxis,:]
is_all_masked = self.N == masked_N
is_none_masked = masked_N == 0
return is_none_masked + is_all_masked
def test_scalar_with_overlap_above_mdtol(self):
# Slice src so result collapses to a scalar.
src_cube = self.src_cube[:, 1, :]
# Regrid to a single cell with 50% overlap with masked src cells.
grid_cube = self.grid_cube[3, 1, 4]
# Set threshold (mdtol) to less than 0.5 (50%).
res = regrid(src_cube, grid_cube, mdtol=0.4)
self.assertEqual(ma.count_masked(res.data), 1)
def interpolate(self):
# If there are no masked values there is nothing to do
if ma.count_masked(self.Values) == 0:
if self.Debug == True:
self.DebugMessages += "Series2.interpolate(): No masked values"
return(len(self.Values))
for vIndex in range(self.width):
self.interpolateItem(vIndex)
def test_masked_pts_lazy(self):
data = self.masked_pts_lazy
computed = data.compute()
self.assertTrue(ma.isMaskedArray(computed))
self.assertTrue(ma.count_masked(computed))
emsg = "points array must not be masked"
with self.assertRaisesRegex(TypeError, emsg):
DimCoord(data)
def action(self, Data):
'''Prepares Data and flags RFI.
Parameters
----------
Data : DataBlock
Contains information in a usable format direct from GBT.
Returns
-------
Data : DataBlock
The input `Data` with RFI flagged. Will also be cal scaled and
rotated to XX,YY... if so chosen.
'''
params = self.params
# Keep track of how many pre existing flags there are for feedback
# purposes.
already_flagged = ma.count_masked(Data.data)
if params["rotate"]:
if (tuple(Data.field['CRVAL4']) == (1, 2, 3, 4)):
rotate_pol.rotate(Data, (-5,-7,-8,-6))
Data.add_history('Rotated to XX,XY,YX,YY')
# Few operations to be performed before flagging.
if params["perform_hanning"] :
hanning.hanning_smooth(Data)
Data.add_history('Hanning smoothed.')
if params["cal_scale"] or params["cal_phase"]:
cal_scale.scale_by_cal(Data, params['cal_scale'], False, False,
False, rotate=params['cal_phase'])
Data.add_history('Converted to units of noise cal temperture.')
# Flag the data.
apply_cuts(Data, sigma_thres=params['sigma_thres'],
badness_thres=params['badness_thres'],
time_cut=params['time_cut'],
submean=params['submean'])
Data.add_history('Flagged Bad Data.', ('Sigma threshold: '
+ str(self.params['sigma_thres']), 'Badness threshold: '
+ str(self.params['badness_thres']), 'Time mask size: '
+ str(self.params['time_cut'])))
# Report the number of new flags.
new_flags = ma.count_masked(Data.data) - already_flagged
percent = float(new_flags) / Data.data.size * 100
self.block_feedback = '%d (%f%%), ' % (new_flags, percent)
return Data
def test_no_masked_bds_lazy(self):
data = self.no_masked_bds_lazy
computed = data.compute()
self.assertTrue(ma.isMaskedArray(computed))
self.assertEqual(ma.count_masked(computed), 0)
coord = DimCoord(self.pts_real, bounds=data)
# DimCoord always realises its bounds.
self.assertFalse(coord.has_lazy_bounds())
self.assertFalse(ma.isMaskedArray(coord.bounds))
def monotonic(array, strict=False, return_direction=False):
"""
Return whether the given 1d array is monotonic.
Note that, the array must not contain missing data.
Kwargs:
* strict (boolean)
Flag to enable strict monotonic checking
* return_direction (boolean)
Flag to change return behaviour to return
(monotonic_status, direction). Direction will be 1 for positive
or -1 for negative. The direction is meaningless if the array is
not monotonic.
Returns:
* monotonic_status (boolean)
Whether the array was monotonic.
If the return_direction flag was given then the returned value
will be:
``(monotonic_status, direction)``
"""
if array.ndim != 1 or len(array) <= 1:
raise ValueError('The array to check must be 1 dimensional and have '
'more than 1 element.')
if ma.isMaskedArray(array) and ma.count_masked(array) != 0:
raise ValueError('The array to check contains missing data.')
# Identify the directions of the largest/most-positive and
# smallest/most-negative steps.
d = np.diff(array)
sign_max_d = np.sign(np.max(d))
sign_min_d = np.sign(np.min(d))
if strict:
monotonic = sign_max_d == sign_min_d and sign_max_d != 0
else:
monotonic = (sign_min_d < 0 and sign_max_d <= 0) or \
(sign_max_d > 0 and sign_min_d >= 0) or \
(sign_min_d == sign_max_d == 0)
if return_direction:
if sign_max_d == 0:
direction = sign_min_d
else:
direction = sign_max_d
return monotonic, direction
return monotonic
def test_no_masked_pts_lazy(self):
data = self.no_masked_pts_lazy
computed = data.compute()
self.assertTrue(ma.isMaskedArray(computed))
self.assertEqual(ma.count_masked(computed), 0)
coord = DimCoord(data)
# DimCoord always realises its points.
self.assertFalse(coord.has_lazy_points())
self.assertFalse(ma.isMaskedArray(coord.points))
def monotonic(array, strict=False, return_direction=False):
"""
Return whether the given 1d array is monotonic.
Note that, the array must not contain missing data.
Kwargs:
* strict (boolean)
Flag to enable strict monotonic checking
* return_direction (boolean)
Flag to change return behaviour to return
(monotonic_status, direction). Direction will be 1 for positive
or -1 for negative. The direction is meaningless if the array is
not monotonic.
Returns:
* monotonic_status (boolean)
Whether the array was monotonic.
If the return_direction flag was given then the returned value
will be:
``(monotonic_status, direction)``
"""
if array.ndim != 1 or len(array) <= 1:
raise ValueError('The array to check must be 1 dimensional and have '
'more than 1 element.')
if ma.isMaskedArray(array) and ma.count_masked(array) != 0:
raise ValueError('The array to check contains missing data.')
# Identify the directions of the largest/most-positive and
# smallest/most-negative steps.
d = np.diff(array)
sign_max_d = np.sign(np.max(d))
sign_min_d = np.sign(np.min(d))
if strict:
monotonic = sign_max_d == sign_min_d and sign_max_d != 0
else:
monotonic = (sign_min_d < 0 and sign_max_d <= 0) or \
(sign_max_d > 0 and sign_min_d >= 0) or \
(sign_min_d == sign_max_d == 0)
if return_direction:
if sign_max_d == 0:
direction = sign_min_d
else:
direction = sign_max_d
return monotonic, direction
return monotonic
def _sample_anchor(img, tile_radius, distant_radius, timestep, size_even):
_, _, img_h, img_w = img.shape
while True:
a_lat = np.random.randint(tile_radius, img_h - tile_radius)
a_lon = np.random.randint(tile_radius, img_w - tile_radius)
lat0, lat1, lon0, lon1 = _get_lat_lon_range(a_lat, a_lon, tile_radius,
size_even)
tile = img[:, timestep, lat0:lat1 + 1, lon0:lon1 + 1]
# guarantee that distant tile can be sampled
d_lat0, d_lat1, d_lon0, d_lon1 = _get_lat_lon_range(
a_lat, a_lon, distant_radius, True)
big_tile = img[:, timestep, d_lat0:d_lat1 + 1, d_lon0:d_lon1 + 1]
if ma.count_masked(tile) == 0 and ma.count_masked(big_tile) == 0:
break
return tile, a_lat, a_lon
def write(self, output_map_name, values):
drv = gdal.GetDriverByName(self.FORMAT)
masked_values = self._mask_values(values)
n = ma.count_masked(masked_values)
self._mem_ds.GetRasterBand(1).SetNoDataValue(self.mv)
self._mem_ds.GetRasterBand(1).WriteArray(masked_values)
out_ds = drv.CreateCopy(output_map_name.encode('utf-8'), self._mem_ds)
self._log(f'{output_map_name} written!', 'INFO')
out_ds = None
del out_ds
def params_to_array_zeromask(self, params):
"""
Returns tuple of arrays + list
Process params input into appropriate arrays by setting them to
zero if param in params in zero and removing them from params,
otherwise they stay in params and value remains masked
Parameters
----------
params : list
list of values to fill in masked values
Returns
-------
lam_0 : numpy array
lam_1 : numpy array
delta_0 : numpy array
delta_1 : numpy array
mu : numpy array
phi : numpy array
sigma : numpy array
guesses : list
List of remaining params after filtering and filling those
that were zero
"""
paramcopy = params[:]
lam_0_e = self.lam_0_e.copy()
lam_1_e = self.lam_1_e.copy()
delta_0_e = self.delta_0_e.copy()
delta_1_e = self.delta_1_e.copy()
mu_e = self.mu_e.copy()
phi_e = self.phi_e.copy()
sigma_e = self.sigma_e.copy()
all_arrays = [
lam_0_e, lam_1_e, delta_0_e, delta_1_e, mu_e, phi_e, sigma_e
]
arg_sep = self._gen_arg_sep([ma.count_masked(struct) for struct in \
all_arrays])
guesses = []
# check if each element is masked or not
for struct in all_arrays:
it = np.nditer(struct.mask, flags=['multi_index'])
while not it.finished:
if it[0]:
val = paramcopy.pop(0)
if val == 0:
struct[it.multi_index] = 0
else:
guesses.append(val)
it.iternext()
return tuple(all_arrays + [guesses])
def params_to_array_zeromask(self, params):
"""
Returns tuple of arrays + list
Process params input into appropriate arrays by setting them to
zero if param in params in zero and removing them from params,
otherwise they stay in params and value remains masked
Parameters
----------
params : list
list of values to fill in masked values
Returns
-------
lam_0 : numpy array
lam_1 : numpy array
delta_0 : numpy array
delta_1 : numpy array
mu : numpy array
phi : numpy array
sigma : numpy array
guesses : list
List of remaining params after filtering and filling those
that were zero
"""
paramcopy = params[:]
lam_0_e = self.lam_0_e.copy()
lam_1_e = self.lam_1_e.copy()
delta_0_e = self.delta_0_e.copy()
delta_1_e = self.delta_1_e.copy()
mu_e = self.mu_e.copy()
phi_e = self.phi_e.copy()
sigma_e = self.sigma_e.copy()
all_arrays = [lam_0_e, lam_1_e, delta_0_e, delta_1_e, mu_e, phi_e,
sigma_e]
arg_sep = self._gen_arg_sep([ma.count_masked(struct) for struct in \
all_arrays])
guesses = []
# check if each element is masked or not
for struct in all_arrays:
it = np.nditer(struct.mask, flags=['multi_index'])
while not it.finished:
if it[0]:
val = paramcopy.pop(0)
if val == 0:
struct[it.multi_index] = 0
else:
guesses.append(val)
it.iternext()
return tuple(all_arrays + [guesses])
def rail_sweep(min_dist, max_dist, dist_step, raw_frame_dict, frame_dict,
frame_count, window, scale_factor, sw_gain, sw_offset, rail,
cam_handle):
logger = logging.getLogger(__name__)
logger.info('Running Rail Sweep')
depthCrop = np.empty([frame_count, window['X'], window['Y']])
results = []
img = None
rail.moveRailMM(int(min_dist))
time.sleep(1) #inital wait
for i, curr_loc in enumerate(
range(int(min_dist), int(max_dist + 1), int(dist_step))):
rail.moveRailMM(curr_loc)
#Wait time for target to stop shaking
time.sleep(5)
frame.dummy_read(cam_handle)
for frame_ind in np.arange(0, frame_count):
depthImage = frame.get_depth_image_df(cam_handle,
raw_frame_dict['width'],
raw_frame_dict['height'])
depthCrop[frame_ind] = frame.crop_center(depthImage,
frame_dict['width'],
frame_dict['height'],
window['X'], window['Y'])
depthMasked = ma.masked_equal(depthCrop, 0)
satCount = ma.count_masked(depthMasked)
meas_depth = np.median(depthCrop)
meas_depth_14b = meas_depth * scale_factor
expected_depth_14b = curr_loc * scale_factor
correction_offset_14b = ((
(curr_loc - sw_offset) / sw_gain) * 4) - meas_depth_14b
meas_depth = (np.median(depthCrop) * sw_gain) + sw_offset
results.append([
0, curr_loc, expected_depth_14b, meas_depth, meas_depth_14b,
correction_offset_14b, satCount
])
logger.debug('expected_depth: %d, measured_depth: %d', curr_loc,
meas_depth)
dfResults = pd.DataFrame(results,
columns=[
'delay', 'expected_depth_12b',
'expected_depth_14b', 'meas_depth_12b',
'meas_depth_14b', 'correction_offset_14b',
'saturation_count'
])
return dfResults
def printstuff(vals,vals_sum,file_count):
# create running sum of values
print file_count
print '----------------'
print 'sum', ma.sum(vals)
print 'total sum', ma.sum(vals_sum)
print 'perc masked:', ma.count_masked(vals_sum)/vals_sum.size*100.,'%'
print '----------------'
print 'max', ma.amax(vals)
print 'min', ma.amin(vals)
print '\n----------------'
def printstuff(vals, vals_sum, file_count):
# create running sum of values
print file_count
print '----------------'
print 'sum', ma.sum(vals)
print 'total sum', ma.sum(vals_sum)
print 'perc masked:', ma.count_masked(vals_sum) / vals_sum.size * 100., '%'
print '----------------'
print 'max', ma.amax(vals)
print 'min', ma.amin(vals)
print '\n----------------'
def format_and_clean_data_main(self):
"""
Main function to format and clean data based on choices by the user.
"""
# Check if over missing_bound percent or missing_bound number of values are missing
too_many_missing = self.has_too_many_missing(self.init_perc_remove)
if ma.any(too_many_missing):
idx, = ma.where(too_many_missing)
self.xs[idx] = ma.mask_rows(self.xs[idx])
# Check array to see if it is filled with values or empty
if ma.all(self.check_for_all()):
return self.xs
# Clean outliers
self.clean_outliers()
# Take average of neighbor values to fill up to a given missing value gap length
self.clean_gaps_w_linspace(fill_gap_length=self.max_gap_length)
if ma.all(ma.count_masked(self.xs[:, :-self.keep_n_values], axis=1)[np.newaxis,:] == 0):
return self.xs # if no masked values remain in values before recent ones
# Remove values if they start the array and are then followed by too many masked values
start_idx = self.find_new_starting_value()
# If there are over x% blank values left in the original data after above changes,
# check to see if x% of the blanks fall after the new start year
too_many_missing = self.has_too_many_missing(self.second_perc_remove) # boolean array
if ma.any(too_many_missing):
n_masked = np.array([ma.count_masked(self.xs[i,s_idx:])
for i, s_idx in enumerate(start_idx)]) / self.N > self.perc_remove_after_start_idx
if ma.any(n_masked):
idx, = ma.where(n_masked)
self.xs[idx] = ma.mask_rows(self.xs[idx])
# To fill in remaining values, run linear regression on non-zero values
self.clean_gaps_w_lin_regress(start_idx)
# If linear regression left negative or zero values, then use linear space to fill in middle gaps
if ma.any(ma.masked_less_equal(self.xs, 0.)):
self.clean_gaps_w_linspace()
def test_002_flagging_cut(self):
ind = (6,1,1,676)
freq = 676
ind2 = (3,2,0,245)
freq2 = 245
self.Data.data[ind] += 0.2
self.Data.data[ind2] += 0.2
flag_data.apply_cuts(self.Data)
self.assertFalse(False in self.Data.data[:,:,:,freq].mask)
self.assertFalse(False in self.Data.data[:,:,:,freq2].mask)
self.assertTrue(float(ma.count_masked(self.Data.data)) /
float(self.Data.data.size) < 0.1)
def test_rebinned_cal(self) :
rebin_freq.rebin(self.CalData, 1.0)
DataCopy = copy.deepcopy(self.Data)
DataCopy.calc_freq()
calibrate.multiply_by_cal(self.Data, self.CalData)
gains = sp.array([1,sp.sqrt(5),sp.sqrt(5),5])
gains.shape = (1,4,1,1)
expected = (DataCopy.data*gains*DataCopy.freq)
self.assertTrue(ma.count_masked(self.Data.data <
sp.size(self.Data.data)*50.0/2000.0))
expected.mask = sp.array(self.Data.data.mask)
self.assertTrue(ma.allclose(expected, self.Data.data, 4))
def _sample_distant_diff(img, tile_radius, timestep, size_even):
_, _, img_h, img_w = img.shape
while True:
d_lat = np.random.randint(tile_radius, img_h - tile_radius)
d_lon = np.random.randint(tile_radius, img_w - tile_radius)
lat0, lat1, lon0, lon1 = _get_lat_lon_range(d_lat, d_lon, tile_radius,
size_even)
tile = img[:, timestep, lat0:lat1 + 1, lon0:lon1 + 1]
if ma.count_masked(tile) == 0:
break
return tile, d_lat, d_lon
def test_cube_slice_w_ignore_dead_traces_nearest():
"""Get cube slice nearest aka Auto4D input, with scrambled data with
dead traces, various YFLIP cases, ignore dead traces."""
cube1 = Cube(XCUB2)
surf1 = RegularSurface()
surf1.from_cube(cube1, 1000.1)
cells = ((18, 12), (20, 2), (0, 4))
surf1.slice_cube(cube1, deadtraces=False)
plotfile = ojn(td, "slice_nea1.png")
title = "Cube with dead traces; nearest; use just values as is"
surf1.quickplot(filename=plotfile, minmax=(-10000, 10000), title=title)
for cell in cells:
icell, jcell = cell
assert surf1.values[icell,
jcell] == pytest.approx(cube1.values[icell, jcell,
0],
abs=0.01)
assert ma.count_masked(surf1.values) == 0 # shall be no masked cells
# swap surface
surf2 = surf1.copy()
surf2.values = 1000.1
surf2.swapaxes()
surf2.slice_cube(cube1, deadtraces=False)
assert surf2.values.mean() == pytest.approx(surf1.values.mean(), rel=0.001)
# swap surface and cube
surf2 = surf1.copy()
surf2.values = 1000.1
surf2.swapaxes()
cube2 = cube1.copy()
cube2.swapaxes()
surf2.slice_cube(cube2, deadtraces=False)
assert surf2.values.mean() == pytest.approx(surf1.values.mean(), rel=0.001)
# swap cube only
surf2 = surf1.copy()
surf2.values = 1000.1
cube2 = cube1.copy()
cube2.swapaxes()
surf2.slice_cube(cube2, deadtraces=False)
assert surf2.values.mean() == pytest.approx(surf1.values.mean(), rel=0.001)
def params_to_array(self, params, return_mask=False):
"""
Returns tuple of arrays
Process params input into appropriate arrays
Parameters
----------
params : list
list of values to fill in masked values
return_mask : boolean
Returns
-------
lam_0 : numpy array
lam_1 : numpy array
delta_0 : numpy array
delta_1 : numpy array
mu : numpy array
phi : numpy array
sigma : numpy array
"""
lam_0_e = self.lam_0_e.copy()
lam_1_e = self.lam_1_e.copy()
delta_0_e = self.delta_0_e.copy()
delta_1_e = self.delta_1_e.copy()
mu_e = self.mu_e.copy()
phi_e = self.phi_e.copy()
sigma_e = self.sigma_e.copy()
all_arrays = [
lam_0_e, lam_1_e, delta_0_e, delta_1_e, mu_e, phi_e, sigma_e
]
arg_sep = self._gen_arg_sep([ma.count_masked(struct) for struct in \
all_arrays])
# DOC: Get first element of parameters
first_el = params[0]
# DOC: Make sure that the type of every parameter element passed in is
# consistent
assert all(isinstance(param, type(first_el)) for param in params), \
"Guess parameters are not consistently typed"
np_dtype = self._choose_numpy_type(first_el)
for pos, struct in enumerate(all_arrays):
struct[ma.getmask(struct)] = params[arg_sep[pos]:arg_sep[pos + 1]]
if not return_mask:
all_arrays[pos] = np.asfortranarray(struct, dtype=np_dtype)
all_arrays.append(np_dtype)
return tuple(all_arrays)
def test_params_to_array_zeromask(self):
"""
Tests if params_to_array_zeromask function works correctly. In order to
pass, params_to_array_zeromask must return masked arrays with the
guess_params elements that are zero unmasked and set to zero in the
appropriate arrays. The new guess_params array is also returned with
those that were 0 removed. If both of these are not returned correctly,
the test fails.
"""
guess_params_arr = np.array(self.guess_params)
neqs = self.affine_obj.neqs
guess_params_arr[:neqs] = 0
guess_params = guess_params_arr.tolist()
guess_length = self.affine_obj._gen_guess_length()
params_guesses = self.affine_obj.params_to_array_zeromask(guess_params)
updated_guesses = params_guesses[-1]
self.assertEqual(len(updated_guesses), len(guess_params) - neqs)
# ensure that number of masked has correctly been set
count_masked_new = ma.count_masked(params_guesses[0])
count_masked_orig = ma.count_masked(self.affine_obj.lam_0_e)
self.assertEqual(count_masked_new, count_masked_orig - neqs)
def has_too_many_missing(self, missing_bound):
"""
Function to find if over missing_bound percent or
missing_bound number of values are missing
Parameters
----------
missing_bound : if < 1 ==> float, otherwise integer
Bound for checking if too many values are missing in the array.
"""
masked_count = ma.count_masked(self.xs, axis=1) * 1.
return [masked_count > missing_bound,
masked_count / self.N > missing_bound][missing_bound < 1]
def setCPU(self):
masked_vals = np.where(self.CPU.Values.mask == True)
self.Missing = np.zeros(len(self.CPU.Values), dtype=np.int32)
if ma.count_masked(self.CPU.Values) != 0:
self.Missing[masked_vals] = 1
n = self.CPU.interpolate()
if n == 0:
handleError(self,
OSSNoCPUDataError,
"OSS.setCPU(): WARNING - No data")
# not reached
self.CPU.stats()
self.haveData = True
def search(self, delta):
self.max_peaks = []
self.min_peaks = []
x = arange(len(self.data))
if not isscalar(delta):
exit('argument delta must be a scalar')
if delta <= 0:
exit('argument delta must be positive')
maxval, minval = self.data[0], self.data[0]
maxpos, minpos = 0, 0
lookformax = True
for i in range(self.n_data):
if ma.count_masked(self.data) > 1 and self.data.mask[i] == True:
continue
now = self.data[i]
if now > maxval:
maxval = now
maxpos = x[i]
if now < minval:
minval = now
minpos = x[i]
if lookformax:
if now < (maxval - delta):
#for j in range(maxpos,0,-1):
# if self.data[j] < (maxval - delta):
# self.max_peaks.append((x[i] + x[j]) / 2)
# break
self.max_peaks.append(maxpos)
minval = now
minpos = x[i]
lookformax = False
else:
if now > (minval + delta):
self.min_peaks.append(minpos)
maxval = now
maxpos = x[i]
lookformax = True
# if no peak found with the given delta, return maximum found
if len(self.max_peaks) == 0:
self.max_peaks.append(maxpos)
return
def params_to_array(self, params, return_mask=False):
"""
Returns tuple of arrays
Process params input into appropriate arrays
Parameters
----------
params : list
list of values to fill in masked values
return_mask : boolean
Returns
-------
lam_0 : numpy array
lam_1 : numpy array
delta_0 : numpy array
delta_1 : numpy array
mu : numpy array
phi : numpy array
sigma : numpy array
"""
lam_0_e = self.lam_0_e.copy()
lam_1_e = self.lam_1_e.copy()
delta_0_e = self.delta_0_e.copy()
delta_1_e = self.delta_1_e.copy()
mu_e = self.mu_e.copy()
phi_e = self.phi_e.copy()
sigma_e = self.sigma_e.copy()
all_arrays = [lam_0_e, lam_1_e, delta_0_e, delta_1_e, mu_e, phi_e,
sigma_e]
arg_sep = self._gen_arg_sep([ma.count_masked(struct) for struct in \
all_arrays])
# DOC: Get first element of parameters
first_el = params[0]
# DOC: Make sure that the type of every parameter element passed in is
# consistent
assert all(isinstance(param, type(first_el)) for param in params), \
"Guess parameters are not consistently typed"
np_dtype = self._choose_numpy_type(first_el)
for pos, struct in enumerate(all_arrays):
struct[ma.getmask(struct)] = params[arg_sep[pos]:arg_sep[pos + 1]]
if not return_mask:
all_arrays[pos] = np.asfortranarray(struct, dtype=np_dtype)
all_arrays.append(np_dtype)
return tuple(all_arrays)
def calc_joint_histogram(data_array1, data_array2, bins_for_data1, bins_for_data2):
''' Calculate a joint histogram of two variables in data_array1 and data_array2
:param data_array1: the first variable
:type data_array1: :class:'numpy.ma.core.MaskedArray'
:param data_array2: the second variable
:type data_array2: :class:'numpy.ma.core.MaskedArray'
:param bins_for_data1: histogram bin edges for data_array1
:type bins_for_data1: :class:'numpy.ndarray'
:param bins_for_data2: histogram bin edges for data_array2
:type bins_for_data2: :class:'numpy.ndarray'
'''
if ma.count_masked(data_array1) != 0 or ma.count_masked(data_array2) != 0:
index = numpy.where((data_array1.mask == False) &
(data_array2.mask == False))
new_array1 = data_array1[index]
new_array2 = data_array2[index]
else:
new_array1 = data_array1.flatten()
new_array2 = data_array2.flatten()
histo2d, xedge, yedge = numpy.histogram2d(new_array1, new_array2, bins=[
bins_for_data1, bins_for_data2])
return histo2d
def wet_spell_analysis(reference_array, threshold=0.1, nyear=1, dt=3.):
''' Characterize wet spells using sub-daily (hourly) data
:param reference_array: an array to be analyzed
:type reference_array: :class:'numpy.ma.core.MaskedArray'
:param threshold: the minimum amount of rainfall [mm/hour]
:type threshold: 'float'
:param nyear: the number of discontinous periods
:type nyear: 'int'
:param dt: the temporal resolution of reference_array
:type dt: 'float'
'''
nt = reference_array.shape[0]
if reference_array.ndim == 3:
reshaped_array = reference_array.reshape([nt, reference_array.size / nt])
else:
reshaped_array = reference_array
if ma.count_masked(reshaped_array[0,:]) != 0:
xy_indices = numpy.where(reshaped_array.mask[0, :] == False)[0]
else:
xy_indices = numpy.arange(reshaped_array.shape[1])
nt_each_year = nt / nyear
spell_duration = []
peak_rainfall = []
total_rainfall = []
for index in xy_indices:
for iyear in numpy.arange(nyear):
data0_temp = reshaped_array[nt_each_year * iyear:nt_each_year * (iyear + 1),
index]
# time indices when precipitation rate is smaller than the
# threshold [mm/hr]
t_index = numpy.where((data0_temp <= threshold) &
(data0_temp.mask == False))[0]
t_index = numpy.insert(t_index, 0, 0)
t_index = t_index + nt_each_year * iyear
for it in numpy.arange(t_index.size - 1):
if t_index[it + 1] - t_index[it] > 1:
data1_temp = data0_temp[t_index[it] + 1:t_index[it + 1]]
if not ma.is_masked(data1_temp):
spell_duration.append(
(t_index[it + 1] - t_index[it] - 1) * dt)
peak_rainfall.append(data1_temp.max())
total_rainfall.append(data1_temp.sum())
return numpy.array(spell_duration), numpy.array(peak_rainfall), numpy.array(total_rainfall)
def differential(self):
"""
The differential is going ot have one fewer elements than the
original series, so we need to decide how to handle that. Setting
the first value to zero is reasonable, but reducing the length
by one is also possible. If we do the latter then that has to be
worked out with the classes that coordinate with sequences of
time steps.
"""
if ma.count_masked(self.Values) != 0:
handleError(self,
SeriesDifferentialError,
"Series2.differential(): Warning - Masked values present. Did you mean to interpolate?")
for vIndex in range(self.width):
self.Values[vIndex, 1:] = np.diff(self.Values[vIndex,:])
self.Values[vIndex,0] = 0.0
def test_set_window(self) :
window_data = np.ones( (self.lat_size_win, self.lon_size_win) )
x = self.w.set_window(window_data)
# check output geometry
self.assertEqual(x.shape[0], 360)
self.assertEqual(x.shape[1], 720)
# check output is masked
self.assertTrue(ma.is_masked(x))
# check that the window is only thing in returned array
win_masked = ma.count_masked(x)
win = ma.count(x)
self.assertEqual(win, window_data.size)
self.assertEqual(win_masked, x.size - window_data.size)
self.assertTrue(np.all(x[self.w._window] == window_data))
def _gen_guess_length(self):
lam_0_e = self.lam_0_e
lam_1_e = self.lam_1_e
delta_0_e = self.delta_0_e
delta_1_e = self.delta_1_e
mu_e = self.mu_e
phi_e = self.phi_e
sigma_e = self.sigma_e
all_arrays = [lam_0_e, lam_1_e, delta_0_e, delta_1_e, mu_e, phi_e,
sigma_e]
count = 0
for struct in all_arrays:
count += ma.count_masked(struct)
return count
def interpolate(self):
"""
The time series gets its differential calculated as well, once
the interpolation is complete.
"""
masked_vals = np.where(self.Values.mask == True)
self.Missing = np.zeros(len(self.Values), dtype=np.int32)
if ma.count_masked(self.Values) != 0:
self.Missing[masked_vals] = 1
count = TimeSeries.TimeSeries.interpolate(self)
if count == 0:
return(0)
if self.Steps == None:
return(0)
Series.Series.differential(self)
self.Values[1:] /= self.Steps.Diff
self.Resets = np.where(self.Values < 0)
self.Values[self.Values < 0] = 0
return(count)
