def mean_and_stdev(raypaths):
minlen = min(map(len, raypaths))
maxlen = max(map(len, raypaths))
# newtimes = empty(maxlen)
meandepth = empty(maxlen)
meanxt = empty(maxlen)
depths = MaskedArray(empty((len(raypaths), maxlen)), mask=numpy.zeros((len(raypaths), maxlen), dtype=bool))
xts = MaskedArray(empty((len(raypaths), maxlen)), mask=numpy.zeros((len(raypaths), maxlen), dtype=bool))
times = MaskedArray(empty((len(raypaths), maxlen)), mask=numpy.zeros((len(raypaths), maxlen), dtype=bool))
# Build up some numpy arrays
for i, r in enumerate(raypaths):
# Set the used values
depths[i, :len(r)] = r.getDepths()
xts[i, :len(r)] = r.getAcrossTracks()
times[i, :len(r)] = r.getTimes()
# For the longest row, save the times to be our return value
if len(r) == maxlen: newtimes = r.getTimes()
# Mask out the unused valuse
mask = [[True for _ in range(maxlen - len(r))]]
depths.mask[i, len(r):] = deepcopy(mask)
xts.mask[i, len(r):] = deepcopy(mask)
times.mask[i, len(r):] = deepcopy(mask)
# Calculate the stats
mean_depths = depths.mean(axis=0) # depths.filled(0).sum(axis=0) / depths.count(axis=0)
mean_xts = xts.mean(axis=0) # xts.filled(0).sum(axis=0) / xts.count(axis=0)
stdev_depths = [depths[:, i].compressed().std() for i in range(maxlen)]
stdev_xts = [xts[:, i].compressed().std() for i in range(maxlen)]
return array([newtimes, mean_depths, mean_xts, stdev_depths, stdev_xts]).transpose()
def test_columns_of_unequal_sizes_raise(dsn, configuration):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'INSERT INTEGER') as table_name:
columns = [MaskedArray([1, 2, 3], mask=False, dtype='int64'),
MaskedArray([1, 2], mask=False, dtype='int64')]
with pytest.raises(turbodbc.InterfaceError):
cursor.executemanycolumns("INSERT INTO {} VALUES (?)".format(table_name), columns)
def _make_masked_array(data, mask):
from numpy.ma import MaskedArray
from numpy import object_
if isinstance(data, list):
return MaskedArray(data=data, mask=mask, dtype=object_)
else:
return MaskedArray(data=data, mask=mask)
def test_numpy_two_columns(dsn, configuration):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'INSERT TWO INTEGER COLUMNS') as table_name:
cursor.executemany("INSERT INTO {} VALUES (?, ?)".format(table_name),
[[1, 42], [2, 41]])
cursor.execute("SELECT a, b FROM {} ORDER BY a".format(table_name))
results = cursor.fetchallnumpy()
assert_equal(results[_fix_case(configuration, 'a')], MaskedArray([1, 2], mask=False))
assert_equal(results[_fix_case(configuration, 'b')], MaskedArray([42, 41], mask=False))
def _make_masked_arrays(result_batch):
from numpy.ma import MaskedArray
from numpy import object_
masked_arrays = []
for data, mask in result_batch:
if isinstance(data, list):
masked_arrays.append(MaskedArray(data=data, mask=mask, dtype=object_))
else:
masked_arrays.append(MaskedArray(data=data, mask=mask))
return masked_arrays
def test_numpy_batch_fetch(dsn, configuration):
with open_cursor(configuration, rows_to_buffer=2) as cursor:
with query_fixture(cursor, configuration, 'INSERT INTEGER') as table_name:
cursor.executemany("INSERT INTO {} VALUES (?)".format(table_name),
[[1], [2], [3], [4], [5]])
cursor.execute("SELECT a FROM {} ORDER BY a".format(table_name))
batches = cursor.fetchnumpybatches()
expected_batches = [MaskedArray([1, 2], mask=False),
MaskedArray([3, 4], mask=False),
MaskedArray([5], mask=False)]
for idx, batch in enumerate(batches):
expected = expected_batches[idx]
assert_equal(batch[_fix_case(configuration, 'a')], expected)
assert_equal(idx, 2)
def condition_data(vis, flags, weight, polarisation):
"""
Make the data ameniable for plotting.
- Convert to stokes I if required, in opts.
- Make vis into a masked array
- Normalise vis by the mean of each spectrum
Returns
=======
visdata = the normalised masked array constructed from vis, flags (same shape as vis)
"""
#Convert to Stokes I
if polarisation == 'I':
vis = vis[:, :, 0] + vis[:, :, 1]
flags = np.logical_or(flags[:, :, 0], flags[:, :, 1])
weight = weight[:, :, 0] + weight[:, :, 1]
else:
vis = np.squeeze(vis)
flags = np.squeeze(flags)
weight = np.squeeze(weight)
# Get the abs (we only care about amplitudes)
visdata = np.abs(vis)
#Make a masked array
visdata = MaskedArray(visdata, mask=flags)
return visdata, flags, weight
def _tidy_ncdf_data(var, start=None, count=None, stride=None):
from cis.utils import listify
from numpy.ma import MaskedArray
start = [] if start is None else listify(start)
count = [] if count is None else listify(count)
stride = [] if stride is None else listify(stride)
dim_len = var.shape
ndim = len(dim_len)
# Assume full read of all omitted dimensions
while len(start) < ndim:
start += [0]
while len(count) < ndim:
count += [-1]
while len(stride) < ndim:
stride += [1]
sl = (slice(x0, n if n >= 0 else l - x0, s)
for x0, n, s, l in zip(start, count, stride, dim_len))
data = var[sl]
if isinstance(data, MaskedArray):
return data
else:
return MaskedArray(data)
def set_vars(variables, dataset, group_name=None):
'''
set NetCDF Variables
use dataset.createVariable(),set Variable's attr and create MaskedArray
:param hdf:
:param dataset:
:return:
'''
for v in variables: # 遍历variables type: orderdict
var = variables[v]
if isinstance(var, Variable):
create_var = dataset.createVariable(varname=var.name, datatype=var.datatype, dimensions=var.dimensions,
endian=var.endian(), chunksizes=None) # 内置方法创建变量
# 设置属性信息
for attr in var.ncattrs(): # 遍历属性信息 type:list
if isinstance(create_var, Variable):
create_var.setncattr(attr, var.getncattr(attr)) # 设置属性
# 设置组信息
if group_name != None:
create_var.setncattr("group_name", group_name) # 设置属性,自定义的信息
var_data = var[::] # 数据信息
if isinstance(var_data, MaskedArray):
fill_value = var_data.get_fill_value() # Return the filling value of the masked array
data = var_data.data # Return the current data, as a view of the original underlying data
# MaskedArray对象
maskedArray = MaskedArray(data, dtype=var_data.dtype, fill_value=fill_value)
create_var[::] = maskedArray # 变量数据赋值
# data2=create_var[::]
return dataset
def any_of(self, *states, **kwargs):
'''
Return a boolean array containing True where the value of the
MappedArray equals any state in states.
:param states: List of states.
:type states: [str]
:param ignore_missing: If this is False, raise an exception if any of
the states are not in the values mapping.
:type ignore_missing: bool
:returns: Boolean array.
:rtype: np.ma.array(bool)
'''
ignore_missing = kwargs.get('ignore_missing', False)
raw_values = []
for state in states:
if state not in self.state:
if ignore_missing:
# do not check array as invalid states cause
# exception level log messages.
continue
else:
raise ValueError(
"State '%s' is not valid. Valid states: '%s'." %
(state, self.state.keys()))
raw_values.extend(self.state[state])
return MaskedArray(in1d(self.raw.data, raw_values), mask=self.mask)
def test_numpy_int_column(dsn, configuration):
with open_cursor(configuration) as cursor:
cursor.execute("SELECT 42 AS a")
results = cursor.fetchallnumpy()
expected = MaskedArray([42], mask=[0])
assert results[_fix_case(configuration, 'a')].dtype == numpy.int64
assert_equal(results[_fix_case(configuration, 'a')], expected)
def __init__(self, *args, **kwds):
TestCase.__init__(self, *args, **kwds)
self.data = np.arange(25)
self.maskeddata = MaskedArray(self.data)
self.maskeddata[10] = masked
self.func_pairs = [(mf.mov_average, ma.mean),
(mf.mov_median, ma.median),
((lambda x, span: mf.mov_std(x, span)), ma.std)]
def test_numpy_double_column(dsn, configuration):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'SELECT DOUBLE') as query:
cursor.execute(query)
results = cursor.fetchallnumpy()
expected = MaskedArray([3.14], mask=[0])
assert results[_fix_case(configuration, 'a')].dtype == numpy.float64
assert_equal(results[_fix_case(configuration, 'a')], expected)
def convert_t(x):
"""Faz a conversão para os tipos esperados."""
if x.dtype == 'object' and isinstance(x[0], date):
return pd.to_datetime(x).values
elif x.dtype == 'datetime64[ns]':
return x
return MaskedArray(x, pd.isnull(x))
def test_column_with_incompatible_dtype_raises(dsn, configuration):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration,
"INSERT INTEGER") as table_name:
columns = [MaskedArray([1, 2, 3], mask=False, dtype="int16")]
with pytest.raises(turbodbc.InterfaceError):
cursor.executemanycolumns(
f"INSERT INTO {table_name} VALUES (?)", columns)
def test_numpy_string_column_with_null(dsn, configuration):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'INSERT STRING') as table_name:
cursor.execute('INSERT INTO {} VALUES (?)'.format(table_name), [None])
cursor.execute('SELECT a FROM {}'.format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray([None], mask=[0], dtype=numpy.object_)
assert_equal(results[_fix_case(configuration, 'a')], expected)
def test_numpy_unicode_column_with_truncation(dsn, configuration):
with open_cursor(configuration, rows_to_buffer=1, varchar_max_character_limit=9, limit_varchar_results_to_max=True) as cursor:
with query_fixture(cursor, configuration, 'INSERT UNICODE MAX') as table_name:
cursor.execute('INSERT INTO {} VALUES (?)'.format(table_name), [u'truncated string'])
cursor.execute('SELECT a FROM {}'.format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray([u'truncated'], mask=[0], dtype=numpy.object_)
assert results[_fix_case(configuration, 'a')].dtype == numpy.object_
assert_equal(results[_fix_case(configuration, 'a')], expected)
def test_numpy_binary_column_larger_than_batch_size(dsn, configuration):
with open_cursor(configuration, rows_to_buffer=2) as cursor:
with query_fixture(cursor, configuration, 'INSERT INTEGER') as table_name:
cursor.executemany("INSERT INTO {} VALUES (?)".format(table_name),
[[1], [2], [3], [4], [5]])
cursor.execute("SELECT a FROM {} ORDER BY a".format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray([1, 2, 3, 4, 5], mask=False)
assert_equal(results[_fix_case(configuration, 'a')], expected)
def test_numpy_unicode_column(dsn, configuration):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'INSERT UNICODE') as table_name:
cursor.execute('INSERT INTO {} VALUES (?)'.format(table_name), [u'unicode \u2665'])
cursor.execute('SELECT a FROM {}'.format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray([u'unicode \u2665'], mask=[0], dtype=numpy.object_)
assert results[_fix_case(configuration, 'a')].dtype == numpy.object_
assert_equal(results[_fix_case(configuration, 'a')], expected)
def test_numpy_binary_column_with_null(dsn, configuration):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'INSERT TWO INTEGER COLUMNS') as table_name:
cursor.executemany("INSERT INTO {} VALUES (?, ?)".format(table_name),
[[42, 1], [None, 2]]) # second column to enforce ordering
cursor.execute("SELECT a FROM {} ORDER BY b".format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray([42, 0], mask=[0, 1])
assert_equal(results[_fix_case(configuration, 'a')], expected)
def unwrap_py(inph, in_p=(), uv=2 * pi):
"""Return the input matrix unwrapped the value given in uv
The same as unwrapv, but using for-s, written in python
"""
if not is_masked(inph):
fasei = MaskedArray(inph, isnan(inph))
else:
fasei = inph
nx, ny = (fasei.shape[0], fasei.shape[1])
# If the initial unwraping point is not given, take the center of the image
# as initial coordinate
if in_p == ():
in_p = (int(nx / 2), int(ny / 2))
# Create a temporal space to mark if the points are already unwrapped
# 0 the point has not been unwrapped
# 1 the point has not been unwrapped, but it is in the unwrapping list
# 2 the point was already unwrapped
fl = zeros((nx, ny))
# List containing the points to unwrap
l_un = [in_p]
fl[in_p] = 1
# unwrapped values
faseo = fasei.copy()
while len(l_un) > 0:
# remove the first value from the list
cx, cy = l_un.pop(0)
# Put the coordinates of unwrapped the neigbors in the list
# And check for wrapping
nv = 0
wv = 0
for i in range(cx - 1, cx + 2):
for j in range(cy - 1, cy + 2):
if (i > -1) and (i < nx) and (j > -1) and (j < ny):
if (fl[i, j] == 0) & (faseo.mask[i, j] == False):
fl[i, j] = 1
l_un.append((i, j))
elif fl[i, j] == 2:
wv = wv + rint((faseo[i, j] - faseo[cx, cy]) / uv)
nv = nv + 1
if nv != 0:
wv = wv / nv
fl[cx, cy] = 2
faseo[cx, cy] = faseo[cx, cy] + wv * uv
return faseo
def test_numpy_boolean_column(dsn, configuration):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'INSERT INDEXED BOOL') as table_name:
cursor.executemany('INSERT INTO {} VALUES (?, ?)'.format(table_name),
[[True, 1], [False, 2], [True, 3]])
cursor.execute('SELECT a FROM {} ORDER BY b'.format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray([True, False, True], mask=[0])
assert results[_fix_case(configuration, 'a')].dtype == numpy.bool_
assert_equal(results[_fix_case(configuration, 'a')], expected)
def _test_single_masked_value(configuration, fixture, values, dtype):
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, fixture) as table_name:
columns = [MaskedArray([values[0]], mask=[True], dtype=dtype)]
cursor.executemanycolumns(
"INSERT INTO {} VALUES (?)".format(table_name), columns)
assert cursor.rowcount == 1
results = cursor.execute(
"SELECT A FROM {} ORDER BY A".format(table_name)).fetchall()
assert results == [[None]]
def test_prepare_data(self):
"""Test prepare_data
"""
from numpy.ma import MaskedArray
from ..plots import prepare_data
data = np.arange(5.)
mask = np.array([True, False])
with self.assertRaises(ValueError):
ma = prepare_data(data, mask)
mask = data == 2
datama = MaskedArray(data, mask)
ma = prepare_data(datama)
self.assertIs(ma, datama)
ma = prepare_data(data)
self.assertTrue(np.allclose(data, ma.data))
self.assertFalse(ma.mask.any())
ma = prepare_data(data, mask)
self.assertTrue(np.allclose(data, ma.data))
self.assertTrue(
(ma.mask == np.array([False, False, True, False, False])).all())
data2 = data.copy()
data2[0] = 0.25
data3 = data.copy()
data3[0] = 0.25
data3[1:] = 0.75
data4 = data.copy()
data4[0] = 0.75
data4[-1] = 3.25
ma = prepare_data(data, mask, clip_lo=0.25)
self.assertTrue(np.allclose(data2, ma.data))
self.assertTrue(
(ma.mask == np.array([False, False, True, False, False])).all())
ma = prepare_data(data, mask, clip_lo=0.25, clip_hi=0.75)
self.assertTrue(np.allclose(data3, ma.data))
self.assertTrue(
(ma.mask == np.array([False, False, True, False, False])).all())
ma = prepare_data(data, mask, clip_lo='25%', clip_hi='75%')
self.assertTrue(np.allclose(data4, ma.data))
self.assertTrue(
(ma.mask == np.array([False, False, True, False, False])).all())
ma = prepare_data(data, mask, clip_lo='!25%', clip_hi='!75%')
self.assertTrue(np.allclose(data4, ma.data))
self.assertTrue((ma.mask == np.array([True, False, True, False,
True])).all())
ma = prepare_data(data,
mask,
clip_lo='25%',
clip_hi='75%',
save_limits=True)
self.assertTrue(np.allclose(data4, ma.data))
self.assertTrue(
(ma.mask == np.array([False, False, True, False, False])).all())
self.assertEqual(ma.vmin, 0.75)
self.assertEqual(ma.vmax, 3.25)
def test_numpy_string_column_larger_than_batch_size(dsn, configuration):
strings = [u'abc', u'def', u'ghi', u'jkl', u'mno']
with open_cursor(configuration, rows_to_buffer=2) as cursor:
with query_fixture(cursor, configuration,
'INSERT STRING') as table_name:
cursor.executemany('INSERT INTO {} VALUES (?)'.format(table_name),
[[string] for string in strings])
cursor.execute('SELECT a FROM {} ORDER BY a'.format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray(strings, mask=[0], dtype=numpy.object_)
assert_equal(results[_fix_case(configuration, 'a')], expected)
def test_numpy_date_column(dsn, configuration):
date = datetime.date(3999, 12, 31)
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'INSERT DATE') as table_name:
cursor.execute('INSERT INTO {} VALUES (?)'.format(table_name), [date])
cursor.execute('SELECT a FROM {}'.format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray([date], mask=[0], dtype='datetime64[D]')
assert results[_fix_case(configuration, 'a')].dtype == numpy.dtype('datetime64[D]')
assert_equal(results[_fix_case(configuration, 'a')], expected)
def test_numpy_timelike_column_with_null(dsn, configuration):
fill_value = 0;
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, 'INSERT TIMESTAMP') as table_name:
cursor.execute('INSERT INTO {} VALUES (?)'.format(table_name), [None])
cursor.execute('SELECT a FROM {}'.format(table_name))
results = cursor.fetchallnumpy()
expected = MaskedArray([42], mask=[1], dtype='datetime64[us]')
assert_equal(results[_fix_case(configuration, 'a')].filled(fill_value),
expected.filled(fill_value))
def __equality__(self, other, invert=False):
'''
Test equality or inequality allowing for multiple raw values matching a
state.
'''
raw_values = self.__raw_values__(other)
if raw_values:
return MaskedArray(in1d(self.raw.data, raw_values, invert=invert),
mask=self.mask)
else:
method = MaskedArray.__ne__ if invert else MaskedArray.__eq__
return method(self.raw, self.__coerce_type(other))
def clipping(cut, sigma_thres=5):
""" Masks array elements with extreme values.
"""
iqr2std = 1 / normal.ppf(0.75) # IQR to SD
x0 = np.median(cut) # mean estimate
std = iqr(cut) * iqr2std # stdev estimate
mask = cut > x0 + sigma_thres * std # flags extreme values
clipped = MaskedArray(cut, mask)
return clipped
def test_numpy_date_column(dsn, configuration):
date = datetime.date(3999, 12, 31)
with open_cursor(configuration) as cursor:
with query_fixture(cursor, configuration, "INSERT DATE") as table_name:
cursor.execute(f"INSERT INTO {table_name} VALUES (?)", [date])
cursor.execute(f"SELECT a FROM {table_name}")
results = cursor.fetchallnumpy()
expected = MaskedArray([date], mask=[0], dtype="datetime64[D]")
assert results[_fix_case(
configuration, "a")].dtype == numpy.dtype("datetime64[D]")
assert_equal(results[_fix_case(configuration, "a")], expected)
