def __init__(self, name, input_shape, output_dim, hidden_dim, hidden_nonlinearity=tf.nn.relu,
lstm_layer_cls=L.LSTMLayer,
output_nonlinearity=None, input_var=None, input_layer=None, forget_bias=1.0, use_peepholes=False,
layer_args=None):
with tf.variable_scope(name):
if input_layer is None:
l_in = L.InputLayer(shape=(None, None) + input_shape, input_var=input_var, name="input")
else:
l_in = input_layer
l_step_input = L.InputLayer(shape=(None,) + input_shape, name="step_input")
# contains previous hidden and cell state
l_step_prev_state = L.InputLayer(shape=(None, hidden_dim * 2), name="step_prev_state")
if layer_args is None:
layer_args = dict()
l_lstm = lstm_layer_cls(l_in, num_units=hidden_dim, hidden_nonlinearity=hidden_nonlinearity,
hidden_init_trainable=False, name="lstm", forget_bias=forget_bias,
cell_init_trainable=False, use_peepholes=use_peepholes, **layer_args)
l_lstm_flat = L.ReshapeLayer(
l_lstm, shape=(-1, hidden_dim),
name="lstm_flat"
)
l_output_flat = L.DenseLayer(
l_lstm_flat,
num_units=output_dim,
nonlinearity=output_nonlinearity,
name="output_flat"
)
l_output = L.OpLayer(
l_output_flat,
op=lambda flat_output, l_input:
tf.reshape(flat_output, tf.pack((tf.shape(l_input)[0], tf.shape(l_input)[1], -1))),
shape_op=lambda flat_output_shape, l_input_shape:
(l_input_shape[0], l_input_shape[1], flat_output_shape[-1]),
extras=[l_in],
name="output"
)
l_step_state = l_lstm.get_step_layer(l_step_input, l_step_prev_state, name="step_state")
l_step_hidden = L.SliceLayer(l_step_state, indices=slice(hidden_dim), name="step_hidden")
l_step_cell = L.SliceLayer(l_step_state, indices=slice(hidden_dim, None), name="step_cell")
l_step_output = L.DenseLayer(
l_step_hidden,
num_units=output_dim,
nonlinearity=output_nonlinearity,
W=l_output_flat.W,
b=l_output_flat.b,
name="step_output"
)
self._l_in = l_in
self._hid_init_param = l_lstm.h0
self._cell_init_param = l_lstm.c0
self._l_lstm = l_lstm
self._l_out = l_output
self._l_step_input = l_step_input
self._l_step_prev_state = l_step_prev_state
self._l_step_hidden = l_step_hidden
self._l_step_cell = l_step_cell
self._l_step_state = l_step_state
self._l_step_output = l_step_output
self._hidden_dim = hidden_dim
def test_buffer(self):
a = array.array(self.typecode, self.example)
m = memoryview(a)
expected = m.tobytes()
self.assertEqual(a.tobytes(), expected)
self.assertEqual(a.tobytes()[0], expected[0])
# Resizing is forbidden when there are buffer exports.
# For issue 4509, we also check after each error that
# the array was not modified.
self.assertRaises(BufferError, a.append, a[0])
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, a.extend, a[0:1])
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, a.remove, a[0])
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, a.pop, 0)
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, a.fromlist, a.tolist())
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, a.frombytes, a.tobytes())
self.assertEqual(m.tobytes(), expected)
if self.typecode == 'u':
self.assertRaises(BufferError, a.fromunicode, a.tounicode())
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, operator.imul, a, 2)
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, operator.imul, a, 0)
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, operator.setitem, a, slice(0, 0), a)
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, operator.delitem, a, 0)
self.assertEqual(m.tobytes(), expected)
self.assertRaises(BufferError, operator.delitem, a, slice(0, 1))
self.assertEqual(m.tobytes(), expected)
def slicer_samples(shape):
""" Generator returns slice samples for given `shape`
"""
ndim = len(shape)
slicers_list = []
for i in range(ndim):
slicers_list.append(_slices_for_len(shape[i]))
for sliceobj in product(*slicers_list):
yield sliceobj
# Nones and ellipses
yield (None,)
if ndim == 0:
return
yield (None, 0)
yield (0, None)
yield (Ellipsis, -1)
yield (-1, Ellipsis)
yield (None, Ellipsis)
yield (Ellipsis, None)
yield (Ellipsis, None, None)
if ndim == 1:
return
yield (0, None, slice(None))
yield (Ellipsis, -1, None)
yield (0, Ellipsis, None)
if ndim == 2:
return
yield (slice(None), 0, -1, None)
def return_patches(input_image, target_image, PATCH_SIZE):
input_patches = []
targets = []
input_image_height = input_image.shape[1]
input_image_width = input_image.shape[2]
for i in xrange(input_image_height-PATCH_SIZE+1):
for j in xrange(input_image_width-PATCH_SIZE+1):
height_slice = slice(i, i+PATCH_SIZE)
width_slice = slice(j, j+PATCH_SIZE)
patch = input_image[:, height_slice, width_slice]
#We pick the target label as the label of the central pixel of the patch
target = int(target_image[i+(PATCH_SIZE-1)/2, j+(PATCH_SIZE-1)/2])
if target==0:
continue
else:
targets.append(target)
input_patches.append(patch)
return input_patches, targets
def test_insert():
x = np.random.randint(10, size=(10, 10))
a = from_array(x, chunks=(5, 5))
y = np.random.randint(10, size=(5, 10))
b = from_array(y, chunks=(4, 4))
assert eq(np.insert(x, 0, -1, axis=0), insert(a, 0, -1, axis=0))
assert eq(np.insert(x, 3, -1, axis=-1), insert(a, 3, -1, axis=-1))
assert eq(np.insert(x, 5, -1, axis=1), insert(a, 5, -1, axis=1))
assert eq(np.insert(x, -1, -1, axis=-2), insert(a, -1, -1, axis=-2))
assert eq(np.insert(x, [2, 3, 3], -1, axis=1),
insert(a, [2, 3, 3], -1, axis=1))
assert eq(np.insert(x, [2, 3, 8, 8, -2, -2], -1, axis=0),
insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0))
assert eq(np.insert(x, slice(1, 4), -1, axis=1),
insert(a, slice(1, 4), -1, axis=1))
assert eq(np.insert(x, [2] * 3 + [5] * 2, y, axis=0),
insert(a, [2] * 3 + [5] * 2, b, axis=0))
assert eq(np.insert(x, 0, y[0], axis=1),
insert(a, 0, b[0], axis=1))
assert raises(NotImplementedError, lambda: insert(a, [4, 2], -1, axis=0))
assert raises(IndexError, lambda: insert(a, [3], -1, axis=2))
assert raises(IndexError, lambda: insert(a, [3], -1, axis=-3))
assert same_keys(insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0),
insert(a, [2, 3, 8, 8, -2, -2], -1, axis=0))
def shift(a, shift, axis=None, circular=True):
"""Shift array along a given axis.
If circular is False, zeros are inserted for elements rolled off
the end of the array.
See Also
--------
np.roll
"""
aroll = np.roll(a, shift, axis)
if not circular and shift != 0:
if axis is None:
arollflattened = aroll.flatten()
if shift > 0:
arollflattened[:shift] = 0
elif shift < 0:
arollflattened[shift:] = 0
aroll = np.reshape(arollflattened, aroll.shape)
else:
index = [slice(None)] * a.ndim
if shift > 0:
index[axis] = slice(0, shift)
elif shift < 0:
index[axis] = slice(shift, None)
aroll[index] = 0
return aroll
def _extract_shifted_mean_gauss(image, mask = slice(None), offset = None, sigma = 1, voxelspacing = None):
"""
Internal, single-image version of `shifted_mean_gauss`.
"""
# set voxel spacing
if voxelspacing is None:
voxelspacing = [1.] * image.ndim
# set offset
if offset is None:
offset = [0] * image.ndim
# determine gaussian kernel size in voxel units
sigma = _create_structure_array(sigma, voxelspacing)
# compute smoothed version of image
smoothed = gaussian_filter(image, sigma)
shifted = numpy.zeros_like(smoothed)
in_slicer = []
out_slicer = []
for o in offset:
in_slicer.append(slice(o, None))
out_slicer.append(slice(None, -1 * o))
shifted[out_slicer] = smoothed[in_slicer]
return _extract_intensities(shifted, mask)
def read(self, filename):
data = pickle.load(open(filename))
self.nkpts = data['nkpts']
if 'swaps' in data:
# This is an old file:
self.op_scc = np.array([np.identity(3, int)[list(swap)]
for swap in data['swaps']])
else:
self.op_scc = data['symmetry operations']
self.weight_k = data['weight_k']
self.spin_k = data['spin_k']
self.dim = data['dim']
k1, k2 = self.k1, self.k2
if k2 is None:
k2 = self.nkpts
a_kci, b_kci = data['ab']
self.a_uci = a_kci[k1:k2].copy()
self.b_uci = b_kci[k1:k2].copy()
if self.wfs is not None and 'arrays' in data:
print('reading arrays')
w_kcG, wold_kcG, y_kcG = data['arrays']
i = [slice(k1, k2), slice(0, self.dim)] + self.wfs.gd.get_slice()
self.w_ucG = w_kcG[i].copy()
self.wold_ucG = wold_kcG[i].copy()
self.y_ucG = y_kcG[i].copy()
def setUp(self):
self.ntime = 2048
self.nfreq = 10
self.data = sp.zeros((self.ntime, 4, self.nfreq))
self.n_bins_cal = 64
# Set channel dependant gain.
self.level = 0.1*(self.nfreq + sp.arange(self.nfreq))
# Add noise.
self.data[:,:,:] += (0.1 * self.level
* rand.randn(self.ntime, 4, self.nfreq))
# Add DC level.
self.dc = 10 * self.level
self.data += self.dc
# First can transition.
self.first_trans = rand.randint(0, self.n_bins_cal // 2)
# The following randomly assigns self.neg to -1 or 1.
self.neg = 0
while not self.neg: self.neg = rand.randint(-1, 2)
# First upward edge:
if self.neg == 1:
self.offset = self.first_trans
else:
self.offset = self.first_trans + self.n_bins_cal // 2
self.data[:,0,:] += self.level
for ii in range(self.ntime//self.n_bins_cal) :
s = slice(self.first_trans + ii*self.n_bins_cal, self.first_trans +
(2*ii+1)*self.n_bins_cal//2)
self.data[s, 0, :] += self.neg * self.level
# Transition values and locations.
self.t_slice = slice(self.first_trans, sys.maxint, self.n_bins_cal//2)
self.t_vals = 0.5 + 0.1 * rand.randn(2*self.ntime//self.n_bins_cal,
self.nfreq)
self.t_vals *= - self.level
def expanded_indexer(key, ndim):
"""Given a key for indexing an ndarray, return an equivalent key which is a
tuple with length equal to the number of dimensions.
The expansion is done by replacing all `Ellipsis` items with the right
number of full slices and then padding the key with full slices so that it
reaches the appropriate dimensionality.
"""
if not isinstance(key, tuple):
# numpy treats non-tuple keys equivalent to tuples of length 1
key = (key,)
new_key = []
# handling Ellipsis right is a little tricky, see:
# http://docs.scipy.org/doc/numpy/reference/arrays.indexing.html#advanced-indexing
found_ellipsis = False
for k in key:
if k is Ellipsis:
if not found_ellipsis:
new_key.extend((ndim + 1 - len(key)) * [slice(None)])
found_ellipsis = True
else:
new_key.append(slice(None))
else:
new_key.append(k)
if len(new_key) > ndim:
raise IndexError('too many indices')
new_key.extend((ndim - len(new_key)) * [slice(None)])
return tuple(new_key)
def run_tests(df, rhs, right):
# label, index, slice
r, i, s = list('bcd'), [1, 2, 3], slice(1, 4)
c, j, l = ['joe', 'jolie'], [1, 2], slice(1, 3)
left = df.copy()
left.loc[r, c] = rhs
tm.assert_frame_equal(left, right)
left = df.copy()
left.iloc[i, j] = rhs
tm.assert_frame_equal(left, right)
left = df.copy()
with catch_warnings(record=True):
left.ix[s, l] = rhs
tm.assert_frame_equal(left, right)
left = df.copy()
with catch_warnings(record=True):
left.ix[i, j] = rhs
tm.assert_frame_equal(left, right)
left = df.copy()
with catch_warnings(record=True):
left.ix[r, c] = rhs
tm.assert_frame_equal(left, right)
def _infer_interval_breaks(coord, axis=0, check_monotonic=False):
"""
>>> _infer_interval_breaks(np.arange(5))
array([-0.5, 0.5, 1.5, 2.5, 3.5, 4.5])
>>> _infer_interval_breaks([[0, 1], [3, 4]], axis=1)
array([[-0.5, 0.5, 1.5],
[ 2.5, 3.5, 4.5]])
"""
coord = np.asarray(coord)
if check_monotonic and not _is_monotonic(coord, axis=axis):
raise ValueError("The input coordinate is not sorted in increasing "
"order along axis %d. This can lead to unexpected "
"results. Consider calling the `sortby` method on "
"the input DataArray. To plot data with categorical "
"axes, consider using the `heatmap` function from "
"the `seaborn` statistical plotting library." % axis)
deltas = 0.5 * np.diff(coord, axis=axis)
if deltas.size == 0:
deltas = np.array(0.0)
first = np.take(coord, [0], axis=axis) - np.take(deltas, [0], axis=axis)
last = np.take(coord, [-1], axis=axis) + np.take(deltas, [-1], axis=axis)
trim_last = tuple(slice(None, -1) if n == axis else slice(None)
for n in range(coord.ndim))
return np.concatenate([first, coord[trim_last] + deltas, last], axis=axis)
def test_lbcode3x23(self):
time_bounds = np.array([[0.875, 1.125], [1.125, 1.375],
[1.375, 1.625], [1.625, 1.875]])
field = mock.MagicMock(
lbproc=0, bzx=0, bdx=0, lbnpt=3, lbrow=4,
t1=nc_datetime(2000, 1, 2, hour=0, minute=0, second=0),
t2=nc_datetime(2000, 1, 3, hour=0, minute=0, second=0),
lbtim=mock.Mock(ia=1, ib=2, ic=2),
lbcode=SplittableInt(31323, {'iy': slice(0, 2),
'ix': slice(2, 4)}),
x_bounds=None,
y_bounds=time_bounds,
_x_coord_name=lambda: 'longitude',
_y_coord_name=lambda: 'latitude')
spec = ['lbtim', 'lbcode', 'lbrow', 'lbnpt', 'lbproc', 'lbsrce',
'lbuser', 'bzx', 'bdx', 'bdy', 'bmdi', 't1', 't2', 'stash',
'x_bounds', 'y_bounds', '_x_coord_name', '_y_coord_name']
field.mock_add_spec(spec)
res = _all_other_rules(field)[DIM_COORDS_INDEX]
expected_time_points = np.array([1, 1.25, 1.5, 1.75]) + (2000 * 360)
expected_unit = Unit('days since 0000-01-01 00:00:00',
calendar=CALENDAR_360_DAY)
expected = [(DimCoord(expected_time_points, standard_name='time',
units=expected_unit, bounds=time_bounds), 0)]
self.assertCoordsAndDimsListsMatch(res, expected)
def load_imgs(folder_path='./tmp/', slices=(slice(70, 195), slice(78, 172))):
filepaths = [os.path.join(folder_path, f) for f in os.listdir(folder_path)]
n_imgs = len(filepaths)
data = np.zeros((n_imgs, 64*64))
target = np.zeros(n_imgs)
for i, f in enumerate(filepaths):
print f
img = imread(f)
if img.shape[0] > 195 and img.shape[1] > 172 :
img = img[slices]
img = np.asarray(img, dtype=np.float32)
img = imresize(img, (64,64))
img = img.mean(2)
img /= 255.0
data[i, ...] = img.ravel()
if f.find('cat') >= 0 :
target[i] = 0
else:
target[i] = 1
indices = np.arange(n_imgs)
np.random.RandomState(42).shuffle(indices)
data, target = data[indices], target[indices]
return data, target
def extract_images(self, overwrite=False):
for setn in ('train', 'val'):
img_dir = os.path.join(self.out_dir, setn)
neon_logger.display("Extracting %s files" % (setn))
toptar = getattr(self, setn + '_tar')
label_dict = getattr(self, setn + '_labels')
name_slice = slice(None, 9) if setn == 'train' else slice(15, -5)
with tarfile.open(toptar) as tf:
for s in tf.getmembers():
label = label_dict[s.name[name_slice]]
subpath = os.path.join(img_dir, str(label))
if not os.path.exists(subpath):
os.makedirs(subpath)
if setn == 'train':
tarfp = tarfile.open(fileobj=tf.extractfile(s))
file_list = tarfp.getmembers()
else:
tarfp = tf
file_list = [s]
for fobj in file_list:
fname = os.path.join(subpath, fobj.name)
if not os.path.exists(fname) or overwrite:
with open(fname, 'wb') as jf:
jf.write(tarfp.extractfile(fobj).read())
def show3D():
data = layer._datasources[0].request((slice(0,1,None),
slice(None,None,None), slice(None,None,None),
slice(None,None,None),
slice(0,1,None))).wait()[0,:,:,:,0]
self.SetColorTable(layer._colorTable)
self.DisplayObjectMeshes(data)#, suppressLabels=(), smooth=True):
def numdiff1(f, x0, dv=1e-8):
'''Returns the derivative of f w.r.t. to multidimensional vector x0
If x0 is of dimension R1 x ... x Rd x Rn dimension of f is assumed to be
in the form S1 x ... x Sf x Rn. The last dimension corresponds to various
observations. The value returned is of dimension :
S1 x ... x Sf x R1 x ... x Rd x Rn
'''
in_shape = x0.shape
nobs = in_shape[-1]
dd = in_shape[:-1]
f0 = f(x0)
assert(f0.shape[-1] == nobs)
f_shape = f0.shape[:-1]
out_shape = f_shape + dd + (nobs,)
ret = np.zeros(out_shape)
for ind in product( *[range(i) for i in dd] ):
sl = ind + (slice(None, None, None), )
x = x0.copy()
x[sl] += dv
x2 = x0.copy()
x2[sl] -= dv
df = (f(x) - f(x2))/dv/2.0
obj = [ Ellipsis] + list(ind) + [slice(None, None, None)]
obj = tuple(obj)
ret[obj] = df
return ret
def Z_LSTM(input_var, z_dim=256, nhid=512, layers=2, gradclip=10, training=True):
ret = {}
state_vars = []
ret['input'] = layer = nn.layers.InputLayer(input_var=input_var, shape=(None, None, z_dim))
batchsize, seqlen, _ = layer.input_var.shape
for lay in xrange(layers):
ret['drop_{}'.format(lay)] = layer = nn.layers.DropoutLayer(layer, p=0.3)
if training:
ret['lstm_{}'.format(lay)] = layer = LSTMSampleableLayer(layer, nhid,
grad_clipping=gradclip, learn_init=True)
else:
cell_var = T.matrix('cell_var_{}'.format(lay))
hid_var = T.matrix('hid_var_{}'.format(lay))
state_vars.append(cell_var)
state_vars.append(hid_var)
ret['lstm_{}'.format(lay)] = layer = LSTMSampleableLayer(layer, nhid,
cell_init=cell_var, hid_init=hid_var)
ret['cell_{}'.format(lay)] = nn.layers.SliceLayer(layer, axis=2,
indices=slice(None,nhid))
ret['hid_{}'.format(lay)] = layer = nn.layers.SliceLayer(layer, axis=2,
indices=slice(nhid,None))
ret['reshape'] = layer = nn.layers.ReshapeLayer(layer, (-1, nhid))
ret['project'] = layer = nn.layers.DenseLayer(layer, num_units=z_dim, nonlinearity=None)
ret['output'] = layer = nn.layers.ReshapeLayer(layer, (batchsize, seqlen, z_dim))
# final state slice layers for passing to next instance of lstm
for lay in xrange(layers):
ret['cellfinal_{}'.format(lay)] = nn.layers.SliceLayer(ret['cell_{}'.format(lay)],
axis=1, indices=-1)
ret['hidfinal_{}'.format(lay)] = nn.layers.SliceLayer(ret['hid_{}'.format(lay)],
axis=1, indices=-1)
return ret, state_vars
def test_loc_non_unique(self):
# GH3659
# non-unique indexer with loc slice
# https://groups.google.com/forum/?fromgroups#!topic/pydata/zTm2No0crYs
# these are going to raise because the we are non monotonic
df = DataFrame({'A': [1, 2, 3, 4, 5, 6],
'B': [3, 4, 5, 6, 7, 8]}, index=[0, 1, 0, 1, 2, 3])
pytest.raises(KeyError, df.loc.__getitem__,
tuple([slice(1, None)]))
pytest.raises(KeyError, df.loc.__getitem__,
tuple([slice(0, None)]))
pytest.raises(KeyError, df.loc.__getitem__, tuple([slice(1, 2)]))
# monotonic are ok
df = DataFrame({'A': [1, 2, 3, 4, 5, 6],
'B': [3, 4, 5, 6, 7, 8]},
index=[0, 1, 0, 1, 2, 3]).sort_index(axis=0)
result = df.loc[1:]
expected = DataFrame({'A': [2, 4, 5, 6], 'B': [4, 6, 7, 8]},
index=[1, 1, 2, 3])
tm.assert_frame_equal(result, expected)
result = df.loc[0:]
tm.assert_frame_equal(result, df)
result = df.loc[1:2]
expected = DataFrame({'A': [2, 4, 5], 'B': [4, 6, 7]},
index=[1, 1, 2])
tm.assert_frame_equal(result, expected)
def array_split(ary, indices_or_sections, axis=0):
"""Splits an array into multiple sub arrays along a given axis.
This function is almost equivalent to :func:`cupy.split`. The only
difference is that this function allows an integer sections that does not
evenly divide the axis.
.. seealso:: :func:`cupy.split` for more detail, :func:`numpy.array_split`
"""
ndim = ary.ndim
if -ndim > axis or ndim <= axis:
raise IndexError('Axis exceeds ndim')
axis %= ndim
size = ary.shape[axis]
if numpy.isscalar(indices_or_sections):
each_size = (size - 1) // indices_or_sections + 1
indices = [i * each_size
for i in six.moves.range(1, indices_or_sections)]
else:
indices = indices_or_sections
if len(indices) == 0:
return [ary]
skip = (slice(None),) * axis
ret = []
i = 0
for index in indices:
ret.append(ary[skip + (slice(i, index),)])
i = index
ret.append(ary[skip + (slice(i, size),)])
return ret
def _defitsify_slice(slices):
"""
Convert a FITS-style slice specification into a python slice.
This means two things:
+ Subtract 1 from starting index because in the FITS
specification arrays are one-based.
+ Do **not** subtract 1 from the ending index because the python
convention for a slice is for the last value to be one less than the
stop value. In other words, this subtraction is already built into
python.
+ Reverse the order of the slices, because the FITS specification dictates
that the first axis is the one along which the index varies most rapidly
(aka FORTRAN order).
"""
python_slice = []
for a_slice in slices[::-1]:
new_start = a_slice.start - 1 if a_slice.start is not None else None
if new_start is not None and new_start < 0:
raise ValueError("Smallest permissible FITS index is 1")
if a_slice.stop is not None and a_slice.stop < 0:
raise ValueError("Negative final index not allowed for FITS slice")
new_slice = slice(new_start, a_slice.stop, a_slice.step)
if (a_slice.start is not None and a_slice.stop is not None and
a_slice.start > a_slice.stop):
# FITS use a positive step index when dimension are inverted
new_step = -1 if a_slice.step is None else -a_slice.step
# Special case to prevent -1 as slice stop value
new_stop = None if a_slice.stop == 1 else a_slice.stop-2
new_slice = slice(new_start, new_stop, new_step)
python_slice.append(new_slice)
return python_slice
def test_decision_function():
"""
Test decision_function
Sanity check, test that decision_function implemented in python
returns the same as the one in libsvm
TODO: proabably could be simplified
"""
clf = svm.SVC(kernel='linear').fit(iris.data, iris.target)
data = iris.data[0]
sv_start = np.r_[0, np.cumsum(clf.n_support_)]
n_class = 3
kvalue = np.dot(data, clf.support_vectors_.T)
dec = np.empty(n_class * (n_class - 1) / 2)
p = 0
for i in range(n_class):
for j in range(i + 1, n_class):
coef1 = clf.dual_coef_[j - 1]
coef2 = clf.dual_coef_[i]
idx1 = slice(sv_start[i], sv_start[i + 1])
idx2 = slice(sv_start[j], sv_start[j + 1])
s = np.dot(coef1[idx1], kvalue[idx1]) + \
np.dot(coef2[idx2], kvalue[idx2]) + \
clf.intercept_[p]
dec[p] = s
p += 1
assert_array_almost_equal(-dec, np.ravel(clf.decision_function(data)))
def PruneArray(data, kernel_shape, scaling):
"""Crop correlation results.
Scipy's :func:`scipy.ndimage.correlate` applies kernel to border units (in the
last two dimensions), and performs no subsampling. This function returns a
cropped view of the result array, in which border units have been removed, and
subsampling has been performed.
:param data: Result from one or more :func:`scipy.ndimage.correlate` calls
:type data: ndarray of float
:param kernel_shape: Shape of the kernel passed to
:func:`scipy.ndimage.correlate`. If None, then cropping is disabled and
only subsampling is performed.
:type kernel_shape: tuple of int
:param scaling: Subsampling factor.
:type scaling: positive int
:return: Cropped, sampled view (not copy) of the input data.
:rtype: ndarray of float
"""
if kernel_shape == None:
# Keep all bands
band_indices = [slice(None)] * (data.ndim - 2)
# Perform subsampling of maps
map_indices = [slice(None, None, scaling)] * 2
else:
# Keep all bands
band_indices = [slice(None)] * (data.ndim - 2)
# Crop borders of maps by half the kernel width, and perform subsampling.
assert len(kernel_shape) >= 2
h, w = kernel_shape[-2:]
hh = int(h / 2)
hw = int(w / 2)
map_indices = [slice(hh, -hh, scaling), slice(hw, -hw, scaling)]
return data[band_indices + map_indices]
def load_datastream(train_batch_size=100):
from fuel.datasets.mnist import MNIST
from fuel.transformers import ScaleAndShift, Cast, Flatten, Mapping
from fuel.streams import DataStream
from fuel.schemes import SequentialScheme, ShuffledScheme
MNIST.default_transformers = (
(ScaleAndShift, [2.0 / 255.0, -1], {'which_sources': 'features'}),
(Cast, [np.float32], {'which_sources': 'features'}),
)
mnist_train = MNIST(('train',), subset=slice(None, 50000))
mnist_train_stream = DataStream.default_stream(
mnist_train,
iteration_scheme=ShuffledScheme(mnist_train.num_examples, train_batch_size)
)
mnist_validation = MNIST(('train',), subset=slice(50000, None))
mnist_validation_stream = DataStream.default_stream(
mnist_validation,
iteration_scheme=SequentialScheme(mnist_validation.num_examples, 250)
)
mnist_test = MNIST(('test',))
mnist_test_stream = DataStream.default_stream(
mnist_test,
iteration_scheme=SequentialScheme(mnist_test.num_examples, 250)
)
return {
'train': mnist_train_stream,
'validation': mnist_validation_stream,
'test': mnist_test_stream
}
def _compute_qth_percentile(sorted, q, axis, out):
if not np.isscalar(q):
p = [_compute_qth_percentile(sorted, qi, axis, None)
for qi in q]
if out is not None:
out.flat = p
return p
q = q / 100.0
if (q < 0) or (q > 1):
raise ValueError, "percentile must be either in the range [0,100]"
indexer = [slice(None)] * sorted.ndim
Nx = sorted.shape[axis]
index = q*(Nx-1)
i = int(index)
if i == index:
indexer[axis] = slice(i, i+1)
weights = np.array(1)
sumval = 1.0
else:
indexer[axis] = slice(i, i+2)
j = i + 1
weights = np.array([(j - index), (index - i)],float)
wshape = [1]*sorted.ndim
wshape[axis] = 2
weights.shape = wshape
sumval = weights.sum()
return np.add.reduce(sorted[indexer]*weights, axis=axis, out=out)/sumval
def by_column(self, mask=None):
"""
Iterate over the columns of the array. Columns will be yielded either
as a 1D array or as a single value (for a flat array).
`mask`: either `None` or a boolean array indicating which columns should be included.
"""
column_indices = numpy.arange(self.ncols)
if mask is not None:
assert len(mask) == self.ncols
column_indices = column_indices[mask]
if isinstance(self.base_value, RandomDistribution) and self.base_value.rng.parallel_safe:
if mask is None:
for j in column_indices:
yield self._apply_operations(self.base_value.next(self.nrows, mask_local=False),
(slice(None), j))
else:
column_indices = numpy.arange(self.ncols)
for j,local in zip(column_indices, mask):
col = self.base_value.next(self.nrows, mask_local=False)
if local:
yield self._apply_operations(col, (slice(None), j))
else:
for j in column_indices:
yield self._partially_evaluate((slice(None), j), simplify=True)
def setUp(self):
self.a = np.arange(np.prod([3,1,5,6])).reshape(3,1,5,6)
self.b = np.empty((3,0,5,6))
self.complex_indices = ['skip', Ellipsis,
0,
# Boolean indices, up to 3-d for some special cases of eating up
# dimensions, also need to test all False
np.array(False),
np.array([True, False, False]),
np.array([[True, False], [False, True]]),
np.array([[[False, False], [False, False]]]),
# Some slices:
slice(-5, 5, 2),
slice(1, 1, 100),
slice(4, -1, -2),
slice(None,None,-3),
# Some Fancy indexes:
np.empty((0,1,1), dtype=np.intp), # empty broadcastable
np.array([0,1,-2]),
np.array([[2],[0],[1]]),
np.array([[0,-1], [0,1]]),
np.array([2,-1]),
np.zeros([1]*31, dtype=int), # trigger too large array.
np.array([0., 1.])] # invalid datatype
# Some simpler indices that still cover a bit more
self.simple_indices = [Ellipsis, None, -1, [1], np.array([True]), 'skip']
# Very simple ones to fill the rest:
self.fill_indices = [slice(None,None), 0]
def test_itemgetter(self):
""" Verifies that itemgetter works. """
helper = helpers.itemgetter
# Verify simple index access
data = list(range(5))
secondary_helper = helper(-1)
output = secondary_helper(data)
self.assertEqual(output, data[-1])
# Verify slicing access
secondary_helper = helper(slice(1, 3))
output = secondary_helper(data)
self.assertEqual(output, data[1:3])
# Verify ellipsis works for strings
data = str(range(10))
secondary_helper = helper(slice(0, 5), ellipsis=True)
output = secondary_helper(data)
self.assertEqual(output, data[:5] + "...")
# Verify ellipsis can be customized
secondary_helper = helper(slice(0, 5), ellipsis="custom")
output = secondary_helper(data)
self.assertEqual(output, data[:5] + "custom")
# Verify ellipsis does nothing for non-string data types
data = range(10)
output = secondary_helper(data)
self.assertEqual(output, data[:5])
def __getitem__(self, index):
"""
Return a new arrayterator.
"""
# Fix index, handling ellipsis and incomplete slices.
if not isinstance(index, tuple):
index = (index,)
fixed = []
length, dims = len(index), len(self.shape)
for slice_ in index:
if slice_ is Ellipsis:
fixed.extend([slice(None)] * (dims - length + 1))
length = len(fixed)
elif isinstance(slice_, (int, long)):
fixed.append(slice(slice_, slice_ + 1, 1))
else:
fixed.append(slice_)
index = tuple(fixed)
if len(index) < dims:
index += (slice(None),) * (dims - len(index))
# Return a new arrayterator object.
out = self.__class__(self.var, self.buf_size)
for i, (start, stop, step, slice_) in enumerate(zip(self.start, self.stop, self.step, index)):
out.start[i] = start + (slice_.start or 0)
out.step[i] = step * (slice_.step or 1)
out.stop[i] = start + (slice_.stop or stop - start)
out.stop[i] = min(stop, out.stop[i])
return out
def launch(self, time_series, nfft=None):
"""
Launch algorithm and build results.
"""
##--------- Prepare a CoherenceSpectrum object for result ------------##
coherence = CoherenceSpectrum(source=time_series,
nfft=self.algorithm.nfft,
storage_path=self.storage_path)
##------------- NOTE: Assumes 4D, Simulator timeSeries. --------------##
node_slice = [slice(self.input_shape[0]), None, slice(self.input_shape[2]), slice(self.input_shape[3])]
##---------- Iterate over slices and compose final result ------------##
small_ts = TimeSeries(use_storage=False)
small_ts.sample_rate = time_series.sample_rate
partial_coh = None
for var in range(self.input_shape[1]):
node_slice[1] = slice(var, var + 1)
small_ts.data = time_series.read_data_slice(tuple(node_slice))
self.algorithm.time_series = small_ts
partial_coh = self.algorithm.evaluate()
coherence.write_data_slice(partial_coh)
coherence.frequency = partial_coh.frequency
coherence.close_file()
return coherence
def test_current_interpolation():
## Passing a Builder will raise an error
pytest.raises(TypeError, plotter.interpolate_current, syst_2d(), None)
def R(theta):
return ta.array([[cos(theta), -sin(theta)], [sin(theta), cos(theta)]])
def make_lattice(a, theta):
x = ta.dot(R(theta), (a, 0))
y = ta.dot(R(theta), (0, a))
return kwant.lattice.general([x, y], norbs=1)
_test_border_0(plotter.interpolate_current)
## Check current through cross section is same for different lattice
## parameters and orientations of the system wrt. the discretization grid
for a, theta, width in [(1, 0, 1),
(1, 0, 0.5),
(2, 0, 1),
(1, 0.2, 1),
(2, 0.4, 1)]:
lat = make_lattice(a, theta)
syst = syst_rect(lat, salt='0').finalized()
psi = kwant.wave_function(syst, energy=3)(0)
def cut(a, b):
return b.tag[0] < 0 and a.tag[0] >= 0
J = kwant.operator.Current(syst).bind()
J_cut = kwant.operator.Current(syst, where=cut, sum=True).bind()
J_exact = J_cut(psi[0])
data = []
for n in [4, 6, 8, 11, 16]:
j0, box = plotter.interpolate_current(syst, J(psi[0]),
n=n, abswidth=width)
x, y = (np.linspace(mn, mx, shape)
for (mn, mx), shape in zip(box, j0.shape))
# slice field perpendicular to a cut along the y axis
y_axis = (np.argmin(np.abs(x)), slice(None), 0)
J_interp = scipy.integrate.simps(j0[y_axis], y)
data.append((n, abs(J_interp - J_exact)))
# 3rd value returned from 'linregress' is 'rvalue'
# TODO: review this value once #280 has been dealt with.
assert scipy.stats.linregress(np.log(data))[2] < -0.7
### Tests on a divergence-free current (closed system)
lat = kwant.lattice.general([(1, 0), (0.5, np.sqrt(3) / 2)], norbs=1)
syst = kwant.Builder()
sites = [lat(0, 0), lat(1, 0), lat(0, 1), lat(2, 2)]
syst[sites] = None
syst[((s, t) for s, t in itertools.product(sites, sites) if s != t)] = None
del syst[lat(0, 0), lat(2, 2)]
syst = syst.finalized()
# generate random divergence-free currents
Js = rotational_currents(syst.graph)
rng = ensure_rng(3)
J0 = sum(rng.rand(len(Js))[:, None] * Js)
J1 = sum(rng.rand(len(Js))[:, None] * Js)
# Sanity check that diverence on the graph is 0
divergence = np.zeros(len(syst.sites))
for (a, _), current in zip(syst.graph, J0):
divergence[a] += current
assert np.allclose(divergence, 0)
j0, _ = plotter.interpolate_current(syst, J0)
j1, _ = plotter.interpolate_current(syst, J1)
## Test linearity of interpolation.
j_tot, _ = plotter.interpolate_current(syst, J0 + 2 * J1)
assert np.allclose(j_tot, j0 + 2 * j1)
## Test that divergence of interpolated current approaches zero as we make
## the interpolation finer.
data = []
for n in [4, 6, 8, 11, 16]:
j, box = plotter.interpolate_current(syst, J0, n=n)
dx = [(mx - mn) / (shape - 1) for (mn, mx), shape in zip(box, j.shape)]
div_j = np.max(np.abs(div(j, dx)))
data.append((n, div_j))
# 3rd value returned from 'linregress' is 'rvalue'
# TODO: review this value once #280 has been dealt with.
assert scipy.stats.linregress(np.log(data))[2] < -0.7
def prepare(
self,
segments: Union[List[Text], Tuple[Text], Text, slice],
col_set=DataHandler.CS_ALL,
data_key=DataHandlerLP.DK_I,
**kwargs,
) -> Union[List[pd.DataFrame], pd.DataFrame]:
"""
Prepare the data for learning and inference.
Parameters
----------
segments : Union[List[Text], Tuple[Text], Text, slice]
Describe the scope of the data to be prepared
Here are some examples:
- 'train'
- ['train', 'valid']
col_set : str
The col_set will be passed to self.handler when fetching data.
data_key : str
The data to fetch: DK_*
Default is DK_I, which indicate fetching data for **inference**.
kwargs :
The parameters that kwargs may contain:
flt_col : str
It only exists in TSDatasetH, can be used to add a column of data(True or False) to filter data.
This parameter is only supported when it is an instance of TSDatasetH.
Returns
-------
Union[List[pd.DataFrame], pd.DataFrame]:
Raises
------
NotImplementedError:
"""
logger = get_module_logger("DatasetH")
fetch_kwargs = {"col_set": col_set}
fetch_kwargs.update(kwargs)
if "data_key" in getfullargspec(self.handler.fetch).args:
fetch_kwargs["data_key"] = data_key
else:
logger.info(f"data_key[{data_key}] is ignored.")
# Handle all kinds of segments format
if isinstance(segments, (list, tuple)):
return [
self._prepare_seg(slice(*self.segments[seg]), **fetch_kwargs)
for seg in segments
]
elif isinstance(segments, str):
return self._prepare_seg(slice(*self.segments[segments]),
**fetch_kwargs)
elif isinstance(segments, slice):
return self._prepare_seg(segments, **fetch_kwargs)
else:
raise NotImplementedError(f"This type of input is not supported")
def _fold_in_i_dir(solution_matrix, dim, fdim, size_in_fdim, fold_idx):
"""
Finds T_|m-n| (Referred to as folding in proceeding documentation)
for a given dimension of a matrix.
Parameters
----------
solution_matrix : ndarray
Polynomial to by folded.
dim : int
The number of dimensions in solution_matrix.
fdim : int
The dimension being folded.
size_in_fdim : int
The size of the solution matrix in the dimension being folded.
fold_idx : int
The index to fold around.
Returns
-------
sol : ndarray
"""
if fold_idx == 0:
return solution_matrix
sol = np.zeros_like(solution_matrix) #Matrix of zeroes used to insert the new values..
slice_0 = slice(None, 1, None) # index to take first slice
slice_1 = slice(fold_idx, fold_idx+1, None) # index to take slice that contains the axis folding around.
#indexers are made with a slice index for every dimension.
indexer1 = [slice(None)]*dim
indexer2 = [slice(None)]*dim
indexer3 = [slice(None)]*dim
#Changes the index in each indexer for the correct dimension
indexer1[fdim] = slice_0
indexer2[fdim] = slice_1
#makes first slice in sol equal to the slice we fold around in solution_matrix
sol[tuple(indexer1)] = solution_matrix[tuple(indexer2)]
#Loop adds the slices above and below the slice we rotate around and inserts solutions in sol.
for n in range(size_in_fdim):
slice_2 = slice(n+1, n+2, None) #Used to imput new values in sol.
slice_3 = slice(fold_idx+n+1, fold_idx+n+2, None) #Used to find slices that are n above fold_idx
slice_4 = slice(fold_idx-n-1, fold_idx-n, None) #Used to find slices that are n below fold_idx
indexer1[fdim] = slice_2
indexer2[fdim] = slice_3
indexer3[fdim] = slice_4
#if statement checks to ensure that slices to be added are contained in the matrix.
if fold_idx-n-1 < 0:
if fold_idx+n+2 > size_in_fdim:
break
else:
sol[tuple(indexer1)] = solution_matrix[tuple(indexer2)]
else:
if fold_idx+n+2 > size_in_fdim:
sol[tuple(indexer1)] = solution_matrix[tuple(indexer3)]
else:
sol[tuple(indexer1)] = solution_matrix[tuple(indexer3)] + solution_matrix[tuple(indexer2)]
return sol
def flip(x, dim=-1):
indices = [slice(None)] * x.dim()
indices[dim] = torch.arange(
x.size(dim) - 1, -1, -1, dtype=torch.long, device=x.device
)
return x[tuple(indices)]
assert test.__doc__ == """original text"""
test.__doc__ = "new text"
assert test.__doc__ == "new text"
# issue 443
class Pepe:
def __getitem__(self, arg):
return arg
pepe = Pepe()
assert pepe[0:1] == slice(0, 1)
assert pepe[1, 0, 0:10:2] == (1, 0, slice(0, 10, 2))
assert pepe[0, 0:10:1] == (0, slice(0, 10, 1))
assert pepe[0, 0] == (0, 0)
assert pepe[0, :] == (0, slice(None, None, None))
assert pepe[0, 1, 1, 1, 2, 3, 4, :] == \
(0, 1, 1, 1, 2, 3, 4, slice(None, None, None))
# issue 448
d = {0: [1]}
d[0] += [2]
assert d == {0: [1, 2]}
# issue 449
a = [[1]]
class GameSim():
_prefix_ = ''
# AB = O+E+K+S+D+T+HR
# PA = O+E+K+BB+IBB+HBP+I+S+D+T+HR
AB = {
'O': 1,
'E': 1,
'SH': 0,
'SF': 0,
'I': 0,
'K': 1,
'BB': 0,
'IBB': 0,
'HBP': 0,
'S': 1,
'D': 1,
'T': 1,
'HR': 1
}
E_STR = ('O', 'E', 'K', 'BB', 'IBB', 'HBP', 'I', 'S', 'D', 'T', 'HR', 'WP',
'PB', 'DI', 'OA', 'RUNEVT', 'BK', 'FLE')
E_PA = (1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0)
#POS = ('P','C','1B','2B','3B','SS','LF','CF','RF','DH','PH','PR')
POS = ('P', 'C', '1B', '2B', '3B', 'SS', 'LF', 'CF', 'RF', 'DH')
FPOS = {
'P': 0,
'C': 1,
'1B': 2,
'2B': 3,
'3B': 4,
'SS': 5,
'LF': 6,
'CF': 7,
'RF': 8,
'DH': 9
}
_FPOS = [
'P', 'C', '1B', '2B', '3B', 'SS', 'LF', 'CF', 'RF', 'DH', 'PH', 'PR'
]
PINCH = ('PH', 'PR')
#E,###,evt,code,mod,ra,ra,ra,ra,assist,putout,error
# (i,t,o) (oinc) (a-score,h-score) (e,evt) (bpid,ppid) (blpid,bfpos) (rbi,runflag) (pr-1b,pr-2b,pr-3b)
#eid,i,t,o,asc,hsc,bpid,rbpid,ppid,rppid,blpos,bfpos
CTX = {
'eid': 0,
'i': 1,
't': 2,
'o': 3,
'score': slice(4, 6),
'bases': 6,
'adv': slice(7, 11),
'bpid': 11,
'rbpid': 12,
'ppid': 13,
'rppid': 14,
'batter': slice(11, 13),
'pitcher': slice(13, 15),
'pid': slice(11, 15, 2),
'rpid': slice(12, 15, 2),
'lpos': 15,
'fpos': 16,
'pos': slice(15, 17),
'bhand': 17,
'phand': 18,
'rbhand': 19,
'rphand': 20,
'hand': slice(17, 21),
}
EVENT = {
'n': 0,
'evt': 1,
'code': 2,
'mod': 3,
'badv': 4,
'radv': slice(5, 8),
'adv': slice(4, 8),
'assist': 8,
'putout': 9,
'error': 10,
'dfn': slice(8, 11)
}
SUB = {
'n': 0,
'pid': 1,
't': 2,
'lpos': 3,
'fpos': 4,
'offense': 5,
'count': 6
}
BOOT = {'n': 0, 't': 1, 'lpos': 2}
ADJ = {'n': 0, 'hand': 1},
INFO = {'dh': 0, 'htbf': 1, 'site': 2}
LINEUP = {
'asp': 0,
'asl': slice(1, 10),
'away': slice(0, 10),
'hsp': 10,
'hsl': slice(11, 20),
'home': slice(10, 20)
}
FINAL = {'wp': 0, 'lp': 1, 'sv': 2, 'er': 3}
IS_RBI = (True, True, True, True, True, True, True, True, True, True, True,
False, False, False, False, False, False, False)
def __init__(self):
# Year
self.year = None
# Game ID YYYYMMDDHHHAAAG [Y=year,M=month,D=day,H=hometeam,A=awayteam,G=game number (always 0 unless double header)]
self.gameid = None
# Current Event Number
self.eid = None
# Current Event Code
self.ecode = None
self.emod = None
# Teams
self.teams = None
# Leagues
self.leagues = None
# Site ID
self.site = None
# If game is using the Designated Hitter rule
self.useDH = False
# Binary Flag - number indicating which bases are currently occupied
self.bflg = 0
# Context (inning,top/bottom,outs)
self.i, self.t, self.o = 0, 0, 0
# Box Score
self.score, self.lob = [0, 0], [0, 0]
# Batting Order
self.lpos = [None] * 9, [None] * 9
# At bat index
self.abinx = [0, 0]
# Batting out of turn - flag indicating if team is the process of batting out of order (very rare)
self.bootflg, self.boot = [0, 0], [[], []]
#------------------------------- (Sim)[frame] -------------------------------#
def df(self):
return self.frame.to_dataframe()
#------------------------------- [cycle](Year) -------------------------------#
def initSeason(self, data):
self.year = data.year
def endSeason(self):
self.year = None
#------------------------------- [simgame] -------------------------------#
# Reads the preformatted gamedata file and simulates the next game
def simGame(self, gl):
i, ginfo = next(gl)
i, lineup = next(gl)
self._initGame(*ginfo)
self._lineup(lineup)
for i, l in gl:
try:
if i == 'E':
self._play(*l)
elif i == 'S':
self._sub(l)
elif i == 'O':
self._boot(l)
elif i == 'B':
self._badj(l)
elif i == 'P':
self._padj(l)
except Exception as e:
raise BBSimError(
self.gameid, self.eid,
f"Error Occured while Processing {i},{l}") from e
self._final(l)
self._endGame()
#------------------------------- [cycle](Game) -------------------------------#
def _initGame(self, gameid, dh, htbf, site, hlg, alg):
self.gameid = gameid
self.eid = 0
self.teams = (gameid[11:14], gameid[8:11])
self.leagues = (alg, hlg)
self.useDH = int(dh)
self.t = int(htbf)
self.site = site
def _endGame(self):
'''Clears the simulator in preparation for next game'''
self.gameid, self.eid, = None, None
self.site, self.teams, self.leagues = None, None, None
self.ecode = None
self.bflg = 0
self.i, self.t, self.o = 0, 0, 0
self.score, self.lob = [0, 0], [0, 0]
self.lpos[0][:], self.lpos[1][:] = [None] * 9, [None] * 9
self.abinx[:] = 0, 0
if self.bootflg[0]: self.bootflg[0], self.boot[0] = 0, []
if self.bootflg[1]: self.bootflg[1], self.boot[1] = 0, []
#------------------------------- [Properties] -------------------------------#
@staticmethod
def _bitindexes(flag):
i = 0
while flag > 0:
if flag & 1:
yield i
flag, i = flag >> 1, i + 1
#------------------------------- [Sim Action] -------------------------------#
def _lineup(self, l):
for x, a, h in zip(range(0, 9), l[self.LINEUP['asl']],
l[self.LINEUP['hsl']]):
self.lpos[0][x] = int(a[-1])
self.lpos[1][x] = int(h[-1])
#------------------------------- [sub] -------------------------------#
def _sub(self, l):
"""Performs linup substitution"""
# order=lpos pos=fpos
t, lpos, fpos, offense = tuple(
int(l[x]) for x in [
self.SUB['t'], self.SUB['lpos'], self.SUB['fpos'],
self.SUB['offense']
])
if offense:
if (fpos > 9):
if fpos == 10:
# PINCH HIT
if self._lpos_ != lpos:
raise BBSimSubstitutionError(
f'Pinchit Discrepancy _lpos_[{self._lpos_}] lpos[{lpos}]'
)
else:
if (lpos >= 0):
self.lpos[t][lpos] = fpos
else:
if fpos > 9:
raise BBSimSubstitutionError(f'defensive pinch sub [{fpos}]')
if (lpos >= 0):
self.lpos[t][lpos] = fpos
#------------------------------- [boot] -------------------------------#
def _boot(self, l):
"""handles the rare case of when a team bats out of order"""
assert int(
l[self.BOOT['t']]
) == self.t, 'BOOT team error b[%s] != sim[%i]' % (l[self.BOOT['t']],
self.t)
self.boot[self.t] += [int(l[self.BOOT['lpos']])]
self.bootflg[self.t] = 1
def _bootcycle(self):
"""Ensures the correct player is currently batting in the event of a team batting out of order"""
i, j = self.abinx[self.t], max(self.boot[self.t])
self.abinx[self.t] = (self.abinx[self.t] + (j - i + 1)) % 9
self.bootflg[self.t], self.boot[self.t] = 0, []
#------------------------------- [adj] -------------------------------#
def _padj(self, l):
pass
def _badj(self, l):
pass
def _final(self, l):
pass
#------------------------------- [Properties] -------------------------------#
@property
def date(self):
return self.gameid[:4] + '-' + self.gameid[4:6] + '-' + self.gameid[6:8]
@property
def hometeam(self):
return self.teams[1]
@property
def awayteam(self):
return self.teams[0]
@property
def homeleague(self):
return self.leagues[1]
@property
def awayleague(self):
return self.leagues[0]
@property
def baseoutstate(self):
return self.o << 3 | self.bflg
@property
def inning(self):
return self.i // 2 + 1
@property
def dt(self):
return self.t ^ 1
#------------------------------- [batting-order] -------------------------------#
@property
def _lpos_(self):
"""lineup position of batter"""
return self.boot[self.t][-1] if self.bootflg[self.t] else self.abinx[
self.t]
@property
def _bpid_fpos_(self):
"""field position of batter"""
return self.lpos[self.t][self._lpos_]
#------------------------------- [play] -------------------------------#
def _advance(self, badv, radv):
advflg = 0
for i, a in enumerate(radv):
if len(a) == 0:
continue
if a[0] == 'X':
self.outinc()
self.bflg ^= 1 << i
elif a[0] == 'H':
self.scorerun(a[2:])
self.bflg ^= 1 << i
elif i != int(a[0]) - 1:
advflg |= 1 << i
for i in reversed([*self._bitindexes(advflg)]):
self.bflg = (self.bflg ^ 1 << i) | (1 << int(radv[i][0]) - 1)
if len(badv) > 0:
if badv[0] == 'X':
self.outinc()
elif badv[0] == 'H':
self.scorerun(badv[2:])
else:
self.bflg |= 1 << int(badv[0]) - 1
self._cycle_lineup()
def scorerun(self, *args):
self.score[self.t] += 1
def _scorerun(self):
"""Shortcut to the basic functionality of scorerun"""
self.score[self.t] += 1
def _check_rbi_(self, rbi, norbi):
if rbi == 1 and norbi == 1:
raise BBSimLogicError("Both rbi and norbi flags present")
if rbi == 1: return True
if norbi == 1: return False
return (self.IS_RBI[self.ecode] and "GDP" not in self.emod)
def outinc(self):
self.o += 1
#------------------------------- [inning]<END> -------------------------------#
def _cycle_inning(self):
while self.bflg > 0:
self.lob[self.t] += (self.bflg & 1)
self.bflg = self.bflg >> 1
self.o = 0
self.i += 1
self.t ^= 1
def _cycle_lineup(self):
"""Cycles to the next batter"""
if self.bootflg[self.t]:
self.bootflg[self.t] <<= 1
else:
self.abinx[self.t] = (self.abinx[self.t] + 1) % 9
#------------------------------- [play] -------------------------------#
def _play(self, l, ctx=None):
""" Takes additional Safety Inputs """
self.eid += 1
self.ecode = int(l[self.EVENT['code']])
self.emod = l[self.EVENT['mod']].split('/')
if self.bootflg[self.t] & 2: self._bootcycle()
if ctx != None:
self._verify(l, ctx)
self._event(l)
self.ecode = None
self.emod = None
#------------------------------- [event] -------------------------------#
def _event(self, l):
""" Idea is this should be executable without ctx safety net """
self._advance(l[self.EVENT['badv']], l[self.EVENT['radv']])
if self.o == 3:
self._cycle_inning()
#------------------------------- [verify] -------------------------------#
def _verify(self, l, ctx):
""" Uses safety line inputs to ensure the simulation is not corrupted """
if self.eid != int(l[0]):
raise BBSimVerifyError(
f'event number discrepency with input line {l[0]}')
if self.gameid != ctx[0][:15] or self.eid != int(ctx[0][-3:]):
raise BBSimVerifyError(
f'gameid/eid discrepency with context line {ctx[0]}')
i, t = (int(ctx[x]) for x in [self.CTX['i'], self.CTX['t']])
if self.inning != i or self.t != t:
raise BBSimVerifyError(
f'Inning Discrepency [{self.inning},{self.t}][{i},{t}]')
a, h = (int(x) for x in ctx[self.CTX['score']])
if self.score[0] != a or self.score[1] != h:
raise BBSimVerifyError(
f'Score Discrepency [{self.score[0]},{self.score[1]}|{a},{h}]')
if self._lpos_ != int(ctx[self.CTX['lpos']]):
raise BBSimVerifyError(
f"lpos sim[{self._lpos_}] evt[{ctx[self.CTX['lpos']]}]")
fpos = int(ctx[self.CTX['fpos']])
if fpos != 0 and fpos != 11 and self._bpid_fpos_ != fpos - 1:
raise BBSimVerifyError(
f"fpos sim[{self._bpid_fpos_}] evt[{fpos-1}]")
#------------------------------- [str] -------------------------------#
@property
def _str_ctx_(self):
return '(%i|%i|%i)' % (self.inning, self.t, self.o)
def work(i):
state_ids = support_state_ids[slice (i, i + chunksize)]
new_rewards = copy.copy(rewards)
values = np.zeros(n_states)
for j, state_id in enumerate(state_ids):
s = support_states[state_id] # state id in states
# vi agent
mdp = vi.valueIterAgent(n_actions, n_states,
roadmap, None, states,
gamma=gamma, T=T)
mdp.set_goal(s)
if support_feature_ids is None:
if new_rewards[s] >= 0.:
new_rewards[s] = 1.
else:
# find all states that gives f_g in states
f_id = support_feature_ids[state_id]
goal_state_ids = support_feature_state_dict[f_id]
if computed_f_id[f_id]:
continue
else:
computed_f_id[f_id] = True
new_rewards[goal_state_ids] = 1.
mdp.set_rewards(new_rewards)
# Store q_mat and validity mat per state
if cstr_fn is not None:
for k in range(len(cstr_fn)):
# check if the goal is isolated
if np.sum(cstr_T[k][goal_state_ids])>0.:
values, param_dict = mdp.solve_mdp(error, init_values=values,
T=cstr_T[k], max_cnt=max_cnt,
goal=s,
return_params=True)
support_q_mat[state_id][k] = param_dict['q']
support_validity[state_id][k] = cstr_fn[k](s)
support_values[state_id][k] = values
if add_no_cstr:
values, param_dict = mdp.solve_mdp(error, init_values=values,
T=T, max_cnt=max_cnt,
## goal=s,
return_params=True)
support_q_mat[state_id][-1] = param_dict['q']
support_validity[state_id][-1] = True
support_values[state_id][-1] = values
else:
values, param_dict = mdp.solve_mdp(error, init_values=values,
max_cnt=max_cnt,
return_params=True)
support_q_mat[state_id] = param_dict['q']
support_values[state_id] = values
# find all states that gives f_g in states
for gs in goal_state_ids:
k = support_states.index(gs)
if k!=state_id:
support_q_mat[k] = support_q_mat[state_id]
# reset
## new_rewards = copy.copy(rewards)
if support_feature_ids is None:
new_rewards[s] = 0.
else:
new_rewards[goal_state_ids] = 0.
phase_zero_moments = np.diff(
(np.angle(x) >= 0 - bias / 360 * 2 * np.pi).astype(int)) > 0
phase_bias = np.angle(y)[1:][phase_zero_moments].mean() / 2 / np.pi * 360
phase_disp = (((np.angle(y)[1:][phase_zero_moments] / 2 / np.pi * 360 -
phase_bias)**2).mean())**0.5
return corr, phase_bias, phase_disp
# load eeg
eeg_df = pd.read_pickle('data/train_test_data.pkl')
# utils
methods = ['cfir', 'rlscfir', 'wcfir', 'rect', 'ffiltar']
colors = ['#0099d8', '#84BCDA', '#FE4A49', '#A2A79E', '#444444']
metrics = ['corr', 'phase_bias', 'phase_abs_bias', 'phase_disp']
slices = dict([('train', slice(None, N_SAMPLES_TRAIN)),
('test', slice(N_SAMPLES_TRAIN,
N_SAMPLES_TRAIN + N_SAMPLES_TEST))])
columns = [
'method', 'dataset', 'snr', 'delay', 'metric', 'train', 'test', 'params'
]
# best stats df
stats_df = pd.DataFrame(columns=columns, dtype=float)
# iterate over methods
for j_method, method_name in enumerate(methods):
# load grid results and params
res = np.load('results/{}.npy'.format(method_name))
kwargs_df = pd.read_csv('results/{}_kwargs.csv'.format(method_name))
def __setitem__(self, indx, val):
self.indx = indx
if indx == slice(0, 4):
raise ValueError
def _border_is_0(field):
borders = [(0, slice(None)), (-1, slice(None)),
(slice(None), 0), (slice(None), -1)]
return all(np.allclose(field[a, b], 0) for a, b in borders)
def _set_view_slice(self):
"""Set the view given the indices to slice with."""
self._new_empty_slice()
not_disp = self._dims_not_displayed
# Check if requested slice outside of data range
indices = np.array(self._slice_indices)
extent = self._extent_data
if np.any(
np.less(
[indices[ax] for ax in not_disp],
[extent[0, ax] for ax in not_disp],
)) or np.any(
np.greater(
[indices[ax] for ax in not_disp],
[extent[1, ax] for ax in not_disp],
)):
return
self._empty = False
if self.multiscale:
# If 3d redering just show lowest level of multiscale
if self._ndisplay == 3:
self.data_level = len(self.data) - 1
# Slice currently viewed level
level = self.data_level
indices = np.array(self._slice_indices)
downsampled_indices = (indices[not_disp] /
self.downsample_factors[level, not_disp])
downsampled_indices = np.round(
downsampled_indices.astype(float)).astype(int)
downsampled_indices = np.clip(
downsampled_indices, 0, self.level_shapes[level, not_disp] - 1)
indices[not_disp] = downsampled_indices
scale = np.ones(self.ndim)
for d in self._dims_displayed:
scale[d] = self.downsample_factors[self.data_level][d]
self._transforms['tile2data'].scale = scale
if self._ndisplay == 2:
for d in self._dims_displayed:
indices[d] = slice(
self.corner_pixels[0, d],
self.corner_pixels[1, d] + 1,
1,
)
self._transforms['tile2data'].translate = (
self.corner_pixels[0] *
self._transforms['tile2data'].scale)
image = self.data[level][tuple(indices)]
image_indices = indices
# Slice thumbnail
indices = np.array(self._slice_indices)
downsampled_indices = (
indices[not_disp] /
self.downsample_factors[self._thumbnail_level, not_disp])
downsampled_indices = np.round(
downsampled_indices.astype(float)).astype(int)
downsampled_indices = np.clip(
downsampled_indices,
0,
self.level_shapes[self._thumbnail_level, not_disp] - 1,
)
indices[not_disp] = downsampled_indices
thumbnail_source = self.data[self._thumbnail_level][tuple(indices)]
else:
self._transforms['tile2data'].scale = np.ones(self.ndim)
image_indices = self._slice_indices
image = self.data[image_indices]
# For single-scale we don't request a separate thumbnail_source
# from the ChunkLoader because in ImageSlice.chunk_loaded we
# call request.thumbnail_source() and it knows to just use the
# image itself is there is no explicit thumbnail_source.
thumbnail_source = None
# Load our images, might be sync or async.
data = SliceDataClass(self, image_indices, image, thumbnail_source)
self._load_slice(data)
def _get_multi_index(self, arr, indices):
"""Mimic multi dimensional indexing.
Parameters
----------
arr : ndarray
Array to be indexed.
indices : tuple of index objects
Returns
-------
out : ndarray
An array equivalent to the indexing operation (but always a copy).
`arr[indices]` should be identical.
no_copy : bool
Whether the indexing operation requires a copy. If this is `True`,
`np.may_share_memory(arr, arr[indices])` should be `True` (with
some exceptions for scalars and possibly 0-d arrays).
Notes
-----
While the function may mostly match the errors of normal indexing this
is generally not the case.
"""
in_indices = list(indices)
indices = []
# if False, this is a fancy or boolean index
no_copy = True
# number of fancy/scalar indexes that are not consecutive
num_fancy = 0
# number of dimensions indexed by a "fancy" index
fancy_dim = 0
# NOTE: This is a funny twist (and probably OK to change).
# The boolean array has illegal indexes, but this is
# allowed if the broadcast fancy-indices are 0-sized.
# This variable is to catch that case.
error_unless_broadcast_to_empty = False
# We need to handle Ellipsis and make arrays from indices, also
# check if this is fancy indexing (set no_copy).
ndim = 0
ellipsis_pos = None # define here mostly to replace all but first.
for i, indx in enumerate(in_indices):
if indx is None:
continue
if isinstance(indx, np.ndarray) and indx.dtype == bool:
no_copy = False
if indx.ndim == 0:
raise IndexError
# boolean indices can have higher dimensions
ndim += indx.ndim
fancy_dim += indx.ndim
continue
if indx is Ellipsis:
if ellipsis_pos is None:
ellipsis_pos = i
continue # do not increment ndim counter
raise IndexError
if isinstance(indx, slice):
ndim += 1
continue
if not isinstance(indx, np.ndarray):
# This could be open for changes in numpy.
# numpy should maybe raise an error if casting to intp
# is not safe. It rejects np.array([1., 2.]) but not
# [1., 2.] as index (same for ie. np.take).
# (Note the importance of empty lists if changing this here)
try:
indx = np.array(indx, dtype=np.intp)
except ValueError:
raise IndexError
in_indices[i] = indx
elif indx.dtype.kind != 'b' and indx.dtype.kind != 'i':
raise IndexError('arrays used as indices must be of '
'integer (or boolean) type')
if indx.ndim != 0:
no_copy = False
ndim += 1
fancy_dim += 1
if arr.ndim - ndim < 0:
# we can't take more dimensions then we have, not even for 0-d
# arrays. since a[()] makes sense, but not a[(),]. We will
# raise an error later on, unless a broadcasting error occurs
# first.
raise IndexError
if ndim == 0 and None not in in_indices:
# Well we have no indexes or one Ellipsis. This is legal.
return arr.copy(), no_copy
if ellipsis_pos is not None:
in_indices[ellipsis_pos:ellipsis_pos + 1] = ([slice(None, None)] *
(arr.ndim - ndim))
for ax, indx in enumerate(in_indices):
if isinstance(indx, slice):
# convert to an index array
indx = np.arange(*indx.indices(arr.shape[ax]))
indices.append(['s', indx])
continue
elif indx is None:
# this is like taking a slice with one element from a new axis:
indices.append(['n', np.array([0], dtype=np.intp)])
arr = arr.reshape((arr.shape[:ax] + (1, ) + arr.shape[ax:]))
continue
if isinstance(indx, np.ndarray) and indx.dtype == bool:
if indx.shape != arr.shape[ax:ax + indx.ndim]:
raise IndexError
try:
flat_indx = np.ravel_multi_index(np.nonzero(indx),
arr.shape[ax:ax +
indx.ndim],
mode='raise')
except Exception:
error_unless_broadcast_to_empty = True
# fill with 0s instead, and raise error later
flat_indx = np.array([0] * indx.sum(), dtype=np.intp)
# concatenate axis into a single one:
if indx.ndim != 0:
arr = arr.reshape(
(arr.shape[:ax] +
(np.prod(arr.shape[ax:ax + indx.ndim]), ) +
arr.shape[ax + indx.ndim:]))
indx = flat_indx
else:
# This could be changed, a 0-d boolean index can
# make sense (even outside the 0-d indexed array case)
# Note that originally this is could be interpreted as
# integer in the full integer special case.
raise IndexError
else:
# If the index is a singleton, the bounds check is done
# before the broadcasting. This used to be different in <1.9
if indx.ndim == 0:
if indx >= arr.shape[ax] or indx < -arr.shape[ax]:
raise IndexError
if indx.ndim == 0:
# The index is a scalar. This used to be two fold, but if
# fancy indexing was active, the check was done later,
# possibly after broadcasting it away (1.7. or earlier).
# Now it is always done.
if indx >= arr.shape[ax] or indx < -arr.shape[ax]:
raise IndexError
if (len(indices) > 0 and indices[-1][0] == 'f'
and ax != ellipsis_pos):
# NOTE: There could still have been a 0-sized Ellipsis
# between them. Checked that with ellipsis_pos.
indices[-1].append(indx)
else:
# We have a fancy index that is not after an existing one.
# NOTE: A 0-d array triggers this as well, while one may
# expect it to not trigger it, since a scalar would not be
# considered fancy indexing.
num_fancy += 1
indices.append(['f', indx])
if num_fancy > 1 and not no_copy:
# We have to flush the fancy indexes left
new_indices = indices[:]
axes = list(range(arr.ndim))
fancy_axes = []
new_indices.insert(0, ['f'])
ni = 0
ai = 0
for indx in indices:
ni += 1
if indx[0] == 'f':
new_indices[0].extend(indx[1:])
del new_indices[ni]
ni -= 1
for ax in range(ai, ai + len(indx[1:])):
fancy_axes.append(ax)
axes.remove(ax)
ai += len(indx) - 1 # axis we are at
indices = new_indices
# and now we need to transpose arr:
arr = arr.transpose(*(fancy_axes + axes))
# We only have one 'f' index now and arr is transposed accordingly.
# Now handle newaxis by reshaping...
ax = 0
for indx in indices:
if indx[0] == 'f':
if len(indx) == 1:
continue
# First of all, reshape arr to combine fancy axes into one:
orig_shape = arr.shape
orig_slice = orig_shape[ax:ax + len(indx[1:])]
arr = arr.reshape(
(arr.shape[:ax] + (np.prod(orig_slice).astype(int), ) +
arr.shape[ax + len(indx[1:]):]))
# Check if broadcasting works
res = np.broadcast(*indx[1:])
# unfortunately the indices might be out of bounds. So check
# that first, and use mode='wrap' then. However only if
# there are any indices...
if res.size != 0:
if error_unless_broadcast_to_empty:
raise IndexError
for _indx, _size in zip(indx[1:], orig_slice):
if _indx.size == 0:
continue
if np.any(_indx >= _size) or np.any(_indx < -_size):
raise IndexError
if len(indx[1:]) == len(orig_slice):
if np.product(orig_slice) == 0:
# Work around for a crash or IndexError with 'wrap'
# in some 0-sized cases.
try:
mi = np.ravel_multi_index(indx[1:],
orig_slice,
mode='raise')
except Exception:
# This happens with 0-sized orig_slice (sometimes?)
# here it is a ValueError, but indexing gives a:
raise IndexError('invalid index into 0-sized')
else:
mi = np.ravel_multi_index(indx[1:],
orig_slice,
mode='wrap')
else:
# Maybe never happens...
raise ValueError
arr = arr.take(mi.ravel(), axis=ax)
try:
arr = arr.reshape(
(arr.shape[:ax] + mi.shape + arr.shape[ax + 1:]))
except ValueError:
# too many dimensions, probably
raise IndexError
ax += mi.ndim
continue
# If we are here, we have a 1D array for take:
arr = arr.take(indx[1], axis=ax)
ax += 1
return arr, no_copy
n_times = 1000
x, roi, _ = sim_single_suj_ephy(n_epochs=n_epochs,
n_times=n_times,
modality=modality,
n_roi=n_roi,
random_state=0)
times = np.linspace(-1, 1, n_times)
###############################################################################
# Simulate spatial correlations
# -----------------------------
#
# Bellow, we are simulating some distant correlations by injecting the
# activity of an ROI to another
x[:, [1], slice(100, 400)] += x[:, [0], slice(100, 400)]
x[:, [2], slice(600, 800)] += x[:, [1], slice(600, 800)]
print(f'Corr 1 : {roi[0]}-{roi[1]} between [{times[100]}-{times[400]}]')
print(f'Corr 2 : {roi[2]}-{roi[1]} between [{times[600]}-{times[800]}]')
###############################################################################
# Define sliding windows
# ----------------------
#
# Next, we define, and plot sliding windows in order to compute the DFC on
# consecutive time windows. In this example we used windows of length 100ms
# and 5ms between each consecutive windows
slwin_len = .1 # 100ms window length
slwin_step = .02 # 20ms between consecutive windows
win_sample = define_windows(times, slwin_len=slwin_len,
def __getitem__(self, indx):
self.indx = indx
if indx == slice(0, 7):
raise ValueError
def overlap_slices(large_array_shape,
small_array_shape,
position,
mode='partial'):
"""
Get slices for the overlapping part of a small and a large array.
Given a certain position of the center of the small array, with
respect to the large array, tuples of slices are returned which can be
used to extract, add or subtract the small array at the given
position. This function takes care of the correct behavior at the
boundaries, where the small array is cut of appropriately.
Integer positions are at the pixel centers.
Parameters
----------
large_array_shape : tuple of int or int
The shape of the large array (for 1D arrays, this can be an
`int`).
small_array_shape : int or tuple thereof
The shape of the small array (for 1D arrays, this can be an
`int`). See the ``mode`` keyword for additional details.
position : number or tuple thereof
The position of the small array's center with respect to the
large array. The pixel coordinates should be in the same order
as the array shape. Integer positions are at the pixel centers.
For any axis where ``small_array_shape`` is even, the position
is rounded up, e.g. extracting two elements with a center of
``1`` will define the extracted region as ``[0, 1]``.
mode : {'partial', 'trim', 'strict'}, optional
In ``'partial'`` mode, a partial overlap of the small and the
large array is sufficient. The ``'trim'`` mode is similar to
the ``'partial'`` mode, but ``slices_small`` will be adjusted to
return only the overlapping elements. In the ``'strict'`` mode,
the small array has to be fully contained in the large array,
otherwise an `~astropy.nddata.utils.PartialOverlapError` is
raised. In all modes, non-overlapping arrays will raise a
`~astropy.nddata.utils.NoOverlapError`.
Returns
-------
slices_large : tuple of slice
A tuple of slice objects for each axis of the large array, such
that ``large_array[slices_large]`` extracts the region of the
large array that overlaps with the small array.
slices_small : tuple of slice
A tuple of slice objects for each axis of the small array, such
that ``small_array[slices_small]`` extracts the region that is
inside the large array.
"""
if mode not in ['partial', 'trim', 'strict']:
raise ValueError('Mode can be only "partial", "trim", or "strict".')
if np.isscalar(small_array_shape):
small_array_shape = (small_array_shape, )
if np.isscalar(large_array_shape):
large_array_shape = (large_array_shape, )
if np.isscalar(position):
position = (position, )
if any(~np.isfinite(position)):
raise ValueError('Input position contains invalid values (NaNs or '
'infs).')
if len(small_array_shape) != len(large_array_shape):
raise ValueError('"large_array_shape" and "small_array_shape" must '
'have the same number of dimensions.')
if len(small_array_shape) != len(position):
raise ValueError('"position" must have the same number of dimensions '
'as "small_array_shape".')
# define the min/max pixel indices
indices_min = [
int(np.ceil(pos - (small_shape / 2.)))
for (pos, small_shape) in zip(position, small_array_shape)
]
indices_max = [
int(np.ceil(pos + (small_shape / 2.)))
for (pos, small_shape) in zip(position, small_array_shape)
]
for e_max in indices_max:
if e_max < 0:
raise NoOverlapError('Arrays do not overlap.')
for e_min, large_shape in zip(indices_min, large_array_shape):
if e_min >= large_shape:
raise NoOverlapError('Arrays do not overlap.')
if mode == 'strict':
for e_min in indices_min:
if e_min < 0:
raise PartialOverlapError('Arrays overlap only partially.')
for e_max, large_shape in zip(indices_max, large_array_shape):
if e_max > large_shape:
raise PartialOverlapError('Arrays overlap only partially.')
# Set up slices
slices_large = tuple(
slice(max(0, indices_min), min(large_shape, indices_max))
for (indices_min, indices_max,
large_shape) in zip(indices_min, indices_max, large_array_shape))
if mode == 'trim':
slices_small = tuple(
slice(0, slc.stop - slc.start) for slc in slices_large)
else:
slices_small = tuple(
slice(max(0, -indices_min),
min(large_shape - indices_min, indices_max - indices_min))
for (indices_min, indices_max, large_shape
) in zip(indices_min, indices_max, large_array_shape))
return slices_large, slices_small
ds.N3_flux.attrs['units']=tar_units
ds['N4_flux'].values = convert_mass_to_mol(ds.N4_flux.values,'g / m**2 / year',get_molar_mass('N'),target_units=tar_units)
ds.N4_flux.attrs['units']=tar_units
#Interpolate to domain
nc = netCDF4.Dataset('domain_cfg.nc')
lon = np.mean(nc.variables['nav_lon'][:],axis=0)
lat = np.mean(nc.variables['nav_lat'][:],axis=1)
nc.close()
ds = ds.interp(y=lat,x=lon)
#Calculate trends and extrapolate to 2020
ds = ds.reindex({'time':np.arange(1985,2021)})
Nyears_ext = len(ds.time)
for v in ['N3_flux','N4_flux']:
c,m = trend_coef(ds[v].isel(time=slice(0,Nyears)).values)
for i in range(Nyears,Nyears_ext):
ds[v][i,:,:] = c + m*i
#Add global attributes
ds.attrs['title'] = 'Nitrogen Deposition for SANH domain provided by ISIMP'
ds = ds.assign_coords({'time':pd.date_range(str(ds.time.values[0]),str(ds.time.values[-1]),freq='YS')})
#Save Out
ds.to_netcdf('Ndep.nc')
for year,idx in ds.groupby('time.year').groups.items():
ofile=f'Ndep_y%s.nc' %year
print('Saving '+ofile)
def iterate_batch(data, batch_size, word_alphabet=None, unk_replace=0., shuffle=False):
data, max_char_length = data
bucket_sizes = [len(data[b]) for b in range(len(_buckets))]
total_size = float(sum(bucket_sizes))
bucket_indices = np.arange(len(_buckets))
if shuffle:
np.random.shuffle((bucket_indices))
for bucket_id in bucket_indices:
bucket_size = bucket_sizes[bucket_id]
if bucket_size == 0:
continue
bucket_length = _buckets[bucket_id]
char_length = min(utils.MAX_CHAR_LENGTH, max_char_length[bucket_id] + utils.NUM_CHAR_PAD)
wid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
cid_inputs = np.empty([bucket_size, bucket_length, char_length], dtype=np.int64)
pid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
chid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
nid_inputs = np.empty([bucket_size, bucket_length], dtype=np.int64)
masks = np.zeros([bucket_size, bucket_length], dtype=np.float32)
single = np.zeros([bucket_size, bucket_length], dtype=np.int64)
for i, inst in enumerate(data[bucket_id]):
wids, cid_seqs, pids, chids, nids = inst
inst_size = len(wids)
# word ids
wid_inputs[i, :inst_size] = wids
wid_inputs[i, inst_size:] = PAD_ID_WORD
for c, cids in enumerate(cid_seqs):
cid_inputs[i, c, :len(cids)] = cids
cid_inputs[i, c, len(cids):] = PAD_ID_CHAR
cid_inputs[i, inst_size:, :] = PAD_ID_CHAR
# pos ids
pid_inputs[i, :inst_size] = pids
pid_inputs[i, inst_size:] = PAD_ID_TAG
# chunk ids
chid_inputs[i, :inst_size] = chids
chid_inputs[i, inst_size:] = PAD_ID_CHUNK
# heads
nid_inputs[i, :inst_size] = nids
nid_inputs[i, inst_size:] = PAD_ID_TAG
# masks
masks[i, :inst_size] = 1.0
if unk_replace:
for j, wid in enumerate(wids):
if word_alphabet.is_singleton(wid):
single[i, j] = 1
if unk_replace:
noise = np.random.binomial(1, unk_replace, size=[bucket_size, bucket_length])
wid_inputs = wid_inputs * (1 - noise * single)
indices = None
if shuffle:
indices = np.arange(bucket_size)
np.random.shuffle(indices)
for start_idx in range(0, bucket_size, batch_size):
if shuffle:
excerpt = indices[start_idx:start_idx + batch_size]
else:
excerpt = slice(start_idx, start_idx + batch_size)
yield wid_inputs[excerpt], cid_inputs[excerpt], pid_inputs[excerpt], chid_inputs[excerpt], \
nid_inputs[excerpt], masks[excerpt]
def test_basic_indices_replaces_whole_axis_slices_with_ellipsis(idx):
# If ... is in the slice, it replaces all ,:, entries for this shape.
assert slice(None) not in idx
import random
from destables import *
#slices 'mb' bit pattern each having 'n' elements
slice = lambda mb, n: [
mb[i:i + n] for i in range(0,
n * (len(mb) // n) + 1, n)
][:-1]
#xor two binary strings 's1' and 's2'
xor = lambda s1, s2: "".join([str(int(a) ^ int(b)) for a, b in zip(s1, s2)])
#parses bit string 'mb' as a ascii string
read = lambda mb: "".join([chr(int(s, 2)) for s in slice(mb, 8)])
#decrypts 64 bit pairs of ciphered text by applying keyset in reverse order
decrypt = lambda ct, keySet: encrypt(ct, list(reversed(keySet)))
#left shifts the 'bits' string by 'n' position
lshift = lambda bits, n: bits[n:] + bits[:n]
# permutes 'bits' according to the 'table' list and returns the permuted string
rearrange = lambda bits, table: "".join(bits[t - 1] for t in table)
#Key related functions
def keygen():
# generates a 64 bit random key and returns it in bit and string format
key = "" #key in bits
keyStr = "" #key in str
for i in range(8):
print('chunk started:', chunk_ds.lat[0].data, chunk_ds.lon[0].data,flush=True)
res = nested_groupby_apply(chunk_ds, ['lat', 'lon'], func_data_consistency)
print(
'chunk finished:',
chunk_ds.lat[0].data,
chunk_ds.lon[0].data,
flush=True
)
return res
# +
chunk_dict = {"lat": 10, "lon": 10} # keep the number of chunks limited, otherwise nothing happens
ds_sub = ds.isel(
lat=slice(0, None, 1), # 24
lon=slice(0, None, 1), # 38
).chunk(chunk_dict)
ds_sub
# +
# create fake data with the structure of the final
# xr.map_blocks_output
fake_data = np.zeros((
len(ds_sub.lat),
len(ds_sub.lon)
))
fake_array = xr.DataArray(
def combine_slices(a, b, c, d):
return (slice(min(a.start, c.start), max(a.stop, c.stop)),
slice(min(b.start, d.start), max(b.stop, d.stop)))
def slice_(x, i, n):
s = x[..., slice(i, i + n)]
return s if n > 1 else tt.addbroadcast(s, -1)
MO *= spin_mask
MO = MO - MO.swapaxes(1, 3)
MO = MO.swapaxes(1, 2)
print('..finished transformation in %.3f seconds.\n' % (time.time() - t))
# Update nocc and nvirt
nocc = ndocc * 2
nvirt = MO.shape[0] - nocc
# Build epsilon tensor
eps = np.repeat(eps, 2)
eocc = eps[:nocc]
evirt = eps[nocc:]
# Create occupied and virtual slices
o = slice(0, nocc)
v = slice(nocc, MO.shape[0])
if num_orbs > nocc:
num_orbs = nocc
ep2_arr = []
for orbital in range(nocc - num_orbs * 2, nocc, 2):
E = eps[orbital]
ep2_conv = False
for ep_iter in range(50):
Eold = E
# Build energy denominators
epsilon1 = 1 / (E + eocc.reshape(-1, 1, 1) - evirt.reshape(-1, 1) -
#To convert the DataFrame to any other convenient representation
#df = dataset.to_dataframe()
#Plotting OAKDALE with a latitude of -34.3075 and a longtitude of 150.58
spei = dataset.spei
spei_site = spei.sel(lat=-34.05305556, lon=150.49722222, method='nearest')
#Start and end day of interested period, which is the same as the anomoly period
start_date = '1994-01'
end_date = '2016-01'
spei_site = spei_site.sel(time=slice(start_date, end_date))
spei_site.plot()
#Read the file
df = read_csv('/home/emma/Downloads/tmp_table_PM10_011996-012016.csv')
df['Date'] = pd.to_datetime(df['Date'])
df['mon'] = df['Date'].dt.month
df = df.set_index('Date')
for i in range(1, 13):
rolling = df[['OAKDALE']][df.mon == i].rolling(window=7, center=True)
rolling_mean = rolling.mean()
def save(
self,
verbose=False,
fid_range=False,
step=False,
progress=False,
silent=False,
stream=sys.stdout,
strict=False,
):
"""
Save the contents from the OGR DataSource Layer into the database
according to the mapping dictionary given at initialization.
Keyword Parameters:
verbose:
If set, information will be printed subsequent to each model save
executed on the database.
fid_range:
May be set with a slice or tuple of (begin, end) feature ID's to map
from the data source. In other words, this keyword enables the user
to selectively import a subset range of features in the geographic
data source.
step:
If set with an integer, transactions will occur at every step
interval. For example, if step=1000, a commit would occur after
the 1,000th feature, the 2,000th feature etc.
progress:
When this keyword is set, status information will be printed giving
the number of features processed and successfully saved. By default,
progress information will pe printed every 1000 features processed,
however, this default may be overridden by setting this keyword with an
integer for the desired interval.
stream:
Status information will be written to this file handle. Defaults to
using `sys.stdout`, but any object with a `write` method is supported.
silent:
By default, non-fatal error notifications are printed to stdout, but
this keyword may be set to disable these notifications.
strict:
Execution of the model mapping will cease upon the first error
encountered. The default behavior is to attempt to continue.
"""
# Getting the default Feature ID range.
default_range = self.check_fid_range(fid_range)
# Setting the progress interval, if requested.
if progress:
if progress is True or not isinstance(progress, int):
progress_interval = 1000
else:
progress_interval = progress
def _save(feat_range=default_range, num_feat=0, num_saved=0):
if feat_range:
layer_iter = self.layer[feat_range]
else:
layer_iter = self.layer
for feat in layer_iter:
num_feat += 1
# Getting the keyword arguments
try:
kwargs = self.feature_kwargs(feat)
except LayerMapError as msg:
# Something borked the validation
if strict:
raise
elif not silent:
stream.write("Ignoring Feature ID %s because: %s\n" %
(feat.fid, msg))
else:
# Constructing the model using the keyword args
is_update = False
if self.unique:
# If we want unique models on a particular field, handle the
# geometry appropriately.
try:
# Getting the keyword arguments and retrieving
# the unique model.
u_kwargs = self.unique_kwargs(kwargs)
m = self.model.objects.using(
self.using).get(**u_kwargs)
is_update = True
# Getting the geometry (in OGR form), creating
# one from the kwargs WKT, adding in additional
# geometries, and update the attribute with the
# just-updated geometry WKT.
geom_value = getattr(m, self.geom_field)
if geom_value is None:
geom = OGRGeometry(kwargs[self.geom_field])
else:
geom = geom_value.ogr
new = OGRGeometry(kwargs[self.geom_field])
for g in new:
geom.add(g)
setattr(m, self.geom_field, geom.wkt)
except ObjectDoesNotExist:
# No unique model exists yet, create.
m = self.model(**kwargs)
else:
m = self.model(**kwargs)
try:
# Attempting to save.
m.save(using=self.using)
num_saved += 1
if verbose:
stream.write(
"%s: %s\n" %
("Updated" if is_update else "Saved", m))
except Exception as msg:
if strict:
# Bailing out if the `strict` keyword is set.
if not silent:
stream.write(
"Failed to save the feature (id: %s) into the "
"model with the keyword arguments:\n" %
feat.fid)
stream.write("%s\n" % kwargs)
raise
elif not silent:
stream.write(
"Failed to save %s:\n %s\nContinuing\n" %
(kwargs, msg))
# Printing progress information, if requested.
if progress and num_feat % progress_interval == 0:
stream.write("Processed %d features, saved %d ...\n" %
(num_feat, num_saved))
# Only used for status output purposes -- incremental saving uses the
# values returned here.
return num_saved, num_feat
if self.transaction_decorator is not None:
_save = self.transaction_decorator(_save)
nfeat = self.layer.num_feat
if step and isinstance(step, int) and step < nfeat:
# Incremental saving is requested at the given interval (step)
if default_range:
raise LayerMapError(
"The `step` keyword may not be used in conjunction with the "
"`fid_range` keyword.")
beg, num_feat, num_saved = (0, 0, 0)
indices = range(step, nfeat, step)
n_i = len(indices)
for i, end in enumerate(indices):
# Constructing the slice to use for this step; the last slice is
# special (e.g, [100:] instead of [90:100]).
if i + 1 == n_i:
step_slice = slice(beg, None)
else:
step_slice = slice(beg, end)
try:
num_feat, num_saved = _save(step_slice, num_feat,
num_saved)
beg = end
except Exception: # Deliberately catch everything
stream.write("%s\nFailed to save slice: %s\n" %
("=-" * 20, step_slice))
raise
else:
# Otherwise, just calling the previously defined _save() function.
_save()
def synsq_stft_inv(Tx, fs, opts, Cs=None, freqband=None):
"""Inverse STFT synchrosqueezing transform of `Tx` with associated
frequencies in `fs` and curve bands in time-frequency plane
specified by `Cs` and `freqband`. This implements Eq. 5 of [1].
# Arguments:
Tx: np.ndarray. Synchrosqueeze-transformed `x` (see `synsq_cwt_fwd`).
fs: np.ndarray. Frequencies associated with rows of Tx.
(see `synsq_cwt_fwd`).
opts. dict. Options:
'type': type of wavelet used in `synsq_cwt_fwd` (required).
other wavelet options ('mu', 's') should also match those
used in `synsq_cwt_fwd`
'Cs': (optional) curve centerpoints
'freqs': (optional) curve bands
# Returns:
x: components of reconstructed signal, and residual error
Example:
Tx, fs = synsq_cwt_fwd(t, x, 32) # synchrosqueezing
Txf = synsq_filter_pass(Tx, fs, -np.inf, 1) # pass band filter
xf = synsq_cwt_inv(Txf, fs) # filtered signal reconstruction
"""
Cs = Cs or np.ones((Tx.shape[1], 1))
freqband = freqband or Tx.shape[0]
windowfunc = wfiltfn(opts['type'], opts, derivative=False)
inf_lim = 1000 # quadpy can't handle np.inf limits
C = quadgk(lambda x: windowfunc(x)**2, -inf_lim, inf_lim)
if opts['type'] == 'bump':
C *= 0.8675
# Invert Tx around curve masks in the time-frequency plane to recover
# individual components; last one is the remaining signal
# Integration over all frequencies recovers original signal
# Factor of 2 is because real parts contain half the energy
x = np.zeros((Cs.shape[0], Cs.shape[1] + 1))
TxRemainder = Tx
for n in range(Cs.shape[1]):
TxMask = np.zeros(Tx.shape)
UpperCs = min(max(Cs[:, n] + freqband[:, n], 1), len(fs))
LowerCs = min(max(Cs[:, n] - freqband[:, n], 1), len(fs))
# Cs==0 corresponds to no curve at that time, so this removes
# such points from the inversion
# NOTE: transposed + flattened to match MATLAB's 'linear indices'
UpperCs[np.where(Cs[:, n].T.flatten() < 1)] = 1
LowerCs[np.where(Cs[:, n].T.flatten() < 1)] = 2
for m in range(Tx.shape[1]):
idxs = slice(LowerCs[m] - 1, UpperCs[m])
TxMask[idxs, m] = Tx[idxs, m]
TxRemainder[idxs, m] = 0
x[:, n] = 1 / (PI * C) * np.sum(np.real(TxMask), axis=0).T
x[:, n + 1] = 1 / (PI * C) * np.sum(np.real(TxRemainder), axis=0).T
x = x.T
return x
def __call__(self, loc, score, anchor, img_size, scale=1.):
"""input should be ndarray
Propose RoIs.
Inputs :obj:`loc, score, anchor` refer to the same anchor when indexed
by the same index.
On notations, :math:`R` is the total number of anchors. This is equal
to product of the height and the width of an image and the number of
anchor bases per pixel.
Type of the output is same as the inputs.
Args:
loc (array): Predicted offsets and scaling to anchors.
Its shape is :math:`(R, 4)`.
score (array): Predicted foreground probability for anchors.
Its shape is :math:`(R,)`.
anchor (array): Coordinates of anchors. Its shape is
:math:`(R, 4)`.
img_size (tuple of ints): A tuple :obj:`height, width`,
which contains image size after scaling.
scale (float): The scaling factor used to scale an image after
reading it from a file.
Returns:
array:
An array of coordinates of proposal boxes.
Its shape is :math:`(S, 4)`. :math:`S` is less than
:obj:`self.n_test_post_nms` in test time and less than
:obj:`self.n_train_post_nms` in train time. :math:`S` depends on
the size of the predicted bounding boxes and the number of
bounding boxes discarded by NMS.
"""
# NOTE: when test, remember
# faster_rcnn.eval()
# to set self.traing = False
if self.parent_model.training:
n_pre_nms = self.n_train_pre_nms
n_post_nms = self.n_train_post_nms
else:
n_pre_nms = self.n_test_pre_nms
n_post_nms = self.n_test_post_nms
# Convert anchors into proposal via bbox transformations.
# roi = loc2bbox(anchor, loc)
roi = loc2bbox(anchor, loc)
# Clip predicted boxes to image.
roi[:, slice(0, 4, 2)] = np.clip(roi[:, slice(0, 4, 2)], 0,
img_size[0])
roi[:, slice(1, 4, 2)] = np.clip(roi[:, slice(1, 4, 2)], 0,
img_size[1])
# Remove predicted boxes with either height or width < threshold.
min_size = self.min_size * scale
hs = roi[:, 2] - roi[:, 0]
ws = roi[:, 3] - roi[:, 1]
keep = np.where((hs >= min_size) & (ws >= min_size))[0]
roi = roi[keep, :]
score = score[keep]
# Sort all (proposal, score) pairs by score from highest to lowest.
# Take top pre_nms_topN (e.g. 6000).
order = score.ravel().argsort()[::-1]
if n_pre_nms > 0:
order = order[:n_pre_nms]
roi = roi[order, :]
# Apply nms (e.g. threshold = 0.7).
# Take after_nms_topN (e.g. 300).
# unNOTE: somthing is wrong here!
# TODO: remove cuda.to_gpu
keep = non_maximum_suppression(cp.ascontiguousarray(cp.asarray(roi)),
thresh=self.nms_thresh)
if n_post_nms > 0:
keep = keep[:n_post_nms]
roi = roi[keep]
return roi
(-1 - curr_enum_dim))
expected = probs_x[x_prev.unsqueeze(-1),
x_curr.unsqueeze(-1),
torch.arange(hidden_dim)]
actual = Vindex(probs_x)[x_prev, x_curr, :]
assert_equal(actual, expected)
@pytest.mark.parametrize(
"args,expected",
[
(0, 0),
(1, 1),
(None, None),
(slice(1, 2, 3), slice(1, 2, 3)),
(Ellipsis, Ellipsis),
(
(0, 1, None, slice(1, 2, 3), Ellipsis),
(0, 1, None, slice(1, 2, 3), Ellipsis),
),
(
((0, 1), (None, slice(1, 2, 3)), Ellipsis),
(0, 1, None, slice(1, 2, 3), Ellipsis),
),
((Ellipsis, None), (Ellipsis, None)),
((Ellipsis, (Ellipsis, None)), (Ellipsis, None)),
((Ellipsis, (Ellipsis, None, None)), (Ellipsis, None, None)),
],
)
def test_index(args, expected):
def readBlockWithMargin(ds, xoff, yoff, xsize, ysize, datatype, margin=0, nullValList=None,
layerselection=None):
"""
A 'drop-in' look-alike for the ReadAsArray function in GDAL,
but with the option of specifying a margin width, such that
the block actually read and returned will be larger by that many pixels.
The returned array will ALWAYS contain these extra rows/cols, and
if they do not exist in the file (e.g. because the margin would push off
the edge of the file) then they will be filled with the given nullVal.
Otherwise they will be read from the file along with the rest of the block.
Variables within this function which have _margin as suffix are intended to
designate variables which include the margin, as opposed to those without.
This routine will cope with any specified region, even if it is entirely outside
the given raster. The returned block would, in that case, be filled
entirely with the null value.
"""
if layerselection is None:
layerselection = [i+1 for i in range(ds.RasterCount)]
nLayers = len(layerselection)
# Create the final array, with margin, but filled with the null value.
xSize_margin = xsize + 2 * margin
ySize_margin = ysize + 2 * margin
outBlockShape = (nLayers, ySize_margin, xSize_margin)
# Create the empty output array, filled with the appropriate null value.
block_margin = numpy.zeros(outBlockShape, dtype=datatype)
if nullValList is not None and len(nullValList) > 0:
# We really need something as a fill value, so if any of the
# null values in the list is None, then replace it with 0.
fillValList = [nullVal for nullVal in nullValList]
for i in range(len(fillValList)):
if fillValList[i] is None:
fillValList[i] = 0
# Now use the appropriate null value for each layer as the
# initial value in the output array for the block.
if len(outBlockShape) == 2:
block_margin.fill(fillValList[0])
else:
for i in range(nLayers):
block_margin[i].fill(fillValList[layerselection[i]-1])
# for (i, fillVal) in enumerate(fillValList):
# block_margin[i].fill(fillVal)
# Calculate the bounds of the block which we will actually read from the file,
# based on what we have been asked for, what margin size, and how close we
# are to the edge of the file.
# The bounds of the whole image in the file
imgLeftBound = 0
imgTopBound = 0
imgRightBound = ds.RasterXSize
imgBottomBound = ds.RasterYSize
# The region we will, in principle, read from the file. Note that xSize_margin
# and ySize_margin are already calculated above
xoff_margin = xoff - margin
yoff_margin = yoff - margin
# Restrict this to what is available in the file
xoff_margin_file = max(xoff_margin, imgLeftBound)
xoff_margin_file = min(xoff_margin_file, imgRightBound)
xright_margin_file = xoff_margin + xSize_margin
xright_margin_file = min(xright_margin_file, imgRightBound)
xSize_margin_file = xright_margin_file - xoff_margin_file
yoff_margin_file = max(yoff_margin, imgTopBound)
yoff_margin_file = min(yoff_margin_file, imgBottomBound)
ySize_margin_file = min(ySize_margin, imgBottomBound - yoff_margin_file)
ybottom_margin_file = yoff_margin + ySize_margin
ybottom_margin_file = min(ybottom_margin_file, imgBottomBound)
ySize_margin_file = ybottom_margin_file - yoff_margin_file
# How many pixels on each edge of the block we end up NOT reading from
# the file, and thus have to leave as null in the array
notRead_left = xoff_margin_file - xoff_margin
notRead_right = xSize_margin - (notRead_left + xSize_margin_file)
notRead_top = yoff_margin_file - yoff_margin
notRead_bottom = ySize_margin - (notRead_top + ySize_margin_file)
# The upper bounds on the slices specified to receive the data
slice_right = xSize_margin - notRead_right
slice_bottom = ySize_margin - notRead_bottom
if xSize_margin_file > 0 and ySize_margin_file > 0:
# Now read in the part of the array which we can actually read from the file.
# Read each layer separately, to honour the layerselection
# The part of the final array we are filling
imageSlice = (slice(notRead_top, slice_bottom), slice(notRead_left, slice_right))
for i in range(nLayers):
band = ds.GetRasterBand(layerselection[i])
block_margin[i][imageSlice] = band.ReadAsArray(xoff_margin_file, yoff_margin_file,
xSize_margin_file, ySize_margin_file)
return block_margin
def main(url_list, sDir, deployment_num, start_time, end_time, preferred_only,
n_std, inpercentile, zcell_size, zdbar):
rd_list = []
for uu in url_list:
elements = uu.split('/')[-2].split('-')
rd = '-'.join((elements[1], elements[2], elements[3], elements[4]))
if rd not in rd_list and 'ENG' not in rd and 'ADCP' not in rd:
rd_list.append(rd)
for r in rd_list:
print('\n{}'.format(r))
datasets = []
for u in url_list:
splitter = u.split('/')[-2].split('-')
rd_check = '-'.join(
(splitter[1], splitter[2], splitter[3], splitter[4]))
if rd_check == r:
udatasets = cf.get_nc_urls([u])
datasets.append(udatasets)
datasets = list(itertools.chain(*datasets))
fdatasets = []
if preferred_only == 'yes':
# get the preferred stream information
ps_df, n_streams = cf.get_preferred_stream_info(r)
for index, row in ps_df.iterrows():
for ii in range(n_streams):
try:
rms = '-'.join((r, row[ii]))
except TypeError:
continue
for dd in datasets:
spl = dd.split('/')[-2].split('-')
catalog_rms = '-'.join(
(spl[1], spl[2], spl[3], spl[4], spl[5], spl[6]))
fdeploy = dd.split('/')[-1].split('_')[0]
if rms == catalog_rms and fdeploy == row['deployment']:
fdatasets.append(dd)
else:
fdatasets = datasets
main_sensor = r.split('-')[-1]
fdatasets_sel = cf.filter_collocated_instruments(
main_sensor, fdatasets)
for fd in fdatasets_sel:
part_d = fd.split('/')[-1]
print('\n{}'.format(part_d))
ds = xr.open_dataset(fd, mask_and_scale=False)
ds = ds.swap_dims({'obs': 'time'})
fname, subsite, refdes, method, stream, deployment = cf.nc_attributes(
fd)
array = subsite[0:2]
sci_vars = cf.return_science_vars(stream)
# if 'CE05MOAS' in r or 'CP05MOAS' in r: # for coastal gliders, get m_water_depth for bathymetry
# eng = '-'.join((r.split('-')[0], r.split('-')[1], '00-ENG000000', method, 'glider_eng'))
# eng_url = [s for s in url_list if eng in s]
# if len(eng_url) == 1:
# eng_datasets = cf.get_nc_urls(eng_url)
# # filter out collocated datasets
# eng_dataset = [j for j in eng_datasets if (eng in j.split('/')[-1] and deployment in j.split('/')[-1])]
# if len(eng_dataset) > 0:
# ds_eng = xr.open_dataset(eng_dataset[0], mask_and_scale=False)
# t_eng = ds_eng['time'].values
# m_water_depth = ds_eng['m_water_depth'].values
#
# # m_altitude = glider height above seafloor
# # m_depth = glider depth in the water column
# # m_altitude = ds_eng['m_altitude'].values
# # m_depth = ds_eng['m_depth'].values
# # calc_water_depth = m_altitude + m_depth
#
# # m_altimeter_status = 0 means a good reading (not nan or -1)
# try:
# eng_ind = ds_eng['m_altimeter_status'].values == 0
# except KeyError:
# eng_ind = (~np.isnan(m_water_depth)) & (m_water_depth >= 0)
#
# m_water_depth = m_water_depth[eng_ind]
# t_eng = t_eng[eng_ind]
#
# # get rid of any remaining nans or fill values
# eng_ind2 = (~np.isnan(m_water_depth)) & (m_water_depth >= 0)
# m_water_depth = m_water_depth[eng_ind2]
# t_eng = t_eng[eng_ind2]
# else:
# print('No engineering file for deployment {}'.format(deployment))
# m_water_depth = None
# t_eng = None
# else:
# m_water_depth = None
# t_eng = None
# else:
# m_water_depth = None
# t_eng = None
if deployment_num is not None:
if int(int(deployment[-4:])) is not deployment_num:
print(type(int(deployment[-4:])), type(deployment_num))
continue
if start_time is not None and end_time is not None:
ds = ds.sel(time=slice(start_time, end_time))
if len(ds['time'].values) == 0:
print(
'No data to plot for specified time range: ({} to {})'.
format(start_time, end_time))
continue
stime = start_time.strftime('%Y-%m-%d')
etime = end_time.strftime('%Y-%m-%d')
ext = stime + 'to' + etime # .join((ds0_method, ds1_method
save_dir_profile = os.path.join(sDir, array, subsite, refdes,
'profile_plots', deployment,
ext)
save_dir_xsection = os.path.join(sDir, array, subsite, refdes,
'xsection_plots', deployment,
ext)
save_dir_4d = os.path.join(sDir, array, subsite, refdes,
'xsection_plots_4d', deployment,
ext)
else:
save_dir_profile = os.path.join(sDir, array, subsite, refdes,
'profile_plots', deployment)
save_dir_xsection = os.path.join(sDir, array, subsite, refdes,
'xsection_plots', deployment)
save_dir_4d = os.path.join(sDir, array, subsite, refdes,
'xsection_plots_4d', deployment)
time1 = ds['time'].values
try:
ds_lat1 = ds['lat'].values
except KeyError:
ds_lat1 = None
print('No latitude variable in file')
try:
ds_lon1 = ds['lon'].values
except KeyError:
ds_lon1 = None
print('No longitude variable in file')
# get pressure variable
pvarname, y1, y_units, press, y_fillvalue = cf.add_pressure_to_dictionary_of_sci_vars(
ds)
for sv in sci_vars:
print('')
print(sv)
if 'pressure' not in sv:
if sv == 'spkir_abj_cspp_downwelling_vector':
pxso.pf_xs_spkir(ds, sv, time1, y1, ds_lat1, ds_lon1,
zcell_size, inpercentile,
save_dir_profile, save_dir_xsection,
deployment, press, y_units, n_std)
elif 'OPTAA' in r:
if sv not in ['wavelength_a', 'wavelength_c']:
pxso.pf_xs_optaa(ds, sv, time1, y1, ds_lat1,
ds_lon1, zcell_size, inpercentile,
save_dir_profile,
save_dir_xsection, deployment,
press, y_units, n_std)
else:
z1 = ds[sv].values
fv = ds[sv]._FillValue
sv_units = ds[sv].units
# Check if the array is all NaNs
if sum(np.isnan(z1)) == len(z1):
print('Array of all NaNs - skipping plot.')
continue
# Check if the array is all fill values
elif len(z1[z1 != fv]) == 0:
print('Array of all fill values - skipping plot.')
continue
else:
# remove unreasonable pressure data (e.g. for surface piercing profilers)
if zdbar:
po_ind = (0 < y1) & (y1 < zdbar)
tm = time1[po_ind]
y = y1[po_ind]
z = z1[po_ind]
ds_lat = ds_lat1[po_ind]
ds_lon = ds_lon1[po_ind]
else:
tm = time1
y = y1
z = z1
ds_lat = ds_lat1
ds_lon = ds_lon1
# reject erroneous data
dtime, zpressure, ndata, lenfv, lennan, lenev, lengr, global_min, global_max, lat, lon = \
cf.reject_erroneous_data(r, sv, tm, y, z, fv, ds_lat, ds_lon)
# get rid of 0.0 data
# if sv == 'practical_salinity':
# ind = ndata > 34
# elif sv == 'sci_seawater_density':
# ind = ndata > 1026
# elif sv == 'sci_water_cond':
# ind = ndata > 3.1
# else:
# ind = ndata > 0
# if sv == 'sci_flbbcd_chlor_units':
# ind = ndata < 7.5
# elif sv == 'sci_flbbcd_cdom_units':
# ind = ndata < 25
# else:
# ind = ndata > 0.0
if 'CTD' in r:
ind = zpressure > 0.0
else:
ind = ndata > 0.0
lenzero = np.sum(~ind)
dtime = dtime[ind]
zpressure = zpressure[ind]
ndata = ndata[ind]
if ds_lat is not None and ds_lon is not None:
lat = lat[ind]
lon = lon[ind]
else:
lat = None
lon = None
if len(dtime) > 0:
# reject time range from data portal file export
t_portal, z_portal, y_portal, lat_portal, lon_portal = \
cf.reject_timestamps_dataportal(subsite, r, dtime, zpressure, ndata, lat, lon)
print(
'removed {} data points using visual inspection of data'
.format(len(ndata) - len(z_portal)))
# create data groups
columns = ['tsec', 'dbar', str(sv)]
min_r = int(
round(np.nanmin(y_portal) - zcell_size))
max_r = int(
round(np.nanmax(y_portal) + zcell_size))
ranges = list(range(min_r, max_r, zcell_size))
groups, d_groups = gt.group_by_depth_range(
t_portal, y_portal, z_portal, columns,
ranges)
if 'scatter' in sv:
n_std = None # to use percentile
else:
n_std = n_std
# get percentile analysis for printing on the profile plot
y_avg, n_avg, n_min, n_max, n0_std, n1_std, l_arr, time_ex = cf.reject_timestamps_in_groups(
groups, d_groups, n_std, inpercentile)
"""
Plot all data
"""
if len(time1) > 0:
cf.create_dir(save_dir_profile)
cf.create_dir(save_dir_xsection)
sname = '-'.join((r, method, sv))
sfileall = '_'.join(
('all_data', sname,
pd.to_datetime(
time1.min()).strftime('%Y%m%d')))
tm0 = pd.to_datetime(
time1.min()).strftime('%Y-%m-%dT%H:%M:%S')
tm1 = pd.to_datetime(
time1.max()).strftime('%Y-%m-%dT%H:%M:%S')
title = ' '.join(
(deployment, refdes,
method)) + '\n' + tm0 + ' to ' + tm1
if 'SPKIR' in r:
title = title + '\nWavelength = 510 nm'
'''
profile plot
'''
xlabel = sv + " (" + sv_units + ")"
ylabel = press[0] + " (" + y_units[0] + ")"
clabel = 'Time'
fig, ax = pf.plot_profiles(z1,
y1,
time1,
ylabel,
xlabel,
clabel,
stdev=None)
ax.set_title(title, fontsize=9)
fig.tight_layout()
pf.save_fig(save_dir_profile, sfileall)
'''
xsection plot
'''
clabel = sv + " (" + sv_units + ")"
ylabel = press[0] + " (" + y_units[0] + ")"
fig, ax, bar = pf.plot_xsection(
subsite,
time1,
y1,
z1,
clabel,
ylabel,
t_eng=None,
m_water_depth=None,
inpercentile=None,
stdev=None)
if fig:
ax.set_title(title, fontsize=9)
fig.tight_layout()
pf.save_fig(save_dir_xsection, sfileall)
"""
Plot cleaned-up data
"""
if len(dtime) > 0:
sfile = '_'.join(
('rm_erroneous_data', sname,
pd.to_datetime(
t_portal.min()).strftime('%Y%m%d')))
t0 = pd.to_datetime(t_portal.min()).strftime(
'%Y-%m-%dT%H:%M:%S')
t1 = pd.to_datetime(t_portal.max()).strftime(
'%Y-%m-%dT%H:%M:%S')
title = ' '.join(
(deployment, refdes,
method)) + '\n' + t0 + ' to ' + t1
if 'SPKIR' in r:
title = title + '\nWavelength = 510 nm'
'''
profile plot
'''
xlabel = sv + " (" + sv_units + ")"
ylabel = press[0] + " (" + y_units[0] + ")"
clabel = 'Time'
fig, ax = pf.plot_profiles(z_portal,
y_portal,
t_portal,
ylabel,
xlabel,
clabel,
stdev=None)
ax.set_title(title, fontsize=9)
ax.plot(n_avg, y_avg, '-k')
ax.fill_betweenx(y_avg,
n0_std,
n1_std,
color='m',
alpha=0.2)
if inpercentile:
leg_text = (
'removed {} fill values, {} NaNs, {} Extreme Values (1e7), {} Global ranges [{} - {}], '
'{} unreasonable values'.format(
lenfv, lennan, lenev, lengr,
global_min, global_max, lenzero) +
'\nexcluded {} suspect data points when inspected visually'
.format(len(ndata) - len(z_portal)) +
'\n(black) data average in {} dbar segments'
.format(zcell_size) +
'\n(magenta) {} percentile envelope in {} dbar segments'
.format(int(100 - inpercentile * 2),
zcell_size), )
elif n_std:
leg_text = (
'removed {} fill values, {} NaNs, {} Extreme Values (1e7), {} Global ranges [{} - {}], '
'{} unreasonable values'.format(
lenfv, lennan, lenev, lengr,
global_min, global_max, lenzero) +
'\nexcluded {} suspect data points when inspected visually'
.format(len(ndata) - len(z_portal)) +
'\n(black) data average in {} dbar segments'
.format(zcell_size) +
'\n(magenta) +/- {} SD envelope in {} dbar segments'
.format(int(n_std), zcell_size), )
ax.legend(leg_text,
loc='upper center',
bbox_to_anchor=(0.5, -0.17),
fontsize=6)
fig.tight_layout()
pf.save_fig(save_dir_profile, sfile)
'''
xsection plot
'''
clabel = sv + " (" + sv_units + ")"
ylabel = press[0] + " (" + y_units[0] + ")"
# plot non-erroneous data
fig, ax, bar = pf.plot_xsection(
subsite,
t_portal,
y_portal,
z_portal,
clabel,
ylabel,
t_eng=None,
m_water_depth=None,
inpercentile=None,
stdev=None)
ax.set_title(title, fontsize=9)
leg_text = (
'removed {} fill values, {} NaNs, {} Extreme Values (1e7), {} Global ranges [{} - {}], '
'{} unreasonable values'.format(
lenfv, lennan, lenev, lengr,
global_min, global_max, lenzero) +
'\nexcluded {} suspect data points when inspected visually'
.format(len(ndata) - len(z_portal)), )
ax.legend(leg_text,
loc='upper center',
bbox_to_anchor=(0.5, -0.17),
fontsize=6)
fig.tight_layout()
pf.save_fig(save_dir_xsection, sfile)
'''
4D plot for gliders only
'''
if 'MOAS' in r:
if ds_lat is not None and ds_lon is not None:
cf.create_dir(save_dir_4d)
clabel = sv + " (" + sv_units + ")"
zlabel = press[0] + " (" + y_units[
0] + ")"
fig = plt.figure()
ax = fig.add_subplot(111,
projection='3d')
sct = ax.scatter(lon_portal,
lat_portal,
y_portal,
c=z_portal,
s=2)
cbar = plt.colorbar(sct,
label=clabel,
extend='both')
cbar.ax.tick_params(labelsize=8)
ax.invert_zaxis()
ax.view_init(25, 32)
ax.invert_xaxis()
ax.invert_yaxis()
ax.set_zlabel(zlabel, fontsize=9)
ax.set_ylabel('Latitude', fontsize=9)
ax.set_xlabel('Longitude', fontsize=9)
ax.set_title(title, fontsize=9)
pf.save_fig(save_dir_4d, sfile)
