def __init__(self, ax, Y, clusters, color=None, pmax=0.05, ptrend=0.1, xdim='time', title=None): uts_args = _base.find_uts_args(Y, False, color) self._bottom, self._top = _base.find_vlim_args(Y) if title: if '{name}' in title: title = title.format(name=Y.name) ax.set_title(title) _plt_uts(ax, Y, xdim=xdim, **uts_args) if np.any(Y.x < 0) and np.any(Y.x > 0): ax.axhline(0, color='k') # pmap self.cluster_plt = _plt_uts_clusters(ax, clusters, pmax, ptrend, color) # save ax attr self.ax = ax x = Y.get_dim(xdim).x self.xlim = (x[0], x[-1]) ax.set_xlim(*self.xlim) ax.set_ylim(bottom=self._bottom, top=self._top)
def _validate_covars(covars, cvtype, nmix, n_dim): from scipy import linalg if cvtype == 'spherical': if len(covars) != nmix: raise ValueError("'spherical' covars must have length nmix") elif np.any(covars <= 0): raise ValueError("'spherical' covars must be non-negative") elif cvtype == 'tied': if covars.shape != (n_dim, n_dim): raise ValueError("'tied' covars must have shape (n_dim, n_dim)") elif (not np.allclose(covars, covars.T) or np.any(linalg.eigvalsh(covars) <= 0)): raise ValueError("'tied' covars must be symmetric, " "positive-definite") elif cvtype == 'diag': if covars.shape != (nmix, n_dim): raise ValueError("'diag' covars must have shape (nmix, n_dim)") elif np.any(covars <= 0): raise ValueError("'diag' covars must be non-negative") elif cvtype == 'full': if covars.shape != (nmix, n_dim, n_dim): raise ValueError("'full' covars must have shape " "(nmix, n_dim, n_dim)") for n, cv in enumerate(covars): if (not np.allclose(cv, cv.T) or np.any(linalg.eigvalsh(cv) <= 0)): raise ValueError("component %d of 'full' covars must be " "symmetric, positive-definite" % n)
def nanallclose(x, y, rtol=1.0e-5, atol=1.0e-8): """Numpy allclose function which allows NaN Input x, y: Either scalars or numpy arrays Output True or False Returns True if all non-nan elements pass. """ xn = numpy.isnan(x) yn = numpy.isnan(y) if numpy.any(xn != yn): # Presence of NaNs is not the same in x and y return False if numpy.all(xn): # Everything is NaN. # This will also take care of x and y being NaN scalars return True # Filter NaN's out if numpy.any(xn): x = x[-xn] y = y[-yn] # Compare non NaN's and return return numpy.allclose(x, y, rtol=rtol, atol=atol)
def _testUniformSampleMultiDimensional(self): # DISABLED: Please enable this test once b/issues/30149644 is resolved. with self.test_session(): batch_size = 2 a_v = [3.0, 22.0] b_v = [13.0, 35.0] a = constant_op.constant([a_v] * batch_size) b = constant_op.constant([b_v] * batch_size) uniform = uniform_lib.Uniform(low=a, high=b) n_v = 100000 n = constant_op.constant(n_v) samples = uniform.sample(n) self.assertEqual(samples.get_shape(), (n_v, batch_size, 2)) sample_values = self.evaluate(samples) self.assertFalse( np.any(sample_values[:, 0, 0] < a_v[0]) or np.any(sample_values[:, 0, 0] >= b_v[0])) self.assertFalse( np.any(sample_values[:, 0, 1] < a_v[1]) or np.any(sample_values[:, 0, 1] >= b_v[1])) self.assertAllClose( sample_values[:, 0, 0].mean(), (a_v[0] + b_v[0]) / 2, atol=1e-2) self.assertAllClose( sample_values[:, 0, 1].mean(), (a_v[1] + b_v[1]) / 2, atol=1e-2)
def tag_sites(self, scaled_positions, symprec=1e-3): """Returns an integer array of the same length as *scaled_positions*, tagging all equivalent atoms with the same index. Example: >>> from ase.lattice.spacegroup import Spacegroup >>> sg = Spacegroup(225) # fcc >>> sg.tag_sites([[0.0, 0.0, 0.0], ... [0.5, 0.5, 0.0], ... [1.0, 0.0, 0.0], ... [0.5, 0.0, 0.0]]) array([0, 0, 0, 1]) """ scaled = np.array(scaled_positions, ndmin=2) scaled %= 1.0 scaled %= 1.0 tags = -np.ones((len(scaled), ), dtype=int) mask = np.ones((len(scaled), ), dtype=np.bool) rot, trans = self.get_op() i = 0 while mask.any(): pos = scaled[mask][0] sympos = np.dot(rot, pos) + trans # Must be done twice, see the scaled_positions.py test sympos %= 1.0 sympos %= 1.0 m = ~np.all(np.any(np.abs(scaled[np.newaxis,:,:] - sympos[:,np.newaxis,:]) > symprec, axis=2), axis=0) assert not np.any((~mask) & m) tags[m] = i mask &= ~m i += 1 return tags
def parseData(data, userType): if not np.any(data.columns == "userid"): sys.exit("'userid' column not found! Module = 'parseData'") if not np.any(data.columns == "adid"): sys.exit("'adid' column not found! Module = 'parseData'") # Handle datatypes try: data["adid"] = data["adid"].astype(str) print(" 'data' datatypes converted successfully.") except: sys.exit(" Could not convert 'data' datatypes: Exit.") # Filter out rows that contain "" in the userid column nbefore = data.shape[0] # Only done for loginids if userType == "login": try: data = data[data["userid"].apply(lambda x: x.isdigit())] except: print(" Warning: Failed to parse 'userid' column.") try: data = data.loc[data["userid"].astype(int) > 999999, :] except: print(" Warning: Failed to filter 'userid' column.") # Check returned results if data.shape[0] < 10: sys.exit(" No data left after filtering. System exit.") print(" Number of rows removed in 'data' = %i" % (data.shape[0] - nbefore)) return data
def testDtype(self): with self.test_session(): d = array_ops.fill([2, 3], 12., name="fill") self.assertEqual(d.get_shape(), [2, 3]) # Test default type for both constant size and dynamic size z = array_ops.zeros([2, 3]) self.assertEqual(z.dtype, dtypes_lib.float32) self.assertEqual([2, 3], z.get_shape()) self.assertAllEqual(z.eval(), np.zeros([2, 3])) z = array_ops.zeros(array_ops.shape(d)) self.assertEqual(z.dtype, dtypes_lib.float32) self.assertEqual([2, 3], z.get_shape()) self.assertAllEqual(z.eval(), np.zeros([2, 3])) # Test explicit type control for dtype in [ dtypes_lib.float32, dtypes_lib.float64, dtypes_lib.int32, dtypes_lib.uint8, dtypes_lib.int16, dtypes_lib.int8, dtypes_lib.complex64, dtypes_lib.complex128, dtypes_lib.int64, dtypes_lib.bool, dtypes_lib.string ]: z = array_ops.zeros([2, 3], dtype=dtype) self.assertEqual(z.dtype, dtype) self.assertEqual([2, 3], z.get_shape()) z_value = z.eval() self.assertFalse(np.any(z_value)) self.assertEqual((2, 3), z_value.shape) z = array_ops.zeros(array_ops.shape(d), dtype=dtype) self.assertEqual(z.dtype, dtype) self.assertEqual([2, 3], z.get_shape()) z_value = z.eval() self.assertFalse(np.any(z_value)) self.assertEqual((2, 3), z_value.shape)
def compare(neurons1, spikes1, neurons2, spikes2): import matplotlib.pyplot as plt [sn1,ss1]=sort(neurons1,spikes1) [sn2,ss2]=sort(neurons2,spikes2) #sn1 = neurons1 #ss1 = spikes1 #sn2 = neurons2 #ss2 = spikes2 in1 = np.in1d(sn1, sn2) in2 = np.in1d(sn2, sn1) nin = len(sn1[in1]) print "Neuron in 1 but not in 2:", len(sn1)-nin print "Neuron in 2 but not in 1:", len(sn2)-nin for i in range(0,nin): if np.any(ss1[in1][i] > 0) and np.any(ss2[in2][i] > 0): if (len(ss1[in1][i]) == len(ss2[in2][i])): plt.plot(ss1[in1][i]-ss2[in2][i], i*np.ones([len(ss1[in1][i]),1]), '*') else: print "For neuron", sn1[in1][i], "difference in length of spiketrains: ", len(ss1[in1][i]) - len(ss2[in2][i]) print "ss1:", ss1[in1][i] print "ss2:", ss2[in2][i] #plt.plot(ss1[in1][i][:np.min(len(ss1[in1][i]), len(ss2[in2][i]))]-ss2[in2][i][:np.min(len(ss1[in1][i]), len(ss2[in2][i]))], i*np.ones([np.min(len(ss1[in1][i]), len(ss2[in2][i])),1]), '*') plt.show()
def predict(self, session, X, y=None): """Make predictions from the provided model.""" # If y is given, the loss is also calculated # We deactivate dropout by setting it to 1 dp = 1 losses = [] results = [] if np.any(y): data = data_iterator(X, y, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) else: data = data_iterator(X, batch_size=self.config.batch_size, label_size=self.config.label_size, shuffle=False) for step, (x, y) in enumerate(data): feed = self.create_feed_dict(input_batch=x, dropout=dp) if np.any(y): feed[self.labels_placeholder] = y loss, preds = session.run( [self.loss, self.predictions], feed_dict=feed) losses.append(loss) else: preds = session.run(self.predictions, feed_dict=feed) predicted_indices = preds.argmax(axis=1) results.extend(predicted_indices) return np.mean(losses), results
def _LoadHeader(self): ConfigLoader._LoadHeader(self) bc = self.Domain.BlockCounts nBlocks = self.Domain.TotalBlocks # Number of blocks in its neighbourhood. self.BlockNeighbourhoodSize = np.zeros(nBlocks, dtype=np.uint8) # Number of blocks in the neighbourhood that are available self.BlockNeighbourhoodAvailable = np.zeros(nBlocks, dtype=np.uint8) # Number of blocks in the neighbourhood that are done self.BlockNeighbourhoodDone = np.zeros(nBlocks, dtype=np.uint8) # Is the block itself done self.IsBlockDone = np.zeros(nBlocks, dtype=np.bool) # Lock to ensure only one thread at a time updates IsBlockDone # and BlockNeighbourhoodDone self.DoneLock = threading.RLock() # Compute the size of each block's neighbourhood for bIjk, bIdx in self.Domain.BlockIndexer.IterBoth(): for i, delta in enumerate(self.NeighbourhoodOffsets): nIdx = bIdx + delta if np.any(nIdx < 0) or np.any(nIdx >= bc): continue nIjk = self.Domain.BlockIndexer.NdToOne(nIdx) self.BlockNeighbourhoodSize[nIjk] += 1 continue continue return
def allowed_region( V_nj, ave_j ): # read PCs PC1 = V_nj[0] PC2 = V_nj[1] n_band = len( PC1 ) band_ticks = np.arange( n_band ) x_ticks = np.linspace(-0.4,0.2,RESOLUTION) y_ticks = np.linspace(-0.2,0.4,RESOLUTION) x_mesh, y_mesh, band_mesh = np.meshgrid( x_ticks, y_ticks, band_ticks, indexing='ij' ) vec_mesh = x_mesh * PC1[ band_mesh ] + y_mesh * PC2[ band_mesh ] + ave_j[ band_mesh ] x_grid, y_grid = np.meshgrid( x_ticks, y_ticks, indexing='ij' ) prohibited_grid = np.zeros_like( x_grid ) for ii in xrange( len( x_ticks ) ) : for jj in xrange( len( y_ticks ) ) : if np.any( vec_mesh[ii][jj] < 0. ) : prohibited_grid[ii][jj] = 1 if np.any( vec_mesh[ii][jj] > 1. ) : prohibited_grid[ii][jj] = 3 elif np.any( vec_mesh[ii][jj] > 1. ) : prohibited_grid[ii][jj] = 2 else : prohibited_grid[ii][jj] = 0 return x_grid, y_grid, prohibited_grid
def __init__(self, x, y): assert np.ndim(x)==2 and np.ndim(y)==2 and np.shape(x)==np.shape(y), \ 'x and y must be 2D arrays of the same size.' if np.any(np.isnan(x)) or np.any(np.isnan(y)): x = np.ma.masked_where( (isnan(x)) | (isnan(y)) , x) y = np.ma.masked_where( (isnan(x)) | (isnan(y)) , y) self.x_vert = x self.y_vert = y mask_shape = tuple([n-1 for n in self.x_vert.shape]) self.mask_rho = np.ones(mask_shape, dtype='d') # If maskedarray is given for verticies, modify the mask such that # non-existant grid points are masked. A cell requires all four # verticies to be defined as a water point. if isinstance(self.x_vert, np.ma.MaskedArray): mask = (self.x_vert.mask[:-1,:-1] | self.x_vert.mask[1:,:-1] | \ self.x_vert.mask[:-1,1:] | self.x_vert.mask[1:,1:]) self.mask_rho = np.asarray(~(~np.bool_(self.mask_rho) | mask), dtype='d') if isinstance(self.y_vert, np.ma.MaskedArray): mask = (self.y_vert.mask[:-1,:-1] | self.y_vert.mask[1:,:-1] | \ self.y_vert.mask[:-1,1:] | self.y_vert.mask[1:,1:]) self.mask_rho = np.asarray(~(~np.bool_(self.mask_rho) | mask), dtype='d') self._calculate_subgrids() self._calculate_metrics()
def fill_betweenx_discontinuous(ax, ymin, ymax, x, freq=1, **kwargs): """Fill betwwen x even if x is discontinuous clusters Parameters ---------- ax : axis x : list Returns ------- ax : axis """ x = np.array(x) min_gap = 1.1 / freq while np.any(x): # If with single time point if len(x) > 1: xmax = np.where((x[1:] - x[:-1]) > min_gap)[0] else: xmax = [0] # If continuous if not np.any(xmax): xmax = [len(x) - 1] ax.fill_betweenx((ymin, ymax), x[0], x[xmax[0]], **kwargs) # remove from list x = x[(xmax[0] + 1) :] return ax
def test_using_gpu_1(self): # I'm checking if this compiles and runs from theano import function, config, shared, sandbox import theano.tensor as T import numpy import time vlen = 10 * 30 * 70 # 10 x #cores x # threads per core iters = 10 rng = numpy.random.RandomState(22) x = shared(numpy.asarray(rng.rand(vlen), config.floatX)) f = function([], T.exp(x)) # print f.maker.fgraph.toposort() t0 = time.time() for i in xrange(iters): r = f() t1 = time.time() print 'Looping %d times took' % iters, t1 - t0, 'seconds' print 'Result is', r if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]): print 'Used the cpu' else: print 'Used the gpu' if theano.config.device.find('gpu') > -1: assert not numpy.any( [isinstance(x.op,T.Elemwise) for x in f.maker.fgraph.toposort()]) else: assert numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()])
def test_using_gpu_3(self): if theano.config.device.find('gpu') > -1: from theano import function, config, shared, sandbox, Out import theano.tensor as T import numpy import time vlen = 10 * 30 * 70 # 10 x #cores x # threads per core iters = 10 rng = numpy.random.RandomState(22) x = shared(numpy.asarray(rng.rand(vlen), config.floatX)) f = function([], Out(sandbox.cuda.basic_ops.gpu_from_host(T.exp(x)), borrow=True)) # print f.maker.fgraph.toposort() t0 = time.time() for i in xrange(iters): r = f() t1 = time.time() print 'Looping %d times took' % iters, t1 - t0, 'seconds' print 'Result is', r print 'Numpy result is', numpy.asarray(r) if numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()]): print 'Used the cpu' else: print 'Used the gpu' assert not numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()])
def test4d(self): g = Graph() oper = OpThresholdOneLevel(graph=g) oper.MinSize.setValue(self.minSize) oper.MaxSize.setValue(self.maxSize) oper.Threshold.setValue(0.5) oper.InputImage.setValue(self.data) output = oper.Output[:].wait() assert numpy.all(output.shape == self.data.shape) clusters = self.generateData((self.nx, self.ny, self.nz)) cluster1 = numpy.logical_and(output, clusters[0]) assert numpy.any(cluster1 != 0) oper.MinSize.setValue(5) output = oper.Output[:].wait() cluster1 = numpy.logical_and(output, clusters[0]) assert numpy.all(cluster1 == 0) cluster4 = numpy.logical_and(output.squeeze(), clusters[3]) assert numpy.all(cluster4 == 0) cluster5 = numpy.logical_and(output.squeeze(), clusters[2]) assert numpy.all(cluster5 == 0) oper.Threshold.setValue(0.2) output = oper.Output[:].wait() cluster5 = numpy.logical_and(output.squeeze(), clusters[2]) assert numpy.any(cluster5 != 0)
def pop_planes(geometry, kwargs): # Convert miller index specifications to normal vectors miller_defs = kwargs.pop("planes_miller", None) if miller_defs is not None: if np.any(np.all(abs(miller_defs[:,0:3]) < EPSILON, axis=1)): error("Emtpy miller index tuple") miller_defs[:,0:3] = miller_to_normal( np.dot(geometry.latvecs, geometry.bravais_cell), miller_defs[:,0:3]) else: miller_defs = np.zeros((0, 4), dtype=float) # Convert plane normal vector specifications into cartesian coords. normal_defs = kwargs.pop("planes_normal", None) if normal_defs is not None: normal_defs[:,0:3] = geometry.coord_transform( normal_defs[:,0:3], kwargs.pop("planes_normal_coordsys", "lattice")) if np.any(np.all(abs(normal_defs[:,0:3]) < EPSILON, axis=1)): error("Emtpy normal vector definition") else: normal_defs = np.zeros((0, 4), dtype=float) # Append two defintions planes_normal = np.vstack(( miller_defs, normal_defs )) return planes_normal
def test_sequential_as_downstream_of_masking_layer(self): inputs = keras.layers.Input(shape=(3, 4)) x = keras.layers.Masking(mask_value=0., input_shape=(3, 4))(inputs) s = keras.Sequential() s.add(keras.layers.Dense(5, input_shape=(4,))) x = keras.layers.wrappers.TimeDistributed(s)(x) model = keras.Model(inputs=inputs, outputs=x) model.compile( optimizer='rmsprop', loss='mse', run_eagerly=testing_utils.should_run_eagerly()) model_input = np.random.randint( low=1, high=5, size=(10, 3, 4)).astype('float32') for i in range(4): model_input[i, i:, :] = 0. model.fit(model_input, np.random.random((10, 3, 5)), epochs=1, batch_size=6) if not context.executing_eagerly(): # Note: this doesn't work in eager due to DeferredTensor/ops compatibility # issue. mask_outputs = [model.layers[1].compute_mask(model.layers[1].input)] mask_outputs += [model.layers[2].compute_mask( model.layers[2].input, mask_outputs[-1])] func = keras.backend.function([model.input], mask_outputs) mask_outputs_val = func([model_input]) self.assertAllClose(mask_outputs_val[0], np.any(model_input, axis=-1)) self.assertAllClose(mask_outputs_val[1], np.any(model_input, axis=-1))
def __call__(self, x, y, xval, bounds_error=True, fill_value=np.nan): if np.isscalar(xval): # xval is a scalar if xval < x[0] or xval > x[-1]: # the value is out of bounds if bounds_error: raise Exception("x value is out of interpolation bounds") else: return np.nan else: # the value is in the bounds return self.f(x, y, xval) else: # xval is an array inside = (xval >= x[0]) & (xval <= x[-1]) outside = ~inside if np.any(outside): # some values are out of bounds if bounds_error: raise Exception("x values are out of interpolation bounds") else: if np.any(inside): yval = np.zeros(xval.shape) yval[inside] = self.f(x, y, xval[inside]) yval[outside] = fill_value return yval else: return np.repeat(fill_value, xval.shape) else: # all values are in the bounds return self.f(x, y, xval)
def test_impute_random(self): nan = numpy.nan data = [ [1.0, nan, 0.0], [2.0, 1.0, 3.0], [nan, nan, nan] ] domain = Orange.data.Domain( (Orange.data.DiscreteVariable("A", values=["0", "1", "2"]), Orange.data.ContinuousVariable("B"), Orange.data.ContinuousVariable("C")) ) data = Orange.data.Table.from_numpy(domain, numpy.array(data)) cimp1 = column_imputer_random(domain[0], data) self.assertTrue(not numpy.any(numpy.isnan(cimp1(data).X))) cimp2 = column_imputer_random(domain[1], data) self.assertTrue(not numpy.any(numpy.isnan(cimp2(data).X))) cimp3 = column_imputer_random(domain[2], data) self.assertTrue(not numpy.any(numpy.isnan(cimp3(data).X))) imputer = ImputerModel( data.domain, {data.domain[0]: cimp1, data.domain[1]: cimp2, data.domain[2]: cimp3} ) idata = imputer(data) self.assertTrue(not numpy.any(numpy.isnan(idata.X))) definedmask = ~numpy.isnan(data.X) self.assertClose(data.X[definedmask], idata.X[definedmask])
def tracks_to_expected(tracks, vol_dims): # simulate expected behavior of module vol_dims = np.array(vol_dims, dtype=np.int32) counts = np.zeros(vol_dims, dtype=np.int32) elements = {} for t_no, t in enumerate(tracks): u_ps = set() ti = np.round(t).astype(np.int32) for p_no, p in enumerate(ti): if np.any(p < 0): p[p<0] = 0 too_high = p >= vol_dims if np.any(too_high): p[too_high] = vol_dims[too_high]-1 p = tuple(p) if p in u_ps: continue u_ps.add(p) val = t_no if counts[p]: elements[p].append(val) else: elements[p] = [val] counts[p] +=1 return counts, elements
def j_roots(n, alpha, beta, mu=0): """[x,w] = j_roots(n,alpha,beta) Returns the roots (x) of the nth order Jacobi polynomial, P^(alpha,beta)_n(x) and weights (w) to use in Gaussian Quadrature over [-1,1] with weighting function (1-x)**alpha (1+x)**beta with alpha,beta > -1. """ if any(alpha <= -1) or any(beta <= -1): raise ValueError("alpha and beta must be greater than -1.") assert n > 0, "n must be positive." (p, q) = (alpha, beta) # from recurrence relations sbn_J = ( lambda k: 2.0 / (2.0 * k + p + q) * sqrt((k + p) * (k + q) / (2 * k + q + p + 1)) * (np.where(k == 1, 1.0, sqrt(k * (k + p + q) / (2.0 * k + p + q - 1)))) ) if any(p == q): # XXX any or all??? an_J = lambda k: 0.0 * k else: an_J = lambda k: np.where( k == 0, (q - p) / (p + q + 2.0), (q * q - p * p) / ((2.0 * k + p + q) * (2.0 * k + p + q + 2)) ) g = cephes.gamma mu0 = 2.0 ** (p + q + 1) * g(p + 1) * g(q + 1) / (g(p + q + 2)) val = gen_roots_and_weights(n, an_J, sbn_J, mu0) if mu: return val + [mu0] else: return val
def get_polar_motion(time): """ gets the two polar motion components in radians for use with apio13 """ # Get the polar motion from the IERS table xp, yp, status = iers.IERS_Auto.open().pm_xy(time, return_status=True) wmsg = None if np.any(status == iers.TIME_BEFORE_IERS_RANGE): wmsg = ('Tried to get polar motions for times before IERS data is ' 'valid. Defaulting to polar motion from the 50-yr mean for those. ' 'This may affect precision at the 10s of arcsec level') xp.ravel()[status.ravel() == iers.TIME_BEFORE_IERS_RANGE] = _DEFAULT_PM[0] yp.ravel()[status.ravel() == iers.TIME_BEFORE_IERS_RANGE] = _DEFAULT_PM[1] warnings.warn(wmsg, AstropyWarning) if np.any(status == iers.TIME_BEYOND_IERS_RANGE): wmsg = ('Tried to get polar motions for times after IERS data is ' 'valid. Defaulting to polar motion from the 50-yr mean for those. ' 'This may affect precision at the 10s of arcsec level') xp.ravel()[status.ravel() == iers.TIME_BEYOND_IERS_RANGE] = _DEFAULT_PM[0] yp.ravel()[status.ravel() == iers.TIME_BEYOND_IERS_RANGE] = _DEFAULT_PM[1] warnings.warn(wmsg, AstropyWarning) return xp.to(u.radian).value, yp.to(u.radian).value
def sample_representer_points(self): # Sample representer points only in the # configuration space by setting all environmental # variables to 1 D = np.where(self.is_env == 0)[0].shape[0] lower = self.lower[np.where(self.is_env == 0)] upper = self.upper[np.where(self.is_env == 0)] self.sampling_acquisition.update(self.model) for i in range(5): restarts = np.random.uniform(low=lower, high=upper, size=(self.Nb, D)) sampler = emcee.EnsembleSampler(self.Nb, D, self.sampling_acquisition_wrapper) self.zb, self.lmb, _ = sampler.run_mcmc(restarts, 50) if not np.any(np.isinf(self.lmb)): break else: print("Infinity") if np.any(np.isinf(self.lmb)): raise ValueError("Could not sample valid representer points! LogEI is -infinity") if len(self.zb.shape) == 1: self.zb = self.zb[:, None] if len(self.lmb.shape) == 1: self.lmb = self.lmb[:, None] # Project representer points to subspace proj = np.ones([self.zb.shape[0], self.upper[self.is_env == 1].shape[0]]) proj *= self.upper[self.is_env == 1].shape[0] self.zb = np.concatenate((self.zb, proj), axis=1)
def algebraic2parametric(coeff): ''' Based on matlab function "ellipse_param.m" which accompanies "Least-Squares Fitting of Circles and Ellipses", W. Gander, G. H. Golub, R. Strebel, BIT Numerical Mathematics, Springer 1994 convert the coefficients (a,b,c,d,e,f) of the algebraic equation: ax^2 + bxy + cy^2 + dx + ey + f = 0 to the parameters of the parametric equation. The parameters are returned as a dictionary containing: center - center of the ellipse a - major axis b - minor axis alpha - angle of major axis convention note: alpha is measured as positive values towards the y-axis ''' #print coeff #print ("A=%.3f B=%.3f C=%.3f D=%.3f E=%.3f F=%.3f" # %(coeff[0], coeff[1], coeff[2], coeff[3], coeff[4], coeff[5],)) if numpy.any(numpy.isnan(coeff)) or numpy.any(numpy.isinf(coeff)): return None A = numpy.array((coeff[0], coeff[1]/2, coeff[1]/2, coeff[2])) A.shape = 2,2 bb = numpy.asarray(coeff[3:5]) c = coeff[5] D,Q = scipy.linalg.eig(A) D = D.real det = D[0]*D[1] if det <= 0: return None else: bs = numpy.dot(Q.transpose(), bb) alpha = numpy.arctan2(Q[1,0], Q[0,0]) zs = scipy.linalg.solve(-2*numpy.diagflat(D), bs) z = numpy.dot(Q, zs) h = numpy.dot(-bs.transpose(), zs) / 2 - c a = numpy.sqrt(h/D[0]) b = numpy.sqrt(h/D[1]) ## correct backwards major/minor axes ## 'major axis as a, minor axis as b' if b > a: temp = b b = a a = temp alpha = math.pi/2 + alpha #print "alpha", alpha if alpha <= -math.pi/2: alpha += math.pi elif alpha > math.pi/2: alpha -= math.pi return {'center':z, 'a':a, 'b':b, 'alpha':alpha}
def testBinds(self): ds = normalFeatureDataset() ds_data = ds.samples.copy() ds_chunks = ds.chunks.copy() self.failUnless(N.all(ds.samples == ds_data)) # sanity check funcs = ['zscore', 'coarsenChunks'] if externals.exists('scipy'): funcs.append('detrend') for f in funcs: eval('ds.%s()' % f) self.failUnless(N.any(ds.samples != ds_data) or N.any(ds.chunks != ds_chunks), msg="We should have modified original dataset with %s" % f) ds.samples = ds_data.copy() ds.chunks = ds_chunks.copy() # and some which should just return results for f in ['aggregateFeatures', 'removeInvariantFeatures', 'getSamplesPerChunkLabel']: res = eval('ds.%s()' % f) self.failUnless(res is not None, msg='We should have got result from function %s' % f) self.failUnless(N.all(ds.samples == ds_data), msg="Function %s should have not modified original dataset" % f)
def collide(self): for a in self.actors: a.collision_prepare() for i, ai in enumerate(self.actors): if isinstance(ai, ParticleActor): ai.collision_self() for j, aj in enumerate(self.actors): if isinstance(aj, ParticleActor): continue if i == j: continue info = spatial.Info(ai.spatial_grid, aj.spatial_mesh, i==j) mask = info.triangle != -1 active = np.flatnonzero(mask) # active vertex idx if np.any(mask):# and not isinstance(ai, StaticActor): triangle = info.triangle[active] bary = info.bary[active] velocity = aj.velocity[aj.mesh.faces[triangle]] velocity = np.einsum('vtc,vt->vc', velocity, bary) relative_velocity = ai.velocity[active] - velocity friction = 1e-2 stiffness = 1e1 force = info.depth[active][:, None] * info.normal[active] * stiffness - relative_velocity * friction assert not np.any(np.isnan(force)) np.add.at(ai.force, active, force) corners = aj.mesh.faces[triangle] for i in range(3): np.add.at(aj.force, corners[:, i], -bary[:, [i]] * force)
def backward(self, top, propagate_down, bottom): h=bottom[0].data.shape[2] w=bottom[0].data.shape[3] num_proposals = bottom[1].data.shape[0] fbox = self.fbox bottom[0].diff[...] = 0 for b in np.arange(num_proposals): if np.any(self.act[b]==True): wb=fbox[b,2]-fbox[b,0] hb=fbox[b,3]-fbox[b,1] if hb==0 or wb==0: #bounding box with size 0 pass elif h==hb and w==wb: #bounding box with size of the image pass else: bottom[0].diff[0,self.act[b],:,:] += bottom[1].data[b,self.act[b],np.newaxis,np.newaxis]/(h*w-hb*wb) #negative bottom[0].diff[0,self.act[b],fbox[b,1]:fbox[b,3]+1,fbox[b,0]:fbox[b,2]+1] += -bottom[1].data[b,self.act[b],np.newaxis,np.newaxis]/(h*w-hb*wb) #negative bottom[0].diff[0,self.act[b],fbox[b,1]:fbox[b,3]+1,fbox[b,0]:fbox[b,2]+1] += -bottom[1].data[b,self.act[b],np.newaxis,np.newaxis]/(hb*wb) #positive bottom[0].diff[...] *= self.myloss_weight bottom[1].diff[...] = self.myloss_weight*self.hinge if np.any(np.isnan(bottom[0].diff)): print "Nan error" dsfsf if np.any(np.isnan(bottom[1].diff)): print "Nan error" dsfsf
def check_obs_scheme(self): " Checks the internal validity of provided observation schemes " # check sub_pops idx_union = np.sort(self._sub_pops[0]) i = 1 while idx_union.size < self._p and i < len(self._sub_pops): idx_union = np.union1d(idx_union, self._sub_pops[i]) i += 1 if idx_union.size != self._p or np.any(idx_union!=np.arange(self._p)): raise Exception(('all subpopulations together have to cover ' 'exactly all included observed varibles y_i in y.' 'This is not the case. Change the difinition of ' 'subpopulations in variable sub_pops or reduce ' 'the number of observed variables p. ' 'The union of indices of all subpopulations is'), idx_union ) # check obs_time if not self._obs_time[-1]==self._T: raise Exception(('Entries of obs_time give the respective ends of ' 'the periods of observation for any ' 'subpopulation. Hence the last entry of obs_time ' 'has to be the full recording length. The last ' 'entry of obs_time before is '), self._obs_time[-1]) if np.any(np.diff(self._obs_time)<1): raise Exception(('lengths of observation have to be at least 1. ' 'Minimal observation time for a subpopulation: '), np.min(np.diff(self._obs_time))) # check obs_pops if not self._obs_time.size == self._obs_pops.size: raise Exception(('each entry of obs_pops gives the index of the ' 'subpopulation observed up to the respective ' 'time given in obs_time. Thus the sizes of the ' 'two arrays have to match. They do not. ' 'no. of subpop. switch points and no. of ' 'subpopulations ovserved up to switch points ' 'are '), (self._obs_time.size, self._obs_pops.size)) idx_pops = np.sort(np.unique(self._obs_pops)) if not np.min(idx_pops)==0: raise Exception(('first subpopulation has to have index 0, but ' 'is given the index '), np.min(idx_pops)) elif not idx_pops.size == len(self._sub_pops): raise Exception(('number of specified subpopulations in variable ' 'sub_pops does not meet the number of ' 'subpopulations indexed in variable obs_pops. ' 'Delete subpopulations that are never observed, ' 'or change the observed subpopulations in ' 'variable obs_pops accordingly. The number of ' 'indexed subpopulations is '), len(self._sub_pops)) elif not np.all(np.diff(idx_pops)==1): raise Exception(('subpopulation indices have to be consecutive ' 'integers from 0 to the total number of ' 'subpopulations. This is not the case. ' 'Given subpopulation indices are '), idx_pops)
def test_unwrap(self): """Test different geometry types are appropriately unwrapped.""" wrapper = Wrapper() path = tempfile.mkdtemp() for desc, geom in self.possible.iteritems(): unwrapped = wrapper.unwrap(geom) if desc in self.actual_unwrapped: self.assertTrue(self.actual_unwrapped[desc].almost_equals(unwrapped, decimal=5)) try: self.assertEqual(type(geom), type(unwrapped)) except AssertionError: if desc == 'axis_polygon': # by necessity of being split on the axis, this will come out as a multipolygon self.assertIsInstance(unwrapped, MultiPolygon) else: raise self.assertFalse(np.any(np.array(unwrapped) < 0.0)) if isinstance(unwrapped, (MultiPolygon, Polygon)): it = get_iter(unwrapped) for polygon in it: self.assertFalse(np.any(np.array(polygon.exterior) > 360.0)) else: self.assertFalse(np.any(np.array(unwrapped) > 360.0))
def process_shoreline(contours, cloud_mask, georef, image_epsg, settings): """ Converts the contours from image coordinates to world coordinates. This function also removes the contours that are too small to be a shoreline (based on the parameter settings['min_length_sl']) KV WRL 2018 Arguments: ----------- contours: np.array or list of np.array image contours as detected by the function find_contours cloud_mask: np.array 2D cloud mask with True where cloud pixels are georef: np.array vector of 6 elements [Xtr, Xscale, Xshear, Ytr, Yshear, Yscale] image_epsg: int spatial reference system of the image from which the contours were extracted settings: dict with the following keys 'output_epsg': int output spatial reference system 'min_length_sl': float minimum length of shoreline contour to be kept (in meters) Returns: ----------- shoreline: np.array array of points with the X and Y coordinates of the shoreline """ # convert pixel coordinates to world coordinates contours_world = SDS_tools.convert_pix2world(contours, georef) # convert world coordinates to desired spatial reference system contours_epsg = SDS_tools.convert_epsg(contours_world, image_epsg, settings['output_epsg']) # remove contours that have a perimeter < min_length_sl (provided in settings dict) # this enables to remove the very small contours that do not correspond to the shoreline contours_long = [] for l, wl in enumerate(contours_epsg): coords = [(wl[k,0], wl[k,1]) for k in range(len(wl))] a = LineString(coords) # shapely LineString structure if a.length >= settings['min_length_sl']: contours_long.append(wl) # format points into np.array x_points = np.array([]) y_points = np.array([]) for k in range(len(contours_long)): x_points = np.append(x_points,contours_long[k][:,0]) y_points = np.append(y_points,contours_long[k][:,1]) contours_array = np.transpose(np.array([x_points,y_points])) shoreline = contours_array # now remove any shoreline points that are attached to cloud pixels if sum(sum(cloud_mask)) > 0: # get the coordinates of the cloud pixels idx_cloud = np.where(cloud_mask) idx_cloud = np.array([(idx_cloud[0][k], idx_cloud[1][k]) for k in range(len(idx_cloud[0]))]) # convert to world coordinates and same epsg as the shoreline points coords_cloud = SDS_tools.convert_epsg(SDS_tools.convert_pix2world(idx_cloud, georef), image_epsg, settings['output_epsg'])[:,:-1] # only keep the shoreline points that are at least 30m from any cloud pixel idx_keep = np.ones(len(shoreline)).astype(bool) for k in range(len(shoreline)): if np.any(np.linalg.norm(shoreline[k,:] - coords_cloud, axis=1) < 30): idx_keep[k] = False shoreline = shoreline[idx_keep] return shoreline
def __init__(self, loss_factory, X, penalty_structure=None, group_weights={}, elastic_net=iq(0, 0, 0, 0), alpha=0., intercept=True, positive_part=None, unpenalized=None, lagrange_proportion=0.05, nstep=100, scale=True, center=True): self.loss_factory = loss_factory self.scale = scale self.center = center # for group lasso weights, if implied by penalty_structure self.group_weights = group_weights # normalize X, adding intercept if needed self.intercept = intercept p = X.shape[1] if self.intercept: self.penalty_structure = np.ones(p + 1) * L1_PENALTY self.penalty_structure[0] = UNPENALIZED if penalty_structure is not None: self.penalty_structure[1:] = penalty_structure if scipy.sparse.issparse(X): self._X1 = scipy.sparse.hstack([np.ones((X.shape[0], 1)), X]).tocsc() else: self._X1 = np.hstack([np.ones((X.shape[0], 1)), X]) if self.scale or self.center: self._Xn = normalize(self._X1, center=self.center, scale=self.scale, intercept_column=0) which_0 = self._Xn.col_stds == 0 else: self._Xn = self._X1 which_0 = np.zeros(self._Xn.shape) else: self.penalty_structure = np.ones(p) * L1_PENALTY if penalty_structure is not None: self.penalty_structure[:] = penalty_structure if self.scale or self.center: self._Xn = normalize(X, center=self.center, scale=self.scale) which_0 = self._Xn.col_stds == 0 else: self._Xn = X which_0 = np.zeros(self._Xn.shape) if np.any(which_0): self._selector = selector(~which_0, self._Xn.input_shape) if self.scale or self.center: self._Xn = self._Xn.slice_columns(~which_0) else: self._Xn = self._Xn[:, ~which_0] else: if self.scale or self.center: self._selector = identity(self._Xn.input_shape) else: self._selector = identity(self._Xn.shape) # the penalty parameters self.alpha = alpha self.lagrange_proportion = lagrange_proportion self.nstep = nstep self._elastic_net = elastic_net.collapsed() self.initial_active = (np.equal(self.penalty_structure, UNPENALIZED) + np.equal(self.penalty_structure, NONNEGATIVE)) self.ever_active = self.initial_active.copy()
def main(self, inner_tol=1.e-5, verbose=False): # scaling will be needed to get coefficients on original scale if self.scale: scalings = np.asarray(self.Xn.col_stds).reshape(-1) else: scalings = np.ones(self.shape[1]) scalings = self.nonzero.adjoint_map(scalings) # take a guess at the inverse step size self.final_step = 1000. / self.lipschitz lseq = self.lagrange_sequence # shorthand # first solution corresponding to all zeros except intercept self.solution[:] = self.null_solution.copy() grad_solution = self.grad().copy() strong, strong_selector = self.strong_set(lseq[0], lseq[1], grad=grad_solution) p = self.shape[0] rescaled_solutions = scipy.sparse.csr_matrix( self.nonzero.adjoint_map(self.solution) / scalings) objective = [self.loss.smooth_objective(self.solution, 'func')] # not quite right -- should check tight constraints dfs = [np.sum(self.initial_active)] retry_counter = 0 all_failing = np.zeros(grad_solution.shape, np.bool) for lagrange_new, lagrange_cur in zip(lseq[1:], lseq[:-1]): self.lagrange = lagrange_new tol = inner_tol active_old = self.active.copy() num_tries = 0 debug = False coef_stop = True while True: strong, strong_selector = self.strong_set(lagrange_cur, lagrange_new, grad=grad_solution) subproblem_set = self.ever_active + all_failing final_step, grad, sub_soln, penalty_structure \ = self.solve_subproblem(subproblem_set, lagrange_new, tol=tol, start_step=self.final_step, debug=debug and verbose, coef_stop=coef_stop) p = self.shape[1] self.solution[subproblem_set][:] = sub_soln # this only corrects the gradient on the subproblem_set grad_solution[subproblem_set][:] = grad strong_problem = self.restricted_problem(strong, lagrange_new)[0] strong_soln = self.solution[strong] strong_grad = ( strong_problem.smooth_objective(strong_soln, mode='grad') + self.elastic_net[strong].objective(strong_soln, mode='grad')) strong_penalty = strong_problem.proximal_atom strong_failing = check_KKT(strong_penalty, strong_grad, strong_soln, lagrange_new) if np.any(strong_failing): all_failing += (strong_selector.adjoint_map(strong_failing) != 0) else: self.solution[subproblem_set][:] = sub_soln grad_solution = self.grad() all_failing = check_KKT(self.penalty, grad_solution, self.solution, lagrange_new) if not all_failing.sum(): self.ever_active += self.solution != 0 self.final_step = final_step break else: if verbose: print('failing:', np.nonzero(all_failing)[0]) retry_counter += 1 self.ever_active += all_failing tol /= 2. num_tries += 1 if num_tries % 5 == 0: self.solution[subproblem_set][:] = sub_soln self.solution[~subproblem_set][:] = 0 grad_solution = self.grad() debug = True tol = inner_tol if num_tries >= 10: warn('convergence not achieved for lagrange=%0.4e' % lagrange_new) break rescaled_solution = self.nonzero.adjoint_map(self.solution) rescaled_solutions = scipy.sparse.vstack( [rescaled_solutions, rescaled_solution]) objective.append( self.loss.smooth_objective(self.solution, mode='func')) dfs.append(self.ever_active.shape[0]) gc.collect() if verbose: print(lagrange_cur / self.lagrange_max, lagrange_new, (self.solution != 0).sum(), 1. - objective[-1] / objective[0], list(self.lagrange_sequence).index(lagrange_new), np.fabs(rescaled_solution).sum()) objective = np.array(objective) output = { 'devratio': 1 - objective / objective.max(), 'df': dfs, 'lagrange': self.lagrange_sequence, 'scalings': scalings, 'beta': rescaled_solutions.T } return output
change_Lr = tf.keras.callbacks.LearningRateScheduler(scheduler) print("OPTIMIZER\n") #select optimizer,选择优化器 optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate, decay=decay_rate) #checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( # weight_path, monitor='val_accuracy', verbose=1, # save_best_only=False, save_weights_only=True, # save_frequency=1 ) Model.compile(optimizer=optimizer, loss=tf.keras.losses.categorical_crossentropy, metrics=['accuracy']) print("Model fit\n") print('is there any NaN in x_train? ', np.any(np.isnan(x_train))) print('is there any NaN in y_train? ', np.any(np.isnan(y_train))) training = Model.fit(x_train, one_hot_y_train, batch_size=BATCH_SIZE, validation_data=[x_validation, one_hot_y_validation], epochs=EPOCHS, callbacks=[change_Lr]) # print(training.history) #To visualize the training process def plot_history(training): plt.plot(training.history['loss']) plt.plot(training.history['val_loss'])
def actor_critic(agent_name, multiple_agents=False, load_agent=False, n_episodes=300, max_t=1000, train_mode=True): """ Batch processed the states in a single forward pass with a single neural network Params ====== multiple_agents (boolean): boolean for multiple agents PER (boolean): n_episodes (int): maximum number of training episodes max_t (int): maximum number of timesteps per episode """ start = time.time() device = get_device() env, env_info, states, state_size, action_size, brain_name, num_agents = initialize_env( multiple_agents, train_mode) states = torch.from_numpy(states).to(device).float() NUM_PROCESSES = num_agents # Scores is Episode Rewards scores = np.zeros(num_agents) scores_window = deque(maxlen=100) scores_episode = [] actor_critic = ActorCritic(state_size, action_size, device).to(device) agent = A2C_ACKTR(agent_name, actor_critic, value_loss_coef=CRITIC_DISCOUNT, entropy_coef=ENTROPY_BETA, lr=LEARNING_RATE, eps=EPS, alpha=ALPHA, max_grad_norm=MAX_GRAD_NORM, acktr=False, load_agent=load_agent) rollouts = SimpleRolloutStorage(NUM_STEPS, NUM_PROCESSES, state_size, action_size) rollouts.to(device) num_updates = NUM_ENV_STEPS // NUM_STEPS // NUM_PROCESSES # num_updates = NUM_ENV_STEPS // NUM_STEPS print("\n## Loaded environment and agent in {} seconds ##\n".format( round((time.time() - start), 2))) update_start = time.time() timesteps = 0 episode = 0 if load_agent != False: episode = agent.episode while True: """CAN INSERT LR DECAY HERE""" # if episode == MAX_EPISODES: # return scores_episode # Adds noise to agents parameters to encourage exploration # agent.add_noise(PARAMETER_NOISE) for step in range(NUM_STEPS): step_start = time.time() # Sample actions with torch.no_grad(): values, actions, action_log_probs, _ = agent.act(states) clipped_actions = np.clip(actions.cpu().numpy(), *ACTION_BOUNDS) env_info = env.step(actions.cpu().numpy())[ brain_name] # send the action to the environment next_states = env_info.vector_observations # get the next state rewards = env_info.rewards # get the reward rewards_tensor = np.array(env_info.rewards) rewards_tensor[rewards_tensor == 0] = NEGATIVE_REWARD rewards_tensor = torch.from_numpy(rewards_tensor).to( device).float().unsqueeze(1) dones = env_info.local_done masks = torch.from_numpy(1 - np.array(dones).astype(int)).to( device).float().unsqueeze(1) rollouts.insert(states, actions, action_log_probs, values, rewards_tensor, masks, masks) next_states = torch.from_numpy(next_states).to(device).float() states = next_states scores += rewards # print(rewards) if timesteps % 100: print('\rTimestep {}\tScore: {:.2f}\tmin: {:.2f}\tmax: {:.2f}'. format(timesteps, np.mean(scores), np.min(scores), np.max(scores)), end="") if np.any(dones): print( '\rEpisode {}\tScore: {:.2f}\tAverage Score: {:.2f}\tMin Score: {:.2f}\tMax Score: {:.2f}' .format(episode, score, np.mean(scores_window), np.min(scores), np.max(scores)), end="\n") update_csv(agent_name, episode, np.mean(scores_window), np.max(scores)) if episode % 20 == 0: agent.save_agent(agent_name, score, episode, save_history=True) else: agent.save_agent(agent_name, score, episode) episode += 1 scores = np.zeros(num_agents) break timesteps += 1 with torch.no_grad(): next_values, _, _, _ = agent.act(next_states) rollouts.compute_returns(next_values, USE_GAE, GAMMA, GAE_LAMBDA) agent.update(rollouts) score = np.mean(scores) scores_window.append(score) # save most recent score scores_episode.append(score) return scores_episode
def stepp_analysis(year, mw, dm=0.1, dt=1, ttol=0.2, iloc=True): """ Stepp algorithm :param year: catalog matrix year column :type year: numpy.ndarray :param mw: catalog matrix magnitude column :type mw: numpy.ndarray :keyword dm: magnitude interval/window :type dm: positive float :keyword dt: time interval :type dt: int :keyword ttol: tolerance threshold :type ttol: positive float :keyword iloc: Fix analysis such that completeness magnitude can only increase with catalogue duration (i.e. completess cannot increase for more recent catalogues) :type iloc: bool :returns: two-column completeness table representing the earliest year at which the catalogue is complete above a given magnitude :rtype: numpy.ndarray """ # Round off the magnitudes to 2 d.p mw = np.around(100.0 * mw) / 100.0 lowm = np.floor(10. * np.min(mw)) / 10. highm = np.ceil(10. * np.max(mw)) / 10. # Determine magnitude bins mbin = np.arange(lowm, highm + dm, dm) ntb = np.max(np.shape(mbin)) # Determine time bins end_time = np.max(year) start_time = np.min(year) time_range = np.arange(dt, end_time - start_time + 2, dt) nt = np.max(np.shape(time_range)) t_upper_bound = end_time * np.ones(nt) t_lower_bound = t_upper_bound - time_range t_rate = 1. / np.sqrt(time_range) # Poisson rate number_obs = np.zeros((nt, ntb - 1)) lamda = np.zeros((nt, ntb - 1)) siglam = np.zeros((nt, ntb - 1)) ii = 0 # count number of events catalogue and magnitude windows while ii <= (nt - 1): # Select earthquakes later than or in Year[ii] yrchk = year >= t_lower_bound[ii] mtmp = mw[yrchk] jj = 0 while jj <= (ntb - 2): #Count earthquakes in magnitude bin if jj == (ntb - 2): number_obs[ii, jj] = np.sum(mtmp >= np.sum(mbin[jj])) else: number_obs[ii, jj] = np.sum( np.logical_and(mtmp >= mbin[jj], mtmp < mbin[jj + 1])) jj = jj + 1 ii = ii + 1 time_diff = (np.log10(t_rate[1:]) - np.log10(t_rate[:-1])) time_diff = time_diff / (np.log10(time_range[1:]) - np.log10(time_range[:-1])) comp_length = np.zeros((ntb - 1, 1)) tloc = np.zeros((ntb - 1, 1), dtype=int) ii = 0 while ii < (ntb - 1): lamda[:, ii] = number_obs[:, ii] / time_range siglam[:, ii] = np.sqrt(lamda[:, ii] / time_range) zero_find = siglam[:, ii] < 1E-14 # To avoid divide by zero siglam[zero_find, ii] = 1E-14 grad1 = (np.log10(siglam[1:, ii]) - np.log10(siglam[:-1, ii])) grad1 = grad1 / (np.log10(time_range[1:]) - np.log10(time_range[:-1])) resid1 = grad1 - time_diff test1 = np.abs(resid1[1:] - resid1[:-1]) tloct = np.nonzero(test1 > ttol)[0] if not (np.any(tloct)): tloct = -1 else: tloct = tloct[-1] if tloct < 0: # No location passes test if ii > 0: # Use previous value tloc[ii] = tloc[ii - 1] else: # Print warning LOGGER.critical( "Fitting tolerance removed all data - change parameter") else: tloc[ii] = tloct if tloct > np.max(np.shape(time_range)): tloc[ii] = np.max(np.shape(time_range)) if ii > 0: # If the increasing completeness is option is set # and the completeness is lower than the previous value # then fix at previous value inc_check = np.logical_and(iloc, (tloc[ii] < tloc[ii - 1])) if inc_check: tloc[ii] = tloc[ii - 1] comp_length[ii] = time_range[tloc[ii]] ii = ii + 1 completeness_table = np.column_stack([end_time - comp_length, mbin[:-1].T]) return completeness_table
def core_test_convolution_double_backward(inshape, kernel, outmaps, pad, stride, dilation, group, channel_last, with_bias, base_axis, seed, ctx, func_name, non_accum_check=True, atol_f=1e-4, atol_b=1e-3, atol_accum=8e-2, dstep=1e-3): from nbla_test_utils import backward_function_tester, grad_function_forward_function_output from nnabla.backward_function.convolution import ConvolutionDataGrad, ConvolutionFilterGrad if func_name == 'ConvolutionCuda': pytest.skip('CUDA Convolution N-D is only supported in CUDNN extension') if channel_last and not func_name.endswith('Cudnn'): pytest.skip( 'channel_last=True is only supported in CUDNN backend so far.') if channel_last and func_name.endswith('Cudnn') and (np.any(np.asarray(dilation) > 1) or group > 1): import nnabla_ext.cuda as nc major, minor, revision = map(int, nc.__cudnn_version__.split('.')) version = major * 1000 + minor * 100 if version < 7200: pytest.skip( 'channel_last dilated convolution not work in CUDNN {}.'.format(version)) # base_axis = len(inshape) - len(kernel) - 1 inmaps = inshape[base_axis] if channel_last: t = refs.ChannelLastToFirstTranspose(len(inshape), len(kernel)) inshape = tuple(inshape[i] for i in t.inv_axes) rng = np.random.RandomState(seed) i = np.clip(rng.randn(*inshape).astype(np.float32), -0.8, 0.8) kshape = (outmaps,) + (inmaps // group,) + kernel if channel_last: t = refs.ChannelLastToFirstTranspose(len(kshape), len(kernel)) kshape = tuple(kshape[i] for i in t.inv_axes) k = np.clip(rng.randn(*kshape).astype(np.float32), -0.8, 0.8) b = None if with_bias: b = np.clip(rng.randn(outmaps).astype(np.float32), -0.8, 0.8) inputs = [i, k, b] atol_half = 1.0 if inmaps > 64 else 1e-1 func_args = [base_axis, pad, stride, dilation, group, channel_last] # Convolution backward_function_tester(rng, F.convolution, inputs, func_args=func_args, atol_f=atol_f, atol_accum=atol_accum, dstep=dstep, ctx=ctx) # DataGrad df, y = grad_function_forward_function_output(ConvolutionDataGrad, F.convolution, ctx, inputs, *func_args) df.xshape = i.shape ginputs = [rng.randn(*y.shape), k] backward_function_tester(rng, df, ginputs, func_args=[], atol_f=atol_f, atol_b=atol_b, atol_accum=atol_accum, dstep=dstep, ctx=ctx, non_accum_check=non_accum_check) # FilterGrad df, y = grad_function_forward_function_output(ConvolutionFilterGrad, F.convolution, ctx, inputs, *func_args) df.wshape = k.shape ginputs = [rng.randn(*y.shape), i] backward_function_tester(rng, df, ginputs, func_args=[], atol_f=atol_f, atol_b=atol_b, atol_accum=atol_accum, dstep=dstep, ctx=ctx, non_accum_check=non_accum_check)
def classify_image_NN(im_ms, im_extra, cloud_mask, min_beach_area, clf): """ Classifies every pixel in the image in one of 4 classes: - sand --> label = 1 - whitewater (breaking waves and swash) --> label = 2 - water --> label = 3 - other (vegetation, buildings, rocks...) --> label = 0 The classifier is a Neural Network that is already trained. KV WRL 2018 Arguments: ----------- im_ms: np.array Pansharpened RGB + downsampled NIR and SWIR im_extra: only used for Landsat 7 and 8 where im_extra is the panchromatic band cloud_mask: np.array 2D cloud mask with True where cloud pixels are min_beach_area: int minimum number of pixels that have to be connected to belong to the SAND class clf: joblib object pre-trained classifier Returns: ----------- im_classif: np.array 2D image containing labels im_labels: np.array of booleans 3D image containing a boolean image for each class (im_classif == label) """ # calculate features vec_features = calculate_features(im_ms, cloud_mask, np.ones(cloud_mask.shape).astype(bool)) vec_features[np.isnan(vec_features)] = 1e-9 # NaN values are create when std is too close to 0 # remove NaNs and cloudy pixels vec_cloud = cloud_mask.reshape(cloud_mask.shape[0]*cloud_mask.shape[1]) vec_nan = np.any(np.isnan(vec_features), axis=1) vec_mask = np.logical_or(vec_cloud, vec_nan) vec_features = vec_features[~vec_mask, :] # classify pixels labels = clf.predict(vec_features) # recompose image vec_classif = np.nan*np.ones((cloud_mask.shape[0]*cloud_mask.shape[1])) vec_classif[~vec_mask] = labels im_classif = vec_classif.reshape((cloud_mask.shape[0], cloud_mask.shape[1])) # create a stack of boolean images for each label im_sand = im_classif == 1 im_swash = im_classif == 2 im_water = im_classif == 3 # remove small patches of sand or water that could be around the image (usually noise) im_sand = morphology.remove_small_objects(im_sand, min_size=min_beach_area, connectivity=2) im_water = morphology.remove_small_objects(im_water, min_size=min_beach_area, connectivity=2) im_labels = np.stack((im_sand,im_swash,im_water), axis=-1) return im_classif, im_labels
def run(model, experiment_args, train=True): total_time_start = time.time() env, comm_env, memory, ounoise, comm_ounoise, config, summaries, saver, start_episode = experiment_args start_episode = start_episode if train else 0 NUM_EPISODES = config['n_episodes'] if train else config['n_episodes_test'] EPISODE_LENGTH = config['episode_length'] if train else config[ 'episode_length_test'] t = 0 episode_rewards_all = [] episode_aux_rewards_all = [] episode_comm_all = [] episode_comm_any_all = [] print('oops changed') for i_episode in range(start_episode, NUM_EPISODES): episode_time_start = time.time() frames = [] episode_comm = 0 episode_comm_any = 0 hier_C = config['hierarchical_time_scale'] if train else 1 # Initialize the environment and state observations = np.stack(env.reset()) observations = K.tensor(observations, dtype=K.float32).unsqueeze(1) cumm_rewards = K.zeros((model.num_agents, 1, 1), dtype=dtype) ounoise.scale = get_noise_scale(i_episode, config) comm_ounoise.scale = get_noise_scale(i_episode, config) # monitoring variables episode_rewards = np.zeros((model.num_agents, 1, 1)) episode_aux_rewards = np.zeros((model.num_agents, 1, 1)) for i_step in range(EPISODE_LENGTH): model.to_cpu() if model.communication == 'hierarchical': if i_step % hier_C == 0: observations_init = observations.clone() cumm_rewards = K.zeros((model.num_agents, 1, 1), dtype=dtype) comm_actions = [] for i in range(model.num_agents): if model.discrete_comm: comm_action = model.select_comm_action( observations_init[[i], ], i, True if train else False) else: comm_action = model.select_comm_action( observations_init[[i], ], i, comm_ounoise if train else False) comm_actions.append(comm_action) comm_actions = K.stack(comm_actions) #medium = observations_init[(comm_actions > .5)[:,0,0]] #medium = (K.mean(observations_init, dim=0) if medium.shape == K.Size([0]) else K.mean(medium, dim=0)).unsqueeze(0) if config['agent_alg'] == 'MAHCDDPG_Multi': medium = [] allocaters = [] for i in range(model.num_agents): medium.append(observations_init[to_onehot( comm_actions[:, :, i])].squeeze(0)) allocaters.append( np.asarray(comm_actions[:, :, i].view(-1).argmax())) allocaters = np.asarray(allocaters) medium = K.stack(medium) else: if model.discrete_comm: medium = observations_init[to_onehot( comm_actions[:, :, 1].unsqueeze(2))] else: medium = observations_init[to_onehot(comm_actions)] allocaters = comm_actions.view(-1).argmax() allocaters = np.asarray(allocaters) else: if config['agent_alg'] == 'MAMDDPG': medium = model.select_comm_action(observations).unsqueeze( 0) #if i_episode > 1000: #pdb.set_trace() actions = [] for i in range(model.num_agents): if config['agent_alg'] == 'MAHCDDPG_Multi': action = model.select_action( K.cat([observations[[i], ], medium[[i], ]], dim=-1), i, ounoise if train else False) else: action = model.select_action( K.cat([observations[[i], ], medium], dim=-1), i, ounoise if train else False) actions.append(action) actions = K.stack(actions) next_observations, rewards, dones, infos = env.step( actions.squeeze(1)) next_observations = K.tensor(next_observations, dtype=dtype).unsqueeze(1) rewards = K.tensor(rewards, dtype=dtype).view(-1, 1, 1) # different types od aux reward to train the second policy if config['agent_alg'] == 'MAHCDDPG_Multi': intr_rewards = intrinsic_reward_multi(env, medium.numpy()) else: intr_rewards = intrinsic_reward(env, medium.numpy()) intr_rewards = K.tensor(intr_rewards, dtype=dtype).view(-1, 1, 1) cumm_rewards += rewards if config['aux_reward_type'] == 'intrinsic': aux_rewards = intr_rewards elif config['aux_reward_type'] == 'cummulative': aux_rewards = rewards # for monitoring episode_rewards += rewards episode_aux_rewards += aux_rewards # if it is the last step we don't need next obs if i_step == EPISODE_LENGTH - 1: next_observations = None # Store the transition in memory if train: memory[0].push(observations, actions, next_observations, aux_rewards, medium, None, None, None, None) if model.communication == 'hierarchical' and (i_step + 1) % hier_C == 0: memory[1].push(observations_init, None, next_observations, cumm_rewards, medium, comm_actions, None, None, None) # Move to the next state observations = next_observations t += 1 # Use experience replay and train the model critic_losses = None actor_losses = None medium_loss = None if train: if (sum([ True for i_memory in memory if len(i_memory) > config['batch_size'] - 1 ]) == len(memory) and t % config['steps_per_update'] == 0): model.to_cuda() critic_losses = [] actor_losses = [] for i in range(env.n): batch = Transition_Comm(*zip( *memory[0].sample(config['batch_size']))) if model.communication == 'hierarchical': batch2 = Transition_Comm(*zip( *memory[1].sample(config['batch_size']))) critic_loss, actor_loss = model.update_parameters( batch, batch2, i) else: critic_loss, actor_loss, medium_loss = model.update_parameters( batch, i) critic_losses.append(critic_loss) actor_losses.append(actor_loss) # Record frames if config['render'] > 0 and i_episode % config['render'] == 0: if config['env_id'] == 'waterworld': frames.append(sc.misc.imresize(env.render(), (300, 300))) else: frames.append(env.render(mode='rgb_array')[0]) if config['render_color_change']: for i_geoms, geoms in enumerate(env.render_geoms): if i_geoms == env.world.leader: geoms.set_color(0.85, 0.35, 0.35, 0.55) else: geoms.set_color(0.35, 0.35, 0.85, 0.55) if i_geoms == env.n - 1: break if config['agent_alg'] == 'MAHCDDPG_Multi': episode_comm += np.all((env.world.leader - 1) % model.num_agents == allocaters) episode_comm_any += np.any((env.world.leader - 1) % model.num_agents == allocaters) elif config['agent_alg'] == 'MAHCDDPG': episode_comm += np.all(env.world.leader == allocaters) episode_comm_any += np.any(env.world.leader == allocaters) # <-- end loop: i_step ### MONITORIRNG ### episode_rewards_all.append(episode_rewards.sum()) episode_aux_rewards_all.append(episode_aux_rewards.sum()) episode_comm_all.append(episode_comm) episode_comm_any_all.append(episode_comm_any) if config['verbose'] > 0: # Printing out if (i_episode + 1) % 100 == 0: print("==> Episode {} of {}".format(i_episode + 1, NUM_EPISODES)) print(' | Id exp: {}'.format(config['exp_id'])) print(' | Exp description: {}'.format(config['exp_descr'])) print(' | Env: {}'.format(config['env_id'])) print(' | Process pid: {}'.format(config['process_pid'])) print(' | Tensorboard port: {}'.format(config['port'])) print(' | Episode total reward: {}'.format( episode_rewards.sum())) print(' | Running mean of total reward: {}'.format( running_mean(episode_rewards_all)[-1])) print(' | Running mean of total comm_reward: {}'.format( running_mean(episode_aux_rewards_all)[-1])) print(' | Medium loss: {}'.format(medium_loss)) print(' | Time episode: {}'.format(time.time() - episode_time_start)) print(' | Time total: {}'.format(time.time() - total_time_start)) if config['verbose'] > 0: ep_save = i_episode + 1 if (i_episode == NUM_EPISODES - 1) else None is_best_save = None is_best_avg_save = None if (not train) or ((np.asarray([ ep_save, is_best_save, is_best_avg_save ]) == None).sum() == 3): to_save = False else: model.to_cpu() saver.save_checkpoint(save_dict={ 'model_params': [entity.state_dict() for entity in model.entities] }, episode=ep_save, is_best=is_best_save, is_best_avg=is_best_avg_save) to_save = True if (i_episode + 1) % 100 == 0: summary = summaries[0] if train else summaries[1] summary.update_log(i_episode, episode_rewards.sum(), list(episode_rewards.reshape(-1, )), critic_loss=critic_losses, actor_loss=actor_losses, to_save=to_save, comm_reward_total=episode_aux_rewards.sum(), comm_reward_agents=list( episode_aux_rewards.reshape(-1, ))) # Save gif dir_monitor = config['dir_monitor_train'] if train else config[ 'dir_monitor_test'] if config['render'] > 0 and i_episode % config['render'] == 0: if config['env_id'] == 'waterworld': imageio.mimsave('{}/{}.gif'.format(dir_monitor, i_episode), frames[0::3]) else: imageio.mimsave('{}/{}.gif'.format(dir_monitor, i_episode), frames) # <-- end loop: i_episode if train: print('Training completed') else: print('Test completed') return (episode_rewards_all, episode_aux_rewards_all, episode_comm_all, episode_comm_any_all)
def find_wl_contours2(im_ms, im_labels, cloud_mask, buffer_size, im_ref_buffer): """ New robust method for extracting shorelines. Incorporates the classification component to refine the treshold and make it specific to the sand/water interface. KV WRL 2018 Arguments: ----------- im_ms: np.array RGB + downsampled NIR and SWIR im_labels: np.array 3D image containing a boolean image for each class in the order (sand, swash, water) cloud_mask: np.array 2D cloud mask with True where cloud pixels are buffer_size: int size of the buffer around the sandy beach over which the pixels are considered in the thresholding algorithm. im_ref_buffer: np.array binary image containing a buffer around the reference shoreline Returns: ----------- contours_wi: list of np.arrays contains the coordinates of the contour lines extracted from the NDWI (Normalized Difference Water Index) image contours_mwi: list of np.arrays contains the coordinates of the contour lines extracted from the MNDWI (Modified Normalized Difference Water Index) image """ nrows = cloud_mask.shape[0] ncols = cloud_mask.shape[1] # calculate Normalized Difference Modified Water Index (SWIR - G) im_mwi = SDS_tools.nd_index(im_ms[:,:,4], im_ms[:,:,1], cloud_mask) # calculate Normalized Difference Modified Water Index (NIR - G) im_wi = SDS_tools.nd_index(im_ms[:,:,3], im_ms[:,:,1], cloud_mask) # stack indices together im_ind = np.stack((im_wi, im_mwi), axis=-1) vec_ind = im_ind.reshape(nrows*ncols,2) # reshape labels into vectors vec_sand = im_labels[:,:,0].reshape(ncols*nrows) vec_water = im_labels[:,:,2].reshape(ncols*nrows) # create a buffer around the sandy beach se = morphology.disk(buffer_size) im_buffer = morphology.binary_dilation(im_labels[:,:,0], se) vec_buffer = im_buffer.reshape(nrows*ncols) # select water/sand/swash pixels that are within the buffer int_water = vec_ind[np.logical_and(vec_buffer,vec_water),:] int_sand = vec_ind[np.logical_and(vec_buffer,vec_sand),:] # make sure both classes have the same number of pixels before thresholding if len(int_water) > 0 and len(int_sand) > 0: if np.argmin([int_sand.shape[0],int_water.shape[0]]) == 1: int_sand = int_sand[np.random.choice(int_sand.shape[0],int_water.shape[0], replace=False),:] else: int_water = int_water[np.random.choice(int_water.shape[0],int_sand.shape[0], replace=False),:] # threshold the sand/water intensities int_all = np.append(int_water,int_sand, axis=0) t_mwi = filters.threshold_otsu(int_all[:,0]) t_wi = filters.threshold_otsu(int_all[:,1]) # find contour with MS algorithm im_wi_buffer = np.copy(im_wi) im_wi_buffer[~im_ref_buffer] = np.nan im_mwi_buffer = np.copy(im_mwi) im_mwi_buffer[~im_ref_buffer] = np.nan contours_wi = measure.find_contours(im_wi_buffer, t_wi) contours_mwi = measure.find_contours(im_mwi_buffer, t_mwi) # remove contour points that are NaNs (around clouds) contours = contours_wi contours_nonans = [] for k in range(len(contours)): if np.any(np.isnan(contours[k])): index_nan = np.where(np.isnan(contours[k]))[0] contours_temp = np.delete(contours[k], index_nan, axis=0) if len(contours_temp) > 1: contours_nonans.append(contours_temp) else: contours_nonans.append(contours[k]) contours_wi = contours_nonans # repeat for MNDWI contours contours = contours_mwi contours_nonans = [] for k in range(len(contours)): if np.any(np.isnan(contours[k])): index_nan = np.where(np.isnan(contours[k]))[0] contours_temp = np.delete(contours[k], index_nan, axis=0) if len(contours_temp) > 1: contours_nonans.append(contours_temp) else: contours_nonans.append(contours[k]) contours_mwi = contours_nonans return contours_wi, contours_mwi
def calc_templatematch(Img, template = []): return match_template(Img, template)[0,0] if np.any(template) else 0
def destruct(image, mask, imagesz=(210, 238, 200), inputsz=(1, 32, 32, 32), outputsz=(2, 14, 14, 14)): """The input is an image and a mask of shape (modalities, imagesz) and (imagesz). The method decompses it into patches of size (inputsz) and (outputsz[1-3]). Finally it return a numpy array of shape (number of patches, inputsz) and (number of patches, 1, outputsz[1-3])""" cut_mask = np.zeros(imagesz) while not np.any(cut_mask): index = random.randrange(imagesz[0]) cut_img = image[:, index] cut_mask = mask[index] n_height = ceil(imagesz[0] / outputsz[1]) n_width = ceil(imagesz[1] / outputsz[2]) n_depth = ceil(imagesz[2] / outputsz[3]) n_patches = n_height * n_width * n_depth patches_mask = np.zeros((n_patches, outputsz[1], outputsz[2], outputsz[3])) patches_image = np.zeros( (n_patches, inputsz[0], inputsz[1], inputsz[2], inputsz[3])) big_cut_mask = np.zeros( (imagesz[0] + inputsz[1] * 2, imagesz[1] + inputsz[2] * 2, imagesz[2] + inputsz[3] * 2)) big_cut_mask[inputsz[1]:inputsz[1] + imagesz[0], inputsz[2]:inputsz[2] + imagesz[1], inputsz[3]:inputsz[3] + imagesz[2]] = mask big_cut_img = np.zeros( (inputsz[0], imagesz[0] + inputsz[1] * 2, imagesz[1] + inputsz[2] * 2, imagesz[2] + inputsz[3] * 2)) big_cut_img[:, inputsz[1]:inputsz[1] + imagesz[0], inputsz[2]:inputsz[2] + imagesz[1], inputsz[3]:inputsz[3] + imagesz[2]] = image for x in range(n_height): for y in range(n_width): for z in range(n_depth): j = x * outputsz[1] k = y * outputsz[2] l = z * outputsz[3] shift_j = int((inputsz[1] - outputsz[1]) / 2) shift_k = int((inputsz[2] - outputsz[2]) / 2) shift_l = int((inputsz[3] - outputsz[3]) / 2) patches_mask[x * n_width * n_depth + y * n_depth + z] = big_cut_mask[inputsz[1] + j:inputsz[1] + j + outputsz[1], inputsz[2] + k:inputsz[2] + k + outputsz[2], inputsz[3] + l:inputsz[3] + l + outputsz[3]] patches_image[x * n_width * n_depth + y * n_depth + z] = big_cut_img[:, inputsz[1] + j - shift_j:inputsz[1] + j + inputsz[1] - shift_j, inputsz[2] + k - shift_k:inputsz[2] + k + inputsz[2] - shift_k, inputsz[3] + l - shift_l:inputsz[3] + l + inputsz[3] - shift_l] patches_mask = np.reshape( patches_mask, (n_patches, 1, outputsz[1], outputsz[2], outputsz[3])) return patches_image, patches_mask
def fmt_export(arr, delimiter='\t', header=True, sig_fig=8, width='auto', justify='left', sign=False, pad=''): """ Create a format string for array `arr` such that columns are aligned in the output file when saving with np.savetxt """ with np.testing.suppress_warnings() as sup: sup.filter(RuntimeWarning) flag1 = '' if justify != 'left' else '-' flag2 = '+' if sign else '' flag3 = '0' if pad == '0' else '' fmt = [] hdr = [] for j, name in enumerate(arr.dtype.names): dtype = arr[name].dtype if dtype.kind in ['b']: specifier = 'i' precision = '' w = 4 if np.all(arr[name]) else 5 elif dtype.kind in ['i', 'u']: specifier = 'i' precision = '' w = _get_f_width(arr[name], sign) elif dtype.kind in ['f', 'c']: specifier = 'g' precision = '.' + str(sig_fig) # float notation width # todo check for nan widths w_f = _get_f_width(arr[name], sign) + sig_fig # scientific notation width i = 1 if sign or np.any(arr[name] < 0) else 0 w_s = sig_fig + 4 + i + 1 # +1 for decimal point which is not always needed w = min(w_f, w_s) + 1 elif dtype.kind in ['U', 'S', 'O']: specifier = 's' precision = '' w = np.max([len(str(item)) for item in arr[name]]) else: raise TypeError(f'Invalid dtype kind {dtype.kind} for field {name}') if width == 'auto': col_w = w elif isinstance(width, int): col_w = width else: raise ValueError('Invalid width') if header: i = 2 if j == 0 else 0 # Additional space for header comment # if width == 'auto': _width = max(col_w, len(name) + i) elif isinstance(width, int): _width = col_w func = str.ljust if justify == 'left' else str.rjust fill = flag3 if flag3 else ' ' h = func(name, _width - i, fill) hdr.append(h) else: _width = col_w s = f'%{flag1}{flag2}{flag3}{_width}{precision}{specifier}' fmt.append(s) fmt = delimiter.join(fmt) hdr = delimiter.join(hdr) return fmt, hdr
def runme(id=None,exclude=None,benchmark='nightly',procedure='check',output='none',rank=1,numprocs=1): """ RUNME - test deck for ISSM nightly runs In a test deck directory (tests/Vertification/NightlyRun for example) The following command will launch all the existing tests: >> runme() To run the tests 101 and 102: >> runme(id=[101,102]) etc... Available options: 'id' followed by the list of ids requested 'exclude' ids to be excluded from the test 'benchmark' 'all' (all of the tests) 'nightly' (nightly run/ daily run) 'ismip' : validation of ismip-hom tests 'eismint': validation of eismint tests 'thermal': validation of thermal tests 'mesh' : validation of mesh tests 'adolc' : validation of adolc tests 'slr' : validation of slr tests 'procedure' 'check' : run the test (default) 'update': update the archive Usage: runme(varargin) Examples: runme() runme(exclude=101) runme(id=102,procedure='update') """ from parallelrange import parallelrange from IdToName import IdToName from arch import archread from arch import archwrite from arch import archdisp #Get ISSM_DIR variable ISSM_DIR=os.environ['ISSM_DIR'] #Process options #GET benchmark {{{ if not benchmark in ['all','nightly','ismip','eismint','thermal','mesh','validation','tranforcing','adolc','slr','referential']: print("runme warning: benchmark '{}' not supported, defaulting to test 'nightly'.".format(benchmark)) benchmark='nightly' # }}} #GET procedure {{{ if not procedure in ['check','update']: print("runme warning: procedure '{}' not supported, defaulting to test 'check'.".format(procedure)) procedure='check' # }}} #GET output {{{ if not output in ['nightly','none']: print("runme warning: output '{}' not supported, defaulting to test 'none'.".format(output)) output='none' # }}} #GET RANK and NUMPROCS for multithreaded runs {{{ if (numprocs<rank): numprocs=1 # }}} #GET ids {{{ flist=glob('test*.py') #File name must start with 'test' and must end by '.py' and must be different than 'test.py' list_ids=[int(file[4:-3]) for file in flist if not file == 'test.py'] #Keep test id only (skip 'test' and '.py') #print 'list_ids =',list_ids i1,i2=parallelrange(rank,numprocs,len(list_ids)) #Get tests for this cpu only list_ids=list_ids[i1:i2+1] #print 'list_ids after parallelrange =',list_ids if id: if isinstance(id,list): test_ids=id else: test_ids=[id] test_ids=set(test_ids).intersection(set(list_ids)) else: test_ids=set(list_ids) #print 'test_ids after list =',test_ids # }}} #GET exclude {{{ if exclude: if isinstance(exclude,list): exclude_ids=exclude else: exclude_ids=[exclude] test_ids=test_ids.difference(set(exclude_ids)) # print 'test_ids after exclude =',test_ids # }}} #Process Ids according to benchmarks {{{ if benchmark=='nightly': test_ids=test_ids.intersection(set(range(1,1000))) elif benchmark=='validation': test_ids=test_ids.intersection(set(range(1001,2000))) elif benchmark=='ismip': test_ids=test_ids.intersection(set(range(1101,1200))) elif benchmark=='eismint': test_ids=test_ids.intersection(set(range(1201,1300))) elif benchmark=='thermal': test_ids=test_ids.intersection(set(range(1301,1400))) elif benchmark=='mesh': test_ids=test_ids.intersection(set(range(1401,1500))) elif benchmark=='tranforcing': test_ids=test_ids.intersection(set(range(1501,1503))) elif benchmark=='referential': test_ids=test_ids.intersection(set(range(1601,1603))) elif benchmark=='slr': test_ids=test_ids.intersection(set(range(2001,2500))) elif benchmark=='adolc': test_ids=test_ids.intersection(set(range(3001,3200))) #print 'test_ids after benchmark =',test_ids test_ids=list(test_ids) test_ids.sort() #print 'test_ids after sort =',test_ids # }}} #Loop over tests and launch sequence root=os.getcwd() for id in test_ids: print "----------------starting:%i-----------------------" % id try: #Execute test os.chdir(root) id_string=IdToName(id) execfile('test'+str(id)+'.py',globals()) #UPDATE ARCHIVE? archive_name='Archive'+str(id) if procedure=='update': archive_file=os.path.join('..','Archives',archive_name+'.arch') if os.path.isfile(archive_file): os.remove(archive_file) for k,fieldname in enumerate(field_names): field=np.array(field_values[k],dtype=float) if len(field.shape) == 1: if np.size(field): field=field.reshape(np.size(field),1) else: field=field.reshape(0,0) elif len(field.shape) == 0: field=field.reshape(1,1) # Matlab uses base 1, so use base 1 in labels archwrite(archive_file,archive_name+'_field'+str(k+1),field) print "File '%s' saved.\n" % os.path.join('..','Archives',archive_name+'.arch') #ELSE: CHECK TEST else: #load archive if os.path.exists(os.path.join('..','Archives',archive_name+'.arch')): archive_file=os.path.join('..','Archives',archive_name+'.arch') else: raise IOError("Archive file '"+os.path.join('..','Archives',archive_name+'.arch')+"' does not exist.") for k,fieldname in enumerate(field_names): try: #Get field and tolerance field=np.array(field_values[k]) if len(field.shape) == 1: if np.size(field): field=field.reshape(np.size(field),1) else: field=field.reshape(0,0) tolerance=field_tolerances[k] #compare to archive # Matlab uses base 1, so use base 1 in labels archive=np.array(archread(archive_file,archive_name+'_field'+str(k+1))) if archive == None: raise NameError("Field name '"+archive_name+'_field'+str(k+1)+"' does not exist in archive file.") error_diff=np.amax(np.abs(archive-field),axis=0)/(np.amax(np.abs(archive),axis=0)+float_info.epsilon) #disp test result if (np.any(error_diff>tolerance) or np.isnan(error_diff)): print('ERROR difference: {} > {} test id: {} test name: {} field: {}'.format(error_diff,tolerance,id,id_string,fieldname)) else: print('SUCCESS difference: {} < {} test id: {} test name: {} field: {}'.format(error_diff,tolerance,id,id_string,fieldname)) except Exception as message: #something went wrong, print failure message: print format_exc() directory=os.getcwd().split('/') # not used? if output=='nightly': fid=open(os.path.join(ISSM_DIR,'nightlylog','pythonerror.log'), 'a') fid.write('%s' % message) fid.write('\n------------------------------------------------------------------\n') fid.close() print('FAILURE difference: N/A test id: {} test name: {} field: {}'.format(id,id_string,fieldname)) else: print('FAILURE difference: N/A test id: {} test name: {} field: {}'.format(id,id_string,fieldname)) raise RuntimeError(message) except Exception as message: #something went wrong, print failure message: print format_exc() directory=os.getcwd().split('/') # not used? if output=='nightly': fid=open(os.path.join(ISSM_DIR,'nightlylog','pythonerror.log'), 'a') fid.write('%s' % message) fid.write('\n------------------------------------------------------------------\n') fid.close() print('FAILURE difference: N/A test id: {} test name: {} field: {}'.format(id,id_string,'N/A')) else: print('FAILURE difference: N/A test id: {} test name: {} field: {}'.format(id,id_string,'N/A')) raise RuntimeError(message) print "----------------finished:%i-----------------------" % id return
def join_bins(bin_edges, probabilities, min_prob=0.05): """Join bins until at least the minimum probability is contained. By joining adjacent bins, find the configuration with the maximum number of bins that each contain at least a certain probability. Parameters ---------- bin_edges : iterable of 2-tuple Upper and lower bounds associated with each bin. probabilities : array-like Probabilities in each bin. The contiguous ranges where bin joining must be attempted will automatically be determined. min_prob : float The minimum probability (inclusive) per (final) bin. Returns ------- bin_edges : list of 2-tuple List containing the new bin edges. probabilities : array Probabilities corresponding to the above bins. Raises ------ ValueError : If the number of bin edges does not match the number of probabilities. ValueError : If the sum of all `probabilities` does not exceed `min_prob`. """ if len(bin_edges) != len(probabilities): raise ValueError("Length of bin_edges and probabilities must match.") probabilities = np.asarray(probabilities) if np.sum(probabilities) <= min_prob: raise ValueError("Sum of probabilities must exceed min_prob.") max_i = probabilities.shape[0] - 1 def _join(start_i, end_i): """Return new bin edges and probabilities after a join. Parameters ---------- start_i : int Beginning of the join. end_i : int End of the join. Returns ------- bin_edges : list of 2-tuple List containing the new bin edges. probabilities : array Probabilities corresponding to the above bins. """ new_bin_edges = [] new_probabilities = [] # Remove all but the first index within the join window. for i in filterfalse( lambda x: x in range(start_i + 1, end_i + 1), range(max_i + 1), ): if i == start_i: new_bin_edges.append((bin_edges[start_i][0], bin_edges[end_i][1],)) new_probabilities.append(np.sum(probabilities[start_i : end_i + 1])) else: new_bin_edges.append(bin_edges[i]) new_probabilities.append(probabilities[i]) return join_bins( bin_edges=new_bin_edges, probabilities=new_probabilities, min_prob=min_prob, ) # Identify regions with low probabilities. join_mask = probabilities < min_prob if not np.any(join_mask): # Joining is complete. return (bin_edges, probabilities) # Find the contiguous clusters. labelled, n_clusters = label(join_mask) variations = [] # Carry out contiguous bin joining around all clusters. for cluster_i in range(1, n_clusters + 1): cluster_indices = np.where(labelled == cluster_i)[0] cluster_bounds = (cluster_indices[0], cluster_indices[-1]) # Also consider the adjacent bins, since this may be needed # in some cases. join_bounds = tuple( np.clip(np.array([cluster_bounds[0] - 1, cluster_bounds[1] + 1]), 0, max_i) ) for start_i in range(*join_bounds): # Optimisation: prevent 'orphan' bins on the left. if join_bounds[0] == 0 and start_i != 0: continue for end_i in range(start_i + 1, join_bounds[1] + 1): # Optimisation: prevent 'orphan' bins on the right. if join_bounds[1] == max_i and end_i != max_i: continue # If the sum of probabilities between `start_i` and `end_i` # exceeds the minimum threshold, join the bins. if np.sum(probabilities[start_i : end_i + 1]) >= min_prob: variations.append(_join(start_i, end_i)) # Return the 'best' variation - the set of bins with the lowest variability, # measured using the standard deviation of the probabilities. Only sets with # the largest number of bins will be considered here. lengths = [len(variation[0]) for variation in variations] max_length = max(lengths) long_variations = [ variation for variation in variations if len(variation[0]) == max_length ] variation_stds = [np.std(variation[1]) for variation in long_variations] min_index = np.argsort(variation_stds)[0] return long_variations[min_index]
def draw_static(self, axis=plt.gca(), use_data=True, filetypes=['eps', 'pdf', 'png'], save=False): self.axis = axis #The orbital view. Draw points, subsolar longitude line, and labels. P = self.planet.P e = self.planet.e a = (self.planet.a / U.AU).value #If use_data is True, the dots showing the equal time intervals over the orbit will be colored according to where the data in each band exist. Note: this works best when the time sampling of the data is finer than the plotting time resolution. if use_data: coverage = N.zeros(self.time_resolution+1, dtype=bool) partial_phase = True t_set = {} for i, band in enumerate(['3p6','4p5','8p0']): if band in self.data: t_set[band] = N.unique(self.data[band]['t']) data = t_set[band] band_range = N.array([N.any(N.abs(t-data) <= 0.5*P/U.d/self.time_resolution) for t in self.times/U.d]) coverage = coverage | band_range #The "stitch" is that the array for checking coverage has its first element overlapping with the last. So if either are covered, both should be covered. coverage_stitch = coverage[0] | coverage[-1] coverage[0] = coverage_stitch coverage[-1] = coverage_stitch #Boolean mask for whether we consider the photometry a partial, rather than full, phase curve. I chose to call any light curve with a time range < 90% of the orbital period as "partial". partial_phase = partial_phase & ((N.max(t_set[band])-N.min(t_set[band]))*U.d/P < 0.8) orbit_range = axis.scatter(self.x_points[band_range], self.y_points[band_range], color=color_datlab[band], s=(6*(i+1)**2), zorder=(3-i)*5) orbit_gap = axis.scatter(self.x_points[~coverage], self.y_points[~coverage], color='k', s=2, zorder=100) else: orbit_points = axis.scatter(self.x_points, self.y_points, color='k', s=2, zorder=1) peri_angle = ((self.planet.w + 180*U.deg)%(360*U.deg)).to(U.deg) orbit_outline = self.axis.add_artist(Ellipse(xy = (a*e*N.cos(peri_angle+180*U.deg), a*e*N.sin(peri_angle+180*U.deg)), #orbit_outline = self.axis.add_artist(Ellipse(xy = (a*e*N.cos(peri_angle+0*U.deg), a*e*N.sin(peri_angle+0*U.deg)), width = 2*a, height = 2*a*N.sqrt(1-e**2), angle = peri_angle.value, fill = False, edgecolor = 'k', alpha = 0.3, linestyle = (0, (0.5, 1.5)) )) #The star point scales with the radius of the actual star, and is set true to the scale of the plotted orbit semimajor axis. star_scale = float(self.planet.R/U.AU) #The color is calculated from the blackbody color at the effective temperature. star_color = cm.irgb_string_from_xyz(bb.blackbody_color(float(self.planet.Teff/U.K))) limb_darkening_span = N.squeeze(N.array([colors.to_rgba_array(cm.irgb_string_from_xyz(bb.blackbody_color(T))) for T in N.linspace(0.8, 1, num=20)*self.planet.Teff/U.K])) #star_point = axis.scatter(0, 0, color=star_color, s=star_scale, edgecolors='k') for n, shade in enumerate(limb_darkening_span[:,:-1]): self.star_point = axis.add_artist(Ellipse(xy=[0,0], width=(star_scale*2)*(1-n/20), height=(star_scale*2)*(1-n/20), angle=0)) self.star_point.set_facecolor(shade) #Draw the line from star to apastron, and the line to periastron. if e > 0: x_apo = N.cos(self.planet.w) * N.array([star_scale, self.orbit_scale*(1+e)]) y_apo = N.sin(self.planet.w) * N.array([star_scale, self.orbit_scale*(1+e)]) apo_line = axis.plot(x_apo, y_apo, color='#444444', linestyle='-', linewidth=1, dashes=[2,4]) x_peri = N.cos(180*U.deg+self.planet.w) * N.array([star_scale, self.orbit_scale*(1-e)]) y_peri = N.sin(180*U.deg+self.planet.w) * N.array([star_scale, self.orbit_scale*(1-e)]) peri_line = axis.plot(x_peri, y_peri, color='#444444', linestyle='-', linewidth=1, dashes=[1,2]) line_of_nodes = axis.axhline(y=0, linewidth=0.5, color='k', alpha=0.5, linestyle=(0, (0.25, 1.75)), zorder=1) #Draw the band representing the geometric transit and eclipse window. occultation_band = axis.axvspan(xmin=-star_scale, xmax=star_scale, facecolor='0.2', alpha=0.25) #Draw some motion arrows following a slightly larger elliptical arc around the orbit points. offset_factor = 0.1 motion_arcs = [ Arc(xy = (a*e*N.cos(peri_angle+180*U.deg), a*e*N.sin(peri_angle+180*U.deg)), width = (1+offset_factor)*(2*a), height = (1+offset_factor)*(2*a*N.sqrt(1-e**2)), angle = peri_angle.value, theta1 = i, theta2 = j ) for (i,j) in [(-20,-10), (160, 170)] ] motion_paths = [(motion_arc.get_transform()).transform_path(motion_arc.get_path()) for motion_arc in motion_arcs] motion_lines = [axis.add_artist(FancyArrowPatch(path=motion_path, arrowstyle='->', mutation_scale=10, color='#444444')) for motion_path in motion_paths] #Remove axis labels and ticks. plt.setp(plt.getp(axis.axes, 'xticklabels'), visible=False) plt.setp(plt.getp(axis.axes, 'yticklabels'), visible=False) axis.set_aspect('equal') #Tight boundaries for the figure, using the span of the orbit. if use_data: if partial_phase: orbit_spanx = [N.min(self.x_points[coverage]) - 0.1*(N.max(self.x_points[coverage])-N.min(self.x_points[coverage])), N.max(self.x_points[coverage]) + 0.1*(N.max(self.x_points[coverage])-N.min(self.x_points[coverage]))] orbit_spany = [N.min(self.y_points[coverage]) - 0.1*(N.max(self.y_points[coverage])-N.min(self.y_points[coverage])), N.max(self.y_points[coverage]) + 0.1*(N.max(self.y_points[coverage])-N.min(self.y_points[coverage]))] axis.set_xlim(orbit_spanx) axis.set_ylim(orbit_spany) else: orbit_spanx = [N.min(self.x_points) - 0.1*(N.max(self.x_points)-N.min(self.x_points)), N.max(self.x_points) + 0.1*(N.max(self.x_points)-N.min(self.x_points))] orbit_spany = [N.min(self.y_points) - 0.1*(N.max(self.y_points)-N.min(self.y_points)), N.max(self.y_points) + 0.1*(N.max(self.y_points)-N.min(self.y_points))] axis.set_xlim(orbit_spanx) axis.set_ylim(orbit_spany) #The observer line points from the star to the bottom of the figure, which represents the direction to the observer. observer_linelength = self.orbit_scale * (1-e**2) / (1 + e*N.cos(self.planet.w+90*U.deg)) xmin, xmax = axis.get_xlim() ymin, ymax = axis.get_ylim() observer_line = axis.arrow(xmin+0.05*(xmax-xmin), ymin+0.16*(ymax-ymin), 0, -0.05*(ymax-ymin), color='#444444', head_width=0.015*(ymax-ymin), linewidth=2, width=0.001*(ymax-ymin)) earth_label = axis.text(xmin+0.05*(xmax-xmin), ymin+0.08*(ymax-ymin),'$\mathbf{\oplus}$', horizontalalignment='center', verticalalignment='top', fontproperties=fontprops) if e > 0: PSR_x = self.x_points[1:-1] PSR_y = self.y_points[1:-1] PSR_lines = [axis.add_artist(FancyArrow(x, y, 0.03*(ymax-ymin)*N.cos(PSR_line), 0.03*(ymax-ymin)*N.sin(PSR_line), color='black', lw=0.25, head_length=0.001, shape='right')) for (x, y, PSR_line) in zip(PSR_x, PSR_y, self.PSR_angle[1:-1])] subsolar_lines = [axis.plot([x, x+0.045*(ymax-ymin)*N.cos(subsolar_line)], [y, y+0.045*(ymax-ymin)*N.sin(subsolar_line)], color='#4E89B6', lw=1) for (x, y, subsolar_line) in zip(self.x_points[:-1], self.y_points[:-1], self.subsolar_angle[:-1])] plt.tick_params( axis='x', # changes apply to the x-axis which='both', # both major and minor ticks are affected bottom='off', # ticks along the bottom edge are off top='off', # ticks along the top edge are off labelbottom='off') # labels along the bottom edge are off plt.tick_params( axis='y', # changes apply to the x-axis which='both', # both major and minor ticks are affected left='off', # ticks along the bottom edge are off right='off', # ticks along the top edge are off labelleft='off') # labels along the bottom edge are off #Instead of explicit axes we opt for a scale bar, here chosen to be 0.01 AU. from mpl_toolkits.axes_grid1.anchored_artists import AnchoredSizeBar scalebar = AnchoredSizeBar(axis.transData, 0.01, r"0.01 AU", loc=1, pad=0.0, borderpad=0.5, sep=5, frameon=False, fontproperties=fontprops) axis.add_artist(scalebar) #If we're plotting the whole orbit, the bounding box for the text showing the orbital period should be able to find room in the lower-right corner. Otherwise, there is a chance something could intersect it, and so we draw a white bounding box around it. if partial_phase: textbox_settings = dict(pad=0, fc='white', ec='white') else: textbox_settings = dict(pad=0, alpha=0) axis.text(0.975, 0.05,'$P = $ {0:.2f}'.format(self.planet.P), horizontalalignment='right', verticalalignment='center', transform=axis.transAxes, fontproperties=fontprops, bbox = textbox_settings) if save: for filetype in filetypes: plt.savefig('{0}_{1}_orbit.{2}'.format(str(datetime.date.today()).replace(" ", ""), (self.planet.name).replace(" ", ""), filetype), bbox_inches='tight', transparent=True)
def add_recommendations(self, is_relevant, user_id): self.users_mask[user_id] = np.any(is_relevant)
def __preprocess_individual(file, outdir, overwrite): """ Internal function for preprocessing raw MEG files :param file: input file along with path :param outdir: where we want to save this :return: save_file_path: a path to the saved and filtered file """ save_file_path = "" no_ecg_removed = False no_eog_removed = False num = os.path.basename(file).split('_')[0] if os.path.basename(file).split('raw')[1] != '.fif': append = os.path.basename(file).split('raw')[1].split('.fif')[0] else: append = '' append = 'raw' + append + '.fif' base = os.path.basename(file).split('.')[0] # check if any of these files exists, if not overwrite then skip and return path # could be any of these names check_fnames = [ f'{outdir}/{num}_noeog_noecg_clean_{append}', f'{outdir}/{num}_noecg_clean_{append}', f'{outdir}/{num}_noeog_clean_{append}', f'{outdir}/{num}_clean_{append}' ] if np.any([os.path.isfile(f) for f in check_fnames]): index = np.where([os.path.isfile(f) for f in check_fnames])[0] if not overwrite: print( f'file for {os.path.basename(file)} already exists, skipping to next' ) save_file_path = check_fnames[index[0]] return save_file_path # read file try: raw = mne.io.read_raw_fif(file, preload=True) except OSError: print('could not read ' + file) return '' raw.filter(0.1, None) raw.notch_filter(np.arange(50, 241, 50), filter_length='auto', phase='zero') # Run ICA on raw data to find blinks and eog try: ica = mne.preprocessing.ICA(n_components=25, method='picard').fit(raw) except: raw.crop(1) # remove the first second due to NaNs ica = mne.preprocessing.ICA(n_components=25, method='picard').fit(raw) try: # look for and remove EOG eog_epochs = mne.preprocessing.create_eog_epochs( raw) # get epochs of eog (if this exists) eog_inds, eog_scores = ica.find_bads_eog( eog_epochs) # try and find correlated components # define flags for tracking if we found components matching or not no_ecg_removed = False no_eog_removed = False # if we have identified something ! if len(eog_inds) > 0: ica.exclude.extend(eog_inds) else: print( f'{os.path.basename(file)} cannot detect eog automatically manual ICA must be done' ) no_eog_removed = True except: print( f'{os.path.basename(file)} cannot detect eog automatically manual ICA must be done' ) no_eog_removed = True # now we do this with hearbeat try: ecg_epochs = mne.preprocessing.create_ecg_epochs( raw) # get epochs of eog (if this exists) ecg_inds, ecg_scores = ica.find_bads_ecg( ecg_epochs) # try and find correlated components # if one component reaches above threshold then remove components automagically if len(ecg_inds) > 0: ica.exclude.extend(ecg_inds) # exclude top 3 components else: # flag for manual ICA inspection and removal print( f'{os.path.basename(file)} cannot detect ecg automatically manual ICA must be done' ) no_ecg_removed = True except: print( f'{os.path.basename(file)} cannot detect ecg automatically manual ICA must be done' ) no_ecg_removed = True # exclude all labelled components raw = ica.apply(inst=raw) # save the file if no_ecg_removed and no_eog_removed: outfname = f'{outdir}/{num}_noeog_noecg_clean_{append}' elif no_ecg_removed: outfname = f'{outdir}/{num}_noecg_clean_{append}' elif no_eog_removed: outfname = f'{outdir}/{num}_noeog_clean_{append}' else: outfname = f'{outdir}/{num}_clean_{append}' raw = raw.resample(250) raw.save(outfname, overwrite=overwrite) save_file_path = outfname # return return save_file_path
def mass_conservation_inversion(gdir, glen_a=None, fs=None, write=True, filesuffix=''): """ Compute the glacier thickness along the flowlines More or less following Farinotti et al., (2009). Parameters ---------- gdir : :py:class:`oggm.GlacierDirectory` the glacier directory to process glen_a : float glen's creep parameter A fs : float sliding parameter write: bool default behavior is to compute the thickness and write the results in the pickle. Set to False in order to spare time during calibration. filesuffix : str add a suffix to the output file """ # Defaults if glen_a is None: glen_a = cfg.PARAMS['inversion_glen_a'] if fs is None: fs = cfg.PARAMS['inversion_fs'] # Check input if fs == 0.: _inv_function = _inversion_simple else: _inv_function = _inversion_poly # Ice flow params fd = 2. / (cfg.PARAMS['glen_n'] + 2) * glen_a a3 = fs / fd rho = cfg.PARAMS['ice_density'] # Shape factor params sf_func = None # Use .get to obatin default None for non-existing key # necessary to pass some tests # TODO: remove after tests are adapted use_sf = cfg.PARAMS.get('use_shape_factor_for_inversion') if use_sf == 'Adhikari' or use_sf == 'Nye': sf_func = utils.shape_factor_adhikari elif use_sf == 'Huss': sf_func = utils.shape_factor_huss sf_tol = 1e-2 # TODO: better as params in cfg? max_sf_iter = 20 # Clip the slope, in degrees clip_angle = cfg.PARAMS['min_slope'] out_volume = 0. cls = gdir.read_pickle('inversion_input') for cl in cls: # Clip slope to avoid negative and small slopes slope = cl['slope_angle'] slope = np.clip(slope, np.deg2rad(clip_angle), np.pi / 2.) # Parabolic bed rock w = cl['width'] a0s = -cl['flux_a0'] / ((rho * cfg.G * slope)**3 * fd) sf = np.ones(slope.shape) # Default shape factor is 1 # TODO: maybe take height update as criterion for iteration end instead # of sf_diff? if sf_func is not None: # Start iteration for shape factor with guess of 1 i = 0 sf_diff = np.ones(slope.shape) while i < max_sf_iter and np.any(sf_diff > sf_tol): out_thick = _compute_thick(gdir, a0s, a3, cl['flux_a0'], sf, _inv_function) sf_diff[:] = sf[:] sf = sf_func(w, out_thick, cl['is_rectangular']) sf_diff = sf_diff - sf i += 1 # TODO: Iteration at the moment for all grid points, # even if some already converged. Change? log.info('Shape factor {:s} used, took {:d} iterations for ' 'convergence.'.format(use_sf, i)) out_thick = _compute_thick(gdir, a0s, a3, cl['flux_a0'], sf, _inv_function) # volume fac = np.where(cl['is_rectangular'], 1, 2. / 3.) volume = fac * out_thick * w * cl['dx'] if write: cl['thick'] = out_thick cl['volume'] = volume out_volume += np.sum(volume) if write: gdir.write_pickle(cls, 'inversion_output', filesuffix=filesuffix) return out_volume, gdir.rgi_area_km2 * 1e6
def generateActivityDiagram(config, shower_data, ax_handle=None, sol_marker=None): """ Generates a plot of shower activity across all solar longitudes. """ shower_data = np.array(shower_data) # Fill in min/max solar longitudes if they are not present for shower in shower_data: sol_min, sol_peak, sol_max = list(shower)[3:6] if np.isnan(sol_min): shower[3] = (sol_peak - config.shower_lasun_threshold) % 360 if np.isnan(sol_max): shower[5] = (sol_peak + config.shower_lasun_threshold) % 360 # Sort showers by duration durations = [(shower[5] - shower[3] + 180) % 360 - 180 for shower in shower_data] shower_data = shower_data[np.argsort(durations)][::-1] # Generate an array of shower activity per 1 deg of solar longitude code_name_dict = {} activity_stack = np.zeros((20, 360), dtype=np.uint16) colors = cm.tab10(np.linspace(0, 1.0, 10)) shower_index = 0 for i in range(20): for sol_plot in range(0, 360): # If the cell is unassigned, check to see if a shower is active if activity_stack[i, sol_plot] > 0: continue for shower in shower_data: code = int(shower[0]) name = shower[1] # Skip assigned showers if code in code_name_dict: continue sol_min, sol_peak, sol_max = list(shower)[3:6] sol_min = int(np.floor(sol_min)) % 360 sol_max = int(np.ceil(sol_max)) % 360 # If the shower is active at the given solar longitude and there aren't any other showers # in the same activity period, assign shower code to this solar longitude shower_active = False if (sol_max - sol_min) < 180: # Check if the shower is active if (sol_plot >= sol_min) and (sol_plot <= sol_max): # Leave a buffer of +/- 3 deg around the shower sol_min_check = sol_min - 3 if sol_min_check < 0: sol_min_check = 0 sol_max_check = sol_max + 3 if sol_max_check > 360: sol_max_check = 360 # Check if the solar longitue range is free of other showers if not np.any( activity_stack[i, sol_min_check:sol_max_check]): # Assign the shower code to activity stack activity_stack[i, sol_min:sol_max] = code code_name_dict[code] = [name, sol_peak] shower_active = True else: if (sol_plot >= sol_min) or (sol_plot <= sol_max): # Check if the solar longitue range is free of other showers if (not np.any(activity_stack[i, 0:sol_max])) and \ (not np.any(activity_stack[i, sol_min:])): # Assign shower code to activity stack activity_stack[i, 0:sol_max] = code activity_stack[i, sol_min:] = code shower_active = True if shower_active: # Get shower color color = colors[shower_index % 10] shower_index += 1 # Assign shower params code_name_dict[code] = [ name, sol_min, sol_peak, sol_max, color ] # If no axis was given, crate one if ax_handle is None: ax_handle = plt.subplot(111, facecolor='black') # Set background color plt.gcf().patch.set_facecolor('black') # Change axis color ax_handle.spines['bottom'].set_color('w') ax_handle.spines['top'].set_color('w') ax_handle.spines['right'].set_color('w') ax_handle.spines['left'].set_color('w') # Change tick color ax_handle.tick_params(axis='x', colors='w') ax_handle.tick_params(axis='y', colors='w') # Change axis label color ax_handle.yaxis.label.set_color('w') ax_handle.xaxis.label.set_color('w') # Plot the activity graph active_shower = 0 vertical_scale_line = 0.5 vertical_shift_text = 0.01 text_size = 8 for i, line in enumerate(activity_stack): for shower_block in line: # If a new shower was found, plot it if (shower_block != active_shower) and (shower_block > 0): # Get shower parameters name, sol_min, sol_peak, sol_max, color = code_name_dict[ shower_block] # Plot the shower activity period if (sol_max - sol_min) < 180: x_arr = np.arange(sol_min, sol_max + 1, 1) ax_handle.plot(x_arr, np.zeros_like(x_arr) + i*vertical_scale_line, linewidth=3, \ color=color, zorder=3) ax_handle.text(round((sol_max + sol_min)/2), i*vertical_scale_line + vertical_shift_text, \ name, ha='center', va='bottom', color='w', size=text_size, zorder=3) else: x_arr = np.arange(0, sol_max + 1, 1) ax_handle.plot(x_arr, np.zeros_like(x_arr) + i*vertical_scale_line, linewidth=3, \ color=color, zorder=3) x_arr = np.arange(sol_min, 361, 1) ax_handle.plot(x_arr, np.zeros_like(x_arr) + i*vertical_scale_line, linewidth=3, \ color=color, zorder=3) ax_handle.text(0, i*vertical_scale_line + vertical_shift_text, name, ha='center', \ va='bottom', color='w', size=text_size, zorder=2) # Plot peak location ax_handle.scatter(sol_peak, i * vertical_scale_line, marker='+', c="w", zorder=4) active_shower = shower_block # Hide y axis ax_handle.get_yaxis().set_visible(False) ax_handle.set_xlabel('Solar longitude (deg)') # Get the plot Y limits y_min, y_max = ax_handle.get_ylim() # Plot a line with given solver longitude if sol_marker is not None: # Plot the solar longitude line behind everything else y_arr = np.linspace(y_min, y_max, 5) if not isinstance(sol_marker, list): sol_marker = [sol_marker] for sol_value in sol_marker: ax_handle.plot(np.zeros_like(y_arr) + sol_value, y_arr, color='r', linestyle='dashed', zorder=2, \ linewidth=1) # Plot month names at the 1st of that month (start in April of this year) for month_no, year_modifier in [[4, 0], [5, 0], [6, 0], [7, 0], [8, 0], [9, 0], [10, 0], [11, 0], [12, 0], [1, 1], [2, 1], [3, 1]]: # Get the solar longitude of the 15th date of the month curr_year = datetime.datetime.now().year dt = datetime.datetime(curr_year + year_modifier, month_no, 15, 0, 0, 0) sol = np.degrees(jd2SolLonSteyaert(datetime2JD(dt))) % 360 # Plot the month name in the background of the plot plt.text(sol, y_max, dt.strftime("%b").upper(), alpha=0.3, rotation=90, size=20, \ zorder=1, color='w', va='top', ha='center') # Get the solar longitude of the 1st date of the month curr_year = datetime.datetime.now().year dt = datetime.datetime(curr_year + year_modifier, month_no, 1, 0, 0, 0) sol = np.degrees(jd2SolLonSteyaert(datetime2JD(dt))) % 360 # Plot the manth beginning line y_arr = np.linspace(y_min, y_max, 5) plt.plot(np.zeros_like(y_arr) + sol, y_arr, linestyle='dotted', alpha=0.3, zorder=3, color='w') # Force Y limits ax_handle.set_ylim([y_min, y_max]) ax_handle.set_xlim([0, 360])
def add_output(self, name, val=1.0, shape=None, units=None, res_units=None, desc='', lower=None, upper=None, ref=1.0, ref0=0.0, res_ref=1.0, var_set=0): """ Add an output variable to the component. Parameters ---------- name : str name of the variable in this component's namespace. val : float or list or tuple or ndarray The initial value of the variable being added in user-defined units. Default is 1.0. shape : int or tuple or list or None Shape of this variable, only required if val is not an array. Default is None. units : str or None Units in which the output variables will be provided to the component during execution. Default is None, which means it has no units. res_units : str or None Units in which the residuals of this output will be given to the user when requested. Default is None, which means it has no units. desc : str description of the variable. lower : float or list or tuple or ndarray or Iterable or None lower bound(s) in user-defined units. It can be (1) a float, (2) an array_like consistent with the shape arg (if given), or (3) an array_like matching the shape of val, if val is array_like. A value of None means this output has no lower bound. Default is None. upper : float or list or tuple or ndarray or or Iterable None upper bound(s) in user-defined units. It can be (1) a float, (2) an array_like consistent with the shape arg (if given), or (3) an array_like matching the shape of val, if val is array_like. A value of None means this output has no upper bound. Default is None. ref : float or ndarray Scaling parameter. The value in the user-defined units of this output variable when the scaled value is 1. Default is 1. ref0 : float or ndarray Scaling parameter. The value in the user-defined units of this output variable when the scaled value is 0. Default is 0. res_ref : float or ndarray Scaling parameter. The value in the user-defined res_units of this output's residual when the scaled value is 1. Default is 1. var_set : hashable object For advanced users only. ID or color for this variable, relevant for reconfigurability. Default is 0. Returns ------- dict metadata for added variable """ if units == 'unitless': warn_deprecation( "Output '%s' has units='unitless' but 'unitless' " "has been deprecated. Use " "units=None instead. Note that connecting a " "unitless variable to one with units is no longer " "an error, but will issue a warning instead." % name) units = None # First, type check all arguments if not isinstance(name, str): raise TypeError('The name argument should be a string') if not np.isscalar(val) and not isinstance( val, (list, tuple, np.ndarray, Iterable)): raise TypeError( 'The val argument should be a float, list, tuple, or ndarray') if not np.isscalar(ref) and not isinstance( val, (list, tuple, np.ndarray, Iterable)): raise TypeError( 'The ref argument should be a float, list, tuple, or ndarray') if not np.isscalar(ref0) and not isinstance( val, (list, tuple, np.ndarray, Iterable)): raise TypeError( 'The ref0 argument should be a float, list, tuple, or ndarray') if not np.isscalar(res_ref) and not isinstance( val, (list, tuple, np.ndarray, Iterable)): raise TypeError( 'The res_ref argument should be a float, list, tuple, or ndarray' ) if shape is not None and not isinstance( shape, (int, tuple, list, np.integer)): raise TypeError( "The shape argument should be an int, tuple, or list but " "a '%s' was given" % type(shape)) if units is not None and not isinstance(units, str): raise TypeError('The units argument should be a str or None') if res_units is not None and not isinstance(res_units, str): raise TypeError('The res_units argument should be a str or None') # Check that units are valid if units is not None and not valid_units(units): raise ValueError("The units '%s' are invalid" % units) metadata = {} # value, shape: based on args, making sure they are compatible metadata['value'], metadata['shape'] = ensure_compatible( name, val, shape) metadata['size'] = np.prod(metadata['shape']) # units, res_units: taken as is metadata['units'] = units metadata['res_units'] = res_units # desc: taken as is metadata['desc'] = desc if lower is not None: lower = ensure_compatible(name, lower, metadata['shape'])[0] if upper is not None: upper = ensure_compatible(name, upper, metadata['shape'])[0] metadata['lower'] = lower metadata['upper'] = upper # All refs: check the shape if necessary for item, item_name in zip([ref, ref0, res_ref], ['ref', 'ref0', 'res_ref']): if not np.isscalar(item): if np.atleast_1d(item).shape != metadata['shape']: raise ValueError('The %s argument has the wrong shape' % item_name) if np.isscalar(ref): self._has_output_scaling |= ref != 1.0 else: self._has_output_scaling |= np.any(ref != 1.0) if np.isscalar(ref0): self._has_output_scaling |= ref0 != 0.0 else: self._has_output_scaling |= np.any(ref0) if np.isscalar(res_ref): self._has_resid_scaling |= res_ref != 1.0 else: self._has_resid_scaling |= np.any(res_ref != 1.0) ref = format_as_float_or_array('ref', ref, flatten=True) ref0 = format_as_float_or_array('ref0', ref0, flatten=True) res_ref = format_as_float_or_array('res_ref', res_ref, flatten=True) # ref, ref0, res_ref: taken as is metadata['ref'] = ref metadata['ref0'] = ref0 metadata['res_ref'] = res_ref # var_set: taken as is metadata['var_set'] = var_set metadata['distributed'] = self.distributed # We may not know the pathname yet, so we have to use name for now, instead of abs_name. if self._static_mode: var_rel2data_io = self._static_var_rel2data_io var_rel_names = self._static_var_rel_names else: var_rel2data_io = self._var_rel2data_io var_rel_names = self._var_rel_names # Disallow dupes if name in var_rel2data_io: msg = "Variable name '{}' already exists.".format(name) raise ValueError(msg) var_rel2data_io[name] = { 'prom': name, 'rel': name, 'my_idx': len(self._var_rel_names['output']), 'type': 'output', 'metadata': metadata } var_rel_names['output'].append(name) return metadata
def prepare_for_inversion(gdir, add_debug_var=False, invert_with_rectangular=True, invert_all_rectangular=False): """Prepares the data needed for the inversion. Mostly the mass flux and slope angle, the rest (width, height) was already computed. It is then stored in a list of dicts in order to be faster. Parameters ---------- gdir : :py:class:`oggm.GlacierDirectory` the glacier directory to process """ # variables fls = gdir.read_pickle('inversion_flowlines') towrite = [] for fl in fls: # Distance between two points dx = fl.dx * gdir.grid.dx # Widths widths = fl.widths * gdir.grid.dx # Heights hgt = fl.surface_h angle = -np.gradient(hgt, dx) # beware the minus sign # Flux needs to be in [m3 s-1] (*ice* velocity * surface) # fl.flux is given in kg m-2 yr-1, rho in kg m-3, so this should be it: rho = cfg.PARAMS['ice_density'] flux = fl.flux * (gdir.grid.dx**2) / cfg.SEC_IN_YEAR / rho # Clip flux to 0 if np.any(flux < -0.1): log.warning('(%s) has negative flux somewhere', gdir.rgi_id) flux = flux.clip(0) if fl.flows_to is None and gdir.inversion_calving_rate == 0: if not np.allclose(flux[-1], 0., atol=0.1): # TODO: this test doesn't seem meaningful here msg = ('({}) flux at terminus should be zero, but is: ' '{.4f} m3 ice s-1'.format(gdir.rgi_id, flux[-1])) raise RuntimeError(msg) flux[-1] = 0. # Shape is_rectangular = fl.is_rectangular if not invert_with_rectangular: is_rectangular[:] = False if invert_all_rectangular: is_rectangular[:] = True # Optimisation: we need to compute this term of a0 only once flux_a0 = np.where(is_rectangular, 1, 1.5) flux_a0 *= flux / widths # Add to output cl_dic = dict(dx=dx, flux_a0=flux_a0, width=widths, slope_angle=angle, is_rectangular=is_rectangular, is_last=fl.flows_to is None) if add_debug_var: cl_dic['flux'] = flux cl_dic['hgt'] = hgt towrite.append(cl_dic) # Write out gdir.write_pickle(towrite, 'inversion_input')
def signaltools_detrend(data, axis=-1, type='linear', bp=0): """ Remove linear trend along axis from data. Parameters ---------- data : array_like The input data. axis : int, optional The axis along which to detrend the data. By default this is the last axis (-1). type : {'linear', 'constant'}, optional The type of detrending. If ``type == 'linear'`` (default), the result of a linear least-squares fit to `data` is subtracted from `data`. If ``type == 'constant'``, only the mean of `data` is subtracted. bp : array_like of ints, optional A sequence of break points. If given, an individual linear fit is performed for each part of `data` between two break points. Break points are specified as indices into `data`. Returns ------- ret : ndarray The detrended input data. """ if type not in ['linear', 'l', 'constant', 'c']: raise ValueError("Trend type must be 'linear' or 'constant'.") data = np.asarray(data) dtype = data.dtype.char if dtype not in 'dfDF': dtype = 'd' if type in ['constant', 'c']: #print('Removing mean') ret = data - np.expand_dims(np.mean(data, axis), axis) return ret else: #print('Removing linear?') dshape = data.shape N = dshape[axis] bp = sort(unique(r_[0, bp, N])) if np.any(bp > N): raise ValueError("Breakpoints must be less than length " "of data along given axis.") Nreg = len(bp) - 1 # Restructure data so that axis is along first dimension and # all other dimensions are collapsed into second dimension rnk = len(dshape) if axis < 0: axis = axis + rnk newdims = r_[axis, 0:axis, axis + 1:rnk] newdata = reshape(np.transpose(data, tuple(newdims)), (N, _prod(dshape) // N)) newdata = newdata.copy() # make sure we have a copy if newdata.dtype.char not in 'dfDF': newdata = newdata.astype(dtype) # Find leastsq fit and remove it for each piece for m in range(Nreg): Npts = bp[m + 1] - bp[m] A = ones((Npts, 2), dtype) A[:, 0] = cast[dtype](np.arange(1, Npts + 1) * 1.0 / Npts) sl = slice(bp[m], bp[m + 1]) coef, resids, rank, s = np.linalg.lstsq(A, newdata[sl]) newdata[sl] = newdata[sl] - dot(A, coef) # Put data back in original shape. tdshape = take(dshape, newdims, 0) ret = np.reshape(newdata, tuple(tdshape)) vals = list(range(1, rnk)) olddims = vals[:axis] + [0] + vals[axis:] ret = np.transpose(ret, tuple(olddims)) return ret
def partition(a): """ given a set, determine if you can partition into two sets s.t. the sum is the same ok so if you sum these numbers and they are odd then it's impossible so first you compute the sum and check that so now you have an even sum what do you do? can you view this as a graph? yeah of course but what's the graph dag? not sure so for each number you can either put it in set 1 or set 2 what's the subproblem? at each point in the list know whether you can separte up tot hat point evenly? seems like it takes the same form as the longest nondecreasing but why? find a subset equal to sum/2 b/c the other subset must therefore equal sum/2 as well ok h = sum(s) / 2 for each number decide whether to add it to the set or not ok 2d array size sum/2 x n+1 where the rows correspond to sums where the cols correspond to whether or not a subset starting at this location and moving backward can sum up to the value how would you fill this? [2,1,1] originally [0,0,0] [0,0,0] [0,1,1] [0,0,0] [0,0,0] [0,0,0] how to fill (2,1)? 0 based indexing - not sure for each row you go through the entire sequence I think it works like this for each possible sum value you see if you could add up the elements to that sum the way you do this efficiently is you if the current sum value (call that i) minus the value you're going to add (v[j-1]) could add up to whatever sum results then it should work so looking at that example [2,1,1] 0 [1,1,1,1] # all ones here indicate that of course you can add to 0 1 [0,0,0,0] 2 [0,0,0,0] deciding the first value value is 2 is 2 less than the current sum? no then no 0 [1,1,1,1] 1 [0,0,0,0] 2 [0,0,0,0] next look at 1 first of all, if you could have added to the current sum by the prior column then you can still add to that value by not considering this value so initialize it to the value in the column before which here is 0 the other condition whereby this cell might be true is that where the current sum minus the current value is achievable without using the current value current sum = 1 current value = 1 table[0,] 0 [1,1,1,1] 1 [0,0,0,0] 2 [0,0,0,0] ok so how does this work? it relies on the fact that the sum is integer only and you basically find out whether you can sum to any value that is less than half of the total it does this by creating a table where the rows are the possible sums i.e., all values < sum/2 and the rows correspond to the elements plus a row without anything so example [2,1,1] sum/2 = 2 table {} {2} {2,1} {2,11} # so the cols reflect the subsets up to that point # and whether that subset can add to that sum 0 [1,1,1,1] # b/c anything can add to 0 if you don't take it 1 [0,0,0,0] 2 [0,0,0,0] # initialize each new cell to the cell left of it # because if the previous set could add to the value # then the new set can as well by not using the new value # to fill in (1, {2,1}) # look at the row minus the element (1 - 1) = 0 // this row # then to know whether you could sum to this value at this column # you ask without this column could you sum to this sum minus this column? the answer to this question is in part[i - arr[j-1]][j-1] this indexing is tricky j-1 indexes into the array where the jth element is whereas it indexes into the container where the sum without this value is """ a, s = np.array(a), np.sum(a) if s % 2 == 1: return False, [] s, n = int(s/2), len(a) t = np.zeros((s+1, n+1)) t[0,:] = 1 for i in range(1, s+1): # j indexes into t for the j-1 element of a for j in range(1, n+1): # if you can sum to the value without this col, then you can with it t[i,j] = t[i,j-1] if s >= a[j-1]: # again if t[i,j-1] true then stay true # become true if the current sum minus the current col value # can be produced using value up to but not including the current # value t[i,j] = t[i,j] or t[i-a[j-1]][j-1] print(t) # how do you get the partitioning? # move left until final true value # subtract that value # repeat # gives the indices of the values contributing to the sum i, j, idxs = s, n, [] while True: if i == 0: break while True: if t[i,j-1]: j -=1 else: break # a[j-1] is part of group idxs.append(j-1) i -= a[j-1] # if you want to know whether you can do the partition, just return # whether anything in the last row is true return np.any(t[-1,:]), idxs
def _inBoard(self, pos): return not np.any([pos < [0, 0], pos >= self._map.shape])
[-LIGHT_X_INSIDE, -0.2, BACK_Z], # 26 [-LIGHT_X_INSIDE + 0.1, -0.3, BACK_Z], # 27 [-X_OUTSIDE + 0.1, BOTTOM_LINE, BACK_Z]] + \ [[np.nan, np.nan, np.nan]] * 30 + \ [[-P_X, P_Y_TOP, FRONT_Z]] + \ [[np.nan, np.nan, np.nan]] + \ [[-P_X, P_Y_BOTTOM, FRONT_Z], # 61 [-P_X, P_Y_TOP, BACK_Z]] + \ [[np.nan, np.nan, np.nan]] * 2 + \ [[-P_X, P_Y_BOTTOM, BACK_Z]]) # 65 CAR_POSE_66 = CAR_POSE_HALF for key in HFLIP_ids: CAR_POSE_66[HFLIP_ids[key], :] = CAR_POSE_HALF[key, :] CAR_POSE_66[HFLIP_ids[key], 0] = -CAR_POSE_HALF[key, 0] assert not np.any(CAR_POSE_66 == np.nan) training_weights_local_centrality = [ 0.890968488270775, 0.716506138617812, 1.05674590410869, 0.764774195768455, 0.637682585483328, 0.686680807728366, 0.955422595797394, 0.936714585642375, 1.34823795445326, 1.38308992581967, 1.32689945125819, 1.38838655605483, 1.18980184904613, 1.02584355494795, 0.90969156732068, 1.24732068576104, 1.11338768064342, 0.933815217550391, 0.852297518872114, 1.04167641424727, 1.01668968075247, 1.34625964088011, 0.911796331039028, 0.866206536337413, 1.55957820407853, 0.730844382675724, 0.651138644197359, 0.758018559633786, 1.31842501396691, 1.32186116654782, 0.744347016851606, 0.636390683664723, 0.715244950821949, 1.63122349407032, 0.849835699185461, 0.910488007220499, 1.44244151650561, 1.14150437331681, 1.19808610191343, 0.960186788642886, 1.05023623286937, 1.19761709710598, 1.3872216313401, 1.01256700741214, 1.1167909667759, 1.27893496336199, 1.54475684725655, 1.40343733870633, 1.45552060866114, 1.47264222155031, 0.970060423999993, 0.944450314768933,
def __contains__(self, key): try: res = self.get_loc(key) return np.isscalar(res) or type(res) == slice or np.any(res) except (KeyError, TypeError, ValueError): return False
def _eq(self, pos1, pos2): return not np.any(pos1 != pos2)
def calc_slope_vars(rn_sect, gain_sect, gdq_sect, group_time, max_seg): """ Calculate the segment-specific variance arrays for the given integration. Parameters ---------- rn_sect : ndarray read noise values for all pixels in data section, 2-D float gain_sect : ndarray gain values for all pixels in data section, 2-D float gdq_sect : ndarray data quality flags for pixels in section, 3-D int group_time : float Time increment between groups, in seconds. max_seg : int maximum number of segments fit Returns ------- den_r3 : ndarray for a given integration, the reciprocal of the denominator of the segment-specific variance of the segment's slope due to read noise, 3-D float den_p3 : ndarray for a given integration, the reciprocal of the denominator of the segment-specific variance of the segment's slope due to Poisson noise, 3-D float num_r3 : ndarray numerator of the segment-specific variance of the segment's slope due to read noise, 3-D float segs_beg_3 : ndarray lengths of segments for all pixels in the given data section and integration, 3-D int """ (nreads, asize2, asize1) = gdq_sect.shape npix = asize1 * asize2 imshape = (asize2, asize1) # Create integration-specific sections of input arrays for determination # of the variances. gdq_2d = gdq_sect[:, :, :].reshape((nreads, npix)) gain_1d = gain_sect.reshape(npix) gdq_2d_nan = gdq_2d.copy() # group dq with SATS will be replaced by nans gdq_2d_nan = gdq_2d_nan.astype(np.float32) wh_sat = np.where(np.bitwise_and(gdq_2d, constants.dqflags["SATURATED"])) if len(wh_sat[0]) > 0: gdq_2d_nan[wh_sat] = np.nan # set all SAT groups to nan del wh_sat # Get lengths of semiramps for all pix [number_of_semiramps, number_of_pix] segs = np.zeros_like(gdq_2d) # Counter of semiramp for each pixel sr_index = np.zeros(npix, dtype=np.uint8) pix_not_done = np.ones(npix, dtype=bool) # initialize to True i_read = 0 # Loop over reads for all pixels to get segments (segments per pixel) while (i_read < nreads and np.any(pix_not_done)): gdq_1d = gdq_2d_nan[i_read, :] wh_good = np.where(gdq_1d == 0) # good groups # if this group is good, increment those pixels' segments' lengths if len(wh_good[0]) > 0: segs[sr_index[wh_good], wh_good] += 1 del wh_good # Locate any CRs that appear before the first SAT group... wh_cr = np.where(gdq_2d_nan[i_read, :].astype(np.int32) & constants.dqflags["JUMP_DET"] > 0) # ... but not on final read: if (len(wh_cr[0]) > 0 and (i_read < nreads - 1)): sr_index[wh_cr[0]] += 1 segs[sr_index[wh_cr], wh_cr] += 1 del wh_cr # If current group is a NaN, this pixel is done (pix_not_done is False) wh_nan = np.where(np.isnan(gdq_2d_nan[i_read, :])) if len(wh_nan[0]) > 0: pix_not_done[wh_nan[0]] = False del wh_nan i_read += 1 segs = segs.astype(np.uint8) segs_beg = segs[:max_seg, :] # the leading nonzero lengths # Create reshaped version [ segs, y, x ] to simplify computation segs_beg_3 = segs_beg.reshape(max_seg, imshape[0], imshape[1]) segs_beg_3 = remove_bad_singles(segs_beg_3) # Create a version 1 less for later calculations for the variance due to # Poisson, with a floor=1 to handle single-group segments wh_pos_3 = np.where(segs_beg_3 > 1) segs_beg_3_m1 = segs_beg_3.copy() segs_beg_3_m1[wh_pos_3] -= 1 segs_beg_3_m1[segs_beg_3_m1 < 1] = 1 # For a segment, the variance due to Poisson noise # = slope/(tgroup * gain * (ngroups-1)), # where slope is the estimated median slope, tgroup is the group time, # and ngroups is the number of groups in the segment. # Here the denominator of this quantity will be computed, which will be # later multiplied by the estimated median slope. # Suppress, then re-enable, harmless arithmetic warnings, as NaN will be # checked for and handled later warnings.filterwarnings("ignore", ".*invalid value.*", RuntimeWarning) warnings.filterwarnings("ignore", ".*divide by zero.*", RuntimeWarning) den_p3 = 1. / (group_time * gain_1d.reshape(imshape) * segs_beg_3_m1) warnings.resetwarnings() # For a segment, the variance due to readnoise noise # = 12 * readnoise**2 /(ngroups_seg**3. - ngroups_seg)/( tgroup **2.) num_r3 = 12. * (rn_sect / group_time)**2. # always >0 # Reshape for every group, every pixel in section num_r3 = np.dstack([num_r3] * max_seg) num_r3 = np.transpose(num_r3, (2, 0, 1)) # Denominator den_r3 = 1./(segs_beg_3 **3.-segs_beg_3). The minimum number # of allowed groups is 2, which will apply if there is actually only 1 # group; in this case den_r3 = 1/6. This covers the case in which there is # only one good group at the beginning of the integration, so it will be # be compared to the plane of (near) zeros resulting from the reset. For # longer segments, this value is overwritten below. den_r3 = num_r3.copy() * 0. + 1. / 6 wh_seg_pos = np.where(segs_beg_3 > 1) # Suppress, then, re-enable harmless arithmetic warnings, as NaN will be # checked for and handled later warnings.filterwarnings("ignore", ".*invalid value.*", RuntimeWarning) warnings.filterwarnings("ignore", ".*divide by zero.*", RuntimeWarning) den_r3[wh_seg_pos] = 1. / ( segs_beg_3[wh_seg_pos]**3. - segs_beg_3[wh_seg_pos] ) # overwrite where segs>1 warnings.resetwarnings() return (den_r3, den_p3, num_r3, segs_beg_3)
"Selección de fechas en Noviembre, Diciembre, Enero, Febrero, Marzo, Abril" pos_2016_04_30 = np.where(fechas == Timestamp('2016-04-30 18:00:00'))[0][0] FCH = fechas[3: pos_2016_04_30 +1 : 4] DT = [] leap_years = np.array([x for x in set(FCH[FCH.is_leap_year].year)]) # Años bisiestos normal_years = np.array([x for x in set(FCH[~FCH.is_leap_year].year)]) # Años normales years = np.array([x for x in set(FCH.year)]) for i in years[:-1]: pos = np.where(FCH == pd.Timestamp(str(i)+'-11-01 18:00:00'))[0][0] if np.any(normal_years == i+1) == True: DT.append(FCH[pos:pos+181]) else: DT.append(FCH[pos:pos+182]) FECHAS = pd.DatetimeIndex(np.concatenate(DT)) "Lectura de Estados" rf = open('/home/yordan/YORDAN/UNAL/TESIS_MAESTRIA/25_expo_2018/States_'+ch+'_NovAbr_anom_925.csv', 'r') reader = csv.reader(rf) states = [row for row in reader][1:] rf.close() print "La extracción de los estados en la exposición 26 para hacer composites de presión, se hace sin los últimos 181 datos, ya que los datos que se tienen de presión sólo llegan hasta el 2016-01-31" states6 = np.array([int(x[5]) for x in states])[:-181]