def train_step_batch(self, epoch): D1 = ma.dot(self.vectors**2, self.weight_matrix) D2 = ma.dot(self.vectors, self.constant_matrix) Dist = D1 - D2 best_nodes = ma.argmin(Dist, 0) distances = ma.min(Dist, 0) ## print "q error:", ma.mean(ma.sqrt(distances + self.dist_cons)), self.radius(epoch) self.qerror.append(ma.mean(ma.sqrt(distances + self.dist_cons))) if self.neighbourhood == Map.NeighbourhoodGaussian: H = numpy.exp(-self.unit_distances / (2 * self.radius(epoch))) * ( self.unit_distances <= self.radius(epoch)) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: H = 1.0 - (self.unit_distances / self.radius(epoch))**2 H = H * (H >= 0.0) else: H = 1.0 * (self.unit_distances <= self.radius(epoch)) P = numpy.zeros((self.vectors.shape[0], self.data.shape[0])) P[(best_nodes, list(range(len(best_nodes))))] = numpy.ones(len(best_nodes)) S = ma.dot(H, ma.dot(P, self.data)) A = ma.dot(H, ma.dot(P, ~self.data._mask)) ## nonzero = (range(epoch%2, len(self.vectors), 2), ) nonzero = (numpy.array(sorted(set(ma.nonzero(A)[0]))), ) self.vectors[nonzero] = S[nonzero] / A[nonzero]
def train_step_sequential(self, epoch, indices=None): """A single step of sequential training algorithm. """ indices = range(len(self.data)) if indices is None else indices for ind in indices: x = self.data[ind] Dx = self.vectors - self.data[ind] Dist = ma.sum(Dx**2, 1) min_dist = ma.min(Dist) bmu = ma.argmin(Dist) self.distances.append(min_dist) iter = epoch * len(self.data) + ind if self.neighbourhood == Map.NeighbourhoodGaussian: h = numpy.exp(-self.unit_distances[:, bmu]**2 / (2 * self.radius_seq(iter)**2)) * ( self.unit_distances[:, bmu]**2 <= self.radius_seq(iter)**2) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: h = 1.0 - (self.unit_distances[:bmu] / self.radius_seq(iter))**2 h = h * (h >= 0.0) else: h = 1.0 * (self.unit_distances[:, bmu] <= self.radius_seq(iter)) h = h * self.alpha(iter) nonzero = ma.nonzero(h) h = h[nonzero] self.vectors[nonzero] = self.vectors[ nonzero] - Dx[nonzero] * numpy.reshape(h, (len(h), 1))
def train_step_batch(self, epoch): """A single step of batch training algorithm. """ D1 = ma.dot(self.vectors**2, self.weight_matrix) D2 = ma.dot(self.vectors, self.constant_matrix) Dist = D1 - D2 best_nodes = ma.argmin(Dist, 0) distances = ma.min(Dist, 0) ## print "q error:", ma.mean(ma.sqrt(distances + self.dist_cons)), self.radius(epoch) self.qerror.append(ma.mean(ma.sqrt(distances + self.dist_cons))) if self.neighbourhood == Map.NeighbourhoodGaussian: H = numpy.exp(-self.unit_distances**2/(2*self.radius(epoch)**2)) * (self.unit_distances**2 <= self.radius(epoch)**2) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: H = 1.0 - (self.unit_distances/self.radius(epoch))**2 H = H * (H >= 0.0) else: H = 1.0*(self.unit_distances <= self.radius(epoch)) P = numpy.zeros((self.vectors.shape[0], self.data.shape[0])) P[(best_nodes, range(len(best_nodes)))] = numpy.ones(len(best_nodes)) S = ma.dot(H, ma.dot(P, self.data)) A = ma.dot(H, ma.dot(P, ~self.data._mask)) ## nonzero = (range(epoch%2, len(self.vectors), 2), ) nonzero = (numpy.array(sorted(set(ma.nonzero(A)[0]))), ) self.vectors[nonzero] = S[nonzero] / A[nonzero]
def find_nearest(self, target_position, target_sublattice=None): """Return the index of the position nearest the target Parameters ---------- target_position : array_like target_sublattice : int Look for a specific sublattice site. By default any will do. Returns ------- int Examples -------- >>> sites = Sites(([0, 1, 1.1], [0, 0, 0], [0, 0, 0]), [0, 1, 0]) >>> sites.find_nearest([1, 0, 0]) 1 >>> sites.find_nearest([1, 0, 0], target_sublattice=0) 2 """ distances = self.distances(target_position) if target_sublattice is None: return np.argmin(distances) else: target_sublattice = self._translate_sublattice(target_sublattice) return ma.argmin(ma.array(distances, mask=(self.sublattices != target_sublattice)))
def find_nearest(self, target_position, target_site_family=""): """Return the index of the position nearest the target Parameters ---------- target_position : array_like target_site_family : Optional[str] Look for a specific sublattice site. By default any will do. Returns ------- int Examples -------- >>> sites = Sites(([0, 1, 1.1], [0, 0, 0], [0, 0, 0]), [0, 1, 0]) >>> sites.find_nearest([1, 0, 0]) 1 >>> sites.find_nearest([1, 0, 0], target_site_family=0) 2 """ distances = self.distances(target_position) if target_site_family == "": return np.argmin(distances) else: return ma.argmin( ma.array(distances, mask=(self.ids != target_site_family)))
def train_step_sequential(self, epoch, indices=None): """A single step of sequential training algorithm. """ indices = range(len(self.data)) if indices == None else indices for ind in indices: x = self.data[ind] Dx = self.vectors - self.data[ind] Dist = ma.sum(Dx**2, 1) min_dist = ma.min(Dist) bmu = ma.argmin(Dist) self.distances.append(min_dist) iter = epoch*len(self.data)+ind if self.neighbourhood == Map.NeighbourhoodGaussian: h = numpy.exp(-self.unit_distances[:, bmu]**2/(2*self.radius_seq(iter)**2)) * (self.unit_distances[:, bmu]**2 <= self.radius_seq(iter)**2) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: h = 1.0 - (self.unit_distances[:bmu]/self.radius_seq(iter))**2 h = h * (h >= 0.0) else: h = 1.0*(self.unit_distances[:, bmu] <= self.radius_seq(iter)) h = h * self.alpha(iter) nonzero = ma.nonzero(h) h = h[nonzero] self.vectors[nonzero] = self.vectors[nonzero] - Dx[nonzero] * numpy.reshape(h, (len(h), 1))
def train_step_sequential(self, epoch, indices=None): indices = list(range(len(self.data))) if indices == None else indices for ind in indices: x = self.data[ind] Dx = self.vectors - self.data[ind] Dist = ma.sum(Dx**2, 1) min_dist = ma.min(Dist) bmu = ma.argmin(Dist) self.distances.append(min_dist) if self.neighbourhood == Map.NeighbourhoodGaussian: h = numpy.exp( -self.unit_distances[:, bmu] / (2 * self.radius(epoch))) * (self.unit_distances[:, bmu] <= self.radius(epoch)) elif self.neighbourhood == Map.NeighbourhoodEpanechicov: h = 1.0 - (self.unit_distances[:bmu] / self.radius(epoch))**2 h = h * (h >= 0.0) else: h = 1.0 * (self.unit_distances[:, bmu] <= self.radius(epoch)) h = h * self.alpha(epoch) nonzero = ma.nonzero(h) h = h[nonzero] self.vectors[nonzero] = self.vectors[ nonzero] - Dx[nonzero] * numpy.reshape(h, (len(h), 1))
def getBestMatchingNode(self, example): """ Return the best matching node """ example, c, w = orange.ExampleTable([example]).toNumpyMA() vectors = self.map.vectors() Dist = vectors - example bmu = ma.argmin(ma.sum(Dist**2, 1)) return list(self.map)[bmu]
def get_best_matching_node(self, instance): """Return the best matching node for a given data instance """ instance, c, w = Orange.data.Table([instance]).toNumpyMA() vectors = self.map.vectors() Dist = vectors - instance bmu = ma.argmin(ma.sum(Dist**2, 1)) return list(self.map)[bmu]
def clipdata(y, xm, x0, x1, minFlag=False): mx = ma.getdata(mask(xm, xm, x0, x1)) my = ma.getdata(mask(y, xm, x0, x1)) if minFlag: # allow clipping from first minimum after the start time u = ma.argmin(mask(y, xm, x0, x1)) mx = mx[u:-1] my = my[u:-1] return (mx, my)
def _lhsmu(N, samples=None, corr=None, random_state=None, M=5): if random_state is None: random_state = np.random.RandomState() elif not isinstance(random_state, np.random.RandomState): random_state = np.random.RandomState(random_state) if samples is None: samples = N I = M * samples rdpoints = random_state.uniform(size=(I, N)) dist = spatial.distance.cdist(rdpoints, rdpoints, metric='euclidean') D_ij = ma.masked_array(dist, mask=np.identity(I)) index_rm = np.zeros(I - samples, dtype=int) i = 0 while i < I - samples: order = ma.sort(D_ij, axis=1) avg_dist = ma.mean(order[:, 0:2], axis=1) min_l = ma.argmin(avg_dist) D_ij[min_l, :] = ma.masked D_ij[:, min_l] = ma.masked index_rm[i] = min_l i += 1 rdpoints = np.delete(rdpoints, index_rm, axis=0) if (corr is not None): #check if covariance matrix is valid assert type(corr) == np.ndarray assert corr.ndim == 2 assert corr.shape[0] == corr.shape[1] assert corr.shape[0] == N norm_u = stats.norm().ppf(rdpoints) L = linalg.cholesky(corr, lower=True) norm_u = np.matmul(norm_u, L) H = stats.norm().cdf(norm_u) else: H = np.zeros_like(rdpoints, dtype=float) rank = np.argsort(rdpoints, axis=0) for l in range(samples): low = float(l) / samples high = float(l + 1) / samples l_pos = rank == l H[l_pos] = random_state.uniform(low, high, size=N) return H
def crossmatch(cat0, cat1, threshold=1., racol0='ra', deccol0='dec', racol1='ra', deccol1='dec', right_join=False): dra = cat0[racol0] - cat1[racol1][:, newaxis] ddec = cat0[deccol0] - cat1[deccol1][:, newaxis] sep = np.sqrt(dra**2 + ddec**2) * 3600. matches = np.min(sep, axis=1) < threshold inds = np.argmin(sep, axis=1) out = Table(cat0[inds], masked=True) if right_join: for col in out.colnames: out[col].mask = ~matches cat1 = cat1.copy() else: out = out[matches] cat1 = cat1[matches] return out, cat1
def computeEdgeDistances(uvframe): """ Create a 2D matrix @edgedists as a companion to @uvframe, containing for each pixel a distance to the nearest edge (more precisely, the nearest 0-valued pixel). We compute @edgedists in a floodfill fashion spreading from zero-areas to the middle of one-areas iteratively, with distances approximated on the pixel grid. We return a tuple (edgedists, edgedirs), where edgedirs contains information about the relative offset of the nearest edge piece. """ # edgedists is a masked array, with only already computed values unmasked; # at first, uvframe == 0 already are computed (as zeros) edgedists = ma.array(numpy.zeros(uvframe.shape, dtype = numpy.float), mask = (uvframe > 0)) edgedirs = ma.array(numpy.zeros(uvframe.shape, dtype = (numpy.float, 2)), mask = [[[j,j] for j in i] for i in uvframe > 0]) #numpy.set_printoptions(threshold=numpy.nan) #print edgedists #print edgedirs flood_spread = scipy.ndimage.morphology.generate_binary_structure(2, 2) neighbor_ofs = [[-1,-1],[-1,0],[-1,1], [0,-1],[0,0],[0,1], [1,-1],[1,0],[1,1]] s2 = math.sqrt(2) neighbor_dist = [s2,1,s2, 1,0,1, s2,1,s2] while ma.getmaskarray(edgedists).any(): # scan masked area for any elements that have unmasked neighbors done_mask = numpy.invert(ma.getmaskarray(edgedists)) todo_mask = done_mask ^ scipy.ndimage.binary_dilation(done_mask, flood_spread) #print_mask(todo_mask) for i in numpy.transpose(numpy.nonzero(todo_mask)): neighbor_val = ma.array([ edge_dist_if_within(edgedists, i + ofs) + dist for ofs, dist in zip(neighbor_ofs, neighbor_dist) ]) nearestnei = ma.argmin(neighbor_val) # We assert that this update never affects value other fields # visited later in this iteration of floodfill edgedists[tuple(i)] = neighbor_val[nearestnei] nearestneicoord = i + neighbor_ofs[nearestnei] #print "-", nearestneicoord, edgedirs[tuple(nearestneicoord)] edgedirs[tuple(i)] = edgedirs[tuple(nearestneicoord)] + tuple(neighbor_ofs[nearestnei]) #print "+", i, edgedirs[tuple(i)] return (edgedists.data, edgedirs.data)
def _select_pivot(self): """ Selects pivot around which we would like to perform optimization. Returns: (i, j): i - row, j - column """ # col - most positive value in objective row col = np.argmax(self.c) masked_column = ma.MaskedArray(self.tableau[:, col], self.tableau[:, col] <= 0) if masked_column.count() == 0: raise UnboundedProblem() ratios = self.b / masked_column # we naively choose the first minimum ratio min_idx = ma.argmin(ratios) return (min_idx, col)
def corr_proba(r, ndata, ndataset=2, dof=False): """Probability of rejecting correlations - **r**: Correlation coefficient - **ndata**: Number of records use for correlations - **ndataset**, optional: Number of datasets (1 for autocorrelations, else 2) [default: 2] .. todo:: This must be rewritten using :mod:`scipy.stats` """ # TODO: use scipy for betai and _gamma? from genutil.salstat import betai,_gammaln # Basic tests ndata = MA.masked_equal(ndata,0,copy=0) r = MV2.masked_where(MA.equal(MA.absolute(r),1.),r,copy=0) # Degree of freedom if dof: df = ndata else: df = ndata-2-ndataset # Advanced test: prevent extreme values by locally decreasing the dof reduc = N.ones(r.shape) z = None while z is None or MA.count(MA.masked_greater(z,-600.)): if z is not None: imax = MA.argmin(z.ravel()) reduc.flat[imax] += 1 dfr = df/reduc t = r*MV2.sqrt(dfr/((1.0-r)* (1.0+r))) a = 0.5*dfr b = 0.5 x = df/(dfr+t**2) z = _gammaln(a+b)-_gammaln(a)-_gammaln(b)+a*MA.log(x)+b*MA.log(1.0-x) # Perfom the test and format the variable prob = MV2.masked_array(betai(a,b,x),axes=r.getAxisList())*100 prob.id = 'corr_proba' ; prob.name = prob.id prob.long_name = 'Probability of rejection' prob.units = '%' return prob
def corr_proba(r, ndata, ndataset=2, dof=False): """Probability of rejecting correlations - **r**: Correlation coefficient - **ndata**: Number of records use for correlations - **ndataset**, optional: Number of datasets (1 for autocorrelations, else 2) [default: 2] .. todo:: This must be rewritten using :mod:`scipy.stats` """ # Basic tests ndata = MA.masked_equal(ndata,0,copy=0) r = MV2.masked_where(MA.equal(MA.absolute(r),1.),r,copy=0) # Degree of freedom if dof: df = ndata else: df = ndata-2-ndataset # Advanced test: prevent extreme values by locally decreasing the dof reduc = N.ones(r.shape) z = None while z is None or MA.count(MA.masked_greater(z,-600.)): if z is not None: imax = MA.argmin(z.ravel()) reduc.flat[imax] += 1 dfr = df/reduc t = r*MV2.sqrt(dfr/((1.0-r)* (1.0+r))) a = 0.5*dfr b = 0.5 x = df/(dfr+t**2) z = _gammaln(a+b)-_gammaln(a)-_gammaln(b)+a*MA.log(x)+b*MA.log(1.0-x) # Perfom the test and format the variable prob = MV2.masked_array(betai(a,b,x),axes=r.getAxisList())*100 prob.id = 'corr_proba' ; prob.name = prob.id prob.long_name = 'Probability of rejection' prob.units = '%' return prob
def _read(self): """ Read the file.""" xml = ET.parse(self.filename).getroot() self.xml = xml # There might be several power curves for different air density data_sets = [] densities = [] for pt in xml.findall('PerformanceTable'): densities.append(float(pt.get('AirDensity'))) data_sets.append(datapoint(pt.findall('DataTable/DataPoint'))) # Looking for the density closest to normal atmospheric conditions: 1.225 id = argmin(abs(array(densities) - 1.225)) self.density = densities[id] a = data_sets[id] ## Sorting with respect of wind speed self.data = a[np.argsort(a[:, 0]), :] self.rotor_diameter = float(xml.get('RotorDiameter')) self.hub_height = float(xml.findall('SuggestedHeights/Height')[0].text) self.manufacturer = xml.get('ManufacturerName')
def _read(self): """ Read the file.""" xml = ET.parse(self.filename).getroot() self.xml = xml # There might be several power curves for different air density data_sets = [] densities = [] for pt in xml.findall("PerformanceTable"): densities.append(float(pt.get("AirDensity"))) data_sets.append(datapoint(pt.findall("DataTable/DataPoint"))) # Looking for the density closest to normal atmospheric conditions: 1.225 id = argmin(abs(array(densities) - 1.225)) self.density = densities[id] a = data_sets[id] ## Sorting with respect of wind speed self.data = a[np.argsort(a[:, 0]), :] self.rotor_diameter = float(xml.get("RotorDiameter")) self.hub_height = float(xml.findall("SuggestedHeights/Height")[0].text) self.manufacturer = xml.get("ManufacturerName")
def measure(mode, x, y, x0, x1, thresh = 0): """ return the a measure of y in the window x0 to x1 """ xt = x.view(numpy.ndarray) # strip Metaarray stuff -much faster! v = y.view(numpy.ndarray) xm = ma.masked_outside(xt, x0, x1).T ym = ma.array(v, mask = ma.getmask(xm)) if mode == 'mean': r1 = ma.mean(ym) r2 = ma.std(ym) if mode == 'max' or mode == 'maximum': r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == 'min' or mode == 'minimum': r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == 'median': r1 = ma.median(ym) r2 = 0 if mode == 'p2p': # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == 'std': # standard deviation r1 = ma.std(ym) r2 = 0 if mode == 'var': # variance r1 = ma.var(ym) r2 = 0 if mode == 'cumsum': # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == 'anom': # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == 'sum': r1 = ma.sum(ym) r2 = 0 if mode == 'area' or mode == 'charge': r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm)) r2 = 0 if mode == 'latency': # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == 'count': r1 = ma.count(ym) r2 = 0 if mode == 'maxslope': return(0,0) slope = numpy.array([]) win = ma.flatnotmasked_contiguous(ym) st = int(len(win)/20) # look over small ranges for k in win: # move through the slope measurementwindow tb = range(k-st, k+st) # get tb array newa = numpy.array(self.dat[i][j, thisaxis, tb]) ppars = numpy.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope = numpy.append(slope, ppars[0]) # keep track of max slope r1 = numpy.amax(slope) r2 = numpy.argmax(slope) return(r1, r2)
def __init__(self, bcfiles, mmfiles, agglvl, metricname): self.nm = len(bcfiles) self.metricname = metricname with nc(bcfiles[0]) as f: self.aggs = f.variables[agglvl][:] self.aggunits = f.variables[agglvl].units self.agglongname = f.variables[agglvl].long_name self.dt = f.variables['dt'].long_name.split(', ') self.mp = f.variables['mp'].long_name.split(', ') self.cr = f.variables['cr'].long_name.split(', ') naggs, ndt, nmp, ncr = len(self.aggs), len(self.dt), len(self.mp), len( self.cr) with nc(mmfiles[0]) as f: self.metricunits = f.variables[metricname].units self.models = [0] * self.nm self.scens = array([]) self.tmin = inf self.tmax = -inf for i in range(self.nm): self.models[i] = basename(bcfiles[i]).split('_')[0] # model names with nc(mmfiles[i]) as f: # scenarios scens = f.variables['scen'].long_name.split(', ') self.scens = unique(list(self.scens) + scens) with nc(bcfiles[i]) as f: # time time = f.variables['time'][:] time_units = f.variables['time'].units time += int(findall(r'\d+', time_units)[0]) self.tmin = min(self.tmin, time[0]) self.tmax = max(self.tmax, time[-1]) self.time = arange(self.tmin, self.tmax + 1) ntime = len(self.time) self.metrics = makeMasked((self.nm, naggs, ndt, nmp, ncr)) self.yield_detr = makeMasked((self.nm, naggs, ntime, ndt, nmp, ncr)) self.yield_retr = makeMasked((self.nm, naggs, ntime, ndt, nmp, ncr)) self.top_scens = makeMasked((self.nm, naggs, ndt, nmp, ncr)) for i in range(self.nm): with nc(mmfiles[i]) as f: metric = f.variables[metricname][:, :, :, :, :, 0] metric = masked_where(metric < 0, metric) # mask negative values scens = f.variables['scen'].long_name.split(', ') scenidx = argmin(self.__cost(metric), axis=1) with nc(bcfiles[i]) as f: time = f.variables['time'][:] time_units = f.variables['time'].units time += int(findall(r'\d+', time_units)[0]) ydt = f.variables['yield_detrend'][:] yrt = f.variables['yield_retrend'][:] inrange = logical_and(self.time >= time[0], self.time <= time[-1]) for a in range(naggs): for d, m, c in product(range(ndt), range(nmp), range(ncr)): si = scenidx[a, d, m, c] self.metrics[i, a, d, m, c] = metric[a, si, d, m, c] self.yield_detr[i, a, inrange, d, m, c] = ydt[a, :, si, d, m, c] self.yield_retr[i, a, inrange, d, m, c] = yrt[a, :, si, d, m, c] self.top_scens[i, a, d, m, c] = where( self.scens == scens[si])[0][0] + 1
def get_fit(theta, height): """ Fitting the local theta profile with three lines """ fitvals = np.zeros_like(theta) RSS = np.empty((290, 290)) + np.nan print RSS[0, 0] for j in range(290): if j > 2: for k in range(290): if k > j + 1 and k < 289: b_1 = ( np.sum(np.multiply(height[:j], theta[:j])) - 1 / j * np.sum(height[:j]) * np.sum(theta[:j]) ) / (np.sum(height[:j] ** 2) - 1 / j * np.sum(height[:j]) ** 2) a_1 = np.sum(np.multiply(height[:j], theta[:j])) / np.sum(height[:j]) - b_1 * np.sum( height[:j] ** 2 ) / np.sum(height[:j]) b_2 = (np.sum(theta[j:k]) - (k - j) * (a_1 + b_1 * height[j])) / ( np.sum(height[j:k]) - (k - j) * height[j] ) a_2 = np.sum(np.multiply(height[j:k], theta[j:k])) / np.sum(height[j:k]) - b_2 * np.sum( height[j:k] ** 2 ) / np.sum(height[j:k]) b_3 = (np.sum(theta[k:290]) - (290 - k) * (a_2 + b_2 * height[k])) / ( np.sum(height[k:290]) - (290 - k) * height[k] ) a_3 = np.sum(np.multiply(height[k:290], theta[k:290])) / np.sum(height[k:290]) - b_3 * np.sum( height[k:290] ** 2 ) / np.sum(height[k:290]) RSS[j, k] = ( np.sum(np.add(theta[2:j], -(a_1 + b_1 * height[2:j])) ** 2) + np.sum(np.add(theta[j:k], -(a_2 + b_2 * height[j:k])) ** 2) + np.sum(np.add(theta[k:290], -(a_3 + b_3 * height[k:290])) ** 2) ) RSS = ma.masked_where(np.isnan(RSS), RSS) [j, k] = np.unravel_index(ma.argmin(RSS), RSS.shape) b_1 = (np.sum(np.multiply(height[:j], theta[:j])) - 1 / j * np.sum(height[:j] * np.sum(theta[:j]))) / ( np.sum(height[:j] ** 2) - 1 / j * np.sum(height[2:j]) ** 2 ) a_1 = np.sum(np.multiply(height[:j], theta[:j])) / np.sum(height[:j]) - b_1 * np.sum(height[:j] ** 2) / np.sum( height[:j] ) b_2 = (np.sum(theta[j:k]) - (k - j) * (a_1 + b_1 * height[j])) / (np.sum(height[j:k]) - (k - j) * height[j]) a_2 = np.sum(np.multiply(height[j:k], theta[j:k])) / np.sum(height[j:k]) - b_2 * np.sum(height[j:k] ** 2) / np.sum( height[j:k] ) b_3 = (np.sum(theta[k:290]) - (290 - k) * (a_2 + b_2 * height[k])) / (np.sum(height[k:290]) - (290 - k) * height[k]) a_3 = np.sum(np.multiply(height[k:290], theta[k:290])) / np.sum(height[k:290]) - b_3 * np.sum( height[k:290] ** 2 ) / np.sum(height[k:290]) fitvals[:j] = b_1 * height[:j] + a_1 fitvals[j:k] = b_2 * height[j:k] + a_2 fitvals[k:290] = b_3 * height[k:290] + a_3 return fitvals, RSS, j, k
def Galaxy_Data(galaxy_ID): ''' PARAMETERS ========== galaxy_ID : string 'Plate-IFU' RETURNS ======= physical properties & data of the galaxy ''' plate, IFU = galaxy_ID.split('-') ''' ############################################################################ # Obtain redshift #--------------------------------------------------------------------------- for i in range(len(DTable2)): if DTable2['MaNGA_plate'][i] == int(plate) and DTable2['MaNGA_IFU'][i] == int(IFU): redshift = z[i] velocity = redshift * c distance = (velocity / H_0) * 1E3 #kpc scale = 0.5 * (distance) / 206265 ############################################################################ ''' ''' ############################################################################ # Obtain inclination #--------------------------------------------------------------------------- incl = 0 for i in range(len(DTable1)): if DTable1['MaNGA_plate'][i] == int(plate) and DTable1['MaNGA_IFU'][i] == int(IFU): incl = np.arccos(rat[i]) ############################################################################ ''' ''' ############################################################################ # Obtain phi #--------------------------------------------------------------------------- ph = 0 for i in range(len(DTable1)): if DTable1['MaNGA_plate'][i] == int(plate) and DTable1['MaNGA_IFU'][i] == int(IFU): ph = phi[i] * np.pi / 180 ############################################################################ ''' ############################################################################ # Obtaining Data Cubes, Inverse Variances, and Masks #--------------------------------------------------------------------------- #cube = fits.open('manga-' + galaxy_ID + '-MAPS-HYB10-GAU-MILESHC.fits.gz') cube = fits.open(VEL_MAP_FOLDER + plate + '/' + IFU + '/manga-' + galaxy_ID + '-MAPS-HYB10-GAU-MILESHC.fits.gz') r_band = cube['SPX_MFLUX'].data Ha_vel = cube['EMLINE_GVEL'].data[18] Ha_vel_ivar = cube['EMLINE_GVEL_IVAR'].data[18] Ha_vel_mask = cube['EMLINE_GVEL_MASK'].data[18] vmasked = ma.array(Ha_vel, mask=Ha_vel_mask) ivar_masked = ma.array(Ha_vel_ivar, mask=Ha_vel_mask) gshape = vmasked.shape ############################################################################ ############################################################################ # Finding the center #--------------------------------------------------------------------------- center_guess = np.unravel_index(ma.argmin(np.abs(vmasked), axis=None), vmasked.shape) x_center_guess = center_guess[0] y_center_guess = center_guess[1] ############################################################################ #return scale, incl, ph, r_band, Ha_vel, Ha_vel_ivar, Ha_vel_mask, vmasked, gshape, x_center_guess, y_center_guess return r_band, Ha_vel, Ha_vel_ivar, Ha_vel_mask, vmasked, ivar_masked, gshape, x_center_guess, y_center_guess
def __init__(self, bcfiles, mmfiles, agglvl, metricname): self.nm = len(bcfiles) self.metricname = metricname with nc(bcfiles[0]) as f: self.aggs = f.variables[agglvl][:] self.aggunits = f.variables[agglvl].units self.agglongname = f.variables[agglvl].long_name self.dt = f.variables['dt'].long_name.split(', ') self.mp = f.variables['mp'].long_name.split(', ') self.cr = f.variables['cr'].long_name.split(', ') naggs, ndt, nmp, ncr = len(self.aggs), len(self.dt), len(self.mp), len(self.cr) with nc(mmfiles[0]) as f: self.metricunits = f.variables[metricname].units self.models = [0] * self.nm self.scens = array([]) self.tmin = inf self.tmax = -inf for i in range(self.nm): self.models[i] = basename(bcfiles[i]).split('_')[0] # model names with nc(mmfiles[i]) as f: # scenarios scens = f.variables['scen'].long_name.split(', ') self.scens = unique(list(self.scens) + scens) with nc(bcfiles[i]) as f: # time time = f.variables['time'][:] time_units = f.variables['time'].units time += int(findall(r'\d+', time_units)[0]) self.tmin = min(self.tmin, time[0]) self.tmax = max(self.tmax, time[-1]) self.time = arange(self.tmin, self.tmax + 1) ntime = len(self.time) self.metrics = makeMasked((self.nm, naggs, ndt, nmp, ncr)) self.yield_detr = makeMasked((self.nm, naggs, ntime, ndt, nmp, ncr)) self.yield_retr = makeMasked((self.nm, naggs, ntime, ndt, nmp, ncr)) self.top_scens = makeMasked((self.nm, naggs, ndt, nmp, ncr)) for i in range(self.nm): with nc(mmfiles[i]) as f: metric = f.variables[metricname][:, :, :, :, :, 0] metric = masked_where(metric < 0, metric) # mask negative values scens = f.variables['scen'].long_name.split(', ') scenidx = argmin(self.__cost(metric), axis = 1) with nc(bcfiles[i]) as f: time = f.variables['time'][:] time_units = f.variables['time'].units time += int(findall(r'\d+', time_units)[0]) ydt = f.variables['yield_detrend'][:] yrt = f.variables['yield_retrend'][:] inrange = logical_and(self.time >= time[0], self.time <= time[-1]) for a in range(naggs): for d, m, c in product(range(ndt), range(nmp), range(ncr)): si = scenidx[a, d, m, c] self.metrics[i, a, d, m, c] = metric[a, si, d, m, c] self.yield_detr[i, a, inrange, d, m, c] = ydt[a, :, si, d, m, c] self.yield_retr[i, a, inrange, d, m, c] = yrt[a, :, si, d, m, c] self.top_scens[i, a, d, m, c] = where(self.scens == scens[si])[0][0] + 1
def measure(mode, x, y, x0, x1, thresh=0): """ return the a measure of y in the window x0 to x1 """ xm = ma.masked_outside(x, x0, x1) # .compressed() ym = ma.array(y, mask=ma.getmask(xm)) # .compressed() if mode == 'mean': r1 = np.mean(ym) r2 = np.std(ym) if mode == 'max' or mode == 'maximum': r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == 'min' or mode == 'minimum': r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == 'minormax': r1p = ma.max(ym) r1n = ma.min(ym) if ma.abs(r1p) > ma.abs(r1n): r1 = r1p r2 = xm[ma.argmax(ym)] else: r1 = r1n r2 = xm[ma.argmin(ym)] if mode == 'median': r1 = ma.median(ym) r2 = 0 if mode == 'p2p': # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == 'std': # standard deviation r1 = ma.std(ym) r2 = 0 if mode == 'var': # variance r1 = ma.var(ym) r2 = 0 if mode == 'cumsum': # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == 'anom': # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == 'sum': r1 = ma.sum(ym) r2 = 0 if mode == 'area' or mode == 'charge': r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm)) r2 = 0 if mode == 'latency': # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == '1090': #measure 10-90% time, also returns max r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] y10 = 0.1 * r1 y90 = 0.9 * r1 sm1 = ma.nonzero(ym >= y10) sm9 = ma.nonzero(ym >= y90) r1 = xm[sm9] - xm[sm1] if mode == 'count': r1 = ma.count(ym) r2 = 0 if mode == 'maxslope': return (0, 0) slope = np.array([]) win = ma.flatnotmasked_contiguous(ym) st = int(len(win) / 20) # look over small ranges for k in win: # move through the slope measurementwindow tb = range(k - st, k + st) # get tb array newa = np.array(self.dat[i][j, thisaxis, tb]) ppars = np.polyfit( x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope = np.append(slope, ppars[0]) # keep track of max slope r1 = np.amax(slope) r2 = np.argmax(slope) return (r1, r2)
#print(adj) #print(maskarr) #adj.mask = ma.nomask #print(adj) #adj.mask = maskarr #print(adj) while not clustered: print(maskarr) print(maskarr.all()) semiclustered = maskarr.all() if not semiclustered: short = ma.argmin(adj) A = int(short//num_ungrp) B = int(short%num_ungrp) first = np.maximum(A,B) second = np.minimum(A,B) temp = [] for i in clus_ind[A]: temp.append(i) for i in clus_ind[B]: temp.append(i) clus_ind.pop(first) clus_ind.pop(second) combined = []
def measure(mode, x, y, x0, x1, thresh=0): """ return the a measure of y in the window x0 to x1 """ xm = ma.masked_outside(x, x0, x1) # .compressed() ym = ma.array(y, mask=ma.getmask(xm)) # .compressed() if mode == "mean": r1 = np.mean(ym) r2 = np.std(ym) if mode == "max" or mode == "maximum": r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == "min" or mode == "minimum": r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == "minormax": r1p = ma.max(ym) r1n = ma.min(ym) if ma.abs(r1p) > ma.abs(r1n): r1 = r1p r2 = xm[ma.argmax(ym)] else: r1 = r1n r2 = xm[ma.argmin(ym)] if mode == "median": r1 = ma.median(ym) r2 = 0 if mode == "p2p": # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == "std": # standard deviation r1 = ma.std(ym) r2 = 0 if mode == "var": # variance r1 = ma.var(ym) r2 = 0 if mode == "cumsum": # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == "anom": # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == "sum": r1 = ma.sum(ym) r2 = 0 if mode == "area" or mode == "charge": r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm)) r2 = 0 if mode == "latency": # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == "1090": # measure 10-90% time, also returns max r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] y10 = 0.1 * r1 y90 = 0.9 * r1 sm1 = ma.nonzero(ym >= y10) sm9 = ma.nonzero(ym >= y90) r1 = xm[sm9] - xm[sm1] if mode == "count": r1 = ma.count(ym) r2 = 0 if mode == "maxslope": return (0, 0) slope = np.array([]) win = ma.flatnotmasked_contiguous(ym) st = int(len(win) / 20) # look over small ranges for k in win: # move through the slope measurementwindow tb = range(k - st, k + st) # get tb array newa = np.array(self.dat[i][j, thisaxis, tb]) ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope = np.append(slope, ppars[0]) # keep track of max slope r1 = np.amax(slope) r2 = np.argmax(slope) return (r1, r2)
def measure(mode, x, y, x0, x1, thresh=0, slopewin=1.0): """ return the a measure of y in the window x0 to x1 """ xm = ma.masked_outside(x, x0, x1)# .compressed() ym = ma.array(y, mask = ma.getmask(xm))# .compressed() if mode == 'mean': r1 = np.mean(ym) r2 = np.std(ym) if mode == 'max' or mode == 'maximum': r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == 'min' or mode == 'minimum': r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == 'minormax': r1p = ma.max(ym) r1n = ma.min(ym) if ma.abs(r1p) > ma.abs(r1n): r1 = r1p r2 = xm[ma.argmax(ym)] else: r1 = r1n r2 = xm[ma.argmin(ym)] if mode == 'median': r1 = ma.median(ym) r2 = 0 if mode == 'p2p': # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == 'std': # standard deviation r1 = ma.std(ym) r2 = 0 if mode == 'var': # variance r1 = ma.var(ym) r2 = 0 if mode == 'cumsum': # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == 'anom': # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == 'sum': r1 = ma.sum(ym) r2 = 0 if mode == 'area' or mode == 'charge': r1 = ma.sum(ym)/(ma.max(xm)-ma.min(xm)) r2 = 0 if mode == 'latency': # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == '1090': #measure 10-90% time, also returns max r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] y10 = 0.1*r1 y90 = 0.9*r1 sm1 = ma.nonzero(ym >= y10) sm9 = ma.nonzero(ym >= y90) r1 = xm[sm9] - xm[sm1] if mode == 'count': r1 = ma.count(ym) r2 = 0 if mode == 'maxslope': slope = [] win = ma.flatnotmasked_contiguous(ym) dt = x[1]-x[0] st = int(slopewin/dt) # use slopewin duration window for fit. print('st: ', st) for k, w in enumerate(win): # move through the slope measurementwindow tb = range(k-st, k+st) # get tb array ppars = np.polyfit(x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope.append(ppars[0]) # keep track of max slope r1 = np.max(slope) r2 = np.argmax(slope) return(r1, r2)
def measure(mode, x, y, x0, x1, thresh=0): """ return the a measure of y in the window x0 to x1 """ xt = x.view(numpy.ndarray) # strip Metaarray stuff -much faster! v = y.view(numpy.ndarray) xm = ma.masked_outside(xt, x0, x1).T ym = ma.array(v, mask=ma.getmask(xm)) if mode == 'mean': r1 = ma.mean(ym) r2 = ma.std(ym) if mode == 'max' or mode == 'maximum': r1 = ma.max(ym) r2 = xm[ma.argmax(ym)] if mode == 'min' or mode == 'minimum': r1 = ma.min(ym) r2 = xm[ma.argmin(ym)] if mode == 'median': r1 = ma.median(ym) r2 = 0 if mode == 'p2p': # peak to peak r1 = ma.ptp(ym) r2 = 0 if mode == 'std': # standard deviation r1 = ma.std(ym) r2 = 0 if mode == 'var': # variance r1 = ma.var(ym) r2 = 0 if mode == 'cumsum': # cumulative sum r1 = ma.cumsum(ym) # Note: returns an array r2 = 0 if mode == 'anom': # anomalies = difference from averge r1 = ma.anom(ym) # returns an array r2 = 0 if mode == 'sum': r1 = ma.sum(ym) r2 = 0 if mode == 'area' or mode == 'charge': r1 = ma.sum(ym) / (ma.max(xm) - ma.min(xm)) r2 = 0 if mode == 'latency': # return first point that is > threshold sm = ma.nonzero(ym > thresh) r1 = -1 # use this to indicate no event detected r2 = 0 if ma.count(sm) > 0: r1 = sm[0][0] r2 = len(sm[0]) if mode == 'count': r1 = ma.count(ym) r2 = 0 if mode == 'maxslope': return (0, 0) slope = numpy.array([]) win = ma.flatnotmasked_contiguous(ym) st = int(len(win) / 20) # look over small ranges for k in win: # move through the slope measurementwindow tb = range(k - st, k + st) # get tb array newa = numpy.array(self.dat[i][j, thisaxis, tb]) ppars = numpy.polyfit( x[tb], ym[tb], 1) # do a linear fit - smooths the slope measures slope = numpy.append(slope, ppars[0]) # keep track of max slope r1 = numpy.amax(slope) r2 = numpy.argmax(slope) return (r1, r2)
ax[1].tick_params(axis='x', which='major', labelsize=14) ax[1].hlines(mlim, np.min(t_in), np.max(t_in), color='black', alpha=0.9, linestyle='dotted') #we stack up the separate instruments tstack = ma.vstack([tmabrite, tmaastep, tmabring]) astack = ma.vstack([amabrite, amaastep, amabring]) estack = ma.vstack([emabrite, emaastep, emabring]) # now find which instrument has the smallest error and make an array from that min_amp_ind = ma.argmin(estack, axis=0) # pull out the lowest values min_amp = astack[min_amp_ind, np.arange(min_amp_ind.size)] min_tim = tstack[min_amp_ind, np.arange(min_amp_ind.size)] min_err = estack[min_amp_ind, np.arange(min_amp_ind.size)] ax[1].errorbar(min_tim, min_amp, yerr=min_err, fmt='none', color='black', label='BRITE', elinewidth=1)