def get_depths(noise_maps, pix_size, mask=None, pixel_weights=None): """Compute depth_i and depth_p (sensitivities) from noise maps. :param noise_maps: the noise maps :param pix_size: the pixel size in arcmin :param mask: the mask to apply :param pixel_weights: weighting of pixels (coverage) :return: depth_i and depth_p of the map """ # apply pixel weights weighted_maps = np.empty_like(noise_maps) weighted_maps[...] = noise_maps[...] * pixel_weights # apply mask noise_ma = ma.array(weighted_maps, mask=mask) # noise estimation (in I component) using the noise maps depth_i = ma.getdata(ma.std(noise_ma[:, 0, :], axis=1)) depth_i *= pix_size # noise estimation (in Q & U components) depth_p = ma.getdata(ma.std(noise_ma[:, 1:, :], axis=(1, 2))) depth_p *= pix_size return depth_i, depth_p
def plot_propagation(self, projection='lambert', inbasemap=None, factor=3, showfig=False): """Plot propagation direction """ if inbasemap == None: m = self._get_basemap(projection=projection) else: m = inbasemap if self.lonArr.shape[0] - 2 == self.grad[0].shape[ 0] and self.lonArr.shape[1] - 2 == self.grad[0].shape[1]: self.cut_edge(1, 1) elif self.lonArr.shape[0] != self.grad[0].shape[ 0] or self.lonArr.shape[1] != self.grad[0].shape[1]: raise ValueError( 'Incompatible shape for gradient and lon/lat array!') normArr = np.sqrt( ma.getdata(self.grad[0])**2 + ma.getdata(self.grad[1])**2) x, y = m(self.lonArr, self.latArr) U = self.grad[1] / normArr V = self.grad[0] / normArr if factor != None: x = x[0:self.Nlat:factor, 0:self.Nlon:factor] y = y[0:self.Nlat:factor, 0:self.Nlon:factor] U = U[0:self.Nlat:factor, 0:self.Nlon:factor] V = V[0:self.Nlat:factor, 0:self.Nlon:factor] Q = m.quiver(x, y, U, V, scale=50, width=0.001) if showfig: plt.show() return
def update_json(props, varname, data, datatype): """ Adds an extra field to the properties of a feature in a geojson file. Provides a consistent way to handle different data types :param props: dictionary of properties :param varname: string variable name :param data: the data to add :param datatype: datatype :return: updated props """ if (isinstance(data, ma.MaskedArray) and (data.shape == ()) and ("int" in str(datatype))): props.update({varname: np.int(ma.getdata(data))}) elif (isinstance(data, ma.MaskedArray) and (data.shape == ()) and ("float" in str(datatype))): props.update({varname: np.float(ma.getdata(data))}) elif isinstance(data, np.float32) or isinstance(data, np.float): props.update({varname: float(data)}) elif isinstance(data, np.int) or isinstance(data, np.uint16): props.update({varname: int(data)}) elif str(datatype) == "string": props.update({varname: str(data)}) else: return
def Fill2ThetaAzimuthMap(masks, TA, tam, image): 'Needs a doc string' Zlim = masks['Thresholds'][1] rings = masks['Rings'] arcs = masks['Arcs'] TA = np.dstack((ma.getdata(TA[1]), ma.getdata(TA[0]), ma.getdata(TA[2]))) #azimuth, 2-theta, dist tax, tay, tad = np.dsplit(TA, 3) #azimuth, 2-theta, dist**2/d0**2 for tth, thick in rings: tam = ma.mask_or( tam.flatten(), ma.getmask( ma.masked_inside(tay.flatten(), max(0.01, tth - thick / 2.), tth + thick / 2.))) for tth, azm, thick in arcs: tamt = ma.getmask( ma.masked_inside(tay.flatten(), max(0.01, tth - thick / 2.), tth + thick / 2.)) tama = ma.getmask(ma.masked_inside(tax.flatten(), azm[0], azm[1])) tam = ma.mask_or(tam.flatten(), tamt * tama) taz = ma.masked_outside(image.flatten(), int(Zlim[0]), Zlim[1]) tabs = np.ones_like(taz) tam = ma.mask_or(tam.flatten(), ma.getmask(taz)) tax = ma.compressed(ma.array(tax.flatten(), mask=tam)) #azimuth tay = ma.compressed(ma.array(tay.flatten(), mask=tam)) #2-theta taz = ma.compressed(ma.array(taz.flatten(), mask=tam)) #intensity tad = ma.compressed(ma.array(tad.flatten(), mask=tam)) #dist**2/d0**2 tabs = ma.compressed(ma.array( tabs.flatten(), mask=tam)) #ones - later used for absorption corr. return tax, tay, taz, tad, tabs
def plot_field(self, projection='lambert', contour=True, geopolygons=None, showfig=True, vmin=None, vmax=None, stations=False, event=False): """Plot data with contour """ m=self._get_basemap(projection=projection, geopolygons=geopolygons) x, y=m(self.lonArr, self.latArr) if event: try: evx, evy=m(self.evlo, self.evla) m.plot(evx, evy, 'yo', markersize=10) except: pass if stations: try: stx, sty=m(self.lonArrIn, self.latArrIn) m.plot(stx, sty, 'y^', markersize=6) except: pass try: stx, sty = m(self.stalons, self.stalats) m.plot(stx, sty, 'b^', markersize=6) except: pass im=m.pcolormesh(x, y, self.Zarr, cmap='gist_ncar_r', shading='gouraud', vmin=vmin, vmax=vmax) cb = m.colorbar(im, "bottom", size="3%", pad='2%') cb.ax.tick_params(labelsize=10) if self.fieldtype=='Tph' or self.fieldtype=='Tgr': cb.set_label('sec', fontsize=12, rotation=0) if self.fieldtype=='Amp': cb.set_label('nm', fontsize=12, rotation=0) if contour: # levels=np.linspace(ma.getdata(self.Zarr).min(), ma.getdata(self.Zarr).max(), 20) levels=np.linspace(ma.getdata(self.Zarr).min(), ma.getdata(self.Zarr).max(), 60) m.contour(x, y, self.Zarr, colors='k', levels=levels, linewidths=0.5) if showfig: plt.show() return m
def plot_field(self, projection='lambert', contour=True, geopolygons=None, showfig=True, vmin=None, vmax=None): """Plot data with contour """ m=self._get_basemap(projection=projection, geopolygons=geopolygons) x, y=m(self.lonArr, self.latArr) try: evx, evy=m(self.evlo, self.evla) m.plot(evx, evy, 'yo', markersize=10) except: pass try: stx, sty=m(self.lonArrIn, self.latArrIn) m.plot(stx, sty, 'y^', markersize=10) except: pass im=m.pcolormesh(x, y, self.Zarr, cmap='gist_ncar_r', shading='gouraud', vmin=vmin, vmax=vmax) cb = m.colorbar(im, "bottom", size="3%", pad='2%') cb.ax.tick_params(labelsize=10) if self.fieldtype=='Tph' or self.fieldtype=='Tgr': cb.set_label('sec', fontsize=12, rotation=0) if self.fieldtype=='Amp': cb.set_label('nm', fontsize=12, rotation=0) if contour: # levels=np.linspace(ma.getdata(self.Zarr).min(), ma.getdata(self.Zarr).max(), 20) levels=np.linspace(ma.getdata(self.Zarr).min(), ma.getdata(self.Zarr).max(), 60) m.contour(x, y, self.Zarr, colors='k', levels=levels, linewidths=0.5) if showfig: plt.show() return
def apply_qc(self, ncvariable): ''' Applies QC to a qartod variable :param netCDF4.Variable ncvariable: A QARTOD Variable ''' qc_tests = { 'flat_line': qc.flat_line_check, 'gross_range': qc.range_check, 'rate_of_change': qc.rate_of_change_check, 'spike': qc.spike_check, 'pressure': gliders_qc.pressure_check } qartod_test = getattr(ncvariable, 'qartod_test', None) if not qartod_test: return standard_name = getattr(ncvariable, 'standard_name').split(' ')[0] parent = self.ncfile.get_variables_by_attributes( standard_name=standard_name)[0] times, values, mask = self.get_unmasked(parent) # There's no data to QC if len(values) == 0: return # If the config isn't set for this test, don't run it. if qartod_test not in self.config[standard_name]: return test_params = {} if qartod_test in 'rate_of_change': times = ma.getdata(times[~mask]) time_units = self.ncfile.variables['time'].units dates = np.array(num2date(times, time_units), dtype='datetime64[ms]') test_params['times'] = dates # Calculate the threshold value test_params['thresh_val'] = self.get_rate_of_change_threshold( values, times, time_units) elif qartod_test == 'spike': test_params['times'] = ma.getdata(times[~mask]) test_params['low_thresh'], test_params[ 'high_thresh'] = self.get_spike_thresholds(values) else: test_params = self.config[standard_name][qartod_test] if qartod_test == 'pressure': test_params['pressure'] = values else: test_params['arr'] = values qc_flags = qc_tests[qartod_test](**test_params) ncvariable[~mask] = qc_flags for test_param in test_params: if test_param in ('arr', 'times', 'pressure'): continue ncvariable.setncattr(test_param, test_params[test_param])
def clipdata(y, xm, x0, x1, minFlag=False): mx = ma.getdata(mask(xm, xm, x0, x1)) my = ma.getdata(mask(y, xm, x0, x1)) if minFlag: # allow clipping from first minimum after the start time u = ma.argmin(mask(y, xm, x0, x1)) mx = mx[u:-1] my = my[u:-1] return (mx, my)
def _dense_fit(self, X, strategy, missing_values, fill_value): """Fit the transformer on dense data.""" mask = _get_mask(X, missing_values) X_np = X.to_ndarray() masked_X = ma.masked_array(X_np, mask=np.isnan(X_np)) # FIXME # Mean if strategy == "mean": mean_masked = ma.mean(masked_X, axis=0) # Avoid the warning "Warning: converting a masked element to nan." mean = ma.getdata(mean_masked) mean[ma.getmask(mean_masked)] = np.nan return mean # Median elif strategy == "median": median_masked = ma.median(masked_X, axis=0) # Avoid the warning "Warning: converting a masked element to nan." median = ma.getdata(median_masked) median[ma.getmaskarray(median_masked)] = np.nan return median # Most frequent elif strategy == "most_frequent": # Avoid use of scipy.stats.mstats.mode due to the required # additional overhead and slow benchmarking performance. # See Issue 14325 and PR 14399 for full discussion. # To be able access the elements by columns X = X.T mask = mask.T npdtype = typemap(X.dtype()) if npdtype.kind == "O": most_frequent = af.constant(0, X.shape[0], dtype=object) else: most_frequent = af.constant(0, X.shape[0]) for i, (row, row_mask) in enumerate(zip(X[:], mask[:])): row_mask = row_mask.logical_not() row = row[row_mask].to_ndarray() #most_frequent[i] = _most_frequent(row, np.nan, 0) most_frequent[i] = _most_frequent(row, np.nan, 0) return most_frequent # Constant elif strategy == "constant": # for constant strategy, self.statistcs_ is used to store # fill_value in each column return af.constant(fill_value, X.shape[1], dtype=X.dtype())
def read_icespeed_nc(sample_step=10, store=True): ''' read nc file :return: ''' nc_file = fileutils.correct_path(settings.ICE_SPEED_NC_FILE) dataset = Dataset(nc_file, mask_and_scale=False, decode_times=False) nc_vars = [var for var in dataset.variables] headers = nc_vars[3:7] print('Reading speeds from file....') mask_lat = dataset.variables[headers[0]][:] print('done with: ' + headers[0]) mask_lon = dataset.variables[headers[1]][:] print('done with: ' + headers[1]) mask_vel_x = dataset.variables[headers[2]][:] print('done with: ' + headers[2]) mask_vel_y = dataset.variables[headers[3]][:] print('done with: ' + headers[3]) print('reading the nonzero...') nonzeroposition_x = ma.nonzero(mask_vel_x) nonzeroposition_y = ma.nonzero(mask_vel_y) print('zip...') nonzeroposition_zip = zip(nonzeroposition_x[0], nonzeroposition_x[1], nonzeroposition_y[0], nonzeroposition_y[1]) print('filtering and casting cooridnates....') nonzeroposition = [] for x in tqdm(nonzeroposition_zip): if x[0] == x[2] & x[1] == x[3]: nonzeroposition.append((x[0], x[1])) lat = [] lon = [] vel_x = [] vel_y = [] print('creatating the dataframe...') for pos in tqdm(nonzeroposition): lat.append(ma.getdata(mask_lat[pos[0], pos[1]])) lon.append(ma.getdata(mask_lon[pos[0], pos[1]])) vel_x.append(ma.getdata(mask_vel_x[pos[0], pos[1]])) vel_y.append(ma.getdata(mask_vel_y[pos[0], pos[1]])) table = { headers[0]: lat, headers[1]: lon, headers[2]: vel_x, headers[3]: vel_y } df = pd.DataFrame(table, columns=headers) if store: print('storing in a csv') df.to_csv(settings.OUTPUT_ICESPEED_CSV, sep=',', encoding='utf-8') print('DONE') return df
def getDataOnly(array1d): """ Removes redundant no data slots in a 1D array Note that sometimes data is missing within the array, so in the majority of cases, the array just needs to be shortened, but in a small number of cases, the 'within array' no data needs to be replaced :param array1d: :return: Simple array of numbers """ if np.any(ma.getmask(array1d)): mymask = np.invert(ma.getmask(array1d)) outarray = ma.getdata(array1d)[mymask] else: outarray = ma.getdata(array1d) return outarray
def apply_on_fields(series, func, *args, **kwargs): """ Apply the function ``func`` to each field of ``series``. Parameters ---------- series : array-like A (subclass) of ndarray. func : function Function to apply. args : var Additional arguments to ``func``. kwargs : var Additional optional parameters to ``func`` """ names = series.dtype.names if names is None: return func(series, *args, **kwargs) results = [func(series[f], *args, **kwargs) for f in names] rdtype = [(f[0], r.dtype) for (f, r) in zip(series.dtype.descr, results)] rdata = [ma.getdata(r) for r in results] rmask = [ma.getmaskarray(r) for r in results] isarray = isinstance(results[0], np.ndarray) if isarray: output = ma.array(zip(*rdata), mask=zip(*rmask), dtype=rdtype).view(type=type(series)) output._update_from(series) else: output = ma.array(tuple(rdata), mask=tuple(rmask), dtype=rdtype) return output
def filter_invalid_value(data_array): """ This filter applies a mask to all numerically invalid inputs on a programing side. Numbers that are usually infinite or some other nonsensical quantity serve no real usage in calculations further downstream. Therefore, they are masked here. See numpy.ma.fix_invalid for what is considered invalid. Parameters ---------- data_array : ndarray The data array that the mask will be calculated from. Returns ------- final_mask : ndarray -> dictionary A boolean array for pixels that are masked (True) or are valid (False). """ # As fixing all invalid data is required, masks might obscure # the data itself. raw_data_array = np_ma.getdata(data_array) # Mask all of the invalid data. final_mask = np_ma.getmaskarray(np_ma.fix_invalid(raw_data_array)) return final_mask
def individual_ftp(kk, args): dset_ftp_d, resfactor, gray, mask_g, ref_m_gray, N_iter_max, n, th, c_rect, sl_rect, mask_of_annulus_alone, lin_min_idx_a, lin_max_idx_a, col_min_idx_a, col_max_idx_a = args # dset_ftp_d, resfactor, gray, mask_g, ref_m_gray, N_iter_max, n, th, c_rect, sl_rect, kk = args def_image = dset_ftp_d[:, :, kk] # 1. Undistort image # def_image = undistort_image(def_image, mapx, mapy) # 2. Substract gray def_m_gray = def_image - resfactor * gray #ELIMINO LOS REFLEJOS A MANO #def_m_gray[(def_m_gray<-30)] = 0 #def_m_gray[(def_m_gray>30)] = 0 # 3. Extrapolate fringes def_m_gray = gerchberg2d(def_m_gray, mask_g, N_iter_max=N_iter_max) # 4. Process by FTP dphase = calculate_phase_diff_map_1D(def_m_gray, ref_m_gray, \ th, n) dphase_rect = np.mean( dphase[(c_rect[0]-sl_rect):(c_rect[0]+sl_rect), \ (c_rect[1]-sl_rect):(c_rect[1]+sl_rect)] ) # 5a). Time unwrapping using rectangle area # 5b). Substract solid movement dphase = dphase - dphase_rect # 6. Calculate height field height = dphase #height_map_from_phase_map(dphase, L, D, pspp) height = (ma.getdata(height) * mask_of_annulus_alone)[lin_min_idx_a:lin_max_idx_a, col_min_idx_a:col_max_idx_a] return height
def make_gene_map_1(self): count = 0 self.iterator = itertools.product(range(self.time_len), range(self.lat_len), range(self.lon_len)) for x_valid in self.iterator: binary_string = bin(count)[2:] while len(binary_string) < self.string_length: binary_string = "0" + binary_string # self.gene_map[binary_string] = {} # self.gene_map[binary_string]['coordinate'] = tuple(x_valid) if ma.getdata(self.array[x_valid]) < 100: # chooses unmasked points? self.gene_map[binary_string] = {} self.gene_map[binary_string]["coordinate"] = tuple(x_valid) self.gene_map[binary_string]["value"] = self.array[x_valid] # count += 1 print count, binary_string, self.gene_map[binary_string]["value"] # debug else: # self.gene_map[binary_string]['value'] = 1E06 pass count += 1 # print count, binary_string, self.gene_map[binary_string]['value'] # debug self.last_valid_binary_string = binary_string not_valid_first = eval("0b" + binary_string) + 1 not_valid_last = eval("0b" + "1" * self.string_length) print not_valid_last print x_valid print ma.isMA(self.array) for x_not_valid in range(not_valid_first, not_valid_last + 1): binary_string = bin(x_not_valid)[2:] self.gene_map[binary_string] = {} self.gene_map[binary_string]["coordinate"] = (999, 999, 999) self.gene_map[binary_string]["value"] = 1e06 print x_not_valid, binary_string, self.gene_map[binary_string]["value"]
def putdata_raw(imgname, data, clone=None): """Store (overwrite) data in an existing CASA image. See getdata_raw(imgname) for the reverse operation. Parameters ---------- imagename : str The (absolute) CASA image filename. It should exist already, unless **clone** was given. data : 2D numpy array or a list of 2D numpy arrays The data... clone : str, optional An optional filename from which to clone the image for output. It needs to be an absolute filename. """ if clone != None: taskinit.ia.fromimage(infile=clone,outfile=imgname,overwrite=True) taskinit.ia.close() # @todo this seems circumvent to have to borrow the odd dimensions (nx,ny,1,1,1) shape was seen if type(data) == type([]): # @todo since this needs to extend the axes, the single plane clone and replace data doesn't work here raise Exception,"Not Implemented Yet" bigim = ia.imageconcat(outfile=imgname, infiles=infiles, axis=2, relax=T, tempclose=F, overwrite=T) bigim.close() else: taskinit.tb.open(imgname,nomodify=False) d = taskinit.tb.getcol('map') pdata = ma.getdata(data).reshape(d.shape) taskinit.tb.putcol('map',pdata) taskinit.tb.flush() taskinit.tb.close() return
def get_unmasked(self, ncvariable): times = self.ncfile.variables['time'][:] values = ncvariable[:] mask = np.zeros(times.shape[0], dtype=bool) if hasattr(values, 'mask'): mask |= values.mask if hasattr(times, 'mask'): mask |= times.mask values_initial = ma.getdata(values[~mask]) config = self.get_config(ncvariable.name) units = getattr(ncvariable, 'units', '1') # If units are not defined or empty, set them to unitless # If the config units are empty, do not attempt to convert units # The latter is necessary as some of the NetCDF files do not have # units attribute under the udunits variable definitions if not units or pd.isnull(config.units): units = '1' values = values_initial # must be a CF unit or this will throw an exception elif ncvariable.units != config.units: try: values = Unit(units).convert(values_initial, config.units) except ValueError as e: exc_text = "Caught exception while converting units: {}".format( e) get_logger().warn(exc_text) values = values_initial return times, values, mask
def create_ac9_hs6_plot(netcdf_file_path): """create a png from a ac9/hs6 netcdf file""" plot_output_filepath = os.path.splitext(netcdf_file_path)[0] + '.png' dataset = Dataset(netcdf_file_path) n_wavelength = dataset.variables['wavelength'].shape[0] # look for main variable for varname in dataset.variables.keys(): dim = dataset.variables[varname].dimensions if 'obs' in dim and 'wavelength' in dim: main_data = dataset.variables[varname] fig = figure(num=None, figsize=(15, 10), dpi=80, facecolor='w', edgecolor='k') labels = [] # only one profile per file in AC9 HS6 files for i_wl in range(n_wavelength): main_var_val = main_data[:, i_wl] # for i_prof depth_val = dataset.variables['DEPTH'][:] wavelength_val = dataset.variables['wavelength'][i_wl] plot(main_var_val, depth_val)#, c=np.random.rand(3, 1)) labels.append('%s nm' % wavelength_val) station_name = ''.join(ma.getdata(dataset.variables['station_name'][dataset.variables['station_index'][0] - 1])) title('%s\nCruise: %s\n Station %s' % (dataset.source, dataset.cruise_id, station_name)) gca().invert_yaxis() xlabel('%s: %s in %s' % (main_data.name, main_data.long_name, main_data.units)) ylabel('%s in %s; positive %s' % (dataset.variables['DEPTH'].long_name, dataset.variables['DEPTH'].units, dataset.variables['DEPTH'].positive)) legend(labels, loc='upper right', prop=fontP, title='Wavelength') savefig(plot_output_filepath) plt.close(fig)
def test_units_qc(self): fd, fake_file = tempfile.mkstemp() os.close(fd) self.addCleanup(os.remove, fake_file) nc = Dataset(fake_file, 'w') nc.createDimension('time', 10) timevar = nc.createVariable('time', np.float64, ('time', ), fill_value=-9999.) timevar.standard_name = 'time' timevar.units = 'seconds since 1970-01-01T00:00:00Z' timevar[np.array([0, 2, 4, 6, 8])] = np.array([0, 2, 4, 6, 8]) tempvar = nc.createVariable('temp', np.float32, ('time', ), fill_value=-9999.) tempvar.standard_name = 'sea_water_temperature' tempvar.units = 'deg_F' tempvar[np.array([0, 1, 2, 3, 4, 9])] = np.array([72.0, 72.1, 72.0, 1.0, 72.03, 72.1]) qc = GliderQC(nc, 'data/qc_config.yml') for qcvarname in qc.create_qc_variables(nc.variables['temp']): qc.apply_qc(nc.variables[qcvarname]) np.testing.assert_equal( nc.variables['qartod_temp_flat_line_flag'][:].mask, ~np.array([1, 0, 1, 0, 1, 0, 0, 0, 0, 0], dtype=bool)) np.testing.assert_equal( ma.getdata(nc.variables['qartod_temp_flat_line_flag'][:]), np.array([1, 9, 1, 9, 1, 9, 9, 9, 9, 9], dtype=np.int8))
def apply_primary_qc(self, ncvariable): ''' Applies the primary QC array which is an aggregate of all the other QC tests. :param netCDF4.Variable ncvariable: NCVariable ''' primary_qc_name = 'qartod_%s_primary_flag' % ncvariable.name if primary_qc_name not in self.ncfile.variables: return qcvar = self.ncfile.variables[primary_qc_name] # Only perform primary QC on variables created by DAC if getattr(qcvar, 'dac_comment', '') != 'ioos_qartod_primary': return qc_variables = self.find_qc_flags(ncvariable) vectors = [] for qc_variable in qc_variables: ncvar = self.ncfile.variables[qc_variable] if getattr(ncvar, 'dac_comment', '') != 'ioos_qartod': continue log.info("Using %s in primary QC for %s", qc_variable, primary_qc_name) vectors.append(ma.getdata(ncvar[:])) log.info("Applying QC for %s", primary_qc_name) flags = qc.qc_compare(vectors) qcvar[:] = flags
def LaplacianEqualXY(self): if self.dx != self.dy: raise ValueError('grid spacing not equal!') self.lplc = scipy.ndimage.filters.laplace(ma.getdata( self.Zarr)) / (self.dx * self.dy) self.lplc = self.lplc[1:-1, 1:-1] return
def ma2np(self): """Convert all the maksed data array to numpy array """ self.Zarr=ma.getdata(self.Zarr) try: self.diffaArr=ma.getdata(self.diffaArr) except: pass try: self.appV=ma.getdata(self.appV) except: pass try: self.lplc=ma.getdata(self.lplc) except: pass return
def height_annulus_polar_coordinates(frame, mask_annulus, phase_width=3000): """ Returns a matrix with the height of the annulus in polar coordinates. """ # mask_annulus = mask[:,:,0] annulus = frame annulus_polar = img2polar(ma.masked_array(annulus, mask=(1 - mask_annulus))) first_idx, last_idx = find_index_of_false_values(annulus_polar) if first_idx != last_idx: annulus_polar_lines = ma.getdata( annulus_polar[first_idx:last_idx, :]) #por si toma mal la mascara prom = np.average(annulus_polar_lines, axis=0) else: annulus_polar_lines = ma.getdata(annulus_polar[first_idx, :]) prom = annulus_polar_lines return prom
def data_bruta(data,x): VAR = False BB = data[x].get('reduct') data2 = ma.getdata(BB)[BB.mask == VAR] poly = np.repeat(x , len(data2)) dataf = {'poly':poly, 'data':data2} dataf = pd.DataFrame(dataf) return dataf
def ma2np(self): """Convert all the maksed data array to numpy array """ self.Zarr = ma.getdata(self.Zarr) try: self.diffaArr = ma.getdata(self.diffaArr) except: pass try: self.appV = ma.getdata(self.appV) except: pass try: self.lplc = ma.getdata(self.lplc) except: pass return
def generate_corrected_map(self, dataid, glbdir, outdir, pers=np.array([]), glbpfx='smpkolya_phv_R_', outpfx='smpkolya_phv_R_'): """ Generate corrected global phave velocity map using a regional phase velocity map. ================================================================================================================= Input Parameters: dataid - dataid for regional phase velocity map glbdir - location of global reference phase velocity map files outdir - output directory pers - period array for correction (default is 4) glbpfx - prefix for global reference phase velocity map files outpfx - prefix for output reference phase velocity map files ----------------------------------------------------------------------------------------------------------------- Output format: outdir/outpfx+str(int(per)) ================================================================================================================= """ if not os.path.isdir(outdir): os.makedirs(outdir) if pers.size == 0: pers = np.append( np.arange(7.) * 10. + 40., np.arange(2.) * 25. + 125.) for per in pers: inglobalfname = glbdir + '/' + glbpfx + str(int(per)) try: self.get_data4plot(dataid=dataid, period=per) except: print 'No regional data for period =', per, 'sec' continue if not os.path.isfile(inglobalfname): print 'No global data for period =', per, 'sec' continue outfname = outdir + '/' + outpfx + '%g' % (per) InglbArr = np.loadtxt(inglobalfname) outArr = InglbArr.copy() lonArr = self.lonArr.reshape(self.lonArr.size) latArr = self.latArr.reshape(self.latArr.size) vel_iso = ma.getdata(self.vel_iso) vel_iso = vel_iso.reshape(vel_iso.size) for i in xrange(InglbArr[:, 0].size): lonG = InglbArr[i, 0] latG = InglbArr[i, 1] phVG = InglbArr[i, 2] for j in xrange(lonArr.size): lonR = lonArr[j] latR = latArr[j] phVR = vel_iso[j] if abs(lonR - lonG) < 0.05 and abs( latR - latG) < 0.05 and phVR != 0: outArr[i, 2] = phVR np.savetxt(outfname, outArr, fmt='%g %g %.4f') return
def Gradient(self, edge_order=1, method='default', order=2): if method == 'default': self.grad = np.gradient(ma.getdata(self.Zarr), self.dx, self.dy, edge_order=edge_order) elif method == 'freq': diff_x = self.fftDiff(m=1, n=0) diff_y = self.fftDiff(m=0, n=1) self.grad = [] self.grad.append(diff_y) self.grad.append(diff_x) elif method == 'convolve': if order == 2: diff_x = convolve(ma.getdata(self.Zarr), X_diff_weight_2) / self.dx diff_y = convolve(ma.getdata(self.Zarr), Y_diff_weight_2) / self.dy elif order == 4: diff_x = convolve(ma.getdata(self.Zarr), X_diff_weight_4) / self.dx diff_y = convolve(ma.getdata(self.Zarr), Y_diff_weight_4) / self.dy elif order == 6: diff_x = convolve(ma.getdata(self.Zarr), X_diff_weight_6) / self.dx diff_y = convolve(ma.getdata(self.Zarr), Y_diff_weight_6) / self.dy self.grad = [] self.grad.append(diff_y) self.grad.append(diff_x) return
def Laplacian(self, method='default', order=2): Zarr = ma.getdata(self.Zarr) if method == 'default': Zarr_yp = Zarr[2:, 1:-1] Zarr_yn = Zarr[:-2, 1:-1] Zarr_xp = Zarr[1:-1, 2:] Zarr_xn = Zarr[1:-1, :-2] Zarr = Zarr[1:-1, 1:-1] self.lplc = (Zarr_yp + Zarr_yn - 2 * Zarr) / (self.dy**2) + ( Zarr_xp + Zarr_xn - 2 * Zarr) / (self.dx**2) elif method == 'convolve': if order == 2: diff2_x = convolve(ma.getdata(self.Zarr), X_diff2_weight_2) / self.dx / self.dx diff2_y = convolve(ma.getdata(self.Zarr), Y_diff2_weight_2) / self.dy / self.dy elif order == 4: diff2_x = convolve(ma.getdata(self.Zarr), X_diff2_weight_4) / self.dx / self.dx diff2_y = convolve(ma.getdata(self.Zarr), Y_diff2_weight_4) / self.dy / self.dy elif order == 6: diff2_x = convolve(ma.getdata(self.Zarr), X_diff2_weight_6) / self.dx / self.dx diff2_y = convolve(ma.getdata(self.Zarr), Y_diff2_weight_6) / self.dy / self.dy self.lplc = diff2_x + diff2_y self.lplc = self.lplc[1:-1, 1:-1] return
def _calculateStats(self, di, dof=0.): """Calculate the core QA statistics on a difference image""" mask = di.getMask() maskArr = di.getMask().getArray() # Create a mask using BAD, SAT, NO_DATA, EDGE bits. Keep detections maskArr &= mask.getPlaneBitMask(["BAD", "SAT", "NO_DATA", "EDGE"]) # Mask out values based on maskArr diArr = ma.array(di.getImage().getArray(), mask=maskArr) varArr = ma.array(di.getVariance().getArray(), mask=maskArr) # Normalize by sqrt variance, units are in sigma diArr /= np.sqrt(varArr) mean = diArr.mean() # This is the maximum-likelihood extimate of the variance stdev**2 stdev = diArr.std() median = ma.extras.median(diArr) # Compute IQR of just un-masked data data = ma.getdata(diArr[~diArr.mask]) iqr = np.percentile(data, 75.) - np.percentile(data, 25.) #Calculte chisquare of the residual chisq=np.sum(np.power(data, 2.)) # Mean squared error: variance + bias**2 # Bias = |data - model| = mean of diffim # Variance = |(data - model)**2| = mean of diffim**2 bias = mean variance = np.power(data, 2.).mean() mseResids = bias**2 + variance # If scipy is not set up, return zero for the stats try: #In try block because of risk of divide by zero rchisq=chisq/(len(data)-1-dof) # K-S test on the diffim to a Normal distribution import scipy.stats D, prob = scipy.stats.kstest(data, 'norm') A2, crit, sig = scipy.stats.anderson(data, 'norm') # Anderson Darling statistic cand be inf for really non-Gaussian distributions. if np.isinf(A2) or np.isnan(A2): A2 = 9999. except ZeroDivisionError: D = 0. prob = 0. A2 = 0. crit = np.zeros(5) sig = np.zeros(5) rchisq = 0 return {"mean": mean, "stdev": stdev, "median": median, "iqr": iqr, "D": D, "prob": prob, "A2": A2, "crit": crit, "sig": sig, "rchisq": rchisq, "mseResids": mseResids}
def _calculateStats(self, di, dof=0.): """Calculate the core QA statistics on a difference image""" mask = di.getMask() maskArr = di.getMask().getArray() # Create a mask using BAD, SAT, NO_DATA, EDGE bits. Keep detections maskArr &= mask.getPlaneBitMask(["BAD", "SAT", "NO_DATA", "EDGE"]) # Mask out values based on maskArr diArr = ma.array(di.getImage().getArray(), mask=maskArr) varArr = ma.array(di.getVariance().getArray(), mask=maskArr) # Normalize by sqrt variance, units are in sigma diArr /= np.sqrt(varArr) mean = diArr.mean() # This is the maximum-likelihood extimate of the variance stdev**2 stdev = diArr.std() median = ma.extras.median(diArr) # Compute IQR of just un-masked data data = ma.getdata(diArr[~diArr.mask]) iqr = np.percentile(data, 75.) - np.percentile(data, 25.) # Calculte chisquare of the residual chisq = np.sum(np.power(data, 2.)) # Mean squared error: variance + bias**2 # Bias = |data - model| = mean of diffim # Variance = |(data - model)**2| = mean of diffim**2 bias = mean variance = np.power(data, 2.).mean() mseResids = bias**2 + variance # If scipy is not set up, return zero for the stats try: # In try block because of risk of divide by zero rchisq = chisq/(len(data) - 1 - dof) # K-S test on the diffim to a Normal distribution import scipy.stats D, prob = scipy.stats.kstest(data, 'norm') A2, crit, sig = scipy.stats.anderson(data, 'norm') # Anderson Darling statistic cand be inf for really non-Gaussian distributions. if np.isinf(A2) or np.isnan(A2): A2 = 9999. except ZeroDivisionError: D = 0. prob = 0. A2 = 0. crit = np.zeros(5) sig = np.zeros(5) rchisq = 0 return {"mean": mean, "stdev": stdev, "median": median, "iqr": iqr, "D": D, "prob": prob, "A2": A2, "crit": crit, "sig": sig, "rchisq": rchisq, "mseResids": mseResids}
def ma2np(self): """Convert the maksed data array to numpy data array """ try: self.mask = self.Zarr.mask self.Zarr = ma.getdata(self.Zarr) except: print 'Data array is already numpy array' return
def test_set_fields(self): # Tests setting fields. base = self.base.copy() mbase = base.view(mrecarray) mbase = mbase.copy() mbase.fill_value = (999999, 1e20, 'N/A') # Change the data, the mask should be conserved mbase.a._data[:] = 5 assert_equal(mbase['a']._data, [5, 5, 5, 5, 5]) assert_equal(mbase['a']._mask, [0, 1, 0, 0, 1]) # Change the elements, and the mask will follow mbase.a = 1 assert_equal(mbase['a']._data, [1]*5) assert_equal(ma.getmaskarray(mbase['a']), [0]*5) # Use to be _mask, now it's recordmask assert_equal(mbase.recordmask, [False]*5) assert_equal(mbase._mask.tolist(), np.array([(0, 0, 0), (0, 1, 1), (0, 0, 0), (0, 0, 0), (0, 1, 1)], dtype=bool)) # Set a field to mask ........................ mbase.c = masked # Use to be mask, and now it's still mask ! assert_equal(mbase.c.mask, [1]*5) assert_equal(mbase.c.recordmask, [1]*5) assert_equal(ma.getmaskarray(mbase['c']), [1]*5) assert_equal(ma.getdata(mbase['c']), [asbytes('N/A')]*5) assert_equal(mbase._mask.tolist(), np.array([(0, 0, 1), (0, 1, 1), (0, 0, 1), (0, 0, 1), (0, 1, 1)], dtype=bool)) # Set fields by slices ....................... mbase = base.view(mrecarray).copy() mbase.a[3:] = 5 assert_equal(mbase.a, [1, 2, 3, 5, 5]) assert_equal(mbase.a._mask, [0, 1, 0, 0, 0]) mbase.b[3:] = masked assert_equal(mbase.b, base['b']) assert_equal(mbase.b._mask, [0, 1, 0, 1, 1]) # Set fields globally.......................... ndtype = [('alpha', '|S1'), ('num', int)] data = ma.array([('a', 1), ('b', 2), ('c', 3)], dtype=ndtype) rdata = data.view(MaskedRecords) val = ma.array([10, 20, 30], mask=[1, 0, 0]) with warnings.catch_warnings(): warnings.simplefilter("ignore") rdata['num'] = val assert_equal(rdata.num, val) assert_equal(rdata.num.mask, [1, 0, 0])
def plot(self, ds=1000, unit='km', cmap='seismic_r', vmin=None, vmax=None, zsize=10): """Plot velocity model ============================================================================= Input Parameters: ds - grid spacing unit - unit vmin, vmax - vmin,vmax for colorbar ============================================================================= """ # XLength=self.xmax-self.xmin # ZLength=self.zmax-self.zmin # xsize=zsize*(XLength/ZLength) # fig = plt.figure(figsize=(xsize, zsize)) if cmap=='ses3d': cmap = colormaps.make_colormap({0.0:[0.1,0.0,0.0], 0.2:[0.8,0.0,0.0], 0.3:[1.0,0.7,0.0],0.48:[0.92,0.92,0.92], 0.5:[0.92,0.92,0.92], 0.52:[0.92,0.92,0.92], 0.7:[0.0,0.6,0.7], 0.8:[0.0,0.0,0.8], 1.0:[0.0,0.0,0.1]}) if self.plotflag==False: raise ValueError('No plot array!') # plt.figure(figsize=(16,13)) if self.regular==True: im=plt.pcolormesh(self.XArrPlot/ds, self.ZArrPlot/ds, self.VsArrPlot/ds, cmap=cmap, vmin=vmin, vmax=vmax) else: xi = np.linspace(self.xmin, self.xmax, self.Nx*10) zi = np.linspace(self.zmin, self.zmax, self.Nz*10) self.xi, self.zi = np.meshgrid(xi, zi) #-- Interpolating at the points in xi, yi self.vi = griddata(self.XArr, self.ZArr, self.VsArr, self.xi, self.zi, 'linear') im=plt.pcolormesh(self.xi/ds, self.zi/ds, ma.getdata(self.vi)/ds, cmap=cmap, vmin=vmin, vmax=vmax, shading='gouraud') ########################################## plt.plot(500., 1000 , 'r*', markersize=30, lw=3) # plt.plot( [0., 4000.], [1000, 1000] , 'b--', lw=3) # plt.plot( [500., 500.], [700., 1300.] , 'g-', lw=3) # plt.plot( [500., 3500.], [700., 700.] , 'g-', lw=3) # plt.plot( [500., 3500.], [1300., 1300.] , 'g-', lw=3) # plt.plot( [3500., 3500.], [700., 1300.] , 'g-', lw=3) # # plt.plot( [0., 0.], [0., 2000.] , 'k-', lw=3) # plt.plot( [0., 4000.], [0., 0.] , 'k-', lw=3) # plt.plot( [4000., 4000.], [0., 2000.] , 'k-', lw=3) # plt.plot( [0., 4000.], [2000., 2000.] , 'k-', lw=3) ########################################## plt.xlabel('x('+unit+')', fontsize=35) plt.ylabel('z('+unit+')', fontsize=35) # plt.axis([self.xmin/ds, self.xmax/ds, self.zmin/ds, self.zmax/ds]) plt.axis('scaled') cb=plt.colorbar(shrink=0.8)#, size="3%", pad='2%') cb.set_label('Vs (km/s)', fontsize=20, rotation=90) plt.yticks(fontsize=30) plt.xticks(fontsize=30) ######################## # plt.ylim([-100, 2100]) # plt.xlim([-100, 4100]) ######################## plt.show() return
def fromarrays( arraylist, dtype=None, shape=None, formats=None, names=None, titles=None, aligned=False, byteorder=None, fill_value=None, ): """ Creates a mrecarray from a (flat) list of masked arrays. Parameters ---------- arraylist : sequence A list of (masked) arrays. Each element of the sequence is first converted to a masked array if needed. If a 2D array is passed as argument, it is processed line by line dtype : {None, dtype}, optional Data type descriptor. shape : {None, integer}, optional Number of records. If None, shape is defined from the shape of the first array in the list. formats : {None, sequence}, optional Sequence of formats for each individual field. If None, the formats will be autodetected by inspecting the fields and selecting the highest dtype possible. names : {None, sequence}, optional Sequence of the names of each field. fill_value : {None, sequence}, optional Sequence of data to be used as filling values. Notes ----- Lists of tuples should be preferred over lists of lists for faster processing. """ datalist = [getdata(x) for x in arraylist] masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist] _array = recfromarrays( datalist, dtype=dtype, shape=shape, formats=formats, names=names, titles=titles, aligned=aligned, byteorder=byteorder, ).view(mrecarray) _array._mask.flat = list(zip(*masklist)) if fill_value is not None: _array.fill_value = fill_value return _array
def forward_fill(marr, maxgap=None): """ Forward fills masked values in a 1-d array when there are less ``maxgap`` consecutive masked values. Parameters ---------- marr : MaskedArray Series to fill maxgap : {int}, optional Maximum gap between consecutive masked values. If ``maxgap`` is not specified, all masked values are forward-filled. Examples -------- >>> x = ma.arange(20) >>> x[(x%5)!=0] = ma.masked >>> print x [0 -- -- -- -- 5 -- -- -- -- 10 -- -- -- -- 15 -- -- -- --] >>> print forward_fill(x) [0 0 0 0 0 5 5 5 5 5 10 10 10 10 10 15 15 15 15 15] """ # !!!: We should probably port that to C. # Initialization .................. if np.ndim(marr) > 1: raise ValueError,"The input array should be 1D only!" a = ma.array(marr, copy=True) amask = getmask(a) if amask is nomask or a.size == 0: return a # adata = getdata(a) # Get the indices of the masked values (except a[0]) idxtofill = amask[1:].nonzero()[0] + 1 currGap = 0 if maxgap is not None: previdx = -1 for i in idxtofill: if i != previdx + 1: currGap = 0 currGap += 1 if currGap <= maxgap and not amask[i-1]: adata[i] = adata[i-1] amask[i] = False previdx = i else: amask[i-maxgap:i] = True else: for i in idxtofill: if not amask[i-1]: adata[i] = adata[i-1] amask[i] = False return a
def forward_fill(marr, maxgap=None): """ Forward fills masked values in a 1-d array when there are less ``maxgap`` consecutive masked values. Parameters ---------- marr : MaskedArray Series to fill maxgap : {int}, optional Maximum gap between consecutive masked values. If ``maxgap`` is not specified, all masked values are forward-filled. Examples -------- >>> x = ma.arange(20) >>> x[(x%5)!=0] = ma.masked >>> print x [0 -- -- -- -- 5 -- -- -- -- 10 -- -- -- -- 15 -- -- -- --] >>> print forward_fill(x) [0 0 0 0 0 5 5 5 5 5 10 10 10 10 10 15 15 15 15 15] """ # !!!: We should probably port that to C. # Initialization .................. if np.ndim(marr) > 1: raise ValueError, "The input array should be 1D only!" a = ma.array(marr, copy=True) amask = getmask(a) if amask is nomask or a.size == 0: return a # adata = getdata(a) # Get the indices of the masked values (except a[0]) idxtofill = amask[1:].nonzero()[0] + 1 currGap = 0 if maxgap is not None: previdx = -1 for i in idxtofill: if i != previdx + 1: currGap = 0 currGap += 1 if currGap <= maxgap and not amask[i - 1]: adata[i] = adata[i - 1] amask[i] = False previdx = i else: amask[i - maxgap:i] = True else: for i in idxtofill: if not amask[i - 1]: adata[i] = adata[i - 1] amask[i] = False return a
def rec_array_to_mgr( data: MaskedRecords | np.recarray | np.ndarray, index, columns, dtype: DtypeObj | None, copy: bool, typ: str, ): """ Extract from a masked rec array and create the manager. """ # essentially process a record array then fill it fdata = ma.getdata(data) if index is None: index = _get_names_from_index(fdata) else: index = ensure_index(index) if columns is not None: columns = ensure_index(columns) arrays, arr_columns = to_arrays(fdata, columns) # fill if needed if isinstance(data, np.ma.MaskedArray): # GH#42200 we only get here with MaskedRecords, but check for the # parent class MaskedArray to avoid the need to import MaskedRecords data = cast("MaskedRecords", data) new_arrays = fill_masked_arrays(data, arr_columns) else: # error: Incompatible types in assignment (expression has type # "List[ExtensionArray]", variable has type "List[ndarray]") new_arrays = arrays # type: ignore[assignment] # create the manager # error: Argument 1 to "reorder_arrays" has incompatible type "List[ndarray]"; # expected "List[ExtensionArray]" arrays, arr_columns = reorder_arrays( new_arrays, arr_columns, columns # type: ignore[arg-type] ) if columns is None: columns = arr_columns mgr = arrays_to_mgr(arrays, arr_columns, index, columns, dtype=dtype, typ=typ) if copy: mgr = mgr.copy() return mgr
def apply_qc(self, ncvariable): ''' Applies QC to a qartod variable :param netCDF4.Variable ncvariable: A QARTOD Variable ''' qc_tests = { 'flat_line': qc.flat_line_check, 'gross_range': qc.range_check, 'rate_of_change': qc.rate_of_change_check, 'spike': qc.spike_check, 'pressure': gliders_qc.pressure_check } # If the qartod_test attribute isn't defined then this isn't a variable # this script created and is not eligble for automatic QC qartod_test = getattr(ncvariable, 'qartod_test', None) if not qartod_test: return # Get a reference to the parent variable using the standard_name attribute standard_name = getattr(ncvariable, 'standard_name').split(' ')[0] parent = self.ncfile.get_variables_by_attributes( standard_name=standard_name)[0] test_params = self.get_test_params(parent.name) # If there is no parameters defined for this test, don't apply QC if qartod_test not in test_params: return test_params = test_params[qartod_test] if 'thresh_val' in test_params: test_params['thresh_val'] = test_params['thresh_val'] / pq.hour times, values, mask = self.get_unmasked(parent) if qartod_test == 'rate_of_change': times = ma.getdata(times[~mask]) dates = np.array(num2date(times, self.ncfile.variables['time'].units), dtype='datetime64[ms]') test_params['times'] = dates if qartod_test == 'pressure': test_params['pressure'] = values else: test_params['arr'] = values qc_flags = qc_tests[qartod_test](**test_params) get_logger().info("Flagged: %s", len(np.where(qc_flags == 4)[0])) get_logger().info("Total Values: %s", len(values)) ncvariable[~mask] = qc_flags
def fromarrays( arraylist, dtype=None, shape=None, formats=None, names=None, titles=None, aligned=False, byteorder=None, fill_value=None, ): """Creates a mrecarray from a (flat) list of masked arrays. Parameters ---------- arraylist : sequence A list of (masked) arrays. Each element of the sequence is first converted to a masked array if needed. If a 2D array is passed as argument, it is processed line by line dtype : {None, dtype}, optional Data type descriptor. shape : {None, integer}, optional Number of records. If None, shape is defined from the shape of the first array in the list. formats : {None, sequence}, optional Sequence of formats for each individual field. If None, the formats will be autodetected by inspecting the fields and selecting the highest dtype possible. names : {None, sequence}, optional Sequence of the names of each field. fill_value : {None, sequence}, optional Sequence of data to be used as filling values. Notes ----- Lists of tuples should be preferred over lists of lists for faster processing. """ datalist = [getdata(x) for x in arraylist] masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist] _array = recfromarrays( datalist, dtype=dtype, shape=shape, formats=formats, names=names, titles=titles, aligned=aligned, byteorder=byteorder, ).view(mrecarray) _array._mask.flat = zip(*masklist) if fill_value is not None: _array.fill_value = fill_value return _array
def natgridInterp(self, interp='nn', copy=True, bounds_error=False, fill_value=np.nan): if self.Xarr.size == self.XarrIn.size: if (np.abs(self.Xarr.reshape(self.Nx*self.Ny)-self.XarrIn)).sum() < 0.01\ and (np.abs(self.Yarr.reshape(self.Nx*self.Ny)-self.YarrIn)).sum() < 0.01: print 'No need to interpolate!' self.Zarr=self.ZarrIn return # self.Zarr = ma.getdata(griddata(self.XarrIn, self.YarrIn, self.ZarrIn, self.Xarr, self.Yarr, interp=interp)) self.Zarr = griddata(self.XarrIn, self.YarrIn, self.ZarrIn, self.Xarr, self.Yarr, interp=interp) self.Zarr = ma.getdata(self.Zarr) # self.Zarr=self.Zarr.reshape(self.Nx*self.Ny) return
def create_pigment_tss_plot(netcdf_file_path): """create a png from a pigment netcdf file""" plot_output_filepath = os.path.splitext(netcdf_file_path)[0] + '.png' dataset = Dataset(netcdf_file_path) profiles = dataset.variables['profile'] n_obs_per_profile = dataset.variables['row_size'] n_profiles = len(profiles) if 'CPHL_a' in dataset.variables.keys(): main_data = dataset.variables['CPHL_a'] elif 'SPM' in dataset.variables.keys(): main_data = dataset.variables['SPM'] else: find_main_var_name = (set(dataset.variables.keys()) - set([u'TIME', u'LATITUDE', u'LONGITUDE', u'station_name', u'station_index', u'profile', u'row_size', u'DEPTH'])).pop() main_data = dataset.variables[find_main_var_name] fig = figure(num=None, figsize=(15, 10), dpi=80, facecolor='w', edgecolor='k') labels = [] ylim([-1, max(dataset.variables['DEPTH'][:]) + 1]) for i_prof in range(n_profiles): # we look for the observations indexes related to the choosen profile idx_obs_start = sum(n_obs_per_profile[0:i_prof]) idx_obs_end = idx_obs_start + n_obs_per_profile[i_prof] - 1 idx_obs = range(idx_obs_start, idx_obs_end + 1) main_var_val = main_data[idx_obs] # for i_prof depth_val = dataset.variables['DEPTH'][idx_obs] if len(main_var_val) == 1: scatter(main_var_val, depth_val) elif len(main_var_val) > 1: plot(main_var_val, depth_val, '--')#, c=np.random.rand(3, 1)) else: scatter(main_var_val, depth_val, c=np.random.rand(3, 1)) station_name = ''.join(ma.getdata(dataset.variables['station_name'][dataset.variables['station_index'][i_prof] - 1])) labels.append(station_name) gca().invert_yaxis() title('%s\nCruise: %s' % (dataset.source, dataset.cruise_id)) xlabel('%s: %s in %s' % (main_data.name, main_data.long_name, main_data.units)) ylabel('%s in %s; positive %s' % (dataset.variables['DEPTH'].long_name, dataset.variables['DEPTH'].units, dataset.variables['DEPTH'].positive)) try: legend(labels, loc='upper left', prop=fontP, title='Station') except: pass savefig(plot_output_filepath) plt.close(fig)
def fftDiff2(self, m, n): Nx=1<<(self.Nx-1).bit_length() Ny=1<<(self.Ny-1).bit_length() # h = np.fft.fft2(self.Zarr, s=[Nx, Ny] ) h = np.fft.fft2(ma.getdata(self.Zarr), s=[Nx, Ny] ) hshift = np.fft.fftshift(h) u = np.arange(Nx) - Nx/2. v = np.arange(Ny) - Ny/2. U,V = np.meshgrid(u,v) hdiff = ( (1j*2*np.pi*U/Nx)**m )*( (1j*2*np.pi*V/Ny)**n ) * hshift out_diff = np.real( np.fft.ifft2( np.fft.ifftshift(hdiff) ) )/(self.dx**m)/(self.dy**n) out_diff = out_diff[:self.Ny, :self.Nx] return out_diff
def Fill2ThetaAzimuthMap(masks,TA,tam,image): 'Needs a doc string' Zlim = masks['Thresholds'][1] rings = masks['Rings'] arcs = masks['Arcs'] TA = np.dstack((ma.getdata(TA[1]),ma.getdata(TA[0]),ma.getdata(TA[2]))) #azimuth, 2-theta, dist tax,tay,tad = np.dsplit(TA,3) #azimuth, 2-theta, dist**2/d0**2 for tth,thick in rings: tam = ma.mask_or(tam.flatten(),ma.getmask(ma.masked_inside(tay.flatten(),max(0.01,tth-thick/2.),tth+thick/2.))) for tth,azm,thick in arcs: tamt = ma.getmask(ma.masked_inside(tay.flatten(),max(0.01,tth-thick/2.),tth+thick/2.)) tama = ma.getmask(ma.masked_inside(tax.flatten(),azm[0],azm[1])) tam = ma.mask_or(tam.flatten(),tamt*tama) taz = ma.masked_outside(image.flatten(),int(Zlim[0]),Zlim[1]) tabs = np.ones_like(taz) tam = ma.mask_or(tam.flatten(),ma.getmask(taz)) tax = ma.compressed(ma.array(tax.flatten(),mask=tam)) #azimuth tay = ma.compressed(ma.array(tay.flatten(),mask=tam)) #2-theta taz = ma.compressed(ma.array(taz.flatten(),mask=tam)) #intensity tad = ma.compressed(ma.array(tad.flatten(),mask=tam)) #dist**2/d0**2 tabs = ma.compressed(ma.array(tabs.flatten(),mask=tam)) #ones - later used for absorption corr. return tax,tay,taz,tad,tabs
def interp_data(var, xpts, ypts, xpts2m, ypts2m, int_meth='cubic'): #interpoalte data onto a regular 2d grid. Used by interp_uv data = ma.getdata(var) mask = ma.getmask(var) index = np.where(mask==False) data_masked = data[index] xpoints = xpts[index] ypoints = ypts[index] points = (xpoints, ypoints) var_int = griddata(points,data_masked,(xpts2m,ypts2m),method=int_meth) var_int = ma.masked_array(var_int,np.isnan(var_int)) return var_int
def generate_corrected_map(self, dataid, glbdir, outdir, pers=np.array([]), glbpfx='smpkolya_phv_R_', outpfx='smpkolya_phv_R_'): """ Generate corrected global phave velocity map using a regional phase velocity map. ================================================================================================================= Input Parameters: dataid - dataid for regional phase velocity map glbdir - location of global reference phase velocity map files outdir - output directory pers - period array for correction (default is 4) glbpfx - prefix for global reference phase velocity map files outpfx - prefix for output reference phase velocity map files ----------------------------------------------------------------------------------------------------------------- Output format: outdir/outpfx+str(int(per)) ================================================================================================================= """ if not os.path.isdir(outdir): os.makedirs(outdir) if pers.size==0: pers=np.append( np.arange(7.)*10.+40., np.arange(2.)*25.+125.) for per in pers: inglobalfname=glbdir+'/'+glbpfx+str(int(per)) try: self.get_data4plot(dataid=dataid, period=per) except: print 'No regional data for period =',per,'sec' continue if not os.path.isfile(inglobalfname): print 'No global data for period =',per,'sec' continue outfname=outdir+'/'+outpfx+'%g' %(per) InglbArr=np.loadtxt(inglobalfname) outArr=InglbArr.copy() lonArr=self.lonArr.reshape(self.lonArr.size) latArr=self.latArr.reshape(self.latArr.size) vel_iso=ma.getdata(self.vel_iso) vel_iso=vel_iso.reshape(vel_iso.size) for i in xrange(InglbArr[:,0].size): lonG=InglbArr[i,0] latG=InglbArr[i,1] phVG=InglbArr[i,2] for j in xrange(lonArr.size): lonR=lonArr[j] latR=latArr[j] phVR=vel_iso[j] if abs(lonR-lonG)<0.05 and abs(latR-latG)<0.05 and phVR!=0: outArr[i,2]=phVR np.savetxt(outfname, outArr, fmt='%g %g %.4f') return
def plot_propagation(self, projection='lambert', inbasemap=None, factor=3, showfig=False): """Plot propagation direction """ if inbasemap==None: m=self._get_basemap(projection=projection) else: m=inbasemap if self.lonArr.shape[0]-2==self.grad[0].shape[0] and self.lonArr.shape[1]-2==self.grad[0].shape[1]: self.cut_edge(1,1) elif self.lonArr.shape[0]!=self.grad[0].shape[0] or self.lonArr.shape[1]!=self.grad[0].shape[1]: raise ValueError('Incompatible shape for gradient and lon/lat array!') normArr = np.sqrt ( ma.getdata(self.grad[0] )** 2 + ma.getdata(self.grad[1]) ** 2) x, y=m(self.lonArr, self.latArr) U=self.grad[1]/normArr V=self.grad[0]/normArr if factor!=None: x=x[0:self.Nlat:factor, 0:self.Nlon:factor] y=y[0:self.Nlat:factor, 0:self.Nlon:factor] U=U[0:self.Nlat:factor, 0:self.Nlon:factor] V=V[0:self.Nlat:factor, 0:self.Nlon:factor] Q = m.quiver(x, y, U, V, scale=50, width=0.001) if showfig: plt.show() return
def test_set_fields(self): "Tests setting fields." base = self.base.copy() mbase = base.view(mrecarray) mbase = mbase.copy() mbase.fill_value = (999999, 1e20, "N/A") # Change the data, the mask should be conserved mbase.a._data[:] = 5 assert_equal(mbase["a"]._data, [5, 5, 5, 5, 5]) assert_equal(mbase["a"]._mask, [0, 1, 0, 0, 1]) # Change the elements, and the mask will follow mbase.a = 1 assert_equal(mbase["a"]._data, [1] * 5) assert_equal(ma.getmaskarray(mbase["a"]), [0] * 5) assert_equal(mbase._mask, [False] * 5) assert_equal( mbase._fieldmask.tolist(), np.array([(0, 0, 0), (0, 1, 1), (0, 0, 0), (0, 0, 0), (0, 1, 1)], dtype=bool) ) # Set a field to mask ........................ mbase.c = masked assert_equal(mbase.c.mask, [1] * 5) assert_equal(ma.getmaskarray(mbase["c"]), [1] * 5) assert_equal(ma.getdata(mbase["c"]), ["N/A"] * 5) assert_equal( mbase._fieldmask.tolist(), np.array([(0, 0, 1), (0, 1, 1), (0, 0, 1), (0, 0, 1), (0, 1, 1)], dtype=bool) ) # Set fields by slices ....................... mbase = base.view(mrecarray).copy() mbase.a[3:] = 5 assert_equal(mbase.a, [1, 2, 3, 5, 5]) assert_equal(mbase.a._mask, [0, 1, 0, 0, 0]) mbase.b[3:] = masked assert_equal(mbase.b, base["b"]) assert_equal(mbase.b._mask, [0, 1, 0, 1, 1]) # Set fields globally.......................... ndtype = [("alpha", "|S1"), ("num", int)] data = ma.array([("a", 1), ("b", 2), ("c", 3)], dtype=ndtype) rdata = data.view(MaskedRecords) val = ma.array([10, 20, 30], mask=[1, 0, 0]) # import warnings warnings.simplefilter("ignore") rdata["num"] = val assert_equal(rdata.num, val) assert_equal(rdata.num.mask, [1, 0, 0])
def create_absorption_plot(netcdf_file_path): """create a png from an absorption netcdf file""" plot_output_filepath = os.path.splitext(netcdf_file_path)[0] + '.png' dataset = Dataset(netcdf_file_path) profiles = dataset.variables['profile'] n_obs_per_profile = dataset.variables['row_size'] n_profiles = len(profiles) # look for main variable for varname in dataset.variables.keys(): dim = dataset.variables[varname].dimensions if 'obs' in dim and 'wavelength' in dim: main_data = dataset.variables[varname] fig = figure(num=None, figsize=(15, 10), dpi=80, facecolor='w', edgecolor='k') labels = [] for i_prof in range(n_profiles): # we look for the observations indexes related to the choosen profile # only depth 0 is plotted depth_to_plot = int(0) idx_obs_start = sum(n_obs_per_profile[0:i_prof]) idx_obs_end = idx_obs_start + n_obs_per_profile[i_prof] - 1 idx_obs = range(idx_obs_start, idx_obs_end + 1) main_var_val = main_data[idx_obs] # for i_prof depth_val = dataset.variables['DEPTH'][idx_obs] depth_val = np.array(depth_val).astype(int) wavelength_val = dataset.variables['wavelength'][:] if not any(depth_val == depth_to_plot): continue df = pd.DataFrame(main_var_val[depth_val == depth_to_plot][0].flatten(), index=wavelength_val) plot(df.index, df, '.') station_name = ''.join(ma.getdata(dataset.variables['station_name'][dataset.variables['station_index'][i_prof] - 1])) labels.append(station_name) title('%s\nCruise: %s\n Depth = %sm' % (dataset.source, dataset.cruise_id, depth_to_plot)) ylabel('%s: %s in %s' % (main_data.name, main_data.long_name, main_data.units)) xlabel('%s in %s' % (dataset.variables['wavelength'].long_name, dataset.variables['wavelength'].units)) legend(labels, loc='upper right', prop=fontP, title='Station') savefig(plot_output_filepath) plt.close(fig)
def __call__ (self, two): "Executes the call behavior." # first argument one = self.obj # carry out basic operation, transfer name and units func = getattr(super(Dvect, one), self.f) if self.inp: func(two) result = one else: result = MaskedArray(func(two), subok=True).view(type(one)) result._name = one._name result._units = one._units # handle the errors. They are worked out in a linear approximation # which requires the user to supply two partial derivative functions # in addition to the basic function. if self.fx is not None and self.fy is not None: if isinstance(two, Dvect): if one._err is noerr: result._err = getdata(np.abs(self.fy(one.dat, two.dat))*two._err) elif two._err is noerr: result._err = getdata(np.abs(self.fx(one.dat, two.dat))*one._err) else: if one is two: # Two inputs are identical and not independent if result.mask is nomask: result._err = np.hypot(self.fx(one.dat, two.dat), self.fy(one.dat, two.dat))*one._err else: result._err = np.where(result.mask, fillerr, getdata(np.hypot(self.fx(one.dat, two.dat), self.fy(one.dat, two.dat)*two._err))) pass else: # Two inputs are assumed to be independent. if result.mask is nomask: result._err = getdata(np.hypot(self.fx(one.dat, two.dat)*one._err, self.fy(one.dat, two.dat)*two._err)) else: result._err = np.where(result.mask, fillerr, getdata(np.hypot(self.fx(one.dat, two.dat)*one._err, self.fy(one.dat, two.dat)*two._err))) else: result._err = getdata(np.abs(self.fx(one.dat, two))*one._err) else: result._err = noerr return result
def fftDiff(self, m, n): try: h = np.fft.fft2(ma.getdata(self.Zarr)) except: h = np.fft.fft2(self.Zarr) hshift = np.fft.fftshift(h) Nx=self.Nx Ny=self.Ny if Nx % 2 ==0: u=np.arange(Nx) - Nx/2. else: u=np.arange(Nx) - (Nx-1)/2. if Ny % 2 ==0: v=np.arange(Ny) - Ny/2. else: v=np.arange(Ny) - (Ny-1)/2. U,V=np.meshgrid(u,v) hdiff = ((1j*2*np.pi*U/Nx)**m)*((1j*2*np.pi*V/Ny)**n) * hshift out_diff = np.real( np.fft.ifft2( np.fft.ifftshift(hdiff) ) )/(self.dx**m)/(self.dy**n) out_diff = out_diff[:self.Ny, :self.Nx] return out_diff
def test_units_qc(self): fd, fake_file = tempfile.mkstemp() os.close(fd) self.addCleanup(os.remove, fake_file) nc = Dataset(fake_file, 'w') nc.createDimension('time', 10) timevar = nc.createVariable('time', np.float64, ('time',), fill_value=-9999.) timevar.standard_name = 'time' timevar.units = 'seconds since 1970-01-01T00:00:00Z' timevar[np.array([0, 2, 4, 6, 8])] = np.array([0, 2, 4, 6, 8]) tempvar = nc.createVariable('temp', np.float32, ('time',), fill_value=-9999.) tempvar.standard_name = 'sea_water_temperature' tempvar.units = 'deg_F' tempvar[np.array([0, 1, 2, 3, 4, 9])] = np.array([72.0, 72.1, 72.0, 1.0, 72.03, 72.1]) qc = GliderQC(nc) for qcvarname in qc.create_qc_variables(nc.variables['temp']): qc.apply_qc(nc.variables[qcvarname]) np.testing.assert_equal(nc.variables['qartod_temp_flat_line_flag'][:].mask, ~np.array([1, 0, 1, 0, 1, 0, 0, 0, 0, 0], dtype=bool)) np.testing.assert_equal(ma.getdata(nc.variables['qartod_temp_flat_line_flag'][:]), np.array([1, 9, 1, 9, 1, 9, 9, 9, 9, 9], dtype=np.int8))
def masked_rec_array_to_mgr(data, index, columns, dtype, copy): """ Extract from a masked rec array and create the manager. """ # essentially process a record array then fill it fill_value = data.fill_value fdata = ma.getdata(data) if index is None: index = get_names_from_index(fdata) if index is None: index = ibase.default_index(len(data)) index = ensure_index(index) if columns is not None: columns = ensure_index(columns) arrays, arr_columns = to_arrays(fdata, columns) # fill if needed new_arrays = [] for fv, arr, col in zip(fill_value, arrays, arr_columns): mask = ma.getmaskarray(data[col]) if mask.any(): arr, fv = maybe_upcast(arr, fill_value=fv, copy=True) arr[mask] = fv new_arrays.append(arr) # create the manager arrays, arr_columns = reorder_arrays(new_arrays, arr_columns, columns) if columns is None: columns = arr_columns mgr = arrays_to_mgr(arrays, arr_columns, index, columns, dtype) if copy: mgr = mgr.copy() return mgr
def Laplacian(self, method='default', order=2): Zarr=ma.getdata(self.Zarr) if method == 'default': Zarr_yp=Zarr[2:, 1:-1] Zarr_yn=Zarr[:-2, 1:-1] Zarr_xp=Zarr[1:-1, 2:] Zarr_xn=Zarr[1:-1, :-2] Zarr=Zarr[1:-1, 1:-1] self.lplc=(Zarr_yp+Zarr_yn-2*Zarr) / (self.dy**2) + (Zarr_xp+Zarr_xn-2*Zarr) / (self.dx**2) elif method == 'convolve': if order==2: diff2_x=convolve( ma.getdata(self.Zarr), X_diff2_weight_2)/self.dx/self.dx diff2_y=convolve(ma.getdata(self.Zarr), Y_diff2_weight_2)/self.dy/self.dy elif order==4: diff2_x=convolve( ma.getdata(self.Zarr), X_diff2_weight_4)/self.dx/self.dx diff2_y=convolve(ma.getdata(self.Zarr), Y_diff2_weight_4)/self.dy/self.dy elif order==6: diff2_x=convolve( ma.getdata(self.Zarr), X_diff2_weight_6)/self.dx/self.dx diff2_y=convolve(ma.getdata(self.Zarr), Y_diff2_weight_6)/self.dy/self.dy self.lplc=diff2_x+diff2_y self.lplc=self.lplc[1:-1, 1:-1] return
def loadVOTable(self): try: self.votable = astropy.io.votable.parse_single_table(self.filename) except: return False else: self.columnTitles= [a.name for a in self.votable.fields] rows=ma.getdata(self.votable.array) for row in rows: s=[] for data in row: try: #s.append(decimal.Decimal(data)) s.append(float(data)) except: s.append(str(data)) self.dataValues.append(s) self.calculateArrayColumns() #Assing a delimiter in case of saving as ASCII table self.delimiter="|" return True
def Gradient(self, edge_order=1, method='default', order=2): if method=='default': self.grad=np.gradient( ma.getdata(self.Zarr), self.dx, self.dy, edge_order=edge_order) elif method=='freq': diff_x=self.fftDiff(m=1, n=0) diff_y=self.fftDiff(m=0, n=1) self.grad=[] self.grad.append(diff_y) self.grad.append(diff_x) elif method == 'convolve': if order==2: diff_x=convolve( ma.getdata(self.Zarr), X_diff_weight_2)/self.dx diff_y=convolve(ma.getdata(self.Zarr), Y_diff_weight_2)/self.dy elif order==4: diff_x=convolve(ma.getdata(self.Zarr), X_diff_weight_4)/self.dx diff_y=convolve(ma.getdata(self.Zarr), Y_diff_weight_4)/self.dy elif order==6: diff_x=convolve(ma.getdata(self.Zarr), X_diff_weight_6)/self.dx diff_y=convolve(ma.getdata(self.Zarr), Y_diff_weight_6)/self.dy self.grad=[] self.grad.append(diff_y) self.grad.append(diff_x) return
def GetLplcCorrection(self, per): omega=2.*np.pi/per Zarr=ma.getdata(self.Zarr) self.lplcCo=self.lplc/Zarr[1:-1, 1:-1]/(omega**2) return
def LaplacianEqualXY(self): if self.dx!=self.dy: raise ValueError('grid spacing not equal!') self.lplc=scipy.ndimage.filters.laplace(ma.getdata(self.Zarr) ) / (self.dx*self.dy) self.lplc=self.lplc[1:-1, 1:-1] return