def get_depths(noise_maps, pix_size, mask=None, pixel_weights=None):
"""Compute depth_i and depth_p (sensitivities) from noise maps.
:param noise_maps: the noise maps
:param pix_size: the pixel size in arcmin
:param mask: the mask to apply
:param pixel_weights: weighting of pixels (coverage)
:return: depth_i and depth_p of the map
"""
# apply pixel weights
weighted_maps = np.empty_like(noise_maps)
weighted_maps[...] = noise_maps[...] * pixel_weights
# apply mask
noise_ma = ma.array(weighted_maps, mask=mask)
# noise estimation (in I component) using the noise maps
depth_i = ma.getdata(ma.std(noise_ma[:, 0, :], axis=1))
depth_i *= pix_size
# noise estimation (in Q & U components)
depth_p = ma.getdata(ma.std(noise_ma[:, 1:, :], axis=(1, 2)))
depth_p *= pix_size
return depth_i, depth_p
def plot_propagation(self,
projection='lambert',
inbasemap=None,
factor=3,
showfig=False):
"""Plot propagation direction
"""
if inbasemap == None:
m = self._get_basemap(projection=projection)
else:
m = inbasemap
if self.lonArr.shape[0] - 2 == self.grad[0].shape[
0] and self.lonArr.shape[1] - 2 == self.grad[0].shape[1]:
self.cut_edge(1, 1)
elif self.lonArr.shape[0] != self.grad[0].shape[
0] or self.lonArr.shape[1] != self.grad[0].shape[1]:
raise ValueError(
'Incompatible shape for gradient and lon/lat array!')
normArr = np.sqrt(
ma.getdata(self.grad[0])**2 + ma.getdata(self.grad[1])**2)
x, y = m(self.lonArr, self.latArr)
U = self.grad[1] / normArr
V = self.grad[0] / normArr
if factor != None:
x = x[0:self.Nlat:factor, 0:self.Nlon:factor]
y = y[0:self.Nlat:factor, 0:self.Nlon:factor]
U = U[0:self.Nlat:factor, 0:self.Nlon:factor]
V = V[0:self.Nlat:factor, 0:self.Nlon:factor]
Q = m.quiver(x, y, U, V, scale=50, width=0.001)
if showfig:
plt.show()
return
def update_json(props, varname, data, datatype):
"""
Adds an extra field to the properties of a feature in a geojson file. Provides a consistent way to handle different data types
:param props: dictionary of properties
:param varname: string variable name
:param data: the data to add
:param datatype: datatype
:return: updated props
"""
if (isinstance(data, ma.MaskedArray) and (data.shape == ())
and ("int" in str(datatype))):
props.update({varname: np.int(ma.getdata(data))})
elif (isinstance(data, ma.MaskedArray) and (data.shape == ())
and ("float" in str(datatype))):
props.update({varname: np.float(ma.getdata(data))})
elif isinstance(data, np.float32) or isinstance(data, np.float):
props.update({varname: float(data)})
elif isinstance(data, np.int) or isinstance(data, np.uint16):
props.update({varname: int(data)})
elif str(datatype) == "string":
props.update({varname: str(data)})
else:
return
def Fill2ThetaAzimuthMap(masks, TA, tam, image):
'Needs a doc string'
Zlim = masks['Thresholds'][1]
rings = masks['Rings']
arcs = masks['Arcs']
TA = np.dstack((ma.getdata(TA[1]), ma.getdata(TA[0]),
ma.getdata(TA[2]))) #azimuth, 2-theta, dist
tax, tay, tad = np.dsplit(TA, 3) #azimuth, 2-theta, dist**2/d0**2
for tth, thick in rings:
tam = ma.mask_or(
tam.flatten(),
ma.getmask(
ma.masked_inside(tay.flatten(), max(0.01, tth - thick / 2.),
tth + thick / 2.)))
for tth, azm, thick in arcs:
tamt = ma.getmask(
ma.masked_inside(tay.flatten(), max(0.01, tth - thick / 2.),
tth + thick / 2.))
tama = ma.getmask(ma.masked_inside(tax.flatten(), azm[0], azm[1]))
tam = ma.mask_or(tam.flatten(), tamt * tama)
taz = ma.masked_outside(image.flatten(), int(Zlim[0]), Zlim[1])
tabs = np.ones_like(taz)
tam = ma.mask_or(tam.flatten(), ma.getmask(taz))
tax = ma.compressed(ma.array(tax.flatten(), mask=tam)) #azimuth
tay = ma.compressed(ma.array(tay.flatten(), mask=tam)) #2-theta
taz = ma.compressed(ma.array(taz.flatten(), mask=tam)) #intensity
tad = ma.compressed(ma.array(tad.flatten(), mask=tam)) #dist**2/d0**2
tabs = ma.compressed(ma.array(
tabs.flatten(), mask=tam)) #ones - later used for absorption corr.
return tax, tay, taz, tad, tabs
def plot_field(self, projection='lambert', contour=True, geopolygons=None, showfig=True, vmin=None, vmax=None, stations=False, event=False):
"""Plot data with contour
"""
m=self._get_basemap(projection=projection, geopolygons=geopolygons)
x, y=m(self.lonArr, self.latArr)
if event:
try:
evx, evy=m(self.evlo, self.evla)
m.plot(evx, evy, 'yo', markersize=10)
except: pass
if stations:
try:
stx, sty=m(self.lonArrIn, self.latArrIn)
m.plot(stx, sty, 'y^', markersize=6)
except: pass
try:
stx, sty = m(self.stalons, self.stalats)
m.plot(stx, sty, 'b^', markersize=6)
except: pass
im=m.pcolormesh(x, y, self.Zarr, cmap='gist_ncar_r', shading='gouraud', vmin=vmin, vmax=vmax)
cb = m.colorbar(im, "bottom", size="3%", pad='2%')
cb.ax.tick_params(labelsize=10)
if self.fieldtype=='Tph' or self.fieldtype=='Tgr':
cb.set_label('sec', fontsize=12, rotation=0)
if self.fieldtype=='Amp':
cb.set_label('nm', fontsize=12, rotation=0)
if contour:
# levels=np.linspace(ma.getdata(self.Zarr).min(), ma.getdata(self.Zarr).max(), 20)
levels=np.linspace(ma.getdata(self.Zarr).min(), ma.getdata(self.Zarr).max(), 60)
m.contour(x, y, self.Zarr, colors='k', levels=levels, linewidths=0.5)
if showfig:
plt.show()
return m
def plot_field(self, projection='lambert', contour=True, geopolygons=None, showfig=True, vmin=None, vmax=None):
"""Plot data with contour
"""
m=self._get_basemap(projection=projection, geopolygons=geopolygons)
x, y=m(self.lonArr, self.latArr)
try:
evx, evy=m(self.evlo, self.evla)
m.plot(evx, evy, 'yo', markersize=10)
except:
pass
try:
stx, sty=m(self.lonArrIn, self.latArrIn)
m.plot(stx, sty, 'y^', markersize=10)
except:
pass
im=m.pcolormesh(x, y, self.Zarr, cmap='gist_ncar_r', shading='gouraud', vmin=vmin, vmax=vmax)
cb = m.colorbar(im, "bottom", size="3%", pad='2%')
cb.ax.tick_params(labelsize=10)
if self.fieldtype=='Tph' or self.fieldtype=='Tgr':
cb.set_label('sec', fontsize=12, rotation=0)
if self.fieldtype=='Amp':
cb.set_label('nm', fontsize=12, rotation=0)
if contour:
# levels=np.linspace(ma.getdata(self.Zarr).min(), ma.getdata(self.Zarr).max(), 20)
levels=np.linspace(ma.getdata(self.Zarr).min(), ma.getdata(self.Zarr).max(), 60)
m.contour(x, y, self.Zarr, colors='k', levels=levels, linewidths=0.5)
if showfig:
plt.show()
return
def apply_qc(self, ncvariable):
'''
Applies QC to a qartod variable
:param netCDF4.Variable ncvariable: A QARTOD Variable
'''
qc_tests = {
'flat_line': qc.flat_line_check,
'gross_range': qc.range_check,
'rate_of_change': qc.rate_of_change_check,
'spike': qc.spike_check,
'pressure': gliders_qc.pressure_check
}
qartod_test = getattr(ncvariable, 'qartod_test', None)
if not qartod_test:
return
standard_name = getattr(ncvariable, 'standard_name').split(' ')[0]
parent = self.ncfile.get_variables_by_attributes(
standard_name=standard_name)[0]
times, values, mask = self.get_unmasked(parent)
# There's no data to QC
if len(values) == 0:
return
# If the config isn't set for this test, don't run it.
if qartod_test not in self.config[standard_name]:
return
test_params = {}
if qartod_test in 'rate_of_change':
times = ma.getdata(times[~mask])
time_units = self.ncfile.variables['time'].units
dates = np.array(num2date(times, time_units),
dtype='datetime64[ms]')
test_params['times'] = dates
# Calculate the threshold value
test_params['thresh_val'] = self.get_rate_of_change_threshold(
values, times, time_units)
elif qartod_test == 'spike':
test_params['times'] = ma.getdata(times[~mask])
test_params['low_thresh'], test_params[
'high_thresh'] = self.get_spike_thresholds(values)
else:
test_params = self.config[standard_name][qartod_test]
if qartod_test == 'pressure':
test_params['pressure'] = values
else:
test_params['arr'] = values
qc_flags = qc_tests[qartod_test](**test_params)
ncvariable[~mask] = qc_flags
for test_param in test_params:
if test_param in ('arr', 'times', 'pressure'):
continue
ncvariable.setncattr(test_param, test_params[test_param])
def clipdata(y, xm, x0, x1, minFlag=False):
mx = ma.getdata(mask(xm, xm, x0, x1))
my = ma.getdata(mask(y, xm, x0, x1))
if minFlag: # allow clipping from first minimum after the start time
u = ma.argmin(mask(y, xm, x0, x1))
mx = mx[u:-1]
my = my[u:-1]
return (mx, my)
def clipdata(y, xm, x0, x1, minFlag=False):
mx = ma.getdata(mask(xm, xm, x0, x1))
my = ma.getdata(mask(y, xm, x0, x1))
if minFlag: # allow clipping from first minimum after the start time
u = ma.argmin(mask(y, xm, x0, x1))
mx = mx[u:-1]
my = my[u:-1]
return (mx, my)
def _dense_fit(self, X, strategy, missing_values, fill_value):
"""Fit the transformer on dense data."""
mask = _get_mask(X, missing_values)
X_np = X.to_ndarray()
masked_X = ma.masked_array(X_np, mask=np.isnan(X_np)) # FIXME
# Mean
if strategy == "mean":
mean_masked = ma.mean(masked_X, axis=0)
# Avoid the warning "Warning: converting a masked element to nan."
mean = ma.getdata(mean_masked)
mean[ma.getmask(mean_masked)] = np.nan
return mean
# Median
elif strategy == "median":
median_masked = ma.median(masked_X, axis=0)
# Avoid the warning "Warning: converting a masked element to nan."
median = ma.getdata(median_masked)
median[ma.getmaskarray(median_masked)] = np.nan
return median
# Most frequent
elif strategy == "most_frequent":
# Avoid use of scipy.stats.mstats.mode due to the required
# additional overhead and slow benchmarking performance.
# See Issue 14325 and PR 14399 for full discussion.
# To be able access the elements by columns
X = X.T
mask = mask.T
npdtype = typemap(X.dtype())
if npdtype.kind == "O":
most_frequent = af.constant(0, X.shape[0], dtype=object)
else:
most_frequent = af.constant(0, X.shape[0])
for i, (row, row_mask) in enumerate(zip(X[:], mask[:])):
row_mask = row_mask.logical_not()
row = row[row_mask].to_ndarray()
#most_frequent[i] = _most_frequent(row, np.nan, 0)
most_frequent[i] = _most_frequent(row, np.nan, 0)
return most_frequent
# Constant
elif strategy == "constant":
# for constant strategy, self.statistcs_ is used to store
# fill_value in each column
return af.constant(fill_value, X.shape[1], dtype=X.dtype())
def read_icespeed_nc(sample_step=10, store=True):
'''
read nc file
:return:
'''
nc_file = fileutils.correct_path(settings.ICE_SPEED_NC_FILE)
dataset = Dataset(nc_file, mask_and_scale=False, decode_times=False)
nc_vars = [var for var in dataset.variables]
headers = nc_vars[3:7]
print('Reading speeds from file....')
mask_lat = dataset.variables[headers[0]][:]
print('done with: ' + headers[0])
mask_lon = dataset.variables[headers[1]][:]
print('done with: ' + headers[1])
mask_vel_x = dataset.variables[headers[2]][:]
print('done with: ' + headers[2])
mask_vel_y = dataset.variables[headers[3]][:]
print('done with: ' + headers[3])
print('reading the nonzero...')
nonzeroposition_x = ma.nonzero(mask_vel_x)
nonzeroposition_y = ma.nonzero(mask_vel_y)
print('zip...')
nonzeroposition_zip = zip(nonzeroposition_x[0], nonzeroposition_x[1],
nonzeroposition_y[0], nonzeroposition_y[1])
print('filtering and casting cooridnates....')
nonzeroposition = []
for x in tqdm(nonzeroposition_zip):
if x[0] == x[2] & x[1] == x[3]:
nonzeroposition.append((x[0], x[1]))
lat = []
lon = []
vel_x = []
vel_y = []
print('creatating the dataframe...')
for pos in tqdm(nonzeroposition):
lat.append(ma.getdata(mask_lat[pos[0], pos[1]]))
lon.append(ma.getdata(mask_lon[pos[0], pos[1]]))
vel_x.append(ma.getdata(mask_vel_x[pos[0], pos[1]]))
vel_y.append(ma.getdata(mask_vel_y[pos[0], pos[1]]))
table = {
headers[0]: lat,
headers[1]: lon,
headers[2]: vel_x,
headers[3]: vel_y
}
df = pd.DataFrame(table, columns=headers)
if store:
print('storing in a csv')
df.to_csv(settings.OUTPUT_ICESPEED_CSV, sep=',', encoding='utf-8')
print('DONE')
return df
def getDataOnly(array1d):
"""
Removes redundant no data slots in a 1D array
Note that sometimes data is missing within the array, so in the majority of cases, the array just needs to be shortened, but in a small number of cases, the 'within array' no data needs to be replaced
:param array1d:
:return: Simple array of numbers
"""
if np.any(ma.getmask(array1d)):
mymask = np.invert(ma.getmask(array1d))
outarray = ma.getdata(array1d)[mymask]
else:
outarray = ma.getdata(array1d)
return outarray
def apply_on_fields(series, func, *args, **kwargs):
"""
Apply the function ``func`` to each field of ``series``.
Parameters
----------
series : array-like
A (subclass) of ndarray.
func : function
Function to apply.
args : var
Additional arguments to ``func``.
kwargs : var
Additional optional parameters to ``func``
"""
names = series.dtype.names
if names is None:
return func(series, *args, **kwargs)
results = [func(series[f], *args, **kwargs) for f in names]
rdtype = [(f[0], r.dtype) for (f, r) in zip(series.dtype.descr, results)]
rdata = [ma.getdata(r) for r in results]
rmask = [ma.getmaskarray(r) for r in results]
isarray = isinstance(results[0], np.ndarray)
if isarray:
output = ma.array(zip(*rdata), mask=zip(*rmask),
dtype=rdtype).view(type=type(series))
output._update_from(series)
else:
output = ma.array(tuple(rdata), mask=tuple(rmask), dtype=rdtype)
return output
def filter_invalid_value(data_array):
""" This filter applies a mask to all numerically invalid inputs
on a programing side.
Numbers that are usually infinite or some other nonsensical
quantity serve no real usage in calculations further downstream.
Therefore, they are masked here.
See numpy.ma.fix_invalid for what is considered invalid.
Parameters
----------
data_array : ndarray
The data array that the mask will be calculated from.
Returns
-------
final_mask : ndarray -> dictionary
A boolean array for pixels that are masked (True) or are
valid (False).
"""
# As fixing all invalid data is required, masks might obscure
# the data itself.
raw_data_array = np_ma.getdata(data_array)
# Mask all of the invalid data.
final_mask = np_ma.getmaskarray(np_ma.fix_invalid(raw_data_array))
return final_mask
def individual_ftp(kk, args):
dset_ftp_d, resfactor, gray, mask_g, ref_m_gray, N_iter_max, n, th, c_rect, sl_rect, mask_of_annulus_alone, lin_min_idx_a, lin_max_idx_a, col_min_idx_a, col_max_idx_a = args
# dset_ftp_d, resfactor, gray, mask_g, ref_m_gray, N_iter_max, n, th, c_rect, sl_rect, kk = args
def_image = dset_ftp_d[:, :, kk]
# 1. Undistort image
# def_image = undistort_image(def_image, mapx, mapy)
# 2. Substract gray
def_m_gray = def_image - resfactor * gray
#ELIMINO LOS REFLEJOS A MANO
#def_m_gray[(def_m_gray<-30)] = 0
#def_m_gray[(def_m_gray>30)] = 0
# 3. Extrapolate fringes
def_m_gray = gerchberg2d(def_m_gray, mask_g, N_iter_max=N_iter_max)
# 4. Process by FTP
dphase = calculate_phase_diff_map_1D(def_m_gray, ref_m_gray, \
th, n)
dphase_rect = np.mean( dphase[(c_rect[0]-sl_rect):(c_rect[0]+sl_rect), \
(c_rect[1]-sl_rect):(c_rect[1]+sl_rect)] )
# 5a). Time unwrapping using rectangle area
# 5b). Substract solid movement
dphase = dphase - dphase_rect
# 6. Calculate height field
height = dphase #height_map_from_phase_map(dphase, L, D, pspp)
height = (ma.getdata(height) *
mask_of_annulus_alone)[lin_min_idx_a:lin_max_idx_a,
col_min_idx_a:col_max_idx_a]
return height
def make_gene_map_1(self):
count = 0
self.iterator = itertools.product(range(self.time_len), range(self.lat_len), range(self.lon_len))
for x_valid in self.iterator:
binary_string = bin(count)[2:]
while len(binary_string) < self.string_length:
binary_string = "0" + binary_string
# self.gene_map[binary_string] = {}
# self.gene_map[binary_string]['coordinate'] = tuple(x_valid)
if ma.getdata(self.array[x_valid]) < 100: # chooses unmasked points?
self.gene_map[binary_string] = {}
self.gene_map[binary_string]["coordinate"] = tuple(x_valid)
self.gene_map[binary_string]["value"] = self.array[x_valid]
# count += 1
print count, binary_string, self.gene_map[binary_string]["value"] # debug
else:
# self.gene_map[binary_string]['value'] = 1E06
pass
count += 1
# print count, binary_string, self.gene_map[binary_string]['value'] # debug
self.last_valid_binary_string = binary_string
not_valid_first = eval("0b" + binary_string) + 1
not_valid_last = eval("0b" + "1" * self.string_length)
print not_valid_last
print x_valid
print ma.isMA(self.array)
for x_not_valid in range(not_valid_first, not_valid_last + 1):
binary_string = bin(x_not_valid)[2:]
self.gene_map[binary_string] = {}
self.gene_map[binary_string]["coordinate"] = (999, 999, 999)
self.gene_map[binary_string]["value"] = 1e06
print x_not_valid, binary_string, self.gene_map[binary_string]["value"]
def putdata_raw(imgname, data, clone=None):
"""Store (overwrite) data in an existing CASA image.
See getdata_raw(imgname) for the reverse operation.
Parameters
----------
imagename : str
The (absolute) CASA image filename. It should exist
already, unless **clone** was given.
data : 2D numpy array or a list of 2D numpy arrays
The data...
clone : str, optional
An optional filename from which to clone the image
for output. It needs to be an absolute filename.
"""
if clone != None:
taskinit.ia.fromimage(infile=clone,outfile=imgname,overwrite=True)
taskinit.ia.close()
# @todo this seems circumvent to have to borrow the odd dimensions (nx,ny,1,1,1) shape was seen
if type(data) == type([]):
# @todo since this needs to extend the axes, the single plane clone and replace data doesn't work here
raise Exception,"Not Implemented Yet"
bigim = ia.imageconcat(outfile=imgname, infiles=infiles, axis=2, relax=T, tempclose=F, overwrite=T)
bigim.close()
else:
taskinit.tb.open(imgname,nomodify=False)
d = taskinit.tb.getcol('map')
pdata = ma.getdata(data).reshape(d.shape)
taskinit.tb.putcol('map',pdata)
taskinit.tb.flush()
taskinit.tb.close()
return
def get_unmasked(self, ncvariable):
times = self.ncfile.variables['time'][:]
values = ncvariable[:]
mask = np.zeros(times.shape[0], dtype=bool)
if hasattr(values, 'mask'):
mask |= values.mask
if hasattr(times, 'mask'):
mask |= times.mask
values_initial = ma.getdata(values[~mask])
config = self.get_config(ncvariable.name)
units = getattr(ncvariable, 'units', '1')
# If units are not defined or empty, set them to unitless
# If the config units are empty, do not attempt to convert units
# The latter is necessary as some of the NetCDF files do not have
# units attribute under the udunits variable definitions
if not units or pd.isnull(config.units):
units = '1'
values = values_initial
# must be a CF unit or this will throw an exception
elif ncvariable.units != config.units:
try:
values = Unit(units).convert(values_initial, config.units)
except ValueError as e:
exc_text = "Caught exception while converting units: {}".format(
e)
get_logger().warn(exc_text)
values = values_initial
return times, values, mask
def apply_on_fields(series, func, *args, **kwargs):
"""
Apply the function ``func`` to each field of ``series``.
Parameters
----------
series : array-like
A (subclass) of ndarray.
func : function
Function to apply.
args : var
Additional arguments to ``func``.
kwargs : var
Additional optional parameters to ``func``
"""
names = series.dtype.names
if names is None:
return func(series, *args, **kwargs)
results = [func(series[f], *args, **kwargs) for f in names]
rdtype = [(f[0], r.dtype) for (f, r) in zip(series.dtype.descr, results)]
rdata = [ma.getdata(r) for r in results]
rmask = [ma.getmaskarray(r) for r in results]
isarray = isinstance(results[0], np.ndarray)
if isarray:
output = ma.array(zip(*rdata), mask=zip(*rmask),
dtype=rdtype).view(type=type(series))
output._update_from(series)
else:
output = ma.array(tuple(rdata), mask=tuple(rmask), dtype=rdtype)
return output
def create_ac9_hs6_plot(netcdf_file_path):
"""create a png from a ac9/hs6 netcdf file"""
plot_output_filepath = os.path.splitext(netcdf_file_path)[0] + '.png'
dataset = Dataset(netcdf_file_path)
n_wavelength = dataset.variables['wavelength'].shape[0]
# look for main variable
for varname in dataset.variables.keys():
dim = dataset.variables[varname].dimensions
if 'obs' in dim and 'wavelength' in dim:
main_data = dataset.variables[varname]
fig = figure(num=None, figsize=(15, 10), dpi=80, facecolor='w', edgecolor='k')
labels = []
# only one profile per file in AC9 HS6 files
for i_wl in range(n_wavelength):
main_var_val = main_data[:, i_wl] # for i_prof
depth_val = dataset.variables['DEPTH'][:]
wavelength_val = dataset.variables['wavelength'][i_wl]
plot(main_var_val, depth_val)#, c=np.random.rand(3, 1))
labels.append('%s nm' % wavelength_val)
station_name = ''.join(ma.getdata(dataset.variables['station_name'][dataset.variables['station_index'][0] - 1]))
title('%s\nCruise: %s\n Station %s' % (dataset.source, dataset.cruise_id, station_name))
gca().invert_yaxis()
xlabel('%s: %s in %s' % (main_data.name, main_data.long_name, main_data.units))
ylabel('%s in %s; positive %s' % (dataset.variables['DEPTH'].long_name,
dataset.variables['DEPTH'].units,
dataset.variables['DEPTH'].positive))
legend(labels, loc='upper right', prop=fontP, title='Wavelength')
savefig(plot_output_filepath)
plt.close(fig)
def test_units_qc(self):
fd, fake_file = tempfile.mkstemp()
os.close(fd)
self.addCleanup(os.remove, fake_file)
nc = Dataset(fake_file, 'w')
nc.createDimension('time', 10)
timevar = nc.createVariable('time',
np.float64, ('time', ),
fill_value=-9999.)
timevar.standard_name = 'time'
timevar.units = 'seconds since 1970-01-01T00:00:00Z'
timevar[np.array([0, 2, 4, 6, 8])] = np.array([0, 2, 4, 6, 8])
tempvar = nc.createVariable('temp',
np.float32, ('time', ),
fill_value=-9999.)
tempvar.standard_name = 'sea_water_temperature'
tempvar.units = 'deg_F'
tempvar[np.array([0, 1, 2, 3, 4,
9])] = np.array([72.0, 72.1, 72.0, 1.0, 72.03, 72.1])
qc = GliderQC(nc, 'data/qc_config.yml')
for qcvarname in qc.create_qc_variables(nc.variables['temp']):
qc.apply_qc(nc.variables[qcvarname])
np.testing.assert_equal(
nc.variables['qartod_temp_flat_line_flag'][:].mask,
~np.array([1, 0, 1, 0, 1, 0, 0, 0, 0, 0], dtype=bool))
np.testing.assert_equal(
ma.getdata(nc.variables['qartod_temp_flat_line_flag'][:]),
np.array([1, 9, 1, 9, 1, 9, 9, 9, 9, 9], dtype=np.int8))
def apply_primary_qc(self, ncvariable):
'''
Applies the primary QC array which is an aggregate of all the other QC
tests.
:param netCDF4.Variable ncvariable: NCVariable
'''
primary_qc_name = 'qartod_%s_primary_flag' % ncvariable.name
if primary_qc_name not in self.ncfile.variables:
return
qcvar = self.ncfile.variables[primary_qc_name]
# Only perform primary QC on variables created by DAC
if getattr(qcvar, 'dac_comment', '') != 'ioos_qartod_primary':
return
qc_variables = self.find_qc_flags(ncvariable)
vectors = []
for qc_variable in qc_variables:
ncvar = self.ncfile.variables[qc_variable]
if getattr(ncvar, 'dac_comment', '') != 'ioos_qartod':
continue
log.info("Using %s in primary QC for %s", qc_variable, primary_qc_name)
vectors.append(ma.getdata(ncvar[:]))
log.info("Applying QC for %s", primary_qc_name)
flags = qc.qc_compare(vectors)
qcvar[:] = flags
def LaplacianEqualXY(self):
if self.dx != self.dy:
raise ValueError('grid spacing not equal!')
self.lplc = scipy.ndimage.filters.laplace(ma.getdata(
self.Zarr)) / (self.dx * self.dy)
self.lplc = self.lplc[1:-1, 1:-1]
return
def ma2np(self):
"""Convert all the maksed data array to numpy array
"""
self.Zarr=ma.getdata(self.Zarr)
try:
self.diffaArr=ma.getdata(self.diffaArr)
except:
pass
try:
self.appV=ma.getdata(self.appV)
except:
pass
try:
self.lplc=ma.getdata(self.lplc)
except:
pass
return
def height_annulus_polar_coordinates(frame, mask_annulus, phase_width=3000):
"""
Returns a matrix with the height of the annulus in polar coordinates.
"""
# mask_annulus = mask[:,:,0]
annulus = frame
annulus_polar = img2polar(ma.masked_array(annulus,
mask=(1 - mask_annulus)))
first_idx, last_idx = find_index_of_false_values(annulus_polar)
if first_idx != last_idx:
annulus_polar_lines = ma.getdata(
annulus_polar[first_idx:last_idx, :]) #por si toma mal la mascara
prom = np.average(annulus_polar_lines, axis=0)
else:
annulus_polar_lines = ma.getdata(annulus_polar[first_idx, :])
prom = annulus_polar_lines
return prom
def data_bruta(data,x):
VAR = False
BB = data[x].get('reduct')
data2 = ma.getdata(BB)[BB.mask == VAR]
poly = np.repeat(x , len(data2))
dataf = {'poly':poly, 'data':data2}
dataf = pd.DataFrame(dataf)
return dataf
def ma2np(self):
"""Convert all the maksed data array to numpy array
"""
self.Zarr = ma.getdata(self.Zarr)
try:
self.diffaArr = ma.getdata(self.diffaArr)
except:
pass
try:
self.appV = ma.getdata(self.appV)
except:
pass
try:
self.lplc = ma.getdata(self.lplc)
except:
pass
return
def generate_corrected_map(self,
dataid,
glbdir,
outdir,
pers=np.array([]),
glbpfx='smpkolya_phv_R_',
outpfx='smpkolya_phv_R_'):
"""
Generate corrected global phave velocity map using a regional phase velocity map.
=================================================================================================================
Input Parameters:
dataid - dataid for regional phase velocity map
glbdir - location of global reference phase velocity map files
outdir - output directory
pers - period array for correction (default is 4)
glbpfx - prefix for global reference phase velocity map files
outpfx - prefix for output reference phase velocity map files
-----------------------------------------------------------------------------------------------------------------
Output format:
outdir/outpfx+str(int(per))
=================================================================================================================
"""
if not os.path.isdir(outdir):
os.makedirs(outdir)
if pers.size == 0:
pers = np.append(
np.arange(7.) * 10. + 40.,
np.arange(2.) * 25. + 125.)
for per in pers:
inglobalfname = glbdir + '/' + glbpfx + str(int(per))
try:
self.get_data4plot(dataid=dataid, period=per)
except:
print 'No regional data for period =', per, 'sec'
continue
if not os.path.isfile(inglobalfname):
print 'No global data for period =', per, 'sec'
continue
outfname = outdir + '/' + outpfx + '%g' % (per)
InglbArr = np.loadtxt(inglobalfname)
outArr = InglbArr.copy()
lonArr = self.lonArr.reshape(self.lonArr.size)
latArr = self.latArr.reshape(self.latArr.size)
vel_iso = ma.getdata(self.vel_iso)
vel_iso = vel_iso.reshape(vel_iso.size)
for i in xrange(InglbArr[:, 0].size):
lonG = InglbArr[i, 0]
latG = InglbArr[i, 1]
phVG = InglbArr[i, 2]
for j in xrange(lonArr.size):
lonR = lonArr[j]
latR = latArr[j]
phVR = vel_iso[j]
if abs(lonR - lonG) < 0.05 and abs(
latR - latG) < 0.05 and phVR != 0:
outArr[i, 2] = phVR
np.savetxt(outfname, outArr, fmt='%g %g %.4f')
return
def Gradient(self, edge_order=1, method='default', order=2):
if method == 'default':
self.grad = np.gradient(ma.getdata(self.Zarr),
self.dx,
self.dy,
edge_order=edge_order)
elif method == 'freq':
diff_x = self.fftDiff(m=1, n=0)
diff_y = self.fftDiff(m=0, n=1)
self.grad = []
self.grad.append(diff_y)
self.grad.append(diff_x)
elif method == 'convolve':
if order == 2:
diff_x = convolve(ma.getdata(self.Zarr),
X_diff_weight_2) / self.dx
diff_y = convolve(ma.getdata(self.Zarr),
Y_diff_weight_2) / self.dy
elif order == 4:
diff_x = convolve(ma.getdata(self.Zarr),
X_diff_weight_4) / self.dx
diff_y = convolve(ma.getdata(self.Zarr),
Y_diff_weight_4) / self.dy
elif order == 6:
diff_x = convolve(ma.getdata(self.Zarr),
X_diff_weight_6) / self.dx
diff_y = convolve(ma.getdata(self.Zarr),
Y_diff_weight_6) / self.dy
self.grad = []
self.grad.append(diff_y)
self.grad.append(diff_x)
return
def Laplacian(self, method='default', order=2):
Zarr = ma.getdata(self.Zarr)
if method == 'default':
Zarr_yp = Zarr[2:, 1:-1]
Zarr_yn = Zarr[:-2, 1:-1]
Zarr_xp = Zarr[1:-1, 2:]
Zarr_xn = Zarr[1:-1, :-2]
Zarr = Zarr[1:-1, 1:-1]
self.lplc = (Zarr_yp + Zarr_yn - 2 * Zarr) / (self.dy**2) + (
Zarr_xp + Zarr_xn - 2 * Zarr) / (self.dx**2)
elif method == 'convolve':
if order == 2:
diff2_x = convolve(ma.getdata(self.Zarr),
X_diff2_weight_2) / self.dx / self.dx
diff2_y = convolve(ma.getdata(self.Zarr),
Y_diff2_weight_2) / self.dy / self.dy
elif order == 4:
diff2_x = convolve(ma.getdata(self.Zarr),
X_diff2_weight_4) / self.dx / self.dx
diff2_y = convolve(ma.getdata(self.Zarr),
Y_diff2_weight_4) / self.dy / self.dy
elif order == 6:
diff2_x = convolve(ma.getdata(self.Zarr),
X_diff2_weight_6) / self.dx / self.dx
diff2_y = convolve(ma.getdata(self.Zarr),
Y_diff2_weight_6) / self.dy / self.dy
self.lplc = diff2_x + diff2_y
self.lplc = self.lplc[1:-1, 1:-1]
return
def _calculateStats(self, di, dof=0.):
"""Calculate the core QA statistics on a difference image"""
mask = di.getMask()
maskArr = di.getMask().getArray()
# Create a mask using BAD, SAT, NO_DATA, EDGE bits. Keep detections
maskArr &= mask.getPlaneBitMask(["BAD", "SAT", "NO_DATA", "EDGE"])
# Mask out values based on maskArr
diArr = ma.array(di.getImage().getArray(), mask=maskArr)
varArr = ma.array(di.getVariance().getArray(), mask=maskArr)
# Normalize by sqrt variance, units are in sigma
diArr /= np.sqrt(varArr)
mean = diArr.mean()
# This is the maximum-likelihood extimate of the variance stdev**2
stdev = diArr.std()
median = ma.extras.median(diArr)
# Compute IQR of just un-masked data
data = ma.getdata(diArr[~diArr.mask])
iqr = np.percentile(data, 75.) - np.percentile(data, 25.)
#Calculte chisquare of the residual
chisq=np.sum(np.power(data, 2.))
# Mean squared error: variance + bias**2
# Bias = |data - model| = mean of diffim
# Variance = |(data - model)**2| = mean of diffim**2
bias = mean
variance = np.power(data, 2.).mean()
mseResids = bias**2 + variance
# If scipy is not set up, return zero for the stats
try:
#In try block because of risk of divide by zero
rchisq=chisq/(len(data)-1-dof)
# K-S test on the diffim to a Normal distribution
import scipy.stats
D, prob = scipy.stats.kstest(data, 'norm')
A2, crit, sig = scipy.stats.anderson(data, 'norm')
# Anderson Darling statistic cand be inf for really non-Gaussian distributions.
if np.isinf(A2) or np.isnan(A2):
A2 = 9999.
except ZeroDivisionError:
D = 0.
prob = 0.
A2 = 0.
crit = np.zeros(5)
sig = np.zeros(5)
rchisq = 0
return {"mean": mean, "stdev": stdev, "median": median, "iqr": iqr,
"D": D, "prob": prob, "A2": A2, "crit": crit, "sig": sig,
"rchisq": rchisq, "mseResids": mseResids}
def _calculateStats(self, di, dof=0.):
"""Calculate the core QA statistics on a difference image"""
mask = di.getMask()
maskArr = di.getMask().getArray()
# Create a mask using BAD, SAT, NO_DATA, EDGE bits. Keep detections
maskArr &= mask.getPlaneBitMask(["BAD", "SAT", "NO_DATA", "EDGE"])
# Mask out values based on maskArr
diArr = ma.array(di.getImage().getArray(), mask=maskArr)
varArr = ma.array(di.getVariance().getArray(), mask=maskArr)
# Normalize by sqrt variance, units are in sigma
diArr /= np.sqrt(varArr)
mean = diArr.mean()
# This is the maximum-likelihood extimate of the variance stdev**2
stdev = diArr.std()
median = ma.extras.median(diArr)
# Compute IQR of just un-masked data
data = ma.getdata(diArr[~diArr.mask])
iqr = np.percentile(data, 75.) - np.percentile(data, 25.)
# Calculte chisquare of the residual
chisq = np.sum(np.power(data, 2.))
# Mean squared error: variance + bias**2
# Bias = |data - model| = mean of diffim
# Variance = |(data - model)**2| = mean of diffim**2
bias = mean
variance = np.power(data, 2.).mean()
mseResids = bias**2 + variance
# If scipy is not set up, return zero for the stats
try:
# In try block because of risk of divide by zero
rchisq = chisq/(len(data) - 1 - dof)
# K-S test on the diffim to a Normal distribution
import scipy.stats
D, prob = scipy.stats.kstest(data, 'norm')
A2, crit, sig = scipy.stats.anderson(data, 'norm')
# Anderson Darling statistic cand be inf for really non-Gaussian distributions.
if np.isinf(A2) or np.isnan(A2):
A2 = 9999.
except ZeroDivisionError:
D = 0.
prob = 0.
A2 = 0.
crit = np.zeros(5)
sig = np.zeros(5)
rchisq = 0
return {"mean": mean, "stdev": stdev, "median": median, "iqr": iqr,
"D": D, "prob": prob, "A2": A2, "crit": crit, "sig": sig,
"rchisq": rchisq, "mseResids": mseResids}
def ma2np(self):
"""Convert the maksed data array to numpy data array
"""
try:
self.mask = self.Zarr.mask
self.Zarr = ma.getdata(self.Zarr)
except:
print 'Data array is already numpy array'
return
def test_set_fields(self):
# Tests setting fields.
base = self.base.copy()
mbase = base.view(mrecarray)
mbase = mbase.copy()
mbase.fill_value = (999999, 1e20, 'N/A')
# Change the data, the mask should be conserved
mbase.a._data[:] = 5
assert_equal(mbase['a']._data, [5, 5, 5, 5, 5])
assert_equal(mbase['a']._mask, [0, 1, 0, 0, 1])
# Change the elements, and the mask will follow
mbase.a = 1
assert_equal(mbase['a']._data, [1]*5)
assert_equal(ma.getmaskarray(mbase['a']), [0]*5)
# Use to be _mask, now it's recordmask
assert_equal(mbase.recordmask, [False]*5)
assert_equal(mbase._mask.tolist(),
np.array([(0, 0, 0),
(0, 1, 1),
(0, 0, 0),
(0, 0, 0),
(0, 1, 1)],
dtype=bool))
# Set a field to mask ........................
mbase.c = masked
# Use to be mask, and now it's still mask !
assert_equal(mbase.c.mask, [1]*5)
assert_equal(mbase.c.recordmask, [1]*5)
assert_equal(ma.getmaskarray(mbase['c']), [1]*5)
assert_equal(ma.getdata(mbase['c']), [asbytes('N/A')]*5)
assert_equal(mbase._mask.tolist(),
np.array([(0, 0, 1),
(0, 1, 1),
(0, 0, 1),
(0, 0, 1),
(0, 1, 1)],
dtype=bool))
# Set fields by slices .......................
mbase = base.view(mrecarray).copy()
mbase.a[3:] = 5
assert_equal(mbase.a, [1, 2, 3, 5, 5])
assert_equal(mbase.a._mask, [0, 1, 0, 0, 0])
mbase.b[3:] = masked
assert_equal(mbase.b, base['b'])
assert_equal(mbase.b._mask, [0, 1, 0, 1, 1])
# Set fields globally..........................
ndtype = [('alpha', '|S1'), ('num', int)]
data = ma.array([('a', 1), ('b', 2), ('c', 3)], dtype=ndtype)
rdata = data.view(MaskedRecords)
val = ma.array([10, 20, 30], mask=[1, 0, 0])
with warnings.catch_warnings():
warnings.simplefilter("ignore")
rdata['num'] = val
assert_equal(rdata.num, val)
assert_equal(rdata.num.mask, [1, 0, 0])
def test_set_fields(self):
# Tests setting fields.
base = self.base.copy()
mbase = base.view(mrecarray)
mbase = mbase.copy()
mbase.fill_value = (999999, 1e20, 'N/A')
# Change the data, the mask should be conserved
mbase.a._data[:] = 5
assert_equal(mbase['a']._data, [5, 5, 5, 5, 5])
assert_equal(mbase['a']._mask, [0, 1, 0, 0, 1])
# Change the elements, and the mask will follow
mbase.a = 1
assert_equal(mbase['a']._data, [1]*5)
assert_equal(ma.getmaskarray(mbase['a']), [0]*5)
# Use to be _mask, now it's recordmask
assert_equal(mbase.recordmask, [False]*5)
assert_equal(mbase._mask.tolist(),
np.array([(0, 0, 0),
(0, 1, 1),
(0, 0, 0),
(0, 0, 0),
(0, 1, 1)],
dtype=bool))
# Set a field to mask ........................
mbase.c = masked
# Use to be mask, and now it's still mask !
assert_equal(mbase.c.mask, [1]*5)
assert_equal(mbase.c.recordmask, [1]*5)
assert_equal(ma.getmaskarray(mbase['c']), [1]*5)
assert_equal(ma.getdata(mbase['c']), [asbytes('N/A')]*5)
assert_equal(mbase._mask.tolist(),
np.array([(0, 0, 1),
(0, 1, 1),
(0, 0, 1),
(0, 0, 1),
(0, 1, 1)],
dtype=bool))
# Set fields by slices .......................
mbase = base.view(mrecarray).copy()
mbase.a[3:] = 5
assert_equal(mbase.a, [1, 2, 3, 5, 5])
assert_equal(mbase.a._mask, [0, 1, 0, 0, 0])
mbase.b[3:] = masked
assert_equal(mbase.b, base['b'])
assert_equal(mbase.b._mask, [0, 1, 0, 1, 1])
# Set fields globally..........................
ndtype = [('alpha', '|S1'), ('num', int)]
data = ma.array([('a', 1), ('b', 2), ('c', 3)], dtype=ndtype)
rdata = data.view(MaskedRecords)
val = ma.array([10, 20, 30], mask=[1, 0, 0])
with warnings.catch_warnings():
warnings.simplefilter("ignore")
rdata['num'] = val
assert_equal(rdata.num, val)
assert_equal(rdata.num.mask, [1, 0, 0])
def plot(self, ds=1000, unit='km', cmap='seismic_r', vmin=None, vmax=None, zsize=10):
"""Plot velocity model
=============================================================================
Input Parameters:
ds - grid spacing
unit - unit
vmin, vmax - vmin,vmax for colorbar
=============================================================================
"""
# XLength=self.xmax-self.xmin
# ZLength=self.zmax-self.zmin
# xsize=zsize*(XLength/ZLength)
# fig = plt.figure(figsize=(xsize, zsize))
if cmap=='ses3d':
cmap = colormaps.make_colormap({0.0:[0.1,0.0,0.0], 0.2:[0.8,0.0,0.0], 0.3:[1.0,0.7,0.0],0.48:[0.92,0.92,0.92],
0.5:[0.92,0.92,0.92], 0.52:[0.92,0.92,0.92], 0.7:[0.0,0.6,0.7], 0.8:[0.0,0.0,0.8], 1.0:[0.0,0.0,0.1]})
if self.plotflag==False:
raise ValueError('No plot array!')
# plt.figure(figsize=(16,13))
if self.regular==True:
im=plt.pcolormesh(self.XArrPlot/ds, self.ZArrPlot/ds, self.VsArrPlot/ds, cmap=cmap, vmin=vmin, vmax=vmax)
else:
xi = np.linspace(self.xmin, self.xmax, self.Nx*10)
zi = np.linspace(self.zmin, self.zmax, self.Nz*10)
self.xi, self.zi = np.meshgrid(xi, zi)
#-- Interpolating at the points in xi, yi
self.vi = griddata(self.XArr, self.ZArr, self.VsArr, self.xi, self.zi, 'linear')
im=plt.pcolormesh(self.xi/ds, self.zi/ds, ma.getdata(self.vi)/ds, cmap=cmap, vmin=vmin, vmax=vmax, shading='gouraud')
##########################################
plt.plot(500., 1000 , 'r*', markersize=30, lw=3)
# plt.plot( [0., 4000.], [1000, 1000] , 'b--', lw=3)
# plt.plot( [500., 500.], [700., 1300.] , 'g-', lw=3)
# plt.plot( [500., 3500.], [700., 700.] , 'g-', lw=3)
# plt.plot( [500., 3500.], [1300., 1300.] , 'g-', lw=3)
# plt.plot( [3500., 3500.], [700., 1300.] , 'g-', lw=3)
#
# plt.plot( [0., 0.], [0., 2000.] , 'k-', lw=3)
# plt.plot( [0., 4000.], [0., 0.] , 'k-', lw=3)
# plt.plot( [4000., 4000.], [0., 2000.] , 'k-', lw=3)
# plt.plot( [0., 4000.], [2000., 2000.] , 'k-', lw=3)
##########################################
plt.xlabel('x('+unit+')', fontsize=35)
plt.ylabel('z('+unit+')', fontsize=35)
# plt.axis([self.xmin/ds, self.xmax/ds, self.zmin/ds, self.zmax/ds])
plt.axis('scaled')
cb=plt.colorbar(shrink=0.8)#, size="3%", pad='2%')
cb.set_label('Vs (km/s)', fontsize=20, rotation=90)
plt.yticks(fontsize=30)
plt.xticks(fontsize=30)
########################
# plt.ylim([-100, 2100])
# plt.xlim([-100, 4100])
########################
plt.show()
return
def fromarrays(
arraylist,
dtype=None,
shape=None,
formats=None,
names=None,
titles=None,
aligned=False,
byteorder=None,
fill_value=None,
):
"""
Creates a mrecarray from a (flat) list of masked arrays.
Parameters
----------
arraylist : sequence
A list of (masked) arrays. Each element of the sequence is first converted
to a masked array if needed. If a 2D array is passed as argument, it is
processed line by line
dtype : {None, dtype}, optional
Data type descriptor.
shape : {None, integer}, optional
Number of records. If None, shape is defined from the shape of the
first array in the list.
formats : {None, sequence}, optional
Sequence of formats for each individual field. If None, the formats will
be autodetected by inspecting the fields and selecting the highest dtype
possible.
names : {None, sequence}, optional
Sequence of the names of each field.
fill_value : {None, sequence}, optional
Sequence of data to be used as filling values.
Notes
-----
Lists of tuples should be preferred over lists of lists for faster processing.
"""
datalist = [getdata(x) for x in arraylist]
masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist]
_array = recfromarrays(
datalist,
dtype=dtype,
shape=shape,
formats=formats,
names=names,
titles=titles,
aligned=aligned,
byteorder=byteorder,
).view(mrecarray)
_array._mask.flat = list(zip(*masklist))
if fill_value is not None:
_array.fill_value = fill_value
return _array
def forward_fill(marr, maxgap=None):
"""
Forward fills masked values in a 1-d array when there are less ``maxgap``
consecutive masked values.
Parameters
----------
marr : MaskedArray
Series to fill
maxgap : {int}, optional
Maximum gap between consecutive masked values.
If ``maxgap`` is not specified, all masked values are forward-filled.
Examples
--------
>>> x = ma.arange(20)
>>> x[(x%5)!=0] = ma.masked
>>> print x
[0 -- -- -- -- 5 -- -- -- -- 10 -- -- -- -- 15 -- -- -- --]
>>> print forward_fill(x)
[0 0 0 0 0 5 5 5 5 5 10 10 10 10 10 15 15 15 15 15]
"""
# !!!: We should probably port that to C.
# Initialization ..................
if np.ndim(marr) > 1:
raise ValueError,"The input array should be 1D only!"
a = ma.array(marr, copy=True)
amask = getmask(a)
if amask is nomask or a.size == 0:
return a
#
adata = getdata(a)
# Get the indices of the masked values (except a[0])
idxtofill = amask[1:].nonzero()[0] + 1
currGap = 0
if maxgap is not None:
previdx = -1
for i in idxtofill:
if i != previdx + 1:
currGap = 0
currGap += 1
if currGap <= maxgap and not amask[i-1]:
adata[i] = adata[i-1]
amask[i] = False
previdx = i
else:
amask[i-maxgap:i] = True
else:
for i in idxtofill:
if not amask[i-1]:
adata[i] = adata[i-1]
amask[i] = False
return a
def forward_fill(marr, maxgap=None):
"""
Forward fills masked values in a 1-d array when there are less ``maxgap``
consecutive masked values.
Parameters
----------
marr : MaskedArray
Series to fill
maxgap : {int}, optional
Maximum gap between consecutive masked values.
If ``maxgap`` is not specified, all masked values are forward-filled.
Examples
--------
>>> x = ma.arange(20)
>>> x[(x%5)!=0] = ma.masked
>>> print x
[0 -- -- -- -- 5 -- -- -- -- 10 -- -- -- -- 15 -- -- -- --]
>>> print forward_fill(x)
[0 0 0 0 0 5 5 5 5 5 10 10 10 10 10 15 15 15 15 15]
"""
# !!!: We should probably port that to C.
# Initialization ..................
if np.ndim(marr) > 1:
raise ValueError, "The input array should be 1D only!"
a = ma.array(marr, copy=True)
amask = getmask(a)
if amask is nomask or a.size == 0:
return a
#
adata = getdata(a)
# Get the indices of the masked values (except a[0])
idxtofill = amask[1:].nonzero()[0] + 1
currGap = 0
if maxgap is not None:
previdx = -1
for i in idxtofill:
if i != previdx + 1:
currGap = 0
currGap += 1
if currGap <= maxgap and not amask[i - 1]:
adata[i] = adata[i - 1]
amask[i] = False
previdx = i
else:
amask[i - maxgap:i] = True
else:
for i in idxtofill:
if not amask[i - 1]:
adata[i] = adata[i - 1]
amask[i] = False
return a
def rec_array_to_mgr(
data: MaskedRecords | np.recarray | np.ndarray,
index,
columns,
dtype: DtypeObj | None,
copy: bool,
typ: str,
):
"""
Extract from a masked rec array and create the manager.
"""
# essentially process a record array then fill it
fdata = ma.getdata(data)
if index is None:
index = _get_names_from_index(fdata)
else:
index = ensure_index(index)
if columns is not None:
columns = ensure_index(columns)
arrays, arr_columns = to_arrays(fdata, columns)
# fill if needed
if isinstance(data, np.ma.MaskedArray):
# GH#42200 we only get here with MaskedRecords, but check for the
# parent class MaskedArray to avoid the need to import MaskedRecords
data = cast("MaskedRecords", data)
new_arrays = fill_masked_arrays(data, arr_columns)
else:
# error: Incompatible types in assignment (expression has type
# "List[ExtensionArray]", variable has type "List[ndarray]")
new_arrays = arrays # type: ignore[assignment]
# create the manager
# error: Argument 1 to "reorder_arrays" has incompatible type "List[ndarray]";
# expected "List[ExtensionArray]"
arrays, arr_columns = reorder_arrays(
new_arrays,
arr_columns,
columns # type: ignore[arg-type]
)
if columns is None:
columns = arr_columns
mgr = arrays_to_mgr(arrays,
arr_columns,
index,
columns,
dtype=dtype,
typ=typ)
if copy:
mgr = mgr.copy()
return mgr
def apply_qc(self, ncvariable):
'''
Applies QC to a qartod variable
:param netCDF4.Variable ncvariable: A QARTOD Variable
'''
qc_tests = {
'flat_line': qc.flat_line_check,
'gross_range': qc.range_check,
'rate_of_change': qc.rate_of_change_check,
'spike': qc.spike_check,
'pressure': gliders_qc.pressure_check
}
# If the qartod_test attribute isn't defined then this isn't a variable
# this script created and is not eligble for automatic QC
qartod_test = getattr(ncvariable, 'qartod_test', None)
if not qartod_test:
return
# Get a reference to the parent variable using the standard_name attribute
standard_name = getattr(ncvariable, 'standard_name').split(' ')[0]
parent = self.ncfile.get_variables_by_attributes(
standard_name=standard_name)[0]
test_params = self.get_test_params(parent.name)
# If there is no parameters defined for this test, don't apply QC
if qartod_test not in test_params:
return
test_params = test_params[qartod_test]
if 'thresh_val' in test_params:
test_params['thresh_val'] = test_params['thresh_val'] / pq.hour
times, values, mask = self.get_unmasked(parent)
if qartod_test == 'rate_of_change':
times = ma.getdata(times[~mask])
dates = np.array(num2date(times,
self.ncfile.variables['time'].units),
dtype='datetime64[ms]')
test_params['times'] = dates
if qartod_test == 'pressure':
test_params['pressure'] = values
else:
test_params['arr'] = values
qc_flags = qc_tests[qartod_test](**test_params)
get_logger().info("Flagged: %s", len(np.where(qc_flags == 4)[0]))
get_logger().info("Total Values: %s", len(values))
ncvariable[~mask] = qc_flags
def fromarrays(
arraylist,
dtype=None,
shape=None,
formats=None,
names=None,
titles=None,
aligned=False,
byteorder=None,
fill_value=None,
):
"""Creates a mrecarray from a (flat) list of masked arrays.
Parameters
----------
arraylist : sequence
A list of (masked) arrays. Each element of the sequence is first converted
to a masked array if needed. If a 2D array is passed as argument, it is
processed line by line
dtype : {None, dtype}, optional
Data type descriptor.
shape : {None, integer}, optional
Number of records. If None, shape is defined from the shape of the
first array in the list.
formats : {None, sequence}, optional
Sequence of formats for each individual field. If None, the formats will
be autodetected by inspecting the fields and selecting the highest dtype
possible.
names : {None, sequence}, optional
Sequence of the names of each field.
fill_value : {None, sequence}, optional
Sequence of data to be used as filling values.
Notes
-----
Lists of tuples should be preferred over lists of lists for faster processing.
"""
datalist = [getdata(x) for x in arraylist]
masklist = [np.atleast_1d(getmaskarray(x)) for x in arraylist]
_array = recfromarrays(
datalist,
dtype=dtype,
shape=shape,
formats=formats,
names=names,
titles=titles,
aligned=aligned,
byteorder=byteorder,
).view(mrecarray)
_array._mask.flat = zip(*masklist)
if fill_value is not None:
_array.fill_value = fill_value
return _array
def natgridInterp(self, interp='nn', copy=True, bounds_error=False, fill_value=np.nan):
if self.Xarr.size == self.XarrIn.size:
if (np.abs(self.Xarr.reshape(self.Nx*self.Ny)-self.XarrIn)).sum() < 0.01\
and (np.abs(self.Yarr.reshape(self.Nx*self.Ny)-self.YarrIn)).sum() < 0.01:
print 'No need to interpolate!'
self.Zarr=self.ZarrIn
return
# self.Zarr = ma.getdata(griddata(self.XarrIn, self.YarrIn, self.ZarrIn, self.Xarr, self.Yarr, interp=interp))
self.Zarr = griddata(self.XarrIn, self.YarrIn, self.ZarrIn, self.Xarr, self.Yarr, interp=interp)
self.Zarr = ma.getdata(self.Zarr)
# self.Zarr=self.Zarr.reshape(self.Nx*self.Ny)
return
def create_pigment_tss_plot(netcdf_file_path):
"""create a png from a pigment netcdf file"""
plot_output_filepath = os.path.splitext(netcdf_file_path)[0] + '.png'
dataset = Dataset(netcdf_file_path)
profiles = dataset.variables['profile']
n_obs_per_profile = dataset.variables['row_size']
n_profiles = len(profiles)
if 'CPHL_a' in dataset.variables.keys():
main_data = dataset.variables['CPHL_a']
elif 'SPM' in dataset.variables.keys():
main_data = dataset.variables['SPM']
else:
find_main_var_name = (set(dataset.variables.keys()) - set([u'TIME', u'LATITUDE', u'LONGITUDE', u'station_name', u'station_index', u'profile', u'row_size', u'DEPTH'])).pop()
main_data = dataset.variables[find_main_var_name]
fig = figure(num=None, figsize=(15, 10), dpi=80, facecolor='w', edgecolor='k')
labels = []
ylim([-1, max(dataset.variables['DEPTH'][:]) + 1])
for i_prof in range(n_profiles):
# we look for the observations indexes related to the choosen profile
idx_obs_start = sum(n_obs_per_profile[0:i_prof])
idx_obs_end = idx_obs_start + n_obs_per_profile[i_prof] - 1
idx_obs = range(idx_obs_start, idx_obs_end + 1)
main_var_val = main_data[idx_obs] # for i_prof
depth_val = dataset.variables['DEPTH'][idx_obs]
if len(main_var_val) == 1:
scatter(main_var_val, depth_val)
elif len(main_var_val) > 1:
plot(main_var_val, depth_val, '--')#, c=np.random.rand(3, 1))
else:
scatter(main_var_val, depth_val, c=np.random.rand(3, 1))
station_name = ''.join(ma.getdata(dataset.variables['station_name'][dataset.variables['station_index'][i_prof] - 1]))
labels.append(station_name)
gca().invert_yaxis()
title('%s\nCruise: %s' % (dataset.source, dataset.cruise_id))
xlabel('%s: %s in %s' % (main_data.name, main_data.long_name, main_data.units))
ylabel('%s in %s; positive %s' % (dataset.variables['DEPTH'].long_name,
dataset.variables['DEPTH'].units,
dataset.variables['DEPTH'].positive))
try:
legend(labels, loc='upper left', prop=fontP, title='Station')
except:
pass
savefig(plot_output_filepath)
plt.close(fig)
def fftDiff2(self, m, n):
Nx=1<<(self.Nx-1).bit_length()
Ny=1<<(self.Ny-1).bit_length()
# h = np.fft.fft2(self.Zarr, s=[Nx, Ny] )
h = np.fft.fft2(ma.getdata(self.Zarr), s=[Nx, Ny] )
hshift = np.fft.fftshift(h)
u = np.arange(Nx) - Nx/2.
v = np.arange(Ny) - Ny/2.
U,V = np.meshgrid(u,v)
hdiff = ( (1j*2*np.pi*U/Nx)**m )*( (1j*2*np.pi*V/Ny)**n ) * hshift
out_diff = np.real( np.fft.ifft2( np.fft.ifftshift(hdiff) ) )/(self.dx**m)/(self.dy**n)
out_diff = out_diff[:self.Ny, :self.Nx]
return out_diff
def Fill2ThetaAzimuthMap(masks,TA,tam,image):
'Needs a doc string'
Zlim = masks['Thresholds'][1]
rings = masks['Rings']
arcs = masks['Arcs']
TA = np.dstack((ma.getdata(TA[1]),ma.getdata(TA[0]),ma.getdata(TA[2]))) #azimuth, 2-theta, dist
tax,tay,tad = np.dsplit(TA,3) #azimuth, 2-theta, dist**2/d0**2
for tth,thick in rings:
tam = ma.mask_or(tam.flatten(),ma.getmask(ma.masked_inside(tay.flatten(),max(0.01,tth-thick/2.),tth+thick/2.)))
for tth,azm,thick in arcs:
tamt = ma.getmask(ma.masked_inside(tay.flatten(),max(0.01,tth-thick/2.),tth+thick/2.))
tama = ma.getmask(ma.masked_inside(tax.flatten(),azm[0],azm[1]))
tam = ma.mask_or(tam.flatten(),tamt*tama)
taz = ma.masked_outside(image.flatten(),int(Zlim[0]),Zlim[1])
tabs = np.ones_like(taz)
tam = ma.mask_or(tam.flatten(),ma.getmask(taz))
tax = ma.compressed(ma.array(tax.flatten(),mask=tam)) #azimuth
tay = ma.compressed(ma.array(tay.flatten(),mask=tam)) #2-theta
taz = ma.compressed(ma.array(taz.flatten(),mask=tam)) #intensity
tad = ma.compressed(ma.array(tad.flatten(),mask=tam)) #dist**2/d0**2
tabs = ma.compressed(ma.array(tabs.flatten(),mask=tam)) #ones - later used for absorption corr.
return tax,tay,taz,tad,tabs
def interp_data(var, xpts, ypts, xpts2m, ypts2m, int_meth='cubic'):
#interpoalte data onto a regular 2d grid. Used by interp_uv
data = ma.getdata(var)
mask = ma.getmask(var)
index = np.where(mask==False)
data_masked = data[index]
xpoints = xpts[index]
ypoints = ypts[index]
points = (xpoints, ypoints)
var_int = griddata(points,data_masked,(xpts2m,ypts2m),method=int_meth)
var_int = ma.masked_array(var_int,np.isnan(var_int))
return var_int
def generate_corrected_map(self, dataid, glbdir, outdir, pers=np.array([]), glbpfx='smpkolya_phv_R_', outpfx='smpkolya_phv_R_'):
"""
Generate corrected global phave velocity map using a regional phase velocity map.
=================================================================================================================
Input Parameters:
dataid - dataid for regional phase velocity map
glbdir - location of global reference phase velocity map files
outdir - output directory
pers - period array for correction (default is 4)
glbpfx - prefix for global reference phase velocity map files
outpfx - prefix for output reference phase velocity map files
-----------------------------------------------------------------------------------------------------------------
Output format:
outdir/outpfx+str(int(per))
=================================================================================================================
"""
if not os.path.isdir(outdir):
os.makedirs(outdir)
if pers.size==0:
pers=np.append( np.arange(7.)*10.+40., np.arange(2.)*25.+125.)
for per in pers:
inglobalfname=glbdir+'/'+glbpfx+str(int(per))
try:
self.get_data4plot(dataid=dataid, period=per)
except:
print 'No regional data for period =',per,'sec'
continue
if not os.path.isfile(inglobalfname):
print 'No global data for period =',per,'sec'
continue
outfname=outdir+'/'+outpfx+'%g' %(per)
InglbArr=np.loadtxt(inglobalfname)
outArr=InglbArr.copy()
lonArr=self.lonArr.reshape(self.lonArr.size)
latArr=self.latArr.reshape(self.latArr.size)
vel_iso=ma.getdata(self.vel_iso)
vel_iso=vel_iso.reshape(vel_iso.size)
for i in xrange(InglbArr[:,0].size):
lonG=InglbArr[i,0]
latG=InglbArr[i,1]
phVG=InglbArr[i,2]
for j in xrange(lonArr.size):
lonR=lonArr[j]
latR=latArr[j]
phVR=vel_iso[j]
if abs(lonR-lonG)<0.05 and abs(latR-latG)<0.05 and phVR!=0:
outArr[i,2]=phVR
np.savetxt(outfname, outArr, fmt='%g %g %.4f')
return
def plot_propagation(self, projection='lambert', inbasemap=None, factor=3, showfig=False):
"""Plot propagation direction
"""
if inbasemap==None:
m=self._get_basemap(projection=projection)
else:
m=inbasemap
if self.lonArr.shape[0]-2==self.grad[0].shape[0] and self.lonArr.shape[1]-2==self.grad[0].shape[1]:
self.cut_edge(1,1)
elif self.lonArr.shape[0]!=self.grad[0].shape[0] or self.lonArr.shape[1]!=self.grad[0].shape[1]:
raise ValueError('Incompatible shape for gradient and lon/lat array!')
normArr = np.sqrt ( ma.getdata(self.grad[0] )** 2 + ma.getdata(self.grad[1]) ** 2)
x, y=m(self.lonArr, self.latArr)
U=self.grad[1]/normArr
V=self.grad[0]/normArr
if factor!=None:
x=x[0:self.Nlat:factor, 0:self.Nlon:factor]
y=y[0:self.Nlat:factor, 0:self.Nlon:factor]
U=U[0:self.Nlat:factor, 0:self.Nlon:factor]
V=V[0:self.Nlat:factor, 0:self.Nlon:factor]
Q = m.quiver(x, y, U, V, scale=50, width=0.001)
if showfig:
plt.show()
return
def test_set_fields(self):
"Tests setting fields."
base = self.base.copy()
mbase = base.view(mrecarray)
mbase = mbase.copy()
mbase.fill_value = (999999, 1e20, "N/A")
# Change the data, the mask should be conserved
mbase.a._data[:] = 5
assert_equal(mbase["a"]._data, [5, 5, 5, 5, 5])
assert_equal(mbase["a"]._mask, [0, 1, 0, 0, 1])
# Change the elements, and the mask will follow
mbase.a = 1
assert_equal(mbase["a"]._data, [1] * 5)
assert_equal(ma.getmaskarray(mbase["a"]), [0] * 5)
assert_equal(mbase._mask, [False] * 5)
assert_equal(
mbase._fieldmask.tolist(), np.array([(0, 0, 0), (0, 1, 1), (0, 0, 0), (0, 0, 0), (0, 1, 1)], dtype=bool)
)
# Set a field to mask ........................
mbase.c = masked
assert_equal(mbase.c.mask, [1] * 5)
assert_equal(ma.getmaskarray(mbase["c"]), [1] * 5)
assert_equal(ma.getdata(mbase["c"]), ["N/A"] * 5)
assert_equal(
mbase._fieldmask.tolist(), np.array([(0, 0, 1), (0, 1, 1), (0, 0, 1), (0, 0, 1), (0, 1, 1)], dtype=bool)
)
# Set fields by slices .......................
mbase = base.view(mrecarray).copy()
mbase.a[3:] = 5
assert_equal(mbase.a, [1, 2, 3, 5, 5])
assert_equal(mbase.a._mask, [0, 1, 0, 0, 0])
mbase.b[3:] = masked
assert_equal(mbase.b, base["b"])
assert_equal(mbase.b._mask, [0, 1, 0, 1, 1])
# Set fields globally..........................
ndtype = [("alpha", "|S1"), ("num", int)]
data = ma.array([("a", 1), ("b", 2), ("c", 3)], dtype=ndtype)
rdata = data.view(MaskedRecords)
val = ma.array([10, 20, 30], mask=[1, 0, 0])
#
import warnings
warnings.simplefilter("ignore")
rdata["num"] = val
assert_equal(rdata.num, val)
assert_equal(rdata.num.mask, [1, 0, 0])
def create_absorption_plot(netcdf_file_path):
"""create a png from an absorption netcdf file"""
plot_output_filepath = os.path.splitext(netcdf_file_path)[0] + '.png'
dataset = Dataset(netcdf_file_path)
profiles = dataset.variables['profile']
n_obs_per_profile = dataset.variables['row_size']
n_profiles = len(profiles)
# look for main variable
for varname in dataset.variables.keys():
dim = dataset.variables[varname].dimensions
if 'obs' in dim and 'wavelength' in dim:
main_data = dataset.variables[varname]
fig = figure(num=None, figsize=(15, 10), dpi=80, facecolor='w', edgecolor='k')
labels = []
for i_prof in range(n_profiles):
# we look for the observations indexes related to the choosen profile
# only depth 0 is plotted
depth_to_plot = int(0)
idx_obs_start = sum(n_obs_per_profile[0:i_prof])
idx_obs_end = idx_obs_start + n_obs_per_profile[i_prof] - 1
idx_obs = range(idx_obs_start, idx_obs_end + 1)
main_var_val = main_data[idx_obs] # for i_prof
depth_val = dataset.variables['DEPTH'][idx_obs]
depth_val = np.array(depth_val).astype(int)
wavelength_val = dataset.variables['wavelength'][:]
if not any(depth_val == depth_to_plot):
continue
df = pd.DataFrame(main_var_val[depth_val == depth_to_plot][0].flatten(), index=wavelength_val)
plot(df.index, df, '.')
station_name = ''.join(ma.getdata(dataset.variables['station_name'][dataset.variables['station_index'][i_prof] - 1]))
labels.append(station_name)
title('%s\nCruise: %s\n Depth = %sm' % (dataset.source, dataset.cruise_id, depth_to_plot))
ylabel('%s: %s in %s' % (main_data.name, main_data.long_name, main_data.units))
xlabel('%s in %s' % (dataset.variables['wavelength'].long_name,
dataset.variables['wavelength'].units))
legend(labels, loc='upper right', prop=fontP, title='Station')
savefig(plot_output_filepath)
plt.close(fig)
def __call__ (self, two):
"Executes the call behavior."
# first argument
one = self.obj
# carry out basic operation, transfer name and units
func = getattr(super(Dvect, one), self.f)
if self.inp:
func(two)
result = one
else:
result = MaskedArray(func(two), subok=True).view(type(one))
result._name = one._name
result._units = one._units
# handle the errors. They are worked out in a linear approximation
# which requires the user to supply two partial derivative functions
# in addition to the basic function.
if self.fx is not None and self.fy is not None:
if isinstance(two, Dvect):
if one._err is noerr:
result._err = getdata(np.abs(self.fy(one.dat, two.dat))*two._err)
elif two._err is noerr:
result._err = getdata(np.abs(self.fx(one.dat, two.dat))*one._err)
else:
if one is two:
# Two inputs are identical and not independent
if result.mask is nomask:
result._err = np.hypot(self.fx(one.dat, two.dat), self.fy(one.dat, two.dat))*one._err
else:
result._err = np.where(result.mask, fillerr,
getdata(np.hypot(self.fx(one.dat, two.dat), self.fy(one.dat, two.dat)*two._err)))
pass
else:
# Two inputs are assumed to be independent.
if result.mask is nomask:
result._err = getdata(np.hypot(self.fx(one.dat, two.dat)*one._err,
self.fy(one.dat, two.dat)*two._err))
else:
result._err = np.where(result.mask, fillerr,
getdata(np.hypot(self.fx(one.dat, two.dat)*one._err,
self.fy(one.dat, two.dat)*two._err)))
else:
result._err = getdata(np.abs(self.fx(one.dat, two))*one._err)
else:
result._err = noerr
return result
def fftDiff(self, m, n):
try:
h = np.fft.fft2(ma.getdata(self.Zarr))
except:
h = np.fft.fft2(self.Zarr)
hshift = np.fft.fftshift(h)
Nx=self.Nx
Ny=self.Ny
if Nx % 2 ==0:
u=np.arange(Nx) - Nx/2.
else:
u=np.arange(Nx) - (Nx-1)/2.
if Ny % 2 ==0:
v=np.arange(Ny) - Ny/2.
else:
v=np.arange(Ny) - (Ny-1)/2.
U,V=np.meshgrid(u,v)
hdiff = ((1j*2*np.pi*U/Nx)**m)*((1j*2*np.pi*V/Ny)**n) * hshift
out_diff = np.real( np.fft.ifft2( np.fft.ifftshift(hdiff) ) )/(self.dx**m)/(self.dy**n)
out_diff = out_diff[:self.Ny, :self.Nx]
return out_diff
def test_units_qc(self):
fd, fake_file = tempfile.mkstemp()
os.close(fd)
self.addCleanup(os.remove, fake_file)
nc = Dataset(fake_file, 'w')
nc.createDimension('time', 10)
timevar = nc.createVariable('time', np.float64, ('time',), fill_value=-9999.)
timevar.standard_name = 'time'
timevar.units = 'seconds since 1970-01-01T00:00:00Z'
timevar[np.array([0, 2, 4, 6, 8])] = np.array([0, 2, 4, 6, 8])
tempvar = nc.createVariable('temp', np.float32, ('time',), fill_value=-9999.)
tempvar.standard_name = 'sea_water_temperature'
tempvar.units = 'deg_F'
tempvar[np.array([0, 1, 2, 3, 4, 9])] = np.array([72.0, 72.1, 72.0, 1.0, 72.03, 72.1])
qc = GliderQC(nc)
for qcvarname in qc.create_qc_variables(nc.variables['temp']):
qc.apply_qc(nc.variables[qcvarname])
np.testing.assert_equal(nc.variables['qartod_temp_flat_line_flag'][:].mask, ~np.array([1, 0, 1, 0, 1, 0, 0, 0, 0, 0], dtype=bool))
np.testing.assert_equal(ma.getdata(nc.variables['qartod_temp_flat_line_flag'][:]), np.array([1, 9, 1, 9, 1, 9, 9, 9, 9, 9], dtype=np.int8))
def masked_rec_array_to_mgr(data, index, columns, dtype, copy):
"""
Extract from a masked rec array and create the manager.
"""
# essentially process a record array then fill it
fill_value = data.fill_value
fdata = ma.getdata(data)
if index is None:
index = get_names_from_index(fdata)
if index is None:
index = ibase.default_index(len(data))
index = ensure_index(index)
if columns is not None:
columns = ensure_index(columns)
arrays, arr_columns = to_arrays(fdata, columns)
# fill if needed
new_arrays = []
for fv, arr, col in zip(fill_value, arrays, arr_columns):
mask = ma.getmaskarray(data[col])
if mask.any():
arr, fv = maybe_upcast(arr, fill_value=fv, copy=True)
arr[mask] = fv
new_arrays.append(arr)
# create the manager
arrays, arr_columns = reorder_arrays(new_arrays, arr_columns, columns)
if columns is None:
columns = arr_columns
mgr = arrays_to_mgr(arrays, arr_columns, index, columns, dtype)
if copy:
mgr = mgr.copy()
return mgr
def Laplacian(self, method='default', order=2):
Zarr=ma.getdata(self.Zarr)
if method == 'default':
Zarr_yp=Zarr[2:, 1:-1]
Zarr_yn=Zarr[:-2, 1:-1]
Zarr_xp=Zarr[1:-1, 2:]
Zarr_xn=Zarr[1:-1, :-2]
Zarr=Zarr[1:-1, 1:-1]
self.lplc=(Zarr_yp+Zarr_yn-2*Zarr) / (self.dy**2) + (Zarr_xp+Zarr_xn-2*Zarr) / (self.dx**2)
elif method == 'convolve':
if order==2:
diff2_x=convolve( ma.getdata(self.Zarr), X_diff2_weight_2)/self.dx/self.dx
diff2_y=convolve(ma.getdata(self.Zarr), Y_diff2_weight_2)/self.dy/self.dy
elif order==4:
diff2_x=convolve( ma.getdata(self.Zarr), X_diff2_weight_4)/self.dx/self.dx
diff2_y=convolve(ma.getdata(self.Zarr), Y_diff2_weight_4)/self.dy/self.dy
elif order==6:
diff2_x=convolve( ma.getdata(self.Zarr), X_diff2_weight_6)/self.dx/self.dx
diff2_y=convolve(ma.getdata(self.Zarr), Y_diff2_weight_6)/self.dy/self.dy
self.lplc=diff2_x+diff2_y
self.lplc=self.lplc[1:-1, 1:-1]
return
def loadVOTable(self):
try:
self.votable = astropy.io.votable.parse_single_table(self.filename)
except:
return False
else:
self.columnTitles= [a.name for a in self.votable.fields]
rows=ma.getdata(self.votable.array)
for row in rows:
s=[]
for data in row:
try:
#s.append(decimal.Decimal(data))
s.append(float(data))
except:
s.append(str(data))
self.dataValues.append(s)
self.calculateArrayColumns()
#Assing a delimiter in case of saving as ASCII table
self.delimiter="|"
return True
def Gradient(self, edge_order=1, method='default', order=2):
if method=='default':
self.grad=np.gradient( ma.getdata(self.Zarr), self.dx, self.dy, edge_order=edge_order)
elif method=='freq':
diff_x=self.fftDiff(m=1, n=0)
diff_y=self.fftDiff(m=0, n=1)
self.grad=[]
self.grad.append(diff_y)
self.grad.append(diff_x)
elif method == 'convolve':
if order==2:
diff_x=convolve( ma.getdata(self.Zarr), X_diff_weight_2)/self.dx
diff_y=convolve(ma.getdata(self.Zarr), Y_diff_weight_2)/self.dy
elif order==4:
diff_x=convolve(ma.getdata(self.Zarr), X_diff_weight_4)/self.dx
diff_y=convolve(ma.getdata(self.Zarr), Y_diff_weight_4)/self.dy
elif order==6:
diff_x=convolve(ma.getdata(self.Zarr), X_diff_weight_6)/self.dx
diff_y=convolve(ma.getdata(self.Zarr), Y_diff_weight_6)/self.dy
self.grad=[]
self.grad.append(diff_y)
self.grad.append(diff_x)
return
def GetLplcCorrection(self, per):
omega=2.*np.pi/per
Zarr=ma.getdata(self.Zarr)
self.lplcCo=self.lplc/Zarr[1:-1, 1:-1]/(omega**2)
return
def LaplacianEqualXY(self):
if self.dx!=self.dy:
raise ValueError('grid spacing not equal!')
self.lplc=scipy.ndimage.filters.laplace(ma.getdata(self.Zarr) ) / (self.dx*self.dy)
self.lplc=self.lplc[1:-1, 1:-1]
return
