def xcorr_old(a,b, nlag):
''' does not normalize'''
import numpy.ma as ma
a = ma.masked_where(np.isnan(a),a)
b = ma.masked_where(np.isnan(b),b)
l = len(a)+len(b)
return np.correlate(a,b, mode='full')[l/2-nlag-1:-l/2+nlag+1]
def plot_cell_feature (data, cell_id_column, cell_lat_column, cell_long_column, richness_column, title=None, second_feature_data=None, second_feature_column=None):
lats = np.asarray(np.unique(data[cell_lat_column]))
lons = np.asarray(np.unique(data[cell_long_column]))
lons, lats = np.meshgrid(lons,lats)
richness = np.array(data[richness_column])
richness.shape = (len(np.unique(lats)), len(np.unique(lons)))
richness_mask = ma.masked_where(np.isnan(richness),richness)
if second_feature_column:
second_feature = np.array(second_feature_data[second_feature_column])
second_feature.shape = (len(np.unique(lats)), len(np.unique(lons)))
second_feature_data_mask = ma.masked_where(np.isnan(second_feature),second_feature)
fig = plt.figure()
m = Basemap(projection='merc',llcrnrlat=23.5,urcrnrlat=57, llcrnrlon=-140,urcrnrlon=-50,lat_ts=20,resolution='l')
m.drawcoastlines(linewidth = 1.25)
if np.nanmin(richness) < 20:
vmin=0
else:
vmin=round(np.nanmin(richness)-20, -1)
im1 = m.pcolormesh(lons,lats,richness_mask,shading='flat',cmap=plt.cm.Blues,latlon=True, vmin=vmin)
if second_feature_column:
im2 = m.pcolormesh(lons,lats,second_feature_data_mask,shading='flat',cmap=plt.cm.RdYlBu,latlon=True)
cb = m.colorbar(im1,"bottom", size="5%", pad="2%")
plt.title(title)
def FoF2(grid):
groups = []
# Isolate LL pixels
grid = ma.masked_where(np.log10(grid) < 17.2, grid) # Isolate LLs Pixels
grid = ma.masked_where(np.log10(grid) > 20.3, grid)
g = 0 # Group Number (index of groups)
while grid.count() != 0: # Iterate until all pixels have been added to a group
indexs = grid.nonzero() # Select a pixel: Find all non-masked pixels
i,j = indexs[0][0], indexs[1][0] # and pick the first one.
groups.append([(i,j)]) # append to group
recent_list = [(i,j)] # Do 1 group:
while len(recent_list) != 0: # Iterate until no more neighbouring pixels are found
grid, neighbours = check_neighbours(grid, recent_list[0]) # Get the coordinates of the pixels that neighbour the pixel being looped over if they have a LLS in
recent_list = recent_list+neighbours # Add all the new neighbours to recent_list
del recent_list[0] # Remove the pixel that is being looped over
grid[i,j] = ma.masked
if len(recent_list) > 0:
groups[g].append(recent_list[0]) # Add new pixel to master list of pixels ( groups ) that will be iterated over next
g += 1 # Finished 1 whole group, so start the next
print "one group done", grid.count()
return groups
def findMean(dataVectorPy,indexList,maskedValue= -9999.):
"""
find the mean of binned variables
"""
dataVector=np.ascontiguousarray(dataVectorPy,dtype=np.float32)
dataVector=dataVector.reshape(-1)
dataList=takeValues(dataVector,indexList)
dataCount=len(dataList)
outList=[]
areaWeightedOutList=[]
gridCounts=[]
for i in range(dataCount):
theData=dataList[i]
if len(theData) > 0:
outList.append(theData.mean())
areaWeightedOutList.append(theData.mean()*len(theData))
#print "appending: ",len(theData)
gridCounts.append(len(theData))
else:
outList.append(maskedValue)
areaWeightedOutList.append(maskedValue)
gridCounts.append(0)
outVec=np.array(outList,dtype=np.float32)
outAreaWeightedVec=np.array(areaWeightedOutList,dtype=np.float32)
outArray=ma.masked_where(outVec==maskedValue,outVec)
outAreaArray=ma.masked_where(outAreaWeightedVec==maskedValue,outAreaWeightedVec)
gridCounts=np.array(gridCounts,dtype=np.int64)
return (outArray,outAreaArray,gridCounts)
def test_mode(self):
a1 = [0,0,0,1,1,1,2,3,3,3,3,4,5,6,7]
a2 = np.reshape(a1, (3,5))
a3 = np.array([1,2,3,4,5,6])
a4 = np.reshape(a3, (3,2))
ma1 = ma.masked_where(ma.array(a1) > 2, a1)
ma2 = ma.masked_where(a2 > 2, a2)
ma3 = ma.masked_where(a3 < 2, a3)
ma4 = ma.masked_where(ma.array(a4) < 2, a4)
assert_equal(mstats.mode(a1, axis=None), (3,4))
assert_equal(mstats.mode(a1, axis=0), (3,4))
assert_equal(mstats.mode(ma1, axis=None), (0,3))
assert_equal(mstats.mode(a2, axis=None), (3,4))
assert_equal(mstats.mode(ma2, axis=None), (0,3))
assert_equal(mstats.mode(a3, axis=None), (1,1))
assert_equal(mstats.mode(ma3, axis=None), (2,1))
assert_equal(mstats.mode(a2, axis=0), ([[0,0,0,1,1]], [[1,1,1,1,1]]))
assert_equal(mstats.mode(ma2, axis=0), ([[0,0,0,1,1]], [[1,1,1,1,1]]))
assert_equal(mstats.mode(a2, axis=-1), ([[0],[3],[3]], [[3],[3],[1]]))
assert_equal(mstats.mode(ma2, axis=-1), ([[0],[1],[0]], [[3],[1],[0]]))
assert_equal(mstats.mode(ma4, axis=0), ([[3,2]], [[1,1]]))
assert_equal(mstats.mode(ma4, axis=-1), ([[2],[3],[5]], [[1],[1],[1]]))
a1_res = mstats.mode(a1, axis=None)
# test for namedtuple attributes
attributes = ('mode', 'count')
check_named_results(a1_res, attributes, ma=True)
def mask_seafloor(input, kmt, axis=0):
""" def mask_seafloor(input, kmt):
default is that input has depth (zt) as first dimension.
axis is axis of depth dimension.
Change axis=1 if time is first and depth is second.
returns input as masked array
"""
tmp = ma.zeros(input.shape)
ndepth = input.shape[axis]
if axis==1:
# need to tile kmt
kmtt = np.tile(kmt,(input.shape[0],1,1))
for lii in np.arange(0,ndepth): # loop through each depth layer
# mask out levels below sea floor
if axis==0:
tmp[lii,...] = ma.masked_where(kmt <= lii, input[lii,...])
elif axis==1:
tmp[:,lii,...] = ma.masked_where(kmtt <= lii, input[:,lii,...])
else:
print 'axis must = 1 or 0'
return tmp
def liquefaction_ratio(current_data):
drytol = getattr(current_data.user, 'drytol', drytol_default)
q = current_data.q
p=ma.masked_where(q[:,:,0]<drytol,q[:,:,4])
litho = lithostaticP(current_data)
ratio = ma.masked_where(q[:,:,0]<drytol,p/litho)
return ratio
def detrend(self, y):
dy = y.copy(); dy[:] = nan
line = y.copy(); line[:] = nan
line[2 : -2] = self.__winave(y, 5, 7)
dy[2 : -2] = y[2 : -2] - line[2 : -2]
dy = masked_where(isnan(dy), dy)
line = masked_where(isnan(line), line)
return dy, line
def loadimg(filename):
img = read(filename)
img = img.astype(numpy.float32)
h, w = img.shape
imgK = img[:h/2]
imgRb = img[h/2:]
return ma.masked_where(imgK==0, imgK*1e-3-1.0), \
ma.masked_where(imgRb==0, imgRb*1e-3-1.0)
def plot_BANDL(ax, h):
chosen = ma.masked_where(~(h["Band"] >= BAND_NAT_RALLY), h["Close"])
if chosen.any():
ax.plot(h.index, chosen, "g-", linewidth=1, alpha=1)
chosen = ma.masked_where(~(h["Band"] <= BAND_NAT_REACT), h["Close"])
if chosen.any():
ax.plot(h.index, chosen, "r-", linewidth=1, alpha=1)
def detrend(self, y):
dy = y.copy(); dy[:] = nan
line = y.copy(); line[:] = nan
ffd = diff(y) / y[: -1]
line[1 :] = self.polytrend(ffd, 1)
dy[1 :] = ffd - line[1 :]
dy = masked_where(isnan(dy), dy)
line = masked_where(isnan(line), line)
return dy, line
def lithostaticP(current_data):
drytol = getattr(current_data.user, 'drytol', drytol_default)
q = current_data.q
h=ma.masked_where(q[:,:,0]<drytol,q[:,:,0])
m=solid_frac(current_data)
rho = density(current_data)
var = ma.masked_where(h<drytol,gmod(current_data)*rho*h)
return var
def interpolate(values, vtx, wts, fill_value=None):
"""Interpolate values using weights and vertices from interp_weights
Ultimately, this does the same job as scipy's interpolate. However, by
calculating the vertices and weights in a previous step, it makes repeated
interpolations much quicker. Useful for say model grids in which
interpolation in x, y occurs over multiple z levels
Idea comes from
``http://stackoverflow.com/questions/20915502/
speedup-scipy-griddata-for-multiple-interpolations-between
-two-irregular-grids``
Inputs
------
values : array (1D or 2D)
The values to interpolate
vtx : N x 3 array
Vertices output from interp_weights
wts : N x 3 array
Weights output from interp_weights
fill_value : None, float, or np.nan
Value for points that are not interpolateable
"""
# Notes on how this works:
# np.einsum('nj,nj->n', A, B) is equivalent to np.sum(A*B, axis=1)
# np.take(values, vtx) is equivalent to values[vtx]
# original
# ret = np.einsum('nj,nj->n', np.take(values, vtx), wts)
# new to allow for values to contain NaNs
# Ensure values array is 1D
if values.ndim > 1:
values = values.flatten()
# indices of where there are NaNs in values
nans = np.array(np.where(np.isnan(values)))
# mask[i, j] is true if vtx[i, j] is contained in nans
mask = np.column_stack((np.in1d(vtx[:, 0], nans),
np.in1d(vtx[:, 1], nans),
np.in1d(vtx[:, 2], nans)))
# copy values[wts] and vtx, but mask where vtx corresponds to a NaN
vals_ma = ma.masked_where(mask, values[vtx])
wts_ma = ma.masked_where(mask, wts)
ret = np.sum(vals_ma*wts_ma, axis=1)/np.sum(wts_ma, axis=1)
# if fill_value has a value other than 'None', then check for negative
# weights. If fill_value=None (ie else case) then do nothing
if fill_value:
holes = np.any(wts < 0, axis=1)
ret[holes] = fill_value
else:
pass
return ret
def plot11(curs):
age,mtot,mf,b=gettable(curs,cols='age,mtot,Massfrac,b_para',where='bpara2 > 3',table='sball')
P.loglog(mtot,age,'Db',label=r'$b>3$')
mtot=masked_where(mf<0.025,mtot)
P.loglog(mtot,age,'Dg',label=r'$b>3, mf>2.5\%$')
mtot=masked_where(b<3,mtot)
P.loglog(mtot,age,'Dr',label=r'$b>3, mf>2.5\%, <b> >3$')
P.xlabel(r'$m_{tot}$')
P.ylabel('age')
P.legend(loc='lower right')
def surface_or_depth(current_data):
from numpy import ma, where
q = current_data.q
h = q[0,:,:]
eta = q[3,:,:]
topo = eta - h
surface = ma.masked_where(h<=drytol, eta)
depth = ma.masked_where(h<=drytol, h)
surface_or_depth = where(topo<0, surface, depth)
return surface_or_depth
def seaoverland(input_matrix, nloop=1): # depth is to select the number of consequential mask points to fill
# depth loop
if np.sum(input_matrix.mask) == 0: # nothing to flood
print('WARNING 23. Field does not have any land point. Exiting.', file=sys.stderr)
return input_matrix
infill_value = input_matrix.fill_value
for loop in range(nloop):
if loop == 0:
original_mask = input_matrix.mask
output_matrix = np.where(input_matrix.filled() == infill_value, 0, input_matrix.filled())
else:
if np.sum(output_matrix.mask) == 0: # nothing more to flood
print('WARNING 23. Field does not have anymore land points,', str(loop), 'steps were sufficient to flood it completely.', file=sys.stderr)
return output_matrix
original_mask = output_matrix.mask
output_matrix = np.where(output_matrix.filled() == infill_value, 0, output_matrix.filled())
# Create a nD 3D matrix that contains values shifted in one of the 8 possible directions
# of the last two axes compared to the original matrix
shift_matrix = ma.array(np.zeros(shape=output_matrix.shape),
mask=False, fill_value=infill_value, dtype=float)
weight_matrix = ma.array(np.zeros(shape=output_matrix.shape),
mask=False, fill_value=infill_value, dtype=float)
# up shift
shift_matrix[..., : - 1, :] = output_matrix[..., 1:, :]
weight_matrix[..., : - 1, :] = np.where(original_mask[..., 1:, :], 0, 1)
# down shift
shift_matrix[..., 1:, :] += output_matrix[..., 0: - 1, :]
weight_matrix[..., 1:, :] += np.where(original_mask[..., 0: - 1, :], 0, 1)
# left shift
shift_matrix[..., :, : - 1] += output_matrix[..., :, 1:]
weight_matrix[..., :, : - 1] += np.where(original_mask[..., :, 1:], 0, 1)
# right shift
shift_matrix[..., :, 1:] += output_matrix[..., :, : - 1]
weight_matrix[..., :, 1:] += np.where(original_mask[..., :, : - 1], 0, 1)
# up-left shift
shift_matrix[..., : - 1, : - 1] += output_matrix[..., 1:, 1:]
weight_matrix[..., : - 1, : - 1] += np.where(original_mask[..., 1:, 1:], 0, 1)
# up-right shift
shift_matrix[..., : - 1, 1:] += output_matrix[..., 1:, : - 1]
weight_matrix[..., : - 1, 1:] += np.where(original_mask[..., 1:, : - 1], 0, 1)
# down-left shift
shift_matrix[..., 1:, : - 1] += output_matrix[..., : - 1, 1:]
weight_matrix[..., 1:, : - 1] += np.where(original_mask[..., : - 1, 1:], 0, 1)
# down-right shift
shift_matrix[..., 1:, 1:] += output_matrix[..., : - 1, : - 1]
weight_matrix[..., 1:, 1:] += np.where(original_mask[..., : - 1, : - 1], 0, 1)
# Mask the matrices where there are zeros
shift_matrix = ma.masked_where(shift_matrix == 0, shift_matrix)
weight_matrix = ma.masked_where(weight_matrix == 0, weight_matrix)
# Mediate the shift matrix among the third dimension
mean_matrix = shift_matrix / weight_matrix
# Replace input masked points with new ones belonging to the mean matrix
output_matrix = ma.array(np.where(mean_matrix.mask + original_mask, mean_matrix, output_matrix),
mask=mean_matrix.mask, fill_value=infill_value, dtype=float)
return output_matrix
def ueval(self, x, fill_value=0, fill_err=0):
"""Return the histogram value and uncertainty at `x`."""
mbins = ma.masked_outside(self.findbin(x), 0, self.hist.size-1)
value, err = ma.masked_where(mbins.mask, self.hist[mbins.filled(0)]), \
ma.masked_where(mbins.mask, self.errs[mbins.filled(0)])
if np.iterable(x):
return uarray((value.filled(fill_value), err.filled(fill_err)))
else:
return ufloat((value.filled(fill_value).item(), \
err.filled(fill_err).item()))
def integrate_over_domain( domain, basis_1, basis_2, nx, ny):
x = np.linspace(0, domain.extent[0], nx)
y = np.linspace(0, domain.extent[1], ny)
dx = domain.extent[0]/nx
dy = domain.extent[1]/ny
xgrid, ygrid = np.meshgrid(x,y)
domain_grid = domain.in_subdomain(xgrid, ygrid)
fn1 = ma.masked_where( ~domain_grid, basis_1(xgrid,ygrid))
fn2 = ma.masked_where( ~domain_grid, basis_2(xgrid,ygrid))
value = np.sum( fn1*fn2 )*dx*dy
return value
def read_latlon(f):
lat, latnam = read_var(f, 'lat')
lon, lonnam = read_var(f, 'lon')
all_masks = [lat < -90., lat > 90., lon < -180., lon > 180.]
total_mask = reduce(np.logical_or, all_masks)
lat = ma.masked_where(total_mask, lat)
lon = ma.masked_where(total_mask, lon)
return lat, lon
def surface_corrector1(current_data): #return a masked array showing the surface where the corrector wave is big from numpy import ma drytol=getattr(current_data.user,'dry_tolderance',drytol_default) q=current_data.q aux=current_data.aux h=q[0,:] eta=aux[0,:]+h water=ma.masked_where(h<=drytol,eta) water=ma.masked_where(aux[2,:]!=1.0,water) return water
def get_roi(template, vct_src, group_by = ''):
# retrieving output specifications from template raster data set
tpl_ds = gdal.Open(tpl)
tpl_geo_transform = tpl_ds.GetGeoTransform()
tpl_x_size, tpl_y_size = tpl_ds.RasterXSize, tpl_ds.RasterYSize
#tpl_ds = None
tpl_bd = tpl_ds.GetRasterBand(12)
tpl_data = tpl_bd.ReadAsArray()
# opening vector base
vct_ds = ogr.Open(vct_src)
vct_ly = vct_ds.GetLayer(0)
polygon_row_cols = dict()
roi_groups = dict()
for ft in vct_ly:
fid = ft.GetFID()
print "Working on FID %d..." % fid
if group_by:
if not roi_groups.has_key(group_by):
roi_groups[group_by] = list()
roi_groups[group_by].append(fid)
mem_ds = gdal.GetDriverByName('MEM').Create("", tpl_x_size, tpl_y_size, 1, gdal.GDT_Byte)
mem_ds.SetGeoTransform(tpl_geo_transform)
mem_ds.SetProjection(vct_ly.GetSpatialRef().ExportToWkt())
where_clause = "WHERE '%s' = %i" % ('FID', fid)
query = "SELECT * FROM '%s' %s" % (vct_ly.GetName(), where_clause)
tmp_ds = ogr.Open(vct_src)
sel_ly = tmp_ds.ExecuteSQL(query)
gdal.RasterizeLayer(mem_ds, (1,), sel_ly, burn_values = (255,))
mem_data = mem_ds.GetRasterBand(1).ReadAsArray()
xxx = ma.masked_where(mem_data != 255, tpl_data)
mask = ma.masked_where(mem_data != 255, mem_data)
rows_cols = list()
shape = list(mask.shape)
shape.reverse()
non_mask_indices = ma.flatnotmasked_contiguous(mask)
for sl in non_mask_indices:
start_row_col = np.unravel_index(sl.start, tuple(shape), order = 'F')
stop_row_col = np.unravel_index(sl.stop, tuple(shape), order = 'F')
#print sl, np.unravel_index(sl.start, mask.shape), np.unravel_index(sl.stop, mask.shape)
rows_cols.append((start_row_col, stop_row_col))
else:
polygon_row_cols[fid] = rows_cols
return polygon_row_cols, roi_groups
def region_plot(fname, time, output_dir, julday):
#print "fname = %s, day = %d\n" % (fname, time);
ncfile = nc.Dataset(fname,'r')
fig = plt.figure(figsize=(20.48,10.24))
ax = plt.axes([0,0,1,1],frameon=False)
ax.set_axis_off()
# set up a Basemap
latrng = (-90.0,90.0)
lonrng = (0,360)
m = Basemap(projection='cyl',llcrnrlat=latrng[0],urcrnrlat=latrng[1],
llcrnrlon=lonrng[0],urcrnrlon=lonrng[1],resolution='h',
area_thresh=800,fix_aspect=False)
# for mom output
lats = ncfile.variables["yt_ocean"][:]
lons = ncfile.variables["xt_ocean"][:]
field = ncfile.variables["eta_t"][time,:,:]
aveNcfile = nc.Dataset('ocean_eta_annual.nc','r')
average = aveNcfile.variables["eta_t"][0,:,:]
field = field - average
field = ma.array(field)
field = ma.masked_where(field == -1.0e10, field)
field = ma.masked_where(field<= -10,field)
field = ma.masked_where(field>= 100, field)
field, lons = addcyclic(field, lons)
#field_min = np.min(field)
#field_max = np.max(field)
x,y = m(*np.meshgrid(lons[:],lats[:]))
colorMap = colormaps.make_colormap({0.:'#191970', 0.2:'#448BB9', 0.35:'#1e90ff', 0.5:'w', 0.6:'#F8BC4E', 0.7:'#ffa500', 0.8:'#ff8c00',0.9:'#F56D2E', 1.0:'#CC4000'})
m.drawmapboundary(fill_color='white')
nlevs = 256
clevs = np.linspace(-0.8,0.8,num=nlevs)
cs = m.contourf(x,y,field, levels=clevs, cmap=colorMap, extend='both', norm=mpl.colors.normalize(),antialiased=False )
m.fillcontinents(color='black', lake_color='black', zorder=1)
save_file = "%s/%f.png" % (output_dir,julday)
plt.savefig(save_file)
#plt.show()
ncfile.close()
def velocity(current_data): """ Return a masked array containing velocity v in wet cells. """ from numpy import ma drytol = getattr(current_data.user, 'drytol', drytol_default) q = current_data.q h = q[:,:,0] hv = q[:,:,2] u = ma.masked_where(h<=drytol, hv/h) hv = q[:,:,2] v = ma.masked_where(h<=drytol, hv/h) return (u,v)
def plot12(curs):
age,mtot,b=gettable(curs,cols='age,mtot,b_para',where='agn=0',table='sball')
b1=masked_where(age>=5E7,b)
b2=masked_where(age<=5E7,b)
b2=masked_where(age>=5E8,b2)
b3=masked_where(age<=5E8,b)
P.semilogx(mtot,b3,',r',label=r'age $> 5E8 yr$')
P.semilogx(mtot,b2,'.b',label=r'$5E7 <$ age $< 5E8 yr$')
P.semilogx(mtot,b1,'.k',label=r'age $< 5E7 yr$')
P.axis([5E8,1E12,-1,15])
P.xlabel(r'$m_{tot}$')
P.ylabel(r'$<b>$')
P.legend(loc='upper right')
def averbins(X,orgX,Y,median=True):
mean=N.zeros(len(X),dtype='Float64')
#median=mean.copy()
#sigma=mean.copy()
for i,x in enumerate(X):
tmp=masked_where(orgX<x,Y)
if i<len(X)-1: tmp=masked_where(orgX>=X[i+1],tmp)
print i,x,tmp
if median: mean[i]=N.ma.median(tmp)
else: mean[i]=tmp.mean()
#median[i]=N.median(tmp)
#sigma[i]=tmp.std()
return mean
def test_mode(self):
a1 = [0,0,0,1,1,1,2,3,3,3,3,4,5,6,7]
a2 = np.reshape(a1, (3,5))
ma1 = ma.masked_where(ma.array(a1) > 2,a1)
ma2 = ma.masked_where(a2 > 2, a2)
assert_equal(mstats.mode(a1, axis=None), (3,4))
assert_equal(mstats.mode(ma1, axis=None), (0,3))
assert_equal(mstats.mode(a2, axis=None), (3,4))
assert_equal(mstats.mode(ma2, axis=None), (0,3))
assert_equal(mstats.mode(a2, axis=0), ([[0,0,0,1,1]],[[1,1,1,1,1]]))
assert_equal(mstats.mode(ma2, axis=0), ([[0,0,0,1,1]],[[1,1,1,1,1]]))
assert_equal(mstats.mode(a2, axis=-1), ([[0],[3],[3]], [[3],[3],[1]]))
assert_equal(mstats.mode(ma2, axis=-1), ([[0],[1],[0]], [[3],[1],[0]]))
def __init__(self, bare_dir, dem_dir, outdir):
#loads the GDAL derived DEMs to produce the crop height model
if not os.path.exists(outdir):
os.mkdir(outdir)
os.path.join(dem_dir,bare_dir)
bare = None
crop_model = None
for bare_dem in os.listdir(bare_dir):
os.chdir(bare_dir)
bare = LoadImage(bare_dem)
bare_filtered = filters.median_filter(bare.stacked,size=(9,9))
bare_spatial = bare.spatial
for survey in os.listdir(dem_dir):
im_path = os.path.join(dem_dir, survey)
os.chdir(im_path)
for image in os.listdir(im_path):
crop = LoadImage(image)
crop_filtered = filters.median_filter(crop.stacked,size=(3,3))
crop_model = crop_filtered-bare_filtered
bare_mask = ma.masked_where(bare_filtered==-999, bare_filtered)
crop_mask = ma.masked_where(crop_filtered==-999, crop_filtered)
#combine these into a single mask so that anything masked in either dataset
#will be masked in the combined output
combined_mask = ma.mask_or(bare_mask.mask, crop_mask.mask)
#use this mask t omask the crop model
crop_model = ma.array(crop_model, mask=combined_mask)
#convert back to a bog-standard numpy array and fill the masked values
crop_model = crop_model.filled(-999)
name = survey+'_CropHeightModel.tif'
self.writeimage(outdir,
name,
crop_model.shape[0],
crop_model.shape[1],
crop_model,
bare_spatial)
def scan_filter(arr, fill_val): """ Filters out invalid scan data from a given array of ranges Args: arr: the array of scan data that should be filtered fill_val: the value to replace 'inf' values with Returns: A copy of the input array, without 'nan' values and with 'inf' values replaces by the given fill_val """ arr = ma.masked_where(np.isnan(arr), arr).compressed() arr = ma.masked_where(np.isinf(arr), arr).filled(fill_val) return arr
def velocity_norm(current_data): """ Return a masked array containing velocity norm in wet cells. """ from numpy import ma drytol = getattr(current_data.user, 'drytol', drytol_default) q = current_data.q h = q[:,:,0] hv = q[:,:,2] u = ma.masked_where(h<=drytol, hv/h) hv = q[:,:,2] v = ma.masked_where(h<=drytol, hv/h) vnorm = np.sqrt(u**2 + v**2) return vnorm
def calculate_moments(d,minchan=False,maxchan=False,vel=False,bestmask=False,mask=False):
nglat = d.shape[1]
nglon = d.shape[2]
nspec = d.shape[0]
maps = np.zeros((nglat,nglon),dtype={'names':['mean','sd','errmn',
'errsd','skew','kurt','error','intint','npix'],
'formats':['f4','f4','f4','f4','f4','f4','f4','f4','f4']})
#These definitions for mask seem backward but are correct.
noise_portion = ma.masked_where(mask == 1,d)
good_d = d[minchan:maxchan,...]
mask2 = mask[minchan:maxchan,...]
#print(mask)
#print(mask2)
print(minchan)
print(maxchan)
signal_portion = ma.masked_where(mask2 == 0,good_d)
maps['error'] = ma.std(noise_portion,axis=0)
maps['intint'] = ma.sum(signal_portion,axis=0)
#print(maps['error'])
for x in range(nglat):
for y in range(nglon):
fullspec = d[...,x,y]#Exract a single spectrum
ind = np.arange(nspec)
velmask = mask[minchan:maxchan,x,y]
if np.sum(velmask) != 0:
velmask = bestmask
npix = max(np.sum(velmask),1)
ind = ind[velmask > 0]
sigma = maps['error'][x,y]
if ind.size > 2 and (sigma > 0):
mom = idl_stats.wt_moment(vel[ind],fullspec[ind],
errors = np.zeros(ind.size)+sigma)
maps['mean'][x,y] = mom['mean']
maps['sd'][x,y] = mom['stdev']
maps['errmn'][x,y] = mom['errmn']
maps['errsd'][x,y] = mom['errsd']
maps['npix'][x,y] = npix
else:
maps['mean'][x,y] = np.nan
maps['sd'][x,y] = np.nan
maps['errmn'][x,y] = np.nan
maps['errsd'][x,y] = np.nan
maps['npix'][x,y] = np.nan
return(maps)
mj2.append([]) ol2.append([]) else: mi2[j].append(int(line.split()[0])) mj2[j].append(int(line.split()[1])) ol2[j].append(float(line.split()[2])) mi2=np.array(mi2) mj2=np.array(mj2) ol2=np.array(ol2) #--------------------------------------------------------------------- Zm = ma.masked_where(mi2 < mj2, ol2) fig=plt.figure(1, figsize=(13,9)) ax=fig.add_subplot(111,autoscale_on=False) cmain=ax.pcolor(mi,mj,ol,vmin=0, vmax=16,cmap=plt.cm.gist_yarg_r) csec=ax.pcolor(mi2,mj2,Zm,vmin=-1, vmax=1,cmap=plt.cm.RdBu_r) ax.set_aspect(1) ax.set_title(ttl) ax.set_xlabel(xlbl) ax.set_xlim(mi.min(), mi.max()) ax.set_ylabel(ylbl) ax.set_ylim(mj.max(), mj.min())
from mpl_toolkits.basemap import Basemap, cm, shiftgrid, interp, maskoceans
from netCDF4 import Dataset as NetCDFFile
import numpy as N
import matplotlib.pyplot as plt
import numpy.ma as ma
import matplotlib.colors as colors
mask = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/Ctry_halfdeg.nc', 'r')
cou1 = mask.variables['MASK_Country'][:, :]
cou1 = ma.masked_where(cou1 <= 0.0, cou1)
region = NetCDFFile(
'/project/projectdirs/m1602/datasets4.full/surfdata_05x05.nc', 'r')
ind = region.variables['REGION_MASK_CRU_NCEP'][:, :]
#area=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/gridareahalf.nc','r')
#gridarea = area.variables['cell_area']
isam = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/rice_cheyenne/his_cru/ric_irr_fert/output/ric_irr_fert.nc',
'r')
lonisam1 = isam.variables['lon'][:]
#ric_i_f=isam.variables['totalyield'][96:103,:,:]
#riceb=N.average(ric_i_f,axis=0)
spam = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/spamsoy_isam.nc', 'r')
spamy = spam.variables['soyy_total'][:, :]
spampro = spam.variables['soyp_total'][:, :]
spamarea = spam.variables['soya_total'][:, :]
from mpl_toolkits.basemap import Basemap, cm, shiftgrid,interp
from netCDF4 import Dataset as NetCDFFile
import numpy as N
import matplotlib.pyplot as plt
import glob
import numpy.ma as ma
from scipy.interpolate import griddata
from scipy.stats import ttest_ind
region1=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/RCP45_crop_150901.nc','r')
maitrop = region1.variables['soy_trop'][4,:,:]
maitemp = region1.variables['soy_temp'][4,:,:]
maitropi = region1.variables['soy_trop_irrig'][4,:,:]
maitempi = region1.variables['soy_temp_irrig'][4,:,:]
maitrop= ma.masked_where(maitrop<=0,maitrop)
maitropi= ma.masked_where(maitropi<=0,maitropi)
maitemp= ma.masked_where(maitemp<=0,maitemp)
maitempi= ma.masked_where(maitempi<=0,maitempi)
maitrop=ma.filled(maitrop, fill_value=0.)
maitropi=ma.filled(maitropi,fill_value=0.)
maitemp=ma.filled(maitemp, fill_value=0.)
maitempi=ma.filled(maitempi, fill_value=0.)
maizeto = maitrop+maitemp
maizetoi = maitropi+maitempi
maitoatemp=maitemp+maitempi
maitoatrop=maitrop+maitropi
maizetotal = maizeto+maizetoi
#clm=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/soytrop_rcp45_constco2_rf_nofert_0.5x0.5.nc','r')
#clm=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/soytrop_rcp45_co2_rf_nofert_0.5x0.5.nc','r')
clmy=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/soytrop_rcp45_co2_rf_fert_0.5x0.5.nc','r')
clm=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/soytrop_rcp45_co2_irrig_fert_0.5x0.5.nc','r')
def annualyield(year, couna, counb):
bb = year - 2006
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/RCP45_crop_150901.nc',
'r')
maitrop = region1.variables['maize_trop'][bb, :, :]
maitemp = region1.variables['maize_temp'][bb, :, :]
maitropi = region1.variables['maize_trop_irrig'][bb, :, :]
maitempi = region1.variables['maize_temp_irrig'][bb, :, :]
gridarea = region1.variables['area'][:, :]
maitrop = ma.masked_where(maitrop <= 0, maitrop)
maitrop = ma.filled(maitrop, fill_value=0.)
maitemp = ma.masked_where(maitemp <= 0, maitemp)
maitemp = ma.filled(maitemp, fill_value=0.)
maitropi = ma.masked_where(maitropi <= 0, maitropi)
maitropi = ma.filled(maitropi, fill_value=0.)
maitempi = ma.masked_where(maitempi <= 0, maitempi)
maitempi = ma.filled(maitempi, fill_value=0.)
maizetro = maitrop + maitropi
maizetem = maitemp + maitempi
maizetor = maitrop + maitemp
maizetoi = maitropi + maitempi
maizeto = maitrop + maitemp + maitropi + maitempi
clm = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_constco2_rf_nofert_0.5x0.5.nc',
'r')
clmtropf = clm.variables['yield'][bb, :, :]
clm1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetemp_rcp45_constco2_rf_nofert_0.5x0.5.nc',
'r')
clmtempf = clm1.variables['yield'][bb, :, :]
clm2 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_co2_rf_nofert_0.5x0.5.nc',
'r')
clmtropfi = clm2.variables['yield'][bb, :, :]
clm3 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetemp_rcp45_co2_rf_nofert_0.5x0.5.nc',
'r')
clmtempfi = clm3.variables['yield'][bb, :, :]
clmtropf = N.flipud(clmtropf)
clmtempf = N.flipud(clmtempf)
clmtropfi = N.flipud(clmtropfi)
clmtempfi = N.flipud(clmtempfi)
clmtropf = ma.masked_where(maizetro <= 0, clmtropf)
clmtempf = ma.masked_where(maizetem <= 0, clmtempf)
clmtropf = ma.filled(clmtropf, fill_value=0.)
clmtempf = ma.filled(clmtempf, fill_value=0.)
clmtropfi = ma.masked_where(maizetro <= 0, clmtropfi)
clmtempfi = ma.masked_where(maizetem <= 0, clmtempfi)
clmtropfi = ma.filled(clmtropfi, fill_value=0.)
clmtempfi = ma.filled(clmtempfi, fill_value=0.)
yield_clmtf = clmtropf + clmtempf
yield_clmtf = ma.masked_where(yield_clmtf <= 0, yield_clmtf)
yield_clmtf = ma.masked_where(maizeto <= 0, yield_clmtf)
yield_clmtf = ma.filled(yield_clmtf, fill_value=0.)
yield_clmtfi = clmtropfi + clmtempfi
yield_clmtfi = ma.masked_where(yield_clmtfi <= 0, yield_clmtfi)
yield_clmtfi = ma.masked_where(maizeto <= 0, yield_clmtfi)
yield_clmtfi = ma.filled(yield_clmtfi, fill_value=0.)
clmn = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_co2_rf_fert_0.5x0.5.nc',
'r')
clmtropfn = clmn.variables['yield'][bb, :, :]
clm1n = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetemp_rcp45_co2_rf_fert_0.5x0.5.nc',
'r')
clmtempfn = clm1n.variables['yield'][bb, :, :]
clm2n = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_co2_irrig_fert_0.5x0.5.nc',
'r')
clmtropfin = clm2n.variables['yield'][bb, :, :]
clm3n = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetemp_rcp45_co2_irrig_fert_0.5x0.5.nc',
'r')
clmtempfin = clm3n.variables['yield'][bb, :, :]
clmtropfn = N.flipud(clmtropfn)
clmtempfn = N.flipud(clmtempfn)
clmtropfin = N.flipud(clmtropfin)
clmtempfin = N.flipud(clmtempfin)
clmtropfn = ma.masked_where(maizetro <= 0, clmtropfn)
clmtempfn = ma.masked_where(maizetem <= 0, clmtempfn)
clmtropfn = ma.filled(clmtropfn, fill_value=0.)
clmtempfn = ma.filled(clmtempfn, fill_value=0.)
clmtropfin = ma.masked_where(maizetro <= 0, clmtropfin)
clmtempfin = ma.masked_where(maizetem <= 0, clmtempfin)
clmtropfin = ma.filled(clmtropfin, fill_value=0.)
clmtempfin = ma.filled(clmtempfin, fill_value=0.)
yield_clmtfn = clmtropfn + clmtempfn
yield_clmtfn = ma.masked_where(yield_clmtfn <= 0, yield_clmtfn)
yield_clmtfn = ma.masked_where(maizeto <= 0, yield_clmtfn)
yield_clmtfn = ma.filled(yield_clmtfn, fill_value=0.)
yield_clmtfin = clmtropfin + clmtempfin
yield_clmtfin = ma.masked_where(yield_clmtfin <= 0, yield_clmtfin)
yield_clmtfin = ma.masked_where(maizeto <= 0, yield_clmtfin)
yield_clmtfin = ma.filled(yield_clmtfin, fill_value=0.)
base = NetCDFFile(
"/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamrcp45/heat/maizetemp_rcp45_constco2_rf_nofert_0.5x0.5.nc",
mode='r')
base2 = NetCDFFile(
"/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamrcp45/heat/maizetemp_rcp45_co2_rf_nofert_0.5x0.5.nc",
mode='r')
lona1 = base.variables["lon"][:]
lata1 = base.variables["lat"][:]
yieldf = base.variables["totalyield"][bb, :, :]
yieldfa = base2.variables["totalyield"][bb, :, :]
basei = NetCDFFile(
"/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamrcp45/heat/maizetemp_rcp45_co2_rf_fert_0.5x0.5.nc",
mode='r')
base2i = NetCDFFile(
"/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamrcp45/heat/maizetemp_rcp45_co2_irrig_fert_0.5x0.5.nc",
mode='r')
yieldfi = basei.variables["totalyield"][bb, :, :]
yieldfai = base2i.variables["totalyield"][bb, :, :]
baseif = NetCDFFile(
"/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamrcp45/heat/maizetrop_rcp45_co2_rf_fert_0.5x0.5.nc",
mode='r')
base2if = NetCDFFile(
"/scratch2/scratchdirs/tslin2/isam/cheyenne/yieldout/isamrcp45/heat/maizetrop_rcp45_co2_irrig_fert_0.5x0.5.nc",
mode='r')
yieldfitr = baseif.variables["totalyield"][bb, :, :]
yieldfaitr = base2if.variables["totalyield"][bb, :, :]
yielda = yieldf
yield_new, lona11 = shiftgrid(180.5, yielda, lona1, start=False)
yieldb = yieldfa
yield_new1, lona11 = shiftgrid(180.5, yieldb, lona1, start=False)
yieldai = yieldfi
yield_newi, lona11 = shiftgrid(180.5, yieldai, lona1, start=False)
yieldbi = yieldfai
yield_new1i, lona11 = shiftgrid(180.5, yieldbi, lona1, start=False)
yieldaitr = yieldfitr
yield_newitr, lona11 = shiftgrid(180.5, yieldaitr, lona1, start=False)
yieldbitr = yieldfaitr
yield_new1itr, lona11 = shiftgrid(180.5, yieldbitr, lona1, start=False)
yield_new[N.isnan(yield_new)] = -9999
yield_new = ma.masked_where(yield_new <= 0, yield_new)
yield_new = ma.masked_where(maizeto <= 0, yield_new)
yield_new = ma.filled(yield_new, fill_value=0.)
yield_new1[N.isnan(yield_new1)] = -9999
yield_new1 = ma.masked_where(yield_new1 <= 0, yield_new1)
yield_new1 = ma.masked_where(maizeto <= 0, yield_new1)
yield_new1 = ma.filled(yield_new1, fill_value=0.)
yield_newi[N.isnan(yield_newi)] = -9999
yield_newi = ma.masked_where(yield_newi <= 0, yield_newi)
yield_newi = ma.masked_where(maizetem <= 0, yield_newi)
yield_newi = ma.filled(yield_newi, fill_value=0.)
yield_new1i[N.isnan(yield_new1i)] = -9999
yield_new1i = ma.masked_where(yield_new1i <= 0, yield_new1i)
yield_new1i = ma.masked_where(maizetem <= 0, yield_new1i)
yield_new1i = ma.filled(yield_new1i, fill_value=0.)
yield_newitr[N.isnan(yield_newitr)] = -9999
yield_newitr = ma.masked_where(yield_newitr <= 0, yield_newitr)
yield_newitr = ma.masked_where(maizetro <= 0, yield_newitr)
yield_newitr = ma.filled(yield_newitr, fill_value=0.)
yield_new1itr[N.isnan(yield_new1i)] = -9999
yield_new1itr = ma.masked_where(yield_new1itr <= 0, yield_new1itr)
yield_new1itr = ma.masked_where(maizetro <= 0, yield_new1itr)
yield_new1itr = ma.filled(yield_new1itr, fill_value=0.)
yield_newi = yield_newi + yield_newitr
yield_new1i = yield_new1i + yield_new1itr
yieldagf = 0.
yieldg = 0.
harea = 0.
a = 0
yieldgid = 0.
yieldgia = 0.
yieldgib = 0.
yieldgic = 0.
yieldgca = 0.
yieldgcb = 0.
yieldgcc = 0.
yieldgcd = 0.
yieldagfa = 0.
yieldagfb = 0.
yieldagfc = 0.
yieldagfd = 0.
yieldagfa1 = 0.
yieldagfb1 = 0.
yieldagfc1 = 0.
yieldagfd1 = 0.
yieldagfa = yield_new
yieldagfb = yield_new1
yieldagfc = yield_newi
yieldagfd = yield_new1i
yieldagfa1 = yield_clmtf
yieldagfb1 = yield_clmtfi
yieldagfc1 = yield_clmtfn
yieldagfd1 = yield_clmtfin
yieldagfa = ma.masked_where(yieldagfa <= 0, yieldagfa)
yieldagfb = ma.masked_where(yieldagfb <= 0, yieldagfb)
yieldagfc = ma.masked_where(yieldagfc <= 0, yieldagfc)
yieldagfd = ma.masked_where(yieldagfd <= 0, yieldagfd)
yieldagfa1 = ma.masked_where(yieldagfa1 <= 0, yieldagfa1)
yieldagfb1 = ma.masked_where(yieldagfb1 <= 0, yieldagfb1)
yieldagfc1 = ma.masked_where(yieldagfc1 <= 0, yieldagfc1)
yieldagfd1 = ma.masked_where(yieldagfd1 <= 0, yieldagfd1)
maizeto = ma.masked_where(maizeto <= 0, maizeto)
return yieldagfa, yieldagfa1, yieldagfb, yieldagfb1, yieldagfc, yieldagfc1, yieldagfd, yieldagfd1, maizeto
def _append_data(drizzle_data, results):
"""Save retrieved fields to the drizzle_data object."""
for key, value in results.items():
value = ma.masked_where(value == 0, value)
drizzle_data.append_data(value, key)
from netCDF4 import Dataset as NetCDFFile
import numpy as N
import matplotlib.pyplot as plt
import glob
import numpy.ma as ma
from scipy.interpolate import griddata
from scipy.stats import ttest_ind
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/RCP45_crop_150901.nc',
'r')
maitrop = region1.variables['maize_trop'][4, :, :]
maitemp = region1.variables['maize_temp'][4, :, :]
maitropi = region1.variables['maize_trop_irrig'][4, :, :]
maitempi = region1.variables['maize_temp_irrig'][4, :, :]
maitrop = ma.masked_where(maitrop <= 0, maitrop)
maitropi = ma.masked_where(maitropi <= 0, maitropi)
maitemp = ma.masked_where(maitemp <= 0, maitemp)
maitempi = ma.masked_where(maitempi <= 0, maitempi)
maitrop = ma.filled(maitrop, fill_value=0.)
maitropi = ma.filled(maitropi, fill_value=0.)
maitemp = ma.filled(maitemp, fill_value=0.)
maitempi = ma.filled(maitempi, fill_value=0.)
maizeto = maitrop + maitemp
maizetoi = maitropi + maitempi
maitoatemp = maitemp + maitempi
maitoatrop = maitrop + maitropi
maizetotal = maizeto + maizetoi
clm = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45rcp45/maizetrop_rcp45_constco2_rf_nofert_0.5x0.5.nc',
'r')
def main():
root = Tk()
root.geometry("300x300")
root.withdraw()
s = simpledialog.askstring(
title=
"1. Print individual file data 2. Print file data and visual overlay 3. Directory",
prompt="select:")
if s == '1':
f1 = simpledialog.askstring(title="file1", prompt="Computer Filepath:")
f2 = simpledialog.askstring(title="file2",
prompt="Radiologist Filepath:")
process(f1, f2)
if s == '2':
print(
"Please input entire file directory if the files are not in the directory of this program"
)
#f1 = input("Input computer generated image filename: ")
#print(f1)
#f2 = input("Input radiologist image filename: ")
#print(f2)
master = Tk()
master.withdraw()
f1 = simpledialog.askstring(title="file1", prompt="Computer Filepath:")
f2 = simpledialog.askstring(title="file2",
prompt="Radiologist Filepath:")
print(
"Input the given slice value for the image display. If you want the program to generate this value for you please input 0"
)
slice = simpledialog.askinteger(title="Slice", prompt="Input slice :")
#global cf
process(f1, f2)
cimage = sitk.ReadImage(f1)
rimage = sitk.ReadImage(f2)
if cimage.GetDepth() != rimage.GetDepth():
print(
"These files have different dimensions, please input files with equal dimensions"
)
return
curr = 0
max = 0
#slice
if slice == 0:
# Find slice of Nifti image
for i in range(0, cimage.GetDepth()):
curr = 0
for j in range(0, cimage.GetHeight()):
for t in range(0, cimage.GetWidth()):
val = cimage.GetPixel(t, j, i)
if val > 0:
curr += 1
if curr > max:
# print("Values: ", i, curr)
max = curr
slice = i
# convert to array
ver1 = sitk.GetArrayFromImage(cimage)
ver2 = sitk.GetArrayFromImage(rimage)
A = ver1[slice, :, :]
B = ver2[slice, :, :]
cv2.imwrite("computer_2d_slice.jpg", A)
cv2.imwrite("radiologist_2d_slice.jpg", B)
c2d = sitk.ReadImage("computer_2d_slice.jpg")
r2d = sitk.ReadImage("radiologist_2d_slice.jpg")
RAR = sitk.GetArrayFromImage(c2d)
RAR2 = sitk.GetArrayFromImage(r2d)
mask = ma.masked_where(RAR2 > 0, RAR)
Image2_mask = ma.masked_array(RAR2, mask)
plt.imshow(RAR, cmap='Reds')
plt.imshow(Image2_mask, cmap='Blues', alpha=0.5)
plt.title('Image Overlay')
plt.show()
if s == '3':
#comp_dir = simpledialog.askstring(title="Test", prompt="Computer Filepath:")
#rad_dir = simpledialog.askstring(title="Test", prompt="Radiologist Filepath:")
accept = Tk()
comp_dir = filedialog.askdirectory(
parent=accept,
initialdir=os.getcwd(),
title="Please select computer algr folder:")
rad_dir = filedialog.askdirectory(
parent=accept,
initialdir=os.getcwd(),
title="Please select radiologist folder:")
# Iterate through both directories and add the values to their respective lists
for roots, dirs, files in os.walk(comp_dir):
for file in files:
#print("File = %s" % file)
filename = comp_dir + "/" + file
#curr = sitk.ReadImage(filename)
comp_list.append(filename)
for roots, dirs, files in os.walk(rad_dir):
for file in files:
#print("File = %s" % file)
filename = rad_dir + "/" + file
#curr = sitk.ReadImage(filename)
rad_list.append(filename)
#print(len(comp_list))
#print(len(rad_list))
# Sort the lists
comp_list.sort()
rad_list.sort()
# Create array of records
records = []
totalIOU = []
totalDice = []
totalHaus = []
totalMCC = []
totalACC = []
avgStats = []
stdStats = []
avgStats.append('Average')
avgStats.append('')
stdStats.append('STD DEV')
stdStats.append('')
# Main loop
for x, y in zip(comp_list, rad_list):
print("==========")
print(x) # X is comp
print(y) # Y is rad
currVal = process(x, y)
if currVal[2] != 0:
totalIOU.append(currVal[2])
totalDice.append(currVal[3])
totalHaus.append(currVal[4])
totalMCC.append(currVal[5])
totalACC.append(currVal[6])
records.append(currVal)
avgStats.append(Average(totalIOU))
avgStats.append(Average(totalDice))
avgStats.append(Average(totalHaus))
avgStats.append(Average(totalMCC))
avgStats.append(Average(totalACC))
stdStats.append(np.std(totalIOU))
stdStats.append(np.std(totalDice))
stdStats.append(np.std(totalHaus))
stdStats.append(np.std(totalMCC))
stdStats.append(np.std(totalACC))
records.append(avgStats)
records.append(stdStats)
# Write to csv
with open(inScan, 'w', newline='') as csvfile:
fieldnames = [
'key', 'filename', 'IOU', 'Dice_Coefficient', 'Hausdorff',
'MCC', 'ACC'
]
# Dictionary writer.
thewriter = csv.DictWriter(csvfile, fieldnames=fieldnames)
thewriter.writeheader()
for disk in records:
thewriter.writerow({
'key': disk[0],
'filename': disk[1],
'IOU': disk[2],
'Dice_Coefficient': disk[3],
'Hausdorff': disk[4],
'MCC': disk[5],
'ACC': disk[6]
})
return
def _import_sgas(self, fhandle, kwlist, metadata, grid, date, fracture):
"""Import SGAS; this may be lack of oil/water (need to verify)"""
s_exists = _chk_kw_date(kwlist, "SGAS", date)
logger.info("SGAS: S_EXISTS %s for date %s", s_exists, date)
flag = 0
if s_exists or metadata["IPHS"] in (0, 5, 7, -2345):
import_eclbinary(
self,
fhandle,
name="SGAS{__}",
etype=5,
grid=grid,
date=date,
fracture=fracture,
_kwlist=kwlist,
)
elif metadata["IPHS"] == 6:
flag = 1
logger.info("SGAS: ask for SWAT")
swat = self.__class__()
import_eclbinary(
swat,
fhandle,
name="SWAT{__}",
etype=5,
grid=grid,
date=date,
fracture=fracture,
_kwlist=kwlist,
)
self.name = "SGAS" + "_" + str(date)
self._nrow = grid.nrow
self._ncol = grid.ncol
self._nlay = grid.nlay
self._date = date
self._values = swat._values * -1 + 1.0
del swat
elif metadata["IPHS"] in (1, 2, 3, 4):
flag = 1
logger.info("SGAS: asked for but 0% or 100%")
self.name = "SGAS" + "_" + str(date)
self._nrow = grid.nrow
self._ncol = grid.ncol
self._nlay = grid.nlay
self._date = date
if metadata["IPHS"] == 4:
self._values = np.ones(
(self._ncol, self._nrow, self.nlay), dtype=np.float64
)
else:
self._values = np.zeros(
(self._ncol, self._nrow, self.nlay), dtype=np.float64
)
if grid.dualporo and flag:
if fracture:
self.name = "SGASF" + "_" + str(date)
self._values[grid._dualactnum.values == 1] = 0.0
else:
self.name = "SGASM" + "_" + str(date)
self._values[grid._dualactnum.values == 2] = 0.0
gactnum = grid.get_actnum().values
self._values = ma.masked_where(gactnum < 1, self._values)
return 1
def plotnow_azelsig(fpath,
yrmoday,
chan,
var,
st_hour,
st_minute,
ed_hour,
ed_minute,
supply_index=False):
flp = select_h5(fpath, yrmoday, st_hour, st_minute, ed_hour, ed_minute)
fld = select_dat(fpath, yrmoday, st_hour, st_minute, ed_hour, ed_minute)
i = 0
while len(flp) < 3:
i += 1
flp = select_h5(fpath, yrmoday, st_hour,
int(st_minute) - i, ed_hour,
int(ed_minute) + i)
pp = get_h5_pointing(flp)
dd = get_demodulated_data_from_list(fld, supply_index=supply_index)
combined = combine_cofe_h5_pointing(dd, pp)
#synchronized data az and el values
az1, el1 = combined['az'], combined['el']
data = combined['sci_data'][chan][var]
steps = len(data)
#set az/el resolution
dx = 1.0
dy = 1.0
#set up bins/grid
x, y = np.arange(0., 360. + dx, dx), np.arange(0., 90. + dy, dy)
AZ, EL = np.meshgrid(x, y)
#small number for comparing floats
epsilon = 1e-6
#set up matrix for signal
z1 = np.zeros(len(x) * len(y))
sig = np.reshape(z1, (len(y), len(x)))
#set up matrix for keeping track of data points in single bin for averaging
z2 = np.zeros(len(x) * len(y))
count = np.reshape(z2, (len(y), len(x)))
for i in range(steps):
#round az/el points for comparison with grid
el1[i] = round_fraction(el1[i], dy)
az1[i] = round_fraction(az1[i], dx)
#find where data points belong in grid
iel = np.where(abs(y - el1[i]) < epsilon)[0][0]
iaz = np.where(abs(x - az1[i]) < epsilon)[0][0]
#add 1 each time data point lands in same bin
count[iel][iaz] += 1
#add total number of data values in bin
sig[iel][iaz] = sig[iel][iaz] + data[i]
#mask 0 count values so they dont show up in color plot
count = ma.masked_where(count == 0.0, count)
#take average of all data points in single bin
sig = sig / count
plt.pcolormesh(AZ, EL, sig)
plt.colorbar(label='Signal, V')
plt.clim(data.min(), data.max())
plt.axis([0., 360., 0., 90.])
plt.ylabel('elevation (deg)')
plt.xlabel('azimuth (deg)')
plt.title('channel %s %s data binned to azimuth and elevation, date %s' %
(chan, var, fld[-1][-21:-13]))
plt.grid()
plt.show()
if a == Min2:
Box_Vch = Vch
Box_N2fixF = N2fixF
else:
Box_Vch = vstack((Box_Vch, Vch))
Box_N2fixF = vstack((Box_N2fixF, N2fixF))
B1A[count] = B1
print(count, 'looptime =', round(time.time() - tl, 2), '(s)')
tl = time.time()
count = count + 1
Box_Vch_mask = ma.masked_where(Box_N2fixF > 6.1e-15, Box_Vch)
B1B = Max1 - argmin(Box_Vch[:, ::-1], axis=1) / (
Max1 / Step1
) #here Argmin gives first min value but at Ef=1, there are multiple and I want it to point to the last min, so I change left and right of Box_Vch
#that is the "1-" and ",[::-1,:]" part for
U = arange(size(Array2))
for i in U:
if i > 0:
if isnan(B1A[i]) == True and (B1A[i - 1] != Min1 and isnan(B1A[i - 1])
== False): # and (B1A[i-1]!=Min1):
B1A[i] = Min1
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#5.Plot
import numpy.ma as ma
fuz = 1 / 4.
if (len(sys.argv) < 2):
print("Usage: \n dataplot.py path_to_binfile [clamp value]")
sys.exit()
elif (len(sys.argv) > 2):
fuz = float(sys.argv[2])
binfile = sys.argv[1]
lengthdata = np.fromfile(binfile + '_lengths.bin', dtype="float32")
minLength = 300.0
winddata = np.fromfile(binfile + '_windings.bin', dtype="float32")
datasize = int(np.sqrt(lengthdata.shape[0]))
lengthdata = lengthdata.reshape(datasize, datasize)
winddata = winddata.reshape(datasize, datasize)
masked = ma.masked_where(lengthdata < minLength, winddata)
clampVal = np.mean(masked)
dev = ma.std(masked)
fractions = sorted([
-1. / 3, -2. / 3, -1. / 5, -2. / 5, -3. / 5, -4. / 5, -1. / 4, -1. / 2,
-3. / 4, -2. / 7, -3. / 7, -4. / 7, -5. / 7, -6. / 7, -3. / 8, -5. / 8,
-7. / 8
])
print(fractions)
cmap = plt.cm.get_cmap('nipy_spectral', 8192)
fig = plt.figure()
img = plt.imshow(masked.filled(0), clim=[-0.667, -0.595], cmap=cmap)
#img.set_cmap('nipy_spectral')
cbar = plt.colorbar(ticks=fractions)
cbar.ax.set_yticklabels(['$2/3$', r'$5/8$', r'$3/5$', r'$5/8$', '$0.62$'])
fig_size = fig.get_size_inches()
#do some setup for the figure
plt.rc('text', usetex=True)
plt.rcParams['text.latex.preamble'] = '\usepackage{relsize}'
plt.rc('font', family='sanserif')
plt.subplots_adjust(wspace=0.3)
plt.subplot(111, aspect='equal')
plt.xlim(left_edge, right_edge)
plt.ylim(bottom_edge, top_edge)
plt.xlabel('Pressure (GPa)')
plt.ylabel('Wave Speed (km/s)')
#plot v_s
vs_hist = ma.masked_where(vs_hist <= 0.0, vs_hist)
c = matplotlib.colors.LinearSegmentedColormap.from_list('vphi', [ (0, '#ffffff'), (0.2, '#eff3ff'), (0.4, '#bdd7e7'), (0.6, '#6baed6'), (0.8, '#3182bd'), (1.0, '#08519c') ] , gamma=gamma)
c.set_bad('w', alpha=1.0)
plt.imshow(vs_hist.transpose(), origin='low', cmap=c, interpolation='gaussian', alpha=.7,\
aspect=aspect_ratio, extent=[vs_xedge[0], vs_xedge[-1], vs_yedge[0], vs_yedge[-1]])
plt.plot(pressure/1.e9,seis_vs/1.e3,linestyle="--",color='k',linewidth=2.0,label='PREM')
#plot v_phi
vphi_hist = ma.masked_where(vphi_hist <= 0.0, vphi_hist)
c = matplotlib.colors.LinearSegmentedColormap.from_list('vphi', [ (0, '#ffffff'), (0.2, '#fee5d9'), (0.4, '#fcae91'), (0.6, '#fb6a4a'), (0.8, '#de2d26'), (1.0, '#a50f15') ] , gamma=gamma)
c.set_bad('w', alpha=1.0)
plt.imshow(vphi_hist.transpose(), origin='low', cmap=c, interpolation='gaussian', alpha=.7, \
aspect=aspect_ratio, extent=[vphi_xedge[0], vphi_xedge[-1], vphi_yedge[0], vphi_yedge[-1]])
plt.plot(pressure/1.e9,seis_vphi/1.e3,linestyle="--",color='k',linewidth=2.0,label='PREM')
#plot density
def compute(self, dataset_pool):
parcels = self.get_dataset()
residential_units = parcels.get_attribute(self.residential_units)
return ma.filled(parcels.get_attribute(self.built_sf) / \
ma.masked_where(residential_units==0, residential_units.astype(float32)), 0.0)
DATAindices = np.logical_and(LONindices, LATindices)
if np.any(DATAindices):
local_mean = ma.mean(DATA_subset[line][DATAindices])
if ma.is_masked(local_mean):
pass
else:
SUM.append(float(local_mean))
if len(SUM) == 0:
AVG[i, j] = 1e+20
else:
AVG[i, j] = np.mean(SUM)
#...
#set mask
AVG = ma.masked_where(AVG >= 1e+19, AVG)
#XC = ma.masked_where(AVG>=1e+19,XC)
#YC = ma.masked_where(AVG>=1e+19,YC)
#---
'''
#prepare current figure
plt.close('all')
plt.figure(figsize=(8,10),dpi=defdpi)
ax = plt.gca()
#prepare map
mymap.drawcoastlines(ax=ax,zorder=500)
#mymap.fillcontinents('0.9',ax=ax,zorder=499)
mymap.drawparallels(pars, dashes=(1, 1),
linewidth=0.15, labels=parslabels, ax=ax,zorder=501)
def r1rho_M61(r1rho_prime=None,
phi_ex=None,
kex=None,
spin_lock_fields2=None,
back_calc=None):
"""Calculate the R2eff values for the M61 model.
See the module docstring for details.
@keyword r1rho_prime: The R1rho_prime parameter value (R1rho with no exchange).
@type r1rho_prime: numpy float array of rank [NE][NS][NM][NO][ND]
@keyword phi_ex: The phi_ex parameter value (pA * pB * delta_omega^2).
@type phi_ex: numpy float array of rank [NE][NS][NM][NO][ND]
@keyword kex: The kex parameter value (the exchange rate in rad/s).
@type kex: float
@keyword spin_lock_fields2: The R1rho spin-lock field strengths squared (in rad^2.s^-2).
@type spin_lock_fields2: numpy float array of rank [NE][NS][NM][NO][ND]
@keyword back_calc: The array for holding the back calculated R1rho values. Each element corresponds to the combination of spin lock field.
@type back_calc: numpy float array of rank [NE][NS][NM][NO][ND]
"""
# Flag to tell if values should be replaced if numer is zero.
t_numer_zero = False
t_denom_zero = False
# Repetitive calculations (to speed up calculations).
kex2 = kex**2
# The numerator.
numer = phi_ex * kex
# Catch zeros (to avoid pointless mathematical operations).
# This will result in no exchange, returning flat lines.
if min(numer) == 0.0:
t_numer_zero = True
mask_numer_zero = masked_where(numer == 0.0, numer)
# Denominator.
denom = kex2 + spin_lock_fields2
# Catch math domain error of dividing with 0.
# This is when denom = 0.
mask_denom_zero = denom == 0.0
if any(mask_denom_zero):
t_denom_zero = True
denom[mask_denom_zero] = 1.0
# R1rho calculation.
back_calc[:] = r1rho_prime + numer / denom
# Replace data in array.
# If numer is zero.
if t_numer_zero:
back_calc[mask_numer_zero.mask] = r1rho_prime[mask_numer_zero.mask]
# If denom is zero.
if t_denom_zero:
back_calc[mask_denom_zero] = 1e100
# Catch errors, taking a sum over array is the fastest way to check for
# +/- inf (infinity) and nan (not a number).
if not isfinite(sum(back_calc)):
# Replaces nan, inf, etc. with fill value.
fix_invalid(back_calc, copy=False, fill_value=1e100)
znc = nclu.variables['level'][:]
lonnc = nclu.variables['longitude'][:]
timenc = nclu.variables['time'][:]
latnc = N.flipud(latnc)
ncvar_maizef = N.zeros((2160, 4320))
ncvar_maizef = ncvar_maize[0, 1, :, :]
ncvar_maize1 = ncvar_maize[0, 4, :, :]
ncvar_mask = N.zeros((2160, 4320))
ncvar_mask = ncvar_maize[0, 0, :, :]
ncvar_maizef[N.isnan(ncvar_maizef)] = -9999
ncvar_mask[N.isnan(ncvar_mask)] = -9999
ncvar_maize1[N.isnan(ncvar_maize1)] = -9999
ncvar_maizef = ma.masked_where(ncvar_maizef <= 0, ncvar_maizef)
#ncvar_maizef= ma.masked_where(ncvar_mask<0.01,ncvar_maizef)
ncvar_maizef = ma.masked_where(ncvar_maize1 <= 0, ncvar_maizef)
ncvar_maize1 = N.flipud(ncvar_maize1)
ncvar_maizef = N.flipud(ncvar_maizef)
ncvar_mask = N.flipud(ncvar_mask)
lon2, lat2 = N.meshgrid(gridlon, gridlat)
iyield = interp(ncvar_maizef, lonnc, latnc, lon2, lat2, order=1)
iarea = interp(ncvar_mask, lonnc, latnc, lon2, lat2, order=1)
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc',
'r')
def _import_soil(self, fhandle, kwlist, metadata, grid, date, fracture):
# pylint: disable=too-many-branches, too-many-statements
s_exists = _chk_kw_date(kwlist, "SOIL", date)
logger.info("SOIL: S_EXISTS %s for date %s", s_exists, date)
phases = metadata["IPHS"]
if not s_exists:
logger.info("SOIL does not exist for date %s, need to estimate...", date)
if metadata["SIMULATOR"] != "E100":
phases = 7 # just assume this; hope its works...
else:
logger.info("SOIL property exists for date %s", date)
flag = 0
if s_exists or phases in (0, -2345):
import_eclbinary(
self,
fhandle,
name="SOIL{__}",
etype=5,
grid=grid,
date=date,
fracture=fracture,
_kwlist=kwlist,
)
elif phases in (3, 5, 7):
sgas = None
swat = None
flag = 1
logger.info("Making SOIL from SWAT and/or SGAS ...")
if phases in (3, 7):
swat = self.__class__()
import_eclbinary(
swat,
fhandle,
name="SWAT{__}",
etype=5,
grid=grid,
date=date,
fracture=fracture,
_kwlist=kwlist,
)
if phases in (5, 7):
sgas = self.__class__()
import_eclbinary(
sgas,
fhandle,
name="SGAS{__}",
etype=5,
grid=grid,
date=date,
fracture=fracture,
_kwlist=kwlist,
)
self.name = "SOIL" + "_" + str(date)
self._nrow = grid.nrow
self._ncol = grid.ncol
self._nlay = grid.nlay
self._date = date
if phases == 7: # owg
self._values = swat._values * -1 - sgas._values + 1.0
self._values[self._values < 0.0] = 0.0
self._values[self._values > 1.0] = 1.0
elif phases == 5: # og
self._values = sgas._values * -1 + 1.0
elif phases == 3: # ow
self._values = swat._values * -1 + 1.0
if swat:
del swat
if sgas:
del sgas
elif phases in (1, 2, 4, 6):
flag = 1
logger.info("SOIL: asked for but 0% or 100%")
self.name = "SOIL" + "_" + str(date)
self._nrow = grid.nrow
self._ncol = grid.ncol
self._nlay = grid.nlay
self._date = date
if phases == 1:
self._values = np.ones(
(self._ncol, self._nrow, self.nlay), dtype=np.float64
)
else:
self._values = np.zeros(
(self._ncol, self._nrow, self.nlay), dtype=np.float64
)
if grid.dualporo and flag:
if fracture:
self.name = "SOILF" + "_" + str(date)
self._values[grid._dualactnum.values == 1] = 0.0
else:
self.name = "SOILM" + "_" + str(date)
self._values[grid._dualactnum.values == 2] = 0.0
gactnum = grid.get_actnum().values
self._values = ma.masked_where(gactnum < 1, self._values)
return 2
dat5 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/fixedirr/mai_cli/output/mai_cli.nc',
'r')
iyield5ynew = dat5.variables['g_ET'][0:115, 0, :, :]
iyield6ynew = N.zeros((115, 360, 720))
dat6 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/code/isamhiscru_mai_mirca_evp.nc',
'r')
lonisam1 = dat6.variables['lon'][:]
for z in range(0, 115):
aa = dat6.variables['yield'][z, :, :]
maizeto1r, lonisam2 = shiftgrid(0.5, aa, lonisam1, start=True)
iyield6ynew[z, :, :] = maizeto1r
iyield1ynew = ma.masked_where(iyield1ynew <= 0., iyield1ynew)
iyield2ynew = ma.masked_where(iyield2ynew <= 0., iyield2ynew)
iyield3ynew = ma.masked_where(iyield3ynew <= 0., iyield3ynew)
iyield4ynew = ma.masked_where(iyield4ynew <= 0., iyield4ynew)
iyield5ynew = ma.masked_where(iyield5ynew <= 0., iyield5ynew)
iyield6ynew = ma.masked_where(iyield6ynew <= 0., iyield6ynew)
eiyield1ynew = ma.masked_where(eiyield1ynew <= 0., eiyield1ynew)
eiyield2ynew = ma.masked_where(eiyield2ynew <= 0., eiyield2ynew)
eiyield3ynew = ma.masked_where(eiyield3ynew <= 0., eiyield3ynew)
eiyield4ynew = ma.masked_where(eiyield4ynew <= 0., eiyield4ynew)
eiyield5ynew = ma.masked_where(eiyield5ynew <= 0., eiyield5ynew)
for i in range(1, 6):
locals()['e{0}p'.format(i)] = N.zeros((nn, year))
locals()['e{0}y'.format(i)] = N.zeros((nn, year))
def test_testOddFeatures(self):
# Test of other odd features
x = arange(20)
x = x.reshape(4, 5)
x.flat[5] = 12
assert_(x[1, 0] == 12)
z = x + 10j * x
assert_(eq(z.real, x))
assert_(eq(z.imag, 10 * x))
assert_(eq((z * conjugate(z)).real, 101 * x * x))
z.imag[...] = 0.0
x = arange(10)
x[3] = masked
assert_(str(x[3]) == str(masked))
c = x >= 8
assert_(count(where(c, masked, masked)) == 0)
assert_(shape(where(c, masked, masked)) == c.shape)
z = where(c, x, masked)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is masked)
assert_(z[7] is masked)
assert_(z[8] is not masked)
assert_(z[9] is not masked)
assert_(eq(x, z))
z = where(c, masked, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
z = masked_where(c, x)
assert_(z.dtype is x.dtype)
assert_(z[3] is masked)
assert_(z[4] is not masked)
assert_(z[7] is not masked)
assert_(z[8] is masked)
assert_(z[9] is masked)
assert_(eq(x, z))
x = array([1., 2., 3., 4., 5.])
c = array([1, 1, 1, 0, 0])
x[2] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
c[0] = masked
z = where(c, x, -x)
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
assert_(eq(masked_where(greater(x, 2), x), masked_greater(x, 2)))
assert_(
eq(masked_where(greater_equal(x, 2), x),
masked_greater_equal(x, 2)))
assert_(eq(masked_where(less(x, 2), x), masked_less(x, 2)))
assert_(eq(masked_where(less_equal(x, 2), x), masked_less_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_where(equal(x, 2), x), masked_equal(x, 2)))
assert_(eq(masked_where(not_equal(x, 2), x), masked_not_equal(x, 2)))
assert_(eq(masked_inside(list(range(5)), 1, 3), [0, 199, 199, 199, 4]))
assert_(eq(masked_outside(list(range(5)), 1, 3), [199, 1, 2, 3, 199]))
assert_(
eq(
masked_inside(array(list(range(5)), mask=[1, 0, 0, 0, 0]), 1,
3).mask, [1, 1, 1, 1, 0]))
assert_(
eq(
masked_outside(array(list(range(5)), mask=[0, 1, 0, 0, 0]), 1,
3).mask, [1, 1, 0, 0, 1]))
assert_(
eq(
masked_equal(array(list(range(5)), mask=[1, 0, 0, 0, 0]),
2).mask, [1, 0, 1, 0, 0]))
assert_(
eq(
masked_not_equal(array([2, 2, 1, 2, 1], mask=[1, 0, 0, 0, 0]),
2).mask, [1, 0, 1, 0, 1]))
assert_(
eq(masked_where([1, 1, 0, 0, 0], [1, 2, 3, 4, 5]),
[99, 99, 3, 4, 5]))
atest = ones((10, 10, 10), dtype=np.float32)
btest = zeros(atest.shape, MaskType)
ctest = masked_where(btest, atest)
assert_(eq(atest, ctest))
z = choose(c, (-x, x))
assert_(eq(z, [1., 2., 0., -4., -5]))
assert_(z[0] is masked)
assert_(z[1] is not masked)
assert_(z[2] is masked)
x = arange(6)
x[5] = masked
y = arange(6) * 10
y[2] = masked
c = array([1, 1, 1, 0, 0, 0], mask=[1, 0, 0, 0, 0, 0])
cm = c.filled(1)
z = where(c, x, y)
zm = where(cm, x, y)
assert_(eq(z, zm))
assert_(getmask(zm) is nomask)
assert_(eq(zm, [0, 1, 2, 30, 40, 50]))
z = where(c, masked, 1)
assert_(eq(z, [99, 99, 99, 1, 1, 1]))
z = where(c, 1, masked)
assert_(eq(z, [99, 1, 1, 99, 99, 99]))
def _import_eclbinary_prop(
self, grid, fhandle, kwname, kwlen, kwtype, kwbyte, name, date, etype
):
"""Import the actual record"""
values = _eclbin.eclbin_record(fhandle, kwname, kwlen, kwtype, kwbyte)
self._isdiscrete = False
use_undef = xtgeo.UNDEF
self.codes = {}
if kwtype == "INTE":
self._isdiscrete = True
use_undef = xtgeo.UNDEF_INT
# make the code list
uniq = np.unique(values).tolist()
codes = dict(zip(uniq, uniq))
codes = {key: str(val) for key, val in codes.items()} # val: strings
self.codes = codes
else:
values = values.astype(np.float64) # cast REAL (float32) to float64
# arrays from Eclipse INIT or UNRST are usually for inactive values only.
# Use the ACTNUM index array for vectorized numpy remapping (need both C
# and F order)
gactnum = grid.get_actnum().values
gactindc = grid.actnum_indices
gactindf = grid.get_actnum_indices(order="F")
allvalues = (
np.zeros((self._ncol * self._nrow * self._nlay), dtype=values.dtype) + use_undef
)
msg = "\n"
msg = msg + "grid.actnum_indices.shape[0] = {}\n".format(
grid.actnum_indices.shape[0]
)
msg = msg + "values.shape[0] = {}\n".format(values.shape[0])
msg = msg + "ncol nrow nlay {} {} {}, nrow*nrow*nlay = {}\n".format(
self._ncol, self._nrow, self._nlay, self._ncol * self._nrow * self._nlay
)
logger.info(msg)
if gactindc.shape[0] == values.shape[0]:
allvalues[gactindf] = values
elif values.shape[0] == self._ncol * self._nrow * self._nlay: # often case for PORV
allvalues = values.copy()
else:
msg = (
"BUG somehow... Is the file corrupt? If not contact "
"the library developer(s)!\n" + msg
)
raise SystemExit(msg)
allvalues = allvalues.reshape((self._ncol, self._nrow, self._nlay), order="F")
allvalues = np.asanyarray(allvalues, order="C")
allvalues = ma.masked_where(gactnum < 1, allvalues)
self._values = allvalues
if etype == 1:
self._name = name
else:
self._name = name + "_" + str(date)
self._date = date
for i in xrange(1, len(data[:, 1])):
if data[i - 2, 1] <= data[i, 1] and data[i + 2, 1] <= data[i, 1]:
print(data[i, :])
fname2 = "Si-Ag-Si-channels-TotalR063.dat"
data2, data_spaced2 = load_data(fname2)
fname3 = "Si-Ag-Si-channels-TotalR081.dat"
data3, data_spaced3 = load_data(fname3)
isAll = False
#isAll = True
############################# Plotting ######################
import numpy.ma as ma
vals = ma.array(data_spaced)
mvals = ma.masked_where(np.nan in data_spaced, vals)
if isAll:
fig, axs = plt.subplots(3, figsize=(4, 6), sharex=True) #, sharey=True)
else:
fig, axs = plt.subplots(2, figsize=(4, 4), sharex=True) #, sharey=True)
AgSi = 0
SiAgSi = 1
SiAgSi2 = 2
for ax in axs:
ax.locator_params(axis='y', nbins=4)
# for label in ['left', 'right', 'top', 'bottom']:
# ax.spines[label].set_position(('outward',-1.3))
#ax.tick_params(axis='x', pad=30)
plotwidth = 2.0
resolution) + 'km_' + versionStr
_, _, lonsIS1, latsIS1, ice_thicknessIS1 = cF.getIS1gridded(
savePathIS1, labelStr, mapProj)
#IS2mask = ma.zeros((ice_thicknessIS2.shape))
#IS2mask[np.isfinite(ice_thicknessIS2)]=1
#IS1mask = ma.zeros((ice_thicknessIS1.shape))
#IS1mask[np.isfinite(ice_thicknessIS1)]=1
#ice_thicknessIS1 = rasterio.fill.fillnodata(ice_thicknessIS1, mask=IS1mask, max_search_distance=2, smoothing_iterations=0)
#ice_thicknessIS2 = rasterio.fill.fillnodata(ice_thicknessIS2, mask=IS2mask, max_search_distance=2, smoothing_iterations=3)
#ice_thicknessIS2zero=np.copy(ice_thicknessIS2)
ice_thicknessIS1 = ma.masked_where(np.isnan(ice_thicknessIS1),
ice_thicknessIS1)
ice_thicknessIS2 = ma.masked_where(np.isnan(ice_thicknessIS2),
ice_thicknessIS2)
#ice_thicknessIS2[where(np.isnan(ice_thicknessIS2))]=0
region_mask, xptsI, yptsI = cF.get_region_mask_sect('../../AncData/',
mapProj,
xypts_return=1)
#region_maskG = griddata((xptsI.flatten(), yptsI.flatten()), region_mask.flatten(), (xptsN, yptsN), method='nearest', rescale=True)
regions = [10, 11, 12, 13, 15]
ice_thicknessIS1 = ma.masked_where(~np.isin(region_mask, regions),
ice_thicknessIS1)
ice_thicknessIS2 = ma.masked_where(~np.isin(region_mask, regions),
ice_thicknessIS2)
def _import_eclbinary_dualporo(
self, grid, fhandle, kwname, kwlen, kwtype, kwbyte, name, date, etype, fracture
):
"""Import the actual record for dual poro scheme"""
#
# TODO, merge this with routine above!
# A lot of code duplication here, as this is under testing
#
values = _eclbin.eclbin_record(fhandle, kwname, kwlen, kwtype, kwbyte)
# arrays from Eclipse INIT or UNRST are usually for inactive values only.
# Use the ACTNUM index array for vectorized numpy remapping (need both C
# and F order)
gactnum = grid.get_actnum().values
gactindc = grid.get_dualactnum_indices(order="C", fracture=fracture)
gactindf = grid.get_dualactnum_indices(order="F", fracture=fracture)
indsize = gactindc.size
if kwlen == 2 * grid.ntotal:
indsize = grid.ntotal # in case of e.g. PORV which is for all cells
if not fracture:
values = values[:indsize]
else:
values = values[values.size - indsize :]
self._isdiscrete = False
self.codes = {}
if kwtype == "INTE":
self._isdiscrete = True
# make the code list
uniq = np.unique(values).tolist()
codes = dict(zip(uniq, uniq))
codes = {key: str(val) for key, val in codes.items()} # val: strings
self.codes = codes
else:
values = values.astype(np.float64) # cast REAL (float32) to float64
allvalues = (
np.zeros((self._ncol * self._nrow * self._nlay), dtype=values.dtype)
+ self.undef
)
if gactindc.shape[0] == values.shape[0]:
allvalues[gactindf] = values
elif values.shape[0] == self._ncol * self._nrow * self._nlay: # often case for PORV
allvalues = values.copy()
else:
msg = (
"BUG somehow reading binary Eclipse! Is the file corrupt? If not contact "
"the library developer(s)!\n"
)
raise SystemExit(msg)
allvalues = allvalues.reshape((self._ncol, self._nrow, self.nlay), order="F")
allvalues = np.asanyarray(allvalues, order="C")
if self._dualporo:
if fracture:
allvalues[grid._dualactnum.values == 1] = 0.0
else:
allvalues[grid._dualactnum.values == 2] = 0.0
allvalues = ma.masked_where(gactnum < 1, allvalues)
self._values = allvalues
append = ""
if self._dualporo:
append = "M"
if fracture:
append = "F"
logger.info("Dual status is %s, and append is %s", self._dualporo, append)
if etype == 1:
self._name = name + append
else:
self._name = name + append + "_" + str(date)
self._date = date
import numpy as np
from numpy import ma
from matplotlib import pyplot as plt
n = 12
x = np.linspace(-1.5, 1.5, n)
y = np.linspace(-1.5, 1.5, n * 2)
X, Y = np.meshgrid(x, y)
Qx = np.cos(Y) - np.cos(X)
Qz = np.sin(Y) + np.sin(X)
Qx = (Qx + 1.1)
Z = np.sqrt(X**2 + Y**2) / 5
Z = (Z - Z.min()) / (Z.max() - Z.min())
# The color array can include masked values:
Zm = ma.masked_where(np.fabs(Qz) < 0.5 * np.amax(Qz), Z)
fig = plt.figure()
ax = fig.add_subplot(121)
ax.set_facecolor("#bdb76b")
ax.pcolormesh(Qx, Qz, Z, shading='gouraud')
ax.set_title('Without masked values')
ax = fig.add_subplot(122)
ax.set_facecolor("#bdb76b")
col = ax.pcolormesh(Qx, Qz, Zm, shading='gouraud')
ax.set_title('With masked values')
serve_figure.serve_figure(fig, port=8888)
def r2eff_B14(r20a=None,
r20b=None,
pA=None,
dw=None,
dw_orig=None,
kex=None,
ncyc=None,
inv_tcpmg=None,
tcp=None,
back_calc=None):
"""Calculate the R2eff values for the CR72 model.
See the module docstring for details.
@keyword r20a: The R20 parameter value of state A (R2 with no exchange).
@type r20a: numpy float array of rank [NE][NS][NM][NO][ND]
@keyword r20b: The R20 parameter value of state B (R2 with no exchange).
@type r20b: numpy float array of rank [NE][NS][NM][NO][ND]
@keyword pA: The population of state A.
@type pA: float
@keyword dw: The chemical exchange difference between states A and B in rad/s.
@type dw: numpy float array of rank [NE][NS][NM][NO][ND]
@keyword dw_orig: The chemical exchange difference between states A and B in ppm. This is only for faster checking of zero value, which result in no exchange.
@type dw_orig: numpy float array of rank-1
@keyword kex: The kex parameter value (the exchange rate in rad/s).
@type kex: float
@keyword ncyc: The matrix exponential power array. The number of CPMG blocks.
@type ncyc: numpy int16 array of rank [NE][NS][NM][NO][ND]
@keyword inv_tcpmg: The inverse of the total duration of the CPMG element (in inverse seconds).
@type inv_tcpmg: numpy float array of rank [NE][NS][NM][NO][ND]
@keyword tcp: The tau_CPMG times (1 / 4.nu1).
@type tcp: numpy float array of rank [NE][NS][NM][NO][ND]
@keyword back_calc: The array for holding the back calculated R2eff values. Each element corresponds to one of the CPMG nu1 frequencies.
@type back_calc: numpy float array of rank [NE][NS][NM][NO][ND]
"""
# Flag to tell if values should be replaced if math function is violated.
t_dw_zero = False
t_max_e = False
t_v3_N_zero = False
t_log_tog_neg = False
t_v1c_less_one = False
# Catch parameter values that will result in no exchange, returning flat R2eff = R20 lines (when kex = 0.0, k_AB = 0.0).
# Test if pA or kex is zero.
if kex == 0.0 or pA == 1.0:
back_calc[:] = r20a
return
# Test if dw is zero. Create a mask for the affected spins to replace these with R20 at the end of the calculationWait for replacement, since this is spin specific.
if min(fabs(dw_orig)) == 0.0:
t_dw_zero = True
mask_dw_zero = masked_where(dw == 0.0, dw)
# Parameter conversions.
pB = 1.0 - pA
k_BA = pA * kex
k_AB = pB * kex
# Repetitive calculations (to speed up calculations).
deltaR2 = r20a - r20b
dw2 = dw**2
two_tcp = 2.0 * tcp
# The Carver and Richards (1972) alpha_minus short notation.
alpha_m = deltaR2 + k_AB - k_BA
zeta = 2.0 * dw * alpha_m
Psi = alpha_m**2 + 4.0 * k_BA * k_AB - dw2
# Get the real and imaginary components of the exchange induced shift.
# Trigonometric functions faster than square roots.
quad_zeta2_Psi2 = (zeta**2 + Psi**2)**0.25
fact = 0.5 * arctan2(-zeta, Psi)
g3 = cos(fact) * quad_zeta2_Psi2
g4 = sin(fact) * quad_zeta2_Psi2
# Repetitive calculations (to speed up calculations).
g32 = g3**2
g42 = g4**2
# Time independent factors.
# N = oG + oE.
N = g3 + g4 * 1j
NNc = g32 + g42
# F0.
F0 = (dw2 + g32) / NNc
# F2.
F2 = (dw2 - g42) / NNc
# t1 = (-dw + g4) * (complex(-dw, -g3)) / NNc #t1.
# t2.
F1b = (dw + g4) * (dw - g3 * 1j) / NNc
# t1 + t2.
F1a_plus_b = (2. * dw2 + zeta * 1j) / NNc
# Derived from relaxation.
# E0 = -2.0 * tcp * (F00R - f11R).
E0 = two_tcp * g3
# Catch math domain error of sinh(val > 710).
# This is when E0 > 710.
if max(E0) > 700:
t_max_e = True
mask_max_e = masked_greater_equal(E0, 700.0)
# To prevent math errors, set e_zero to 1.
E0[mask_max_e.mask] = 1.0
# Derived from chemical shifts #E2 = complex(0, -2.0 * tcp * (F00I - f11I)).
E2 = two_tcp * g4
# Mixed term (complex) (E0 - iE2)/2.
E1 = (g3 - g4 * 1j) * tcp
# Complex.
v1s = F0 * sinh(E0) - F2 * sin(E2) * 1j
# -2 * oG * t2.
v4 = F1b * (-alpha_m - g3) + F1b * (dw - g4) * 1j
# Complex.
ex1c = sinh(E1)
# Off diagonal common factor. sinh fuctions.
v5 = (-deltaR2 + kex + dw * 1j) * v1s - 2. * (v4 +
k_AB * F1a_plus_b) * ex1c
# Real. The v_1c in paper.
v1c = F0 * cosh(E0) - F2 * cos(E2)
# Catch math domain error of sqrt of negative.
# This is when v1c is less than 1.
mask_v1c_less_one = v1c < 1.0
if any(mask_v1c_less_one):
t_v1c_less_one = True
v1c[mask_v1c_less_one] = 1.0
# Exact result for v2v3.
v3 = sqrt(v1c**2 - 1.)
y = power((v1c - v3) / (v1c + v3), ncyc)
Tog_div = 2. * v3 * N
# Catch math domain error of division with 0.
# This is when Tog_div is zero.
mask_v3_N_zero = Tog_div == 0.0
if any(mask_v3_N_zero):
t_v3_N_zero = True
Tog_div[mask_v3_N_zero] = 1.0
Tog = 0.5 * (1. + y) + (1. - y) * v5 / Tog_div
## -1/Trel * log(LpreDyn).
# Rpre = (r20a + r20b + kex) / 2.0
## Carver and Richards (1972)
# R2eff_CR72 = Rpre - inv_tcpmg * ncyc * arccosh(v1c.real)
## Baldwin final.
# Estimate R2eff. relax_time = Trel = 1/inv_tcpmg.
# R2eff = R2eff_CR72 - inv_tcpmg * log(Tog.real)
# Catch math domain error of log of negative.
# This is when Tog.real is negative.
mask_log_tog_neg = Tog.real < 0.0
if any(mask_log_tog_neg):
t_log_tog_neg = True
Tog.real[mask_log_tog_neg] = 1.0
# Fastest calculation.
back_calc[:] = (r20a + r20b + kex) / 2.0 - inv_tcpmg * (
ncyc * arccosh(v1c.real) + log(Tog.real))
# Replace data in array.
# If dw is zero.
if t_dw_zero:
back_calc[mask_dw_zero.mask] = r20a[mask_dw_zero.mask]
# If E0 is above 700.
if t_max_e:
back_calc[mask_max_e.mask] = r20a[mask_max_e.mask]
# If v1c is less than 1.
if t_v1c_less_one:
back_calc[mask_v1c_less_one] = 1e100
# If Tog_div is zero.
if t_v3_N_zero:
back_calc[mask_v3_N_zero] = 1e100
# If Tog.real is negative.
if t_log_tog_neg:
back_calc[mask_log_tog_neg] = 1e100
# Catch errors, taking a sum over array is the fastest way to check for
# +/- inf (infinity) and nan (not a number).
if not isfinite(sum(back_calc)):
# Replaces nan, inf, etc. with fill value.
fix_invalid(back_calc, copy=False, fill_value=1e100)
iizumi=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/iizumi/iizumi.2013JAN29.maize.1982-2006.30min.nc4','r')
#print iizumi
iyield = iizumi.variables['yield50'][18,:,:]
iarea =iizumi.variables['area'][18,:,:]
la=iizumi.variables['lat'][:]
lo=iizumi.variables['lon'][:]
iyield=N.flipud(iyield)
iarea=N.flipud(iarea)
region1=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc','r')
maitrop = region1.variables['maize_trop'][99,:,:]
maitemp = region1.variables['maize_temp'][99,:,:]
maitropi=region1.variables['maize_trop_irrig'][99,:,:]
maitempi=region1.variables['maize_temp_irrig'][99,:,:]
maitrop=ma.masked_where(maitrop<=0,maitrop)
maitrop=ma.filled(maitrop, fill_value=0.)
maitemp=ma.masked_where(maitemp<=0,maitemp)
maitemp=ma.filled(maitemp, fill_value=0.)
maitropi=ma.masked_where(maitropi<=0,maitropi)
maitropi=ma.filled(maitropi, fill_value=0.)
maitempi=ma.masked_where(maitempi<=0,maitempi)
maitempi=ma.filled(maitempi, fill_value=0.)
maizetor=maitrop+maitemp
maizetoi=maitropi+maitempi
maizeto = maitrop+maitemp+maitropi+maitempi
clm=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/clm45historical/maizetrop_historical_co2_rf_fert_0.5x0.5.nc','r')
clmtrop = clm.variables['yield'][96:103,:,:]
clmtropf=N.average(clmtrop,axis=0)
gridarea1= area.variables['cell_area'][:,:]
gridlon = area.variables['lon'][:]
gridlat=area.variables['lat'][:]
#print gridlon
nclu=NetCDFFile('/scratch2/scratchdirs/tslin2/plot/globalcrop/data/m3yield_isam.nc','r')
ncvar_maize = nclu.variables['soyy'][0,:,:]
marea = nclu.variables['soy_area'][0,:,:]
region1=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/code/isamhiscru_soy_luh2.nc','r')
maitrop = region1.variables['area'][:,:,:]
lonisam1=region1.variables['lon'][:]
maitrop=ma.masked_where(maitrop<=0,maitrop)
maitrop=ma.filled(maitrop, fill_value=0.)
maizeto = maitrop
edat=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/equili/maiequilibrium/output/maiequilibrium.nc','r')
eiyield1ynew = edat.variables['totalyield'][0:115,:,:]
edat2=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/equili/maiequilibrium_irr/output/maiequilibrium_irr.nc','r')
eiyield2ynew = edat2.variables['totalyield'][0:115,:,:]
edat3=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/equili/maiequilibrium_fert/output/maiequilibrium_fert.nc','r')
eiyield3ynew = edat3.variables['totalyield'][0:115,:,:]
edat4=NetCDFFile('/scratch2/scratchdirs/tslin2/isam/maisoy_cheyenne/his_cru/equili/maiequilibrium_co2/output/maiequilibrium_co2.nc','r')
eiyield4ynew = edat4.variables['totalyield'][0:115,:,:]
znc = nclu.variables['level'][:]
lonnc = nclu.variables['longitude'][:]
timenc = nclu.variables['time'][:]
latnc = N.flipud(latnc)
ncvar_maizef = N.zeros((2160, 4320))
ncvar_maizef = ncvar_maize[0, 1, :, :]
ncvar_maize1 = ncvar_maize[0, 4, :, :]
ncvar_mask = N.zeros((2160, 4320))
ncvar_mask = ncvar_maize[0, 0, :, :]
ncvar_maizef[N.isnan(ncvar_maizef)] = -9999
ncvar_mask[N.isnan(ncvar_mask)] = -9999
ncvar_maize1[N.isnan(ncvar_maize1)] = -9999
ncvar_maizef = ma.masked_where(ncvar_maizef <= 0, ncvar_maizef)
#ncvar_maizef= ma.masked_where(ncvar_mask<0.01,ncvar_maizef)
ncvar_maizef = ma.masked_where(ncvar_maize1 <= 0, ncvar_maizef)
ncvar_maize1 = N.flipud(ncvar_maize1)
ncvar_maizef = N.flipud(ncvar_maizef)
ncvar_mask = N.flipud(ncvar_mask)
lon2, lat2 = N.meshgrid(gridlon, gridlat)
iyield = interp(ncvar_maizef, lonnc, latnc, lon2, lat2, order=1)
iarea = interp(ncvar_mask, lonnc, latnc, lon2, lat2, order=1)
region1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/clm/HistoricalGLM_crop_150901.nc',
'r')
def region(x, ryield):
region = NetCDFFile(
'/global/project/projectdirs/m1602/datasets4.full/arbit_init_state_05x05.nc',
'r')
ind = region.variables['REGION_MASK'][:, :]
area = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/gridareahalf_isam.nc',
'r')
gridarea1 = area.variables['cell_area']
isam1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/luh2area_850_2015_corrcrop.nc',
'r')
meareaisam = isam1.variables['fmai_tt'][1051:1166, :, :] #1901-2015
meareaisam = ma.masked_where(meareaisam <= 0.0, meareaisam)
isam1 = NetCDFFile(
'/scratch2/scratchdirs/tslin2/plot/globalcrop/data/mirca_isam.nc', 'r')
meareaisam1 = isam1.variables['amai_tt'][:, :] #mirca2000
meareaisam1 = ma.masked_where(meareaisam1 <= 0.0, meareaisam1)
areaa1 = N.zeros((x, 360, 720))
ind1 = N.zeros((x, 360, 720))
gridarea = N.zeros((x, 360, 720))
for i in range(0, x):
ind1[i, :, :] = ind[:, :]
gridarea[i, :, :] = gridarea1[:, :]
areaa1[i, :, :] = meareaisam1[:, :]
ryield = ma.masked_where(ind1 == 0.0, ryield)
gridarea = ma.masked_where(ind1 == 0.0, gridarea)
meareaisam = ma.masked_where(ind1 == 0.0, meareaisam)
areaa1 = ma.masked_where(ind1 == 0.0, areaa1)
isamp = ryield * meareaisam
isamp1 = ryield * areaa1
name = [
"Global", "NA", "SA", "EU", "Africa", "PD", "USSR", "China", "SSEA"
]
num = 9
allarea1 = N.zeros((num))
allproisam = N.zeros((num, x))
allyisam = N.zeros((num, x))
allgrid = N.zeros((num))
isamgrid = N.zeros((num, x))
luharea = N.zeros((num, x))
for idx in range(0, num):
isampmask = N.zeros((115, 360, 720))
isamarea = N.zeros((115, 360, 720))
areak = N.zeros((115, 360, 720))
areak = gridarea
isampmask = isamp1
isamarea = areaa1
if idx == 5:
idx = 9
if idx == 4:
isampmask = ma.masked_where(ind1 > 5.0, isampmask)
isamarea = ma.masked_where(ind1 > 5.0, isamarea)
areak = ma.masked_where(ind1 > 5.0, areak)
areak = ma.masked_where(ind1 < 4.0, areak)
isampmask = ma.masked_where(ind1 < 4.0, isampmask)
isamarea = ma.masked_where(ind1 < 4.0, isamarea)
elif idx == 0:
isampmask = ma.masked_where(ind1 == idx, isampmask)
isamarea = ma.masked_where(ind1 == idx, isamarea)
areak = ma.masked_where(ind1 == idx, areak)
else:
isampmask = ma.masked_where(ind1 != idx, isampmask)
isamarea = ma.masked_where(ind1 != idx, isamarea)
areak = ma.masked_where(ind1 != idx, areak)
if idx == 9:
idx = 5
luharea[idx, :] = N.sum(isamarea, axis=(1, 2))
allproisam[idx, :] = (N.sum(isampmask, axis=(1, 2)))
allyisam[idx, :] = N.sum(isampmask, axis=(1, 2)) / (N.sum(isamarea,
axis=(1, 2)))
isamgrid[idx, :] = (N.sum(isampmask, axis=(1, 2)) /
(N.sum(areak, axis=(1, 2))) * 10000)
return allproisam, allyisam, isamgrid
def get_pitch(filename):
from aubio import source, pitch
downsample = 1
samplerate = 44100 // downsample
win_s = 4096 // downsample # fft size
hop_s = 512 // downsample # hop size
s = source(filename, samplerate, hop_s)
samplerate = s.samplerate
tolerance = 0.8
pitch_o = pitch("yin", win_s, hop_s, samplerate)
pitch_o.set_unit("midi")
pitch_o.set_tolerance(tolerance)
pitches = []
confidences = []
# total number of frames read
total_frames = 0
while True:
samples, read = s()
pitch = pitch_o(samples)[0]
#pitch = int(round(pitch))
confidence = pitch_o.get_confidence()
#if confidence < 0.8: pitch = 0.
#print("%f %f %f" % (total_frames / float(samplerate), pitch, confidence))
pitches += [pitch]
confidences += [confidence]
total_frames += read
if read < hop_s: break
if 0: sys.exit(0)
#print pitches
import os.path
from numpy import array, ma
import matplotlib.pyplot as plt
from demo_waveform_plot import get_waveform_plot, set_xlabels_sample2time
skip = 1
pitches = array(pitches[skip:])
confidences = array(confidences[skip:])
times = [t * hop_s for t in range(len(pitches))]
fig = plt.figure()
ax1 = fig.add_subplot(311)
ax1 = get_waveform_plot(filename,
samplerate=samplerate,
block_size=hop_s,
ax=ax1)
plt.setp(ax1.get_xticklabels(), visible=False)
ax1.set_xlabel('')
def array_from_text_file(filename, dtype='float'):
filename = os.path.join(os.path.dirname(__file__), filename)
return array([line.split() for line in open(filename).readlines()],
dtype=dtype)
ax2 = fig.add_subplot(312, sharex=ax1)
ground_truth = os.path.splitext(filename)[0] + '.f0.Corrected'
if os.path.isfile(ground_truth):
ground_truth = array_from_text_file(ground_truth)
true_freqs = ground_truth[:, 2]
true_freqs = ma.masked_where(true_freqs < 2, true_freqs)
true_times = float(samplerate) * ground_truth[:, 0]
ax2.plot(true_times, true_freqs, 'r')
ax2.axis(ymin=0.9 * true_freqs.min(), ymax=1.1 * true_freqs.max())
ax2.plot(times, pitches, '.g')
cleaned_pitches = pitches
cleaned_pitches = ma.masked_where(confidences < tolerance, cleaned_pitches)
ax2.plot(times, cleaned_pitches, '.-')
plt.setp(ax2.get_xticklabels(), visible=False)
ax2.set_ylabel('f0 (midi)')
ax3 = fig.add_subplot(313, sharex=ax1)
ax3.plot(times, confidences)
ax3.plot(times, [tolerance] * len(confidences))
ax3.axis(xmin=times[0], xmax=times[-1])
ax3.set_ylabel('condidence')
set_xlabels_sample2time(ax3, times[-1], samplerate)
# plt.show()
plt.savefig(filename.replace('.mp3', '.pdf'))
significant = confidences > tolerance
return array(
times)[significant], pitches[significant], confidences[significant]
