def demarkate_process(self, frac_nodes_end, frac_msg_end): #------------------------------------------------- # create two mat, msg_end, msg_exo # nodes_end , nodes_exo # frac_nodes_end , frac_msg_end # return nodes_end , msg_end #------------------------------------------------- max_end_user = int(frac_nodes_end*self.num_nodes) max_end_msg = int(frac_msg_end * self.num_train) nodes_end = ma.make_mask(np.zeros(self.num_nodes)) nodes_exo = ma.make_mask(np.ones(self.num_nodes)) msg_end = ma.make_mask(np.zeros(self.num_train)) msg_exo = ma.make_mask(np.ones(self.num_train)) while msg_end.count() < max_end_msg: # check count if nodes_end.count() < max_end_user: msg_no , user = self.find_argmax(nodes_end, nodes_exo, msg_end, msg_exo) nodes_exo[user] = False #.remove(user) nodes_end[user] = True #.insert(user) else: msg_no = self.find_argmax() msg_exo[ msg_no ] = False #.remove(msg) msg_end[ msg_no ] = True #.insert(msg) self.mask_nodes_end = nodes_end self.mask_msg_end= msg_end return
def demarkate_process(self, frac_nodes_end, frac_msg_end):
#---------------CHANGE----------------------------------
# create two mat, msg_end, msg_exo
# nodes_end , nodes_exo
# frac_nodes_end , frac_msg_end
# return nodes_end , msg_end
#-------------------------------------------------
max_end_user = int(frac_nodes_end * self.num_nodes)
max_end_msg = int(frac_msg_end * self.num_train)
self.nodes_end = ma.make_mask(np.zeros(self.num_nodes))
self.nodes_exo = ma.make_mask(np.ones(self.num_nodes))
self.msg_end = ma.make_mask(np.zeros(self.num_train))
self.msg_exo = ma.make_mask(np.ones(self.num_train))
self.create_influence_matrix() # self.influence_matrix
self.create_covariance_matrix() # self.covariance
self.create_function_val() # self.curr_function_val
while np.count_nonzero(self.msg_end) < max_end_msg:
if np.count_nonzero(self.nodes_end) < max_end_user:
msg_no, user = self.obtain_most_endogenius_msg_user()
self.update(msg_no, user)
else:
msg_no = self.obtain_most_endogenius_msg_user(
flag_send_msg_only=True)
self.update(msg_no, user=-1)
# -----------------------------------------------
# delete extra files
if hasattr(self, 'covariance'):
del self.covariance
if hasattr(self, 'influence_matrix'):
del self.influence_matrix
return
def regions(cube,clim=False,maxf=True):
mons = 4
lag = 1
max_i = 35 + lag
max_f = max_i + mons
min_i = 11 + lag
min_f = min_i + mons
if maxf:
forc_i = max_i
forc_f = max_f
else:
forc_i = min_i
forc_f = min_f
if clim:
cube_max = cube.collapsed('time',iris.analysis.MEAN)
else:
cube_max = cube[forc_i:forc_f,:,::].collapsed('time',iris.analysis.MEAN)
# cube_anom = cube-cube_mean
cube_max.coord('latitude').guess_bounds()
cube_max.coord('longitude').guess_bounds()
print cube_max.shape
# ---------- define sst/tland----------
lsmask = iris.load_cube(ncfile_path + 'lsmask.nc')[0,0,::]
landmask = ~(ma.make_mask(lsmask.data.copy()) + np.zeros(cube_max.shape)).astype(bool) # mask sea, show land
# mask out the enso region: masked = 1, not masked = 0
# mask from x = 43:73, y = 32:40
enso_mask = np.zeros(cube_max.shape)
enso_mask[0,30:42,43:75] = 1
seamask = (ma.make_mask(lsmask.data.copy()) + np.zeros(cube_max.shape) + enso_mask).astype(bool) # mask land, show sea
Cocean = cube_max.copy()
Cland = cube_max.copy()
Cocean.data = ma.array(Cocean.data, mask=seamask)
Cland.data = ma.array(Cland.data, mask=landmask)
# --------------
land_cube = Cland
ocean_cube = Cocean
oclat10, oclat10_mean = regmean(ocean_cube,loni=0,lonf=360,lati=-10,latf=10)
oclatp20, oclatp20_mean = regmean(ocean_cube,loni=0,lonf=360,lati=10,latf=20)
oclatm20, oclatm20_mean = regmean(ocean_cube,loni=0,lonf=360,lati=-20,latf=-10)
oclatp30, oclatp30_mean = regmean(ocean_cube,loni=0,lonf=360,lati=20,latf=30)
oclatm30, oclatm30_mean = regmean(ocean_cube,loni=0,lonf=360,lati=-30,latf=-20)
lat10, lat10_mean = regmean(land_cube,loni=0,lonf=360,lati=-10,latf=10)
latp20, latp20_mean = regmean(land_cube,loni=0,lonf=360,lati=10,latf=20)
latm20, latm20_mean = regmean(land_cube,loni=0,lonf=360,lati=-20,latf=-10)
latp30, latp30_mean = regmean(land_cube,loni=0,lonf=360,lati=20,latf=30)
latm30, latm30_mean = regmean(land_cube,loni=0,lonf=360,lati=-30,latf=-20)
# lat20mean = latp20
return [oclat10_mean, lat10_mean, oclatp20_mean, latp20_mean, oclatm20_mean, latm20_mean, oclatp30_mean, latp30_mean, oclatm30_mean, latm30_mean]
def regions(cube,clim=False,maxf=True):
mons = 4
lag = 1
max_i = 35 + lag
max_f = max_i + mons
min_i = 11 + lag
min_f = min_i + mons
if maxf:
forc_i = max_i
forc_f = max_f
else:
forc_i = min_i
forc_f = min_f
if clim:
cube_max = cube.collapsed('time',iris.analysis.MEAN)
else:
cube_max = cube[forc_i:forc_f,:,::].collapsed('time',iris.analysis.MEAN)
# cube_anom = cube-cube_mean
cube_max.coord('latitude').guess_bounds()
cube_max.coord('longitude').guess_bounds()
print cube_max.shape
# ---------- define sst/tland----------
lsmask = iris.load_cube(ncfile_path + 'lsmask.nc')[0,0,::]
landmask = ~(ma.make_mask(lsmask.data.copy()) + np.zeros(cube_max.shape)).astype(bool) # mask sea, show land
seamask = (ma.make_mask(lsmask.data.copy()) + np.zeros(cube_max.shape)).astype(bool) # mask land, show sea
Cocean = cube_max.copy()
Cland = cube_max.copy()
Cocean.data = ma.array(Cocean.data, mask=seamask)
Cland.data = ma.array(Cland.data, mask=landmask)
# --------------
land_cube = Cland
India, India_mean = regmean(land_cube,loni=60,lonf=75,lati=0,latf=30)
MC, MC_mean = regmean(land_cube,loni=90,lonf=140,lati=-10,latf=10)
TropSthAm, TropSthAm_mean = regmean(land_cube,loni=290,lonf=315,lati=-23,latf=0)
SthSthAm, SthSthAm_mean = regmean(land_cube,loni=270,lonf=315,lati=-60,latf=-24)
NthWestAfr, NthWestAfr_mean = regmean(land_cube,loni=-15,lonf=15,lati=10,latf=30)
NthEastAfr, NthEastAfr_mean = regmean(land_cube,loni=15,lonf=50,lati=10,latf=30)
TropAfr, TropAfr_mean = regmean(land_cube,loni=12,lonf=40,lati=-15,latf=5)
SthAfr, SthAfr_mean = regmean(land_cube,loni=12,lonf=40,lati=-35,latf=-15)
Aus, Aus_mean = regmean(land_cube,loni=120,lonf=140,lati=-30,latf=-17)
lat10, lat10_mean = regmean(land_cube,loni=0,lonf=360,lati=-10,latf=10)
latp20, latp20_mean = regmean(land_cube,loni=0,lonf=360,lati=10,latf=20)
latm20, latm20_mean = regmean(land_cube,loni=0,lonf=360,lati=-20,latf=-10)
latp30, latp30_mean = regmean(land_cube,loni=0,lonf=360,lati=20,latf=30)
latm30, latm30_mean = regmean(land_cube,loni=0,lonf=360,lati=-30,latf=-20)
# lat20mean = latp20
return [India_mean , MC_mean , TropSthAm_mean , SthSthAm_mean , NthWestAfr_mean , NthEastAfr_mean, TropAfr_mean, SthAfr_mean, Aus_mean, lat10_mean, latp20_mean, latm20_mean, latp30_mean, latm30_mean]
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
assert_(m is m2)
m3 = make_mask(m, copy=True)
assert_(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
assert_(y1._data is not x1)
assert_(allequal(x1, y1._data))
assert_(y1._mask is m)
y1a = array(y1, copy=0)
# For copy=False, one might expect that the array would just
# passed on, i.e., that it would be "is" instead of "==".
# See gh-4043 for discussion.
assert_(y1a._mask.__array_interface__ ==
y1._mask.__array_interface__)
y2 = array(x1, mask=m3, copy=0)
assert_(y2._mask is m3)
assert_(y2[2] is masked)
y2[2] = 9
assert_(y2[2] is not masked)
assert_(y2._mask is m3)
assert_(allequal(y2.mask, 0))
y2a = array(x1, mask=m, copy=1)
assert_(y2a._mask is not m)
assert_(y2a[2] is masked)
y2a[2] = 9
assert_(y2a[2] is not masked)
assert_(y2a._mask is not m)
assert_(allequal(y2a.mask, 0))
y3 = array(x1 * 1.0, mask=m)
assert_(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
assert_(eq(concatenate([x4, x4]), y4))
assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
assert_(eq(y5, y6))
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
assert_(m is m2)
m3 = make_mask(m, copy=1)
assert_(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
assert_(y1._data is not x1)
assert_(allequal(x1, y1._data))
assert_(y1._mask is m)
y1a = array(y1, copy=0)
# For copy=False, one might expect that the array would just
# passed on, i.e., that it would be "is" instead of "==".
# See gh-4043 for discussion.
assert_(y1a._mask.__array_interface__ ==
y1._mask.__array_interface__)
y2 = array(x1, mask=m3, copy=0)
assert_(y2._mask is m3)
assert_(y2[2] is masked)
y2[2] = 9
assert_(y2[2] is not masked)
assert_(y2._mask is m3)
assert_(allequal(y2.mask, 0))
y2a = array(x1, mask=m, copy=1)
assert_(y2a._mask is not m)
assert_(y2a[2] is masked)
y2a[2] = 9
assert_(y2a[2] is not masked)
assert_(y2a._mask is not m)
assert_(allequal(y2a.mask, 0))
y3 = array(x1 * 1.0, mask=m)
assert_(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
assert_(eq(concatenate([x4, x4]), y4))
assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
assert_(eq(y5, y6))
def Make2ThetaAzimuthMap(data, masks, iLim, jLim,
times): #most expensive part of integration!
'Needs a doc string'
#transforms 2D image from x,y space to 2-theta,azimuth space based on detector orientation
pixelSize = data['pixelSize']
scalex = pixelSize[0] / 1000.
scaley = pixelSize[1] / 1000.
tay, tax = np.mgrid[iLim[0] + 0.5:iLim[1] + .5,
jLim[0] + .5:jLim[1] + .5] #bin centers not corners
tax = np.asfarray(tax * scalex, dtype=np.float32)
tay = np.asfarray(tay * scaley, dtype=np.float32)
nI = iLim[1] - iLim[0]
nJ = jLim[1] - jLim[0]
t0 = time.time()
#make position masks here
frame = masks['Frames']
tam = ma.make_mask_none((nI, nJ))
if frame:
tamp = ma.make_mask_none((1024 * 1024))
tamp = ma.make_mask(
pm.polymask(
nI * nJ, tax.flatten(), tay.flatten(), len(frame), frame,
tamp)[:nI * nJ]) - True #switch to exclude around frame
tam = ma.mask_or(tam.flatten(), tamp)
polygons = masks['Polygons']
for polygon in polygons:
if polygon:
tamp = ma.make_mask_none((1024 * 1024))
tamp = ma.make_mask(
pm.polymask(nI * nJ, tax.flatten(), tay.flatten(),
len(polygon), polygon, tamp)[:nI * nJ])
tam = ma.mask_or(tam.flatten(), tamp)
if tam.shape: tam = np.reshape(tam, (nI, nJ))
spots = masks['Points']
for X, Y, diam in spots:
tamp = ma.getmask(
ma.masked_less((tax - X)**2 + (tay - Y)**2, (diam / 2.)**2))
tam = ma.mask_or(tam, tamp)
times[0] += time.time() - t0
t0 = time.time()
TA = np.array(GetTthAzmG(
tax, tay,
data)) #includes geom. corr. as dist**2/d0**2 - most expensive step
times[1] += time.time() - t0
TA[1] = np.where(TA[1] < 0, TA[1] + 360, TA[1])
return np.array(
TA), tam #2-theta, azimuth & geom. corr. arrays & position mask
def MakeFrameMask(data, frame):
pixelSize = data['pixelSize']
scalex = pixelSize[0] / 1000.
scaley = pixelSize[1] / 1000.
blkSize = 512
Nx, Ny = data['size']
nXBlks = (Nx - 1) / blkSize + 1
nYBlks = (Ny - 1) / blkSize + 1
tam = ma.make_mask_none(data['size'])
for iBlk in range(nXBlks):
iBeg = iBlk * blkSize
iFin = min(iBeg + blkSize, Nx)
for jBlk in range(nYBlks):
jBeg = jBlk * blkSize
jFin = min(jBeg + blkSize, Ny)
nI = iFin - iBeg
nJ = jFin - jBeg
tax, tay = np.mgrid[iBeg + 0.5:iFin + .5,
jBeg + .5:jFin + .5] #bin centers not corners
tax = np.asfarray(tax * scalex, dtype=np.float32)
tay = np.asfarray(tay * scaley, dtype=np.float32)
tamp = ma.make_mask_none((1024 * 1024))
tamp = ma.make_mask(
pm.polymask(
nI * nJ, tax.flatten(), tay.flatten(), len(frame), frame,
tamp)[:nI * nJ]) - True #switch to exclude around frame
if tamp.shape:
tamp = np.reshape(tamp[:nI * nJ], (nI, nJ))
tam[iBeg:iFin,
jBeg:jFin] = ma.mask_or(tamp[0:nI, 0:nJ], tam[iBeg:iFin,
jBeg:jFin])
else:
tam[iBeg:iFin, jBeg:jFin] = True
return tam.T
def bit_pixels(bit_shape="cylinder", diameter=3):
radius = diameter / 2.0
if diameter % 2:
size = diameter
x, y = mgrid[:size, :size]
x = x + 0.5
y = y + 0.5
else:
size = diameter + 1
x, y = mgrid[:size, :size]
sphere = (x - radius) ** 2 + (y - radius) ** 2
circle_mask = ma.make_mask(sphere > radius ** 2) # true when outside the circle
high = ones(circle_mask.shape) * 10000
if bit_shape in ["cylinder", "ball", "sphere"]:
if bit_shape == "cylinder":
output = circle_mask * high
else:
# print "test"
output = (circle_mask == False) * sphere + circle_mask * high
elif bit_shape.startswith("v"):
angle = float(bit_shape[1:]) / 2.0
angle = radians(90 - angle)
step = tan(angle)
cone = sqrt(sphere) * step
output = (circle_mask == False) * cone + circle_mask * high
return output
def __setmask__(self, mask):
"Sets the mask and update the fieldmask."
fmask = self.__dict__['_fieldmask']
names = fmask.dtype.names
#
if isinstance(mask, ndarray) and mask.dtype.names == names:
for n in names:
fmask[n] = mask[n].astype(bool)
# self.__dict__['_fieldmask'] = fmask.view(recarray)
return
newmask = make_mask(mask, copy=False)
if names is not None:
if self._hardmask:
for n in names:
fmask[n].__ior__(newmask)
else:
for n in names:
current = fmask[n]
if current.shape == newmask.shape or newmask.size == 1:
current.flat = newmask
else:
for (i, n) in enumerate(newmask):
current[i] = n
return
def MakeFrameMask(data,frame):
pixelSize = data['pixelSize']
scalex = pixelSize[0]/1000.
scaley = pixelSize[1]/1000.
blkSize = 512
Nx,Ny = data['size']
nXBlks = (Nx-1)/blkSize+1
nYBlks = (Ny-1)/blkSize+1
tam = ma.make_mask_none(data['size'])
for iBlk in range(nXBlks):
iBeg = iBlk*blkSize
iFin = min(iBeg+blkSize,Nx)
for jBlk in range(nYBlks):
jBeg = jBlk*blkSize
jFin = min(jBeg+blkSize,Ny)
nI = iFin-iBeg
nJ = jFin-jBeg
tax,tay = np.mgrid[iBeg+0.5:iFin+.5,jBeg+.5:jFin+.5] #bin centers not corners
tax = np.asfarray(tax*scalex,dtype=np.float32)
tay = np.asfarray(tay*scaley,dtype=np.float32)
tamp = ma.make_mask_none((1024*1024))
tamp = ma.make_mask(pm.polymask(nI*nJ,tax.flatten(),
tay.flatten(),len(frame),frame,tamp)[:nI*nJ])-True #switch to exclude around frame
if tamp.shape:
tamp = np.reshape(tamp[:nI*nJ],(nI,nJ))
tam[iBeg:iFin,jBeg:jFin] = ma.mask_or(tamp[0:nI,0:nJ],tam[iBeg:iFin,jBeg:jFin])
else:
tam[iBeg:iFin,jBeg:jFin] = True
return tam.T
def regions(cube,clim=False,maxf=True, maskprint=False):
mons = 4
lag = 1
max_i = 35 + lag
max_f = max_i + mons
min_i = 11 + lag
min_f = min_i + mons
if maxf:
forc_i = max_i
forc_f = max_f
else:
forc_i = min_i
forc_f = min_f
if clim:
cube_max = cube.collapsed('time',iris.analysis.MEAN)
else:
cube_max = cube[forc_i:forc_f,:,::].collapsed('time',iris.analysis.MEAN)
# cube_anom = cube-cube_mean
cube_max.coord('latitude').guess_bounds()
cube_max.coord('longitude').guess_bounds()
print cube_max.shape
# ---------- define sst/tland----------
lsmask = iris.load_cube(ncfile_path + 'lsmask.nc')[0,0,::]
landmask = ~(ma.make_mask(lsmask.data.copy()) + np.zeros(cube_max.shape)).astype(bool) # mask sea, show land
# mask out the enso region: masked = 1, not masked = 0
# mask from x = 43:73, y = 32:40
enso_mask = np.zeros(cube_max.shape)
enso_mask[:,20:56,43:75] = 1
seamask = (ma.make_mask(lsmask.data.copy()) + np.zeros(cube_max.shape) + enso_mask).astype(bool) # mask land, show sea
# if maskprint:
ocean_cube = cube_max.copy()
land_cube = cube_max.copy()
ocean_cube.data = ma.array(ocean_cube.data, mask=seamask)
land_cube.data = ma.array(land_cube.data, mask=landmask)
# --------------
oc_zonmean = ocean_cube.collapsed(['longitude'],
iris.analysis.MEAN)
ln_zonmean = land_cube.collapsed(['longitude'],
iris.analysis.MEAN)
return [oc_zonmean, ln_zonmean]
def downsample_to_bit_diameter(image, scale, bit_diameter, bit="square"):
# print image.shape
bit_diameter = int(bit_diameter)
height, width = image.shape
scaled_width = width * scale
scaled_height = height * scale
output = zeros((scaled_height, scaled_width), dtype=integer)
# coordinates are height,width
print "output is %s" % str(output.shape)
print "target width, target height, scale = ", scaled_width, scaled_height, scale
# print "downsample_to_bit_diamter width=%i height=%i" % (width,height)
# print "downsample_to_bit_diameter shape:", output.shape
# print len(image[::bit_diameter,::bit-_diameter].tolist()[0])
# print "bit_diameter:",bit_diameter
for y in range(int(scaled_height)):
for x in range(int(scaled_width)):
# print "%s:%s,%s:%s = %s" % ( (y)/scale,(y+bit_diameter)/scale,x/scale,(x+bit_diameter)/scale,amax(image[(y)/scale:(y+bit_diameter)/scale,x/scale:(x+bit_diameter)/scale]))
left = (x - bit_diameter / 2) / scale
right = (x + bit_diameter / 2 + 1) / scale
top = (y - bit_diameter / 2) / scale
bottom = (y + bit_diameter / 2 + 1) / scale
if bit == "square":
left = max(left, 0)
right = min(right, width)
top = max(top, 0)
bottom = min(bottom, height)
# this line will have to be a bit more precise for final cuts
output[y, x] = amax(image[top:bottom, left:right])
else:
# print "-"*80
# print left,right,top,bottom, "at %s,%s" % (y,x)
my, mx = mgrid[top:bottom, left:right]
mask_for_bit_check = ma.make_mask((mx >= 0) * (mx < width) * (my >= 0) * (my < height))
left = max(left, 0)
right = min(right, width)
top = max(top, 0)
bottom = min(bottom, height)
surface_subset = image[top:bottom, left:right]
# print "surface:",surface_subset
surface_subset = surface_subset.flatten()
bit_subset = extract(mask_for_bit_check, bit)
# print "surface:",surface_subset.tolist()
# print "bit:",bit_subset.tolist()
# print dir(bit_subset)
# print "image:",surface_subset.shape,"vs","bit:",bit_subset.shape
try:
output[y, x] = amax(surface_subset - bit_subset)
except:
print mask_for_bit_check.sum()
raise
# print "result:",output[y,x]
# if I save that range instead of doing amax on it, I could:
# 1. do amax on it for zigzag or trace cuts
# or
# 2. subtract a grid of values from it where 0 is the tip of the bit and then amax the result
# how do I get a grid that represents the shape of the bit?
# will that affect the speed too much?
# print "downsample_to_bit_diameter shape:", output.shape
return output
def latin_sampler(locator, num_samples, variables, region):
"""
This script creates a matrix of m x n samples using the latin hypercube sampler.
for this, it uses the database of probability distribtutions stored in locator.get_uncertainty_db()
it returns clean and normalized samples.
:param locator: pointer to locator of files of CEA
:param num_samples: number of samples to do
:param variables: list of variables to sample
:return:
1. design: a matrix m x n with the samples where each feature is normalized from [0,1]
2. design_norm: a matrix m x n with the samples where each feature is normalized from [0,1]
3. pdf_list: a dataframe with properties of the probability density functions used in the exercise.
"""
# get probability density function PDF of variables of interest
variable_groups = ('ENVELOPE', 'INDOOR_COMFORT', 'INTERNAL_LOADS',
'SYSTEMS')
database = pd.concat([
pd.read_excel(locator.get_uncertainty_db(region), group, axis=1)
for group in variable_groups
])
pdf_list = database[database['name'].isin(variables)].set_index('name')
# get number of variables
num_vars = pdf_list.shape[0] # alternatively use len(variables)
# get design of experiments
samples = latin_hypercube.lhs(num_vars,
samples=num_samples,
criterion='maximin')
for i, variable in enumerate(variables):
distribution = pdf_list.loc[variable, 'distribution']
#sampling into lhs
min = pdf_list.loc[variable, 'min']
max = pdf_list.loc[variable, 'max']
mu = pdf_list.loc[variable, 'mu']
stdv = pdf_list.loc[variable, 'stdv']
if distribution == 'triangular':
loc = min
scale = max - min
c = (mu - min) / (max - min)
samples[:, i] = triang(loc=loc, c=c, scale=scale).ppf(samples[:,
i])
elif distribution == 'normal':
samples[:, i] = norm(loc=mu, scale=stdv).ppf(samples[:, i])
elif distribution == 'boolean': # converts a uniform (0-1) into True/False
samples[:, i] = ma.make_mask(
np.rint(uniform(loc=min, scale=max).ppf(samples[:, i])))
else: # assume it is uniform
samples[:, i] = uniform(loc=min, scale=max).ppf(samples[:, i])
min_max_scaler = preprocessing.MinMaxScaler(copy=True,
feature_range=(0, 1))
samples_norm = min_max_scaler.fit_transform(samples)
return samples, samples_norm, pdf_list
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
assert_(m is m2)
m3 = make_mask(m, copy=1)
assert_(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
assert_(y1._data is not x1)
assert_(allequal(x1, y1._data))
assert_(y1.mask is m)
y1a = array(y1, copy=0)
assert_(y1a.mask is y1.mask)
y2 = array(x1, mask=m3, copy=0)
assert_(y2.mask is m3)
assert_(y2[2] is masked)
y2[2] = 9
assert_(y2[2] is not masked)
assert_(y2.mask is m3)
assert_(allequal(y2.mask, 0))
y2a = array(x1, mask=m, copy=1)
assert_(y2a.mask is not m)
assert_(y2a[2] is masked)
y2a[2] = 9
assert_(y2a[2] is not masked)
assert_(y2a.mask is not m)
assert_(allequal(y2a.mask, 0))
y3 = array(x1 * 1.0, mask=m)
assert_(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8, ))
assert_(eq(concatenate([x4, x4]), y4))
assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
assert_(eq(y5, y6))
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
assert_(m is m2)
m3 = make_mask(m, copy=1)
assert_(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
assert_(y1._data is not x1)
assert_(allequal(x1, y1._data))
assert_(y1.mask is m)
y1a = array(y1, copy=0)
assert_(y1a.mask is y1.mask)
y2 = array(x1, mask=m3, copy=0)
assert_(y2.mask is m3)
assert_(y2[2] is masked)
y2[2] = 9
assert_(y2[2] is not masked)
assert_(y2.mask is m3)
assert_(allequal(y2.mask, 0))
y2a = array(x1, mask=m, copy=1)
assert_(y2a.mask is not m)
assert_(y2a[2] is masked)
y2a[2] = 9
assert_(y2a[2] is not masked)
assert_(y2a.mask is not m)
assert_(allequal(y2a.mask, 0))
y3 = array(x1 * 1.0, mask=m)
assert_(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
assert_(eq(concatenate([x4, x4]), y4))
assert_(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
assert_(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
assert_(eq(y5, y6))
def test_hdmedian():
# 1-D array
x = ma.arange(11)
assert_allclose(ms.hdmedian(x), 5, rtol=1e-14)
x.mask = ma.make_mask(x)
x.mask[:7] = False
assert_allclose(ms.hdmedian(x), 3, rtol=1e-14)
# Check that `var` keyword returns a value. TODO: check whether returned
# value is actually correct.
assert_(ms.hdmedian(x, var=True).size == 2)
# 2-D array
x2 = ma.arange(22).reshape((11, 2))
assert_allclose(ms.hdmedian(x2, axis=0), [10, 11])
x2.mask = ma.make_mask(x2)
x2.mask[:7, :] = False
assert_allclose(ms.hdmedian(x2, axis=0), [6, 7])
def test_hdmedian():
# 1-D array
x = ma.arange(11)
assert_allclose(ms.hdmedian(x), 5, rtol=1e-14)
x.mask = ma.make_mask(x)
x.mask[:7] = False
assert_allclose(ms.hdmedian(x), 3, rtol=1e-14)
# Check that `var` keyword returns a value. TODO: check whether returned
# value is actually correct.
assert_(ms.hdmedian(x, var=True).size == 2)
# 2-D array
x2 = ma.arange(22).reshape((11, 2))
assert_allclose(ms.hdmedian(x2, axis=0), [10, 11])
x2.mask = ma.make_mask(x2)
x2.mask[:7, :] = False
assert_allclose(ms.hdmedian(x2, axis=0), [6, 7])
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
with suppress_warnings() as sup:
sup.filter(
np.ma.core.MaskedArrayFutureWarning,
"setting an item on a masked array which has a "
"shared mask will not copy")
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
self.assertTrue(m is m2)
m3 = make_mask(m, copy=1)
self.assertTrue(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
self.assertTrue(y1._data is not x1)
self.assertTrue(allequal(x1, y1._data))
self.assertTrue(y1.mask is m)
y1a = array(y1, copy=0)
self.assertTrue(y1a.mask is y1.mask)
y2 = array(x1, mask=m, copy=0)
self.assertTrue(y2.mask is m)
self.assertTrue(y2[2] is masked)
y2[2] = 9
self.assertTrue(y2[2] is not masked)
self.assertTrue(y2.mask is not m)
self.assertTrue(allequal(y2.mask, 0))
y3 = array(x1 * 1.0, mask=m)
self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
self.assertTrue(eq(concatenate([x4, x4]), y4))
self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
self.assertTrue(eq(y5, y6))
def test_testCopySize(self):
# Tests of some subtle points of copying and sizing.
with suppress_warnings() as sup:
sup.filter(
np.ma.core.MaskedArrayFutureWarning,
"setting an item on a masked array which has a "
"shared mask will not copy")
n = [0, 0, 1, 0, 0]
m = make_mask(n)
m2 = make_mask(m)
self.assertTrue(m is m2)
m3 = make_mask(m, copy=1)
self.assertTrue(m is not m3)
x1 = np.arange(5)
y1 = array(x1, mask=m)
self.assertTrue(y1._data is not x1)
self.assertTrue(allequal(x1, y1._data))
self.assertTrue(y1.mask is m)
y1a = array(y1, copy=0)
self.assertTrue(y1a.mask is y1.mask)
y2 = array(x1, mask=m, copy=0)
self.assertTrue(y2.mask is m)
self.assertTrue(y2[2] is masked)
y2[2] = 9
self.assertTrue(y2[2] is not masked)
self.assertTrue(y2.mask is not m)
self.assertTrue(allequal(y2.mask, 0))
y3 = array(x1 * 1.0, mask=m)
self.assertTrue(filled(y3).dtype is (x1 * 1.0).dtype)
x4 = arange(4)
x4[2] = masked
y4 = resize(x4, (8,))
self.assertTrue(eq(concatenate([x4, x4]), y4))
self.assertTrue(eq(getmask(y4), [0, 0, 1, 0, 0, 0, 1, 0]))
y5 = repeat(x4, (2, 2, 2, 2), axis=0)
self.assertTrue(eq(y5, [0, 0, 1, 1, 2, 2, 3, 3]))
y6 = repeat(x4, 2, axis=0)
self.assertTrue(eq(y5, y6))
def immask(img, mask, P, Q):
'''
Mask and crop an image given mask image
'''
#print img[mask].shape
#return img[mask].reshape((P,Q))
maskArray = ma.make_mask(mask)
imgMasked = ma.array(img, mask=~maskArray)
return imgMasked.compressed().reshape((P, Q))
def Cylindrical_warping(middle, right, left):
# Write your codes here
middle = cv2.copyMakeBorder(middle, 50, 50, 300, 300, cv2.BORDER_CONSTANT)
h, w = middle.shape
f = 400
K = np.array([[f, 0, w / 2], [0, f, h / 2],
[0, 0, 1]]) # mock calibration matrix
middlecl, maskmiddle = cylindricalWarpImage(middle, K)
h, w = left.shape
f = 400
K = np.array([[f, 0, w / 2], [0, f, h / 2],
[0, 0, 1]]) # mock calibration matrix
leftcl, maskleft = cylindricalWarpImage(left, K)
h, w = right.shape
f = 400
K = np.array([[f, 0, w / 2], [0, f, h / 2],
[0, 0, 1]]) # mock calibration matrix
rightcl, maskright = cylindricalWarpImage(right, K)
(M, pts1, pts2, mask) = getTransform(rightcl, middlecl)
out = cv2.warpAffine(rightcl,
M, (middlecl.shape[1], middlecl.shape[0]),
dst=middlecl.copy(),
borderMode=cv2.BORDER_TRANSPARENT)
(M, pts1, pts2, mask) = getTransform(leftcl, out)
outR = cv2.warpAffine(leftcl,
M, (out.shape[1], out.shape[0]),
dst=out.copy(),
borderMode=cv2.BORDER_TRANSPARENT)
# maskR = cv2.warpAffine(maskleft, M, (out.shape[1], out.shape[0]), dst=maskmiddle.copy(),borderMode=cv2.BORDER_TRANSPARENT)
mask_31 = ma.make_mask(outR)
new = outR + out * np.invert(mask_31)
mask_new = ma.make_mask(new)
output_image = new + middlecl * np.invert(mask_new)
# output_image = outR + out * (1 - maskR / 255)
# Write out the result
output_name = sys.argv[5] + "output_cylindrical.png"
cv2.imwrite(output_name, output_image)
return True
def setup(self):
"Generic setup"
d = np.arange(5)
m = ma.make_mask([1, 0, 0, 1, 1])
base_d = np.r_[d, d[::-1]].reshape(2, -1).T
base_m = np.r_[[m, m[::-1]]].T
base = ma.array(base_d, mask=base_m)
mrec = mr.fromarrays(base.T,)
dlist = ['2007-%02i' % (i + 1) for i in d]
dates = date_array(dlist)
mts = time_series(mrec, dates)
rts = time_records(mrec, dates)
self.data = [d, m, mrec, dlist, dates, mts, rts]
def setup(self):
"Generic setup"
d = np.arange(5)
m = ma.make_mask([1, 0, 0, 1, 1])
base_d = np.r_[d, d[::-1]].reshape(2, -1).T
base_m = np.r_[[m, m[::-1]]].T
base = ma.array(base_d, mask=base_m)
mrec = mr.fromarrays(base.T, )
dlist = ['2007-%02i' % (i + 1) for i in d]
dates = date_array(dlist)
mts = time_series(mrec, dates)
rts = time_records(mrec, dates)
self.data = [d, m, mrec, dlist, dates, mts, rts]
def Make2ThetaAzimuthMap(data,masks,iLim,jLim,times): #most expensive part of integration!
'Needs a doc string'
#transforms 2D image from x,y space to 2-theta,azimuth space based on detector orientation
pixelSize = data['pixelSize']
scalex = pixelSize[0]/1000.
scaley = pixelSize[1]/1000.
tay,tax = np.mgrid[iLim[0]+0.5:iLim[1]+.5,jLim[0]+.5:jLim[1]+.5] #bin centers not corners
tax = np.asfarray(tax*scalex,dtype=np.float32)
tay = np.asfarray(tay*scaley,dtype=np.float32)
nI = iLim[1]-iLim[0]
nJ = jLim[1]-jLim[0]
t0 = time.time()
#make position masks here
frame = masks['Frames']
tam = ma.make_mask_none((nI,nJ))
if frame:
tamp = ma.make_mask_none((1024*1024))
tamp = ma.make_mask(pm.polymask(nI*nJ,tax.flatten(),
tay.flatten(),len(frame),frame,tamp)[:nI*nJ])-True #switch to exclude around frame
tam = ma.mask_or(tam.flatten(),tamp)
polygons = masks['Polygons']
for polygon in polygons:
if polygon:
tamp = ma.make_mask_none((1024*1024))
tamp = ma.make_mask(pm.polymask(nI*nJ,tax.flatten(),
tay.flatten(),len(polygon),polygon,tamp)[:nI*nJ])
tam = ma.mask_or(tam.flatten(),tamp)
if tam.shape: tam = np.reshape(tam,(nI,nJ))
spots = masks['Points']
for X,Y,diam in spots:
tamp = ma.getmask(ma.masked_less((tax-X)**2+(tay-Y)**2,(diam/2.)**2))
tam = ma.mask_or(tam,tamp)
times[0] += time.time()-t0
t0 = time.time()
TA = np.array(GetTthAzmG(tax,tay,data)) #includes geom. corr. as dist**2/d0**2 - most expensive step
times[1] += time.time()-t0
TA[1] = np.where(TA[1]<0,TA[1]+360,TA[1])
return np.array(TA),tam #2-theta, azimuth & geom. corr. arrays & position mask
def __getitem__(self, indx):
"""Returns all the fields sharing the same fieldname base.
The fieldname base is either `_data` or `_mask`."""
_localdict = self.__dict__
# We want a field ........
if indx in ndarray.__getattribute__(self, 'dtype').names:
obj = self._data[indx].view(TimeSeries)
obj._dates = _localdict['_dates']
obj._mask = make_mask(_localdict['_mask'][indx])
return obj
# We want some elements ..
obj = TimeSeries.__getitem__(self, indx)
if isinstance(obj, MaskedArray) and not isinstance(obj, TimeSeries):
obj = ndarray.view(obj, MaskedRecords)
return obj
def test_hardmask(self):
# Test hardmask
base = self.base.copy()
mbase = base.view(mrecarray)
mbase.harden_mask()
assert_(mbase._hardmask)
mbase.mask = nomask
assert_equal_records(mbase._mask, base._mask)
mbase.soften_mask()
assert_(not mbase._hardmask)
mbase.mask = nomask
# So, the mask of a field is no longer set to nomask...
assert_equal_records(mbase._mask, ma.make_mask_none(base.shape, base.dtype))
assert_(ma.make_mask(mbase["b"]._mask) is nomask)
assert_equal(mbase["a"]._mask, mbase["b"]._mask)
def __getitem__(self, indx):
"""Returns all the fields sharing the same fieldname base.
The fieldname base is either `_data` or `_mask`."""
_localdict = self.__dict__
# We want a field ........
if indx in ndarray.__getattribute__(self, "dtype").names:
obj = self._data[indx].view(TimeSeries)
obj._dates = _localdict["_dates"]
obj._mask = make_mask(_localdict["_mask"][indx])
return obj
# We want some elements ..
obj = TimeSeries.__getitem__(self, indx)
if isinstance(obj, MaskedArray) and not isinstance(obj, TimeSeries):
obj = ndarray.view(obj, MaskedRecords)
return obj
def test_hardmask(self):
# Test hardmask
base = self.base.copy()
mbase = base.view(mrecarray)
mbase.harden_mask()
self.assertTrue(mbase._hardmask)
mbase.mask = nomask
assert_equal_records(mbase._mask, base._mask)
mbase.soften_mask()
self.assertTrue(not mbase._hardmask)
mbase.mask = nomask
# So, the mask of a field is no longer set to nomask...
assert_equal_records(mbase._mask, ma.make_mask_none(base.shape, base.dtype))
self.assertTrue(ma.make_mask(mbase["b"]._mask) is nomask)
assert_equal(mbase["a"]._mask, mbase["b"]._mask)
def update():
radialImg = np.cos(distImg * float(int(freqRad)))
circumImg = np.cos((angleImg + angle) * float(int(freqAng)))
mergeImg = (radialImg + circumImg)
maskImg = (mergeImg > thresh)
xmask = ma.make_mask(maskImg)
filtImg = fourShft * xmask
filtLog = np.log(np.maximum(np.abs(filtImg), 1.))
fourPlot.set_data(filtLog)
plt.sca(axFourInv)
fourIshft = np.fft.ifftshift(filtImg)
fourInv = np.fft.ifft2(fourIshft)
fourReal = np.real(fourInv)
# invPlot = plt.imshow(fourReal, cmap='gray')
invPlot.set_data(fourReal)
plt.pause(.001)
def duplicate_train_samples(train_X, train_Y, dup_target_y, multiplier=2):
"""duplicate samples. train_Y has N 1,0 tuples. dup_target_y in [0,N). multiplier must be >=2. append to end of input arrays"""
target_y = train_Y[:, dup_target_y]
#print(target_y)
mask = ma.make_mask(target_y)
additional_X = train_X[mask]
#print(additional_X)
additional_Y = train_Y[mask]
#print(additional_Y)
for i in range(1, multiplier):
train_X = np.concatenate((train_X, additional_X), axis=0)
train_Y = np.concatenate((train_Y, additional_Y), axis=0)
return train_X, train_Y
def make_mask_circle(img_shape, center, R):
"""
Make a circular mask, where everything inside a radius R around the center
is False and outside is True
Input:
img_shape: the shape of the image in (row, col)
center: the center of the mask, in (row, col)
R: the radius of the circle
Returns:
mask: a masked array of shape img_shape
"""
mask = np.zeros(shape=img_shape, dtype=np.int)
centered_grid = get_centered_grid(img_shape, center)
rad_grid = np.linalg.norm(centered_grid, axis=0)
mask[rad_grid <= R] = 1
return ~ma.make_mask(mask)
def update():
radialImg = np.cos(distImg * float(int(freqRad)))
radialPlot.set_data(radialImg)
circImg = np.cos(angleImg * float(int(freqAng)))
circPlot.set_data(circImg)
sumImg = (radialImg + circImg)
prodImg = (radialImg * circImg)
mergeImg = (sumVal * sumImg) + (prodVal * prodImg)
maskImg = (np.fabs(mergeImg) > thresh)
xmask = ma.make_mask(maskImg)
filtImg = mergeImg * xmask
mergePlot.set_data(filtImg)
plt.pause(.001)
def update():
plt.sca(axFour)
maskR1 = (distImg > rad1)
maskR2 = (distImg < rad2)
maskRadial = np.logical_and(maskR1, maskR2)
maskAngle = (np.sin(angleImg * 2. + angle) >= angleThresh)
maskImg = np.logical_and(maskAngle, maskRadial)
maskImg[hafY, hafX] = True
xmask = ma.make_mask(maskImg)
filtImg = fourShft * xmask
filtLog = np.log(np.maximum(np.abs(filtImg), 1.))
fourPlot.set_data(filtLog)
plt.sca(axFourInv)
fourIshft = np.fft.ifftshift(filtImg)
fourInv = np.fft.ifft2(fourIshft)
fourReal = np.real(fourInv)
invPlot.set_data(fourReal)
plt.pause(.001)
def update(val):
global rad1, rad2, filtImg
plt.sca(axFour)
rad1 = slider1.val
rad2 = slider2.val
mask1 = (distImg > rad1)
mask2 = (distImg < rad2)
maskImg = np.logical_and(mask1, mask2)
maskImg[hafY, hafX] = True
xmask = ma.make_mask(maskImg)
filtImg = fourShft * xmask
filtLog = np.log(np.maximum(np.abs(filtImg), 1.))
fourPlot.set_data(filtLog)
plt.sca(axFourInv)
fourIshft = np.fft.ifftshift(filtImg)
fourInv = np.fft.ifft2(fourIshft)
fourReal = np.real(fourInv)
invPlot = plt.imshow(fourReal, cmap='gray')
plt.pause(.001)
def test_testPut(self):
# Test of put
d = arange(5)
n = [0, 0, 0, 1, 1]
m = make_mask(n)
x = array(d, mask=m)
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is masked)
x[[1, 4]] = [10, 40]
self.assertTrue(x.mask is not m)
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is not masked)
self.assertTrue(eq(x, [0, 10, 2, -1, 40]))
x = array(d, mask=m)
x.put([0, 1, 2], [-1, 100, 200])
self.assertTrue(eq(x, [-1, 100, 200, 0, 0]))
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is masked)
def __setmask__(self, mask):
"Sets the mask and update the fieldmask."
names = self.dtype.names
fmask = self.__dict__['_fieldmask']
#
if isinstance(mask,ndarray) and mask.dtype.names == names:
for n in names:
fmask[n] = mask[n].astype(bool)
# self.__dict__['_fieldmask'] = fmask.view(recarray)
return
newmask = make_mask(mask, copy=False)
if names is not None:
if self._hardmask:
for n in names:
fmask[n].__ior__(newmask)
else:
for n in names:
fmask[n].flat = newmask
return
def test_testPut(self):
# Test of put
d = arange(5)
n = [0, 0, 0, 1, 1]
m = make_mask(n)
x = array(d, mask=m)
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is masked)
x[[1, 4]] = [10, 40]
self.assertTrue(x.mask is not m)
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is not masked)
self.assertTrue(eq(x, [0, 10, 2, -1, 40]))
x = array(d, mask=m)
x.put([0, 1, 2], [-1, 100, 200])
self.assertTrue(eq(x, [-1, 100, 200, 0, 0]))
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is masked)
def maskedarray(self, center, left, right):
# check whether we have enough data left and right:
if len(self.data.shape) > 1:
mask = \
np.zeros((self.data.shape[0], int((right+left)/self.dt)))
maskedarray = \
ma.zeros((self.data.shape[0], int((right+left)/self.dt)))
else:
mask = np.zeros((int((right + left) / self.dt)))
maskedarray = ma.zeros((int((right + left) / self.dt)))
offset = 0
if center - left < 0:
if len(self.data.shape) > 1:
mask[:, :int((left - center) / self.dt)] = 1
else:
mask[:int((left - center) / self.dt)] = 1
leftindex = 0
offset = int((left - center) / self.dt)
else:
leftindex = int((center - left) / self.dt)
if center + right >= len(self.data) * self.dt:
endtime = len(self.data) * self.dt
if len(self.data.shape) > 1:
mask[:, -int((center + right - endtime) / self.dt):] = 1
else:
mask[-int((center + right - endtime) / self.dt):] = 1
rightindex = int(endtime / self.dt)
else:
rightindex = int((center + right) / self.dt)
for timest in range(leftindex, rightindex):
if len(self.data.shape) > 1:
if timest-leftindex+offset < maskedarray.shape[1] and \
timest < self.data.shape[1]:
maskedarray[:, timest-leftindex+offset] = \
self.data[:, timest]
else:
if timest - leftindex + offset < len(maskedarray):
maskedarray[timest-leftindex+offset] = \
self.data[timest]
maskedarray.mask = ma.make_mask(mask)
return Timeseries(maskedarray, self.dt)
def maskedarray(self, center, left, right):
# check whether we have enough data left and right:
if len(self.data.shape) > 1:
mask = \
np.zeros((self.data.shape[0], int((right+left)/self.dt)))
maskedarray = \
ma.zeros((self.data.shape[0], int((right+left)/self.dt)))
else:
mask = np.zeros((int((right+left)/self.dt)))
maskedarray = ma.zeros((int((right+left)/self.dt)))
offset = 0
if center - left < 0:
if len(self.data.shape) > 1:
mask[:, :int((left-center)/self.dt)] = 1
else:
mask[:int((left-center)/self.dt)] = 1
leftindex = 0
offset = int((left-center)/self.dt)
else:
leftindex = int((center-left)/self.dt)
if center + right >= len(self.data) * self.dt:
endtime = len(self.data) * self.dt
if len(self.data.shape) > 1:
mask[:, -int((center+right-endtime)/self.dt):] = 1
else:
mask[-int((center+right-endtime)/self.dt):] = 1
rightindex = int(endtime/self.dt)
else:
rightindex = int((center+right)/self.dt)
for timest in range(leftindex, rightindex):
if len(self.data.shape) > 1:
if timest-leftindex+offset < maskedarray.shape[1] and \
timest < self.data.shape[1]:
maskedarray[:, timest-leftindex+offset] = \
self.data[:, timest]
else:
if timest-leftindex+offset < len(maskedarray):
maskedarray[timest-leftindex+offset] = \
self.data[timest]
maskedarray.mask = ma.make_mask(mask)
return Timeseries(maskedarray, self.dt)
def test_testPut(self):
# Test of put
d = arange(5)
n = [0, 0, 0, 1, 1]
m = make_mask(n)
m2 = m.copy()
x = array(d, mask=m)
assert_(x[3] is masked)
assert_(x[4] is masked)
x[[1, 4]] = [10, 40]
assert_(x._mask is m)
assert_(x[3] is masked)
assert_(x[4] is not masked)
assert_(eq(x, [0, 10, 2, -1, 40]))
x = array(d, mask=m2, copy=True)
x.put([0, 1, 2], [-1, 100, 200])
assert_(x._mask is not m2)
assert_(x[3] is masked)
assert_(x[4] is masked)
assert_(eq(x, [-1, 100, 200, 0, 0]))
def combine_masks(imarray, mask_paths, verbose = False, transpose = False):
"""
Takes a list of paths to .npy mask files and returns a numpy array
consisting of those masks ANDed together.
"""
# Initialize the mask based on zero values in imarray.
import numpy.ma as ma
base_mask = ma.make_mask(np.ones(np.shape(imarray)))
base_mask[imarray == 0.] = False
if not mask_paths:
log( "No additional masks provided")
return base_mask
else:
# Data arrays must be transposed here for the same reason that they
# are in data_extractor.
if transpose:
masks = [np.load(path).T for path in mask_paths]
else:
masks = [np.load(path) for path in mask_paths]
log( "Applying mask(s): ", mask_paths)
return base_mask & reduce(lambda x, y: x & y, masks)
def combine_masks(imarray, mask_paths, verbose=False, transpose=False):
"""
Takes a list of paths to .npy mask files and returns a numpy array
consisting of those masks ANDed together.
"""
# Initialize the mask based on zero values in imarray.
import numpy.ma as ma
base_mask = ma.make_mask(np.ones(np.shape(imarray)))
base_mask[imarray == 0.] = False
if not mask_paths:
log("No additional masks provided")
return base_mask
else:
# Data arrays must be transposed here for the same reason that they
# are in data_extractor.
if transpose:
masks = [np.load(path).T for path in mask_paths]
else:
masks = [np.load(path) for path in mask_paths]
log("Applying mask(s): ", mask_paths)
return base_mask & reduce(lambda x, y: x & y, masks)
def test_testPut(self):
# Test of put
d = arange(5)
n = [0, 0, 0, 1, 1]
m = make_mask(n)
m2 = m.copy()
x = array(d, mask=m)
assert_(x[3] is masked)
assert_(x[4] is masked)
x[[1, 4]] = [10, 40]
assert_(x._mask is m)
assert_(x[3] is masked)
assert_(x[4] is not masked)
assert_(eq(x, [0, 10, 2, -1, 40]))
x = array(d, mask=m2, copy=True)
x.put([0, 1, 2], [-1, 100, 200])
assert_(x._mask is not m2)
assert_(x[3] is masked)
assert_(x[4] is masked)
assert_(eq(x, [-1, 100, 200, 0, 0]))
def __setmask__(self, mask):
"Sets the mask and update the fieldmask."
names = self.dtype.names
fmask = self.__dict__['_fieldmask']
#
if isinstance(mask, ndarray) and mask.dtype.names == names:
for n in names:
fmask[n] = mask[n].astype(bool)
# self.__dict__['_fieldmask'] = fmask.view(recarray)
return
newmask = make_mask(mask, copy=False)
if names is not None:
if self._hardmask:
for n in names:
fmask[n].__ior__(newmask)
else:
for n in names:
fmask[n].flat = newmask
return
def __getitem__(self, idx):
slide_path = self.slide_path[idx]
mask_path = self.mask_path[idx]
slide = cv2.imread(slide_path)
slide = cv2.cvtColor(slide, cv2.COLOR_BGR2RGB)
mask = cv2.imread(mask_path)
mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)
# mask = np.transpose(mask, axes=(2, 0, 1))
mask = mask / 255.0
augmented = self.transforms(image=slide, mask=mask)
slide = augmented['image']
mask = augmented['mask']
mask = ma.make_mask(mask, shrink=False)
mask = mask.squeeze()
mask = np.expand_dims(mask, axis=0)
if self.preprocessing:
preprocessed = self.preprocessing(image=slide, mask=mask)
slide = preprocessed['image']
mask = preprocessed['mask']
return slide, mask
def test_testPut(self):
# Test of put
with suppress_warnings() as sup:
sup.filter(
np.ma.core.MaskedArrayFutureWarning,
"setting an item on a masked array which has a "
"shared mask will not copy")
d = arange(5)
n = [0, 0, 0, 1, 1]
m = make_mask(n)
x = array(d, mask=m)
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is masked)
x[[1, 4]] = [10, 40]
self.assertTrue(x.mask is not m)
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is not masked)
self.assertTrue(eq(x, [0, 10, 2, -1, 40]))
x = array(d, mask=m)
x.put([0, 1, 2], [-1, 100, 200])
self.assertTrue(eq(x, [-1, 100, 200, 0, 0]))
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is masked)
def test_testPut(self):
# Test of put
with suppress_warnings() as sup:
sup.filter(
np.ma.core.MaskedArrayFutureWarning,
"setting an item on a masked array which has a "
"shared mask will not copy")
d = arange(5)
n = [0, 0, 0, 1, 1]
m = make_mask(n)
x = array(d, mask=m)
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is masked)
x[[1, 4]] = [10, 40]
self.assertTrue(x.mask is not m)
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is not masked)
self.assertTrue(eq(x, [0, 10, 2, -1, 40]))
x = array(d, mask=m)
x.put([0, 1, 2], [-1, 100, 200])
self.assertTrue(eq(x, [-1, 100, 200, 0, 0]))
self.assertTrue(x[3] is masked)
self.assertTrue(x[4] is masked)
def make_mask_half_img(img_shape, center, angle):
"""
Mask half the image, cutting it through the center at an arbitrary angle.
Angle is measured *counterclockwise* from vertical, and should be an
astropy units object, otherwise assume degrees.
Input:
img_shape: shape of image in (row, col)
center: the center of the mask in (row, col)
angle: angle measured counterclockwise from vertical, default in deg
Output:
mask: masked_array with a plane running through point (center) at angle
(angle)
"""
if type(angle) != units.quantity.Quantity:
angle = angle * units.degree
mask = np.zeros(shape=img_shape, dtype=np.int)
centered_grid = get_centered_grid(img_shape, center)
rad_grid = np.linalg.norm(centered_grid, axis=0)
ang_grid = np.arctan2(centered_grid[0], centered_grid[1]) * units.radian
rot_grid = ang_grid - angle
#dx = centered_grid[1] + rad_grid * np.cos(rot_grid)
dx = rad_grid * np.cos(rot_grid)
mask[dx >= 0] = 1
return ~ma.make_mask(mask)
def __setmask__(self, mask):
"Sets the mask and update the fieldmask."
fmask = self.__dict__['_fieldmask']
names = fmask.dtype.names
#
if isinstance(mask,ndarray) and mask.dtype.names == names:
for n in names:
fmask[n] = mask[n].astype(bool)
# self.__dict__['_fieldmask'] = fmask.view(recarray)
return
newmask = make_mask(mask, copy=False)
if names is not None:
if self._hardmask:
for n in names:
fmask[n].__ior__(newmask)
else:
for n in names:
current = fmask[n]
if current.shape == newmask.shape or newmask.size == 1:
current.flat = newmask
else:
for (i,n) in enumerate(newmask):
current[i] = n
return
else:
print "t coord changed to time"
# Define regions
theta_plv.coord('latitude').guess_bounds()
theta_plv.coord('longitude').guess_bounds()
thetav_plv.coord('latitude').guess_bounds()
thetav_plv.coord('longitude').guess_bounds()
temp_plv.coord('latitude').guess_bounds()
temp_plv.coord('longitude').guess_bounds()
# pres.coord('latitude').guess_bounds()
# pres.coord('longitude').guess_bounds()
# ---------- define sst/tland----------
lsmask = iris.load_cube(ncfile_path + 'lsmask.nc')[0,0,::]
landmask = ~(ma.make_mask(lsmask.data.copy()) + np.zeros(theta_plv.shape)).astype(bool) # mask sea, show land
seamask = (ma.make_mask(lsmask.data.copy()) + np.zeros(theta_plv.shape)).astype(bool) # mask land, show sea
# sys.exit('exiiiiiiitt')
Tocean = theta_plv.copy()
Tland = theta_plv.copy()
Tocean.data = ma.array(Tocean.data, mask=seamask)
Tland.data = ma.array(Tland.data, mask=landmask)
# --------------
tropics = iris.Constraint(latitude = lambda v: -30 <= v <= 30)
Tocean_trop = Tocean.extract(tropics)
grid_areas_trop = iris.analysis.cartography.area_weights(Tocean_trop)
mons = 3
lag = 3
max_i = 35 + lag
def __call__(self, X, alpha=1.0, bytes=False):
if self.bad_set:
if not isinstance(X, numpy.ma.masked_array):
X = numpy.ma.asarray(X)
X.mask = ma.make_mask(~numpy.isfinite(X))
return Colormap.__call__(self, X, alpha, bytes)
days = WeekdayLocator(MONDAY)
months = MonthLocator(range(1, 13), bymonthday=1, interval=1) # every month
monthsFmt = DateFormatter("%b")
dayFmt = DateFormatter("%d")
ax.xaxis.set_major_locator(months)
ax.xaxis.set_major_formatter(monthsFmt)
ax.xaxis.set_minor_locator(days)
ax.grid(True)
#ax.xaxis.set_minor_formatter(dayFmt)
ax.xaxis.grid(False, which='major')
ax.xaxis.grid(True, which='minor')
for level in range(DD, UD + 1):
mask = ma.make_mask(df.index)
mask = ma.masked_where(band == level, mask)
chosen = ma.masked_where(~mask.mask, price)
#if chosen.any():
plt.plot(df.index, chosen, style_dict[level], alpha=alpha)
# upward trend
mask = ma.make_mask(df.index)
mask = ma.masked_where((band == NY) | (band == SY) | (band == UD), mask)
chosen = ma.masked_where(~mask.mask, price)
plt.plot(df.index, chosen, "g-", alpha=alpha)
# downward trend
mask = ma.make_mask(df.index)
mask = ma.masked_where((band == NN) | (band == SN) | (band == DD), mask)
chosen = ma.masked_where(~mask.mask, price)
def regions(cube,tsfc_cube,name2='name2'):
"""
Calculates a regression between variable and tsfc for different regions
Input: cube, tsfc_cube
Output: array with all regions
"""
cube = nt.remove_seascyc(cube)
tsfc_cube = nt.remove_seascyc(tsfc_cube)
cube = try_cube(cube)
tsfc_cube = try_cube(tsfc_cube)
# cube = anmeananom(cube)
# tsfc_cube = anmeananom(tsfc_cube)
if cube.ndim==4:
cube = cube[:,0,::]
# ---------- define sst/tland----------
lsmask = iris.load_cube(ncfile_path + 'lsmask.nc')[0,0,::]
landmask = ~(ma.make_mask(lsmask.data.copy()) + np.zeros(cube.shape)).astype(bool) # mask sea, show land
seamask = (ma.make_mask(lsmask.data.copy()) + np.zeros(cube.shape)).astype(bool) # mask land, show sea
ocean_cube = tsfc_cube.copy()
land_cube = tsfc_cube.copy()
ocean_cube.data = ma.array(ocean_cube.data, mask=seamask)
land_cube.data = ma.array(land_cube.data, mask=landmask)
# --------------
rorc = 1 # regression or correlation: 0 = reg, 1 = cor
print "Calc reg/cor for India"
India, India_mean = regmean(land_cube,loni=60,lonf=90,lati=0,latf=30)
name1 = 'India tsfc'
India_reg = nt.linregts(cube,India_mean,name1,name2)[rorc]
print "Calc reg/cor for MC"
MC, MC_mean = regmean(land_cube,loni=90,lonf=140,lati=-10,latf=10)
name1 = 'MC tsfc'
MC_reg = nt.linregts(cube,MC_mean,name1,name2)[rorc]
print "Calc reg/cor for TropSthAm"
TropSthAm, TropSthAm_mean = regmean(land_cube,loni=290,lonf=315,lati=-23,latf=0)
name1 = 'TropSthAm tsfc'
TropSthAm_reg = nt.linregts(cube,TropSthAm_mean,name1,name2)[rorc]
print "Calc reg/cor for SthSthAm"
SthSthAm, SthSthAm_mean = regmean(land_cube,loni=270,lonf=315,lati=-60,latf=-24)
name1 = 'SthSthAm tsfc'
SthSthAm_reg = nt.linregts(cube,SthSthAm_mean,name1,name2)[rorc]
print "Calc reg/cor for NthWestAfr"
NthWestAfr, NthWestAfr_mean = regmean(land_cube,loni=-15,lonf=15,lati=10,latf=30,wrap=True)
name1 = 'NthWestAfr tsfc'
NthWestAfr_reg = nt.linregts(cube,NthWestAfr_mean,name1,name2)[rorc]
print "Calc reg/cor for NthEastAfr"
NthEastAfr, NthEastAfr_mean = regmean(land_cube,loni=15,lonf=50,lati=10,latf=30)
name1 = 'NthEastAfr tsfc'
NthEastAfr_reg = nt.linregts(cube,NthEastAfr_mean,name1,name2)[rorc]
print "Calc reg/cor for TropAfr"
TropAfr, TropAfr_mean = regmean(land_cube,loni=12,lonf=40,lati=-15,latf=5)
name1 = 'TropAfr tsfc'
TropAfr_reg = nt.linregts(cube,TropAfr_mean,name1,name2)[rorc]
print "Calc reg/cor for SthAfr"
SthAfr, SthAfr_mean = regmean(land_cube,loni=12,lonf=40,lati=-35,latf=-15)
name1 = 'SthAfr tsfc'
SthAfr_reg = nt.linregts(cube,SthAfr_mean,name1,name2)[rorc]
print "Calc reg/cor for Aus"
Aus, Aus_mean = regmean(land_cube,loni=120,lonf=140,lati=-30,latf=-17)
name1 = 'Aus tsfc'
Aus_reg = nt.linregts(cube,Aus_mean,name1,name2)[rorc]
return [India_reg, MC_reg, TropSthAm_reg , SthSthAm_reg , NthWestAfr_reg , NthEastAfr_reg, TropAfr_reg, SthAfr_reg, Aus_reg]
import numpy as np
import numpy.ma as ma
m = [True, False, True, True]
ma.make_mask(m)
m = [1, 0, 1, 1]
ma.make_mask(m)
m = [1, 0, 2, -3]
ma.make_mask(m)
m = np.zeros(4)
m
ma.make_mask(m)
ma.make_mask(m, shrink=False)
m = [1, 0, 1, 1]
n = [0, 1, 0, 0]
arr = []
for man, mouse in zip(m, n):
arr.append((man, mouse))
arr
dtype = np.dtype({'names': ['man', 'mouse'], })
arr = np.array(arr, dtype=dtype)
arr
ma.make_mask(arr, dtype=dtype)
import scipy as sp
import numpy.ma as ma
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
#import pylab
from math import pow, sqrt, exp, log, sin, cos, pi
#from mpl_toolkits.mplot3d import Axes3D
#from matplotlib import cm
#fig = pylab.figure()
#ax = Axes3D(fig)
m = np.zeros((2594, 2774))
m[2422, 477] = 1
mask = ma.make_mask(m)
mean, std = np.load('21_Si1g_5N_nomvt_mean.npy'), np.load('21_Si1g_5N_nomvt_std.npy')
mean[2422, 477] = (mean[2421:2424, 476:479].sum())/8
"""
# for displaying
numx, numy = 50, 100
scalex, scaley = full2theta, fullgamma
func = gaussian
ar = np.arange(0, scalex+scalex/numx, scalex/numx)
x = np. zeros((numx+1, numy+1))
for i in range(numy+1):
x[:, i] = ar
def testmask():
tmp = np.zeros((100,100))
tmp[20:40,30:50] = 1
return ma.make_mask(tmp)
forc_f = min_f
if clim:
cube_max = cube.collapsed('time',iris.analysis.MEAN)
else:
cube_max = cube[forc_i:forc_f,:,::].collapsed('time',iris.analysis.MEAN)
# cube_anom = cube-cube_mean
cube_max.coord('latitude').guess_bounds()
cube_max.coord('longitude').guess_bounds()
print cube_max.shape
# ---------- define sst/tland----------
lsmask = iris.load_cube(ncfile_path + 'lsmask.nc')[0,0,::]
landmask = ~(ma.make_mask(lsmask.data.copy()) + np.zeros(cube_max.shape)).astype(bool) # mask sea, show land
# mask out the enso region: masked = 1, not masked = 0
# mask from x = 43:73, y = 32:40
enso_mask = np.zeros(cube_max.shape)
enso_mask[0,30:42,43:75] = 1
seamask = (ma.make_mask(lsmask.data.copy()) + np.zeros(cube_max.shape) + enso_mask).astype(bool) # mask land, show sea
Cocean = cube_max.copy()
Cland = cube_max.copy()
Cocean.data = ma.array(Cocean.data, mask=seamask)
Cland.data = ma.array(Cland.data, mask=landmask)
# --------------
land_cube = Cland
ocean_cube = Cocean
def calculate_beam_uniformity (imagearray,roidim=100,neighborhood_size=400):
''' Function receives image array and optionally ROI dimension (#pixels). The optional neighborhood_size parameter determines the scale of the distances between local maxima.
Four ROIs are defined and for each ROI a mean, std and snr is calculated from the image array. The result is returned as a dictionary.
'''
thr = imagearray.min()/4.0
output = imagearray
x,y = findmax.find_max(imagearray,neighborhood_size, threshold=thr)
width = roidim
height = roidim
xdim ,ydim = np.shape(imagearray)
middle = [elem for elem in zip(x,y) if xdim/2 - neighborhood_size < elem[0] < xdim/2 + neighborhood_size and ydim/2 - neighborhood_size < elem[1] < ydim/2 + neighborhood_size]
x0 = int(middle[0][0]/2)
x1 = int(xdim + middle[0][0])/2
y0 = int(middle[0][1]/2)
y1 = int(ydim + middle[0][1])/2
widthx = int(xdim/10)
widthy = int(xdim/10)
deltax = int(xdim/10)
deltay = int(ydim/10)
roi1 = np.zeros(np.shape(imagearray))
roi1[x0 - widthx : x0 +widthx, y0 - widthy:y0 + widthy] = 1
roi1 = ma.make_mask(roi1)
roi2 = np.zeros(np.shape(imagearray))
roi2[x1 - widthx : x1 + widthx, y0 - widthy:y0 + widthy]=1
roi2 = ma.make_mask(roi2)
roi3 = np.zeros(np.shape(imagearray))
roi3[x0 - widthx : x0 + widthx, y1 - widthy:y1 + widthy]=1
roi3 = ma.make_mask(roi3)
roi4 = np.zeros(np.shape(imagearray))
roi4[x1 - widthx : x1 + widthx, y1 - widthy:y1 + widthy]=1
roi4 = ma.make_mask(roi4)
results = {}
tmp1 = ma.array(imagearray,mask=1-roi1)
print ma.count(tmp1)
results['roi1']={'mean':np.mean(tmp1),'std':np.std(tmp1)}
tmp2 = ma.array(imagearray,mask=1-roi2)
results['roi2']={'mean':np.mean(tmp2),'std':np.std(tmp2)}
tmp3 = ma.array(imagearray,mask=1-roi3)
results['roi3']={'mean':np.mean(tmp3),'std':np.std(tmp3)}
tmp4 = ma.array(imagearray,mask=1-roi4)
results['roi4']={'mean':np.mean(tmp4),'std':np.std(tmp4)}
avgmean = (results['roi1']['mean']+results['roi2']['mean']+results['roi3']['mean']+results['roi4']['mean'])/4.0
avgstd = (results['roi1']['std']+results['roi2']['std']+results['roi3']['std']+results['roi4']['std'])/4.0
if avgstd > 0:
tmpsnr = avgmean/avgstd
else:
tmpsnr = -999
results['avg']={'mean':avgmean,'std':avgstd,'snr':tmpsnr}
results['middle'] = middle
results['image'] = imagearray
return results
