def get_sky_ref_patch(f_gray_one_third, sky_prob_map):
"""
Return mean and std of the *largest* sky patch detected on the input image
+TODO: clear patches other than the largest (set sky_prob to 0)
Will MODIFY sky_prob_map
"""
lbl, nlbl = ndi.label(f_gray_one_third)
lbls = np.arange(1, nlbl+1)
_res = ndi.labeled_comprehension(f_gray_one_third, lbl, lbls, _var_lenpos, object, 0, True)
val = []; lenpos = []; pos_arr = []
for idx, (_val, _lenpos, _pos) in np.ndenumerate(_res):
val.append(_val); lenpos.append(_lenpos); pos_arr.append(_pos)
pos_am = np.array(lenpos).argmax() # pos of largest sky ref patch
print 'max patch amount, patch: ', lenpos[pos_am], val[pos_am]
mean = np.mean(val[pos_am])
std = np.std(val[pos_am])
# clear patches other than the largest (set sky_prob to 0)
# import ipdb; ipdb.set_trace()
for i, _p in enumerate(pos_arr):
if i != pos_am:
sky_prob_map.ravel()[_p] = 0.
return mean, std
def ehabitat(ecor,nw,nwpathout): # main fonction (names of ecoregion, network directory input, network directory input) rq: let the network directory input, network directory input empty as we have all the data on the same local folder
global nwpath
if nw=='':
nwpath = os.getcwd()
else:
nwpath = nw
# to create directory :
#~~~~~~~~~~~~~~~~~~~~
if gmaps == 0: # to check if global maps (input variables) are loaded, if not, do it !
initglobalmaps()
if nwpathout=='':
#outdir = 'results' # ToDo: locally create folder "results" if it does not exist!
outdir = os.path.join(os.path.sep, os.getcwd(), 'results')
safelyMakeDir(outdir)
else:
#outdir = nwpathout+'/results' # SHARED FOLDER PATH
outdir = os.path.join(os.path.sep, nwpathout, 'results')
safelyMakeDir(outdir)
#~~~~~~~~~~~~~~~~~~~~~~~
# to create variables
tseaspamin = tseaspamax = tmax_warm_Mpamin = tmax_warm_Mpamax = prepamin = prepamax = preD_Mpamin = preD_Mpamax = aridpamin = aridpamax = ndviminpamin = ndviminpamax = ndvimaxpamin = ndvimaxpamax = treehpamin = treehpamax = treeDpamin = treeDpamax = soilpamin = soilpamax = slopepamin = slopepamax = None #= = treepamin = treepamax = = ndwipamin = ndwipamax
tseaspamean = tmax_warm_Mpamean = prepamean = preD_Mpamean = aridpamean = ndviminpamean = ndvimaxpamean = treehpamean = treeDpamean = soilpamean = slopepamean = None # = treepamean ndwipamean =
s = nd.generate_binary_structure(2,2) # pattern to know wgich pixels are agggregated, usefull if we want to work on similar areas most restrictive pattern for the landscape patches , for landscape patern analyses
# LOCAL FOLDER
csvname1 = os.path.join(os.path.sep, outdir, 'ecoregs_done.csv') # create name of the file
print csvname1
if os.path.isfile(csvname1) == False: # if the file ecoregs_done doesn't exist :
wb = open(csvname1,'a') # create the file
wb.write('None') # first line
wb.write('\n') # next line
wb.close() # close the file
# LOCAL FOLDER
csvname = os.path.join(os.path.sep, outdir, 'hri_results.csv') # same than before
print csvname
if os.path.isfile(csvname) == False:
wb = open(csvname,'a')
wb.write('ecoregion wdpaid averpasim hr2aver pxpa hr1insumaver hriaver nfeatsaver lpratio lpratio2 numpszok lpmaxsize aggregation tseaspamin tseaspamax tmax_warm_Mpamin tmax_warm_Mpamax prepamin prepamax preD_Mpamin preD_Mpamax aridpamin aridpamax ndviminpamin ndviminpamax ndvimaxpamin ndvimaxpamax treehpamin treehpamax treeDpamin treeDpamax soilpamin soilpamax slopepamin slopepamax tseaspamean tmax_warm_Mpamean prepamean preD_Mpamean aridpamean ndviminpamean ndvimaxpamean treehpamean treeDpamean soilpamean slopepamean') # treepamin treepamax treepamean ndwipamin ndwipamax ndwipamean
wb.write('\n')
wb.close()
ef = 'eco_'+str(ecor)+'.tif' # crate a name of a tif file based on the extent of the eco_ (input)
ecofile = os.path.join(os.path.sep, nwpath, 'ecoregs', ef) # file + path
#ecofile = os.path.join(os.path.sep, nwpath, os.path.sep,'ecoregs', os.path.sep, ef)
print ecofile
avail = os.path.isfile(ecofile) # does this ecoregion file exist in the folder? T/F
if avail == True: # if the file exist in the path mentionned (if it doesn't exit, it's ignore this ecoregion without error !!):
eco_csv = str(ecor)+'.csv' # str = to convert ecor to a string, to have a name
print eco_csv
ecoparksf = os.path.join(os.path.sep, nwpath, 'pas', eco_csv)
#ecoparksf = os.path.join(os.path.sep, nwpath, os.path.sep, 'pas', os.path.sep, eco_csv)
print ecoparksf
#ecoparksf = nwpath+'/pas/'+str(ecor)+'.csv'
src_ds_eco = gdal.Open(ecofile) # for each ecoregion, open the tif
eco = src_ds_eco.GetRasterBand(1) # band 1 called eco
eco_mask0 = eco.ReadAsArray(0,0,eco.XSize,eco.YSize).astype(np.int32) # read the values of the raster as a matrix as integers (red-yellow 1/0 raster values ex:555.tif), with the number of cells we want to read 'eco.XSize,eco.YSize' (read total size of rows and columns)
eco_mask = eco_mask0.flatten() # convert columns and rows of a matrix to one vector (eco.XSize*eco.YSize)
gt_eco = src_ds_eco.GetGeoTransform() # to get the coordinates, properties of the raster map, to begin by a corner
print 'eco mask'
xoff = int((gt_eco[0]-gt_tseas_global[0])/5000)
yoff = int((gt_tseas_global[3]-gt_eco[3])/5000)
tseas_eco_bb0 = tseas_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
tseas_eco_bb = tseas_eco_bb0.flatten()
tseas_eco0 = np.where(eco_mask == 1,(tseas_eco_bb),(0))
tseas_eco = np.where(tseas_eco0 == 65535.0, (float('NaN')),(tseas_eco0))
masktseas = np.isnan(tseas_eco)
tseas_eco[masktseas] = np.interp(np.flatnonzero(masktseas), np.flatnonzero(~masktseas), tseas_eco[~masktseas])
print 'eco tseas'
xoff = int((gt_eco[0]-gt_tmax_warm_M_global[0])/5000)
yoff = int((gt_tmax_warm_M_global[3]-gt_eco[3])/5000)
tmax_warm_M_eco_bb0 = tmax_warm_M_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
tmax_warm_M_eco_bb = tmax_warm_M_eco_bb0.flatten()
tmax_warm_M_eco0 = np.where(eco_mask == 1, (tmax_warm_M_eco_bb),(0))
tmax_warm_M_eco = np.where(tmax_warm_M_eco0 == 65535.0, (float('NaN')),(tmax_warm_M_eco0))
masktmax_warm_M = np.isnan(tmax_warm_M_eco)
tmax_warm_M_eco[masktmax_warm_M] = np.interp(np.flatnonzero(masktmax_warm_M), np.flatnonzero(~masktmax_warm_M), tmax_warm_M_eco[~masktmax_warm_M])
print 'eco tmax_warm_M'
xoff = int((gt_eco[0]-gt_pre_global[0])/5000)
yoff = int((gt_pre_global[3]-gt_eco[3])/5000)
pre_eco_bb0 = pre_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
pre_eco_bb = pre_eco_bb0.flatten()
pre_eco0 = np.where(eco_mask == 1, (pre_eco_bb),(0))
pre_eco = np.where(pre_eco0 == 65535.0, (float('NaN')),(pre_eco0))
maskpre = np.isnan(pre_eco)
pre_eco[maskpre] = np.interp(np.flatnonzero(maskpre), np.flatnonzero(~maskpre), pre_eco[~maskpre])
print 'eco pre'
xoff = int((gt_eco[0]-gt_preD_M_global[0])/5000)
yoff = int((gt_preD_M_global[3]-gt_eco[3])/5000)
preD_M_eco_bb0 = preD_M_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
preD_M_eco_bb = preD_M_eco_bb0.flatten()
preD_M_eco0 = np.where(eco_mask == 1, (preD_M_eco_bb),(0))
preD_M_eco = np.where(preD_M_eco0 == 65535.0, (float('NaN')),(preD_M_eco0))
maskpreD_M = np.isnan(preD_M_eco)
preD_M_eco[maskpreD_M] = np.interp(np.flatnonzero(maskpreD_M), np.flatnonzero(~maskpreD_M), preD_M_eco[~maskpreD_M])
print 'eco preD_M'
xoff = int((gt_eco[0]-gt_arid_global[0])/5000)
yoff = int((gt_arid_global[3]-gt_eco[3])/5000)
arid_eco_bb0 = arid_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
arid_eco_bb = arid_eco_bb0.flatten()
arid_eco0 = np.where(eco_mask == 1, (arid_eco_bb),(0))
arid_eco = np.where(arid_eco0 == 65535.0, (float('NaN')),(arid_eco0))
maskarid = np.isnan(arid_eco)
arid_eco[maskarid] = np.interp(np.flatnonzero(maskarid), np.flatnonzero(~maskarid), arid_eco[~maskarid])
print 'eco arid'
xoff = int((gt_eco[0]-gt_ndvimin_global[0])/5000)
yoff = int((gt_ndvimin_global[3]-gt_eco[3])/5000)
ndvimin_eco_bb0 = ndvimin_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
ndvimin_eco_bb = ndvimin_eco_bb0.flatten()
ndvimin_eco0 = np.where(eco_mask == 1, (ndvimin_eco_bb),(0))
ndvimin_eco = np.where(ndvimin_eco0 == 65535.0, (float('NaN')),(ndvimin_eco0))
maskndvimin = np.isnan(ndvimin_eco)
ndvimin_eco[maskndvimin] = np.interp(np.flatnonzero(maskndvimin), np.flatnonzero(~maskndvimin), ndvimin_eco[~maskndvimin])
print 'eco ndvimin'
xoff = int((gt_eco[0]-gt_ndvimax_global[0])/5000)
yoff = int((gt_ndvimax_global[3]-gt_eco[3])/5000)
ndvimax_eco_bb0 = ndvimax_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
ndvimax_eco_bb = ndvimax_eco_bb0.flatten()
ndvimax_eco0 = np.where(eco_mask == 1, (ndvimax_eco_bb),(0))
ndvimax_eco = np.where(ndvimax_eco0 == 65535.0, (float('NaN')),(ndvimax_eco0))
maskndvimax = np.isnan(ndvimax_eco)
ndvimax_eco[maskndvimax] = np.interp(np.flatnonzero(maskndvimax), np.flatnonzero(~maskndvimax), ndvimax_eco[~maskndvimax])
print 'eco ndvimax'
xoff = int((gt_eco[0]-gt_treeh_global[0])/5000)
yoff = int((gt_treeh_global[3]-gt_eco[3])/5000)
treeh_eco_bb0 = treeh_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
treeh_eco_bb = treeh_eco_bb0.flatten()
treeh_eco0 = np.where(eco_mask == 1, (treeh_eco_bb),(0))
treeh_eco = np.where(treeh_eco0 == 65535.0, (float('NaN')),(treeh_eco0))
masktreeh = np.isnan(treeh_eco)
treeh_eco[masktreeh] = np.interp(np.flatnonzero(masktreeh), np.flatnonzero(~masktreeh), treeh_eco[~masktreeh])
print 'eco treeh'
xoff = int((gt_eco[0]-gt_treeD_global[0])/5000)
yoff = int((gt_treeD_global[3]-gt_eco[3])/5000)
treeD_eco_bb0 = treeD_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
treeD_eco_bb = treeD_eco_bb0.flatten()
treeD_eco0 = np.where(eco_mask == 1, (treeD_eco_bb),(0))
treeD_eco = np.where(treeD_eco0 == 1.8e+308, (float('NaN')),(treeD_eco0))
masktreeD = np.isnan(treeD_eco)
treeD_eco[masktreeD] = np.interp(np.flatnonzero(masktreeD), np.flatnonzero(~masktreeD), treeD_eco[~masktreeD])
print 'eco treeD'
xoff = int((gt_eco[0]-gt_soil_global[0])/5000) # compute the begining of the ecoregion (x and y): gt_eco = ecoregion map , gt_soil_global = global map, we convert it at the resolution we want
yoff = int((gt_soil_global[3]-gt_eco[3])/5000)
soil_eco_bb0 = soil_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32) # as before eco_mask0 but for global, to knoe where and how big is the ecoregion at the global (bounding box = square), astype(np.float32) to keep values
soil_eco_bb = soil_eco_bb0.flatten() # to put vales in one single vector
soil_eco0 = np.where(eco_mask == 1, (soil_eco_bb),(0)) # where you have values of 1 in the ecoregion (= ecoregion present) => take the values of the inVars raster> because bounding box is a square wich contain the ecoregion
soil_eco = np.where(soil_eco0 == 65535.0, (float('NaN')),(soil_eco0)) # where you have 65535 (= NULL from treeD for instence), replace by NaN
masksoil = np.isnan(soil_eco) # to create new mask, when you have nan => TRUE if not => FALSE
soil_eco[masksoil] = np.interp(np.flatnonzero(masksoil), np.flatnonzero(~masksoil), soil_eco[~masksoil]) # where you have TRUE, you do an interpolation with nearbourhood to get values instead of NaN
print 'eco soil'
xoff = int((gt_eco[0]-gt_slope_global[0])/5000)
yoff = int((gt_slope_global[3]-gt_eco[3])/5000)
slope_eco_bb0 = slope_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
slope_eco_bb = slope_eco_bb0.flatten()
slope_eco0 = np.where(eco_mask == 1, (slope_eco_bb),(0))
slope_eco = np.where(slope_eco0 == 65535.0, (float('NaN')),(slope_eco0))
maskslope = np.isnan(slope_eco)
slope_eco[maskslope] = np.interp(np.flatnonzero(maskslope), np.flatnonzero(~maskslope), slope_eco[~maskslope])
print 'eco slope'
ind_eco0 = np.column_stack((tseas_eco,tmax_warm_M_eco,pre_eco,preD_M_eco,arid_eco,ndvimin_eco,ndvimax_eco,treeh_eco,treeD_eco,soil_eco,slope_eco)) # we make an array putting all the columns of each variable together # tree_eco,,ndwi_eco
print 'ecovars stacked'
print ecoparksf
pa_list0 = np.genfromtxt(ecoparksf,dtype='int') # crear este archivo en subpas! read the txt file (eco_csv file = 555.csv), reand pas od one ecoreg
pa_list = np.unique(pa_list0) # take each pas of the list just once (in case the pa name appears several times)
n = len(pa_list) # to have the number of pas computed
for px in range(0,n): # 0,n
pa = pa_list[px] # for each pa
print pa
outfile = os.path.join(os.path.sep, outdir, str(ecor)+'_'+str(pa)+'.tif') # prepare the .tif files to be filled
outfile2 = os.path.join(os.path.sep, outdir, str(ecor)+'_'+str(pa)+'_lp.tif')
outfile3 = os.path.join(os.path.sep, outdir, str(ecor)+'_'+str(pa)+'_mask.tif')
#outfile = outdir+'/'+str(ecor)+'_'+str(pa)+'.tif' # LOCAL FOLDER
pa_infile = 'pa_'+str(pa)+'.tif'
pa4 = os.path.join(os.path.sep, nwpath, 'pas', pa_infile) # pas input
#pa4 = os.path.join(os.path.sep, nwpath, os.path.sep, 'pas', os.path.sep, pa_infile)
print pa4
#pa4 = nwpath+'/pas/pa_'+str(pa)+'.tif'
dropcols = np.arange(10,dtype=int) # create vector from 0 to 10 for 11 variables
done = os.path.isfile(outfile) # outfile is already created
avail2 = os.path.isfile(pa4) # check if input is available
if done == False and avail2 == True: # if the files pa4 exists but the HRI computing is not done yet (* see ptt)
pafile=pa4
src_ds_pa = gdal.Open(pafile) # open the pa with gdal
par = src_ds_pa.GetRasterBand(1) # same than previously with ecoreg
pa_mask0 = par.ReadAsArray(0,0,par.XSize,par.YSize).astype(np.int32)
pa_mask = pa_mask0.flatten()
ind = pa_mask > 0 # create index T/F, T if >0.
go = 1
sum_pa_mask = sum(pa_mask[ind])# create the sum of the 1 values to know the size of the pa
if sum_pa_mask < 2: go = 0 # not processing areas smaller than 2 pixels, if size too small, don't go
print sum_pa_mask # print the size of the pa
sum_pa_mask_inv = len(pa_mask[pa_mask == 0]) # what is the size of 0 pixels inside the pa
print sum_pa_mask_inv # print size of 0
print len(pa_mask)
ratiogeom = 5000 # when bounding box is bigger than pa (due to wwpa database error)
if sum_pa_mask > 0: ratiogeom = sum_pa_mask_inv/sum_pa_mask # total pixel outside much bigger than pixel in pa ?
#print ratiogeom
gt_pa = src_ds_pa.GetGeoTransform() # get coordinates of pa
xoff = int((gt_pa[0]-gt_pre_global[0])/5000) # compute from the global map xoff, yoff of pa
yoff = int((gt_pre_global[3]-gt_pa[3])/5000)
if xoff>=0 and yoff>=0 and go == 1: # xoff>0 to be sure we are not in a boarder of ecoreg and our pa is > 2
num_bands=src_ds_eco.RasterCount # to have the number of bands in one ecoreg, determined as 1 in the script
driver = gdal.GetDriverByName("GTiff") # create a new tif file where to store the output (hri)
dst_options = ['COMPRESS=LZW'] # compression method of the files as many will be processed
dst_ds = driver.Create( outfile,src_ds_eco.RasterXSize,src_ds_eco.RasterYSize,num_bands,gdal.GDT_Float32,dst_options) # to create the tif file empty
dst_ds.SetGeoTransform( src_ds_eco.GetGeoTransform())
dst_ds.SetProjection(src_ds_eco.GetProjectionRef()) # determine the file projection. preparation of the tif file over
# 1. tseas
xoff = int((gt_pa[0]-gt_tseas_global[0])/5000)
yoff = int((gt_tseas_global[3]-gt_pa[3])/5000)
tseas_pa_bb0 = tseas_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
tseas_pa_bb = tseas_pa_bb0.flatten()
tseas_pa0 = tseas_pa_bb[ind]
tseas_pa = np.where(tseas_pa0 == 65535.0, (float('NaN')),(tseas_pa0))
mask2tseas = np.isnan(tseas_pa)
if mask2tseas.all() == True: # if all the variable values in a pa are only NA, put -8 in dropcol
dropcols[0] = -0
else: # if there are also non NA values
tseas_pa[mask2tseas] = np.interp(np.flatnonzero(mask2tseas), np.flatnonzero(~mask2tseas), tseas_pa[~mask2tseas])
tseas_pa = np.random.random_sample(len(tseas_pa),)/5000 + tseas_pa
print 'pa tseas'
tseaspamin = round(tseas_pa.min(),2)
tseaspamax = round(tseas_pa.max(),2)
tseaspamean = round(np.mean(tseas_pa),2)
print tseaspamin
print tseaspamax
tseasdiff = abs(tseas_pa.min()-tseas_pa.max())
if tseasdiff < 0.001: dropcols[0] = -0
# 2. tmax_warm_M
xoff = int((gt_pa[0]-gt_tmax_warm_M_global[0])/5000)
yoff = int((gt_tmax_warm_M_global[3]-gt_pa[3])/5000)
tmax_warm_M_pa_bb0 = tmax_warm_M_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
tmax_warm_M_pa_bb = tmax_warm_M_pa_bb0.flatten()
tmax_warm_M_pa0 = tmax_warm_M_pa_bb[ind]
tmax_warm_M_pa = np.where(tmax_warm_M_pa0 == 65535.0, (float('NaN')),(tmax_warm_M_pa0))
mask2tmax_warm_M = np.isnan(tmax_warm_M_pa)
if mask2tmax_warm_M.all() == True:
dropcols[1] = -1
else:
tmax_warm_M_pa[mask2tmax_warm_M] = np.interp(np.flatnonzero(mask2tmax_warm_M), np.flatnonzero(~mask2tmax_warm_M), tmax_warm_M_pa[~mask2tmax_warm_M])
tmax_warm_M_pa = np.random.random_sample(len(tmax_warm_M_pa),)/5000 + tmax_warm_M_pa
print 'pa tmax_warm_M'
tmax_warm_Mpamin = round(tmax_warm_M_pa.min(),2)
tmax_warm_Mpamax = round(tmax_warm_M_pa.max(),2)
tmax_warm_Mpamean = round(np.mean(tmax_warm_M_pa),2)
print tmax_warm_Mpamin
print tmax_warm_Mpamax
tmax_warm_Mdiff = abs(tmax_warm_M_pa.min()-tmax_warm_M_pa.max())
if tmax_warm_Mdiff < 0.001: dropcols[1] = -1
# 3. pre
xoff = int((gt_pa[0]-gt_pre_global[0])/5000)
yoff = int((gt_pre_global[3]-gt_pa[3])/5000)
pre_pa_bb0 = pre_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
pre_pa_bb = pre_pa_bb0.flatten()
pre_pa0 = pre_pa_bb[ind]
pre_pa = np.where(pre_pa0 == 65535.0, (float('NaN')),(pre_pa0))
mask2pre = np.isnan(pre_pa)
if mask2pre.all() == True:
dropcols[2] = -2
else:
pre_pa[mask2pre] = np.interp(np.flatnonzero(mask2pre), np.flatnonzero(~mask2pre), pre_pa[~mask2pre])
pre_pa = np.random.random_sample(len(pre_pa),)/5000 + pre_pa
print 'pa pre'
prepamin = round(pre_pa.min(),2)
prepamax = round(pre_pa.max(),2)
prepamean = round(np.mean(pre_pa),2)
print prepamin
print prepamax
prediff = abs(pre_pa.min()-pre_pa.max())
if prediff < 0.001: dropcols[2] = -2
# # 4. preD_M
xoff = int((gt_pa[0]-gt_preD_M_global[0])/5000)
yoff = int((gt_preD_M_global[3]-gt_pa[3])/5000)
preD_M_pa_bb0 = preD_M_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
preD_M_pa_bb = preD_M_pa_bb0.flatten()
preD_M_pa0 = preD_M_pa_bb[ind]
preD_M_pa = np.where(preD_M_pa0 == 65535.0, (float('NaN')),(preD_M_pa0))
mask2preD_M = np.isnan(preD_M_pa)
if mask2preD_M.all() == True:
dropcols[3] = -3
else:
preD_M_pa[mask2preD_M] = np.interp(np.flatnonzero(mask2preD_M), np.flatnonzero(~mask2preD_M), preD_M_pa[~mask2preD_M])
preD_M_pa = np.random.random_sample(len(preD_M_pa),)/5000 + preD_M_pa
print 'pa preD_M'
preD_Mpamin = round(preD_M_pa.min(),2)
preD_Mpamax = round(preD_M_pa.max(),2)
preD_Mpamean = round(np.mean(preD_M_pa),2)
print preD_Mpamin
print preD_Mpamax
preD_Mdiff = abs(preD_M_pa.min()-preD_M_pa.max())
if preD_Mdiff < 0.001: dropcols[3] = -3
# 5. arid
xoff = int((gt_pa[0]-gt_arid_global[0])/5000)
yoff = int((gt_arid_global[3]-gt_pa[3])/5000)
arid_pa_bb0 = arid_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
arid_pa_bb = arid_pa_bb0.flatten()
arid_pa0 = arid_pa_bb[ind]
arid_pa = np.where(arid_pa0 == 65535.0, (float('NaN')),(arid_pa0))
mask2arid = np.isnan(arid_pa)
if mask2arid.all() == True:
dropcols[4] = -4
else:
arid_pa[mask2arid] = np.interp(np.flatnonzero(mask2arid), np.flatnonzero(~mask2arid), arid_pa[~mask2arid])
arid_pa = np.random.random_sample(len(arid_pa),)/5000 + arid_pa
print 'pa arid'
aridpamin = round(arid_pa.min(),2)
aridpamax = round(arid_pa.max(),2)
aridpamean = round(np.mean(arid_pa),2)
print aridpamin
print aridpamax
ariddiff = abs(arid_pa.min()-arid_pa.max())
if ariddiff < 0.001: dropcols[4] = -4
# 6. ndvimin
xoff = int((gt_pa[0]-gt_ndvimin_global[0])/5000)
yoff = int((gt_ndvimin_global[3]-gt_pa[3])/5000)
ndvimin_pa_bb0 = ndvimin_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
ndvimin_pa_bb = ndvimin_pa_bb0.flatten()
ndvimin_pa0 = ndvimin_pa_bb[ind]
ndvimin_pa = np.where(ndvimin_pa0 == 65535.0, (float('NaN')),(ndvimin_pa0))
mask2ndvimin = np.isnan(ndvimin_pa)
if mask2ndvimin.all() == True:
dropcols[5] = -5
else:
ndvimin_pa[mask2ndvimin] = np.interp(np.flatnonzero(mask2ndvimin), np.flatnonzero(~mask2ndvimin), ndvimin_pa[~mask2ndvimin])
ndvimin_pa = np.random.random_sample(len(ndvimin_pa),)/5000 + ndvimin_pa
print 'pa ndvimin'
ndviminpamin = round(ndvimin_pa.min(),2)
ndviminpamax = round(ndvimin_pa.max(),2)
ndviminpamean = round(np.mean(ndvimin_pa),2)
print ndviminpamin
print ndviminpamax
ndvimindiff = abs(ndvimin_pa.min()-ndvimin_pa.max())
if ndvimindiff < 0.001: dropcols[5] = -5
# 7. ndvimax
xoff = int((gt_pa[0]-gt_ndvimax_global[0])/5000)
yoff = int((gt_ndvimax_global[3]-gt_pa[3])/5000)
ndvimax_pa_bb0 = ndvimax_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
ndvimax_pa_bb = ndvimax_pa_bb0.flatten()
ndvimax_pa0 = ndvimax_pa_bb[ind]
ndvimax_pa = np.where(ndvimax_pa0 == 65535.0, (float('NaN')),(ndvimax_pa0))
mask2ndvimax = np.isnan(ndvimax_pa)
if mask2ndvimax.all() == True:
dropcols[6] = -6
else:
ndvimax_pa[mask2ndvimax] = np.interp(np.flatnonzero(mask2ndvimax), np.flatnonzero(~mask2ndvimax), ndvimax_pa[~mask2ndvimax])
ndvimax_pa = np.random.random_sample(len(ndvimax_pa),)/5000 + ndvimax_pa
print 'pa ndvimax'
ndvimaxpamin = round(ndvimax_pa.min(),2)
ndvimaxpamax = round(ndvimax_pa.max(),2)
ndvimaxpamean = round(np.mean(ndvimax_pa),2)
print ndvimaxpamin
print ndvimaxpamax
ndvimaxdiff = abs(ndvimax_pa.min()-ndvimax_pa.max())
if ndvimaxdiff < 0.001: dropcols[6] = -6
# 8. treeh
xoff = int((gt_pa[0]-gt_treeh_global[0])/5000) # start to read the trrh cover in the pa
yoff = int((gt_treeh_global[3]-gt_pa[3])/5000)
treeh_pa_bb0 = treeh_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
treeh_pa_bb = treeh_pa_bb0.flatten()
treeh_pa0 = treeh_pa_bb[ind] # to read the values inside the pa when we have 1 (higher than 0)
treeh_pa = np.where(treeh_pa0 == 255.0, (float('NaN')),(treeh_pa0))
mask2treeh = np.isnan(treeh_pa) # isnan makes an index of T/F, to know which is nan
if mask2treeh.all() == True: # if all the variable values in a pa are only NA, put -8 in dropcol
dropcols[7] = -7
else: # if there are also non NA values
treeh_pa[mask2treeh] = np.interp(np.flatnonzero(mask2treeh), np.flatnonzero(~mask2treeh), treeh_pa[~mask2treeh]) # we do the neirbourood interpolation
treeh_pa = np.random.random_sample(len(treeh_pa),)/5000 + treeh_pa # add a random noise with an insignificant value to avoid to have all the pixels (from input variables) with the same value and to avoid perfect inverse correlation (between 2 variables ex:treeD/treehs), otherwise Mahalanobis distance can't work
print 'pa treeh'
treehpamin = round(treeh_pa.min(),2) #to compute th min of the variable
treehpamax = round(treeh_pa.max(),2) #to compute the max of the variable
treehpamean = round(np.mean(treeh_pa),2) #to compute the mean of the variable
print treehpamin
print treehpamax
print treehpamean
treehdiff = abs(treeh_pa.min()-treeh_pa.max())
if treehdiff < 0.001: dropcols[7] = -7 # if the difference is too tiny = if the value of the variable don't change => don't use it
# 9. treeD
xoff = int((gt_pa[0]-gt_treeD_global[0])/5000)
yoff = int((gt_treeD_global[3]-gt_pa[3])/5000)
treeD_pa_bb0 = treeD_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
treeD_pa_bb = treeD_pa_bb0.flatten()
treeD_pa0 = treeD_pa_bb[ind]
treeD_pa = np.where(treeD_pa0 == 1.8e+308, (float('NaN')),(treeD_pa0))
mask2treeD = np.isnan(treeD_pa)
if mask2treeD.all() == True:
dropcols[8] = -8
else:
treeD_pa[mask2treeD] = np.interp(np.flatnonzero(mask2treeD), np.flatnonzero(~mask2treeD), treeD_pa[~mask2treeD])
treeD_pa = np.random.random_sample(len(treeD_pa),)/5000 + treeD_pa
print 'pa treeD'
hpamin = round(treeD_pa.min(),2)
hpamax = round(treeD_pa.max(),2)
hpamean = round(np.mean(treeD_pa),2)
print hpamin
print hpamax
hdiff = abs(treeD_pa.min()-treeD_pa.max())
if hdiff < 0.001: dropcols[8] = -8
# 10. soil
xoff = int((gt_pa[0]-gt_soil_global[0])/5000)
yoff = int((gt_soil_global[3]-gt_pa[3])/5000)
soil_pa_bb0 = soil_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
soil_pa_bb = soil_pa_bb0.flatten()
soil_pa0 = soil_pa_bb[ind]
soil_pa = np.where(soil_pa0 == 65535.0, (float('NaN')),(soil_pa0))
mask2soil = np.isnan(soil_pa)
if mask2soil.all() == True:
dropcols[9] = -9
else:
soil_pa[mask2soil] = np.interp(np.flatnonzero(mask2soil), np.flatnonzero(~mask2soil), soil_pa[~mask2soil])
soil_pa = np.random.random_sample(len(soil_pa),)/5000 + soil_pa
print 'pa soil'
soilpamin = round(soil_pa.min(),2)
soilpamax = round(soil_pa.max(),2)
soilpamean = round(np.mean(soil_pa),2)
print soilpamin
print soilpamax
soildiff = abs(soil_pa.min()-soil_pa.max())
if soildiff < 0.001: dropcols[9] = -9
# 11. slope
xoff = int((gt_pa[0]-gt_slope_global[0])/5000)
yoff = int((gt_slope_global[3]-gt_pa[3])/5000)
slope_pa_bb0 = slope_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
slope_pa_bb = slope_pa_bb0.flatten()
slope_pa0 = slope_pa_bb[ind]
slope_pa = np.where(slope_pa0 == 65535.0, (float('NaN')),(slope_pa0))
mask2slope = np.isnan(slope_pa)
if mask2slope.all() == True:
dropcols[10] = -10
else:
slope_pa[mask2slope] = np.interp(np.flatnonzero(mask2slope), np.flatnonzero(~mask2slope), slope_pa[~mask2slope])
slope_pa = np.random.random_sample(len(slope_pa),)/5000 + slope_pa
print 'pa slope'
slopepamin = round(slope_pa.min(),2)
slopepamax = round(slope_pa.max(),2)
slopepamean = round(np.mean(slope_pa),2)
print slopepamin
print slopepamax
slopediff = abs(slope_pa.min()-slope_pa.max())
if slopediff < 0.001: dropcols[10] = -10
cols = dropcols[dropcols>=0] # select the "columns" which are positive ex: if hdiff < 0.001: dropcols[3] = -3 => we not use it
ind_pa0 = np.column_stack((tseas_pa,tmax_warm_M_pa,pre_pa,preD_M_pa,arid_pa,ndvimin_pa,ndvimax_pa,treeh_pa,treeD_pa,soil_pa,slope_pa)) # stack of all the columns (even the negative) tree_pa, ndwi_pa,
ind_pa = ind_pa0[:,cols] # we select from the previoous stack only positive columns for pas
ind_eco = ind_eco0[:,cols] # we select from the previoous stack only positive columns for ecoreg
print ind_pa.shape
hr1sum = hr1insum = indokpsz = pszok = sumpszok = lpratio2 = numpszok = hr1averpa = hr3aver = hr2aver = pszmax = num_featuresaver = lpratio = hr1medianpa = hr1insumaver = pxpa = aggregation = None
print "PA masked"
#print ind_pa
if ind_pa.shape[0]>2 and ind_pa.shape[1]>1: #if we have at least 3 pixels per pa and 2 variables, then we start mahalanobis computation
Ymean = np.mean(ind_pa,axis=0) #mean of each of the positive variables for one pa
print 'Max. mean value is '+ str(Ymean.max()) # print the maximum of the mean of the 6 variables
print "Ymean ok"
Ycov = np.cov(ind_pa,rowvar=False) # do the covariance among the positive variables
print 'Max. cov value is '+ str(Ycov.max())
print "Ycov ok"
#mh = mahalanobis_distances(Ymean, Ycov, ind_eco, parallel=False)
#mh = mahalanobis_distances(Ymean, Ycov, ind_eco, parallel=True)
mh2 = mahalanobis_distances_scipy(Ymean, Ycov, ind_eco, parallel=True) # to compute the mahalanobis distance (in parallel)
#mh2 = mahalanobis_distances_scipy(Ymean, Ycov, ind_eco, parallel=False)
maxmh=mh2.max() # max to check there is no NA
print 'Max. mh value is '+ str(maxmh)
print 'Max. mh value is nan: '+ str(np.isnan(maxmh))
mh = mh2*mh2 #multiply the mahalanobis distance by itself, to make chi2 (because sqrt is needed by mahalanobis)
print "mh ok" # transform with the chi2 the values in 0/1
pmh = chisqprob(mh,len(cols)).reshape((eco.YSize,eco.XSize)) # chi2 with mh with 6 variables ( or change chisqprob(mh,6) by: chisqprob(mh,len(cols)) ), and transform the vector in 2D matrix
# pmhh = np.where(pmh <= 0.001,None, pmh) # if the value in pmh is really low, put NA (chi2 values goes from 0 to 1)
# print "pmh ok" # quitar valores muy bajos!
# pmhhmax = pmhh.max() # max should be close to 1
# print 'Max. similarity value is '+ str(pmhhmax)
# dst_ds.GetRasterBand(1).WriteArray(pmhh) #put values of pmhh in ecoregion map (dst_ds, l.399)
# dst_ds = None # close the file and save it (with hri inside)
# hr11 = np.where(pmhh>0,1,0) # 0.5
print "pmh ok" # quitar valores muy bajos!
pmhhmax = pmh.max() # max should be close to 1
print 'Max. similarity value is '+ str(pmhhmax)
dst_ds.GetRasterBand(1).WriteArray(pmh) #put values of pmh in ecoregion map (dst_ds, l.399)
dst_ds = None # close the file and save it (with hri inside)
hr11 = np.where(pmh>0,1,0) # 0.5
hr1 = hr11.flatten()
hr1sum = sum(hr1)
print 'Number of pixels with similarity higher than 0 is '+str(hr1sum)
hr1insumaver = hr1insum = 0
hr1sumaver = hr1sum
src_ds_sim = gdal.Open(outfile)
sim = src_ds_sim.GetRasterBand(1)
gt_sim = src_ds_sim.GetGeoTransform()
xoff = int((gt_pa[0]-gt_sim[0])/5000)
yoff = int((gt_sim[3]-gt_pa[3])/5000)
xextentpa = xoff + par.XSize
yextentpa = yoff + par.YSize
xless = sim.XSize - xextentpa
yless = sim.YSize - yextentpa
xsize = par.XSize
ysize = par.YSize
if xoff>0 and yoff>0 and pmhhmax>0.01 and hr1sum>1 and maxmh!=float('NaN'):#and ratiogeom < 100: # also checks if results are not empty
# reading the similarity ecoregion without the PA (tmp mask) : for landscape metrics
os.system('gdal_merge.py '+str(ecofile)+' '+str(pa4)+' -o '+str(outfile3)+' -ot Int32') # gdal tools is needed !!! otherwise => error
hri_pa_bb03 = sim.ReadAsArray().astype(np.float32)
hri_pa_bb3 = hri_pa_bb03.flatten()
src_ds_sim2 = gdal.Open(outfile3)
sim2 = src_ds_sim2.GetRasterBand(1)
gt_sim2 = src_ds_sim2.GetGeoTransform()
hri_pa_bb02 = sim2.ReadAsArray().astype(np.int32)
#hri_pa_bb2 = hri_pa_bb02.flatten()
hri_pa_bb02_max = hri_pa_bb02.max()
print 'PA: '+str(pa)
print 'PA (= max) value from mask = '+str(hri_pa_bb02_max)
if hri_pa_bb02.shape == hri_pa_bb03.shape:
hri_pa02 = np.where(hri_pa_bb02 == pa,0,hri_pa_bb03) # hri_pa_bb02_max
if xless < 0: xsize = xsize + xless
if yless < 0: ysize = ysize + yless
hri_pa_bb0 = sim.ReadAsArray(xoff,yoff,xsize,ysize).astype(np.float32)
hri_pa_bb = hri_pa_bb0.flatten()
indd = hri_pa_bb > 0
hri_pa0 = hri_pa_bb[indd]
print 'Total number of pixels with similarity values in PA: '+str(len(hri_pa0))
hr1averpa = round(np.mean(hri_pa0[~np.isnan(hri_pa0)]),2) # compute the mean similarity inside a pa
#print hr1averpa
#hr1medianpa = np.median(hri_pa0[~np.isnan(hri_pa0)])
print 'mean similarity in the park is '+str(hr1averpa)
#hr1insum = sum(np.where(hri_pa0 >= 0.5, 1,0)) # use hr1averpa as threshold instead!
##hr1inaver = np.where(hri_pa0 >= hr1averpa, 1,0)
##hr1insumaver = sum(hr1inaver)
#print hr1insum
##labeled_arrayin, num_featuresin = nd.label(hr1inaver, structure=s)
hr1averr = np.where(hri_pa02 >= hr1averpa, 1,0) # pmhh
hr1aver = hr1averr.flatten()
print 'Total number of pixels with similarity values in ECO: '+str(sum(hr1aver))
labeled_arrayaver, num_featuresaver = nd.label(hr1averr, structure=s)
print 'Nr of similar patches found: '+str(num_featuresaver)
if num_featuresaver > 0:
lbls = np.arange(1, num_featuresaver+1)
psizes = nd.labeled_comprehension(labeled_arrayaver, labeled_arrayaver, lbls, np.count_nonzero, float, 0) #-1
pszmax = psizes.max()#-hr1insumaver
dst_ds2 = driver.Create(outfile2,src_ds_eco.RasterXSize,src_ds_eco.RasterYSize,num_bands,gdal.GDT_Int32,dst_options)
dst_ds2.SetGeoTransform(src_ds_eco.GetGeoTransform())
dst_ds2.SetProjection(src_ds_eco.GetProjectionRef())
dst_ds2.GetRasterBand(1).WriteArray(labeled_arrayaver)
dst_ds2 = None
#num_feats = num_features - num_featuresaver
hr1sumaver = sum(hr1aver)
hr2aver = hr1sumaver #- hr1insumaver , number of pixel in the all ecoregion which have similarity values higher or equal than the average inside a pa
pxpa = ind_pa.shape[0] # size of the pa used for computation
indokpsz = psizes >= pxpa
pszsok = psizes[indokpsz] # NEW
sumpszok = sum(pszsok)
lpratio=round(float(pszmax/pxpa),2)
lpratio2=round(float(sumpszok/pxpa),2)
numpszok = len(pszsok)
hr3aver = round(float(hr2aver/pxpa),2)
aggregation = round(float(hr2aver/num_featuresaver),2)
#hr2 = hr1sumaver - hr1insumaver
#print hr2
#hr3 = float(hr2/ind_pa.shape[0])
#print hr3
wb = open(csvname,'a')
var = str(ecor)+' '+str(pa)+' '+str(hr1averpa)+' '+str(hr2aver)+' '+str(pxpa)+' '+str(hr3aver)+' '+str(num_featuresaver)+' '+str(lpratio)+' '+str(lpratio2)+' '+str(numpszok)+' '+str(pszmax)+' '+str(aggregation)+' '+str(tseasmin)+' '+str(tseasmax)+' '+str(tmax_warm_Mmin)+' '+str(tmax_warm_Mmax)+' '+str(premin)+' '+str(premax)+' '+str(preD_Mmin)+' '+str(preD_Mmax)+' '+str(aridmin)+' '+str(aridmax)+' '+str(ndviminmin)+' '+str(ndviminmax)+' '+str(ndvimaxmin)+' '+str(ndvimaxmax)+' '+str(treehmin)+' '+str(treehmax)+' '+str(treeDmin)+' '+str(treeDmax)+' '+str(soilmin)+' '+str(soilmax)+' '+str(slopemin)+' '+str(slopemax)+' '+str(tseasmean)+' '+str(tmax_warm_Mmean)+' '+str(premean)+' '+str(preD_Mmean)+' '+str(aridmean)+' '+str(ndviminmean)+' '+str(ndvimaxmean)+' '+str(treehmean)+' '+str(treeDmean)+' '+str(soilmean)+' '+str(slopemean) # exclude PA! '+str(treemin)+' '+str(treemax)+' '+str(treemean)+' '+str(ndwimin)+' '+str(ndwimax)+' '+str(ndwimean)+'
wb.write(var)
wb.write('\n')
wb.close()
print "results exported"
os.system('rm '+str(outfile3))
wb = open(csvname1,'a') # where we write the final results (LOCAL FOLDER)
var = str(ecor)
wb.write(var)
wb.write('\n')
wb.close()
print "END ECOREG: " + str(ecor)
def objstats(args):
# Open and read from image and segmentation
try:
img_ds = gdal.Open(args.image, gdal.GA_ReadOnly)
except:
logger.error('Could not open image: {}'.format(args.image))
sys.exit(1)
try:
seg_ds = ogr.Open(args.segment, 0)
seg_layer = seg_ds.GetLayer()
except:
logger.error('Could not open segmentation vector file: {}'.format(
args.segment))
sys.exit(1)
cols, rows = img_ds.RasterXSize, img_ds.RasterYSize
bands = range(1, img_ds.RasterCount + 1)
if args.bands is not None:
bands = args.bands
# Rasterize segments
logger.debug('About to rasterize segment vector file')
img_srs = osr.SpatialReference()
img_srs.ImportFromWkt(img_ds.GetProjectionRef())
mem_raster = gdal.GetDriverByName('MEM').Create(
'', cols, rows, 1, gdal.GDT_UInt32)
mem_raster.SetProjection(img_ds.GetProjection())
mem_raster.SetGeoTransform(img_ds.GetGeoTransform())
# Create artificial 'FID' field
fid_layer = seg_ds.ExecuteSQL(
'select FID, * from "{l}"'.format(l=seg_layer.GetName()))
gdal.RasterizeLayer(mem_raster, [1], fid_layer, options=['ATTRIBUTE=FID'])
logger.debug('Rasterized segment vector file')
seg = mem_raster.GetRasterBand(1).ReadAsArray()
logger.debug('Read segmentation image into memory')
mem_raster = None
seg_ds = None
# Get list of unique segments
useg = np.unique(seg)
# If calc is num, do only for 1 band
out_bands = 0
for stat in args.stat:
if stat == 'num':
out_bands += 1
else:
out_bands += len(bands)
# Create output driver
driver = gdal.GetDriverByName(args.format)
out_ds = driver.Create(args.output, cols, rows, out_bands,
gdal.GDT_Float32)
# Loop through image bands
out_b = 0
out_2d = np.empty_like(seg, dtype=np.float32)
for i_b, b in enumerate(bands):
img_band = img_ds.GetRasterBand(b)
ndv = img_band.GetNoDataValue()
band_name = img_band.GetDescription()
if not band_name:
band_name = 'Band {i}'.format(i=b)
logger.info('Processing input band {i}, "{b}"'.format(
i=b, b=band_name))
img = img_band.ReadAsArray().astype(
gdal_array.GDALTypeCodeToNumericTypeCode(img_band.DataType))
logger.debug('Read image band {i}, "{b}" into memory'.format(
i=b, b=band_name))
for stat in args.stat:
logger.debug(' calculating {s}'.format(s=stat))
if stat == 'mean':
out = ndimage.mean(img, seg, useg)
elif stat == 'var':
out = ndimage.variance(img, seg, useg)
elif stat == 'num':
# Remove from list of stats so it is only calculated once
args.stat.remove('num')
count = np.ones_like(seg)
out = ndimage.sum(count, seg, useg)
elif stat == 'sum':
out = ndimage.sum(img, seg, useg)
elif stat == 'min':
out = ndimage.minimum(img, seg, useg)
elif stat == 'max':
out = ndimage.maximum(img, seg, useg)
elif stat == 'mode':
out = ndimage.labeled_comprehension(img, seg, useg,
scipy_mode,
out_2d.dtype, ndv)
else:
logger.error('Unknown stat. Not sure how you got here')
sys.exit(1)
# Transform to 2D
out_2d = out[seg - seg.min()]
# Fill in NDV
if ndv is not None:
out_2d[np.where(img == ndv)] = ndv
# Write out the data
out_band = out_ds.GetRasterBand(out_b + 1)
out_band.SetDescription(band_name)
if ndv is not None:
out_band.SetNoDataValue(ndv)
logger.debug(' Writing object statistic for band {b}'.format(
b=b + 1))
out_band.WriteArray(out_2d, 0, 0)
out_band.FlushCache()
logger.debug(' Wrote out object statistic for band {b}'.format(
b=b + 1))
out_b += 1
out_ds.SetGeoTransform(img_ds.GetGeoTransform())
out_ds.SetProjection(img_ds.GetProjection())
img_ds = None
seg_ds = None
out_ds = None
logger.info('Completed object statistic calculation')
def _significant_features(radar,
field,
gatefilter=None,
min_size=None,
size_bins=75,
size_limits=(0, 300),
structure=None,
remove_size_field=True,
fill_value=None,
size_field=None,
debug=False,
verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if size_field is None:
size_field = '{}_feature_size'.format(field)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Parse binary structuring element
if structure is None:
structure = ndimage.generate_binary_structure(2, 1)
# Initialize echo feature size array
size_data = np.zeros_like(radar.fields[field]['data'],
subok=False,
dtype=np.int32)
# Loop over all sweeps
feature_sizes = []
for sweep in radar.iter_slice():
# Parse radar sweep data and define only valid gates
is_valid_gate = ~radar.fields[field]['data'][sweep].mask
# Label the connected features in radar sweep data and create index
# array which defines each unique label (feature)
labels, nlabels = ndimage.label(is_valid_gate,
structure=structure,
output=None)
index = np.arange(1, nlabels + 1, 1)
if debug:
print 'Number of unique features for {}: {}'.format(sweep, nlabels)
# Compute the size (in radar gates) of each echo feature
# Check for case where no echo features are found, e.g., no data in
# sweep
if nlabels > 0:
sweep_sizes = ndimage.labeled_comprehension(
is_valid_gate, labels, index, np.count_nonzero, np.int32, 0)
feature_sizes.append(sweep_sizes)
# Set each label (feature) to its total size (in radar gates)
for label, size in zip(index, sweep_sizes):
size_data[sweep][labels == label] = size
# Stack sweep echo feature sizes
feature_sizes = np.hstack(feature_sizes)
# Compute histogram of echo feature sizes, bin centers and bin
# width
counts, bin_edges = np.histogram(feature_sizes,
bins=size_bins,
range=size_limits,
normed=False,
weights=None,
density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {} gate(s)'.format(bin_width)
# Compute the peak of the echo feature size distribution
# We expect the peak of the echo feature size distribution to be close to 1
# radar gate
peak_size = bin_centers[counts.argmax()] - bin_width / 2.0
if debug:
print 'Feature size at peak: {} gate(s)'.format(peak_size)
# Determine the first instance when the count (sample size) for an echo
# feature size bin reaches 0 after the distribution peak
# This will define the minimum echo feature size
is_zero_size = np.logical_and(bin_centers > peak_size,
np.isclose(counts, 0, atol=1.0e-5))
min_size = bin_centers[is_zero_size].min() - bin_width / 2.0
if debug:
_range = [0.0, min_size]
print 'Insignificant feature size range: {} gates'.format(_range)
# Mask invalid feature sizes, e.g., zero-size features
size_data = np.ma.masked_equal(size_data, 0, copy=False)
size_data.set_fill_value(fill_value)
# Add echo feature size field to radar
size_dict = {
'data': size_data.astype(np.int32),
'standard_name': size_field,
'long_name': '',
'_FillValue': size_data.fill_value,
'units': 'unitless',
}
radar.add_field(size_field, size_dict, replace_existing=True)
# Update gate filter
gatefilter.include_above(size_field, min_size, op='and', inclusive=False)
# Remove eacho feature size field
if remove_size_field:
radar.fields.pop(size_field, None)
return gatefilter
def main():
# Parse command line options
parser = argparse.ArgumentParser(description='Test different nets with 3D data.')
parser.add_argument('--flair', action='store', dest='flair', default='FLAIR_preprocessed.nii.gz')
parser.add_argument('--pd', action='store', dest='pd', default='DP_preprocessed.nii.gz')
parser.add_argument('--t2', action='store', dest='t2', default='T2_preprocessed.nii.gz')
parser.add_argument('--t1', action='store', dest='t1', default='T1_preprocessed.nii.gz')
parser.add_argument('--output', action='store', dest='output', default='output.nii.gz')
parser.add_argument('--no-docker', action='store_false', dest='docker', default=True)
c = color_codes()
patch_size = (15, 15, 15)
options = vars(parser.parse_args())
batch_size = 10000
min_size = 30
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Loading the net ' + c['b'] + '1' + c['nc'] + c['g'] + '>' + c['nc'])
net_name = '/usr/local/nets/deep-challenge2016.init.model_weights.pkl' if options['docker'] \
else './deep-challenge2016.init.model_weights.pkl'
net = NeuralNet(
layers=[
(InputLayer, dict(name='in', shape=(None, 4, 15, 15, 15))),
(Conv3DDNNLayer, dict(name='conv1_1', num_filters=32, filter_size=(5, 5, 5), pad='same')),
(Pool3DDNNLayer, dict(name='avgpool_1', pool_size=2, stride=2, mode='average_inc_pad')),
(Conv3DDNNLayer, dict(name='conv2_1', num_filters=64, filter_size=(5, 5, 5), pad='same')),
(Pool3DDNNLayer, dict(name='avgpool_2', pool_size=2, stride=2, mode='average_inc_pad')),
(DropoutLayer, dict(name='l2drop', p=0.5)),
(DenseLayer, dict(name='l1', num_units=256)),
(DenseLayer, dict(name='out', num_units=2, nonlinearity=nonlinearities.softmax)),
],
objective_loss_function=objectives.categorical_crossentropy,
update=updates.adam,
update_learning_rate=0.0001,
verbose=10,
max_epochs=50,
train_split=TrainSplit(eval_size=0.25),
custom_scores=[('dsc', lambda p, t: 2 * np.sum(p * t[:, 1]) / np.sum((p + t[:, 1])))],
)
net.load_params_from(net_name)
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Creating the probability map ' + c['b'] + '1' + c['nc'] + c['g'] + '>' + c['nc'])
names = np.array([options['flair'], options['pd'], options['t2'], options['t1']])
image_nii = load_nii(options['flair'])
image1 = np.zeros_like(image_nii.get_data())
print('0% of data tested', end='\r')
sys.stdout.flush()
for batch, centers, percent in load_patch_batch_percent(names, batch_size, patch_size):
y_pred = net.predict_proba(batch)
print('%f%% of data tested' % percent, end='\r')
sys.stdout.flush()
[x, y, z] = np.stack(centers, axis=1)
image1[x, y, z] = y_pred[:, 1]
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Loading the net ' + c['b'] + '2' + c['nc'] + c['g'] + '>' + c['nc'])
net_name = '/usr/local/nets/deep-challenge2016.final.model_weights.pkl' if options['docker'] \
else './deep-challenge2016.final.model_weights.pkl'
net = NeuralNet(
layers=[
(InputLayer, dict(name='in', shape=(None, 4, 15, 15, 15))),
(Conv3DDNNLayer, dict(name='conv1_1', num_filters=32, filter_size=(5, 5, 5), pad='same')),
(Pool3DDNNLayer, dict(name='avgpool_1', pool_size=2, stride=2, mode='average_inc_pad')),
(Conv3DDNNLayer, dict(name='conv2_1', num_filters=64, filter_size=(5, 5, 5), pad='same')),
(Pool3DDNNLayer, dict(name='avgpool_2', pool_size=2, stride=2, mode='average_inc_pad')),
(DropoutLayer, dict(name='l2drop', p=0.5)),
(DenseLayer, dict(name='l1', num_units=256)),
(DenseLayer, dict(name='out', num_units=2, nonlinearity=nonlinearities.softmax)),
],
objective_loss_function=objectives.categorical_crossentropy,
update=updates.adam,
update_learning_rate=0.0001,
batch_iterator_train=BatchIterator(batch_size=4096),
verbose=10,
max_epochs=2000,
train_split=TrainSplit(eval_size=0.25),
custom_scores=[('dsc', lambda t, p: 2 * np.sum(t * p[:, 1]) / np.sum((t + p[:, 1])))],
)
net.load_params_from(net_name)
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Creating the probability map ' + c['b'] + '2' + c['nc'] + c['g'] + '>' + c['nc'])
image2 = np.zeros_like(image_nii.get_data())
print('0% of data tested', end='\r')
sys.stdout.flush()
for batch, centers, percent in load_patch_batch_percent(names, batch_size, patch_size):
y_pred = net.predict_proba(batch)
print('%f%% of data tested' % percent, end='\r')
sys.stdout.flush()
[x, y, z] = np.stack(centers, axis=1)
image2[x, y, z] = y_pred[:, 1]
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Saving to file ' + c['b'] + options['output'] + c['nc'] + c['g'] + '>' + c['nc'])
image = (image1 * image2) > 0.5
# filter candidates < min_size
labels, num_labels = ndimage.label(image)
lesion_list = np.unique(labels)
num_elements_by_lesion = ndimage.labeled_comprehension(image, labels, lesion_list, np.sum, float, 0)
filt_min_size = num_elements_by_lesion >= min_size
lesion_list = lesion_list[filt_min_size]
image = reduce(np.logical_or, map(lambda lab: lab == labels, lesion_list))
image_nii.get_data()[:] = np.roll(np.roll(image, 1, axis=0), 1, axis=1)
path = '/'.join(options['t1'].rsplit('/')[:-1])
outputname = options['output'].rsplit('/')[-1]
image_nii.to_filename(os.path.join(path, outputname))
if not options['docker']:
path = '/'.join(options['output'].rsplit('/')[:-1])
case = options['output'].rsplit('/')[-1]
gt = load_nii(os.path.join(path, 'Consensus.nii.gz')).get_data().astype(dtype=np.bool)
dsc = np.sum(2.0 * np.logical_and(gt, image)) / (np.sum(gt) + np.sum(image))
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<DSC value for ' + c['c'] + case + c['g'] + ' = ' + c['b'] + str(dsc) + c['nc'] + c['g'] + '>' + c['nc'])
def Channel_Head_Definition(skeletonFromFlowAndCurvatureArray,
geodesicDistanceArray):
# Locating end points
print 'Locating skeleton end points'
structure = np.ones((3, 3))
skeletonLabeledArray, skeletonNumConnectedComponentsList =\
ndimage.label(skeletonFromFlowAndCurvatureArray,
structure=structure)
"""
Through the histogram of skeletonNumElementsSortedList
(skeletonNumElementsList minus the maximum value which
corresponds to the largest connected element of the skeleton) we get the
size of the smallest elements of the skeleton, which will likely
correspond to small isolated convergent areas. These elements will be
excluded from the search of end points.
"""
print 'Counting the number of elements of each connected component'
lbls = np.arange(1, skeletonNumConnectedComponentsList + 1)
skeletonLabeledArrayNumtuple = ndimage.labeled_comprehension(skeletonFromFlowAndCurvatureArray,\
skeletonLabeledArray,\
lbls,np.count_nonzero,\
int,0)
skeletonNumElementsSortedList = np.sort(skeletonLabeledArrayNumtuple)
histarray,skeletonNumElementsHistogramX=np.histogram(\
skeletonNumElementsSortedList[0:len(skeletonNumElementsSortedList)-1],
int(np.floor(np.sqrt(len(skeletonNumElementsSortedList)))))
if defaults.doPlot == 1:
raster_plot(skeletonLabeledArray,
'Skeleton Labeled Array elements Array')
# Create skeleton gridded array
skeleton_label_set, label_indices = np.unique(skeletonLabeledArray,
return_inverse=True)
skeletonNumElementsGriddedArray = np.array([
skeletonLabeledArrayNumtuple[x - 1] for x in skeleton_label_set
])[label_indices].reshape(skeletonLabeledArray.shape)
if defaults.doPlot == 1:
raster_plot(skeletonNumElementsGriddedArray,
'Skeleton Num elements Array')
# Elements smaller than skeletonNumElementsThreshold are not considered in the
# skeletonEndPointsList detection
skeletonNumElementsThreshold = skeletonNumElementsHistogramX[2]
print 'skeletonNumElementsThreshold', str(skeletonNumElementsThreshold)
# Scan the array for finding the channel heads
print 'Continuing to locate skeleton endpoints'
skeletonEndPointsList = []
nrows = skeletonFromFlowAndCurvatureArray.shape[0]
ncols = skeletonFromFlowAndCurvatureArray.shape[1]
for i in range(nrows):
for j in range(ncols):
if skeletonLabeledArray[i,j]!=0 \
and skeletonNumElementsGriddedArray[i,j]>=skeletonNumElementsThreshold:
# Define search box and ensure it fits within the DTM bounds
my = i - 1
py = nrows - i
mx = j - 1
px = ncols - j
xMinus = np.min([defaults.endPointSearchBoxSize, mx])
xPlus = np.min([defaults.endPointSearchBoxSize, px])
yMinus = np.min([defaults.endPointSearchBoxSize, my])
yPlus = np.min([defaults.endPointSearchBoxSize, py])
# Extract the geodesic distances geodesicDistanceArray for pixels within the search box
searchGeodesicDistanceBox = geodesicDistanceArray[i -
yMinus:i +
yPlus, j -
xMinus:j +
xPlus]
# Extract the skeleton labels for pixels within the search box
searchLabeledSkeletonBox = skeletonLabeledArray[i - yMinus:i +
yPlus,
j - xMinus:j +
xPlus]
# Look in the search box for skeleton points with the same label
# and greater geodesic distance than the current pixel at (i,j)
# - if there are none, then add the current point as a channel head
v = searchLabeledSkeletonBox == skeletonLabeledArray[i, j]
v1 = v * searchGeodesicDistanceBox > geodesicDistanceArray[i,
j]
v3 = np.where(np.any(v1 == True, axis=0))
if len(v3[0]) == 0:
skeletonEndPointsList.append([i, j])
# For loop ends here
skeletonEndPointsListArray = np.transpose(skeletonEndPointsList)
if defaults.doPlot == 1:
raster_point_plot(skeletonFromFlowAndCurvatureArray,
skeletonEndPointsListArray,
'Skeleton Num elements Array with channel heads',
cm.binary, 'ro')
if defaults.doPlot == 1:
raster_point_plot(geodesicDistanceArray, skeletonEndPointsListArray,
'Geodesic distance Array with channel heads',
cm.coolwarm, 'ro')
xx = skeletonEndPointsListArray[1]
yy = skeletonEndPointsListArray[0]
# Write shapefiles of channel heads
write_drainage_nodes(xx, yy, "ChannelHead", Parameters.pointFileName,
Parameters.pointshapefileName)
# Write raster of channel heads
channelheadArray = np.zeros((geodesicDistanceArray.shape))
channelheadArray[skeletonEndPointsListArray[0],
skeletonEndPointsListArray[1]] = 1
outfilepath = Parameters.geonetResultsDir
demName = Parameters.demFileName
outfilename = demName.split('.')[0] + '_channelHeads.tif'
write_geotif_generic(channelheadArray,\
outfilepath,outfilename)
return xx, yy
def detect_objects(self, img, outline='GREEN'):
original = img.copy()
post_processed = img.copy()
if img.mode != 'L':
img = img.convert('L')
x_step = 1
y_step = 1
scale = 1.25
count = 0
confidence_threshold = 3
reduced_windows = []
original_draw = ImageDraw.Draw(original)
window_width = self.classifier.image_width
window_height = self.classifier.image_height
while window_width < img.size[0] and window_height < img.size[1]:
window_stamps = np.zeros_like(img, np.uint16)
integral_image = get_integral_image(img)
for y in range(0, img.size[1] - window_height, y_step):
for x in range(0, img.size[0] - window_width, x_step):
window = integral_window(integral_image, x, y,
window_width, window_height)
if self.classifier.classify(window):
window_stamps[y, x] = window_width
original_draw.rectangle(
(x * original.size[0] // img.size[0],
y * original.size[1] // img.size[1],
(x + window_width) * original.size[0] //
img.size[0], (y + window_height) *
original.size[1] // img.size[1]),
outline=outline)
def representative_windows(values, indexes):
left = np.inf
right = -1
top = np.inf
bottom = -1
for index in indexes:
x = index % img.size[0]
y = index // img.size[0]
if x < left:
left = x
if x > right:
right = x
if y < top:
top = y
if y > bottom:
bottom = y
confidence = indexes.size # / (right - left + values[0])
if confidence > confidence_threshold:
# return (left, top, right + values[0], bottom + values[0]), confidence
return ((left + right) // 2, (top + bottom) // 2,
(left + right) // 2 + values[0],
(top + bottom) // 2 + values[0]), confidence
else:
return None
labeled, num_of_windows = label(window_stamps,
np.ones((3, 3), np.int))
if num_of_windows != 0:
merged_windows = list(
filter(
lambda w: w is not None,
labeled_comprehension(window_stamps, labeled,
np.arange(1, num_of_windows + 1),
representative_windows, tuple, 0,
True)))
if len(merged_windows) != 0:
reduced_windows += [
(tuple(v * original.size[0] // img.size[0]
for v in x[0]), ) +
((x[0][2] - x[0][0]) * original.size[0] // img.size[0],
x[1]) for x in merged_windows
]
count += 1
img.thumbnail((img.size[0] / scale, img.size[1] / scale),
Image.ANTIALIAS)
original.show()
reduced_windows.sort(key=lambda x: x[1])
def center_of_rectangle(x1, y1, x2, y2):
return (x1 + x2) / 2, (y1 + y2) / 2
i = 0
while i < len(reduced_windows):
i_center_x, i_center_y = center_of_rectangle(
*reduced_windows[i][0])
i_confidence = reduced_windows[i][2]
j = i + 1
while j < len(reduced_windows):
j_x1, j_y1, j_x2, j_y2 = reduced_windows[j][0]
j_confidence = reduced_windows[j][2]
if j_x1 <= i_center_x <= j_x2 and j_y1 <= i_center_y <= j_y2:
if i_confidence > j_confidence:
del reduced_windows[j]
j -= 1
else:
del reduced_windows[i]
i -= 1
j = len(reduced_windows)
j += 1
i += 1
post_processed_draw = ImageDraw.Draw(post_processed)
for rectangle in [x[0] for x in reduced_windows]:
post_processed_draw.rectangle(rectangle, outline=outline)
post_processed.show()
def depth_cal(wet_raster, width_raster, ele_raster, pix_per_m, min_slope, max_slope, min_pos_slope,
wet_pos_depth_raster, water_ele_raster):
corner = arcpy.Point(arcpy.Describe(wet_raster).Extent.XMin, arcpy.Describe(wet_raster).Extent.YMin)
dx = arcpy.Describe(wet_raster).meanCellWidth
org_wet = arcpy.RasterToNumPyArray(wet_raster, nodata_to_value=0)
wet = ndimage.morphology.binary_fill_holes(org_wet).astype(np.uint8)
ele = arcpy.RasterToNumPyArray(ele_raster, nodata_to_value=0)
common_border = find_sep_border(org_wet, 7, corner, dx)
wet = ndimage.binary_dilation(org_wet, iterations=2).astype(np.uint8)
wet = np.where(common_border == 1, 0, wet)
common_border = find_sep_border(wet, 7, corner, dx)
wet = ndimage.morphology.binary_fill_holes(wet).astype(np.uint8)
wet = np.where(common_border == 1, 0, wet)
wet = np.where(org_wet == 1, 1, wet)
er_wet_1 = ndimage.binary_erosion(wet).astype(np.uint8)
bound_1 = np.where((wet == 1) & (er_wet_1 == 0), 1, 0).astype(np.uint8)
ele_1 = np.where(bound_1 == 1, ele, 0)
temp_ele_1 = arcpy.NumPyArrayToRaster(ele_1, corner, dx, dx, value_to_nodata=0)
ele_1_thick = FocalStatistics(temp_ele_1, NbrRectangle(3, 3, "CELL"), "MEAN", "DATA")
del ele_1
er_wet_2 = ndimage.binary_erosion(er_wet_1).astype(np.uint8)
bound_2 = np.where((er_wet_1 == 1) & (er_wet_2 == 0), 1, 0).astype(np.uint8)
ele_2 = np.where(bound_2 == 1, ele, 0)
ele_1 = arcpy.RasterToNumPyArray(ele_1_thick, nodata_to_value=0)
ele_1 = np.where(bound_2 == 0, 0, ele_1)
slope = (ele_1 - ele_2) / pix_per_m
slope_sign = np.where(slope > 0, 1, 0)
Lab_wet, num_label = ndimage.label(wet, structure=np.ones((3, 3)))
labels = np.arange(1, num_label + 1)
slope_label = ndimage.labeled_comprehension(slope, Lab_wet, labels, np.sum, float, 0)
slope_sign_label = ndimage.labeled_comprehension(slope_sign, Lab_wet, labels, np.sum, float, 0)
count_2 = ndimage.labeled_comprehension(bound_2, Lab_wet, labels, np.sum, float, 0)
water_ele_label = ndimage.labeled_comprehension(ele, Lab_wet, labels, np.min, float, 0)
slope_seg = np.zeros_like(org_wet).astype(np.float32)
slope_seg = slope_label[Lab_wet - 1].astype(np.float32) / count_2[Lab_wet - 1].astype(np.float32)
slope_seg = np.where((Lab_wet == 0) | (org_wet == 0), 0, slope_seg)
new_wet = np.where((slope_seg > min_slope) & (slope_seg < max_slope), 1, 0).astype(np.uint8)
width = arcpy.RasterToNumPyArray(width_raster, nodata_to_value=0)
depth = slope_seg * width / 2
del slope_seg, slope_label, width
slope_sign_seg = np.zeros_like(org_wet).astype(np.float32)
slope_sign_seg = slope_sign_label[Lab_wet - 1].astype(np.float32) / count_2[Lab_wet - 1].astype(np.float32)
slope_sign_seg = np.where((Lab_wet == 0) | (org_wet == 0), 0, slope_sign_seg)
new_wet = np.where((slope_sign_seg <= min_pos_slope), 0, new_wet).astype(np.uint8)
del slope_sign_seg, slope_sign_label, count_2
arcpy.NumPyArrayToRaster(new_wet, corner, dx, dx, value_to_nodata=0).save(wet_pos_depth_raster)
water_ele = water_ele_label[Lab_wet - 1].astype(np.float32) + depth
water_ele = np.where((Lab_wet == 0) | (new_wet == 0), 0, water_ele)
arcpy.NumPyArrayToRaster(water_ele, corner, dx, dx, value_to_nodata=0).save(water_ele_raster)
del new_wet, water_ele, depth
def significant_features(
radar, fields, gatefilter=None, size_bins=100, size_limits=(0, 400),
structure=None, save_size_field=False, fill_value=None, debug=False,
verbose=False):
"""
Determine significant radar echo features on a sweep by sweep basis by
computing the size each echo feature. Here an echo feature is defined as
multiple connected radar gates with valid data, where the connection
structure is defined by the user.
Parameters
----------
radar : Radar
Py-ART Radar containing
fields : str or list or tuple
Radar fields to be used to identify significant echo featues.
gatefilter : GateFilter
Py-ART GateFilter instance.
size_bins : int, optional
Number of size bins used to bin feature size distribution.
size_limits : list or tuple, optional
Lower and upper limits of the feature size distribution. This together
with size_bins defines the bin width of the feature size distribution.
structure : array_like, optional
Binary structuring element used to define connected radar gates. The
default defines a structuring element in which diagonal radar gates are
not considered connected.
save_size_field : bool, optional
True to save size fields in the radar object, False to discard.
debug : bool, optional
True to print debugging information, False to suppress.
Returns
-------
gf : GateFilter
Py-ART GateFilter.
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse radar fields
if isinstance(fields, str):
fields = [fields]
# Parse gate filter
if gatefilter is None:
gf = GateFilter(radar, exclude_based=False)
# Parse binary structuring element
if structure is None:
structure = ndimage.generate_binary_structure(2, 1)
for field in fields:
if verbose:
print 'Processing echo features: {}'.format(field)
# Initialize echo feature size array
size_data = np.zeros_like(
radar.fields[field]['data'], subok=False, dtype=np.int32)
feature_sizes = []
for sweep, _slice in enumerate(radar.iter_slice()):
# Parse radar sweep data and define only valid gates
data = radar.get_field(sweep, field, copy=False)
is_valid_gate = ~np.ma.getmaskarray(data)
# Label the connected features in the radar sweep data and create
# index array which defines each unique label (feature)
labels, nlabels = ndimage.label(
is_valid_gate, structure=structure, output=None)
index = np.arange(1, nlabels + 1, 1)
if debug:
print 'Unique features in sweep {}: {}'.format(sweep, nlabels)
# Compute the size (in radar gates) of each echo feature
# Check for case where no echo features are found, e.g., no data in
# sweep
if nlabels > 0:
sweep_sizes = ndimage.labeled_comprehension(
is_valid_gate, labels, index, np.count_nonzero,
np.int32, 0)
feature_sizes.append(sweep_sizes)
# Set each label (feature) to its total size (in radar gates)
for label, size in zip(index, sweep_sizes):
size_data[_slice][labels == label] = size
# Stack sweep echo feature sizes
feature_sizes = np.hstack(feature_sizes)
# Bin and compute feature size occurrences
counts, bin_edges = np.histogram(
feature_sizes, bins=size_bins, range=size_limits, normed=False,
weights=None, density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {} gate(s)'.format(bin_width)
# Compute the peak of the echo feature size distribution. We expect the
# peak of the echo feature size distribution to be close to 1 radar
# gate
peak_size = bin_centers[counts.argmax()] - bin_width / 2.0
if debug:
print 'Feature size at peak: {} gate(s)'.format(peak_size)
# Determine the first instance when the count (sample size) for an echo
# feature size bin reaches 0 after the distribution peak. This will
# define the minimum echo feature size
is_zero_size = np.logical_and(
bin_centers > peak_size, np.isclose(counts, 0, atol=1.0e-1))
min_size = bin_centers[is_zero_size].min() - bin_width / 2.0
if debug:
_range = [0.0, min_size]
print 'Insignificant feature size range: {} gates'.format(_range)
# Mask invalid feature sizes, e.g., zero-size features
size_data = np.ma.masked_equal(size_data, 0, copy=False)
size_data.set_fill_value(fill_value)
# Parse echo feature size field name
size_field = '{}_feature_size'.format(field)
# Add echo feature size field to radar
size_dict = {
'data': size_data.astype(np.int32),
'long_name': 'Echo feature size in number of radar gates',
'_FillValue': size_data.fill_value,
'units': 'unitless',
'comment': None,
}
radar.add_field(size_field, size_dict, replace_existing=True)
# Update gate filter
gf.include_above(size_field, min_size, op='and', inclusive=False)
# Remove eacho feature size field if specified
if not save_size_field:
radar.fields.pop(size_field, None)
return gf
def find_star(self, raw, ref=False, count=8, showLBL=False, name=''):
'''
找星程序
首先对图片进行中值滤波 获取滤波图img_med
将滤波图中 滤波图 < [滤波图中值 + N * 原图标准差] 的位置定义为背景
余下连通区进行标记 根据大小进行排序 最大的count个连通区定义为星
随后记录星的几何半径与流量中心 并返回
para:
img: ndarr 图片
ref: bool True:返回pandas.DataFrame对象 False:返回dict对象
count: int 找多少颗星 默认:0 假如是0时, 则取对象中的count参数
return:
dic: dict/df 返回的找星数据 有两个关键词
radius: 几何半径
centers: 流量中心
'''
if ref:
# 叠加图背景去除
img = raw - nd.median_filter(raw, footprint=np.array(self.fpa))
else:
# 信号增强
raw = nd.median_filter(raw, 5)
img = nd.convolve(raw, self.kernel)
# 计算背景标准差
sky, std = bginfo(img, self.mask)
# 标记背景
mark = img > sky + self.N * std
# # 清除跨蒙版星
# lbl, _ = nd.measurements.label(mark | ~self.mask)
# mark = mark & (lbl != 1)
# 计算连通区
lbl, num = nd.measurements.label(mark)
idx = np.arange(num) + 1
# 根据连通区计算半径
r_arr = nd.labeled_comprehension(
input=lbl,
labels=lbl,
index=idx,
func=lambda x: np.sqrt(len(x) / np.pi),
out_dtype=float,
default=0)
# 第一次筛选连通区
# jud = r_arr > 9
# idx = idx[jud]
# r_arr = r_arr[jud]
if ref:
jud = r_arr > 3
idx = idx[jud]
r_arr = r_arr[jud]
# 根据R排序
jud = np.argsort(-r_arr)
idx = idx[jud]
r_arr = r_arr[jud]
if len(idx) > count:
idx = idx[:count]
r_arr = r_arr[:count]
else:
# jud = r_arr > 6
# idx = idx[jud]
# r_arr = r_arr[jud]
# 获取SNR列表
F = nd.sum(img, lbl, idx) # OR MAX
# # 第二次筛选连通区
# jud = Fmax > np.pi * 9**2 * 0
# idx = idx[jud]
# r_arr = r_arr[jud]
# Fmax = Fmax[jud]
# 根据SNR最大值排序
jud = np.argsort(-F)
idx = idx[jud]
r_arr = r_arr[jud]
F = F[jud]
# 第三次筛选连通区
if len(idx) > 8:
idx = idx[:8]
r_arr = r_arr[:8]
F = F[:8]
if showLBL:
save = np.zeros_like(lbl).astype(int)
for i in idx:
save += (lbl == i) * 1
if not os.path.exists('local/lbl/'):
os.mkdir('local/lbl/')
plt.imsave('local/lbl/' + name + '_lbl.png', save)
plt.close()
# 计算质心
centers = nd.measurements.center_of_mass(input=raw,
labels=lbl,
index=idx)
centers = [complex(*center) for center in centers]
if ref:
return pd.DataFrame({
'radius': pd.Series(r_arr),
'centers': pd.Series(centers)
})
return {'radius': pd.Series(r_arr), 'centers': pd.Series(centers)}
def label_size(radar,
field,
structure=None,
rays_wrap_around=False,
size_field=None,
debug=False,
verbose=False):
"""
Label connected pixels in a binary radar field and compute the size of each
label. Unexpected results may occur if the specified field is not binary.
Parameters
----------
radar : Radar
Py-ART Radar containing the specified field to be labeled.
field : str
Radar field to be labeled.
structure : array_like, optional
Binary structuring element used in labeling. The default structuring
element has a squared connectivity equal to 1, i.e., only nearest
neighbours are connected to the structure origin and
diagonally-connected elements are not considered neighbours.
rays_wrap_around : bool, optional
True if all sweeps have contiguous rays, e.g., 360 deg PPI radar
volumes.
debug : bool, optional
True to print debugging information, False to suppress.
verbose : bool, optional
True to print relevant information, False to suppress.
"""
if verbose:
print 'Performing binary labeling: {}'.format(field)
if size_field is None:
size_field = '{}_feature_size'.format(field)
size_dict = {
'data': np.zeros_like(radar.fields[field]['data'], dtype=np.int32),
'units': 'unitless',
'valid_min': 0,
'comment': 'size in pixels of connected features',
}
radar.add_field(size_field, size_dict, replace_existing=True)
for sweep, slc in enumerate(radar.iter_slice()):
# Parse radar sweep data
data = np.ma.getdata(radar.get_field(sweep, field))
# Label connected pixels defined by the structuring element
labels, nlabels = ndimage.label(data, structure=structure)
index = np.arange(1, nlabels + 1)
if debug:
print 'Unique features in sweep {}: {}'.format(sweep, nlabels)
if nlabels > 0:
# Compute the size in number of pixels of each labeled feature
sizes = ndimage.labeled_comprehension(data, labels, index,
np.count_nonzero, np.int32,
0)
# Set each labeled feature to its total size in radar gates
for label, size in zip(index, sizes):
radar.fields[size_field]['data'][slc][labels == label] = size
return
def ehabitat(ecor, nw, nwpathout):
global nwpath
if nw == '':
nwpath = os.getcwd()
else:
nwpath = nw
if gmaps == 0:
initglobalmaps()
if nwpathout == '':
#outdir = 'results' # ToDo: locally create folder "results" if it does not exist!
outdir = os.path.join(os.path.sep, os.getcwd(), 'results')
safelyMakeDir(outdir)
else:
#outdir = nwpathout+'/results' # SHARED FOLDER PATH
outdir = os.path.join(os.path.sep, nwpathout, 'results')
safelyMakeDir(outdir)
treepamin = treepamax = eprpamin = eprpamax = prepamin = prepamax = biopamin = biopamax = slopepamin = slopepamax = ndwipamin = ndwipamax = ndvimaxpamin = ndvimaxpamax = ndviminpamin = ndviminpamax = hpamin = hpamax = None
s = nd.generate_binary_structure(
2, 2) # most restrictive pattern for the landscape patches
# LOCAL FOLDER
csvname1 = os.path.join(os.path.sep, outdir, 'ecoregs_done.csv')
print csvname1
if os.path.isfile(csvname1) == False:
wb = open(csvname1, 'a')
wb.write('None')
wb.write('\n')
wb.close()
# LOCAL FOLDER
csvname = os.path.join(os.path.sep, outdir, 'hri_results.csv')
print csvname
if os.path.isfile(csvname) == False:
wb = open(csvname, 'a')
wb.write(
'ecoregion wdpaid averpasim hr2aver pxpa hr1insumaver hriaver nfeatsaver lpratio lpratio2 numpszok lpmaxsize aggregation treepamin treepamax eprpamin eprpamax prepamin prepamax biopamin biopamax slopepamin slopepamax ndwipamin ndwipamax ndvimaxpamin ndvimaxpamax ndviminpamin ndviminpamax hpamin hpamax treepamean eprpamean prepamean biopamean slopepamean ndwipamean ndvimaxpamean ndviminpamean hpamean'
)
wb.write('\n')
wb.close()
treepamean = eprpamean = prepamean = biopamean = slopepamean = ndwipamean = ndvimaxpamean = ndviminpamean = hpamean = None
ef = 'eco_' + str(ecor) + '.tif'
ecofile = os.path.join(os.path.sep, nwpath, 'ecoregs', ef)
#ecofile = os.path.join(os.path.sep, nwpath, os.path.sep,'ecoregs', os.path.sep, ef)
print ecofile
avail = os.path.isfile(ecofile)
if avail == True:
eco_csv = str(ecor) + '.csv'
print eco_csv
ecoparksf = os.path.join(os.path.sep, nwpath, 'pas', eco_csv)
#ecoparksf = os.path.join(os.path.sep, nwpath, os.path.sep, 'pas', os.path.sep, eco_csv)
print ecoparksf
#ecoparksf = nwpath+'/pas/'+str(ecor)+'.csv'
src_ds_eco = gdal.Open(ecofile)
eco = src_ds_eco.GetRasterBand(1)
eco_mask0 = eco.ReadAsArray(0, 0, eco.XSize,
eco.YSize).astype(np.int32)
eco_mask = eco_mask0.flatten()
gt_eco = src_ds_eco.GetGeoTransform()
print 'eco mask'
xoff = int((gt_eco[0] - gt_epr_global[0]) / 1000)
yoff = int((gt_epr_global[3] - gt_eco[3]) / 1000)
epr_eco_bb0 = epr_global.ReadAsArray(xoff, yoff, eco.XSize,
eco.YSize).astype(np.float32)
epr_eco_bb = epr_eco_bb0.flatten()
epr_eco0 = np.where(eco_mask == 1, (epr_eco_bb), (0))
epr_eco = np.where(epr_eco0 == 65535.0, (float('NaN')), (epr_eco0))
maskepr = np.isnan(epr_eco)
epr_eco[maskepr] = np.interp(np.flatnonzero(maskepr),
np.flatnonzero(~maskepr),
epr_eco[~maskepr])
print 'eco epr'
xoff = int((gt_eco[0] - gt_slope_global[0]) / 1000)
yoff = int((gt_slope_global[3] - gt_eco[3]) / 1000)
slope_eco_bb0 = slope_global.ReadAsArray(xoff, yoff, eco.XSize,
eco.YSize).astype(np.float32)
slope_eco_bb = slope_eco_bb0.flatten()
slope_eco0 = np.where(eco_mask == 1, (slope_eco_bb), (0))
slope_eco = np.where(slope_eco0 == 65535.0, (float('NaN')),
(slope_eco0))
maskslope = np.isnan(slope_eco)
slope_eco[maskslope] = np.interp(np.flatnonzero(maskslope),
np.flatnonzero(~maskslope),
slope_eco[~maskslope])
print 'eco slope'
xoff = int((gt_eco[0] - gt_ndvimax_global[0]) / 1000)
yoff = int((gt_ndvimax_global[3] - gt_eco[3]) / 1000)
ndvimax_eco_bb0 = ndvimax_global.ReadAsArray(
xoff, yoff, eco.XSize, eco.YSize).astype(np.float32)
ndvimax_eco_bb = ndvimax_eco_bb0.flatten()
ndvimax_eco0 = np.where(eco_mask == 1, (ndvimax_eco_bb), (0))
ndvimax_eco = np.where(ndvimax_eco0 == 65535.0, (float('NaN')),
(ndvimax_eco0))
maskndvimax = np.isnan(ndvimax_eco)
ndvimax_eco[maskndvimax] = np.interp(np.flatnonzero(maskndvimax),
np.flatnonzero(~maskndvimax),
ndvimax_eco[~maskndvimax])
print 'eco ndvimax'
xoff = int((gt_eco[0] - gt_ndvimin_global[0]) / 1000)
yoff = int((gt_ndvimin_global[3] - gt_eco[3]) / 1000)
ndvimin_eco_bb0 = ndvimin_global.ReadAsArray(
xoff, yoff, eco.XSize, eco.YSize).astype(np.float32)
ndvimin_eco_bb = ndvimin_eco_bb0.flatten()
ndvimin_eco0 = np.where(eco_mask == 1, (ndvimin_eco_bb), (0))
ndvimin_eco = np.where(ndvimin_eco0 == 65535.0, (float('NaN')),
(ndvimin_eco0))
maskndvimin = np.isnan(ndvimin_eco)
ndvimin_eco[maskndvimin] = np.interp(np.flatnonzero(maskndvimin),
np.flatnonzero(~maskndvimin),
ndvimin_eco[~maskndvimin])
print 'eco ndvimin'
xoff = int((gt_eco[0] - gt_ndwi_global[0]) / 1000)
yoff = int((gt_ndwi_global[3] - gt_eco[3]) / 1000)
ndwi_eco_bb0 = ndwi_global.ReadAsArray(xoff, yoff, eco.XSize,
eco.YSize).astype(np.float32)
ndwi_eco_bb = ndwi_eco_bb0.flatten()
ndwi_eco0 = np.where(eco_mask == 1, (ndwi_eco_bb), (0))
ndwi_eco = np.where(ndwi_eco0 == 255.0, (float('NaN')), (ndwi_eco0))
maskndwi = np.isnan(ndwi_eco)
ndwi_eco[maskndwi] = np.interp(np.flatnonzero(maskndwi),
np.flatnonzero(~maskndwi),
ndwi_eco[~maskndwi])
print 'eco ndwi'
xoff = int((gt_eco[0] - gt_pre_global[0]) / 1000)
yoff = int((gt_pre_global[3] - gt_eco[3]) / 1000)
pre_eco_bb0 = pre_global.ReadAsArray(xoff, yoff, eco.XSize,
eco.YSize).astype(np.float32)
pre_eco_bb = pre_eco_bb0.flatten()
pre_eco0 = np.where(eco_mask == 1, (pre_eco_bb), (0))
pre_eco = np.where(pre_eco0 == 65535.0, (float('NaN')), (pre_eco0))
maskpre = np.isnan(pre_eco)
pre_eco[maskpre] = np.interp(np.flatnonzero(maskpre),
np.flatnonzero(~maskpre),
pre_eco[~maskpre])
print 'eco pre'
xoff = int((gt_eco[0] - gt_bio_global[0]) / 1000)
yoff = int((gt_bio_global[3] - gt_eco[3]) / 1000)
bio_eco_bb0 = bio_global.ReadAsArray(xoff, yoff, eco.XSize,
eco.YSize).astype(np.float32)
bio_eco_bb = bio_eco_bb0.flatten()
bio_eco0 = np.where(eco_mask == 1, (bio_eco_bb), (0))
bio_eco = np.where(bio_eco0 == 65535.0, (float('NaN')), (bio_eco0))
maskbio = np.isnan(bio_eco)
bio_eco[maskbio] = np.interp(np.flatnonzero(maskbio),
np.flatnonzero(~maskbio),
bio_eco[~maskbio])
print 'eco bio'
xoff = int((gt_eco[0] - gt_tree_global[0]) / 1000)
yoff = int((gt_tree_global[3] - gt_eco[3]) / 1000)
tree_eco_bb0 = tree_global.ReadAsArray(xoff, yoff, eco.XSize,
eco.YSize).astype(np.float32)
tree_eco_bb = tree_eco_bb0.flatten()
tree_eco0 = np.where(eco_mask == 1, (tree_eco_bb), (0))
tree_eco = np.where(tree_eco0 == 255.0, (float('NaN')), (tree_eco0))
masktree = np.isnan(tree_eco)
tree_eco[masktree] = np.interp(np.flatnonzero(masktree),
np.flatnonzero(~masktree),
tree_eco[~masktree])
print 'eco tree'
xoff = int((gt_eco[0] - gt_herb_global[0]) / 1000)
yoff = int((gt_herb_global[3] - gt_eco[3]) / 1000)
herb_eco_bb0 = herb_global.ReadAsArray(xoff, yoff, eco.XSize,
eco.YSize).astype(np.float32)
herb_eco_bb = herb_eco_bb0.flatten()
herb_eco0 = np.where(eco_mask == 1, (herb_eco_bb), (0))
herb_eco = np.where(herb_eco0 == 255.0, (float('NaN')), (herb_eco0))
maskherb = np.isnan(herb_eco)
herb_eco[maskherb] = np.interp(np.flatnonzero(maskherb),
np.flatnonzero(~maskherb),
herb_eco[~maskherb])
print 'eco herb'
ind_eco0 = np.column_stack(
(bio_eco, pre_eco, epr_eco, herb_eco, ndvimax_eco, ndvimin_eco,
ndwi_eco, slope_eco, tree_eco))
print 'ecovars stacked'
print ecoparksf
pa_list0 = np.genfromtxt(
ecoparksf, dtype='string') # crear este archivo en subpas!
pa_list = np.unique(pa_list0)
n = len(pa_list)
for px in range(0, n): # 0,n
pa = pa_list[px]
print pa
outfile = os.path.join(os.path.sep, outdir,
str(ecor) + '_' + str(pa) + '.tif')
outfile2 = os.path.join(os.path.sep, outdir,
str(ecor) + '_' + str(pa) + '_lp.tif')
outfile3 = os.path.join(os.path.sep, outdir,
str(ecor) + '_' + str(pa) + '_mask.tif')
#outfile = outdir+'/'+str(ecor)+'_'+str(pa)+'.tif' # LOCAL FOLDER
pa_infile = 'pa_' + str(pa) + '.tif'
pa4 = os.path.join(os.path.sep, nwpath, 'pas', pa_infile)
#pa4 = os.path.join(os.path.sep, nwpath, os.path.sep, 'pas', os.path.sep, pa_infile)
print pa4
#pa4 = nwpath+'/pas/pa_'+str(pa)+'.tif'
dropcols = np.arange(9, dtype=int)
done = os.path.isfile(outfile)
avail2 = os.path.isfile(pa4)
if done == False and avail2 == True:
pafile = pa4
src_ds_pa = gdal.Open(pafile)
par = src_ds_pa.GetRasterBand(1)
pa_mask0 = par.ReadAsArray(0, 0, par.XSize,
par.YSize).astype(np.int32)
pa_mask = pa_mask0.flatten()
ind = pa_mask > 0 #==int(pa)
go = 1
sum_pa_mask = sum(pa_mask[ind]) #/int(pa)
if sum_pa_mask < 3:
go = 0 # not processing areas smaller than 3 pixels
print sum_pa_mask
sum_pa_mask_inv = len(pa_mask[pa_mask == 0])
print sum_pa_mask_inv
print len(pa_mask)
ratiogeom = 10000
if sum_pa_mask > 0: ratiogeom = sum_pa_mask_inv / sum_pa_mask
#print ratiogeom
gt_pa = src_ds_pa.GetGeoTransform()
xoff = int((gt_pa[0] - gt_pre_global[0]) / 1000)
yoff = int((gt_pre_global[3] - gt_pa[3]) / 1000)
if xoff > 0 and yoff > 0 and go == 1:
num_bands = src_ds_eco.RasterCount
driver = gdal.GetDriverByName("GTiff")
dst_options = ['COMPRESS=LZW']
dst_ds = driver.Create(outfile, src_ds_eco.RasterXSize,
src_ds_eco.RasterYSize, num_bands,
gdal.GDT_Float32, dst_options)
dst_ds.SetGeoTransform(src_ds_eco.GetGeoTransform())
dst_ds.SetProjection(src_ds_eco.GetProjectionRef())
xoff = int((gt_pa[0] - gt_tree_global[0]) / 1000)
yoff = int((gt_tree_global[3] - gt_pa[3]) / 1000)
tree_pa_bb0 = tree_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
tree_pa_bb = tree_pa_bb0.flatten()
tree_pa0 = tree_pa_bb[ind]
tree_pa = np.where(tree_pa0 == 255.0, (float('NaN')),
(tree_pa0))
mask2tree = np.isnan(tree_pa)
if mask2tree.all() == True:
dropcols[8] = -8
else:
tree_pa[mask2tree] = np.interp(
np.flatnonzero(mask2tree),
np.flatnonzero(~mask2tree), tree_pa[~mask2tree])
tree_pa = np.random.random_sample(
len(tree_pa), ) / 1000 + tree_pa
print 'pa tree'
treepamin = round(tree_pa.min(), 2)
treepamax = round(tree_pa.max(), 2)
treepamean = round(np.mean(tree_pa), 2)
print treepamin
print treepamax
treediff = abs(tree_pa.min() - tree_pa.max())
if treediff < 0.001: dropcols[8] = -8
xoff = int((gt_pa[0] - gt_epr_global[0]) / 1000)
yoff = int((gt_epr_global[3] - gt_pa[3]) / 1000)
epr_pa_bb0 = epr_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
epr_pa_bb = epr_pa_bb0.flatten()
epr_pa0 = epr_pa_bb[ind]
epr_pa = np.where(epr_pa0 == 65535.0, (float('NaN')),
(epr_pa0))
mask2epr = np.isnan(epr_pa)
if mask2epr.all() == True:
dropcols[2] = -2
else:
epr_pa[mask2epr] = np.interp(np.flatnonzero(mask2epr),
np.flatnonzero(~mask2epr),
epr_pa[~mask2epr])
epr_pa = np.random.random_sample(
len(epr_pa), ) / 1000 + epr_pa
print 'pa epr'
eprpamin = round(epr_pa.min(), 2)
eprpamax = round(epr_pa.max(), 2)
eprpamean = round(np.mean(epr_pa), 2)
print eprpamin
print eprpamax
eprdiff = abs(epr_pa.min() - epr_pa.max())
if eprdiff < 0.001: dropcols[2] = -2
xoff = int((gt_pa[0] - gt_pre_global[0]) / 1000)
yoff = int((gt_pre_global[3] - gt_pa[3]) / 1000)
pre_pa_bb0 = pre_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
pre_pa_bb = pre_pa_bb0.flatten()
pre_pa0 = pre_pa_bb[ind]
pre_pa = np.where(pre_pa0 == 65535.0, (float('NaN')),
(pre_pa0))
mask2pre = np.isnan(pre_pa)
if mask2pre.all() == True:
dropcols[1] = -1
else:
pre_pa[mask2pre] = np.interp(np.flatnonzero(mask2pre),
np.flatnonzero(~mask2pre),
pre_pa[~mask2pre])
pre_pa = np.random.random_sample(
len(pre_pa), ) / 1000 + pre_pa
print 'pa pre'
prepamin = round(pre_pa.min(), 2)
prepamax = round(pre_pa.max(), 2)
prepamean = round(np.mean(pre_pa), 2)
print prepamin
print prepamax
prediff = abs(pre_pa.min() - pre_pa.max())
if prediff < 0.001: dropcols[1] = -1
xoff = int((gt_pa[0] - gt_bio_global[0]) / 1000)
yoff = int((gt_bio_global[3] - gt_pa[3]) / 1000)
bio_pa_bb0 = bio_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
bio_pa_bb = bio_pa_bb0.flatten()
bio_pa0 = bio_pa_bb[ind]
bio_pa = np.where(bio_pa0 == 65535.0, (float('NaN')),
(bio_pa0))
mask2bio = np.isnan(bio_pa)
if mask2bio.all() == True:
dropcols[0] = -0
else:
bio_pa[mask2bio] = np.interp(np.flatnonzero(mask2bio),
np.flatnonzero(~mask2bio),
bio_pa[~mask2bio])
bio_pa = np.random.random_sample(
len(bio_pa), ) / 1000 + bio_pa
print 'pa bio'
biopamin = round(bio_pa.min(), 2)
biopamax = round(bio_pa.max(), 2)
biopamean = round(np.mean(bio_pa), 2)
print biopamin
print biopamax
biodiff = abs(bio_pa.min() - bio_pa.max())
if biodiff < 0.001: dropcols[0] = -0
xoff = int((gt_pa[0] - gt_slope_global[0]) / 1000)
yoff = int((gt_slope_global[3] - gt_pa[3]) / 1000)
slope_pa_bb0 = slope_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
slope_pa_bb = slope_pa_bb0.flatten()
slope_pa0 = slope_pa_bb[ind]
slope_pa = np.where(slope_pa0 == 65535.0, (float('NaN')),
(slope_pa0))
mask2slope = np.isnan(slope_pa)
if mask2slope.all() == True:
dropcols[7] = -7
else:
slope_pa[mask2slope] = np.interp(
np.flatnonzero(mask2slope),
np.flatnonzero(~mask2slope), slope_pa[~mask2slope])
slope_pa = np.random.random_sample(
len(slope_pa), ) / 1000 + slope_pa
print 'pa slope'
slopepamin = round(slope_pa.min(), 2)
slopepamax = round(slope_pa.max(), 2)
slopepamean = round(np.mean(slope_pa), 2)
print slopepamin
print slopepamax
slopediff = abs(slope_pa.min() - slope_pa.max())
if slopediff < 0.001: dropcols[7] = -7
xoff = int((gt_pa[0] - gt_ndwi_global[0]) / 1000)
yoff = int((gt_ndwi_global[3] - gt_pa[3]) / 1000)
ndwi_pa_bb0 = ndwi_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
ndwi_pa_bb = ndwi_pa_bb0.flatten()
ndwi_pa0 = ndwi_pa_bb[ind]
ndwi_pa = np.where(ndwi_pa0 == 255.0, (float('NaN')),
(ndwi_pa0))
mask2ndwi = np.isnan(ndwi_pa)
if mask2ndwi.all() == True:
dropcols[6] = -6
else:
ndwi_pa[mask2ndwi] = np.interp(
np.flatnonzero(mask2ndwi),
np.flatnonzero(~mask2ndwi), ndwi_pa[~mask2ndwi])
ndwi_pa = np.random.random_sample(
len(ndwi_pa), ) / 1000 + ndwi_pa
print 'pa ndwi'
ndwipamin = round(ndwi_pa.min(), 2)
ndwipamax = round(ndwi_pa.max(), 2)
ndwipamean = round(np.mean(ndwi_pa), 2)
print ndwipamin
print ndwipamax
ndwidiff = abs(ndwi_pa.min() - ndwi_pa.max())
if ndwidiff < 0.001: dropcols[6] = -6
xoff = int((gt_pa[0] - gt_ndvimax_global[0]) / 1000)
yoff = int((gt_ndvimax_global[3] - gt_pa[3]) / 1000)
ndvimax_pa_bb0 = ndvimax_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
ndvimax_pa_bb = ndvimax_pa_bb0.flatten()
ndvimax_pa0 = ndvimax_pa_bb[ind]
ndvimax_pa = np.where(ndvimax_pa0 == 65535.0,
(float('NaN')), (ndvimax_pa0))
mask2ndvimax = np.isnan(ndvimax_pa)
if mask2ndvimax.all() == True:
dropcols[4] = -4
else:
ndvimax_pa[mask2ndvimax] = np.interp(
np.flatnonzero(mask2ndvimax),
np.flatnonzero(~mask2ndvimax),
ndvimax_pa[~mask2ndvimax])
ndvimax_pa = np.random.random_sample(
len(ndvimax_pa), ) / 1000 + ndvimax_pa
print 'pa ndvimax'
ndvimaxpamin = round(ndvimax_pa.min(), 2)
ndvimaxpamax = round(ndvimax_pa.max(), 2)
ndvimaxpamean = round(np.mean(ndvimax_pa), 2)
print ndvimaxpamin
print ndvimaxpamax
ndvimaxdiff = abs(ndvimax_pa.min() - ndvimax_pa.max())
if ndvimaxdiff < 0.001: dropcols[4] = -4
xoff = int((gt_pa[0] - gt_ndvimin_global[0]) / 1000)
yoff = int((gt_ndvimin_global[3] - gt_pa[3]) / 1000)
ndvimin_pa_bb0 = ndvimin_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
ndvimin_pa_bb = ndvimin_pa_bb0.flatten()
ndvimin_pa0 = ndvimin_pa_bb[ind]
ndvimin_pa = np.where(ndvimin_pa0 == 65535.0,
(float('NaN')), (ndvimin_pa0))
mask2ndvimin = np.isnan(ndvimin_pa)
if mask2ndvimin.all() == True:
dropcols[5] = -5
else:
ndvimin_pa[mask2ndvimin] = np.interp(
np.flatnonzero(mask2ndvimin),
np.flatnonzero(~mask2ndvimin),
ndvimin_pa[~mask2ndvimin])
ndvimin_pa = np.random.random_sample(
len(ndvimin_pa), ) / 1000 + ndvimin_pa
print 'pa ndvimin'
ndviminpamin = round(ndvimin_pa.min(), 2)
ndviminpamax = round(ndvimin_pa.max(), 2)
ndviminpamean = round(np.mean(ndvimin_pa), 2)
print ndviminpamin
print ndviminpamax
ndvimindiff = abs(ndvimin_pa.min() - ndvimin_pa.max())
if ndvimindiff < 0.001: dropcols[5] = -5
xoff = int((gt_pa[0] - gt_herb_global[0]) / 1000)
yoff = int((gt_herb_global[3] - gt_pa[3]) / 1000)
herb_pa_bb0 = herb_global.ReadAsArray(
xoff, yoff, par.XSize, par.YSize).astype(np.float32)
herb_pa_bb = herb_pa_bb0.flatten()
herb_pa0 = herb_pa_bb[ind]
herb_pa = np.where(herb_pa0 == 255.0, (float('NaN')),
(herb_pa0))
mask2herb = np.isnan(herb_pa)
if mask2herb.all() == True:
dropcols[3] = -3
else:
herb_pa[mask2herb] = np.interp(
np.flatnonzero(mask2herb),
np.flatnonzero(~mask2herb), herb_pa[~mask2herb])
herb_pa = np.random.random_sample(
len(herb_pa), ) / 1000 + herb_pa
print 'pa herb'
hpamin = round(herb_pa.min(), 2)
hpamax = round(herb_pa.max(), 2)
hpamean = round(np.mean(herb_pa), 2)
print hpamin
print hpamax
hdiff = abs(herb_pa.min() - herb_pa.max())
if hdiff < 0.001: dropcols[3] = -3
cols = dropcols[dropcols >= 0]
ind_pa0 = np.column_stack(
(bio_pa, pre_pa, epr_pa, herb_pa, ndvimax_pa,
ndvimin_pa, ndwi_pa, slope_pa, tree_pa))
ind_pa = ind_pa0[:, cols]
ind_eco = ind_eco0[:, cols]
print ind_pa.shape
hr1sum = hr1insum = indokpsz = pszok = sumpszok = lpratio2 = numpszok = hr1averpa = hr3aver = hr2aver = pszmax = num_featuresaver = lpratio = hr1medianpa = hr1insumaver = pxpa = aggregation = None
print "PA masked"
#print ind_pa
if ind_pa.shape[0] > 4 and ind_pa.shape[1] > 1:
Ymean = np.mean(ind_pa, axis=0)
print 'Max. mean value is ' + str(Ymean.max())
print "Ymean ok"
Ycov = np.cov(ind_pa, rowvar=False)
print 'Max. cov value is ' + str(Ycov.max())
print "Ycov ok"
#mh = mahalanobis_distances(Ymean, Ycov, ind_eco, parallel=False)
#mh = mahalanobis_distances(Ymean, Ycov, ind_eco, parallel=True)
mh2 = mahalanobis_distances_scipy(Ymean,
Ycov,
ind_eco,
parallel=True)
#mh2 = mahalanobis_distances_scipy(Ymean, Ycov, ind_eco, parallel=False)
maxmh = mh2.max()
print 'Max. mh value is ' + str(maxmh)
print 'Max. mh value is nan: ' + str(np.isnan(maxmh))
mh = mh2 * mh2
print "mh ok"
pmh = chisqprob(mh, len(cols)).reshape(
(eco.YSize, eco.XSize)) # 9
pmhh = np.where(pmh <= 0.001, None, pmh)
print "pmh ok" # quitar valores muy bajos!
pmhhmax = pmhh.max()
print 'Max. similarity value is ' + str(pmhhmax)
dst_ds.GetRasterBand(1).WriteArray(pmhh)
dst_ds = None
hr11 = np.where(pmhh > 0, 1, 0) # 0.5
hr1 = hr11.flatten()
hr1sum = sum(hr1)
print 'Number of pixels with similarity higher than 0 is ' + str(
hr1sum)
hr1insumaver = hr1insum = 0
hr1sumaver = hr1sum
src_ds_sim = gdal.Open(outfile)
sim = src_ds_sim.GetRasterBand(1)
gt_sim = src_ds_sim.GetGeoTransform()
xoff = int((gt_pa[0] - gt_sim[0]) / 1000)
yoff = int((gt_sim[3] - gt_pa[3]) / 1000)
xextentpa = xoff + par.XSize
yextentpa = yoff + par.YSize
xless = sim.XSize - xextentpa
yless = sim.YSize - yextentpa
xsize = par.XSize
ysize = par.YSize
if xoff > 0 and yoff > 0 and pmhhmax > 0.01 and hr1sum > 1 and maxmh != float(
'NaN'
): #and ratiogeom < 100: # also checks if results are not empty
# reading the similarity ecoregion without the PA (tmp mask)
os.system('gdal_merge.py ' + str(ecofile) + ' ' +
str(pa4) + ' -o ' + str(outfile3) +
' -ot Int32')
hri_pa_bb03 = sim.ReadAsArray().astype(np.float32)
hri_pa_bb3 = hri_pa_bb03.flatten()
src_ds_sim2 = gdal.Open(outfile3)
sim2 = src_ds_sim2.GetRasterBand(1)
gt_sim2 = src_ds_sim2.GetGeoTransform()
hri_pa_bb02 = sim2.ReadAsArray().astype(np.int32)
#hri_pa_bb2 = hri_pa_bb02.flatten()
hri_pa_bb02_max = hri_pa_bb02.max()
print 'PA: ' + str(pa)
print 'PA (= max) value from mask = ' + str(
hri_pa_bb02_max)
if hri_pa_bb02.shape == hri_pa_bb03.shape:
hri_pa02 = np.where(
hri_pa_bb02 == pa, 0,
hri_pa_bb03) # hri_pa_bb02_max
if xless < 0: xsize = xsize + xless
if yless < 0: ysize = ysize + yless
hri_pa_bb0 = sim.ReadAsArray(
xoff, yoff, xsize,
ysize).astype(np.float32)
hri_pa_bb = hri_pa_bb0.flatten()
indd = hri_pa_bb > 0
hri_pa0 = hri_pa_bb[indd]
print 'Total number of pixels with similarity values in PA: ' + str(
len(hri_pa0))
hr1averpa = round(
np.mean(hri_pa0[~np.isnan(hri_pa0)]), 2)
#print hr1averpa
#hr1medianpa = np.median(hri_pa0[~np.isnan(hri_pa0)])
print 'mean similarity in the park is ' + str(
hr1averpa)
#hr1insum = sum(np.where(hri_pa0 >= 0.5, 1,0)) # use hr1averpa as threshold instead!
hr1inaver = np.where(hri_pa0 >= hr1averpa, 1,
0)
hr1insumaver = sum(hr1inaver)
#print hr1insum
##labeled_arrayin, num_featuresin = nd.label(hr1inaver, structure=s)
hr1averr = np.where(hri_pa02 >= hr1averpa, 1,
0) # pmhh
hr1aver = hr1averr.flatten()
print 'Total number of pixels with similarity values in ECO: ' + str(
sum(hr1aver))
labeled_arrayaver, num_featuresaver = nd.label(
hr1averr, structure=s)
print 'Nr of similar patches found: ' + str(
num_featuresaver)
if num_featuresaver > 0:
lbls = np.arange(1, num_featuresaver + 1)
psizes = nd.labeled_comprehension(
labeled_arrayaver, labeled_arrayaver,
lbls, np.count_nonzero, float, 0) #-1
pszmax = psizes.max() #-hr1insumaver
dst_ds2 = driver.Create(
outfile2, src_ds_eco.RasterXSize,
src_ds_eco.RasterYSize, num_bands,
gdal.GDT_Int32, dst_options)
dst_ds2.SetGeoTransform(
src_ds_eco.GetGeoTransform())
dst_ds2.SetProjection(
src_ds_eco.GetProjectionRef())
dst_ds2.GetRasterBand(1).WriteArray(
labeled_arrayaver)
dst_ds2 = None
#num_feats = num_features - num_featuresaver
hr1sumaver = sum(hr1aver)
hr2aver = hr1sumaver #- hr1insumaver
pxpa = ind_pa.shape[0]
indokpsz = psizes >= pxpa
pszsok = psizes[indokpsz] # NEW
sumpszok = sum(pszsok)
lpratio = round(float(pszmax / pxpa), 2)
lpratio2 = round(float(sumpszok / pxpa), 2)
numpszok = len(pszsok)
hr3aver = round(float(hr2aver / pxpa), 2)
aggregation = round(
float(hr2aver / num_featuresaver), 2)
#hr2 = hr1sumaver - hr1insumaver
#print hr2
#hr3 = float(hr2/ind_pa.shape[0])
#print hr3
wb = open(csvname, 'a')
var = str(ecor) + ' ' + str(pa) + ' ' + str(
hr1averpa
) + ' ' + str(hr2aver) + ' ' + str(pxpa) + ' ' + str(
hr1insumaver
) + ' ' + str(hr3aver) + ' ' + str(
num_featuresaver
) + ' ' + str(lpratio) + ' ' + str(lpratio2) + ' ' + str(
numpszok
) + ' ' + str(pszmax) + ' ' + str(aggregation) + ' ' + str(
treepamin
) + ' ' + str(treepamax) + ' ' + str(eprpamin) + ' ' + str(
eprpamax
) + ' ' + str(prepamin) + ' ' + str(prepamax) + ' ' + str(
biopamin
) + ' ' + str(biopamax) + ' ' + str(slopepamin) + ' ' + str(
slopepamax
) + ' ' + str(ndwipamin) + ' ' + str(ndwipamax) + ' ' + str(
ndvimaxpamin
) + ' ' + str(
ndvimaxpamax
) + ' ' + str(
ndviminpamin
) + ' ' + str(
ndviminpamax
) + ' ' + str(
hpamin
) + ' ' + str(
hpamax
) + ' ' + str(
treepamean
) + ' ' + str(eprpamean) + ' ' + str(
prepamean
) + ' ' + str(biopamean) + ' ' + str(
slopepamean
) + ' ' + str(
ndwipamean
) + ' ' + str(ndvimaxpamean) + ' ' + str(
ndviminpamean
) + ' ' + str(
hpamean
) # exclude PA! #+' '+str(hr1p25pa)# '+str(hr3)+' +' '+str(hr1medianpa)+' '+str(num_features)+' '
wb.write(var)
wb.write('\n')
wb.close()
print "results exported"
os.system('rm ' + str(outfile3))
wb = open(csvname1, 'a') # LOCAL FOLDER
var = str(ecor)
wb.write(var)
wb.write('\n')
wb.close()
print "END ECOREG: " + str(ecor)
def plot_cloud_alpha(data, time, n_bins, size_min, size_max, ref_min,
n_cloud_min):
"""
Written by Till Vondenhoff, 20-03-25
Calculates slopes using 3 different methods (described by Newman)
Parameters:
data: unfiltered cloud data including multiple timesteps
time: array with all timesteps
n_bins: number of bins
ref_min: threhold for smallest value to be counted as cloud
size_min: value of the first bin
size_max: value of the last bin
n_cloud_min: minimum number of clouds per timestep required to calculate slope
Returns:
valid_time: updated time -> timesteps where n_cloud_min is not matched are not included
valid_n_clouds: updated number of clouds -> n_clouds where n_cloud_min is not matched are not included
m1: slope linear regression of power-law distribution with linear binning
m2: slope linear regression of power-law distribution with logarithmic binning
m3: slope linear regression of cumulative distribution (alpha-1)
"""
slope_lin = []
slope_log = []
slope_cum = []
valid_n_clouds = []
valid_time = []
for timestep in time:
timestep_data = data[timestep, :, :]
# marks everything above ref_min as a cloud
cloud_2D_mask = np.zeros_like(timestep_data)
cloud_2D_mask[timestep_data > ref_min] = 1
# calculates how many clouds exist in cloud_2D_mask, returns total number of clouds
labeled_clouds, n_clouds = ndi.label(cloud_2D_mask)
labels = np.arange(1, n_clouds + 1)
if (n_clouds <= n_cloud_min):
slope_lin.append(np.NaN)
slope_log.append(np.NaN)
slope_cum.append(np.NaN)
valid_n_clouds.append(n_clouds)
valid_time.append(timestep)
print('timestap', timestep, 'has too few clouds:', n_clouds)
continue
valid_n_clouds.append(n_clouds)
valid_time.append(timestep)
# Calculating how many cells belong to each labeled cloud using ndi.labeled_comprehension
# returns cloud_area and therefore it's 2D size
cloud_number_cells = ndi.labeled_comprehension(cloud_2D_mask,
labeled_clouds, labels,
np.size, float, 0)
cloud_area = np.sqrt(cloud_number_cells) * 25
# linear power-law distribution of the data (a,b)
f, slope, intercept = lin_binning(cloud_area,
n_bins,
size_min,
size_max,
show_plt=0)
slope_lin.append(slope)
# logarithmic binning of the data (c)
f, slope, intercept = log_binning(cloud_area,
n_bins,
size_min,
size_max,
show_plt=0)
slope_log.append(slope)
# cumulative distribution by sorting the data (d)
f, slope, intercept = cum_dist(cloud_area,
size_min,
size_max,
show_plt=0)
slope_cum.append(slope)
return valid_time, valid_n_clouds, slope_lin, slope_log, slope_cum
train_batch_loader = lytic_HU_3cto3v_Loader_patch.data_loader(
train_records, resize_option)
MAX_ITERATION = len(train_records)
for itr in range(MAX_ITERATION):
train_batch_pre_images, train_batch_main_images, train_batch_post_images, train_batch_main_annotation, file_list = train_batch_loader.get_next_batch(
FLAGS.training_batch_size)
# print(file_list)
batch_main_training_image = np.squeeze(train_batch_main_images)
batch_main_training_annotation = np.squeeze(train_batch_main_annotation)
batch_main_multiple_image = batch_main_training_image * batch_main_training_annotation
lbl, nlbl = nd.label(batch_main_multiple_image)
lbls = np.arange(1, nlbl + 1)
get_median_value = nd.labeled_comprehension(batch_main_multiple_image, lbl,
lbls, np.median, float, -1)
print('Median_HU_Value:', get_median_value)
if np.all(get_median_value < 1300):
print('\r')
print('==============================')
print(file_list)
print(get_median_value)
print('==============================')
print('\r')
file_list = sum(file_list, [])
low_HU_records.append(file_list)
with open('Osteolyitc_low_value_HU_fold1.pickle', 'wb') as f:
pickle.dump(low_HU_records, f, protocol=pickle.HIGHEST_PROTOCOL)
def objstats(args):
# Open and read from image and segmentation
try:
img_ds = gdal.Open(args.image, gdal.GA_ReadOnly)
except:
logger.error("Could not open image: {}".format(i=args.image))
sys.exit(1)
try:
seg_ds = ogr.Open(args.segment, 0)
seg_layer = seg_ds.GetLayer()
except:
logger.error("Could not open segmentation vector file: {}".format(args.segment))
sys.exit(1)
cols, rows = img_ds.RasterXSize, img_ds.RasterYSize
bands = range(1, img_ds.RasterCount + 1)
if args.bands is not None:
bands = args.bands
# Rasterize segments
logger.debug("About to rasterize segment vector file")
img_srs = osr.SpatialReference()
img_srs.ImportFromWkt(img_ds.GetProjectionRef())
mem_raster = gdal.GetDriverByName("MEM").Create("", cols, rows, 1, gdal.GDT_UInt32)
mem_raster.SetProjection(img_ds.GetProjection())
mem_raster.SetGeoTransform(img_ds.GetGeoTransform())
# Create artificial 'FID' field
fid_layer = seg_ds.ExecuteSQL('select FID, * from "{l}"'.format(l=seg_layer.GetName()))
gdal.RasterizeLayer(mem_raster, [1], fid_layer, options=["ATTRIBUTE=FID"])
logger.debug("Rasterized segment vector file")
seg = mem_raster.GetRasterBand(1).ReadAsArray()
logger.debug("Read segmentation image into memory")
mem_raster = None
seg_ds = None
# Get list of unique segments
useg = np.unique(seg)
# If calc is num, do only for 1 band
out_bands = 0
for stat in args.stat:
if stat == "num":
out_bands += 1
else:
out_bands += len(bands)
# Create output driver
driver = gdal.GetDriverByName(args.format)
out_ds = driver.Create(args.output, cols, rows, out_bands, gdal.GDT_Float32)
# Loop through image bands
out_b = 0
out_2d = np.empty_like(seg, dtype=np.float32)
for i_b, b in enumerate(bands):
img_band = img_ds.GetRasterBand(b)
ndv = img_band.GetNoDataValue()
band_name = img_band.GetDescription()
if not band_name:
band_name = "Band {i}".format(i=b)
logger.info('Processing input band {i}, "{b}"'.format(i=b, b=band_name))
img = img_band.ReadAsArray().astype(gdal_array.GDALTypeCodeToNumericTypeCode(img_band.DataType))
logger.debug('Read image band {i}, "{b}" into memory'.format(i=b, b=band_name))
for stat in args.stat:
logger.debug(" calculating {s}".format(s=stat))
if stat == "mean":
out = ndimage.mean(img, seg, useg)
elif stat == "var":
out = ndimage.variance(img, seg, useg)
elif stat == "num":
# Remove from list of stats so it is only calculated once
args.stat.remove("num")
count = np.ones_like(seg)
out = ndimage.sum(count, seg, useg)
elif stat == "sum":
out = ndimage.sum(img, seg, useg)
elif stat == "min":
out = ndimage.minimum(img, seg, useg)
elif stat == "max":
out = ndimage.maximum(img, seg, useg)
elif stat == "mode":
out = ndimage.labeled_comprehension(img, seg, useg, scipy_mode, out_2d.dtype, ndv)
else:
logger.error("Unknown stat. Not sure how you got here")
sys.exit(1)
# Transform to 2D
out_2d = out[seg - seg.min()]
# Fill in NDV
if ndv is not None:
out_2d[np.where(img == ndv)] = ndv
# Write out the data
out_band = out_ds.GetRasterBand(out_b + 1)
out_band.SetDescription(band_name)
if ndv is not None:
out_band.SetNoDataValue(ndv)
logger.debug(" Writing object statistic for band {b}".format(b=b + 1))
out_band.WriteArray(out_2d, 0, 0)
out_band.FlushCache()
logger.debug(" Wrote out object statistic for band {b}".format(b=b + 1))
out_b += 1
out_ds.SetGeoTransform(img_ds.GetGeoTransform())
out_ds.SetProjection(img_ds.GetProjection())
img_ds = None
seg_ds = None
out_ds = None
logger.info("Completed object statistic calculation")
def detection_by_scene_segmentation():
'''Detection of moving object by using Distortion field of centers of mass
of background objects'''
if co.counters.im_number == 0:
co.segclass.needs_segmentation = 1
if not co.segclass.exists_previous_segmentation:
print 'No existing previous segmentation. The initialisation will delay...'
else:
print 'Checking similarity..'
old_im = co.meas.background
check_if_segmented = np.sqrt(np.sum(
(co.data.depth_im[co.meas.trusty_pixels.astype(bool)].astype(float) -
old_im[co.meas.trusty_pixels.astype(bool)].astype(float))**2))
print 'Euclidean Distance of old and new background is ' +\
str(check_if_segmented)
print 'Minimum Distance to approve previous segmentation is ' +\
str(co.CONST['similar_bg_min_dist'])
if check_if_segmented < co.CONST['similar_bg_min_dist']:
print 'No need to segment again'
co.segclass.needs_segmentation = 0
else:
print 'Segmentation is needed'
if co.segclass.needs_segmentation and co.counters.im_number >= 1:
if co.counters.im_number == (co.CONST['framerate'] *
co.CONST['calib_secs'] - 1):
co.segclass.flush_previous_segmentation()
co.segclass.nz_objects.image = np.zeros_like(co.data.depth_im) - 1
co.segclass.z_objects.image = np.zeros_like(co.data.depth_im) - 1
levels_num = 8
levels = np.linspace(np.min(co.data.depth_im[co.data.depth_im > 0]), np.max(
co.data.depth_im), levels_num)
co.segclass.segment_values = np.zeros_like(co.data.depth_im)
for count in range(levels_num - 1):
co.segclass.segment_values[(co.data.depth_im >= levels[count]) *
(co.data.depth_im <= levels[count + 1])] = count + 1
elif co.counters.im_number == (co.CONST['framerate'] *
co.CONST['calib_secs']):
co.segclass.nz_objects.count = -1
co.segclass.z_objects.count = -1
co.segclass.segment_values = co.segclass.segment_values * co.meas.trusty_pixels
for val in np.unique(co.segclass.segment_values):
objs = np.ones_like(co.data.depth_im) * \
(val == co.segclass.segment_values)
labeled, nr_objects =\
ndimage.label(objs * co.edges.calib_frame)
lbls = np.arange(1, nr_objects + 1)
if val > 0:
ndimage.labeled_comprehension(objs, labeled, lbls,
co.segclass.nz_objects.process, float, 0,
True)
else:
ndimage.labeled_comprehension(objs, labeled, lbls,
co.segclass.z_objects.process,
float, 0, True)
for (points, pixsize,
xsize, ysize) in (co.segclass.nz_objects.untrusty +
co.segclass.z_objects.untrusty):
co.segclass.z_objects.count += 1
co.segclass.z_objects.image[
tuple(points)] = co.segclass.z_objects.count
co.segclass.z_objects.pixsize.append(pixsize)
co.segclass.z_objects.xsize.append(xsize)
co.segclass.z_objects.ysize.append(ysize)
print 'Found or partitioned',\
co.segclass.nz_objects.count +\
co.segclass.z_objects.count + 2, 'background objects'
co.segclass.needs_segmentation = 0
with open(co.CONST['segmentation_data'] + '.pkl', 'wb') as output:
pickle.dump((co.segclass, co.meas), output, -1)
print 'Saved segmentation data for future use.'
elif (not co.segclass.needs_segmentation) and co.counters.im_number >= 2:
if not co.segclass.initialised_centers:
try:
co.segclass.nz_objects.find_object_center(1)
except BaseException as err:
print 'Centers initialisation Exception'
raise err
try:
co.segclass.z_objects.find_object_center(0)
except BaseException as err:
print 'Centers initialisation Exception'
raise err
co.segclass.initialise_neighborhoods()
co.segclass.initialised_centers = 1
else:
try:
co.segclass.nz_objects.find_object_center(1)
except BaseException as err:
print 'Centers calculation Exception'
raise err
if co.segclass.nz_objects.center.size > 0:
co.segclass.nz_objects.find_centers_displacement()
co.meas.found_objects_mask = co.segclass.find_objects()
points_on_im = co.data.depth3d.copy()
# points_on_im[np.sum(points_on_im,axis=2)==0,:]=np.array([1,0,1])
for calc, point1, point2 in zip(
co.segclass.nz_objects.centers_to_calculate,
co.segclass.nz_objects.initial_center,
co.segclass.nz_objects.center):
if point1[0] != -1:
if calc:
cv2.arrowedLine(points_on_im,
(point1[1], point1[0]),
(point2[1], point2[0]), [0, 1, 0], 2, 1)
else:
cv2.arrowedLine(points_on_im,
(point1[1], point1[0]),
(point2[1], point2[0]), [0, 0, 1], 2, 1)
struct_el = cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, tuple(2 * [5]))
co.meas.found_objects_mask = cv2.morphologyEx(
co.meas.found_objects_mask.astype(np.uint8), cv2.MORPH_CLOSE, struct_el)
struct_el = cv2.getStructuringElement(
cv2.MORPH_ELLIPSE, tuple(2 * [10]))
co.meas.found_objects_mask = cv2.morphologyEx(
co.meas.found_objects_mask.astype(np.uint8), cv2.MORPH_OPEN, struct_el)
hand_patch, hand_patch_pos = hsa.main_process(
co.meas.found_objects_mask.astype(
np.uint8), co.meas.all_positions, 1)
co.meas.hand_patch = hand_patch
co.meas.hand_patch_pos = hand_patch_pos
if len(co.im_results.images) == 1:
co.im_results.images.append(
(255 * co.meas.found_objects_mask).astype(np.uint8))
co.im_results.images.append(points_on_im)
# elif len(co.im_results.images)==2:
# co.im_results.images[1][co.im_results.images[1]==0]=(255*points_on_im).astype(np.uint8)[co.im_results.images[1]==0]
# if hand_points.shape[1]>1:
# points_on_im[tuple(hand_points.T)]=[1,0,0]
# co.im_results.images.append(points_on_im)
return 1
def computeFeature(self, parameters, context, feedback, clip_flag=True):
step_feedback = feedbacks.ProgressMultiStepFeedback(6, feedback)
clipped_path = QgsProcessingUtils.generateTempFilename(
"labeled_clipped" + self.curr_suffix + ".tif")
clipped = qgsTreatments.clipRasterFromVector(
self.labeled_path,
self.report_layer,
clipped_path,
crop_cutline=True,
nodata=self.nodata,
data_type=self.label_out_type,
context=context,
feedback=step_feedback)
# clipped = qgsTreatments.clipRasterAllTouched(self.labeled_path,self.report_layer,
# self.input_crs,out_path=clipped_path,nodata=self.nodata,
# data_type=self.label_out_type,resolution=self.resolution,
# context=context,feedback=step_feedback)
step_feedback.setCurrentStep(1)
clip_report = qgsTreatments.getRasterUniqueValsReport(
clipped, context, step_feedback)
step_feedback.pushDebugInfo("clip_report = " + str(clip_report))
step_feedback.setCurrentStep(2)
if clip_flag:
input_clipped_path = QgsProcessingUtils.generateTempFilename(
"input_clipped_clipped" + self.curr_suffix + ".tif")
input_clipped = qgsTreatments.clipRasterFromVector(
self.input_clipped,
self.report_layer,
input_clipped_path,
crop_cutline=True,
data_type=0,
nodata=255,
context=context,
feedback=step_feedback)
else:
input_clipped_path = self.input_clipped
# input_clipped = qgsTreatments.clipRasterAllTouched(self.input_clipped,self.report_layer,
# self.input_crs,out_path=input_clipped_path,nodata=self.nodata,
# data_type=0,resolution=self.resolution,
# context=context,feedback=step_feedback)
step_feedback.setCurrentStep(3)
input_clip_report = qgsTreatments.getRasterUniqueValsReport(
input_clipped_path, context, step_feedback)
step_feedback.pushDebugInfo("input_clip_report = " +
str(input_clip_report))
clip_classes, clip_array = qgsUtils.getRasterValsAndArray(
str(clipped_path))
clip_labels = [int(cl) for cl in clip_classes]
step_feedback.setCurrentStep(4)
if 0 in clip_labels:
clip_labels.remove(0)
nb_patches_clipped = len(clip_labels)
# feedback.pushDebugInfo("nb_patches = " + str(nb_patches))
# feedback.pushDebugInfo("nb labels = " + str(len(labels)))
step_feedback.pushDebugInfo("nb clip labels = " +
str(len(clip_labels)))
# Patches length
if clip_labels:
patches_len2 = ndimage.labeled_comprehension(
clip_array, clip_array, clip_labels, len, int, 0)
step_feedback.pushDebugInfo("patches_len2 = " + str(patches_len2))
step_feedback.pushDebugInfo("nb patches_len2 = " +
str(len(patches_len2)))
step_feedback.setCurrentStep(5)
sum_ai = 0
sum_ai_sq = 0
sum_ai_sq_cbc = 0
for cpt, lbl in enumerate(clip_labels):
lbl_val = int(lbl)
cbc_len = self.patches_len[lbl_val - 1]
patch_len = patches_len2[cpt]
if cbc_len < patch_len:
utils.internal_error("CBC len " + str(cbc_len) +
" < patch_len " + str(patch_len))
ai = patch_len * self.pix_area
ai_cbc = cbc_len * self.pix_area
sum_ai_sq += ai * ai
sum_ai_sq_cbc += ai * ai_cbc
sum_ai += ai
step_feedback.pushDebugInfo("sum_ai = " + str(sum_ai))
step_feedback.pushDebugInfo("sum_ai_sq = " + str(sum_ai_sq))
step_feedback.pushDebugInfo("sum_ai_sq_cbc = " + str(sum_ai_sq_cbc))
step_feedback.pushDebugInfo("unit_divisor = " + str(self.unit_divisor))
if sum_ai_sq == 0:
step_feedback.reportError(
"Empty area for patches, please check your selection.")
nb_pix_old = len(clip_array[clip_array != self.nodata])
nb_pix2 = input_clip_report['TOTAL_PIXEL_COUNT']
nb_pix_nodata2 = input_clip_report['NODATA_PIXEL_COUNT']
nb_pix = nb_pix2 - nb_pix_nodata2
nb_pix3 = clip_report['TOTAL_PIXEL_COUNT']
nb_pix_nodata3 = clip_report['NODATA_PIXEL_COUNT']
nb_pix33 = nb_pix3 - nb_pix_nodata3
nb_0 = len(clip_array[clip_array == 0])
nb_not_0 = len(clip_array[clip_array != 0])
step_feedback.pushDebugInfo("nb_pix_old = " + str(nb_pix_old))
step_feedback.pushDebugInfo("nb_pix2 = " + str(nb_pix2))
step_feedback.pushDebugInfo("nb_pix_nodata2 = " + str(nb_pix_nodata2))
step_feedback.pushDebugInfo("nb_pix = " + str(nb_pix))
step_feedback.pushDebugInfo("nb_pix3 = " + str(nb_pix3))
step_feedback.pushDebugInfo("nb_pix_nodata3 = " + str(nb_pix_nodata3))
step_feedback.pushDebugInfo("nb_pix33 = " + str(nb_pix33))
step_feedback.pushDebugInfo("nb_0 = " + str(nb_0))
step_feedback.pushDebugInfo("nb_not_0 = " + str(nb_not_0))
tot_area = nb_pix * self.pix_area
step_feedback.pushDebugInfo("tot_area = " + str(tot_area))
#area_sq = math.pow(nb_pix,2)
if nb_pix == 0:
step_feedback.reportError(
"Unexpected error : empty area for input layer")
res_dict = {
self.REPORT_AREA: tot_area,
self.SUM_AI: sum_ai,
self.SUM_AI_SQ: sum_ai_sq,
self.SUM_AI_SQ_CBC: sum_ai_sq_cbc,
self.NB_PATCHES: nb_patches_clipped,
self.DIVISOR: self.unit_divisor,
}
res = self.mkOutputs(parameters, res_dict, context)
step_feedback.setCurrentStep(6)
return res
def connect_wet_segments(valley_raster, water_ele_raster, wet_raster, flowdir_raster, ele_raster, \
width_raster, order_raster, connected_wet_raster, max_gap, pix_per_m):
corner = arcpy.Point(arcpy.Describe(valley_raster).Extent.XMin, arcpy.Describe(valley_raster).Extent.YMin)
dx = arcpy.Describe(valley_raster).meanCellWidth
valley = arcpy.RasterToNumPyArray(valley_raster, nodata_to_value=0).astype(np.uint8)
dep_ele = arcpy.RasterToNumPyArray(water_ele_raster, nodata_to_value=0)
wet = arcpy.RasterToNumPyArray(wet_raster, nodata_to_value=0).astype(np.uint8)
ele = arcpy.RasterToNumPyArray(ele_raster, nodata_to_value=0)
width = arcpy.RasterToNumPyArray(width_raster, nodata_to_value=0)
width = np.where(wet == 1, width, 0)
dep_ele = np.where(wet == 1, dep_ele, 0)
Lab_wet, num_label_wet = ndimage.label(wet, structure=np.ones((3, 3)))
# find dry and wet valley
tran = disconnect_valley_segments(valley_raster, flowdir_raster, 3, order_raster)
dry_valley = np.where((valley == 1) & (wet == 0), 1, 0).astype(np.uint8)
wet_valley = np.where((valley == 1) & (wet == 1), 1, 0).astype(np.uint8)
temp_dry_valley = ndimage.binary_dilation(dry_valley, iterations=1, structure=np.ones((3, 3))).astype(np.uint8)
dry_valley = np.where(((wet_valley == 1) & (temp_dry_valley == 1)) | (dry_valley == 1), 1, 0).astype(np.uint8)
dry_valley = np.where(tran == 1, 0, dry_valley).astype(np.uint8)
del temp_dry_valley
Lab_dry, num_label_dry = ndimage.label(dry_valley, structure=np.ones((3, 3)))
labels = np.arange(1, num_label_dry + 1)
# 1. connection of dry segments
num_conn_label = ndimage.labeled_comprehension(Lab_wet, Lab_dry, labels, count_unique, int, 0)
num_conn = np.zeros_like(wet)
num_conn = num_conn_label[Lab_dry - 1]
num_conn = np.where(Lab_dry == 0, 0, num_conn)
conn_dry_valley = np.where(num_conn >= 3, 1, 0)
del num_conn, num_conn_label
# 2. elevation of dry segments
Lab_dry, num_label_dry = ndimage.label(conn_dry_valley, structure=np.ones((3, 3)))
labels = np.arange(1, num_label_dry + 1)
ave_ele_label = ndimage.labeled_comprehension(dep_ele, Lab_dry, labels, np.max, float, 0)
max_ele_label = ndimage.labeled_comprehension(ele, Lab_dry, labels, np.max, float, 0)
diff_ele = np.zeros_like(ele)
diff_ele = ave_ele_label[Lab_dry - 1] - max_ele_label[Lab_dry - 1]
diff_ele = np.where(Lab_dry == 0, 0, diff_ele)
conn_dry_valley = np.where(diff_ele < 0, 0, conn_dry_valley)
del diff_ele, max_ele_label, ave_ele_label
# 3. length of dry segment
Lab_dry, num_label_dry = ndimage.label(conn_dry_valley, structure=np.ones((3, 3)))
labels = np.arange(1, num_label_dry + 1)
area_gap_label = ndimage.labeled_comprehension(conn_dry_valley, Lab_dry, labels, np.sum, float, 0) * pix_per_m ** 2
area_dry = np.zeros_like(ele)
area_dry = area_gap_label[Lab_dry - 1]
area_dry = np.where((Lab_dry == 0), 0, area_dry)
conn_dry_valley = np.where(area_dry > max_gap, 0, conn_dry_valley)
del area_dry, ele
# find width of added segments
Lab_dry, num_label_dry = ndimage.label(conn_dry_valley, structure=np.ones((3, 3)))
labels = np.arange(1, num_label_dry + 1)
width_gap_label = ndimage.labeled_comprehension(width, Lab_dry, labels, np.sum, float, 0)
num_conn_label = ndimage.labeled_comprehension(Lab_wet, Lab_dry, labels, count_unique, int, 0)
width_conn = np.zeros_like(width)
width_conn = (width_gap_label[Lab_dry - 1] / num_conn_label[Lab_dry - 1]).astype(np.float32)
width_conn = np.where(Lab_dry == 0, 0, width_conn)
add_wet = add_width(width_conn)
wet = np.where((add_wet == 1) | (wet == 1), 1, 0)
arcpy.NumPyArrayToRaster(wet, corner, dx, dx).save(connected_wet_raster)
def label_size(radar, field, structure=None, rays_wrap_around=False,
size_field=None, debug=False, verbose=False):
"""
Label connected pixels in a binary radar field and compute the size of each
label. Unexpected results may occur if the specified field is not binary.
Parameters
----------
radar : Radar
Py-ART Radar containing the specified field to be labeled.
field : str
Radar field to be labeled.
structure : array_like, optional
Binary structuring element used in labeling. The default structuring
element has a squared connectivity equal to 1, i.e., only nearest
neighbours are connected to the structure origin and
diagonally-connected elements are not considered neighbours.
rays_wrap_around : bool, optional
True if all sweeps have contiguous rays, e.g., 360 deg PPI radar
volumes.
debug : bool, optional
True to print debugging information, False to suppress.
verbose : bool, optional
True to print relevant information, False to suppress.
"""
if verbose:
print 'Performing binary labeling: {}'.format(field)
if size_field is None:
size_field = '{}_feature_size'.format(field)
size_dict = {
'data': np.zeros_like(radar.fields[field]['data'], dtype=np.int32),
'units': 'unitless',
'valid_min': 0,
'comment': 'size in pixels of connected features',
}
radar.add_field(size_field, size_dict, replace_existing=True)
for sweep, slc in enumerate(radar.iter_slice()):
# Parse radar sweep data
data = np.ma.getdata(radar.get_field(sweep, field))
# Label connected pixels defined by the structuring element
labels, nlabels = ndimage.label(data, structure=structure)
index = np.arange(1, nlabels + 1)
if debug:
print 'Unique features in sweep {}: {}'.format(sweep, nlabels)
if nlabels > 0:
# Compute the size in number of pixels of each labeled feature
sizes = ndimage.labeled_comprehension(
data, labels, index, np.count_nonzero, np.int32, 0)
# Set each labeled feature to its total size in radar gates
for label, size in zip(index, sizes):
radar.fields[size_field]['data'][slc][labels == label] = size
return
def shapes_from_labels(pixels, labeled_image, labels, shape_class, shape_data_type, *args, **kwargs):
return ndimage.labeled_comprehension(
pixels, labeled_image, labels,
lambda shape_pixels: shape_class(shape_pixels, *args, **kwargs),
shape_data_type, None, False
).tolist()
def phot(self, im, showmask=True):
# TODO if we switch to astropy.photometry then we can have that
# do the work with subpixels properly, but for now they don't
# do rms of the bg correctly so we can't use their stuff yet.
mask = self.setmask(im)
if showmask:
cmap1 = pl.matplotlib.colors.LinearSegmentedColormap.from_list(
'my_cmap', ["black", "blue"], 2)
cmap1._init()
cmap1._lut[:, -1] = pl.array([0, 0.5, 0, 0, 0])
pl.imshow(mask > 0,
origin="bottom",
interpolation="nearest",
cmap=cmap1)
from scipy import ndimage
from scipy.ndimage import measurements as m
nin = len(pl.where(mask == 1)[0])
nout = len(pl.where(mask == 2)[0])
floor = pl.nanmin(im.data)
if floor < 0: floor = 0
raw = m.sum(im.data, mask, 1) - floor * nin
#bg=m.mean(im.data,mask,2)
#bgsig=m.standard_deviation(im.data,mask,2)
# from astropy.stats import sigma_clip
# clipped = sigma_clip(im.data,sig=3,iters=2)
# # http://astropy.readthedocs.org/en/latest/api/astropy.stats.sigma_clip.html#astropy.stats.sigma_clip
# # TODO what we really want is to sigma-clip only the BG array/mask
# # because including the source will probably just be domimated by the
# # source...
# bg =m.mean( clipped,mask,2)-floor
# bgsig=m.standard_deviation(clipped,mask,2)
# sigma_clip doesn't handle nans
from scipy import stats
def mymode(x):
return stats.mode(x, axis=None)[0][0]
# pdb.set_trace()
# xx=stats.mode(im.data,axis=None)
# print xx
bg = ndimage.labeled_comprehension(im.data, mask, 2, mymode, "float",
0) - floor
# bg = ndimage.labeled_comprehension(im.data,mask,2,pl.mean,"float",0)
bgsig = m.standard_deviation(im.data, mask, 2)
# assume uncert dominated by BG level.
# TODO add sqrt(cts in source) Poisson - need gain or explicit err/pix
uncert = bgsig * nin / pl.sqrt(nout)
results = raw, bg, raw - bg * nin, uncert
f = self.photfactor(im)
if f:
if self.debug: print "phot factor = ", f
results = pl.array(results) * f
if self.debug:
# print "max=", m.maximum(im.data,mask,1), m.maximum(im.data,mask,2)
# print "nin,nout=",nin,nout
print "raw, bg, bgsubbed, uncert=", results
pdb.set_trace()
return results
def ehabitat(ecor,nw,nwpathout):
global nwpath
if nw=='':
nwpath = os.getcwd()
else:
nwpath = nw
if gmaps == 0:
initglobalmaps()
if nwpathout=='':
#outdir = 'results' # ToDo: locally create folder "results" if it does not exist!
outdir = os.path.join(os.path.sep, os.getcwd(), 'results')
safelyMakeDir(outdir)
else:
#outdir = nwpathout+'/results' # SHARED FOLDER PATH
outdir = os.path.join(os.path.sep, nwpathout, 'results')
safelyMakeDir(outdir)
treepamin = treepamax = eprpamin = eprpamax = prepamin = prepamax = biopamin = biopamax = slopepamin = slopepamax = ndwipamin = ndwipamax = ndvimaxpamin = ndvimaxpamax = ndviminpamin = ndviminpamax = hpamin = hpamax = None
s = nd.generate_binary_structure(2,2) # most restrictive pattern for the landscape patches
# LOCAL FOLDER
csvname1 = os.path.join(os.path.sep, outdir, 'ecoregs_done.csv')
print csvname1
if os.path.isfile(csvname1) == False:
wb = open(csvname1,'a')
wb.write('None')
wb.write('\n')
wb.close()
# LOCAL FOLDER
csvname = os.path.join(os.path.sep, outdir, 'hri_results.csv')
print csvname
if os.path.isfile(csvname) == False:
wb = open(csvname,'a')
wb.write('ecoregion wdpaid averpasim hr2aver pxpa hr1insumaver hriaver nfeatsaver lpratio lpratio2 numpszok lpmaxsize aggregation treepamin treepamax eprpamin eprpamax prepamin prepamax biopamin biopamax slopepamin slopepamax ndwipamin ndwipamax ndvimaxpamin ndvimaxpamax ndviminpamin ndviminpamax hpamin hpamax treepamean eprpamean prepamean biopamean slopepamean ndwipamean ndvimaxpamean ndviminpamean hpamean')
wb.write('\n')
wb.close()
treepamean = eprpamean = prepamean = biopamean = slopepamean = ndwipamean = ndvimaxpamean = ndviminpamean = hpamean = None
ef = 'eco_'+str(ecor)+'.tif'
ecofile = os.path.join(os.path.sep, nwpath, 'ecoregs', ef)
#ecofile = os.path.join(os.path.sep, nwpath, os.path.sep,'ecoregs', os.path.sep, ef)
print ecofile
avail = os.path.isfile(ecofile)
if avail == True:
eco_csv = str(ecor)+'.csv'
print eco_csv
ecoparksf = os.path.join(os.path.sep, nwpath, 'pas', eco_csv)
#ecoparksf = os.path.join(os.path.sep, nwpath, os.path.sep, 'pas', os.path.sep, eco_csv)
print ecoparksf
#ecoparksf = nwpath+'/pas/'+str(ecor)+'.csv'
src_ds_eco = gdal.Open(ecofile)
eco = src_ds_eco.GetRasterBand(1)
eco_mask0 = eco.ReadAsArray(0,0,eco.XSize,eco.YSize).astype(np.int32)
eco_mask = eco_mask0.flatten()
gt_eco = src_ds_eco.GetGeoTransform()
print 'eco mask'
xoff = int((gt_eco[0]-gt_epr_global[0])/1000)
yoff = int((gt_epr_global[3]-gt_eco[3])/1000)
epr_eco_bb0 = epr_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
epr_eco_bb = epr_eco_bb0.flatten()
epr_eco0 = np.where(eco_mask == 1, (epr_eco_bb),(0))
epr_eco = np.where(epr_eco0 == 65535.0, (float('NaN')),(epr_eco0))
maskepr = np.isnan(epr_eco)
epr_eco[maskepr] = np.interp(np.flatnonzero(maskepr), np.flatnonzero(~maskepr), epr_eco[~maskepr])
print 'eco epr'
xoff = int((gt_eco[0]-gt_slope_global[0])/1000)
yoff = int((gt_slope_global[3]-gt_eco[3])/1000)
slope_eco_bb0 = slope_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
slope_eco_bb = slope_eco_bb0.flatten()
slope_eco0 = np.where(eco_mask == 1, (slope_eco_bb),(0))
slope_eco = np.where(slope_eco0 == 65535.0, (float('NaN')),(slope_eco0))
maskslope = np.isnan(slope_eco)
slope_eco[maskslope] = np.interp(np.flatnonzero(maskslope), np.flatnonzero(~maskslope), slope_eco[~maskslope])
print 'eco slope'
xoff = int((gt_eco[0]-gt_ndvimax_global[0])/1000)
yoff = int((gt_ndvimax_global[3]-gt_eco[3])/1000)
ndvimax_eco_bb0 = ndvimax_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
ndvimax_eco_bb = ndvimax_eco_bb0.flatten()
ndvimax_eco0 = np.where(eco_mask == 1, (ndvimax_eco_bb),(0))
ndvimax_eco = np.where(ndvimax_eco0 == 65535.0, (float('NaN')),(ndvimax_eco0))
maskndvimax = np.isnan(ndvimax_eco)
ndvimax_eco[maskndvimax] = np.interp(np.flatnonzero(maskndvimax), np.flatnonzero(~maskndvimax), ndvimax_eco[~maskndvimax])
print 'eco ndvimax'
xoff = int((gt_eco[0]-gt_ndvimin_global[0])/1000)
yoff = int((gt_ndvimin_global[3]-gt_eco[3])/1000)
ndvimin_eco_bb0 = ndvimin_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
ndvimin_eco_bb = ndvimin_eco_bb0.flatten()
ndvimin_eco0 = np.where(eco_mask == 1, (ndvimin_eco_bb),(0))
ndvimin_eco = np.where(ndvimin_eco0 == 65535.0, (float('NaN')),(ndvimin_eco0))
maskndvimin = np.isnan(ndvimin_eco)
ndvimin_eco[maskndvimin] = np.interp(np.flatnonzero(maskndvimin), np.flatnonzero(~maskndvimin), ndvimin_eco[~maskndvimin])
print 'eco ndvimin'
xoff = int((gt_eco[0]-gt_ndwi_global[0])/1000)
yoff = int((gt_ndwi_global[3]-gt_eco[3])/1000)
ndwi_eco_bb0 = ndwi_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
ndwi_eco_bb = ndwi_eco_bb0.flatten()
ndwi_eco0 = np.where(eco_mask == 1, (ndwi_eco_bb),(0))
ndwi_eco = np.where(ndwi_eco0 == 255.0, (float('NaN')),(ndwi_eco0))
maskndwi = np.isnan(ndwi_eco)
ndwi_eco[maskndwi] = np.interp(np.flatnonzero(maskndwi), np.flatnonzero(~maskndwi), ndwi_eco[~maskndwi])
print 'eco ndwi'
xoff = int((gt_eco[0]-gt_pre_global[0])/1000)
yoff = int((gt_pre_global[3]-gt_eco[3])/1000)
pre_eco_bb0 = pre_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
pre_eco_bb = pre_eco_bb0.flatten()
pre_eco0 = np.where(eco_mask == 1, (pre_eco_bb),(0))
pre_eco = np.where(pre_eco0 == 65535.0, (float('NaN')),(pre_eco0))
maskpre = np.isnan(pre_eco)
pre_eco[maskpre] = np.interp(np.flatnonzero(maskpre), np.flatnonzero(~maskpre), pre_eco[~maskpre])
print 'eco pre'
xoff = int((gt_eco[0]-gt_bio_global[0])/1000)
yoff = int((gt_bio_global[3]-gt_eco[3])/1000)
bio_eco_bb0 = bio_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
bio_eco_bb = bio_eco_bb0.flatten()
bio_eco0 = np.where(eco_mask == 1, (bio_eco_bb),(0))
bio_eco = np.where(bio_eco0 == 65535.0, (float('NaN')),(bio_eco0))
maskbio = np.isnan(bio_eco)
bio_eco[maskbio] = np.interp(np.flatnonzero(maskbio), np.flatnonzero(~maskbio), bio_eco[~maskbio])
print 'eco bio'
xoff = int((gt_eco[0]-gt_tree_global[0])/1000)
yoff = int((gt_tree_global[3]-gt_eco[3])/1000)
tree_eco_bb0 = tree_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
tree_eco_bb = tree_eco_bb0.flatten()
tree_eco0 = np.where(eco_mask == 1, (tree_eco_bb),(0))
tree_eco = np.where(tree_eco0 == 255.0, (float('NaN')),(tree_eco0))
masktree = np.isnan(tree_eco)
tree_eco[masktree] = np.interp(np.flatnonzero(masktree), np.flatnonzero(~masktree), tree_eco[~masktree])
print 'eco tree'
xoff = int((gt_eco[0]-gt_herb_global[0])/1000)
yoff = int((gt_herb_global[3]-gt_eco[3])/1000)
herb_eco_bb0 = herb_global.ReadAsArray(xoff,yoff,eco.XSize,eco.YSize).astype(np.float32)
herb_eco_bb = herb_eco_bb0.flatten()
herb_eco0 = np.where(eco_mask == 1, (herb_eco_bb),(0))
herb_eco = np.where(herb_eco0 == 255.0, (float('NaN')),(herb_eco0))
maskherb = np.isnan(herb_eco)
herb_eco[maskherb] = np.interp(np.flatnonzero(maskherb), np.flatnonzero(~maskherb), herb_eco[~maskherb])
print 'eco herb'
ind_eco0 = np.column_stack((bio_eco,pre_eco,epr_eco,herb_eco,ndvimax_eco,ndvimin_eco,ndwi_eco,slope_eco,tree_eco))
print 'ecovars stacked'
print ecoparksf
pa_list0 = np.genfromtxt(ecoparksf,dtype='string') # crear este archivo en subpas!
pa_list = np.unique(pa_list0)
n = len(pa_list)
for px in range(0,n): # 0,n
pa = pa_list[px]
print pa
outfile = os.path.join(os.path.sep, outdir, str(ecor)+'_'+str(pa)+'.tif')
outfile2 = os.path.join(os.path.sep, outdir, str(ecor)+'_'+str(pa)+'_lp.tif')
outfile3 = os.path.join(os.path.sep, outdir, str(ecor)+'_'+str(pa)+'_mask.tif')
#outfile = outdir+'/'+str(ecor)+'_'+str(pa)+'.tif' # LOCAL FOLDER
pa_infile = 'pa_'+str(pa)+'.tif'
pa4 = os.path.join(os.path.sep, nwpath, 'pas', pa_infile)
#pa4 = os.path.join(os.path.sep, nwpath, os.path.sep, 'pas', os.path.sep, pa_infile)
print pa4
#pa4 = nwpath+'/pas/pa_'+str(pa)+'.tif'
dropcols = np.arange(9,dtype=int)
done = os.path.isfile(outfile)
avail2 = os.path.isfile(pa4)
if done == False and avail2 == True:
pafile=pa4
src_ds_pa = gdal.Open(pafile)
par = src_ds_pa.GetRasterBand(1)
pa_mask0 = par.ReadAsArray(0,0,par.XSize,par.YSize).astype(np.int32)
pa_mask = pa_mask0.flatten()
ind = pa_mask > 0 #==int(pa)
go = 1
sum_pa_mask = sum(pa_mask[ind])#/int(pa)
if sum_pa_mask < 3: go = 0 # not processing areas smaller than 3 pixels
print sum_pa_mask
sum_pa_mask_inv = len(pa_mask[pa_mask == 0])
print sum_pa_mask_inv
print len(pa_mask)
ratiogeom = 10000
if sum_pa_mask > 0: ratiogeom = sum_pa_mask_inv/sum_pa_mask
#print ratiogeom
gt_pa = src_ds_pa.GetGeoTransform()
xoff = int((gt_pa[0]-gt_pre_global[0])/1000)
yoff = int((gt_pre_global[3]-gt_pa[3])/1000)
if xoff>0 and yoff>0 and go == 1:
num_bands=src_ds_eco.RasterCount
driver = gdal.GetDriverByName("GTiff")
dst_options = ['COMPRESS=LZW']
dst_ds = driver.Create( outfile,src_ds_eco.RasterXSize,src_ds_eco.RasterYSize,num_bands,gdal.GDT_Float32,dst_options)
dst_ds.SetGeoTransform( src_ds_eco.GetGeoTransform())
dst_ds.SetProjection( src_ds_eco.GetProjectionRef())
xoff = int((gt_pa[0]-gt_tree_global[0])/1000)
yoff = int((gt_tree_global[3]-gt_pa[3])/1000)
tree_pa_bb0 = tree_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
tree_pa_bb = tree_pa_bb0.flatten()
tree_pa0 = tree_pa_bb[ind]
tree_pa = np.where(tree_pa0 == 255.0, (float('NaN')),(tree_pa0))
mask2tree = np.isnan(tree_pa)
if mask2tree.all() == True:
dropcols[8] = -8
else:
tree_pa[mask2tree] = np.interp(np.flatnonzero(mask2tree), np.flatnonzero(~mask2tree), tree_pa[~mask2tree])
tree_pa = np.random.random_sample(len(tree_pa),)/1000 + tree_pa
print 'pa tree'
treepamin = round(tree_pa.min(),2)
treepamax = round(tree_pa.max(),2)
treepamean = round(np.mean(tree_pa),2)
print treepamin
print treepamax
treediff = abs(tree_pa.min()-tree_pa.max())
if treediff < 0.001: dropcols[8] = -8
xoff = int((gt_pa[0]-gt_epr_global[0])/1000)
yoff = int((gt_epr_global[3]-gt_pa[3])/1000)
epr_pa_bb0 = epr_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
epr_pa_bb = epr_pa_bb0.flatten()
epr_pa0 = epr_pa_bb[ind]
epr_pa = np.where(epr_pa0 == 65535.0, (float('NaN')),(epr_pa0))
mask2epr = np.isnan(epr_pa)
if mask2epr.all() == True:
dropcols[2] = -2
else:
epr_pa[mask2epr] = np.interp(np.flatnonzero(mask2epr), np.flatnonzero(~mask2epr), epr_pa[~mask2epr])
epr_pa = np.random.random_sample(len(epr_pa),)/1000 + epr_pa
print 'pa epr'
eprpamin = round(epr_pa.min(),2)
eprpamax = round(epr_pa.max(),2)
eprpamean = round(np.mean(epr_pa),2)
print eprpamin
print eprpamax
eprdiff = abs(epr_pa.min()-epr_pa.max())
if eprdiff < 0.001: dropcols[2] = -2
xoff = int((gt_pa[0]-gt_pre_global[0])/1000)
yoff = int((gt_pre_global[3]-gt_pa[3])/1000)
pre_pa_bb0 = pre_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
pre_pa_bb = pre_pa_bb0.flatten()
pre_pa0 = pre_pa_bb[ind]
pre_pa = np.where(pre_pa0 == 65535.0, (float('NaN')),(pre_pa0))
mask2pre = np.isnan(pre_pa)
if mask2pre.all() == True:
dropcols[1] = -1
else:
pre_pa[mask2pre] = np.interp(np.flatnonzero(mask2pre), np.flatnonzero(~mask2pre), pre_pa[~mask2pre])
pre_pa = np.random.random_sample(len(pre_pa),)/1000 + pre_pa
print 'pa pre'
prepamin = round(pre_pa.min(),2)
prepamax = round(pre_pa.max(),2)
prepamean = round(np.mean(pre_pa),2)
print prepamin
print prepamax
prediff = abs(pre_pa.min()-pre_pa.max())
if prediff < 0.001: dropcols[1] = -1
xoff = int((gt_pa[0]-gt_bio_global[0])/1000)
yoff = int((gt_bio_global[3]-gt_pa[3])/1000)
bio_pa_bb0 = bio_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
bio_pa_bb = bio_pa_bb0.flatten()
bio_pa0 = bio_pa_bb[ind]
bio_pa = np.where(bio_pa0 == 65535.0, (float('NaN')),(bio_pa0))
mask2bio = np.isnan(bio_pa)
if mask2bio.all() == True:
dropcols[0] = -0
else:
bio_pa[mask2bio] = np.interp(np.flatnonzero(mask2bio), np.flatnonzero(~mask2bio), bio_pa[~mask2bio])
bio_pa = np.random.random_sample(len(bio_pa),)/1000 + bio_pa
print 'pa bio'
biopamin = round(bio_pa.min(),2)
biopamax = round(bio_pa.max(),2)
biopamean = round(np.mean(bio_pa),2)
print biopamin
print biopamax
biodiff = abs(bio_pa.min()-bio_pa.max())
if biodiff < 0.001: dropcols[0] = -0
xoff = int((gt_pa[0]-gt_slope_global[0])/1000)
yoff = int((gt_slope_global[3]-gt_pa[3])/1000)
slope_pa_bb0 = slope_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
slope_pa_bb = slope_pa_bb0.flatten()
slope_pa0 = slope_pa_bb[ind]
slope_pa = np.where(slope_pa0 == 65535.0, (float('NaN')),(slope_pa0))
mask2slope = np.isnan(slope_pa)
if mask2slope.all() == True:
dropcols[7] = -7
else:
slope_pa[mask2slope] = np.interp(np.flatnonzero(mask2slope), np.flatnonzero(~mask2slope), slope_pa[~mask2slope])
slope_pa = np.random.random_sample(len(slope_pa),)/1000 + slope_pa
print 'pa slope'
slopepamin = round(slope_pa.min(),2)
slopepamax = round(slope_pa.max(),2)
slopepamean = round(np.mean(slope_pa),2)
print slopepamin
print slopepamax
slopediff = abs(slope_pa.min()-slope_pa.max())
if slopediff < 0.001: dropcols[7] = -7
xoff = int((gt_pa[0]-gt_ndwi_global[0])/1000)
yoff = int((gt_ndwi_global[3]-gt_pa[3])/1000)
ndwi_pa_bb0 = ndwi_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
ndwi_pa_bb = ndwi_pa_bb0.flatten()
ndwi_pa0 = ndwi_pa_bb[ind]
ndwi_pa = np.where(ndwi_pa0 == 255.0, (float('NaN')),(ndwi_pa0))
mask2ndwi = np.isnan(ndwi_pa)
if mask2ndwi.all() == True:
dropcols[6] = -6
else:
ndwi_pa[mask2ndwi] = np.interp(np.flatnonzero(mask2ndwi), np.flatnonzero(~mask2ndwi), ndwi_pa[~mask2ndwi])
ndwi_pa = np.random.random_sample(len(ndwi_pa),)/1000 + ndwi_pa
print 'pa ndwi'
ndwipamin = round(ndwi_pa.min(),2)
ndwipamax = round(ndwi_pa.max(),2)
ndwipamean = round(np.mean(ndwi_pa),2)
print ndwipamin
print ndwipamax
ndwidiff = abs(ndwi_pa.min()-ndwi_pa.max())
if ndwidiff < 0.001: dropcols[6] = -6
xoff = int((gt_pa[0]-gt_ndvimax_global[0])/1000)
yoff = int((gt_ndvimax_global[3]-gt_pa[3])/1000)
ndvimax_pa_bb0 = ndvimax_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
ndvimax_pa_bb = ndvimax_pa_bb0.flatten()
ndvimax_pa0 = ndvimax_pa_bb[ind]
ndvimax_pa = np.where(ndvimax_pa0 == 65535.0, (float('NaN')),(ndvimax_pa0))
mask2ndvimax = np.isnan(ndvimax_pa)
if mask2ndvimax.all() == True:
dropcols[4] = -4
else:
ndvimax_pa[mask2ndvimax] = np.interp(np.flatnonzero(mask2ndvimax), np.flatnonzero(~mask2ndvimax), ndvimax_pa[~mask2ndvimax])
ndvimax_pa = np.random.random_sample(len(ndvimax_pa),)/1000 + ndvimax_pa
print 'pa ndvimax'
ndvimaxpamin = round(ndvimax_pa.min(),2)
ndvimaxpamax = round(ndvimax_pa.max(),2)
ndvimaxpamean = round(np.mean(ndvimax_pa),2)
print ndvimaxpamin
print ndvimaxpamax
ndvimaxdiff = abs(ndvimax_pa.min()-ndvimax_pa.max())
if ndvimaxdiff < 0.001: dropcols[4] = -4
xoff = int((gt_pa[0]-gt_ndvimin_global[0])/1000)
yoff = int((gt_ndvimin_global[3]-gt_pa[3])/1000)
ndvimin_pa_bb0 = ndvimin_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
ndvimin_pa_bb = ndvimin_pa_bb0.flatten()
ndvimin_pa0 = ndvimin_pa_bb[ind]
ndvimin_pa = np.where(ndvimin_pa0 == 65535.0, (float('NaN')),(ndvimin_pa0))
mask2ndvimin = np.isnan(ndvimin_pa)
if mask2ndvimin.all() == True:
dropcols[5] = -5
else:
ndvimin_pa[mask2ndvimin] = np.interp(np.flatnonzero(mask2ndvimin), np.flatnonzero(~mask2ndvimin), ndvimin_pa[~mask2ndvimin])
ndvimin_pa = np.random.random_sample(len(ndvimin_pa),)/1000 + ndvimin_pa
print 'pa ndvimin'
ndviminpamin = round(ndvimin_pa.min(),2)
ndviminpamax = round(ndvimin_pa.max(),2)
ndviminpamean = round(np.mean(ndvimin_pa),2)
print ndviminpamin
print ndviminpamax
ndvimindiff = abs(ndvimin_pa.min()-ndvimin_pa.max())
if ndvimindiff < 0.001: dropcols[5] = -5
xoff = int((gt_pa[0]-gt_herb_global[0])/1000)
yoff = int((gt_herb_global[3]-gt_pa[3])/1000)
herb_pa_bb0 = herb_global.ReadAsArray(xoff,yoff,par.XSize,par.YSize).astype(np.float32)
herb_pa_bb = herb_pa_bb0.flatten()
herb_pa0 = herb_pa_bb[ind]
herb_pa = np.where(herb_pa0 == 255.0, (float('NaN')),(herb_pa0))
mask2herb = np.isnan(herb_pa)
if mask2herb.all() == True:
dropcols[3] = -3
else:
herb_pa[mask2herb] = np.interp(np.flatnonzero(mask2herb), np.flatnonzero(~mask2herb), herb_pa[~mask2herb])
herb_pa = np.random.random_sample(len(herb_pa),)/1000 + herb_pa
print 'pa herb'
hpamin = round(herb_pa.min(),2)
hpamax = round(herb_pa.max(),2)
hpamean = round(np.mean(herb_pa),2)
print hpamin
print hpamax
hdiff = abs(herb_pa.min()-herb_pa.max())
if hdiff < 0.001: dropcols[3] = -3
cols = dropcols[dropcols>=0]
ind_pa0 = np.column_stack((bio_pa,pre_pa,epr_pa,herb_pa,ndvimax_pa,ndvimin_pa,ndwi_pa,slope_pa,tree_pa))
ind_pa = ind_pa0[:,cols]
ind_eco = ind_eco0[:,cols]
print ind_pa.shape
hr1sum = hr1insum = indokpsz = pszok = sumpszok = lpratio2 = numpszok = hr1averpa = hr3aver = hr2aver = pszmax = num_featuresaver = lpratio = hr1medianpa = hr1insumaver = pxpa = aggregation = None
print "PA masked"
#print ind_pa
if ind_pa.shape[0]>4 and ind_pa.shape[1]>1:
Ymean = np.mean(ind_pa,axis=0)
print 'Max. mean value is '+ str(Ymean.max())
print "Ymean ok"
Ycov = np.cov(ind_pa,rowvar=False)
print 'Max. cov value is '+ str(Ycov.max())
print "Ycov ok"
#mh = mahalanobis_distances(Ymean, Ycov, ind_eco, parallel=False)
#mh2 = mahalanobis_distances(Ymean, Ycov, ind_eco, parallel=True)
mh2 = mahalanobis_distances_scipy(Ymean, Ycov, ind_eco, parallel=True) # previous working version
#mh2 = mahalanobis_distances_scipy(Ymean, Ycov, ind_eco, parallel=False)
maxmh=mh2.max()
print 'Max. mh value is '+ str(maxmh)
print 'Max. mh value is nan: '+ str(np.isnan(maxmh))
mh = mh2*mh2
print "mh ok"
pmh = chi2.sf(mh,len(cols)).reshape((eco.YSize,eco.XSize)) # chisqprob
pmhh = np.where(pmh <= 0.001,None, pmh)
print "pmh ok" # quitar valores muy bajos!
pmhhmax = pmhh.max()
print 'Max. similarity value is '+ str(pmhhmax)
dst_ds.GetRasterBand(1).WriteArray(pmhh)
dst_ds = None
hr11 = np.where(pmhh>0,1,0) # 0.5
hr1 = hr11.flatten()
hr1sum = sum(hr1)
print 'Number of pixels with similarity higher than 0 is '+str(hr1sum)
hr1insumaver = hr1insum = 0
hr1sumaver = hr1sum
src_ds_sim = gdal.Open(outfile)
sim = src_ds_sim.GetRasterBand(1)
gt_sim = src_ds_sim.GetGeoTransform()
xoff = int((gt_pa[0]-gt_sim[0])/1000)
yoff = int((gt_sim[3]-gt_pa[3])/1000)
xextentpa = xoff + par.XSize
yextentpa = yoff + par.YSize
xless = sim.XSize - xextentpa
yless = sim.YSize - yextentpa
xsize = par.XSize
ysize = par.YSize
if xoff>0 and yoff>0 and pmhhmax>0.01 and hr1sum>1 and maxmh!=float('NaN'):#and ratiogeom < 100: # also checks if results are not empty
# reading the similarity ecoregion without the PA (tmp mask)
os.system('gdal_merge.py '+str(ecofile)+' '+str(pa4)+' -o '+str(outfile3)+' -ot Int32')
hri_pa_bb03 = sim.ReadAsArray().astype(np.float32)
hri_pa_bb3 = hri_pa_bb03.flatten()
src_ds_sim2 = gdal.Open(outfile3)
sim2 = src_ds_sim2.GetRasterBand(1)
gt_sim2 = src_ds_sim2.GetGeoTransform()
hri_pa_bb02 = sim2.ReadAsArray().astype(np.int32)
#hri_pa_bb2 = hri_pa_bb02.flatten()
hri_pa_bb02_max = hri_pa_bb02.max()
print 'PA: '+str(pa)
print 'PA (= max) value from mask = '+str(hri_pa_bb02_max)
if hri_pa_bb02.shape == hri_pa_bb03.shape:
hri_pa02 = np.where(hri_pa_bb02 == pa,0,hri_pa_bb03) # hri_pa_bb02_max
if xless < 0: xsize = xsize + xless
if yless < 0: ysize = ysize + yless
hri_pa_bb0 = sim.ReadAsArray(xoff,yoff,xsize,ysize).astype(np.float32)
hri_pa_bb = hri_pa_bb0.flatten()
indd = hri_pa_bb > 0
hri_pa0 = hri_pa_bb[indd]
print 'Total number of pixels with similarity values in PA: '+str(len(hri_pa0))
hr1averpa = round(np.mean(hri_pa0[~np.isnan(hri_pa0)]),2)
#print hr1averpa
#hr1medianpa = np.median(hri_pa0[~np.isnan(hri_pa0)])
print 'mean similarity in the park is '+str(hr1averpa)
#hr1insum = sum(np.where(hri_pa0 >= 0.5, 1,0)) # use hr1averpa as threshold instead!
hr1inaver = np.where(hri_pa0 >= hr1averpa, 1,0)
hr1insumaver = sum(hr1inaver)
#print hr1insum
##labeled_arrayin, num_featuresin = nd.label(hr1inaver, structure=s)
hr1averr = np.where(hri_pa02 >= hr1averpa, 1,0) # pmhh
hr1aver = hr1averr.flatten()
print 'Total number of pixels with similarity values in ECO: '+str(sum(hr1aver))
labeled_arrayaver, num_featuresaver = nd.label(hr1averr, structure=s)
print 'Nr of similar patches found: '+str(num_featuresaver)
if num_featuresaver > 0:
lbls = np.arange(1, num_featuresaver+1)
psizes = nd.labeled_comprehension(labeled_arrayaver, labeled_arrayaver, lbls, np.count_nonzero, float, 0) #-1
pszmax = psizes.max()#-hr1insumaver
dst_ds2 = driver.Create(outfile2,src_ds_eco.RasterXSize,src_ds_eco.RasterYSize,num_bands,gdal.GDT_Int32,dst_options)
dst_ds2.SetGeoTransform(src_ds_eco.GetGeoTransform())
dst_ds2.SetProjection(src_ds_eco.GetProjectionRef())
dst_ds2.GetRasterBand(1).WriteArray(labeled_arrayaver)
dst_ds2 = None
#num_feats = num_features - num_featuresaver
hr1sumaver = sum(hr1aver)
hr2aver = hr1sumaver #- hr1insumaver
pxpa = ind_pa.shape[0]
indokpsz = psizes >= pxpa
pszsok = psizes[indokpsz] # NEW
sumpszok = sum(pszsok)
lpratio=round(float(pszmax/pxpa),2)
lpratio2=round(float(sumpszok/pxpa),2)
numpszok = len(pszsok)
hr3aver = round(float(hr2aver/pxpa),2)
aggregation = round(float(hr2aver/num_featuresaver),2)
#hr2 = hr1sumaver - hr1insumaver
#print hr2
#hr3 = float(hr2/ind_pa.shape[0])
#print hr3
wb = open(csvname,'a')
var = str(ecor)+' '+str(pa)+' '+str(hr1averpa)+' '+str(hr2aver)+' '+str(pxpa)+' '+str(hr1insumaver)+' '+str(hr3aver)+' '+str(num_featuresaver)+' '+str(lpratio)+' '+str(lpratio2)+' '+str(numpszok)+' '+str(pszmax)+' '+str(aggregation)+' '+str(treepamin)+' '+str(treepamax)+' '+str(eprpamin)+' '+str(eprpamax)+' '+str(prepamin)+' '+str(prepamax)+' '+str(biopamin)+' '+str(biopamax)+' '+str(slopepamin)+' '+str(slopepamax)+' '+str(ndwipamin)+' '+str(ndwipamax)+' '+str(ndvimaxpamin)+' '+str(ndvimaxpamax)+' '+str(ndviminpamin)+' '+str(ndviminpamax)+' '+str(hpamin)+' '+str(hpamax)+' '+str(treepamean)+' '+str(eprpamean)+' '+str(prepamean)+' '+str(biopamean)+' '+str(slopepamean)+' '+str(ndwipamean)+' '+str(ndvimaxpamean)+' '+str(ndviminpamean)+' '+str(hpamean)# exclude PA! #+' '+str(hr1p25pa)# '+str(hr3)+' +' '+str(hr1medianpa)+' '+str(num_features)+' '
wb.write(var)
wb.write('\n')
wb.close()
print "results exported"
os.system('rm '+str(outfile3))
wb = open(csvname1,'a') # LOCAL FOLDER
var = str(ecor)
wb.write(var)
wb.write('\n')
wb.close()
print "END ECOREG: " + str(ecor)
def main():
# Parse command line options
parser = argparse.ArgumentParser(
description='Test different nets with 3D data.')
parser.add_argument('--flair',
action='store',
dest='flair',
default='FLAIR_preprocessed.nii.gz')
parser.add_argument('--pd',
action='store',
dest='pd',
default='DP_preprocessed.nii.gz')
parser.add_argument('--t2',
action='store',
dest='t2',
default='T2_preprocessed.nii.gz')
parser.add_argument('--t1',
action='store',
dest='t1',
default='T1_preprocessed.nii.gz')
parser.add_argument('--output',
action='store',
dest='output',
default='output.nii.gz')
parser.add_argument('--no-docker',
action='store_false',
dest='docker',
default=True)
c = color_codes()
patch_size = (15, 15, 15)
options = vars(parser.parse_args())
batch_size = 10000
min_size = 30
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Loading the net ' + c['b'] + '1' + c['nc'] + c['g'] + '>' +
c['nc'])
net_name = '/usr/local/nets/deep-challenge2016.init.model_weights.pkl' if options['docker'] \
else './deep-challenge2016.init.model_weights.pkl'
net = NeuralNet(
layers=[
(InputLayer, dict(name='in', shape=(None, 4, 15, 15, 15))),
(Conv3DDNNLayer,
dict(name='conv1_1',
num_filters=32,
filter_size=(5, 5, 5),
pad='same')),
(Pool3DDNNLayer,
dict(name='avgpool_1',
pool_size=2,
stride=2,
mode='average_inc_pad')),
(Conv3DDNNLayer,
dict(name='conv2_1',
num_filters=64,
filter_size=(5, 5, 5),
pad='same')),
(Pool3DDNNLayer,
dict(name='avgpool_2',
pool_size=2,
stride=2,
mode='average_inc_pad')),
(DropoutLayer, dict(name='l2drop', p=0.5)),
(DenseLayer, dict(name='l1', num_units=256)),
(DenseLayer,
dict(name='out', num_units=2,
nonlinearity=nonlinearities.softmax)),
],
objective_loss_function=objectives.categorical_crossentropy,
update=updates.adam,
update_learning_rate=0.0001,
verbose=10,
max_epochs=50,
train_split=TrainSplit(eval_size=0.25),
custom_scores=[('dsc', lambda p, t: 2 * np.sum(p * t[:, 1]) / np.sum(
(p + t[:, 1])))],
)
net.load_params_from(net_name)
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Creating the probability map ' + c['b'] + '1' + c['nc'] + c['g'] +
'>' + c['nc'])
names = np.array(
[options['flair'], options['pd'], options['t2'], options['t1']])
image_nii = load_nii(options['flair'])
image1 = np.zeros_like(image_nii.get_data())
print('0% of data tested', end='\r')
sys.stdout.flush()
for batch, centers, percent in load_patch_batch_percent(
names, batch_size, patch_size):
y_pred = net.predict_proba(batch)
print('%f%% of data tested' % percent, end='\r')
sys.stdout.flush()
[x, y, z] = np.stack(centers, axis=1)
image1[x, y, z] = y_pred[:, 1]
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Loading the net ' + c['b'] + '2' + c['nc'] + c['g'] + '>' +
c['nc'])
net_name = '/usr/local/nets/deep-challenge2016.final.model_weights.pkl' if options['docker'] \
else './deep-challenge2016.final.model_weights.pkl'
net = NeuralNet(
layers=[
(InputLayer, dict(name='in', shape=(None, 4, 15, 15, 15))),
(Conv3DDNNLayer,
dict(name='conv1_1',
num_filters=32,
filter_size=(5, 5, 5),
pad='same')),
(Pool3DDNNLayer,
dict(name='avgpool_1',
pool_size=2,
stride=2,
mode='average_inc_pad')),
(Conv3DDNNLayer,
dict(name='conv2_1',
num_filters=64,
filter_size=(5, 5, 5),
pad='same')),
(Pool3DDNNLayer,
dict(name='avgpool_2',
pool_size=2,
stride=2,
mode='average_inc_pad')),
(DropoutLayer, dict(name='l2drop', p=0.5)),
(DenseLayer, dict(name='l1', num_units=256)),
(DenseLayer,
dict(name='out', num_units=2,
nonlinearity=nonlinearities.softmax)),
],
objective_loss_function=objectives.categorical_crossentropy,
update=updates.adam,
update_learning_rate=0.0001,
batch_iterator_train=BatchIterator(batch_size=4096),
verbose=10,
max_epochs=2000,
train_split=TrainSplit(eval_size=0.25),
custom_scores=[('dsc', lambda t, p: 2 * np.sum(t * p[:, 1]) / np.sum(
(t + p[:, 1])))],
)
net.load_params_from(net_name)
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Creating the probability map ' + c['b'] + '2' + c['nc'] + c['g'] +
'>' + c['nc'])
image2 = np.zeros_like(image_nii.get_data())
print('0% of data tested', end='\r')
sys.stdout.flush()
for batch, centers, percent in load_patch_batch_percent(
names, batch_size, patch_size):
y_pred = net.predict_proba(batch)
print('%f%% of data tested' % percent, end='\r')
sys.stdout.flush()
[x, y, z] = np.stack(centers, axis=1)
image2[x, y, z] = y_pred[:, 1]
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<Saving to file ' + c['b'] + options['output'] + c['nc'] + c['g'] +
'>' + c['nc'])
image = (image1 * image2) > 0.5
# filter candidates < min_size
labels, num_labels = ndimage.label(image)
lesion_list = np.unique(labels)
num_elements_by_lesion = ndimage.labeled_comprehension(
image, labels, lesion_list, np.sum, float, 0)
filt_min_size = num_elements_by_lesion >= min_size
lesion_list = lesion_list[filt_min_size]
image = reduce(np.logical_or, map(lambda lab: lab == labels, lesion_list))
image_nii.get_data()[:] = np.roll(np.roll(image, 1, axis=0), 1, axis=1)
path = '/'.join(options['t1'].rsplit('/')[:-1])
outputname = options['output'].rsplit('/')[-1]
image_nii.to_filename(os.path.join(path, outputname))
if not options['docker']:
path = '/'.join(options['output'].rsplit('/')[:-1])
case = options['output'].rsplit('/')[-1]
gt = load_nii(os.path.join(
path, 'Consensus.nii.gz')).get_data().astype(dtype=np.bool)
dsc = np.sum(
2.0 * np.logical_and(gt, image)) / (np.sum(gt) + np.sum(image))
print(c['c'] + '[' + strftime("%H:%M:%S") + '] ' + c['g'] +
'<DSC value for ' + c['c'] + case + c['g'] + ' = ' + c['b'] +
str(dsc) + c['nc'] + c['g'] + '>' + c['nc'])
def cloud_size_dist(dist, time, n_bins, size_min, size_max, ref_min, file,
show_plt):
"""
Written by Lennéa Hayo, 2019
Edited by Till Vondenhoff, 20-03-28
Creates Newmanns figure 3 of cloud size distribution
Parameters:
dist: netcdf file of satelite shot with clouds
n_bins: number of bins
ref_min: smallest value
bin_min: value of the first bin
bin_max: value of the last bin
file: name given to dataset of netcdf file
min_pixel: smallest cloud size value for which the power law holds (used in alpha_newman5 to calculate the
alpha of cumulative dist.)
Added Parameters:
show_plt: Creates 4 tile plot with different slopes if show_plt=True
Returns:
fig: plots that resemble Newmans figure 3
m1: slope linear regression of power-law distribution with log scales
m2: slope linear regression of power-law distribution with log binning
m3: slope linear regression of cumulative distribution (alpha-1)
"""
r1 = file[dist][time]
# marks everything above ref_min as a cloud
cloud_2D_mask = np.zeros_like(r1)
cloud_2D_mask[r1 > ref_min] = 1
# calculates how many clouds exist in cloud_2D_mask, returns total number of clouds
labeled_clouds, n_clouds = ndi.label(cloud_2D_mask)
labels = np.arange(1, n_clouds + 1)
print('\n---------------------------------------------')
print(' number of clouds in this timestep:', n_clouds)
print('---------------------------------------------\n')
# Calculating how many cells belong to each labeled cloud using ndi.labeled_comprehension
# returns cloud_area and therefore its 2D size
cloud_pixel = ndi.labeled_comprehension(cloud_2D_mask, labeled_clouds,
labels, np.size, float, 0)
cloud_area = np.sqrt(cloud_pixel) * 25.
cloud_area_min = np.min(cloud_area)
cloud_area_max = np.max(cloud_area)
if show_plt:
CSD, bins = np.histogram(cloud_area, bins=n_bins)
bin_width = (bins[-1] - bins[0]) / len(bins)
bins = bins[1:] / 2 + bins[:-1] / 2
plt.axvline(x=size_min,
color='red',
linewidth=1.5,
alpha=0.8,
linestyle='--')
plt.axvline(x=size_max,
color='red',
linewidth=1.5,
alpha=0.8,
linestyle='--')
plt.plot(bins, CSD / bin_width)
plt.axvspan(0, size_min, color='gray', alpha=0.4, lw=0)
plt.axvspan(size_max, cloud_area_max, color='gray', alpha=0.4, lw=0)
plt.xlabel('cloud size [m]')
plt.ylabel('probability density function')
plt.title('linear histogram')
plt.xlim(0, cloud_area_max)
plt.show()
# linear power-law distribution of the data
f_lin, slope_lin, intercept_lin = lin_binning(cloud_area, n_bins, size_min,
size_max, show_plt)
# logarithmic binning of the data
f_log, slope_log, intercept_log = log_binning(cloud_area, n_bins, size_min,
size_max, show_plt)
# cumulative distribution by sorting the data
f_cum, slope_cum, intercept_cum = cum_dist(cloud_area, size_min, size_max,
show_plt)
#fig, m1, m2, m3 = func_newmann3(cloud_area, n_bins, bin_min, bin_max, cloud_area_min, cloud_area_max, min_pixel, show_plt)
#fig = four_tile_plot()
return slope_lin, slope_log, slope_cum
def NotePlumeCoordinates(daily_data_dict, basepath, lonres, latres, params,
compare_gfed_edgar):
"""
Parameters
----------
daily_data_dict : dictionary
daily_data[<day>], contains data about TROPOMI measurement per day.
TODO!!!!!!!!!!!!!!ct (txt file) of plume coordinates will be stored.
Returns
-------
Saves .txt file in coord_directory.
"""
# Output directories, for coordinates and figures
coord_directory = ut.DefineAndCreateDirectory(
os.path.join(basepath + r'\04_output\plume_coordinates'))
# Defining boundaries
lat_min = daily_data_dict['lat_min']
lat_max = daily_data_dict['lat_max']
lon_min = daily_data_dict['lon_min']
lon_max = daily_data_dict['lon_max']
# Deciding on the nlon_t and nlat_t
field_t = daily_data_dict['CO_ppb']
nlon_t = len(field_t[0])
nlat_t = len(field_t)
if compare_gfed_edgar:
# Define the plumes
plumes = daily_data_dict['plumes_explained']
# Label the plumes
labels, nlabels = ndimage.label(plumes)
# Get some statistics
daily_data_dict = GetStats(daily_data_dict, labels, nlabels)
else:
# Define the plumes
plumes = daily_data_dict['plume_mask']
# Label the plumes
labels, nlabels = ndimage.label(plumes)
# Get some label statistics, for file
max_xco = ndimage.measurements.maximum_position(field_t, labels,
np.arange(nlabels) + 1)
mean_xco_raw = ndimage.measurements.mean(field_t, labels,
np.arange(nlabels) + 1)
mean_xco = [round(num, 3) for num in mean_xco_raw]
plume_size = ndimage.labeled_comprehension(plumes, labels,
np.arange(nlabels) + 1, np.sum,
float, 0)
unexplained = round(
(100 - (daily_data_dict['explained plumes'] / nlabels) * 100), 2)
exp_by_gfed = round(
((daily_data_dict['explained by gfed'] / nlabels) * 100), 2)
exp_by_edgar = round(
((daily_data_dict['explained by edgar'] / nlabels) * 100), 2)
# Generate coordinate meshgrid
lon_t = np.linspace(lon_min, lon_max, nlon_t)
lat_t = np.linspace(lat_min, lat_max, nlat_t)
lon, lat = np.meshgrid(lon_t, lat_t)
# Write to txt file
day = daily_data_dict['day']
month = daily_data_dict['month']
year = daily_data_dict['year']
curr_time = ut.GetCurrentTime()
total = daily_data_dict['total_plumes']
bufferarea = round((np.pi * ((params[0] * ((lonres + latres) / 2))**2)), 1)
filename = os.path.join(coord_directory + \
'Plume_coordinates_{}_{}_{}.txt'.format(month, day, year))
headerstring = f"""#----------------------------------------
#----------------------------------------
This file was automatically generated at: {curr_time['year']}/{curr_time['month']}/{curr_time['day']} {curr_time['hour']}:{curr_time['minute']}
This file contains a list with information on Carbon Monoxide plumes at {month}/{day}/{year}, between:
longitudes: [{lon_min}, {lon_max}]
latitudes: [{lat_min}, {lat_max}]
column descriptions:
- type: Plume centre of mass origin: (Unknown, TROPOMI, TROPOMI+GFED, TROPOMI+EDGAR, TROPOMI+GFED+EDGAR)
- latitude: Latitude of northernmost edge of plume
- longitude: Longitude of westermost edge of plume
- grid_cells: Amount of grid cells (~{lonres}x{latres}km) in plume
- CO_max: Highest Carbon Monoxide concentration measured in plume (ppb)
- CO_average: Average Carbon Monoxide concentration measured in plume (ppb)
Total amount of plumes identified by TROPOMI: {total}
Percentage of TROPOMI plumes that can be explained by GFED: {exp_by_gfed}%
Percentage of TROPOMI plumes that can be explained by EDGAR: {exp_by_edgar}%
Percentage of TROPOMI plumes that cannot be explained by EDGAR or GFED:{unexplained}%
Note: a TROPOMI grid cell can be explained by EDGAR or GFED when an EDGAR or GFED enhancement was detected within
a circular buffer with radius {params[0]} (~{bufferarea} km^2) around a TROPOMI plume.
Other parameter settings:
Moving window size: {params[2]}x{params[2]} grid cells
Moving window step size: {params[3]} grid cells
Standard deviations treshold: {params[1]}
#----------------------------------------
#----------------------------------------
plume_origin; latitude; longitude; grid_cells; CO_max; CO_average;
"""
f = open(filename, 'w+')
f.write(headerstring)
for i in range(1, nlabels + 1):
plume = np.copy(labels)
plume[plume != i] = 0 # Keep only the labelled plume
# Retrieve plume value
y, x = np.where(
plume != 0) # Get the indices of the left top corner of plume
y = y[0] # Get only northernmost grid cell coordinate
x = x[0] # Get only westermost grid cell coordinate
x_co_max = int(max_xco[i - 1][1])
y_co_max = int(max_xco[i - 1][0])
plume_origin = 'Unknown' if (plumes[y,x] == 1) else 'Wildfire' \
if (plumes[y,x] == 11) else 'Anthropogenic' if (plumes[y,x] == 101) \
else 'Wildfire/anthropogenic' if (plumes[y,x] == 111) else 'Error'
f.write(
f"{plume_origin}; {lat[y, x]}; {lon[y, x]}; {plume_size[i-1]}; {round(field_t[y_co_max, x_co_max], 3)}; {mean_xco[i-1]}\n"
)
f.close()
# Notify a text file has been generated
#print(f'Generated: {filename}')
return
def NEMASubvols(input_vol,
voxsizes,
relTh=0.2,
minvol=300,
margin=9,
nbins=100,
zignore=38,
bgSignal=None):
""" Segment a complete NEMA PET volume with several hot sphere in different subvolumes containing
only one sphere
Parameters
----------
input_vol : 3D numpy array
the volume to be segmented
voxsizes a 1D numpy array
containing the voxelsizes
relTh : float, optional
the relative threshold used to find spheres
minvol : float, optional
minimum volume of spheres to be segmented (same unit as voxel size^3)
margin : int, optional
margin around segmented spheres (same unit as voxel size)
nbins : int, optional
number of bins used in histogram for background determination
zignore : float, optional
distance to edge of FOV that is ignored (same unit as voxelsize)
bgSignal : float or None, optional
the signal intensity of the background
if None, it is auto determined from a histogram analysis
Returns
-------
list
of slices to access the subvolumes from the original volume
"""
vol = input_vol.copy()
xdim, ydim, zdim = vol.shape
minvol = int(minvol / np.prod(voxsizes))
dx = int(np.ceil(margin / voxsizes[0]))
dy = int(np.ceil(margin / voxsizes[1]))
dz = int(np.ceil(margin / voxsizes[2]))
nzignore = int(np.ceil(zignore / voxsizes[2]))
vol[:, :, :nzignore] = 0
vol[:, :, -nzignore:] = 0
# first do a quick search for the biggest sphere (noisy edge of FOV can spoil max value!)
histo = py.histogram(vol[vol > 0.01 * vol.max()], nbins)
#bgSignal = histo[1][argrelextrema(histo[0], np.greater)[0][0]]
if bgSignal is None:
bgSignal = histo[1][find_peaks_cwt(histo[0],
np.arange(nbins / 6, nbins))[0]]
thresh = bgSignal + relTh * (vol.max() - bgSignal)
vol2 = np.zeros(vol.shape, dtype=np.int)
vol2[vol > thresh] = 1
vol3, nrois = label(vol2)
rois = np.arange(1, nrois + 1)
roivols = labeled_comprehension(vol, vol3, rois, len, int, 0)
i = 1
for roi in rois:
if (roivols[roi - 1] < minvol): vol3[vol3 == roi] = 0
else:
vol3[vol3 == roi] = i
i = i + 1
nspheres = vol3.max()
spherelabels = np.arange(1, nspheres + 1)
bboxes = find_objects(vol3)
nmaskvox = list()
slices = list()
for bbox in bboxes:
xstart = max(0, bbox[0].start - dx)
xstop = min(xdim, bbox[0].stop + dx + 1)
ystart = max(0, bbox[1].start - dy)
ystop = min(xdim, bbox[1].stop + dy + 1)
zstart = max(0, bbox[2].start - dz)
zstop = min(xdim, bbox[2].stop + dz + 1)
slices.append((slice(xstart, xstop,
None), slice(ystart, ystop,
None), slice(zstart, zstop, None)))
nmaskvox.append((xstop - xstart) * (ystop - ystart) * (zstop - zstart))
# sort subvols acc to number of voxel
slices = [slices[kk] for kk in np.argsort(nmaskvox)[::-1]]
return slices
def main():
print "current working directory", os.getcwd()
print "Reading input file path :",Parameters.demDataFilePath
print "Reading input file :",Parameters.demFileName
defaults.figureNumber = 0
rawDemArray = read_dem_from_geotiff(Parameters.demFileName,\
Parameters.demDataFilePath)
nanDemArraylr=np.array(rawDemArray)
nanDemArray = nanDemArraylr
nanDemArray[nanDemArray < defaults.demNanFlag]= np.nan
Parameters.minDemValue= np.min(nanDemArray[:])
Parameters.maxDemValue= np.max(nanDemArray[:])
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(nanDemArray,cmap=cm.coolwarm)
plt.xlabel('X[m]')
plt.ylabel('Y[m]')
plt.title('Input DEM')
if defaults.doPlot==1:
plt.show()
# Area of analysis
Parameters.xDemSize=np.size(nanDemArray,0)
Parameters.yDemSize=np.size(nanDemArray,1)
# Calculate pixel length scale and assume square
Parameters.maxLowerLeftCoord = np.max([Parameters.xDemSize, \
Parameters.yDemSize])
print 'DTM size: ',Parameters.xDemSize, 'x' ,Parameters.yDemSize
#-----------------------------------------------------------------------------
# Compute slope magnitude for raw and filtered DEMs
print 'Computing slope of raw DTM'
print 'DEM pixel scale:',Parameters.demPixelScale
print np.array(nanDemArray).shape
slopeXArray,slopeYArray = np.gradient(np.array(nanDemArray),\
Parameters.demPixelScale)
slopeMagnitudeDemArray = np.sqrt(slopeXArray**2 + slopeYArray**2)
# plot the slope DEM array
slopeMagnitudeDemArrayNp = np.array(slopeMagnitudeDemArray)
print slopeMagnitudeDemArrayNp.shape
# plotting the slope DEM of non filtered DEM
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(slopeMagnitudeDemArrayNp,cmap=cm.coolwarm)
plt.xlabel('X[m]')
plt.ylabel('Y[m]')
plt.title('Slope of unfiltered DEM')
if defaults.doPlot==1:
plt.show()
# Computation of the threshold lambda used in Perona-Malik nonlinear
# filtering. The value of lambda (=edgeThresholdValue) is given by the 90th
# quantile of the absolute value of the gradient.
print'Computing lambda = q-q-based nonlinear filtering threshold'
slopeMagnitudeDemArrayQ = slopeMagnitudeDemArrayNp
slopeMagnitudeDemArrayQ = np.reshape(slopeMagnitudeDemArrayQ,\
np.size(slopeMagnitudeDemArrayQ))
slopeMagnitudeDemArrayQ = slopeMagnitudeDemArrayQ[~np.isnan(slopeMagnitudeDemArrayQ)]
print 'dem smoothing Quantile',defaults.demSmoothingQuantile
edgeThresholdValuescipy = mquantiles(np.absolute(slopeMagnitudeDemArrayQ),\
defaults.demSmoothingQuantile)
print 'edgeThresholdValuescipy :', edgeThresholdValuescipy
# performing PM filtering using the anisodiff
print 'Performing Perona-Malik nonlinear filtering'
filteredDemArray = anisodiff(nanDemArray, defaults.nFilterIterations, \
edgeThresholdValuescipy,\
defaults.diffusionTimeIncrement, \
(Parameters.demPixelScale,\
Parameters.demPixelScale),2)
# plotting the filtered DEM
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(filteredDemArray,cmap=cm.coolwarm)
plt.xlabel('X[m]')
plt.ylabel('Y[m]')
plt.title('Filtered DEM')
if defaults.doPlot==1:
plt.show()
# Writing the filtered DEM as a tif
write_geotif_filteredDEM(filteredDemArray,Parameters.demDataFilePath,\
Parameters.demFileName)
# Computing slope of filtered DEM
print 'Computing slope of filtered DTM'
filteredDemArraynp = filteredDemArray#np.gradient only takes an array as input
slopeXArray,slopeYArray = np.gradient(filteredDemArraynp,Parameters.demPixelScale)
slopeDemArray = np.sqrt(slopeXArray**2 + slopeYArray**2)
slopeMagnitudeDemArrayQ = slopeDemArray
slopeMagnitudeDemArrayQ = np.reshape(slopeMagnitudeDemArrayQ,\
np.size(slopeMagnitudeDemArrayQ))
slopeMagnitudeDemArrayQ = slopeMagnitudeDemArrayQ[~np.isnan(slopeMagnitudeDemArrayQ)]
print ' angle min:', np.arctan(np.percentile(slopeMagnitudeDemArrayQ,0.1))*180/np.pi
print ' angle max:', np.arctan(np.percentile(slopeMagnitudeDemArrayQ,99.9))*180/np.pi
print 'mean slope:',np.nanmean(slopeDemArray[:])
print 'stdev slope:',np.nanstd(slopeDemArray[:])
#Computing curvature
print 'computing curvature'
curvatureDemArrayIn= filteredDemArraynp
#curvatureDemArrayIn[curvatureDemArrayIn== defaults.demErrorFlag]=np.nan
curvatureDemArray = compute_dem_curvature(curvatureDemArrayIn,\
Parameters.demPixelScale,\
defaults.curvatureCalcMethod)
#Writing the curvature array
outfilepath = Parameters.geonetResultsDir
outfilename = Parameters.demFileName
outfilename = outfilename.split('.')[0]+'_curvature.tif'
write_geotif_generic(curvatureDemArray,outfilepath,outfilename)
#Computation of statistics of curvature
print 'Computing curvature statistics'
print curvatureDemArray.shape
tt = curvatureDemArray[~np.isnan(curvatureDemArray[:])]
print tt.shape
finiteCurvatureDemList = curvatureDemArray[np.isfinite(curvatureDemArray[:])]
print finiteCurvatureDemList.shape
curvatureDemMean = np.nanmean(finiteCurvatureDemList)
curvatureDemStdDevn = np.nanstd(finiteCurvatureDemList)
print ' mean: ', curvatureDemMean
print ' standard deviation: ', curvatureDemStdDevn
# plotting only for testing purposes
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(curvatureDemArray,cmap=cm.coolwarm)
plt.xlabel('X[m]')
plt.ylabel('Y[m]')
plt.title('Curvature DEM')
plt.colorbar()
if defaults.doPlot==1:
plt.show()
#*************************************************
#Compute curvature quantile-quantile curve
# This seems to take a long time ... is commented for now
print 'Computing curvature quantile-quantile curve'
#osm,osr = compute_quantile_quantile_curve(finiteCurvatureDemList)
#print osm[0]
#print osr[0]
thresholdCurvatureQQxx = 1
# have to add method to automatically compute the thresold
# .....
# .....
#*************************************************
# Computing contributing areas
print 'Computing upstream accumulation areas using MFD from GRASS GIS'
"""
return {'outlets':outlets, 'fac':nanDemArrayfac ,\
'fdr':nanDemArrayfdr ,'basins':nanDemArraybasins,\
'outletsxxProj':outletsxxProj, 'outletsyyProj':outletsyyProj,\
'bigbasins':allbasins}
"""
# Call the flow accumulation function
flowroutingresults = flowaccumulation(filteredDemArray)
# Read out the flowroutingresults into appropriate variables
outletPointsList = flowroutingresults['outlets']
flowArray = flowroutingresults['fac']
flowDirectionsArray = flowroutingresults['fdr']
# These are actually not sub basins, if the basin threshold
# is large, then you might have as nulls, so best
# practice is to keep the basin threshold close to 1000
# default value is 10,000
#subBasinIndexArray = flowroutingresults['basins']
#subBasinIndexArray[subBasinIndexArray==-9999]=np.nan
basinIndexArray = flowroutingresults['bigbasins']
flowArray[np.isnan(filteredDemArray)]=np.nan
flowMean = np.mean(flowArray[~np.isnan(flowArray[:])])
print 'Mean upstream flow: ', flowMean
# plotting only for testing purposes
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
drainageMeasure = -np.sqrt(np.log10(flowArray))
plt.imshow(drainageMeasure,cmap=cm.coolwarm)
plt.plot(outletPointsList[1],outletPointsList[0],'go')
plt.xlabel('X[m]')
plt.ylabel('Y[m]')
plt.title('flowArray with outlets')
plt.colorbar()
if defaults.doPlot==1:
plt.show()
# plotting only for testing purposes
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(basinIndexArray.T,cmap=cm.Dark2)
plt.plot(outletPointsList[1],outletPointsList[0],'go')
plt.xlabel('X[m]')
plt.ylabel('Y[m]')
plt.title('basinIndexArray with outlets')
if defaults.doPlot==1:
plt.show()
# Define a skeleton based on flow alone
skeletonFromFlowArray = \
compute_skeleton_by_single_threshold(flowArray.T,\
defaults.flowThresholdForSkeleton)
# Define a skeleton based on curvature alone
skeletonFromCurvatureArray =\
compute_skeleton_by_single_threshold(curvatureDemArray.T,\
curvatureDemMean+thresholdCurvatureQQxx*curvatureDemStdDevn)
# Define a skeleton based on curvature and flow
skeletonFromFlowAndCurvatureArray =\
compute_skeleton_by_dual_threshold(curvatureDemArray.T, flowArray.T, \
curvatureDemMean+thresholdCurvatureQQxx*curvatureDemStdDevn, \
defaults.flowThresholdForSkeleton)
# plotting only for testing purposes
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(skeletonFromFlowAndCurvatureArray.T,cmap=cm.binary)
plt.plot(outletPointsList[1],outletPointsList[0],'go')
plt.xlabel('X[m]')
plt.ylabel('Y[m]')
plt.title('Curvature with outlets')
if defaults.doPlot==1:
plt.show()
# Writing the skeletonFromFlowAndCurvatureArray array
outfilepath = Parameters.geonetResultsDir
outfilename = Parameters.demFileName
outfilename = outfilename.split('.')[0]+'_skeleton.tif'
write_geotif_generic(skeletonFromFlowAndCurvatureArray.T,\
outfilepath,outfilename)
# Computing the percentage drainage areas
print 'Computing percentage drainage area of each indexed basin'
fastMarchingStartPointList = np.array(outletPointsList)
print fastMarchingStartPointList
#fastMarchingStartPointListFMM = np.zeros((fastMarchingStartPointList.shape))
fastMarchingStartPointListFMMx = []
fastMarchingStartPointListFMMy = []
basinsUsedIndexList = np.zeros((len(fastMarchingStartPointList[0]),1))
nx = Parameters.xDemSize
ny = Parameters.yDemSize
nDempixels = float(nx*ny)
basinIndexArray = basinIndexArray.T
for label in range(0,len(fastMarchingStartPointList[0])):
outletbasinIndex = basinIndexArray[fastMarchingStartPointList[0,label],\
fastMarchingStartPointList[1,label]]
print outletbasinIndex
numelments = basinIndexArray[basinIndexArray==outletbasinIndex]
#print type(numelments), len(numelments)
percentBasinArea = float(len(numelments)) * 100/nDempixels
print 'Basin: ',outletbasinIndex,\
'@ : ',fastMarchingStartPointList[:,label],' #Elements ',len(numelments),\
' area ',percentBasinArea,' %'
if percentBasinArea > defaults.thresholdPercentAreaForDelineation and\
len(numelments) > Parameters.numBasinsElements:
# Get the watersheds used
basinsUsedIndexList[label]= label
# Preparing the outlets used for fast marching in ROI
#fastMarchingStartPointListFMM[:,label] = fastMarchingStartPointList[:,label]
fastMarchingStartPointListFMMx.append(fastMarchingStartPointList[0,label])
fastMarchingStartPointListFMMy.append(fastMarchingStartPointList[1,label])
# finishing Making outlets for FMM
#Closing Basin area computation
fastMarchingStartPointListFMM = np.array([fastMarchingStartPointListFMMx,\
fastMarchingStartPointListFMMy])
# Computing the local cost function
print 'Preparing to calculate cost function'
# lets normalize the curvature first
if defaults.doNormalizeCurvature ==1:
curvatureDemArrayNor = normalize(curvatureDemArray)
del curvatureDemArray
curvatureDemArray = curvatureDemArrayNor
del curvatureDemArrayNor
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.figure(defaults.figureNumber)
plt.imshow(curvatureDemArray,cmap=cm.coolwarm)
plt.title('Curvature after normalization')
plt.colorbar()
if defaults.doPlot==1:
plt.show()
print 'Curvature min: ' ,str(np.min(curvatureDemArray[~np.isnan(curvatureDemArray)])), \
' exp(min): ',str(np.exp(3*np.min(curvatureDemArray[~np.isnan(curvatureDemArray)])))
print 'Curvature max: ' ,str(np.max(curvatureDemArray[~np.isnan(curvatureDemArray)])),\
' exp(max): ',str(np.exp(3*np.max(curvatureDemArray[~np.isnan(curvatureDemArray)])))
# set all the nan's to zeros before cost function is computed
curvatureDemArray[np.isnan(curvatureDemArray)] = 0
print 'Computing cost function & geodesic distance'
# Calculate the local reciprocal cost (weight, or propagation speed in the
# eikonal equation sense). If the cost function isn't defined, default to
# old cost function.
flowArray = flowArray.T
curvatureDemArray = curvatureDemArray.T
if hasattr(defaults, 'reciprocalLocalCostFn'):
print 'Evaluating local cost func.'
reciprocalLocalCostArray = eval(defaults.reciprocalLocalCostFn)
else:
print 'Evaluating local cost func. (default)'
reciprocalLocalCostArray = flowArray + \
(flowMean*skeletonFromFlowAndCurvatureArray)\
+ (flowMean*curvatureDemArray)
del reciprocalLocalCostArray
# Forcing the evaluations
reciprocalLocalCostArray = flowArray + \
(flowMean*skeletonFromFlowAndCurvatureArray)\
+ (flowMean*curvatureDemArray)
if hasattr(defaults,'reciprocalLocalCostMinimum'):
if defaults.reciprocalLocalCostMinimum != 'nan':
reciprocalLocalCostArray[reciprocalLocalCostArray[:]\
< defaults.reciprocalLocalCostMinimum]=1.0
print '1/cost min: ', np.nanmin(reciprocalLocalCostArray[:])
print '1/cost max: ', np.nanmax(reciprocalLocalCostArray[:])
# Writing the reciprocal array
outfilepath = Parameters.geonetResultsDir
outfilename = Parameters.demFileName
outfilename = outfilename.split('.')[0]+'_costfunction.tif'
write_geotif_generic(reciprocalLocalCostArray,outfilepath,outfilename)
# Fast marching
print 'Performing fast marching'
print '# of unique basins:',np.size(np.unique(basinIndexArray))
# Now access each unique basin and get the
# outlets for it
basinIndexList = np.unique(basinIndexArray)
print 'basinIndexList:', str(basinIndexList)
print reciprocalLocalCostArray.shape
#stop
# Do fast marching for each sub basin
geodesicDistanceArray = np.zeros((basinIndexArray.shape))
geodesicDistanceArray[geodesicDistanceArray==0]=np.Inf
geodesicDistanceArray = geodesicDistanceArray.T
filteredDemArrayTr = filteredDemArray.T
basinIndexArray = basinIndexArray.T
# create a watershed outlet dictionary
outletwatersheddict = {}
defaults.figureNumber = defaults.figureNumber + 1
for i in range(0,len(fastMarchingStartPointListFMM[0])):
basinIndexList = basinIndexArray[fastMarchingStartPointListFMM[1,i],\
fastMarchingStartPointListFMM[0,i]]
print 'basin Index:',basinIndexList
print 'start point :', fastMarchingStartPointListFMM[:,i]
outletwatersheddict[basinIndexList]=fastMarchingStartPointListFMM[:,i]
maskedBasin = np.zeros((basinIndexArray.shape))
maskedBasin[basinIndexArray==basinIndexList]=1
# For the masked basin get the maximum accumulation are
# location and use that as an outlet for the basin.
maskedBasinFAC = np.zeros((basinIndexArray.shape))
maskedBasinFAC[basinIndexArray==basinIndexList]=\
flowArray[basinIndexArray==basinIndexList]
maskedBasinFAC[maskedBasinFAC==0]=np.nan
# Get the outlet of subbasin
maskedBasinFAC[np.isnan(maskedBasinFAC)]=0
# print subBasinoutletindices
# outlets locations in projection of the input dataset
outletsxx = fastMarchingStartPointList[0,i]
outletsyy = fastMarchingStartPointList[1,i]
# call the fast marching here
phi = np.nan * np.ones((reciprocalLocalCostArray.shape))
speed = np.ones((reciprocalLocalCostArray.shape))* np.nan
phi[maskedBasinFAC!=0] = 1
speed[maskedBasinFAC!=0] = reciprocalLocalCostArray[maskedBasinFAC!=0]
phi[fastMarchingStartPointListFMM[1,i],\
fastMarchingStartPointListFMM[0,i]] =-1
try:
travelTimearray = skfmm.travel_time(phi,speed, dx=1)
except IOError as e:
print 'Error in calculating skfmm travel time'
print 'Error in catchment: ',basinIndexList
# setting travel time to empty array
travelTimearray = np.nan * np.zeros((reciprocalLocalCostArray.shape))
plt.figure(defaults.figureNumber+1)
plt.imshow(speed.T,cmap=cm.coolwarm)
plt.plot(fastMarchingStartPointListFMM[1,i],\
fastMarchingStartPointListFMM[0,i],'ok')
#plt.contour(speed.T,cmap=cm.coolwarm)
plt.title('speed basin Index'+str(basinIndexList))
plt.colorbar()
plt.show()
plt.figure(defaults.figureNumber+1)
plt.imshow(phi.T,cmap=cm.coolwarm)
plt.plot(fastMarchingStartPointListFMM[1,i],\
fastMarchingStartPointListFMM[0,i],'ok')
#plt.contour(speed.T,cmap=cm.coolwarm)
plt.title('phi basin Index'+str(basinIndexList))
plt.colorbar()
plt.show()
print "I/O error({0}): {1}".format(e.errno, e.strerror)
#stop
#print travelTimearray.shape
geodesicDistanceArray[maskedBasin ==1]= travelTimearray[maskedBasin ==1]
#-----------------------------------
#-----------------------------------
# Plot the geodesic array
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(np.log10(geodesicDistanceArray.T),cmap=cm.coolwarm)
plt.contour(geodesicDistanceArray.T,140,cmap=cm.coolwarm)
plt.title('Geodesic distance array (travel time)')
plt.colorbar()
if defaults.doPlot==1:
plt.show()
print geodesicDistanceArray.shape
# Writing the geodesic distance array
outfilepath = Parameters.geonetResultsDir
outfilename = Parameters.demFileName
outfilename = outfilename.split('.')[0]+'_geodesicDistance.tif'
write_geotif_generic(geodesicDistanceArray.T,outfilepath,outfilename)
# Locating end points
print 'Locating skeleton end points'
xySkeletonSize = skeletonFromFlowAndCurvatureArray.shape
skeletonLabeledArray, skeletonNumConnectedComponentsList =\
ndimage.label(skeletonFromFlowAndCurvatureArray)
#print skeletonNumConnectedComponentsList
"""
Through the histogram of skeletonNumElementsSortedList
(skeletonNumElementsList minus the maximum value which
corresponds to the largest connected element of the skeleton) we get the
size of the smallest elements of the skeleton, which will likely
correspond to small isolated convergent areas. These elements will be
excluded from the search of end points.
"""
print 'Counting the number of elements of each connected component'
#print "ndimage.labeled_comprehension"
lbls = np.arange(1, skeletonNumConnectedComponentsList+1)
skeletonLabeledArrayNumtuple = ndimage.labeled_comprehension(skeletonFromFlowAndCurvatureArray,\
skeletonLabeledArray,\
lbls,np.count_nonzero,\
int,0)
skeletonNumElementsSortedList = np.sort(skeletonLabeledArrayNumtuple)
print np.sqrt(len(skeletonNumElementsSortedList))
histarray,skeletonNumElementsHistogramX=np.histogram(\
skeletonNumElementsSortedList[0:len(skeletonNumElementsSortedList)-1],
np.sqrt(len(skeletonNumElementsSortedList)))
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(skeletonLabeledArray.T,cmap=cm.coolwarm)
plt.title('Skeleton Labeled Array elements Array')
plt.colorbar()
if defaults.doPlot==1:
plt.show()
# Create skeleton gridded array
skeletonNumElementsGriddedArray = np.zeros(xySkeletonSize)
#"""
for i in range(0,xySkeletonSize[0]):
for j in range(0,xySkeletonSize[1]):
#Gets the watershed label for this specified cell and checked in
#subsequent if statement
basinIndex = basinIndexArray[i,j]
if skeletonLabeledArray[i, j] > 0:
skeletonNumElementsGriddedArray[i,j] = \
skeletonLabeledArrayNumtuple[skeletonLabeledArray[i,j]-1]
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(skeletonNumElementsGriddedArray.T,cmap=cm.coolwarm)
plt.title('Skeleton Num elements Array')
plt.colorbar()
if defaults.doPlot==1:
plt.show()
#"""
# Elements smaller than skeletonNumElementsThreshold are not considered in the
# skeletonEndPointsList detection
print skeletonNumElementsHistogramX
skeletonNumElementsThreshold = skeletonNumElementsHistogramX[2]
print 'skeletonNumElementsThreshold',str(skeletonNumElementsThreshold)
# Scan the array for finding the channel heads
print 'Continuing to locate skeleton endpoints'
#"""
skeletonEndPointsList = []
for i in range(0,xySkeletonSize[0]):
for j in range(0,xySkeletonSize[1]):
#print i,j
# Skip this pixel if the current point is not a labeled or if the
# number of connected skeleton elements is too small
if skeletonLabeledArray[i,j]!=0 \
and skeletonNumElementsGriddedArray[i,j]>=skeletonNumElementsThreshold:
# Define search box and ensure it fits within the DTM bounds
mx = i-1
px = xySkeletonSize[0]-i
my = j-1
py = xySkeletonSize[1]-j
xMinus = np.min([defaults.endPointSearchBoxSize, mx])
xPlus = np.min([defaults.endPointSearchBoxSize, px])
yMinus = np.min([defaults.endPointSearchBoxSize, my])
yPlus = np.min([defaults.endPointSearchBoxSize, py])
# Extract the geodesic distances geodesicDistanceArray for pixels within the search box
searchGeodesicDistanceBox = geodesicDistanceArray[i-xMinus:i+xPlus, j-yMinus:j+yPlus]
# Extract the skeleton labels for pixels within the search box
searchLabeledSkeletonBox = skeletonLabeledArray[i-xMinus:i+xPlus, j-yMinus:j+yPlus]
# Look in the search box for skeleton points with the same label
# and greater geodesic distance than the current pixel at (i,j)
# - if there are none, then add the current point as a channel head
v = searchLabeledSkeletonBox==skeletonLabeledArray[i,j]
v1 = v * searchGeodesicDistanceBox > geodesicDistanceArray[i,j]
v3 = np.where(np.any(v1==True,axis=0))
if len(v3[0])==0:
skeletonEndPointsList.append([i,j])
# For loop ends here
skeletonEndPointsListArray = np.array(skeletonEndPointsList)
xx = skeletonEndPointsListArray[0:len(skeletonEndPointsListArray),0]
yy = skeletonEndPointsListArray[0:len(skeletonEndPointsListArray),1]
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(skeletonFromFlowAndCurvatureArray.T,cmap=cm.binary)
plt.plot(xx,yy,'or')
plt.title('Skeleton Num elements Array with channel heads')
plt.colorbar()
if defaults.doPlot==1:
plt.show()
defaults.figureNumber = defaults.figureNumber + 1
plt.figure(defaults.figureNumber)
plt.imshow(np.log(geodesicDistanceArray.T),cmap=cm.coolwarm)
plt.plot(xx,yy,'or')
plt.title('Geodesic distance Array with channel heads')
plt.colorbar()
if defaults.doPlot==1:
plt.show()
# Write shapefiles of channel heads
write_channel_heads(xx,yy)
# Do compute discrete geodesics
print 'Computing discrete geodesics'
compute_discrete_geodesic_v1()
print 'Finished pyGeoNet'
def Channel_Head_Definition(skeletonFromFlowAndCurvatureArray, geodesicDistanceArray):
# Locating end points
print 'Locating skeleton end points'
structure = np.ones((3, 3))
skeletonLabeledArray, skeletonNumConnectedComponentsList = \
ndimage.label(skeletonFromFlowAndCurvatureArray,
structure=structure)
"""
Through the histogram of skeletonNumElementsSortedList
(skeletonNumElementsList minus the maximum value which
corresponds to the largest connected element of the skeleton) we get the
size of the smallest elements of the skeleton, which will likely
correspond to small isolated convergent areas. These elements will be
excluded from the search of end points.
"""
print('Counting the number of elements of each connected component')
lbls = np.arange(1, skeletonNumConnectedComponentsList + 1)
skeletonLabeledArrayNumtuple = ndimage.labeled_comprehension(skeletonFromFlowAndCurvatureArray,
skeletonLabeledArray,
lbls, np.count_nonzero,
int, 0)
skeletonNumElementsSortedList = np.sort(skeletonLabeledArrayNumtuple)
histarray, skeletonNumElementsHistogramX = np.histogram(
skeletonNumElementsSortedList[0:len(skeletonNumElementsSortedList) - 1],
int(np.floor(np.sqrt(len(skeletonNumElementsSortedList)))))
if defaults.doPlot == 1:
pyg_plt.raster_plot(skeletonLabeledArray,
'Skeleton Labeled Array elements Array')
# Create skeleton gridded array
skeleton_label_set, label_indices = np.unique(skeletonLabeledArray, return_inverse=True)
skeletonNumElementsGriddedArray = \
np.array([skeletonLabeledArrayNumtuple[x - 1] for x in skeleton_label_set])[
label_indices].reshape(skeletonLabeledArray.shape)
if defaults.doPlot == 1:
pyg_plt.raster_plot(skeletonNumElementsGriddedArray,
'Skeleton Num elements Array')
# Elements smaller than skeletonNumElementsThreshold are not considered in the
# skeletonEndPointsList detection
skeletonNumElementsThreshold = skeletonNumElementsHistogramX[2]
print('skeletonNumElementsThreshold {}'.format(str(skeletonNumElementsThreshold)))
# Scan the array for finding the channel heads
print('Continuing to locate skeleton endpoints')
skeletonEndPointsList = []
nrows = skeletonFromFlowAndCurvatureArray.shape[0]
ncols = skeletonFromFlowAndCurvatureArray.shape[1]
for i in range(nrows):
for j in range(ncols):
if skeletonLabeledArray[i, j] != 0 \
and skeletonNumElementsGriddedArray[i, j] >= skeletonNumElementsThreshold:
# Define search box and ensure it fits within the DTM bounds
my = i - 1
py = nrows - i
mx = j - 1
px = ncols - j
xMinus = np.min([defaults.endPointSearchBoxSize, mx])
xPlus = np.min([defaults.endPointSearchBoxSize, px])
yMinus = np.min([defaults.endPointSearchBoxSize, my])
yPlus = np.min([defaults.endPointSearchBoxSize, py])
# Extract the geodesic distances geodesicDistanceArray for pixels within
# the search box
searchGeodesicDistanceBox = geodesicDistanceArray[i - yMinus:i + yPlus,
j - xMinus:j + xPlus]
# Extract the skeleton labels for pixels within the search box
searchLabeledSkeletonBox = skeletonLabeledArray[i - yMinus:i + yPlus,
j - xMinus:j + xPlus]
# Look in the search box for skeleton points with the same label
# and greater geodesic distance than the current pixel at (i,j)
# - if there are none, then add the current point as a channel head
v = searchLabeledSkeletonBox == skeletonLabeledArray[i, j]
v1 = v * searchGeodesicDistanceBox > geodesicDistanceArray[i, j]
v3 = np.where(np.any(v1 == True, axis=0))
if len(v3[0]) == 0:
skeletonEndPointsList.append([i, j])
# For loop ends here
skeletonEndPointsListArray = np.transpose(skeletonEndPointsList)
if defaults.doPlot == 1:
pyg_plt.raster_point_plot(skeletonFromFlowAndCurvatureArray,
skeletonEndPointsListArray,
'Skeleton Num elements Array with channel heads',
cm.binary, 'ro')
if defaults.doPlot == 1:
pyg_plt.raster_point_plot(geodesicDistanceArray,
skeletonEndPointsListArray,
'Geodesic distance Array with channel heads',
cm.coolwarm, 'ro')
xx = skeletonEndPointsListArray[1]
yy = skeletonEndPointsListArray[0]
# Write shapefiles of channel heads
pyg_vio.write_drainage_nodes(xx, yy, "ChannelHead",
parameters.pointFileName,
parameters.pointshapefileName)
# Write raster of channel heads
channelheadArray = np.zeros((geodesicDistanceArray.shape))
channelheadArray[skeletonEndPointsListArray[0],
skeletonEndPointsListArray[1]] = 1
outfilepath = parameters.geonetResultsDir
demName = parameters.demFileName
outfilename = demName.split('.')[0] + '_channelHeads.tif'
pyg_rio.write_geotif_generic(channelheadArray,
outfilepath, outfilename)
return xx, yy
def _significant_features(
radar, field, gatefilter=None, min_size=None, size_bins=75,
size_limits=(0, 300), structure=None, remove_size_field=True,
fill_value=None, size_field=None, debug=False, verbose=False):
"""
"""
# Parse fill value
if fill_value is None:
fill_value = get_fillvalue()
# Parse field names
if size_field is None:
size_field = '{}_feature_size'.format(field)
# Parse gate filter
if gatefilter is None:
gatefilter = GateFilter(radar, exclude_based=False)
# Parse binary structuring element
if structure is None:
structure = ndimage.generate_binary_structure(2, 1)
# Initialize echo feature size array
size_data = np.zeros_like(
radar.fields[field]['data'], subok=False, dtype=np.int32)
# Loop over all sweeps
feature_sizes = []
for sweep in radar.iter_slice():
# Parse radar sweep data and define only valid gates
is_valid_gate = ~radar.fields[field]['data'][sweep].mask
# Label the connected features in radar sweep data and create index
# array which defines each unique label (feature)
labels, nlabels = ndimage.label(
is_valid_gate, structure=structure, output=None)
index = np.arange(1, nlabels + 1, 1)
if debug:
print 'Number of unique features for {}: {}'.format(sweep, nlabels)
# Compute the size (in radar gates) of each echo feature
# Check for case where no echo features are found, e.g., no data in
# sweep
if nlabels > 0:
sweep_sizes = ndimage.labeled_comprehension(
is_valid_gate, labels, index, np.count_nonzero, np.int32, 0)
feature_sizes.append(sweep_sizes)
# Set each label (feature) to its total size (in radar gates)
for label, size in zip(index, sweep_sizes):
size_data[sweep][labels == label] = size
# Stack sweep echo feature sizes
feature_sizes = np.hstack(feature_sizes)
# Compute histogram of echo feature sizes, bin centers and bin
# width
counts, bin_edges = np.histogram(
feature_sizes, bins=size_bins, range=size_limits, normed=False,
weights=None, density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Bin width: {} gate(s)'.format(bin_width)
# Compute the peak of the echo feature size distribution
# We expect the peak of the echo feature size distribution to be close to 1
# radar gate
peak_size = bin_centers[counts.argmax()] - bin_width / 2.0
if debug:
print 'Feature size at peak: {} gate(s)'.format(peak_size)
# Determine the first instance when the count (sample size) for an echo
# feature size bin reaches 0 after the distribution peak
# This will define the minimum echo feature size
is_zero_size = np.logical_and(
bin_centers > peak_size, np.isclose(counts, 0, atol=1.0e-5))
min_size = bin_centers[is_zero_size].min() - bin_width / 2.0
if debug:
_range = [0.0, min_size]
print 'Insignificant feature size range: {} gates'.format(_range)
# Mask invalid feature sizes, e.g., zero-size features
size_data = np.ma.masked_equal(size_data, 0, copy=False)
size_data.set_fill_value(fill_value)
# Add echo feature size field to radar
size_dict = {
'data': size_data.astype(np.int32),
'standard_name': size_field,
'long_name': '',
'_FillValue': size_data.fill_value,
'units': 'unitless',
}
radar.add_field(size_field, size_dict, replace_existing=True)
# Update gate filter
gatefilter.include_above(size_field, min_size, op='and', inclusive=False)
# Remove eacho feature size field
if remove_size_field:
radar.fields.pop(size_field, None)
return gatefilter
def nema_2008_small_animal_pet_rois(vol, voxsize, lp_voxel = 'max', rod_th = 0.15,
phantom = 'standard'):
""" generate a label volume indicating the ROIs needed in the analysis of the
NEMA small animal PET IQ phantom
Parameters
----------
vol : 3D numpy float array
containing the image
voxsize : 3 element 1D numpy array
containing the voxel size
lp_voxel: string, optional
method of how to compute the pixel used to draw the line profiles
in the rods. 'max' means the maximum voxels in the summed 2D image.
anything else means use the center of mass.
rod_th : float, optional
threshold to find the rod in the summed 2D image relative to the
mean of the big uniform region
phantom : string
phantom version ('standard' or 'mini')
Returns
-------
a 3D integer numpy array
encoding the following ROIs:
1 ... ROI of the big uniform region
2 ... first cold insert
3 ... second cold insert
4 ... central line profile in 5mm rod
5 ... central line profile in 4mm rod
6 ... central line profile in 3mm rod
7 ... central line profile in 2mm rod
8 ... central line profile in 1mm rod
Note
----
The rod ROIs in the summed 2D image are found by thresholding.
If the activity in the small rods is too low, they might be missed.
"""
roi_vol = np.zeros(vol.shape, dtype = np.uint)
# calculate the summed z profile to place the ROIs
zprof = vol.sum(0).sum(0)
zprof_grad = np.gradient(zprof)
zprof_grad[np.abs(zprof_grad) < 0.13*np.abs(zprof_grad).max()] = 0
rising_edges = argrelextrema(zprof_grad, np.greater, order = 10)[0]
falling_edges = argrelextrema(zprof_grad, np.less, order = 10)[0]
# if we only have 2 falling edges because the volume is cropped, we add the last slices as
# falling edge
if falling_edges.shape[0] == 2:
falling_edges = np.concatenate([falling_edges,[vol.shape[2]]])
# define and analyze the big uniform ROI
uni_region_start_slice = rising_edges[1]
uni_region_end_slice = falling_edges[1]
uni_region_center_slice = 0.5*(uni_region_start_slice + uni_region_end_slice)
uni_roi_start_slice = int(np.floor(uni_region_center_slice - 5./voxsize[2]))
uni_roi_end_slice = int(np.ceil(uni_region_center_slice + 5./voxsize[2]))
uni_com = np.array(center_of_mass(vol[:,:,uni_roi_start_slice:(uni_roi_end_slice+1)]))
x0 = (np.arange(vol.shape[0]) - uni_com[0]) * voxsize[0]
x1 = (np.arange(vol.shape[1]) - uni_com[1]) * voxsize[1]
x2 = (np.arange(vol.shape[2]) - uni_com[2]) * voxsize[2]
X0,X1,X2 = np.meshgrid(x0,x1,x2,indexing='ij')
RHO = np.sqrt(X0**2 + X1**2)
uni_mask = np.zeros(vol.shape, dtype = np.uint)
if phantom == 'standard':
uni_mask[RHO <= 11.25] = 1
elif phantom == 'mini':
uni_mask[RHO <= 6.25] = 1
uni_mask[:,:,:uni_roi_start_slice] = 0
uni_mask[:,:,(uni_roi_end_slice+1):] = 0
uni_inds = np.where(uni_mask == 1)
roi_vol[uni_inds] = 1
# define and analyze the two cold ROIs
insert_region_start_slice = falling_edges[1]
insert_region_end_slice = falling_edges[2]
insert_region_center_slice = 0.5*(insert_region_start_slice + insert_region_end_slice)
insert_roi_start_slice = int(np.floor(insert_region_center_slice - 3.75/voxsize[2]))
insert_roi_end_slice = int(np.ceil(insert_region_center_slice + 3.75/voxsize[2]))
# sum the insert slices and subtract them from the max to find the two cold inserts
sum_insert_img = vol[:,:,insert_roi_start_slice:(insert_roi_end_slice+1)].mean(2)
ref = np.percentile(sum_insert_img,99)
if phantom == 'standard':
insert_label_img, nlab_insert = label(sum_insert_img <= 0.5*ref)
elif phantom == 'mini':
# reset pixels outside the phantom, since inserts sometimes leak into background
tmp_inds = RHO[:,:,0] > 9
sum_insert_img[tmp_inds] = ref
insert_label_img, nlab_insert = label(binary_erosion(sum_insert_img <= 0.5*ref))
# add backgroud low activity ROI to be compliant with standard phantom
insert_label_img[tmp_inds] = 3
nlab_insert += 1
insert_labels = np.arange(1,nlab_insert+1)
# sort the labels according to volume
npix_insert = labeled_comprehension(sum_insert_img, insert_label_img, insert_labels, len, int, 0)
insert_sort_inds = npix_insert.argsort()[::-1]
insert_labels = insert_labels[insert_sort_inds]
npix_insert = npix_insert[insert_sort_inds]
for i_insert in [1,2]:
tmp = insert_label_img.copy()
tmp[insert_label_img != insert_labels[i_insert]] = 0
com_pixel = np.round(np.array(center_of_mass(tmp)))
x0 = (np.arange(vol.shape[0]) - com_pixel[0]) * voxsize[0]
x1 = (np.arange(vol.shape[1]) - com_pixel[1]) * voxsize[1]
x2 = (np.arange(vol.shape[2])) * voxsize[2]
X0,X1,X2 = np.meshgrid(x0,x1,x2,indexing='ij')
RHO = np.sqrt(X0**2 + X1**2)
insert_mask = np.zeros(vol.shape, dtype = np.uint)
insert_mask[RHO <= 2] = 1
insert_mask[:,:,:insert_roi_start_slice] = 0
insert_mask[:,:,(insert_roi_end_slice+1):] = 0
insert_inds = np.where(insert_mask == 1)
roi_vol[insert_inds] = i_insert + 1
# find the rod z slices
rod_start_slice = falling_edges[0]
rod_end_slice = rising_edges[1]
rod_center = 0.5*(rod_start_slice + rod_end_slice)
rod_roi_start_slice = int(np.floor(rod_center - 5./voxsize[2]))
rod_roi_end_slice = int(np.ceil(rod_center + 5./voxsize[2]))
# sum the rod slices
sum_img = vol[:,:,rod_roi_start_slice:(rod_roi_end_slice+1)].mean(2)
# label the summed image
label_img, nlab = label(sum_img > rod_th*sum_img.max())
labels = np.arange(1,nlab+1)
# sort the labels according to volume
npix = labeled_comprehension(sum_img, label_img, labels, len, int, 0)
sort_inds = npix.argsort()[::-1]
labels = labels[sort_inds]
npix = npix[sort_inds]
# find the center for the line profiles
for i, lab in enumerate(labels):
rod_sum_img = sum_img.copy()
rod_sum_img[label_img != lab] = 0
if lp_voxel == 'max':
central_pixel = np.unravel_index(rod_sum_img.argmax(),rod_sum_img.shape)
else:
central_pixel = np.round(np.array(center_of_mass(rod_sum_img))).astype(np.int)
roi_vol[central_pixel[0],central_pixel[1],rod_roi_start_slice:(rod_roi_end_slice+1)] = i + 4
#-------------------------------------------------------
# if we only have 4 labels (rods), we find the last (smallest) one based on symmetries
if nlab == 4:
roi_img = roi_vol[...,rod_roi_start_slice]
com = center_of_mass(roi_vol == 1)
x0 = (np.arange(sum_img.shape[0]) - com[0]) * voxsize[0]
x1 = (np.arange(sum_img.shape[1]) - com[1]) * voxsize[1]
X0,X1 = np.meshgrid(x0, x1, indexing = 'ij')
RHO = np.sqrt(X0**2 + X1**2)
PHI = np.arctan2(X1,X0)
rod_phis = np.array([PHI[roi_img == x][0] for x in np.arange(4,nlab+4)])
PHI = ((PHI - rod_phis[3]) % (2*np.pi)) - np.pi
rod_phis = ((rod_phis - rod_phis[3]) % (2*np.pi)) - np.pi
missing_phi = ((rod_phis[3] - rod_phis[2]) % (2*np.pi)) - np.pi
mask = np.logical_and(np.abs(PHI - missing_phi) < 0.25, np.abs(RHO - 6.4) < 2)
central_pixel = np.unravel_index(np.argmax(sum_img*mask), sum_img.shape)
roi_vol[central_pixel[0],central_pixel[1],rod_roi_start_slice:(rod_roi_end_slice+1)] = 8
nlab += 1
#-------------------------------------------------------
return roi_vol
def fit_nema_2008_cylinder_profiles(vol,
voxsize,
Rrod_init = [2.5,2,1.5,1,0.5],
fwhm_init = 1.5,
S_init = 1,
fix_S = True,
fix_R = False,
fix_fwhm = False,
nrods = 4,
phantom = 'standard'):
""" Fit the radial profiles of the rods in a nema 2008 small animal PET phantom
Parameters
----------
vol : 3D numpy float array
containing the image
voxsize : 3 element 1D numpy array
containing the voxel size
Rrod_init : list or 1D numpy array of floats, optional
containing the initial values of the rod radii
S_init, fwhm_init: float, optional
initial values for the signal and the FWHM in the fit
fix_S, fix_R, fix_fwhm : bool, optional
whether to keep the initial values of signal, radius and FWHM fixed during the fix
nrods: int, optional
number of rods to fit
phantom : string
phantom version ('standard' or 'mini')
Returns
-------
a list of lmfit fit results
Note
----
The axial direction should be the right most direction in the 3D numpy array.
The slices containing the rods are found automatically and summed.
In the summed image, all rods (disks) are segmented followed by a fit
of the radial profile.
"""
roi_vol = nema_2008_small_animal_pet_rois(vol, voxsize, phantom = phantom)
rod_bbox = find_objects(roi_vol==4)
# find the rods in the summed image
sum_img = vol[:,:,rod_bbox[0][2].start:rod_bbox[0][2].stop].mean(2)
label_img, nlab = label(sum_img > 0.1*sum_img.max())
labels = np.arange(1,nlab+1)
# sort the labels according to volume
npix = labeled_comprehension(sum_img, label_img, labels, len, int, 0)
sort_inds = npix.argsort()[::-1]
labels = labels[sort_inds]
npix = npix[sort_inds]
#----------------------------------------------------------------------
ncols = 2
nrows = int(np.ceil(nrods/ncols))
fig, ax = py.subplots(nrows,ncols,figsize = (12,7*nrows/2), sharey = True, sharex = True)
retval = []
for irod in range(nrods):
rod_bbox = find_objects(label_img == labels[irod])
rod_bbox = [(slice(rod_bbox[0][0].start - 2,rod_bbox[0][0].stop + 2),
slice(rod_bbox[0][1].start - 2,rod_bbox[0][1].stop + 2))]
rod_img = sum_img[rod_bbox[0]]
com = np.array(center_of_mass(rod_img))
x0 = (np.arange(rod_img.shape[0]) - com[0]) * voxsize[0]
x1 = (np.arange(rod_img.shape[1]) - com[1]) * voxsize[1]
X0, X1 = np.meshgrid(x0, x1, indexing = 'ij')
RHO = np.sqrt(X0**2 + X1**2)
rho = RHO.flatten()
signal = rod_img.flatten()
# sort the values according to rho
sort_inds = rho.argsort()
rho = rho[sort_inds]
signal = signal[sort_inds]
pmodel = Model(cylinder_profile)
params = pmodel.make_params(S = S_init, R = Rrod_init[irod], fwhm = fwhm_init)
if fix_S:
params['S'].vary = False
if fix_R:
params['R'].vary = False
if fix_fwhm:
params['fwhm'].vary = False
fitres = pmodel.fit(signal, r = rho, params = params)
retval.append(fitres)
fit_report = fitres.fit_report()
iplot = np.unravel_index(irod, ax.shape)
ax[iplot].plot(rho,signal,'k.')
rfit = np.linspace(0,rho.max(),100)
ax[iplot].plot(rfit,fitres.eval(r = rfit),'r-')
ax[iplot].text(0.99, 0.99, fit_report, fontsize = 6, transform = ax[iplot].transAxes,
verticalalignment='top', horizontalalignment = 'right',
backgroundcolor = 'white', bbox = {'pad':0, 'facecolor':'white','lw':0})
ax[iplot].grid()
for axx in ax[-1,:]: axx.set_xlabel('R (mm)')
for axx in ax[:,0]: axx.set_ylabel('signal')
fig.tight_layout()
fig.show()
return retval
def _binary_significant_features(
radar, binary_field, size_bins=75, size_limits=(0, 300),
structure=None, debug=False, verbose=False):
"""
Objectively determine the minimum echo feature size (in radar gates) and
remove features smaller than this from the specified radar field. This
function can be used to objectively remove salt and pepper noise from
binary (mask) radar fields. Unexpected results may occur if the specified
radar field is not a binary field.
Parameters
----------
radar : Radar
Radar object containing the specified binary field.
binary_field : str
The binary radar field that will have insignificant echo features
removed.
size_bins : int, optional
Number of bins used to bin echo feature sizes and thus define its
distribution.
size_limits : list or tuple, optional
Limits of the echo feature size distribution. The upper limit needs to
be large enough to include the minimum feature size. The size bin width
is defined by both the size_bins and size_limits parameters.
structure : array_like, optional
Binary structuring element used to define connected features. The
default structuring element has a squared connectivity equal to one.
debug, verbose : bool, optional
True to print debugging and progress information, respectively, False
to suppress.
"""
# Parse binary structuring element
if structure is None:
structure = ndimage.generate_binary_structure(2, 1)
# Parse feature size arrays
size_data = np.zeros_like(
radar.fields[binary_field]['data'], dtype=np.int32)
feature_sizes = []
for i, sweep in enumerate(radar.iter_slice()):
# Parse radar sweep data
# Non-zero elements of the array form the subset to be dilated
data = radar.fields[binary_field]['data'][sweep]
is_valid_gate = np.ma.filled(data, 0)
# Label the connected features in the sweep data and create index
# array which defines each unique labeled feature
labels, nlabels = ndimage.label(
is_valid_gate, structure=structure, output=None)
index = np.arange(1, nlabels + 1, 1)
if debug:
print 'Unique features in sweep {}: {}'.format(i, nlabels)
# Compute the size in radar gates of each labeled feature
sweep_sizes = ndimage.labeled_comprehension(
is_valid_gate, labels, index, np.count_nonzero, np.int32, 0)
feature_sizes.append(sweep_sizes)
# Set each labeled feature to its total size in radar gates
for label, size in zip(index, sweep_sizes):
size_data[sweep][labels == label] = size
feature_sizes = np.hstack(feature_sizes)
# Bin and count occurrences of labeled feature sizes
# Compute bin centers and bin width
counts, bin_edges = np.histogram(
feature_sizes, bins=size_bins, range=size_limits, normed=False,
weights=None, density=False)
bin_centers = bin_edges[:-1] + np.diff(bin_edges) / 2.0
bin_width = np.diff(bin_edges).mean()
if debug:
print 'Feature size bin width: {} gate(s)'.format(bin_width)
# Compute the peak of the labeled feature size distribution
# We expect the peak of this distribution to be close to 1 radar gate
peak_size = bin_centers[counts.argmax()] - bin_width / 2.0
if debug:
print 'Feature size at peak: {} gate(s)'.format(peak_size)
# Determine the first instance when the count (sample size) of a feature
# size bin reaches 0 in the right side of the feature size distribution
# This will define the minimum feature size
is_zero_size = np.logical_and(
bin_centers > peak_size, np.isclose(counts, 0, atol=1.0e-5))
min_size = bin_centers[is_zero_size].min() - bin_width / 2.0
if debug:
_range = [0.0, min_size]
print 'Insignificant feature size range: {} gates'.format(_range)
# Remove insignificant features from the binary radar field
radar.fields[binary_field]['data'][size_data < min_size] = 0
return
def detection_by_scene_segmentation():
'''Detection of moving object by using Distortion field of centers of mass
of background objects'''
if co.counters.im_number == 0:
co.segclass.needs_segmentation = 1
if not co.segclass.exists_previous_segmentation:
print 'No existing previous segmentation. The initialisation will delay...'
else:
print 'Checking similarity..'
old_im = co.meas.background
check_if_segmented = np.sqrt(
np.sum(
(co.data.depth_im[co.meas.trusty_pixels.astype(
bool)].astype(float) -
old_im[co.meas.trusty_pixels.astype(bool)].astype(float)
)**2))
print 'Euclidean Distance of old and new background is ' +\
str(check_if_segmented)
print 'Minimum Distance to approve previous segmentation is ' +\
str(co.CONST['similar_bg_min_dist'])
if check_if_segmented < co.CONST['similar_bg_min_dist']:
print 'No need to segment again'
co.segclass.needs_segmentation = 0
else:
print 'Segmentation is needed'
if co.segclass.needs_segmentation and co.counters.im_number >= 1:
if co.counters.im_number == (
co.CONST['framerate'] * co.CONST['calib_secs'] - 1):
co.segclass.flush_previous_segmentation()
co.segclass.nz_objects.image = np.zeros_like(co.data.depth_im) - 1
co.segclass.z_objects.image = np.zeros_like(co.data.depth_im) - 1
levels_num = 8
levels = np.linspace(
np.min(co.data.depth_im[co.data.depth_im > 0]),
np.max(co.data.depth_im), levels_num)
co.segclass.segment_values = np.zeros_like(co.data.depth_im)
for count in range(levels_num - 1):
co.segclass.segment_values[
(co.data.depth_im >= levels[count]) *
(co.data.depth_im <= levels[count + 1])] = count + 1
elif co.counters.im_number == (co.CONST['framerate'] *
co.CONST['calib_secs']):
co.segclass.nz_objects.count = -1
co.segclass.z_objects.count = -1
co.segclass.segment_values = co.segclass.segment_values * co.meas.trusty_pixels
for val in np.unique(co.segclass.segment_values):
objs = np.ones_like(co.data.depth_im) * \
(val == co.segclass.segment_values)
labeled, nr_objects =\
ndimage.label(objs * co.edges.calib_frame)
lbls = np.arange(1, nr_objects + 1)
if val > 0:
ndimage.labeled_comprehension(
objs, labeled, lbls, co.segclass.nz_objects.process,
float, 0, True)
else:
ndimage.labeled_comprehension(
objs, labeled, lbls, co.segclass.z_objects.process,
float, 0, True)
for (points, pixsize, xsize,
ysize) in (co.segclass.nz_objects.untrusty +
co.segclass.z_objects.untrusty):
co.segclass.z_objects.count += 1
co.segclass.z_objects.image[tuple(
points)] = co.segclass.z_objects.count
co.segclass.z_objects.pixsize.append(pixsize)
co.segclass.z_objects.xsize.append(xsize)
co.segclass.z_objects.ysize.append(ysize)
print 'Found or partitioned',\
co.segclass.nz_objects.count +\
co.segclass.z_objects.count + 2, 'background objects'
co.segclass.needs_segmentation = 0
with open(co.CONST['segmentation_data'] + '.pkl', 'wb') as output:
pickle.dump((co.segclass, co.meas), output, -1)
print 'Saved segmentation data for future use.'
elif (not co.segclass.needs_segmentation) and co.counters.im_number >= 2:
if not co.segclass.initialised_centers:
try:
co.segclass.nz_objects.find_object_center(1)
except BaseException as err:
print 'Centers initialisation Exception'
raise err
try:
co.segclass.z_objects.find_object_center(0)
except BaseException as err:
print 'Centers initialisation Exception'
raise err
co.segclass.initialise_neighborhoods()
co.segclass.initialised_centers = 1
else:
try:
co.segclass.nz_objects.find_object_center(1)
except BaseException as err:
print 'Centers calculation Exception'
raise err
if co.segclass.nz_objects.center.size > 0:
co.segclass.nz_objects.find_centers_displacement()
co.meas.found_objects_mask = co.segclass.find_objects()
points_on_im = co.data.depth3d.copy()
# points_on_im[np.sum(points_on_im,axis=2)==0,:]=np.array([1,0,1])
for calc, point1, point2 in zip(
co.segclass.nz_objects.centers_to_calculate,
co.segclass.nz_objects.initial_center,
co.segclass.nz_objects.center):
if point1[0] != -1:
if calc:
cv2.arrowedLine(points_on_im, (point1[1], point1[0]),
(point2[1], point2[0]), [0, 1, 0], 2,
1)
else:
cv2.arrowedLine(points_on_im, (point1[1], point1[0]),
(point2[1], point2[0]), [0, 0, 1], 2,
1)
struct_el = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
tuple(2 * [5]))
co.meas.found_objects_mask = cv2.morphologyEx(
co.meas.found_objects_mask.astype(np.uint8), cv2.MORPH_CLOSE,
struct_el)
struct_el = cv2.getStructuringElement(cv2.MORPH_ELLIPSE,
tuple(2 * [10]))
co.meas.found_objects_mask = cv2.morphologyEx(
co.meas.found_objects_mask.astype(np.uint8), cv2.MORPH_OPEN,
struct_el)
hand_patch, hand_patch_pos = hsa.main_process(
co.meas.found_objects_mask.astype(np.uint8),
co.meas.all_positions, 1)
co.meas.hand_patch = hand_patch
co.meas.hand_patch_pos = hand_patch_pos
if len(co.im_results.images) == 1:
co.im_results.images.append(
(255 * co.meas.found_objects_mask).astype(np.uint8))
co.im_results.images.append(points_on_im)
# elif len(co.im_results.images)==2:
# co.im_results.images[1][co.im_results.images[1]==0]=(255*points_on_im).astype(np.uint8)[co.im_results.images[1]==0]
# if hand_points.shape[1]>1:
# points_on_im[tuple(hand_points.T)]=[1,0,0]
# co.im_results.images.append(points_on_im)
return 1
