def read_multiband(objid, objdir, band=('g', 'r', 'z'), pixscale=0.262):
"""Read the multi-band images, construct the residual image, and then create a
masked array from the corresponding inverse variances image. Finally,
convert to surface brightness by dividing by the pixel area.
"""
import fitsio
from scipy.ndimage.morphology import binary_dilation
data = dict()
for filt in band:
image = fitsio.read(
os.path.join(objdir, '{}-image-{}.fits.fz'.format(objid, filt)))
model = fitsio.read(
os.path.join(objdir, '{}-model-{}.fits.fz'.format(objid, filt)))
invvar = fitsio.read(
os.path.join(objdir, '{}-invvar-{}.fits.fz'.format(objid, filt)))
# Mask pixels with ivar<=0. Also build an object mask from the model
# image, to handle systematic residuals.
sig1 = 1.0 / np.sqrt(np.median(invvar[invvar > 0]))
mask = (invvar <= 0) * 1 # 1=bad, 0=good
mask = np.logical_or(mask, (model > (2 * sig1)) * 1)
mask = binary_dilation(mask, iterations=5) * 1
data[filt] = (image - model) / pixscale**2 # [nanomaggies/arcsec**2]
data['{}_mask'.format(filt)] = mask == 0 # 1->bad
data['{}_masked'.format(filt)] = ma.masked_array(
data[filt], ~data['{}_mask'.format(filt)]) # 0->bad
ma.set_fill_value(data['{}_masked'.format(filt)], 0)
return data
def rm_centre_colomn(ssh_model, ssh_obs, lon, lat, x_ac, x_al, time_sec):
fill_value = ssh_model.fill_value
enssize = ssh_model.shape[0]
dimtime = ssh_model.shape[1]
dimnc = ssh_model.shape[2]-1
lon = np.delete(lon, (60), axis=1)
lat = np.delete(lat, (60), axis=1)
x_ac = np.delete(x_ac, (60))
ssh_model_rmd = np.zeros([enssize,dimtime,dimnc]) # [layer, time, nC]
ssh_obs_rmd = np.zeros([enssize,dimtime,dimnc])
for k in range (0,enssize):
ssh_model_rmd[k,:,:] = np.delete(ssh_model[k,:,:], (60), axis=1)
ssh_obs_rmd[k,:,:] = np.delete(ssh_obs[k,:,:], (60), axis=1)
ssh_model_rmd = ma.masked_where(ssh_model_rmd==fill_value,ssh_model_rmd)
ma.set_fill_value(ssh_model_rmd, fill_value)
ssh_obs_rmd = ma.masked_where(ssh_obs_rmd==fill_value,ssh_obs_rmd)
ma.set_fill_value(ssh_obs_rmd, fill_value)
return ssh_model_rmd, ssh_obs_rmd, lon, lat, x_ac, x_al, time_sec
def func(V, Y):
count_each_feature_value = Counter(V)
feature_value = np.unique(V)
num_value_feature = len(feature_value)
subtotal = 0
count = 0
for ele in feature_value: # each value in this feature
weight = count_each_feature_value[feature_value[count]] / float(
sampleNum)
after_mask = ma.masked_not_equal(V, ele)
ma.set_fill_value(after_mask, 0)
after_mask = after_mask.filled()
mat_each_value_feature = np.dot(after_mask, Y) #
tmp = mat_each_value_feature
tmp_sum = np.sum(mat_each_value_feature)
mat_each_value_feature = np.dot(mat_each_value_feature, 1.0 / tmp_sum)
current = 0
#after_compress = ma.masked_equal(mat_each_value_feature, 0)
#after_compress = after_compress.compressed()
#t5 = time.clock()
#current = np.dot(mat_each_value_feature, np.log2(after_compress))
#t6 = time.clock()
#mat_each_value_feature = mat_each_value_feature[-mat_each_value_feature.mask]
#ma.set_fill_value(mat_each_value_feature,1)
#mat_each_value_feature = mat_each_value_feature.filled()
#current = np.dot(mat_each_value_feature, np.log2(mat_each_value_feature))
#current = 0
for k in mat_each_value_feature:
if k == 0:
continue
current += k * math.log(k, 2)
subtotal += current * weight
count += 1
subtotal = -subtotal
return subtotal
def func(V, Y): count_each_feature_value = Counter(V) feature_value = np.unique(V) num_value_feature = len(feature_value) subtotal = 0 count = 0 for ele in feature_value:# each value in this feature weight = count_each_feature_value[feature_value[count]] / float(sampleNum) after_mask = ma.masked_not_equal(V,ele) ma.set_fill_value(after_mask,0) after_mask = after_mask.filled() mat_each_value_feature = np.dot(after_mask, Y) # tmp = mat_each_value_feature tmp_sum = np.sum(mat_each_value_feature) mat_each_value_feature = np.dot(mat_each_value_feature, 1.0 / tmp_sum) current = 0 #after_compress = ma.masked_equal(mat_each_value_feature, 0) #after_compress = after_compress.compressed() #t5 = time.clock() #current = np.dot(mat_each_value_feature, np.log2(after_compress)) #t6 = time.clock() #mat_each_value_feature = mat_each_value_feature[-mat_each_value_feature.mask] #ma.set_fill_value(mat_each_value_feature,1) #mat_each_value_feature = mat_each_value_feature.filled() #current = np.dot(mat_each_value_feature, np.log2(mat_each_value_feature)) #current = 0 for k in mat_each_value_feature: if k == 0: continue; current += k * math.log(k,2) subtotal += current * weight count += 1 subtotal = - subtotal return subtotal
def resample_from_array(
in_raster=None, in_affine=None, out_tile=None, resampling="nearest",
nodataval=0
):
"""
Extract and resample from array to target tile.
Parameters
----------
in_raster : array
in_affine : ``Affine``
out_tile : ``BufferedTile``
resampling : string
one of rasterio's resampling methods (default: nearest)
nodataval : integer or float
raster nodata value (default: 0)
Returns
-------
resampled array : array
"""
if isinstance(in_raster, ma.MaskedArray):
pass
if isinstance(in_raster, np.ndarray):
in_raster = ma.MaskedArray(in_raster, mask=in_raster == nodataval)
elif isinstance(in_raster, ReferencedRaster):
in_affine = in_raster.affine
in_raster = in_raster.data
elif isinstance(in_raster, tuple):
in_raster = ma.MaskedArray(
data=np.stack(in_raster),
mask=np.stack([
band.mask
if isinstance(band, ma.masked_array)
else np.where(band == nodataval, True, False)
for band in in_raster
]),
fill_value=nodataval
)
else:
raise TypeError("wrong input data type: %s" % type(in_raster))
if in_raster.ndim == 2:
in_raster = ma.expand_dims(in_raster, axis=0)
elif in_raster.ndim == 3:
pass
else:
raise TypeError("input array must have 2 or 3 dimensions")
if in_raster.fill_value != nodataval:
ma.set_fill_value(in_raster, nodataval)
out_shape = (in_raster.shape[0], ) + out_tile.shape
dst_data = np.empty(out_shape, in_raster.dtype)
in_raster = ma.masked_array(
data=in_raster.filled(), mask=in_raster.mask, fill_value=nodataval)
reproject(
in_raster, dst_data, src_transform=in_affine, src_crs=out_tile.crs,
dst_transform=out_tile.affine, dst_crs=out_tile.crs,
resampling=Resampling[resampling])
return ma.MaskedArray(dst_data, mask=dst_data == nodataval)
def output_indices(bands, indices, meta, output_path):
for name, formula in indices.items():
image = formula(bands)
image = utils.image_histogram_equalization(image)
ma.set_fill_value(image, 0)
store_as_single_color_geotiff(image, f"{output_path}/{name} (I).tif",
meta)
def read_ens(ensemble):
"""Read arrays from an ensemble of netcdf files.
Parameters:
----------
ensemble: input ensemble name
Returns:
-------
ssh_model, ssh_obs, lat, lon, x_ac, time_sec: arrays
"""
# function reads an ensemble and returns a matrix containing the ssh at all grid points of all members #
### verify name of ensemble ###
#regexEnsname = re.compile(r'[\w]\d{4,4}.nc.bas') #<name><nnnn>.nc.bas
ensname = ensemble.split('/')[-1]
#ensdir = ensemble.split(ensname)[0]
#match = regexEnsname.search(ensname)
#if match == None:
#raise NameError("Ensemble not correctly named - see http://pp.ige-grenoble.fr/pageperso/brankarj/SESAM/ for naming input files")
### extract number of members of ensemble ###
enssizelist = re.findall(r'\d{4,4}', ensname)
enssize = int(enssizelist[0])
### get grid dimension ###
membername1 = ensemble+'/vctgridSWOT0001.nc'
"""eventually remove _denoised"""
with xr.open_dataset(membername1,mask_and_scale=False) as dsmember1:
time = dsmember1.time.size
nc = dsmember1.nC.size
lat = dsmember1.lat[:,:].values
lon = dsmember1.lon[:,:].values
x_ac = dsmember1.x_ac[:].values
x_al = dsmember1.x_al[:].values
time_sec = dsmember1.time_sec[:].values
# first member of ensemble is later used to mask all members
fill_value = dsmember1.ssh_model._FillValue
### load data ###
ssh_model = np.zeros([enssize,time,nc]) # [layer, time, nC]
ssh_obs = np.zeros([enssize,time,nc])
for k in range (1,enssize+1):
membername = (ensemble+'/vctgridSWOT{:04d}.nc').format(k)
with xr.open_dataset(membername,mask_and_scale=False) as ds:
buf_model = ds.ssh_model[:,:]
buf_obs = ds.ssh_obs[:,:]
ssh_model[k-1,:,:] = buf_model
ssh_obs[k-1,:,:] = buf_obs
ssh_obs = ma.masked_where(ssh_obs==fill_value, ssh_obs)
ssh_model = ma.masked_where(ssh_model==fill_value, ssh_model)
ma.set_fill_value(ssh_obs, fill_value)
ma.set_fill_value(ssh_model, fill_value)
### return data ###
return ssh_model, ssh_obs, lat, lon, x_ac, x_al, time_sec
def constructY(Y, width):
Y = ma.masked_equal(Y, value=10, copy=True)
ma.set_fill_value(Y, fill_value=0)
Y = Y.filled()
Ynew = np.zeros((len(Y), width))
for i in range(len(Y)):
Ynew[i][Y[i]] = 1
return Ynew
def output_composite(bands, indices, meta, output_path):
for name, formula in indices.items():
image = [formula["R"](bands), formula["G"](bands), formula["B"](bands)]
image = [utils.image_histogram_equalization(c) for c in image]
for c in image:
ma.set_fill_value(c, 0)
store_as_rgb_geotiff(image, f"{output_path}/{name} (C).tif", meta)
def projectToLatLong(ifile, outfile, img_bnd_coords, proj4string):
####################################
# Inputs
# For details see http://fcm7/projects/SPS/browser/SPS/trunk/data/products/MSG_Products.nl
ulx_ll, uly_ll, lrx_ll, lry_ll = img_bnd_coords # Bounding coordinates in latlon, although the the image is in mercator projection!!!
# epsg_code = 3395 # Mercator projection code of the imagery
####################################
# For testing
# ifile = '/data/users/hadhy/HyVic/Obs/OLR_noborders/EIAM50_201901210230.png'
# Open the image file
ds = gdal.Open(ifile)
band = ds.GetRasterBand(1)
arr = band.ReadAsArray()
# Mask out pixel values with no data
arr = ma.masked_less(arr, 4)
arr = ma.masked_greater(arr, 254)
# Apply equation to retrieve brightness temperatures. See Chamberlain et al. 2013 https://rmets.onlinelibrary.wiley.com/doi/full/10.1002/met.1403
arrbt = (-0.44 * (arr - 4.)) + 308.
# Fill masked values with -9999 (because the Geotiff format doesn't accept masked arrays)
ma.set_fill_value(arrbt, -9999)
arrbt = arrbt.filled()
# Set some image info ...
## Gets the bounding coordinates in Mercator projection
ulx, uly = latlon2projected(ulx_ll, uly_ll, proj4string)
lrx, lry = latlon2projected(lrx_ll, lry_ll, proj4string)
## Calculate geotransform object
nx, ny = [ds.RasterXSize, ds.RasterYSize]
## NB: The coordinates start in the top left corner (usually), so
## xDist needs to be positive, and yDist negative
xDist = (lrx - ulx) / nx if lrx > ulx else -1 * (lrx - ulx) / nx
yDist = -1 * (uly - lry) / ny if uly > lry else (uly - lry) / ny
rtnX, rtnY = [0, 0]
gt = [ulx, xDist, rtnX, uly, rtnY, yDist]
## Projection information
srs = osr.SpatialReference()
srs.ImportFromProj4(proj4string)
# srs.ImportFromEPSG(epsg_code) # Documentation says it is mercator
# Create dataset in mercator projection
memfile = ''
ds_merc = makeGDALds(arrbt, nx, ny, gt, srs, 'MEM', memfile)
# Regrid to latlon
gdal.Warp(outfile, ds_merc, dstSRS='EPSG:4326', dstNodata=-9999)
# Convert geotiff to a cube and save
timestamp = os.path.basename(outfile).split('.')[0].split('_')[1]
cube = sf.geotiff2cube(outfile, timestamp)
return cube
def mask_outside_disk(inst_map):
# Find coordinates and radius
hpc_coords = all_coordinates_from_map(inst_map)
r = np.sqrt(hpc_coords.Tx**2 + hpc_coords.Ty**2) / inst_map.rsun_obs
# Mask everything outside of the solar disk
mask = ma.masked_greater_equal(r, 1)
ma.set_fill_value(mask, np.nan)
where_disk = np.where(mask.mask == 1)
return where_disk
def _standardize_table(self, tab):
new_tab = tab.copy()
for col in tab.columns:
if tab[col].name == 'source_id':
new_tab[col] = tab[col].astype(np.int64)
if tab[col].unit is not None:
colunit = new_tab[col].unit.to_string()
new_tab[col].unit = GaiaQuery.unit_conversion.get(colunit)
if tab[col].name == 'pos':
ma.set_fill_value(new_tab[col], 180.0)
new_tab[col] = new_tab[col].astype(np.float64)
new_tab[col].name = 'dist'
return new_tab
def extractvel(i):
fcontent2 = wrl.io.read_iris(i)
ftype = fcontent2['product_hdr']['product_configuration'][
"product_name"].strip()
for _, data in fcontent2['data'].items():
vel = data["sweep_data"]["DB_VEL"]
vel_data = vel['data']
ma.set_fill_value(vel_data, 0)
vel_data = vel_data.filled()
vel_angel = vel['ele_start'][0]
break
cdict = [
'#E6E6E6', '#00E0FE', '#0080FF', '#320096', '#00FB90', '#00BB99',
'#008F00', '#CDC99F', '#767676', '#F88700', '#FFCF00', '#FFFF00',
'#AE0000', '#D07000', '#FF0000', '#FF007D'
]
cmap = colors.ListedColormap(cdict)
norm = colors.Normalize(vmin=-50, vmax=50)
bins = 778
if ftype == 'VOL_A':
bins = 831
elif ftype == 'VOL_B': # 无标题文档
bins = 414
el = np.zeros((360, bins)) # 仰角
for i in range(360):
for j in range(bins):
el[i, j] = vel_angel
az = np.zeros((360, bins)) # 方位角
for i in range(360):
for j in range(bins):
az[i, j] = i
rl = np.zeros((360, bins)) # 径向长度
for i in range(360):
for j in range(bins):
rl[i, j] = j
# dbz = np.zeros((len(data), 460)) # 反射率
x, y, h = sph2cord(el, az, rl)
x = np.concatenate((x, [x[0]])) # 闭合
y = np.concatenate((y, [y[0]])) # 闭合
plt.pcolor(x, y, vel_data, norm=norm, cmap=cmap)
plt.title('Velocity')
plt.axis('square')
plt.colorbar()
plt.show()
def read_ens(ensemble, varname='ssh_model'):
"""Read arrays from an ensemble of netcdf files.
Parameters:
----------
ensemble: input ensemble name
Returns:
-------
ssh_all, lat, lon: arrays
"""
regexEnsname = re.compile('[\\w]\\d{4,4}.nc.bas')
ensname = ensemble.split('/')[-1]
match = regexEnsname.search(ensname)
if match == None:
raise NameError(
'Ensemble not correctly named - see http://pp.ige-grenoble.fr/pageperso/brankarj/SESAM/ for naming input files'
)
enssizelist = re.findall('\\d{4,4}', ensname)
enssize = int(enssizelist[0])
membername1 = ensemble + '/vctgridSWOT0001_denoised.nc' #_denoised
with xr.open_dataset(membername1, mask_and_scale=False) as (dsmember1):
time = dsmember1.time.size
nc = dsmember1.nC.size
lat = dsmember1.lat[:, :].values
lon = dsmember1.lon[:, :].values
if varname == 'ssh_obs':
fill_value = dsmember1.ssh_obs.fill_value #_FillValue # we used with data written with SWOTdenoise, eventually change to ssh_obs.fill_value #_obs
else:
fill_value = dsmember1.ssh_model.fill_value #_FillValue # we used with data written with SWOTdenoise, eventually change to ssh_obs.fill_value
ssh_all = np.zeros([enssize, time, nc])
for k in range(1, enssize + 1):
membername = (ensemble + '/vctgridSWOT{:04d}_denoised.nc').format(k)
with xr.open_dataset(membername, mask_and_scale=False) as (ds):
if varname == 'ssh_obs':
buf = ds.ssh_obs[:, :] #_obs
else:
buf = ds.ssh_model[:, :]
ssh_all[k - 1, :, :] = buf
ssh_all = ma.masked_where(ssh_all == fill_value, ssh_all)
ma.set_fill_value(ssh_all, fill_value)
return (ssh_all, lat, lon)
def generate_d2(ssh_all):
"""Calculate the second derivations of the ssh across and along track (central difference)
Parameters:
----------
ssh_all: array
Returns:
-------
d2c,d2a: arrays"""
fill_value = ssh_all.fill_value
if ssh_all.ndim == 3:
isobs = False
enssize = ssh_all.shape[0]
dimtime = ssh_all.shape[1]
dimnc = ssh_all.shape[2]
if ssh_all.ndim == 2:
isobs = True
enssize = 1
dimtime = ssh_all.shape[0]
dimnc = ssh_all.shape[1]
tmp = np.zeros([enssize, dimtime, dimnc])
tmp[0, :, :] = ssh_all
ssh_all = tmp
ssh_all = ma.masked_where(ssh_all == fill_value, ssh_all)
operand1 = ssh_all[:, :, 2:]
operand2 = ssh_all[:, :, 1:dimnc - 1] * 2
operand3 = ssh_all[:, :, :dimnc - 2]
d2c = operand1 - operand2 + operand3
d2c = ma.masked_where(d2c == fill_value, d2c)
ma.set_fill_value(d2c, fill_value)
operand1 = ssh_all[:, 2:, :]
operand2 = ssh_all[:, 1:dimtime - 1, :] * 2
operand3 = ssh_all[:, :dimtime - 2, :]
d2a = operand1 - operand2 + operand3
d2a = ma.masked_where(d2a == fill_value, d2a)
ma.set_fill_value(d2a, fill_value)
if isobs:
return (d2a[0, :, :], d2c[0, :, :])
if ssh_all.ndim == 3:
return (d2a, d2c)
def save_the_mat_file(lat,lon,data_mean,p_values, field, difference_on,pole,title,layer_interface,file_type,lat_lon_bounds,cscale,axes_direction,input_folder,\
start_year,end_year,months_str,lon_sector ):
#Setting absent variables to zero for saving purposes (can't save None)
#if p_values==None:
if p_values is None:
p_values=0
if layer_interface is None:
#if layer_interface==None:
layer_interface=0
if cscale==None:
cscale=0
if field=='rho' and file_type=='ice_month':
field='SSD'
#Creating filename
exp_name=str.split(input_folder,'_bergs_')[-1]
if len(exp_name)>15:
exp_name='Control'
mat_filename='processed_data/'+exp_name+'_'+field+'_'+str(start_year)+'_to_'+ \
str(end_year)+'_'+months_str+'_'+pole + '_' + axes_direction
if difference_on==1:
mat_filename=mat_filename+'_anomaly'
if axes_direction!='xy' and lon_sector!='all':
mat_filename=mat_filename+'_'+lon_sector
mat_filename=mat_filename +'.mat'
#Setting the mask equal to 1.e20 for saving reasons, to avoid the format being screwed up.
#if axes_direction=='yz':
fill_extra_values=False #This is a hack.
if fill_extra_values==True:
ma.set_fill_value(data_mean,1.e20)
data_mean=data_mean.filled()
print p_values
sc.savemat(mat_filename, {'lat':lat, 'lon':lon, 'data':data_mean,'p_values':p_values,'field':field, \
'difference_on':difference_on, 'cscale':cscale,'pole': pole, 'title':title, 'layer_interface':layer_interface, \
'file_type': file_type, 'axes_direction':axes_direction, 'lat_lon_bounds' : lat_lon_bounds})
print 'File: ' + mat_filename + ' saved.'
def generate_d1(ssh_all):
"""Calculate the first derivations of the ssh across and along track (right difference)
Parameters:
----------
ssh_all: array
Returns:
-------
d1c,d1a: arrays
PROBLEM: SWOT simulator does not fill grid points that are not within the swaths. (ncdump -v ssh_obs xyz.nc shows a _ at these points). ncview fills these points with the _FillValue=2147483647.0 whereas python would like to fill them with missing_value which is not declared in the SWOT simulator output files. Python therefore takes an own value=9.96920996839e+36. SOLUTION: i) mask arrays with _PythonFillValue, ii) add a missing_value=2147483647.0 to variable ssh_obs in netcdf files and adjust this script to mask nan"""
fill_value = ssh_all.fill_value
if ssh_all.ndim == 3:
isobs = False
enssize = ssh_all.shape[0]
dimtime = ssh_all.shape[1]
dimnc = ssh_all.shape[2]
if ssh_all.ndim == 2:
isobs = True
enssize = 1
dimtime = ssh_all.shape[0]
dimnc = ssh_all.shape[1]
tmp = np.zeros([enssize, dimtime, dimnc])
tmp[0, :, :] = ssh_all
ssh_all = tmp
ssh_all = ma.masked_where(ssh_all == fill_value, ssh_all)
operand1 = ssh_all[:, :, 1:]
operand2 = ssh_all[:, :, :dimnc - 1]
d1c = operand1 - operand2
d1c = ma.masked_where(d1c == fill_value, d1c)
ma.set_fill_value(d1c, fill_value)
operand1 = ssh_all[:, 1:, :]
operand2 = ssh_all[:, :dimtime - 1, :]
d1a = operand1 - operand2
d1a = ma.masked_where(d1a == fill_value, d1a)
ma.set_fill_value(d1a, fill_value)
if isobs:
return (d1a[0, :, :], d1c[0, :, :])
return (d1a, d1c)
def read_obs(observation):
"""Read arrays from a netcdf file.
Parameters:
----------
observation: input observation name
Returns:
-------
ssh_obs, lat, lon: arrays
"""
with xr.open_dataset(observation, mask_and_scale=False) as (ds):
time = ds.time.size
nc = ds.nC.size
lat = ds.lat[:, :].values
lon = ds.lon[:, :].values
ssh_obs = np.zeros([time, nc])
ssh_obs[:, :] = ds.ssh_obs[:, :]
fill_value = ds.ssh_obs._FillValue # we used with data written with SWOTdenoise, eventually change to ssh_obs.fill_value
ssh_obs = ma.masked_where(ssh_obs == fill_value, ssh_obs)
ma.set_fill_value(ssh_obs, fill_value)
return (ssh_obs, lat, lon)
def NDVI(rast_1_path,rast_2_path,mask_path,outpath):
############################################################ I/O setup, read data sets and process into masked arrays
#open mask band
try:
image_mask=gdal_io.multi_band(mask_path,False)
image_mask.get_multiband_array()
mask = image_mask.array !=4096
image_mask = None #free up the memory
#open red band
image_red=gdal_io.multi_band(rast_1_path,False)
image_red.get_multiband_array()
#get a masked array from the red data object
red_array=ma.masked_array(deepcopy(image_red.array),mask)
#create the output object using the red band as a template
ndvi_image=gdal_io.raster_output(outpath,image_red,False)
ndvi_image.datatype=3 #set to int16
ndvi_image.Create_Dataset()
ndvi_image.get_multiband_array()
image_red=None #free up the memory
#open nir band
image_nir=gdal_io.multi_band(rast_2_path,False)
image_nir.get_multiband_array()
nir_array=ma.masked_array(deepcopy(image_nir.array),mask)
image_nir=None #free up the memory
########################################################### NDVI Processing
ndvi_arr=((((nir_array-red_array)/(nir_array+red_array).astype('float16'))/SCALEF).astype('int16'))
ma.set_fill_value(ndvi_arr,INT16_NODATA)
ndvi_image.array =ndvi_arr.data
ndvi_image.write_array()
########################################################### Clean Up
ndvi_image=None
nir_array=None
red_array=None
return 0
except Exception:
return -1
def main():
"""Create the model and start the evaluation process."""
args = Parameters().parse()
# #
# args.method = 'student_res18_pre'
args.method = 'student_esp_d'
args.dataset = 'camvid_light'
args.data_list = "/ssd/yifan/SegNet/CamVid/test.txt"
args.data_dir = "/ssd/yifan/"
args.num_classes = 11
# args.method='psp_dsn_floor'
args.restore_from = "./checkpoint/Camvid/ESP/base_57.8.pth"
# args.restore_from="/teamscratch/msravcshare/v-yifan/ESPNet/train/0.4results_enc_01_enc_2_8/model_298.pth"
# args.restore_from = "/teamscratch/msravcshare/v-yifacd n/sd_pytorch0.5/checkpoint/snapshots_psp_dsn_floor_1e-2_40000_TEACHER864/CS_scenes_40000.pth"
# args.restore_from = "/teamscratch/msravcshare/v-yifan/sd_pytorch0.5/checkpoint/snapshots_psp_dsn_floor_1e-2_40000_TEACHER5121024_esp/CS_scenes_40000.pth"
# args.data_list = '/teamscratch/msravcshare/v-yifan/deeplab_v3/dataset/list/cityscapes/train.lst'
args.batch_size = 1
print("Input arguments:")
for key, val in vars(args).items():
print("{:16} {}".format(key, val))
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
print(args)
output_path = args.output_path
if not os.path.exists(output_path):
os.makedirs(output_path)
# args.method='psp_dsn'
deeplab = get_segmentation_model(args.method, num_classes=args.num_classes)
ignore_label = 255
id_to_trainid = {
-1: ignore_label,
0: ignore_label,
1: ignore_label,
2: ignore_label,
3: ignore_label,
4: ignore_label,
5: ignore_label,
6: ignore_label,
7: 0,
8: 1,
9: ignore_label,
10: ignore_label,
11: 2,
12: 3,
13: 4,
14: ignore_label,
15: ignore_label,
16: ignore_label,
17: 5,
18: ignore_label,
19: 6,
20: 7,
21: 8,
22: 9,
23: 10,
24: 11,
25: 12,
26: 13,
27: 14,
28: 15,
29: ignore_label,
30: ignore_label,
31: 16,
32: 17,
33: 18
}
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
# args.restore_from="/teamscratch/msravcshare/v-yifan/sd_pytorch0.3/checkpoint/snapshots_resnet_psp_dsn_1e-4_5e-4_8_20000_DSN_0.4_769light/CS_scenes_20000.pth"
# if 'dense' in args.method:
#
if args.restore_from is not None:
saved_state_dict = torch.load(args.restore_from)
c_keys = saved_state_dict.keys()
for i in c_keys:
flag = i.split('.')[0]
if 'module' in flag:
deeplab = nn.DataParallel(deeplab)
deeplab.load_state_dict(saved_state_dict)
if 'module' not in flag:
deeplab = nn.DataParallel(deeplab)
# if 'dense' not in args.method:
# deeplab = nn.DataParallel(deeplab)
model = deeplab
model.eval()
model.cuda()
# args.dataset='cityscapes_light'
testloader = data.DataLoader(get_segmentation_dataset(
args.dataset,
root=args.data_dir,
list_path=args.data_list,
crop_size=(360, 480),
mean=IMG_MEAN,
scale=False,
mirror=False),
batch_size=args.batch_size,
shuffle=False,
pin_memory=True)
data_list = []
confusion_matrix = np.zeros((args.num_classes, args.num_classes))
palette = get_palette(20)
image_id = 0
for index, batch in enumerate(testloader):
if index % 100 == 0:
print('%d processd' % (index))
if args.side:
image, label, _, size, name = batch
elif 'sd' in args.dataset:
_, image, label, size, name = batch
else:
image, label, size, name = batch
# print('image name: {}'.format(name))
size = size[0].numpy()
output = predict_esp(model, image)
# seg_pred = np.asarray(np.argmax(output, axis=3), dtype=np.uint8)
result = np.asarray(np.argmax(output, axis=3), dtype=np.uint8)
# result=cv2.resize(result, (1024, 1024), interpolation=cv2.INTER_NEAREST)
m_seg_pred = ma.masked_array(result, mask=torch.eq(label, 255))
ma.set_fill_value(m_seg_pred, 20)
seg_pred = m_seg_pred
for i in range(image.size(0)):
image_id += 1
print('%d th segmentation map generated ...' % (image_id))
args.store_output = 'True'
output_path = './esp_camvid_base/'
if not os.path.exists(output_path):
os.mkdir(output_path)
if args.store_output == 'True':
# print('a')
output_im = PILImage.fromarray(seg_pred[i])
output_im.putpalette(palette)
output_im.save(output_path + '/' + name[i] + '.png')
seg_gt = np.asarray(label.numpy()[:, :size[0], :size[1]], dtype=np.int)
ignore_index = seg_gt != 255
seg_gt = seg_gt[ignore_index]
seg_pred = seg_pred[ignore_index]
confusion_matrix += get_confusion_matrix(seg_gt, seg_pred,
args.num_classes)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IU = IU_array.mean()
print({'meanIU': mean_IU, 'IU_array': IU_array})
print("confusion matrix\n")
print(confusion_matrix)
def createImgSCISAT(fileAbsPath):
# read info from netcdf
ncfile = Dataset(fileAbsPath, 'r')
latitude = ncfile.groups['ACE-FTS-v2.2'].latitude
longitude = ncfile.groups['ACE-FTS-v2.2'].longitude
datestart = datetime.strptime(ncfile.groups['ACE-FTS-v2.2'].start_time,
'%Y-%m-%d %H:%M:%S+00')
dateend = datetime.strptime(ncfile.groups['ACE-FTS-v2.2'].end_time,
'%Y-%m-%d %H:%M:%S+00')
ozone = ncfile.groups['ACE-FTS-v2.2'].groups['Data-L2_1km_grid'].variables[
'O3'][:]
heightLevels = ncfile.groups['ACE-FTS-v2.2'].groups[
'Data-L2_1km_grid'].variables['z'][:]
numBand = len(ozone)
ncfile.close()
#common vars
no_value = -9999
minValue = ma.min(ozone)
maxValue = ma.max(ozone)
ma.set_fill_value(ozone, no_value)
ozone = ozone.filled()
#ma.set_fill_value(heightLevels, no_value)
#heightLevels = heightLevels.filled()
sizeX = 1
sizeY = 1
dataType = gdal.GDT_Float32
resolution = 1.0 # in degree
driver = gdal.GetDriverByName('GTiff')
outFile = 'ACE-FTS_L2_ozone_' + datestart.strftime(
'%Y%m%d.%H%M%S') + '.tif'
#create tiff
dst_ds = driver.Create(outFile, sizeX, sizeY, numBand, dataType)
for i in range(numBand):
dst_ds.GetRasterBand(i + 1).WriteArray(
np.expand_dims(np.expand_dims(ozone[i], axis=0), axis=0))
# The computed stat produces this warning
# Warning 1: Lost metadata writing to GeoTIFF ... too large to fit in tag.
# An additional *.aux.xml is added
#if ozone[i] != no_value:
# dst_ds.GetRasterBand(i+1).ComputeStatistics(False)
dst_ds.GetRasterBand(i + 1).SetNoDataValue(no_value)
#set geotrasform matrix
top_left_x = longitude - (resolution / 2)
w_e_pixel_resolution = resolution
top_left_y = latitude - (resolution / 2)
n_s_pixel_resolution = -resolution
coord = [
top_left_x, w_e_pixel_resolution, 0, top_left_y, 0,
n_s_pixel_resolution
]
dst_ds.SetGeoTransform(coord)
srs = osr.SpatialReference()
srs.SetWellKnownGeogCS('WGS84')
dst_ds.SetProjection(srs.ExportToWkt())
#set metadata
dst_ds.SetMetadataItem('GLOBAL_MAX', str(maxValue))
dst_ds.SetMetadataItem('GLOBAL_MIN', str(minValue))
dst_ds.SetMetadataItem('TIME_END', dateend.strftime('%Y-%m-%dT%H:%M:%SZ'))
dst_ds.SetMetadataItem('TIME_START',
datestart.strftime('%Y-%m-%dT%H:%M:%SZ'))
dst_ds.SetMetadataItem('VERTICAL_LEVELS_NUMBER', str(len(heightLevels)))
dst_ds.SetMetadataItem('VERTICAL_LEVELS',
','.join(str(x) for x in heightLevels))
dst_ds = None
return [outFile]
def main():
"""Create the model and start the evaluation process."""
args = Parameters().parse()
# file_log = open(args.log_file, "w")
# sys.stdout = sys.stderr = file_log
print("Input arguments:")
sys.stdout.flush()
for key, val in vars(args).items():
print("{:16} {}".format(key, val))
h, w = map(int, args.input_size.split(','))
input_size = (h, w)
output_path = args.output_path
if not os.path.exists(output_path):
os.makedirs(output_path)
deeplab = get_segmentation_model("_".join([args.network, args.method]),
num_classes=args.num_classes)
ignore_label = 255
id_to_trainid = {
-1: ignore_label,
0: ignore_label,
1: ignore_label,
2: ignore_label,
3: ignore_label,
4: ignore_label,
5: ignore_label,
6: ignore_label,
7: 0,
8: 1,
9: ignore_label,
10: ignore_label,
11: 2,
12: 3,
13: 4,
14: ignore_label,
15: ignore_label,
16: ignore_label,
17: 5,
18: ignore_label,
19: 6,
20: 7,
21: 8,
22: 9,
23: 10,
24: 11,
25: 12,
26: 13,
27: 14,
28: 15,
29: ignore_label,
30: ignore_label,
31: 16,
32: 17,
33: 18
}
os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
import urllib.request
local_checkpoint, _ = urllib.request.urlretrieve(
'http://sceneparsing.csail.mit.edu/model/pretrained_resnet/resnet101-imagenet.pth',
'resnet101-imagenet.pth')
saved_state_dict = torch.load(local_checkpoint)
deeplab.load_state_dict(saved_state_dict)
model = nn.DataParallel(deeplab)
model.eval()
model.cuda()
testloader = data.DataLoader(get_segmentation_dataset(
args.dataset,
root=args.data_dir,
list_path=args.data_list,
crop_size=(1024, 2048),
scale=False,
mirror=False,
network=args.network),
batch_size=args.batch_size,
shuffle=False,
pin_memory=True)
data_list = []
confusion_matrix = np.zeros((args.num_classes, args.num_classes))
palette = get_palette(20)
image_id = 0
for index, batch in enumerate(testloader):
if index % 100 == 0:
print('%d processd' % (index))
sys.stdout.flush()
image, label, size, name = batch
size = size[0].numpy()
if torch_ver == '0.3':
if args.use_ms == 'True':
output = predict_multi_scale(model, image.numpy(),
([0.75, 1, 1.25]), input_size,
args.num_classes, args.use_flip,
args.method)
else:
if args.use_flip == 'True':
output = predict_multi_scale(model, image.numpy(),
([args.whole_scale]),
input_size, args.num_classes,
args.use_flip, args.method)
else:
if 'gt' in args.method:
label = Variable(label.long().cuda())
output = predict_whole_img_w_label(
model,
image.numpy(),
args.num_classes,
args.method,
scale=float(args.whole_scale),
label=label)
else:
output = predict_whole_img(model,
image.numpy(),
args.num_classes,
args.method,
scale=float(
args.whole_scale))
else:
with torch.no_grad():
if args.use_ms == 'True':
output = predict_multi_scale(model, image.numpy(),
([0.75, 1, 1.25]), input_size,
args.num_classes,
args.use_flip, args.method)
else:
if args.use_flip == 'True':
output = predict_multi_scale(model, image.numpy(),
([args.whole_scale]),
input_size,
args.num_classes,
args.use_flip,
args.method)
else:
if 'gt' in args.method:
output = predict_whole_img_w_label(
model,
image.numpy(),
args.num_classes,
args.method,
scale=float(args.whole_scale),
label=Variable(label.long().cuda()))
else:
output = predict_whole_img(model,
image.numpy(),
args.num_classes,
args.method,
scale=float(
args.whole_scale))
seg_pred = np.asarray(np.argmax(output, axis=3), dtype=np.uint8)
m_seg_pred = ma.masked_array(seg_pred, mask=torch.eq(label, 255))
ma.set_fill_value(m_seg_pred, 20)
seg_pred = m_seg_pred
for i in range(image.size(0)):
image_id += 1
print('%d th segmentation map generated ...' % (image_id))
sys.stdout.flush()
if args.store_output == 'True':
output_im = PILImage.fromarray(seg_pred[i])
output_im.putpalette(palette)
output_im.save(output_path + '/' + name[i] + '.png')
seg_gt = np.asarray(label.numpy()[:, :size[0], :size[1]], dtype=np.int)
ignore_index = seg_gt != 255
seg_gt = seg_gt[ignore_index]
seg_pred = seg_pred[ignore_index]
confusion_matrix += get_confusion_matrix(seg_gt, seg_pred,
args.num_classes)
pos = confusion_matrix.sum(1)
res = confusion_matrix.sum(0)
tp = np.diag(confusion_matrix)
IU_array = (tp / np.maximum(1.0, pos + res - tp))
mean_IU = IU_array.mean()
print({'meanIU': mean_IU, 'IU_array': IU_array})
print("confusion matrix\n")
print(confusion_matrix)
sys.stdout.flush()
def __init__(self, filename):
'''
Handles setting up a reader.
'''
self.fields = {}
self.info = {}
self.nc_dataset = Dataset(filename)
self.filename = filename
self.time = common.ncvar_to_dict(self.nc_dataset.variables['Time'])
self.time['data'] = ma.array(
get_datetime_from_epoch(self.time['data'][:]))
self.time['units'] = "Datetime objects"
del self.time['_FillValue']
self.fields['Nd'] = common.ncvar_to_dict(
self.nc_dataset.variables['VolumetricDrops'])
self.fields['Nd']['data'] = ma.transpose(
np.power(10, self.fields['Nd']['data']))
ma.set_fill_value(self.fields['Nd']['data'], 0.)
self.fields['Nd']['data'] = self.fields['Nd']['data'].filled()
del self.fields['Nd']['_FillValue']
self.fields['Nd']['units'] = "1/m^3 1/mm"
self.fields['RR'] = common.ncvar_to_dict(
self.nc_dataset.variables['ParsivelIntensity'])
self.fields['RR']['data'] = ma.masked_array(self.fields['RR']['data'])
ma.set_fill_value(self.fields['RR']['data'],
self.fields['RR']['_FillValue'])
del self.fields['RR']['_FillValue']
self.fields['Zh'] = common.ncvar_to_dict(
self.nc_dataset.variables['Reflectivity'])
del self.fields['Zh']['_FillValue']
self.fields['num_particles'] = common.ncvar_to_dict(
self.nc_dataset.variables['RawDrops'])
self.fields['num_particles']['data'] = ma.masked_array(
self.fields['num_particles']['data'])
ma.set_fill_value(self.fields['num_particles']['data'],
self.fields['num_particles']['_FillValue'])
del self.fields['num_particles']['_FillValue']
self.fields['terminal_velocity'] = \
common.ncvar_to_dict(self.nc_dataset.variables['VelocityDrops'])
self.fields['terminal_velocity']['data'] = \
ma.transpose(ma.masked_array(self.fields['terminal_velocity']['data']))
ma.set_fill_value(self.fields['terminal_velocity']['data'],
self.fields['terminal_velocity']['_FillValue'])
del self.fields['terminal_velocity']['_FillValue']
self.fields['Precip_Code'] = common.ncvar_to_dict(
self.nc_dataset.variables['PrecipCode'])
diameter = ma.array([
0.0625, 0.1875, 0.3125, 0.4375, 0.5625, 0.6875, 0.8125, 0.9375,
1.0625, 1.1875, 1.375, 1.625, 1.875, 2.125, 2.375, 2.75, 3.25,
3.75, 4.25, 4.75, 5.5, 6.5, 7.5, 8.5, 9.5, 11., 13., 15., 17., 19.,
21.5, 24.5
])
spread = ma.array([
0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125, 0.125,
0.125, 0.250, 0.250, 0.250, 0.250, 0.250, 0.500, 0.500, 0.500,
0.500, 0.500, 1.000, 1.000, 1.000, 1.000, 1.000, 2.000, 2.000,
2.000, 2.000, 2.000, 3.000, 3.000
])
# velocity = ma.array([0.05, 0.15, 0.25, 0.35, 0.45, 0.55, 0.65, 0.75, 0.85, 0.95, 1.1, 1.3,
# 1.5, 1.7, 1.9, 2.2, 2.6, 3.0, 3.4, 3.8, 4.4, 5.2, 6.0, 6.8, 7.6, 8.8,
# 10.4, 12.0, 13.6, 15.2, 17.6, 20.8])
self.bin_edges = common.var_to_dict(
'bin_edges', np.hstack((0, diameter + np.array(spread) / 2)), 'mm',
'Boundaries of bin sizes')
self.spread = common.var_to_dict('spread', spread, 'mm',
'Bin size spread of bins')
self.diameter = common.var_to_dict('diameter', diameter, 'mm',
'Particle diameter of bins')
for key in self.nc_dataset.ncattrs():
self.info[key] = self.nc_dataset.getncattr(key)
np_mask = torch.squeeze(
flags.eq(2)).cpu().data.numpy().astype(float)
rho = torch.squeeze(density).cpu().data.numpy()
p = torch.squeeze(pressure).cpu().data.numpy()
img_norm_vel = torch.squeeze(
torch.norm(tensor_vel, dim=1,
keepdim=True)).cpu().data.numpy()
img_velx = torch.squeeze(tensor_vel[:, 0]).cpu().data.numpy()
img_vely = torch.squeeze(tensor_vel[:, 1]).cpu().data.numpy()
img_vel_norm = torch.squeeze( \
torch.norm(tensor_vel, dim=1, keepdim=True)).cpu().data.numpy()
img_velx_masked = ma.array(img_velx, mask=np_mask)
img_vely_masked = ma.array(img_vely, mask=np_mask)
img_vel_norm_masked = ma.array(img_vel_norm, mask=np_mask)
ma.set_fill_value(img_velx_masked, np.nan)
ma.set_fill_value(img_vely_masked, np.nan)
ma.set_fill_value(img_vel_norm_masked, np.nan)
img_velx_masked = img_velx_masked.filled()
img_vely_masked = img_vely_masked.filled()
img_vel_norm_masked = img_vel_norm_masked.filled()
if real_time:
cax_rho.clear()
cax_velx.clear()
cax_vely.clear()
cax_p.clear()
cax_div.clear()
fig.suptitle("it = " + str(it), fontsize=16)
im0 = ax_rho.imshow(rho[minY:maxY, minX:maxX],
cmap=my_map,
import numpy as np import numpy.ma as ma a = np.arange(5) a a = ma.masked_where(a < 3, a) a ma.set_fill_value(a, -999) a a = range(5) a ma.set_fill_value(a, 100) a a = np.arange(5) a ma.set_fill_value(a, 100) a
def resample_from_array(in_raster=None,
in_affine=None,
out_tile=None,
in_crs=None,
resampling="nearest",
nodataval=None,
nodata=0):
"""
Extract and resample from array to target tile.
Parameters
----------
in_raster : array
in_affine : ``Affine``
out_tile : ``BufferedTile``
resampling : string
one of rasterio's resampling methods (default: nearest)
nodata : integer or float
raster nodata value (default: 0)
Returns
-------
resampled array : array
"""
if nodataval is not None:
warnings.warn("'nodataval' is deprecated, please use 'nodata'")
nodata = nodata or nodataval
# TODO rename function
if isinstance(in_raster, ma.MaskedArray):
pass
elif isinstance(in_raster, np.ndarray):
in_raster = ma.MaskedArray(in_raster, mask=in_raster == nodata)
elif isinstance(in_raster, ReferencedRaster):
in_affine = in_raster.affine
in_crs = in_raster.crs
in_raster = in_raster.data
elif isinstance(in_raster, tuple):
in_raster = ma.MaskedArray(
data=np.stack(in_raster),
mask=np.stack([
band.mask if isinstance(band, ma.masked_array) else np.where(
band == nodata, True, False) for band in in_raster
]),
fill_value=nodata)
else:
raise TypeError("wrong input data type: %s" % type(in_raster))
if in_raster.ndim == 2:
in_raster = ma.expand_dims(in_raster, axis=0)
elif in_raster.ndim == 3:
pass
else:
raise TypeError("input array must have 2 or 3 dimensions")
if in_raster.fill_value != nodata:
ma.set_fill_value(in_raster, nodata)
dst_data = np.empty((in_raster.shape[0], ) + out_tile.shape,
in_raster.dtype)
logger.debug(in_raster)
logger.debug(in_affine)
logger.debug(out_tile.affine)
reproject(in_raster.filled(),
dst_data,
src_transform=in_affine,
src_crs=in_crs or out_tile.crs,
src_nodata=nodata,
dst_transform=out_tile.affine,
dst_crs=out_tile.crs,
dst_nodata=nodata,
resampling=Resampling[resampling])
logger.debug(dst_data)
hanse = ma.MaskedArray(dst_data,
mask=dst_data == nodata,
fill_value=nodata)
logger.debug(hanse)
return hanse
#create geotransform
xres = (xmax - xmin) / float(ncols)
yres = (ymax - ymin) / float(nrows)
geotransform = (xmin,xres,0,ymax,0, -yres)
#create mask
mask_DS = gdal.GetDriverByName('MEM').Create('', ncols, nrows, 1 ,gdal.GDT_Int32)
mask_RB = mask_DS.GetRasterBand(1)
mask_RB.Fill(0) #initialise raster with zeros
mask_RB.SetNoDataValue(-32767)
mask_DS.SetGeoTransform(geotransform)
maskvalue = 1
err = gdal.RasterizeLayer(mask_DS, [maskvalue], shpLyr)
mask_DS.FlushCache()
mask_array = mask_DS.GetRasterBand(1).ReadAsArray()
mask_RES = ma.masked_equal(mask_array, 255)
ma.set_fill_value(mask_RES, -32767)
########################################################################
#subset
########################################################################
var_subset = varData_Ori[:,min(lat_inds[0]):max(lat_inds[0]) + 1, min(lon_inds[0]):max(lon_inds[0]) + 1]
var_subset._set_mask(np.logical_not(np.flipud(mask_RES.mask))) # update mask (flipud is reverse 180)
lon_subset = lon_Ori[lon_inds]
lat_subset = lat_Ori[lat_inds]
###################################################################################
# Open a new NetCDF file to write the data to. For format, you can choose
# from
# 'NETCDF3_CLASSIC', 'NETCDF3_64BIT', 'NETCDF4_CLASSIC', and 'NETCDF4'
###################################################################################
#create file path and name
InputFileDir,InputFile = os.path.split(file_name)
InputDir,InputFileDirName = os.path.split(InputFileDir)
def NC_CLIP_VIA_SHP(NcFilePath,ShpFilePath,OutFilePath):
strShpFilePath = ShpFilePath
strNetCDFPathInput = NC
#read shapefile
shpDS = ogr.Open(strShpFilePath)
shpLyr = shpDS.GetLayer()
Envelop = shpLyr.GetExtent()
xmin,xmax,ymin,ymax = [np.round(Envelop[0]),
np.round(Envelop[1]),
np.round(Envelop[2]),
np.round(Envelop[3])] #Your extents as given above
mask_RES = []
######################################################
# Process #
######################################################
leadtime = 0
EnsembleMember = 0
ncInput = Dataset(strNetCDFPathInput)
var_name = list(ncInput.variables.keys())
lon_Ori = getDimVar(ncInput,var_name,'X')
lat_Ori = getDimVar(ncInput,var_name,'Y')
reftime_Ori = getDimVar(ncInput,var_name,'S')
#time = getDimVar(ncInput,varList,'time')
name = 'prec'
varData_Ori = getDataVar(ncInput,name)
######################################################
# Create mask #
######################################################
if len(mask_RES) == 0 :
#get boundary and xs ys
lat_bnds, lon_bnds = [ymin, ymax], [xmin+180, xmax+180]
lat_inds = np.where((lat_Ori >= (lat_bnds[0])) & (lat_Ori <= lat_bnds[1]))
lon_inds = np.where((lon_Ori >= (lon_bnds[0])) & (lon_Ori <= lon_bnds[1]))
ncols = len(lon_inds[0])
nrows = len(lat_inds[0])
nreftime = len(reftime_Ori)
#create geotransform
xres = (xmax - xmin) / float(ncols)
yres = (ymax - ymin) / float(nrows)
geotransform = (xmin-3,xres,0,ymax,0,-yres)
#create mask
mask_DS = gdal.GetDriverByName('MEM').Create('', ncols, nrows, 1 ,gdal.GDT_Int32)
mask_RB = mask_DS.GetRasterBand(1)
mask_RB.Fill(0) #initialise raster with zeros
mask_RB.SetNoDataValue(-32767)
mask_DS.SetGeoTransform(geotransform)
maskvalue = 1
err = gdal.RasterizeLayer(mask_DS, [maskvalue], shpLyr)
mask_DS.FlushCache()
mask_array = mask_DS.GetRasterBand(1).ReadAsArray()
mask_RES = ma.masked_equal(mask_array, 255)
ma.set_fill_value(mask_RES, -32767)
######################################################
# Subset #
######################################################
var_subset = varData_Ori[:,min(lat_inds[0])-1:max(lat_inds[0]), min(lon_inds[0])-1:max(lon_inds[0])]
var_subset.__setmask__(np.logical_not(np.flipud(mask_RES.mask))) # update mask (flipud is reverse 180)
#var_subset = var_subset.data
lon_subset = lon_Ori[lon_inds]-180
lat_subset = lat_Ori[lat_inds]
######################################################
# Open a new NetCDF file to for data svaing. You can #
# choose one of the formats from 'NETCDF3_CLASSIC', #
# 'NETCDF3_64BIT', 'NETCDF4_CLASSIC', and 'NETCDF4' #
######################################################
# Using our previous dimension info, we can create the new time dimension
# Even though we know the size, we are going to set the size to unknown
OutputFileDirName = OutFilePath
ncOutput = Dataset(OutputFileDirName, 'w', format='NETCDF4')
#ncOutput.createDimension('time', None)
ncOutput.createDimension('lon', ncols)
ncOutput.createDimension('lat', nrows)
ncOutput.createDimension('reftime', nreftime)
# Add lat Variable
var_out_lat = ncOutput.createVariable('lat','f',("lat"))
ncOutput.variables['lat'][:] = lat_subset[:]
# Add lon Variable
var_out_lon = ncOutput.createVariable('lon','f',("lon"))
ncOutput.variables['lon'][:] = lon_subset[:]
# Add leadtime Variable
var_out_reftime = ncOutput.createVariable('reftime','f',("reftime"))
ncOutput.variables['reftime'][:] = reftime_Ori[:]
# Add data Variable
var_out_data = ncOutput.createVariable('Precip', 'f',("reftime","lat","lon"))
for i in range(np.size(reftime_Ori)):
ncOutput.variables['Precip'][i,:,:] = var_subset[i,:,:]
# attr
ncOutput.history = "CLIP Created datatime" + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + " by LujunZ at OU"
ncOutput.source = "netCDF4 under python 3.6.5"
######################################################
# Write close #
######################################################
ncOutput.close() # close the new file
print('Done')
return
print('NetCDF file is not right')
continue
else:
nc = ncfile(local_store[0])
if (count == 0): # Just do this once to get the dimensionality of the variable array
data2 = nc.variables[variable][:]
no_dims = len(np.shape(data2))
# Load variable of interest, plus dimensions
requested_months = (eyear - syear + 1)*12 #how many months span requested time period
if (no_dims == 4): data2 = nc.variables[variable][date_diff:date_diff+requested_months,:,:,:]
if (no_dims == 3): data2 = nc.variables[variable][date_diff:date_diff+requested_months:,:,:]
ma.set_fill_value(data2, np.nan) # Change any masked values to NaN to help with better interpolation (later)
lat = nc.variables['lat'][:]
lon = nc.variables['lon'][:]
if (no_dims == 4):
p_mod = nc.variables['plev'][:]
p_units = str(nc.variables['plev'].units)
# Load relevant metadata
model_name = str(nc.source_id)
ensemble_name = str(nc.variant_label)
expid = str(nc.parent_source_id)
grid_label = str(nc.grid_label)
variant_label = str(nc.variant_label)
# Create name of output file, and check if it has already been created
if (inter == False): out_name = data_path + variable + '_' + vtype + '_' + expid +'_'+experiment+'_'+variant_label+'_'+grid_label+'_'+str(syear)+'01-'+str(eyear)+'12.nc'
objtype = np.char.strip(ma.getdata(NGC['type']))
# Keep all globular clusters and planetary nebulae
keeptype = ('PN', 'GCl')
keep = np.zeros(len(NGC), dtype=bool)
for otype in keeptype:
ww = [otype == tt for tt in objtype]
keep = np.logical_or(keep, ww)
print(np.sum(keep))
clusters = NGC[keep]
# Fill missing major axes with a nominal 0.4 arcmin (roughly works
# for NGC7009, which is the only missing PN in the footprint).
ma.set_fill_value(clusters['majax'], 0.4)
clusters['majax'] = ma.filled(clusters['majax'].data)
# Increase the radius of IC4593
# https://github.com/legacysurvey/legacypipe/issues/347
clusters[clusters['name'] == 'IC4593']['majax'] = 0.5
#indesi = desimodel.footprint.is_point_in_desi(tiles, ma.getdata(clusters['ra']),
# ma.getdata(clusters['dec']))
#print(np.sum(indesi))
#bb = clusters[indesi]
#bb[np.argsort(bb['majax'])[::-1]]['name', 'ra', 'dec', 'majax', 'type']
# Build the output catalog: select a subset of the columns and rename
# majax-->radius (arcmin-->degree)
out = Table()
import matplotlib.pyplot as plt
import sys
import os
#vmin = -2
#vmax = 30
### figure 1
nc = Dataset(sys.argv[2])
var1=nc.variables[''.join([sys.argv[1],"Atl"])]
values=var1[:]
xs = nc["lat"][:]
ys = nc["zoc"][:]
error_value = -1.e30
values = ma.masked_values(values, error_value)
ma.set_fill_value(values, 0)
#title = sys.argv[2].split("/")[-1].replace(".nc","")
title = ''.join([sys.argv[1],"Atl"])
ylabel=var1.dimensions[0]
xlabel=var1.dimensions[1]
#clabel=var1.units
siz=16
siz_tick=14
X, Y = np.meshgrid(xs, ys)
fig, ax = plt.subplots()
plt.contour(X, Y, values,colors='k')
plt.contourf(X, Y, values,50,cmap=plt.cm.Spectral_r)
plt.gca().invert_yaxis()
plt.title(title,size=siz)
