def gridInit(self, dstpath, bb, zoom, min_dile, max_dile):
ncg = NetcdfGeometry()
#z = nco.getZoomLevel(self.rgrp.variables['lat'], self.rgrp.variables['lon'])
#bb = nco.getBoundingBox(self.rgrp.variables['lat'], self.rgrp.variables['lon'])
min_dile = ncg.getMinDile(bb,zoom)
max_dile = ncg.getMaxDile(bb,zoom)
bb_min = min_dile.asBoundingBox()
increment = 1/float(2**z)
# xsize/ysize calculated as the indices distance
# adding +1 to consider the lower bound
'''
explanation of the size formula:
problem:
retrive how many diles cover a certain bounding box
then create a grid that represent the diles merged
in one grid
data:
x_max -> the highest x index of the diles covering the bb
x_min -> the lowest "" "" ""
y_max -> the highest y index of the diles covering the bb
y_min -> the lowest "" "" ""
XSIZE -> the size of the x axis of a single dile grid
YSIZE -> "" "" "" "" y "" "" "" ""
analysis:
the distance between indices both for x and y is
(x_max - x_min + 1)
(y_max - y_min + 1)
where the +1 is needed to take in account the min value
the multiplication for the size provides the number of
elements for each axis, but there's a catch:
the size of a dile's grid is 181x361, which means that the
borders are both included in the grid, consequentially,
if we consider 2 adjacent diles, they will share 1 border.
Logically speaking this is not an hard constraint, but when
calculating the grid size, it must be taken in consideration.
ex:
| |
v v
*---*---*---*
| a | b | c |
->*---*---*---*
| d | e | f |
*---*---*---*
along the x axis a,b and b,c share a common vertical edge
along the y axis a,d and b,e and c,f shere a common horizontal edge.
to obtain the correct size along the axis we must consider this edges
only once, therefore the number of the internal edges has to be subtracted
from the total number of elements of the grid. This number is equal to
the number of the diles along that axis minus one, in the previous
example infact that number is 2.
There we have it:
Number of total elements of the grid along x axis:
n_x = (x_max - x_min +1)*Dile_x_size
Number of internal edges:
i_e_x = n_x - 1
Number of elements of along the x axis:
n_x - i_e_x =
(x_max - x_min + 1)*dile_x_size - (x_max - x_min + 1) - 1 =
(x_max - x_min + 1)*dile_x_size - x_max + x_min
'''
xsize = (max_dile.x - min_dile.x + 1)*min_dile.XSIZE - max_dile.x + min_dile.x #max_dile would be good as well
ysize = (max_dile.y - min_dile.y + 1)*min_dile.YSIZE - max_dile.y + min_dile.y
xfirst = bb_min["lon_min"]
yfirst = bb_min["lat_min"]
xinc = increment
yinc = increment
gf = GridFile("lonlat", xsize, ysize, xfirst, xinc, yfirst, yinc)
return gf.createGrid(dstpath)
z += 1 printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname) y += 1 x += 1 return iter_len if __name__ == '__main__': timer = Chrono() ncg = NetcdfGeometry() lpw = LukePathWalker() mydir = "/sdiles/ubuntu/sdiles" myext = ["nc"] paths = [] for file in lpw.getDirectoryContent(mydir): if lpw.checkExtention(myext, file): paths.append(file) for i in range(50,len(paths)): timer.start()
def createDiles(self, dstpath, bb, zoom):
df = DileFactory()
ng = NetcdfGeometry()
basename = pathLeaf(dstpath)
# delimiters for the diles contained in the matrix
indx, indy = ng.getDileIndexTable(bb,zoom)
# information extracted from the netcdf (dimensions, variables and diles' coverage)
dimensions = [d.name for d in self.rgrp.dimensions.values()]
variables = [v for v in self.rgrp.variables.values() if v.name not in dimensions]
diles = df.fromBoundingBox(bb['lon_min'], bb['lat_min'], bb['lon_max'], bb['lat_max'],zoom)
# calculating the number of iterations required, for printing purposes
iter_len = 0
for var in variables:
if len(var.shape) > 2:
iter_len += reduce(lambda x, y: x*y, var.shape[:-2])*len(diles)
else:
iter_len += len(diles)
# x,y,z indices for printing purposes
x = 0
for var in variables:
y = 0
timeind = None
levelind = None
try:
timeind = var.dimensions.index('time')
except:
pass
try:
levelind = var.dimensions.index('level')
except:
pass
for ind in ndindex(var.shape[:-2]):
z = 0
for dile in diles:
indstr = []
if timeind is not None: indstr.append(str(ind[timeind]))
if levelind is not None: indstr.append(str(ind[levelind]))
fname = dile.getFileName('dile_'+''.join([i+'_' for i in indstr]))
fpath = ''.join([i+'/' for i in indstr])+dile.getRelativePath()
path = dstpath+'/'+var.name+'/'+fpath+fname
a = indy[dile.y]["min"]
b = indy[dile.y]["max"]
c = indx[dile.x]["min"]
d = indx[dile.x]["max"]
if isinstance(var[ind], MaskedArray):
matrix = var[ind].data
else:
matrix = var[ind]
attributes = {}
for attr in var.ncattrs():
attributes.update({attr:var.getncattr(attr)})
dile.variable = var.name
dile.attributes = attributes
if '_FillValue' in var.ncattrs():
dile.createNetCDF4(path,matrix[a:b+1,c:d+1],var.dtype,var.getncattr('_FillValue'))
else:
dile.createNetCDF4(path,matrix[a:b+1,c:d+1])
z += 1
printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname)
y += 1
x += 1
return iter_len
z += 1 printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname) y += 1 x += 1 return iter_len def onClose(self): self.client.close() if __name__ == '__main__': ng = NetcdfGeometry() df = DileFactory() timer = Chrono() path = "/sdiles/ubuntu/sdiles/sdile_tasmax_2_0_1.nc" fname = pathLeaf(path,False) print "computing md5 for ", fname, "..." timer.start() md5 = getMD5(path) timer.stop() print "md5 computed in: ", timer.formatted() rgrp = ncOpen(path, mode='r') bb = ng.getBoundingBox(rgrp['lat'],rgrp['lon'])