def createDiles(self, dstpath, bb, zoom):
df = DileFactory()
ng = NetcdfGeometry()
basename = pathLeaf(dstpath)
# delimiters for the diles contained in the matrix
indx, indy = ng.getDileIndexTable(bb,zoom)
# information extracted from the netcdf (dimensions, variables and diles' coverage)
dimensions = [d.name for d in self.rgrp.dimensions.values()]
variables = [v for v in self.rgrp.variables.values() if v.name not in dimensions]
diles = df.fromBoundingBox(bb['lon_min'], bb['lat_min'], bb['lon_max'], bb['lat_max'],zoom)
# calculating the number of iterations required, for printing purposes
iter_len = 0
for var in variables:
if len(var.shape) > 2:
iter_len += reduce(lambda x, y: x*y, var.shape[:-2])*len(diles)
else:
iter_len += len(diles)
# x,y,z indices for printing purposes
x = 0
for var in variables:
y = 0
timeind = None
levelind = None
try:
timeind = var.dimensions.index('time')
except:
pass
try:
levelind = var.dimensions.index('level')
except:
pass
for ind in ndindex(var.shape[:-2]):
z = 0
for dile in diles:
indstr = []
if timeind is not None: indstr.append(str(ind[timeind]))
if levelind is not None: indstr.append(str(ind[levelind]))
fname = dile.getFileName('dile_'+''.join([i+'_' for i in indstr]))
fpath = ''.join([i+'/' for i in indstr])+dile.getRelativePath()
path = dstpath+'/'+var.name+'/'+fpath+fname
a = indy[dile.y]["min"]
b = indy[dile.y]["max"]
c = indx[dile.x]["min"]
d = indx[dile.x]["max"]
if isinstance(var[ind], MaskedArray):
matrix = var[ind].data
else:
matrix = var[ind]
attributes = {}
for attr in var.ncattrs():
attributes.update({attr:var.getncattr(attr)})
dile.variable = var.name
dile.attributes = attributes
if '_FillValue' in var.ncattrs():
dile.createNetCDF4(path,matrix[a:b+1,c:d+1],var.dtype,var.getncattr('_FillValue'))
else:
dile.createNetCDF4(path,matrix[a:b+1,c:d+1])
z += 1
printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname)
y += 1
x += 1
return iter_len
def onIngest(self, srcpath, dstpath, bb, zoom):
df = DileFactory()
ng = NetcdfGeometry()
basename = pathLeaf(srcpath)
dimensions = [d.name for d in self.rgrp.dimensions.values()]
variables = [v for v in self.rgrp.variables.values() if v.name not in dimensions]
diles = df.fromBoundingBox(bb['lon_min'], bb['lat_min'], bb['lon_max'], bb['lat_max'],zoom)
dates = None
try:
dates = num2date(rgrp['time'][:],rgrp['time'].units)
except:
pass
# calculating the number of iterations required, for printing purposes
iter_len = 0
for var in variables:
iter_len += reduce(lambda x, y: x*y, var.shape[:-2])*len(diles)
x = 0
for var in variables:
timeind = None
levelind = None
try:
timeind = var.dimensions.index('time')
except:
pass
try:
levelind = var.dimensions.index('level')
except:
pass
y = 0
for ind in ndindex(var.shape[:-2]):
z = 0
for dile in diles:
docurl = dstpath
item = dile.asDocument()
indstr = []
if timeind is not None:
item['time'] = dates[ind[timeind]]
indstr.append(str(ind[timeind]))
if levelind is not None:
item['level'] = ind[levelind]
indstr.append(str(ind[levelind]))
docurl += ''.join([i+'/' for i in indstr])
docurl += dile.getRelativePath()
fname = dile.getFileName('dile_'+''.join([i+'_' for i in indstr]))
docurl += fname
item['md5'] = md5
item['variable'] = var.name
item['attributes'] = [{str(key): str(var.getncattr(key))} for key in var.ncattrs()]
item['uri'] = docurl
item['source'] = basename
item['zoom'] = zoom
item['cdate'] = datetime.now()
self.onInsert(item)
z += 1
printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname)
y += 1
x += 1
return iter_len