dimensions = nctile.dimensions.values() variables = [v for v in nctile.variables.values() if v.name not in [d.name for d in dimensions ]] for v in variables: if len(v.shape) > 2: sdiles[v.name][v.name][...,latind,lonind] = nctile[v.name][...,:,:] else: sdiles[v.name][v.name][latind,lonind] = nctile[v.name][:,:] serialize(i+1, indexfolder, name) nctile.close() remove(src) timer.stop() printProgress(i, len(urls), urls[i]['name'],"last tile armonized in: "+timer.formatted()) timer.reset() ''' except: raise exit() print 'saving!' serialize(i, indexfolder, name) # closing the super diles for key, value in sdiles.iteritems(): value.close() else: '''
def onIngest(self, srcpath, dstpath, bb, zoom):
df = DileFactory()
ng = NetcdfGeometry()
basename = pathLeaf(srcpath)
dimensions = [d.name for d in self.rgrp.dimensions.values()]
variables = [v for v in self.rgrp.variables.values() if v.name not in dimensions]
diles = df.fromBoundingBox(bb['lon_min'], bb['lat_min'], bb['lon_max'], bb['lat_max'],zoom)
dates = None
try:
dates = num2date(rgrp['time'][:],rgrp['time'].units)
except:
pass
# calculating the number of iterations required, for printing purposes
iter_len = 0
for var in variables:
iter_len += reduce(lambda x, y: x*y, var.shape[:-2])*len(diles)
x = 0
for var in variables:
timeind = None
levelind = None
try:
timeind = var.dimensions.index('time')
except:
pass
try:
levelind = var.dimensions.index('level')
except:
pass
y = 0
for ind in ndindex(var.shape[:-2]):
z = 0
for dile in diles:
docurl = dstpath
item = dile.asDocument()
indstr = []
if timeind is not None:
item['time'] = dates[ind[timeind]]
indstr.append(str(ind[timeind]))
if levelind is not None:
item['level'] = ind[levelind]
indstr.append(str(ind[levelind]))
docurl += ''.join([i+'/' for i in indstr])
docurl += dile.getRelativePath()
fname = dile.getFileName('dile_'+''.join([i+'_' for i in indstr]))
docurl += fname
item['md5'] = md5
item['variable'] = var.name
item['attributes'] = [{str(key): str(var.getncattr(key))} for key in var.ncattrs()]
item['uri'] = docurl
item['source'] = basename
item['zoom'] = zoom
item['cdate'] = datetime.now()
self.onInsert(item)
z += 1
printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname)
y += 1
x += 1
return iter_len
