dimensions = nctile.dimensions.values() variables = [v for v in nctile.variables.values() if v.name not in [d.name for d in dimensions ]] for v in variables: if len(v.shape) > 2: sdiles[v.name][v.name][...,latind,lonind] = nctile[v.name][...,:,:] else: sdiles[v.name][v.name][latind,lonind] = nctile[v.name][:,:] serialize(i+1, indexfolder, name) nctile.close() remove(src) timer.stop() printProgress(i, len(urls), urls[i]['name'],"last tile armonized in: "+timer.formatted()) timer.reset() ''' except: raise exit() print 'saving!' serialize(i, indexfolder, name) # closing the super diles for key, value in sdiles.iteritems(): value.close() else: '''
def onIngest(self, srcpath, dstpath, bb, zoom): df = DileFactory() ng = NetcdfGeometry() basename = pathLeaf(srcpath) dimensions = [d.name for d in self.rgrp.dimensions.values()] variables = [v for v in self.rgrp.variables.values() if v.name not in dimensions] diles = df.fromBoundingBox(bb['lon_min'], bb['lat_min'], bb['lon_max'], bb['lat_max'],zoom) dates = None try: dates = num2date(rgrp['time'][:],rgrp['time'].units) except: pass # calculating the number of iterations required, for printing purposes iter_len = 0 for var in variables: iter_len += reduce(lambda x, y: x*y, var.shape[:-2])*len(diles) x = 0 for var in variables: timeind = None levelind = None try: timeind = var.dimensions.index('time') except: pass try: levelind = var.dimensions.index('level') except: pass y = 0 for ind in ndindex(var.shape[:-2]): z = 0 for dile in diles: docurl = dstpath item = dile.asDocument() indstr = [] if timeind is not None: item['time'] = dates[ind[timeind]] indstr.append(str(ind[timeind])) if levelind is not None: item['level'] = ind[levelind] indstr.append(str(ind[levelind])) docurl += ''.join([i+'/' for i in indstr]) docurl += dile.getRelativePath() fname = dile.getFileName('dile_'+''.join([i+'_' for i in indstr])) docurl += fname item['md5'] = md5 item['variable'] = var.name item['attributes'] = [{str(key): str(var.getncattr(key))} for key in var.ncattrs()] item['uri'] = docurl item['source'] = basename item['zoom'] = zoom item['cdate'] = datetime.now() self.onInsert(item) z += 1 printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname) y += 1 x += 1 return iter_len