def downloadFile(self,url,dst): try: response = urlopen(url) except: print "couldn't reach ", url return False else: try: if not path.exists(dst): makedirs(dst) except: print "couldn't create the destination folder for the file!" raise exit() else: CHUNK = 16 * 1024 fname = pathLeaf(url) with open(dst+fname, 'wb') as f: while True: chunk = response.read(CHUNK) if not chunk: break f.write(chunk) #print url + "-->" + " Downloded!" return str(dst+fname)
def createTileGrid(self, src, dst, latidx, lonidx, z): srcfile = ncOpen(src,"r") fname = pathLeaf(src,False) bbox = self.indices_to_bbox(latidx, lonidx) inc = 1/float(2**z) xsize = len(srcfile.variables['lon'][:]) ysize = len(srcfile.variables['lon'][:]) xfirst = bbox["lon_min"] yfirst = bbox["lat_min"] xinc = inc yinc = inc gf = GridFile("latlon", xsize, ysize, xfirst, xinc, yfirst, yinc) fgrid = gf.createGrid(dst+fname+".grid") return str(dst+fname+".grid")
def getUriList(self, bbidxs, src): uri = [] for latidx in range(bbidxs['latidx_min'],bbidxs['latidx_max']+1): for lonidx in range(bbidxs['lonidx_min'],bbidxs['lonidx_max']+1): try: url = "%s/%04d/clim_%04d_%04d.tile.nc4" %(src,latidx,latidx,lonidx) urlopen(url) except: pass else: uri.append({'name': pathLeaf(url), 'uri':url, 'latidx':latidx, 'lonidx':lonidx}) return uri
def select_dile_by_uri(): """Download a dile given a uri. :example: /select/dile?uri=http://s3.amazonaws.com/edu-uchicago-rdcep-diles/fd65252e41e3cf0b431a07ad6e2cbe85/sdile_pr_2_1_1/pr/0/2/1/1/dile_0_2_1_1.nc :param: uri -- a valid uri to access the dile :returns: netcdf4 -- the return of the dile. ------------------------------------------------------------------------------------------- """ uri=request.args.get('uri') if uri is not None: if uri.startswith("http://s3.amazonaws.com/"): path = uri.replace("http://s3.amazonaws.com/","") bname, kstr = path.split("/",1) # split the bname from the key string conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY) try: bucket = conn.get_bucket(bname) except: print "BUCKET NOT FOUND" return str("ERROR: bucket "+bname+" not found") else: print "BUCKET CONNECTED" try: key = bucket.get_key(kstr) print "KEY: ", key except: print "KEY NOT FOUND" return str("ERROR: key "+kstr+"not found") else: try: key.open_read() # opens the file headers = dict(key.resp.getheaders()) # request the headers headers["Content-Disposition"] = "inline; filename="+str(pathLeaf(key.name)) return Response(key, headers=headers) # return a response except S3ResponseError as e: return Response(e.body, status=e.status, headers=key.resp.getheaders()) abort(400)
def regridTile(self, src, dst, fgrid, remap): fname = pathLeaf(src) cdo = CdoRegridder(src, dst+fname, fgrid) cdo.regridLocal(remap) return str(dst+fname)
def onIngest(self, srcpath, dstpath, bb, zoom): df = DileFactory() ng = NetcdfGeometry() basename = pathLeaf(srcpath) dimensions = [d.name for d in self.rgrp.dimensions.values()] variables = [v for v in self.rgrp.variables.values() if v.name not in dimensions] diles = df.fromBoundingBox(bb['lon_min'], bb['lat_min'], bb['lon_max'], bb['lat_max'],zoom) dates = None try: dates = num2date(rgrp['time'][:],rgrp['time'].units) except: pass # calculating the number of iterations required, for printing purposes iter_len = 0 for var in variables: iter_len += reduce(lambda x, y: x*y, var.shape[:-2])*len(diles) x = 0 for var in variables: timeind = None levelind = None try: timeind = var.dimensions.index('time') except: pass try: levelind = var.dimensions.index('level') except: pass y = 0 for ind in ndindex(var.shape[:-2]): z = 0 for dile in diles: docurl = dstpath item = dile.asDocument() indstr = [] if timeind is not None: item['time'] = dates[ind[timeind]] indstr.append(str(ind[timeind])) if levelind is not None: item['level'] = ind[levelind] indstr.append(str(ind[levelind])) docurl += ''.join([i+'/' for i in indstr]) docurl += dile.getRelativePath() fname = dile.getFileName('dile_'+''.join([i+'_' for i in indstr])) docurl += fname item['md5'] = md5 item['variable'] = var.name item['attributes'] = [{str(key): str(var.getncattr(key))} for key in var.ncattrs()] item['uri'] = docurl item['source'] = basename item['zoom'] = zoom item['cdate'] = datetime.now() self.onInsert(item) z += 1 printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname) y += 1 x += 1 return iter_len
return iter_len def onClose(self): self.client.close() if __name__ == '__main__': ng = NetcdfGeometry() df = DileFactory() timer = Chrono() path = "/sdiles/ubuntu/sdiles/sdile_tasmax_2_0_1.nc" fname = pathLeaf(path,False) print "computing md5 for ", fname, "..." timer.start() md5 = getMD5(path) timer.stop() print "md5 computed in: ", timer.formatted() rgrp = ncOpen(path, mode='r') bb = ng.getBoundingBox(rgrp['lat'],rgrp['lon']) zoom = ng.getZoomLevel(rgrp['lat'], rgrp['lon']) mim = MetaIngesterMongo(rgrp) url = " http://s3.amazonaws.com/edu-uchicago-rdcep-diles/"