def downloadFile(self,url,dst): try: response = urlopen(url) except: print "couldn't reach ", url return False else: try: if not path.exists(dst): makedirs(dst) except: print "couldn't create the destination folder for the file!" raise exit() else: CHUNK = 16 * 1024 fname = pathLeaf(url) with open(dst+fname, 'wb') as f: while True: chunk = response.read(CHUNK) if not chunk: break f.write(chunk) #print url + "-->" + " Downloded!" return str(dst+fname)
def createTileGrid(self, src, dst, latidx, lonidx, z):
srcfile = ncOpen(src,"r")
fname = pathLeaf(src,False)
bbox = self.indices_to_bbox(latidx, lonidx)
inc = 1/float(2**z)
xsize = len(srcfile.variables['lon'][:])
ysize = len(srcfile.variables['lon'][:])
xfirst = bbox["lon_min"]
yfirst = bbox["lat_min"]
xinc = inc
yinc = inc
gf = GridFile("latlon", xsize, ysize, xfirst, xinc, yfirst, yinc)
fgrid = gf.createGrid(dst+fname+".grid")
return str(dst+fname+".grid")
def getUriList(self, bbidxs, src):
uri = []
for latidx in range(bbidxs['latidx_min'],bbidxs['latidx_max']+1):
for lonidx in range(bbidxs['lonidx_min'],bbidxs['lonidx_max']+1):
try:
url = "%s/%04d/clim_%04d_%04d.tile.nc4" %(src,latidx,latidx,lonidx)
urlopen(url)
except:
pass
else:
uri.append({'name': pathLeaf(url), 'uri':url, 'latidx':latidx, 'lonidx':lonidx})
return uri
def select_dile_by_uri():
"""Download a dile given a uri.
:example: /select/dile?uri=http://s3.amazonaws.com/edu-uchicago-rdcep-diles/fd65252e41e3cf0b431a07ad6e2cbe85/sdile_pr_2_1_1/pr/0/2/1/1/dile_0_2_1_1.nc
:param: uri -- a valid uri to access the dile
:returns: netcdf4 -- the return of the dile.
-------------------------------------------------------------------------------------------
"""
uri=request.args.get('uri')
if uri is not None:
if uri.startswith("http://s3.amazonaws.com/"):
path = uri.replace("http://s3.amazonaws.com/","")
bname, kstr = path.split("/",1) # split the bname from the key string
conn = S3Connection(AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
try:
bucket = conn.get_bucket(bname)
except:
print "BUCKET NOT FOUND"
return str("ERROR: bucket "+bname+" not found")
else:
print "BUCKET CONNECTED"
try:
key = bucket.get_key(kstr)
print "KEY: ", key
except:
print "KEY NOT FOUND"
return str("ERROR: key "+kstr+"not found")
else:
try:
key.open_read() # opens the file
headers = dict(key.resp.getheaders()) # request the headers
headers["Content-Disposition"] = "inline; filename="+str(pathLeaf(key.name))
return Response(key, headers=headers) # return a response
except S3ResponseError as e:
return Response(e.body, status=e.status, headers=key.resp.getheaders())
abort(400)
def regridTile(self, src, dst, fgrid, remap): fname = pathLeaf(src) cdo = CdoRegridder(src, dst+fname, fgrid) cdo.regridLocal(remap) return str(dst+fname)
def onIngest(self, srcpath, dstpath, bb, zoom):
df = DileFactory()
ng = NetcdfGeometry()
basename = pathLeaf(srcpath)
dimensions = [d.name for d in self.rgrp.dimensions.values()]
variables = [v for v in self.rgrp.variables.values() if v.name not in dimensions]
diles = df.fromBoundingBox(bb['lon_min'], bb['lat_min'], bb['lon_max'], bb['lat_max'],zoom)
dates = None
try:
dates = num2date(rgrp['time'][:],rgrp['time'].units)
except:
pass
# calculating the number of iterations required, for printing purposes
iter_len = 0
for var in variables:
iter_len += reduce(lambda x, y: x*y, var.shape[:-2])*len(diles)
x = 0
for var in variables:
timeind = None
levelind = None
try:
timeind = var.dimensions.index('time')
except:
pass
try:
levelind = var.dimensions.index('level')
except:
pass
y = 0
for ind in ndindex(var.shape[:-2]):
z = 0
for dile in diles:
docurl = dstpath
item = dile.asDocument()
indstr = []
if timeind is not None:
item['time'] = dates[ind[timeind]]
indstr.append(str(ind[timeind]))
if levelind is not None:
item['level'] = ind[levelind]
indstr.append(str(ind[levelind]))
docurl += ''.join([i+'/' for i in indstr])
docurl += dile.getRelativePath()
fname = dile.getFileName('dile_'+''.join([i+'_' for i in indstr]))
docurl += fname
item['md5'] = md5
item['variable'] = var.name
item['attributes'] = [{str(key): str(var.getncattr(key))} for key in var.ncattrs()]
item['uri'] = docurl
item['source'] = basename
item['zoom'] = zoom
item['cdate'] = datetime.now()
self.onInsert(item)
z += 1
printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname)
y += 1
x += 1
return iter_len
return iter_len def onClose(self): self.client.close() if __name__ == '__main__': ng = NetcdfGeometry() df = DileFactory() timer = Chrono() path = "/sdiles/ubuntu/sdiles/sdile_tasmax_2_0_1.nc" fname = pathLeaf(path,False) print "computing md5 for ", fname, "..." timer.start() md5 = getMD5(path) timer.stop() print "md5 computed in: ", timer.formatted() rgrp = ncOpen(path, mode='r') bb = ng.getBoundingBox(rgrp['lat'],rgrp['lon']) zoom = ng.getZoomLevel(rgrp['lat'], rgrp['lon']) mim = MetaIngesterMongo(rgrp) url = " http://s3.amazonaws.com/edu-uchicago-rdcep-diles/"
