def onUpload(self, ext, src='',folder='', dst=''):
lpw = LukePathWalker()
print "fetching files..."
files = []
for file in lpw.getDirectoryContent(src+folder):
if lpw.checkExtention(ext, file):
files.append(file)
print len(files)," files fetched. Uploading..."
n = 0 # number of iteration completed for printing pourposes
m = 0 # number of files skipped
for fname in files:
srcpath = fname
# allows for custom path in the destination folder to be placed before the path
dstpath = os.path.join(dst,fname[len(src):])
# compute the size of the file
fsize = os.path.getsize(srcpath)
# create a key to use as destination
k = boto.s3.key.Key(self.bucket)
# it will be set as the destination path
k.key = dstpath
if not self.keyExists(k):
# selection between single-part upload and multipart (in case is too big)
if fsize > self.MAX_SIZE:
mp = self.bucket.initiate_multipart_upload(dstpath)
fp = open(srcpath,'rb')
fpnum = 0
while (fp.tell() < fsize):
fpnum += 1
# function to upload a file in chunks
mp.upload_part_from_file(fp, fpnum, size=self.PART_SIZE)
mp.complete_upload() # function to notify the completion of the multipart upload
else:
# uploads a file in its entirety
k.set_contents_from_filename(srcpath)
else:
m += 1
printProgress(n, len(files),"uploading on: "+self.bname, pathLeaf(fname))
n += 1
return [n-m,m,n]
def asJson(self, show = False):
jdoc = {}
if self.path:
jdoc.update({"File": pathLeaf()})
if self.dims:
jdoc.update({"dimensions": self.dims})
if self.vars:
jdoc.update({"variables": self.vars})
if self.glob:
jdoc.update({"globals": self.glob})
if show:
print json.dumps(jdoc, indent = 2)
return jdoc
def createDiles(self, dstpath, bb, zoom):
df = DileFactory()
ng = NetcdfGeometry()
basename = pathLeaf(dstpath)
# delimiters for the diles contained in the matrix
indx, indy = ng.getDileIndexTable(bb,zoom)
# information extracted from the netcdf (dimensions, variables and diles' coverage)
dimensions = [d.name for d in self.rgrp.dimensions.values()]
variables = [v for v in self.rgrp.variables.values() if v.name not in dimensions]
diles = df.fromBoundingBox(bb['lon_min'], bb['lat_min'], bb['lon_max'], bb['lat_max'],zoom)
# calculating the number of iterations required, for printing purposes
iter_len = 0
for var in variables:
if len(var.shape) > 2:
iter_len += reduce(lambda x, y: x*y, var.shape[:-2])*len(diles)
else:
iter_len += len(diles)
# x,y,z indices for printing purposes
x = 0
for var in variables:
y = 0
timeind = None
levelind = None
try:
timeind = var.dimensions.index('time')
except:
pass
try:
levelind = var.dimensions.index('level')
except:
pass
for ind in ndindex(var.shape[:-2]):
z = 0
for dile in diles:
indstr = []
if timeind is not None: indstr.append(str(ind[timeind]))
if levelind is not None: indstr.append(str(ind[levelind]))
fname = dile.getFileName('dile_'+''.join([i+'_' for i in indstr]))
fpath = ''.join([i+'/' for i in indstr])+dile.getRelativePath()
path = dstpath+'/'+var.name+'/'+fpath+fname
a = indy[dile.y]["min"]
b = indy[dile.y]["max"]
c = indx[dile.x]["min"]
d = indx[dile.x]["max"]
if isinstance(var[ind], MaskedArray):
matrix = var[ind].data
else:
matrix = var[ind]
attributes = {}
for attr in var.ncattrs():
attributes.update({attr:var.getncattr(attr)})
dile.variable = var.name
dile.attributes = attributes
if '_FillValue' in var.ncattrs():
dile.createNetCDF4(path,matrix[a:b+1,c:d+1],var.dtype,var.getncattr('_FillValue'))
else:
dile.createNetCDF4(path,matrix[a:b+1,c:d+1])
z += 1
printProgress(x*(len(ind)*len(diles))+(y*len(diles)+z), iter_len, basename, fname)
y += 1
x += 1
return iter_len
myext = ["nc"]
paths = []
for file in lpw.getDirectoryContent(mydir):
if lpw.checkExtention(myext, file):
paths.append(file)
for i in range(50,len(paths)):
timer.start()
md5 = getMD5(paths[i])
timer.stop()
print i, ") md5 for ",pathLeaf(paths[i],ext=True), " computed in: ", timer.formatted()
src = ncOpen(paths[i], mode='r')
bb = ncg.getBoundingBox(src['lat'],src['lon'])
z = ncg.getZoomLevel(src['lat'],src['lon'])
ncs = NetcdfSlicer(src)
timer.reset()
timer.start()
ndiles = ncs.createDiles("/sdiles/ubuntu/diles/"+md5+"/"+pathLeaf(paths[i],ext=False),bb,int(z))
timer.stop()
print ndiles, " created. task completed in: ", timer.formatted()
src.close()