def test_write(self):
ncvariable = NcVariable("Prcp","mm",constant=5)
bounds = Polygon(((0,0),(10,0),(10,15),(0,15)))
res = 5
ncspatial = NcSpatial(bounds,res)
rng = [datetime.datetime(2007,10,1),datetime.datetime(2007,10,3)]
interval = datetime.timedelta(days=1)
nctime = NcTime(rng,interval)
ncw = NcWrite(ncvariable,ncspatial,nctime)
path = get_temp_path(suffix='.nc')
rootgrp = ncw.get_rootgrp(path)
self.assertEquals(
rootgrp.variables["Prcp"][:].shape,
(3, 4, 3)
)
ncw = NcWrite(ncvariable,ncspatial,nctime,nlevels=4)
path = get_temp_path(suffix='.nc')
rootgrp = ncw.get_rootgrp(path)
self.assertEquals(
rootgrp.variables["Prcp"][:].shape,
(3, 4, 4, 3)
)
## check spatial dimensions
self.assertEqual(ncw.ncspatial.dim_col.shape[0],3)
self.assertEqual(ncw.ncspatial.dim_row.shape[0],4)
## write to a file
ncw.write()
def init_db(engine=None, to_disk=False, procs=1):
from sqlalchemy import create_engine
from sqlalchemy.orm.session import sessionmaker
from util.ncconv.experimental import db
from sqlalchemy.pool import NullPool
if engine is None:
path = 'sqlite://'
if to_disk or procs > 1:
path = path + '/' + get_temp_path('.sqlite', nest=True)
db.engine = create_engine(path, poolclass=NullPool)
else:
db.engine = create_engine(
path,
# connect_args={'check_same_thread':False},
# poolclass=StaticPool
)
else:
db.engine = engine
db.metadata.bind = db.engine
db.Session = sessionmaker(bind=db.engine)
db.metadata.create_all()
return (db)
def init_db(engine=None,to_disk=False,procs=1):
from sqlalchemy import create_engine
from sqlalchemy.orm.session import sessionmaker
from util.ncconv.experimental import db
from sqlalchemy.pool import NullPool
if engine is None:
path = 'sqlite://'
if to_disk or procs > 1:
path = path + '/' + get_temp_path('.sqlite',nest=True)
db.engine = create_engine(path,
poolclass=NullPool)
else:
db.engine = create_engine(path,
# connect_args={'check_same_thread':False},
# poolclass=StaticPool
)
else:
db.engine = engine
db.metadata.bind = db.engine
db.Session = sessionmaker(bind=db.engine)
db.metadata.create_all()
return(db)
def test_triangle_url(self):
ext = 'kml'
sop = 'intersects'
agg = 'false'
url = '/api/archive/usgs-cida-maurer/model/miroc3.2(medres)/scenario/sres-a1b/run/2/temporal/2000-01-01+2000-03-01/spatial/{1}+polygon((-94+39.75,-93.75+39.75,-93.75+40,-94+39.75))/aggregate/{2}/variable/pr.{0}'.format(ext,sop,agg)
response = self.client.get(url)
with open(get_temp_path(suffix='.'+ext),'w') as f:
f.write(response.content)
self.assertEqual(response.status_code,200)
def as_tabular(elements, var, wkt=False, wkb=False, path=None):
"""writes output in a tabular, CSV format
geometry output is optional"""
import osgeo.ogr as ogr
if path is None:
path = get_temp_path(suffix=".txt")
# define spatial references for the projection
sr = ogr.osr.SpatialReference()
sr.ImportFromEPSG(4326)
sr2 = ogr.osr.SpatialReference()
sr2.ImportFromEPSG(3005) # Albers Equal Area is used to ensure legitimate area values
with open(path, "w") as f:
for ii, element in enumerate(elements):
# convert area from degrees to m^2
geo = ogr.CreateGeometryFromWkb(element["geometry"].wkb)
geo.AssignSpatialReference(sr)
geo.TransformTo(sr2)
area = geo.GetArea()
# write id, timestamp, variable
f.write(
",".join(
[
repr(ii + 1),
element["properties"]["timestamp"].strftime("%Y-%m-%d %H:%M:%S"),
repr(element["properties"][var]),
]
)
)
# write level if the dataset has levels
if "level" in element["properties"].keys():
f.write("," + repr(element["properties"]["level"]))
# write the area
f.write("," + repr(area))
# write wkb if requested
if wkb:
f.write("," + repr(element["geometry"].wkb))
# write wkt if requested
if wkt:
f.write("," + repr(element["geometry"].wkt))
f.write("\n")
f.close()
return path
def test_write(self):
ncvariable = NcVariable("Prcp", "mm", constant=5)
bounds = Polygon(((0, 0), (10, 0), (10, 15), (0, 15)))
res = 5
ncspatial = NcSpatial(bounds, res)
rng = [datetime.datetime(2007, 10, 1), datetime.datetime(2007, 10, 3)]
interval = datetime.timedelta(days=1)
nctime = NcTime(rng, interval)
ncw = NcWrite(ncvariable, ncspatial, nctime)
path = get_temp_path(suffix='.nc')
rootgrp = ncw.get_rootgrp(path)
self.assertEquals(rootgrp.variables["Prcp"][:].shape, (3, 4, 3))
ncw = NcWrite(ncvariable, ncspatial, nctime, nlevels=4)
path = get_temp_path(suffix='.nc')
rootgrp = ncw.get_rootgrp(path)
self.assertEquals(rootgrp.variables["Prcp"][:].shape, (3, 4, 4, 3))
## check spatial dimensions
self.assertEqual(ncw.ncspatial.dim_col.shape[0], 3)
self.assertEqual(ncw.ncspatial.dim_row.shape[0], 4)
## write to a file
ncw.write()
def __init__(self, *args, **kwds):
self.layer = kwds.pop('layer', 'lyr')
self.srid = kwds.pop('srid', 4326)
## call the superclass
super(ShpConverter, self).__init__(*args, **kwds)
## generate dataset path
self.path = get_temp_path(name=self.base_name, nest=True)
## create shapefile base attributes
self.fcache = FieldCache()
self.ogr_fields = []
self._set_ogr_fields_()
## get the geometry in order
# self.ogr_geom = OGRGeomType(self.sub_ocg_dataset.geometry[0].geometryType()).num
self.ogr_geom = 6 ## assumes multipolygon
self.srs = osr.SpatialReference()
self.srs.ImportFromEPSG(self.srid)
def __init__(self,*args,**kwds):
self.layer = kwds.pop('layer','lyr')
self.srid = kwds.pop('srid',4326)
## call the superclass
super(ShpConverter,self).__init__(*args,**kwds)
## generate dataset path
self.path = get_temp_path(name=self.base_name,nest=True)
## create shapefile base attributes
self.fcache = FieldCache()
self.ogr_fields = []
self._set_ogr_fields_()
## get the geometry in order
# self.ogr_geom = OGRGeomType(self.sub_ocg_dataset.geometry[0].geometryType()).num
self.ogr_geom = 6 ## assumes multipolygon
self.srs = osr.SpatialReference()
self.srs.ImportFromEPSG(self.srid)
def handle_uploaded_shapefile(filename, uid_field=None):
path = get_temp_path(nest=True, suffix='.zip')
dir = os.path.split(path)[0]
## write the data to file
with open(path, 'wb+') as dest:
for chunk in filename.chunks():
dest.write(chunk)
## unzip the file
zip = zipfile.ZipFile(path, 'r')
try:
zip.extractall(os.path.split(path)[0])
finally:
zip.close()
## get the shapefile path
for f in os.listdir(dir):
if f.endswith('.shp'):
break
## extract the wkt
wkt_data = get_shp_as_multi(os.path.join(dir, f), uid_field=uid_field)
return (wkt_data)
def handle_uploaded_shapefile(filename,uid_field=None):
path = get_temp_path(nest=True,suffix='.zip')
dir = os.path.split(path)[0]
## write the data to file
with open(path,'wb+') as dest:
for chunk in filename.chunks():
dest.write(chunk)
## unzip the file
zip = zipfile.ZipFile(path,'r')
try:
zip.extractall(os.path.split(path)[0])
finally:
zip.close()
## get the shapefile path
for f in os.listdir(dir):
if f.endswith('.shp'):
break
## extract the wkt
wkt_data = get_shp_as_multi(os.path.join(dir,f),uid_field=uid_field)
return(wkt_data)
def write(self):
zip_stream = self.response()
path = get_temp_path(suffix='.zip')
with open(path, 'wb') as f:
f.write(zip_stream)
return (path)
def _convert_(self, ocg_dataset, has_levels=False, fill_value=1e20):
if self.use_stat:
sub = self.sub.sub
substat = self.sub
else:
sub = self.sub
substat = None
print('starting convert...')
## create the dataset object
path = get_temp_path(name=self.base_name, nest=True)
tdataset = nc.Dataset(path, 'w')
try:
## return the grid dictionary
grid = sub.to_grid_dict(ocg_dataset)
## initialize the element classes
checks = []
for check in element.PolyElement.get_checks():
try:
obj = check(ocg_dataset.dataset)
checks.append(obj)
except PolyElementNotFound:
if not has_levels:
pass
else:
raise
except:
import ipdb
ipdb.set_trace()
## first do a loop over the dataset attributes
for attr in ocg_dataset.dataset.ncattrs():
captured = None
for check in checks:
if check.name == attr and isinstance(
check, element.DatasetPolyElement):
captured = check
break
if captured is None:
calc = getattr(ocg_dataset.dataset, attr)
else:
if isinstance(captured,
element.SimpleTranslationalElement):
calc = captured.calculate()
elif isinstance(captured,
element.SpatialTranslationalElement):
calc = captured.calculate(grid)
elif isinstance(captured,
element.TemporalTranslationalElement):
calc = captured.calculate(sub.timevec)
elif isinstance(captured, models.FileName):
calc = self.base_name
else:
raise (ValueError)
try:
setattr(tdataset, attr, calc)
except:
## need to account for unicode
setattr(tdataset, attr, str(calc))
## create the dimensions
for dim in ocg_dataset.dataset.dimensions.keys():
for check in checks:
if check.name == dim and isinstance(
check, element.DimensionElement):
captured = check
break
if isinstance(captured, element.TemporalDimensionElement):
if self.use_stat:
continue
calc = captured.calculate(sub.timevec)
elif isinstance(captured, element.SpatialDimensionElement):
calc = captured.calculate(grid)
elif isinstance(captured, element.LevelDimensionElement):
calc = captured.calculate(sub.levelvec)
else:
raise (ValueError)
tdataset.createDimension(captured.name, calc)
## create the variables
for var in ocg_dataset.dataset.variables.keys():
captured = None
for check in checks:
if check.name == var and isinstance(
check, element.VariablePolyElement):
captured = check
break
if captured is None: continue
if isinstance(captured, models.Row):
calc = captured.make_dimension_tup(
models.LatitudeDimension(ocg_dataset.dataset))
elif isinstance(captured, models.Column):
calc = captured.make_dimension_tup(
models.LongitudeDimension(ocg_dataset.dataset))
elif isinstance(captured, models.RowBounds):
calc = captured.make_dimension_tup(
models.LatitudeDimension(ocg_dataset.dataset),
models.BoundsDimension(ocg_dataset.dataset))
elif isinstance(captured, models.ColumnBounds):
calc = captured.make_dimension_tup(
models.LongitudeDimension(ocg_dataset.dataset),
models.BoundsDimension(ocg_dataset.dataset))
elif isinstance(captured, models.Time):
if self.use_stat:
continue
calc = captured.make_dimension_tup(
models.TimeDimension(ocg_dataset.dataset))
else:
raise
tdataset.createVariable(captured.name, captured._dtype, calc)
## set the variable's data
if isinstance(captured, element.TemporalTranslationalElement):
calc = captured.calculate(sub.timevec)
elif isinstance(captured, element.SpatialTranslationalElement):
calc = captured.calculate(grid)
# elif isinstance(captured,element.LevelDimensionElement):
# calc = captured.calculate(sub.levelvec)
else:
raise (ValueError)
tdataset.variables[captured.name][:] = calc
## set the variable's attrs
for attr in ocg_dataset.dataset.variables[
captured.name].ncattrs():
setattr(
tdataset.variables[captured.name], attr,
getattr(ocg_dataset.dataset.variables[captured.name],
attr))
## set the actual value
if self.use_stat:
if has_levels:
raise (NotImplementedError)
else:
## these are the columns to exclude
exclude = ['ocgid', 'gid', 'level', 'geometry']
## get the columns we want to write to the netcdf
cs = [c for c in substat.stats.keys() if c not in exclude]
## loop through the columns and generate the numpy arrays to
## to populate.
print('making variables...')
for ii, c in enumerate(cs):
## get the correct python type from the column type
if type(substat.stats[c][0]) == float:
nctype = 'f4'
if type(substat.stats[c][0]) == int:
nctype = 'i4'
## make the netcdf variable
tdataset.createVariable(c, nctype,
('latitude', 'longitude'))
## check for parallel
if settings.MAXPROCESSES > 1:
manager = Manager()
data = manager.list()
print('configuring processes...')
## create the indices over which to split jobs
count = len(substat.stats['gid'])
indices = [[min(ary), max(ary)] for ary in array_split(
range(0, count + 1), settings.MAXPROCESSES)]
## construct the processes
procs = [
Process(target=self.f_fill,
args=(data, rng, sub, substat,
grid['gidx'].reshape(-1), cs))
for rng in indices
]
pmanager = ProcessManager(procs, settings.MAXPROCESSES)
## run the processes
print('executing processes...')
pmanager.run()
## reshape/transform list data into numpy arrays
## the dictionary to hold merged data
merged = dict.fromkeys(
data[0].keys(),
np.zeros(len(grid['gidx'].reshape(-1))))
## merge the data
for dd in data:
for key, value in dd.iteritems():
merged[key] = merged[key] + value
print('reformatting arrays...')
for key, value in merged.iteritems():
tary = value.reshape(len(grid['y']),
len(grid['x']))
tary[grid['gidx'].mask] = fill_value
merged.update({key: tary})
else:
raise (NotImplementedError)
## set the variable value in the nc dataset
for key, value in merged.iteritems():
tdataset.variables[key].missing_value = fill_value
tdataset.variables[key][:] = value
else:
gidx = grid['gidx']
if has_levels:
raise (NotImplementedError)
else:
value = np.empty(
(len(sub.timevec), len(grid['y']), len(grid['x'])),
dtype=float)
for dt in sub.dim_time:
for ii, jj in itr_array(gidx):
if not hasattr(gidx[ii, jj], 'mask'):
tgidx = gidx[ii, jj]
value[dt, ii, jj] = sub.value[dt, 0, tgidx]
else:
value[dt, ii, jj] = fill_value
tdataset.createVariable('value', 'f4',
('time', 'latitude', 'longitude'))
tdataset.variables['value'].missing_value = fill_value
tdataset.variables['value'][:] = value
tdataset.sync()
return (path)
finally:
tdataset.close()
def as_shp(elements,path=None):
if path is None:
path = get_temp_path(suffix='.shp')
ocs = OpenClimateShp(path,elements)
ocs.write()
return(path)
def as_tabular(elements,
var,
wkt=False,
wkb=False,
todisk=False,
area_srid=3005,
path=None):
'''writes output in a tabular, CSV format geometry output is optional
elements -- standard geojson-like data representation
var -- name of the output variable
wkt=False -- set to True to write wkt to text file
wkb=False -- set to True to write wkb to text file
todisk=False -- set to True to write output to disk as well. if no path
name is specified in the |path| argument, then one is generated.
path=None -- specify an output file name. under the default, no file is
written.
'''
## define spatial references
sr = get_sr(4326)
sr2 = get_sr(area_srid)
## this will hold the data to write
data = []
## prepare column header
header = ['id', 'timestamp', var]
if 'level' in elements[0]['properties'].keys():
header += ['level']
header += ['area_m2']
if wkb:
header += ['wkb']
if wkt:
header += ['wkt']
data.append(header)
for ii, element in enumerate(elements):
## will hold data to append to global list
subdata = []
#convert area from degrees to m^2
geo = ogr.CreateGeometryFromWkb(element['geometry'].wkb)
geo.AssignSpatialReference(sr)
geo.TransformTo(sr2)
area = geo.GetArea()
## retrieve additional elements ----------------------------------------
props = element['properties']
geom = element['geometry']
subdata.append(ii + 1)
subdata.append(props['timestamp'])
subdata.append(props[var])
subdata.append(area)
if 'level' in props.keys(): subdata.append(props['level'])
if wkb: subdata.append(geom.wkb)
if wkt: subdata.append(geom.wkt)
## append to global data
data.append(subdata)
## write to the buffer
buffer = io.BytesIO()
writer = csv.writer(buffer)
writer.writerows(data)
## set-up disk writing
if todisk:
if path is None:
path = get_temp_path(suffix='.txt')
with open(path, 'w') as f:
f.write(buffer.getvalue())
try:
return (buffer.getvalue())
finally:
buffer.close()
def as_keyTabular(elements,
var,
wkt=False,
wkb=False,
path=None,
area_srid=3005):
'''writes output as tabular csv files, but uses foreign keys
on time and geometry to reduce file size'''
if path is None:
path = get_temp_path(suffix='')
# prefix = os.path.splitext(path)[0]
#
# if len(path)>4 and path[-4] == '.':
# path = path[:-4]
patht = path + "_time.csv"
pathg = path + "_geometry.csv"
pathd = path + "_data.csv"
paths = [patht, pathg, pathd]
#define spatial references for the projection
sr = get_sr(4326)
sr2 = get_sr(area_srid)
data = {}
#sort the data into dictionaries so common times and geometries can be identified
for ii, element in enumerate(elements):
#record new element ids (otherwise threads will produce copies of ids)
element['id'] = ii + 1
#get the time and geometry
time = element['properties']['timestamp'].strftime("%Y-%m-%d %H:%M:%S")
ewkt = element['geometry'].wkt
if not (time in data):
data[time] = {}
#put the data into the dictionary
if not (ewkt in data[time]):
data[time][ewkt] = [element]
else:
data[time][ewkt].append(element)
#get a unique set of geometry keys
locs = []
for key in data:
locs.extend(data[key].keys())
locs = set(locs)
ft = open(patht, 'w')
fg = open(pathg, 'w')
fd = open(pathd, 'w')
ft.write(','.join(['tid', 'timestamp']))
ft.write('\n')
fgheader = ['gid', 'area_m2']
if wkt:
fgheader += ['wkt']
if wkb:
fgheader += ['wkb']
fg.write(','.join(fgheader))
fg.write('\n')
fdheader = ['id', 'tid', 'gid', var]
if 'level' in elements[0]['properties']:
fdheader += ['level']
fd.write(','.join(fdheader))
fd.write('\n')
#write the features to file
for ii, time in enumerate(data.keys()):
#write out id's and time values to the time file
tdat = data[time]
ft.write(repr(ii + 1) + ',' + time + '\n')
for jj, loc in enumerate(locs):
if ii == 0:
#find the geometry area
geo = ogr.CreateGeometryFromWkt(loc)
geo.AssignSpatialReference(sr)
geo.TransformTo(sr2)
#write the id and area
fg.write(repr(jj + 1))
fg.write(',' + repr(geo.GetArea()))
#write out optional geometry
if wkt:
fg.write(',' + loc)
if wkb:
fg.write(
',' +
repr(ogr.CreateGeometryFromWkt(loc).ExportToWkb()))
fg.write('\n')
if loc in tdat:
for element in tdat[loc]:
#write out id, foreign keys (time then geometry) and the variable value
fd.write(','.join([
repr(element['id']),
repr(ii + 1),
repr(jj + 1),
repr(element['properties'][var])
]))
#write out level if appropriate
if 'level' in element['properties']:
fd.write(',' + repr(element['properties']['level']))
fd.write('\n')
ft.close()
fg.close()
fd.close()
return (paths)
def as_keyTabular(elements,var,wkt=False,wkb=False,path=None,area_srid=3005):
'''writes output as tabular csv files, but uses foreign keys
on time and geometry to reduce file size'''
if path is None:
path = get_temp_path(suffix='')
# prefix = os.path.splitext(path)[0]
#
# if len(path)>4 and path[-4] == '.':
# path = path[:-4]
patht = path+"_time.csv"
pathg = path+"_geometry.csv"
pathd = path+"_data.csv"
paths = [patht,pathg,pathd]
#define spatial references for the projection
sr = get_sr(4326)
sr2 = get_sr(area_srid)
data = {}
#sort the data into dictionaries so common times and geometries can be identified
for ii,element in enumerate(elements):
#record new element ids (otherwise threads will produce copies of ids)
element['id'] = ii + 1
#get the time and geometry
time = element['properties']['timestamp'].strftime("%Y-%m-%d %H:%M:%S")
ewkt = element['geometry'].wkt
if not (time in data):
data[time] = {}
#put the data into the dictionary
if not (ewkt in data[time]):
data[time][ewkt] = [element]
else:
data[time][ewkt].append(element)
#get a unique set of geometry keys
locs = []
for key in data:
locs.extend(data[key].keys())
locs = set(locs)
ft = open(patht,'w')
fg = open(pathg,'w')
fd = open(pathd,'w')
ft.write(','.join(['tid','timestamp']))
ft.write('\n')
fgheader = ['gid','area_m2']
if wkt:
fgheader += ['wkt']
if wkb:
fgheader += ['wkb']
fg.write(','.join(fgheader))
fg.write('\n')
fdheader = ['id','tid','gid',var]
if 'level' in elements[0]['properties']:
fdheader += ['level']
fd.write(','.join(fdheader))
fd.write('\n')
#write the features to file
for ii,time in enumerate(data.keys()):
#write out id's and time values to the time file
tdat = data[time]
ft.write(repr(ii+1)+','+time+'\n')
for jj,loc in enumerate(locs):
if ii==0:
#find the geometry area
geo = ogr.CreateGeometryFromWkt(loc)
geo.AssignSpatialReference(sr)
geo.TransformTo(sr2)
#write the id and area
fg.write(repr(jj+1))
fg.write(','+repr(geo.GetArea()))
#write out optional geometry
if wkt:
fg.write(','+loc)
if wkb:
fg.write(','+repr(ogr.CreateGeometryFromWkt(loc).ExportToWkb()))
fg.write('\n')
if loc in tdat:
for element in tdat[loc]:
#write out id, foreign keys (time then geometry) and the variable value
fd.write(','.join([repr(element['id']),repr(ii+1),repr(jj+1),repr(element['properties'][var])]))
#write out level if appropriate
if 'level' in element['properties']:
fd.write(','+repr(element['properties']['level']))
fd.write('\n')
ft.close()
fg.close()
fd.close()
return(paths)
def as_tabular(elements,var,wkt=False,wkb=False,todisk=False,area_srid=3005,path=None):
'''writes output in a tabular, CSV format geometry output is optional
elements -- standard geojson-like data representation
var -- name of the output variable
wkt=False -- set to True to write wkt to text file
wkb=False -- set to True to write wkb to text file
todisk=False -- set to True to write output to disk as well. if no path
name is specified in the |path| argument, then one is generated.
path=None -- specify an output file name. under the default, no file is
written.
'''
## define spatial references
sr = get_sr(4326)
sr2 = get_sr(area_srid)
## this will hold the data to write
data = []
## prepare column header
header = ['id','timestamp',var]
if 'level' in elements[0]['properties'].keys():
header += ['level']
header += ['area_m2']
if wkb:
header += ['wkb']
if wkt:
header += ['wkt']
data.append(header)
for ii,element in enumerate(elements):
## will hold data to append to global list
subdata = []
#convert area from degrees to m^2
geo = ogr.CreateGeometryFromWkb(element['geometry'].wkb)
geo.AssignSpatialReference(sr)
geo.TransformTo(sr2)
area = geo.GetArea()
## retrieve additional elements ----------------------------------------
props = element['properties']
geom = element['geometry']
subdata.append(ii+1)
subdata.append(props['timestamp'])
subdata.append(props[var])
subdata.append(area)
if 'level' in props.keys(): subdata.append(props['level'])
if wkb: subdata.append(geom.wkb)
if wkt: subdata.append(geom.wkt)
## append to global data
data.append(subdata)
## write to the buffer
buffer = io.BytesIO()
writer = csv.writer(buffer)
writer.writerows(data)
## set-up disk writing
if todisk:
if path is None:
path = get_temp_path(suffix='.txt')
with open(path,'w') as f:
f.write(buffer.getvalue())
try:
return(buffer.getvalue())
finally:
buffer.close()
def _convert_(self,ocg_dataset,has_levels=False,fill_value=1e20):
if self.use_stat:
sub = self.sub.sub
substat = self.sub
else:
sub = self.sub
substat = None
print('starting convert...')
## create the dataset object
path = get_temp_path(name=self.base_name,nest=True)
tdataset = nc.Dataset(path,'w')
try:
## return the grid dictionary
grid = sub.to_grid_dict(ocg_dataset)
## initialize the element classes
checks = []
for check in element.PolyElement.get_checks():
try:
obj = check(ocg_dataset.dataset)
checks.append(obj)
except PolyElementNotFound:
if not has_levels:
pass
else:
raise
except:
import ipdb;ipdb.set_trace()
## first do a loop over the dataset attributes
for attr in ocg_dataset.dataset.ncattrs():
captured = None
for check in checks:
if check.name == attr and isinstance(check,element.DatasetPolyElement):
captured = check
break
if captured is None:
calc = getattr(ocg_dataset.dataset,attr)
else:
if isinstance(captured,element.SimpleTranslationalElement):
calc = captured.calculate()
elif isinstance(captured,element.SpatialTranslationalElement):
calc = captured.calculate(grid)
elif isinstance(captured,element.TemporalTranslationalElement):
calc = captured.calculate(sub.timevec)
elif isinstance(captured,models.FileName):
calc = self.base_name
else:
raise(ValueError)
try:
setattr(tdataset,attr,calc)
except:
## need to account for unicode
setattr(tdataset,attr,str(calc))
## create the dimensions
for dim in ocg_dataset.dataset.dimensions.keys():
for check in checks:
if check.name == dim and isinstance(check,element.DimensionElement):
captured = check
break
if isinstance(captured,element.TemporalDimensionElement):
if self.use_stat:
continue
calc = captured.calculate(sub.timevec)
elif isinstance(captured,element.SpatialDimensionElement):
calc = captured.calculate(grid)
elif isinstance(captured,element.LevelDimensionElement):
calc = captured.calculate(sub.levelvec)
else:
raise(ValueError)
tdataset.createDimension(captured.name,calc)
## create the variables
for var in ocg_dataset.dataset.variables.keys():
captured = None
for check in checks:
if check.name == var and isinstance(check,element.VariablePolyElement):
captured = check
break
if captured is None: continue
if isinstance(captured,models.Row):
calc = captured.make_dimension_tup(models.LatitudeDimension(ocg_dataset.dataset))
elif isinstance(captured,models.Column):
calc = captured.make_dimension_tup(models.LongitudeDimension(ocg_dataset.dataset))
elif isinstance(captured,models.RowBounds):
calc = captured.make_dimension_tup(models.LatitudeDimension(ocg_dataset.dataset),
models.BoundsDimension(ocg_dataset.dataset))
elif isinstance(captured,models.ColumnBounds):
calc = captured.make_dimension_tup(models.LongitudeDimension(ocg_dataset.dataset),
models.BoundsDimension(ocg_dataset.dataset))
elif isinstance(captured,models.Time):
if self.use_stat:
continue
calc = captured.make_dimension_tup(models.TimeDimension(ocg_dataset.dataset))
else:
raise
tdataset.createVariable(captured.name,captured._dtype,calc)
## set the variable's data
if isinstance(captured,element.TemporalTranslationalElement):
calc = captured.calculate(sub.timevec)
elif isinstance(captured,element.SpatialTranslationalElement):
calc = captured.calculate(grid)
# elif isinstance(captured,element.LevelDimensionElement):
# calc = captured.calculate(sub.levelvec)
else:
raise(ValueError)
tdataset.variables[captured.name][:] = calc
## set the variable's attrs
for attr in ocg_dataset.dataset.variables[captured.name].ncattrs():
setattr(tdataset.variables[captured.name],attr,getattr(ocg_dataset.dataset.variables[captured.name],attr))
## set the actual value
if self.use_stat:
if has_levels:
raise(NotImplementedError)
else:
## these are the columns to exclude
exclude = ['ocgid','gid','level','geometry']
## get the columns we want to write to the netcdf
cs = [c for c in substat.stats.keys() if c not in exclude]
## loop through the columns and generate the numpy arrays to
## to populate.
print('making variables...')
for ii,c in enumerate(cs):
## get the correct python type from the column type
if type(substat.stats[c][0]) == float:
nctype = 'f4'
if type(substat.stats[c][0]) == int:
nctype = 'i4'
## make the netcdf variable
tdataset.createVariable(c,nctype,('latitude','longitude'))
## check for parallel
if settings.MAXPROCESSES > 1:
manager = Manager()
data = manager.list()
print('configuring processes...')
## create the indices over which to split jobs
count = len(substat.stats['gid'])
indices = [[min(ary),max(ary)]
for ary in array_split(range(0,count+1),
settings.MAXPROCESSES)]
## construct the processes
procs = [Process(target=self.f_fill,
args=(data,rng,sub,substat,grid['gidx'].reshape(-1),cs))
for rng in indices]
pmanager = ProcessManager(procs,settings.MAXPROCESSES)
## run the processes
print('executing processes...')
pmanager.run()
## reshape/transform list data into numpy arrays
## the dictionary to hold merged data
merged = dict.fromkeys(data[0].keys(),np.zeros(len(grid['gidx'].reshape(-1))))
## merge the data
for dd in data:
for key,value in dd.iteritems():
merged[key] = merged[key] + value
print('reformatting arrays...')
for key,value in merged.iteritems():
tary = value.reshape(len(grid['y']),len(grid['x']))
tary[grid['gidx'].mask] = fill_value
merged.update({key:tary})
else:
raise(NotImplementedError)
## set the variable value in the nc dataset
for key,value in merged.iteritems():
tdataset.variables[key].missing_value = fill_value
tdataset.variables[key][:] = value
else:
gidx = grid['gidx']
if has_levels:
raise(NotImplementedError)
else:
value = np.empty((len(sub.timevec),len(grid['y']),len(grid['x'])),dtype=float)
for dt in sub.dim_time:
for ii,jj in itr_array(gidx):
if not hasattr(gidx[ii,jj],'mask'):
tgidx = gidx[ii,jj]
value[dt,ii,jj] = sub.value[dt,0,tgidx]
else:
value[dt,ii,jj] = fill_value
tdataset.createVariable('value','f4',('time','latitude','longitude'))
tdataset.variables['value'].missing_value = fill_value
tdataset.variables['value'][:] = value
tdataset.sync()
return(path)
finally:
tdataset.close()
def write(self,path=None):
if path is None:
path = get_temp_path('.nc')
rootgrp = self.get_rootgrp(path)
rootgrp.close()
return(path)
def as_keyTabular(elements, var, wkt=False, wkb=False, path=None):
"""writes output as tabular csv files, but uses foreign keys
on time and geometry to reduce file size"""
import osgeo.ogr as ogr
if path is None:
path = get_temp_path(suffix="")
if len(path) > 4 and path[-4] == ".":
path = path[:-4]
patht = path + "_time.txt"
pathg = path + "_geometry.txt"
pathd = path + "_data.txt"
# define spatial references for the projection
sr = ogr.osr.SpatialReference()
sr.ImportFromEPSG(4326)
sr2 = ogr.osr.SpatialReference()
sr2.ImportFromEPSG(3005)
data = {}
# sort the data into dictionaries so common times and geometries can be identified
for ii, element in enumerate(elements):
# record new element ids (otherwise threads will produce copies of ids)
element["id"] = ii
# get the time and geometry
time = element["properties"]["timestamp"].strftime("%Y-%m-%d %H:%M:%S")
ewkt = element["geometry"].wkt
if not (time in data):
data[time] = {}
# put the data into the dictionary
if not (ewkt in data[time]):
data[time][ewkt] = [element]
else:
data[time][ewkt].append(element)
# get a unique set of geometry keys
locs = []
for key in data:
locs.extend(data[key].keys())
locs = set(locs)
ft = open(patht, "w")
fg = open(pathg, "w")
fd = open(pathd, "w")
# write the features to file
for ii, time in enumerate(data.keys()):
# write out id's and time values to the time file
tdat = data[time]
ft.write(repr(ii + 1) + "," + time + "\n")
for jj, loc in enumerate(locs):
if ii == 0:
# find the geometry area
geo = ogr.CreateGeometryFromWkt(loc)
geo.AssignSpatialReference(sr)
geo.TransformTo(sr2)
# write the id and area
fg.write(repr(jj + 1))
fg.write("," + repr(geo.GetArea()))
# write out optional geometry
if wkt:
fg.write("," + loc)
if wkb:
fg.write("," + repr(ogr.CreateGeometryFromWkt(loc).ExportToWkb()))
fg.write("\n")
if loc in tdat:
for element in tdat[loc]:
# write out id, foreign keys (time then geometry) and the variable value
fd.write(
",".join([repr(element["id"]), repr(ii + 1), repr(jj + 1), repr(element["properties"][var])])
)
# write out level if appropriate
if "level" in element["properties"]:
fd.write("," + repr(element["properties"]["level"]))
fd.write("\n")
ft.close()
fg.close()
fd.close()
def write(self, path=None):
if path is None:
path = get_temp_path(".nc")
rootgrp = self.get_rootgrp(path)
rootgrp.close()
return path
def write(self):
zip_stream = self.response()
path = get_temp_path(suffix='.zip')
with open(path,'wb') as f:
f.write(zip_stream)
return(path)