def crop_image(self, im):
im_ul = (self.point_px[0] - self.target_size[0] // 2,
self.point_px[1] - self.target_size[1] // 2)
box = (*im_ul, im_ul[0] + self.target_size[0],
im_ul[1] + self.target_size[1])
return im.crop(box), BBox(
(self.to_world(box[0], box[3])) + self.to_world(box[2], box[1]),
projection=Proj(init='epsg:3857'))
def bbox(self):
half_x_pixel_size = self.x.pixel_size / 2.0
half_y_pixel_size = self.y.pixel_size / 2.0
return BBox(
(self.x.min - half_x_pixel_size, self.y.min - half_y_pixel_size,
self.x.max + half_x_pixel_size, self.y.max + half_y_pixel_size),
self.projection)
def test_SpatialCoordinateVariables_slice_by_bbox():
lat = SpatialCoordinateVariable(numpy.arange(19, -1, -1))
lon = SpatialCoordinateVariable(numpy.arange(10))
proj = Proj(init='EPSG:4326')
coords = SpatialCoordinateVariables(lon, lat, proj)
subset = coords.slice_by_bbox(BBox((1.75, 3.7, 6.2, 16.7), proj))
assert numpy.array_equal(subset.x.values, numpy.arange(2, 6))
assert subset.x.values[0] == 2
assert subset.x.values[-1] == 5
assert subset.y.values[0] == 16
assert subset.y.values[-1] == 4
def map_eems(
eems_file,
# output_directory,
scale,
format,
src_crs,
resampling):
"""
Render a NetCDF EEMS model to a web map.
"""
from EEMSBasePackage import EEMSCmd, EEMSProgram
model = EEMSProgram(eems_file)
# For each data producing command, store the netcdf file that contains it
file_vars = dict()
raw_variables = set()
for cmd in model.orderedCmds: # This is bottom up, may want to invert
filename = None
variable = None
if cmd.HasResultName():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetResultName()
elif cmd.IsReadCmd():
filename = cmd.GetParam('OutFileName')
variable = cmd.GetParam('NewFieldName')
raw_variables.add(variable)
if filename and variable:
if not filename in file_vars:
file_vars[filename] = []
file_vars[filename].append(variable)
filenames = file_vars.keys()
for filename in filenames:
if not os.path.exists(filename):
raise click.ClickException(
'Could not find data file from EEMS model: {0}'.format(
filename))
dst_crs = 'EPSG:3857'
output_directory = tempfile.mkdtemp()
click.echo('Using temp directory: {0}'.format(output_directory))
# if not os.path.exists(output_directory):
# os.makedirs(output_directory)
# Since fuzzy renderer is hardcoded, we can output it now
fuzzy_renderer = palette_to_stretched_renderer(DEFAULT_PALETTES['fuzzy'],
'1,-1')
fuzzy_renderer.get_legend(image_height=150)[0].to_image().save(
os.path.join(output_directory, 'fuzzy_legend.png'))
template_filename = filenames[0]
template_var = file_vars[template_filename][0]
with Dataset(template_filename) as ds:
var_obj = ds.variables[template_var]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions != 2:
raise click.ClickException(
'Only 2 dimensions are allowed on data variables for now')
ds_crs = get_crs(ds, template_var)
if not ds_crs and is_geographic(ds, template_var):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS(
{'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds, x_dim, y_dim, projection=Proj(src_crs) if src_crs else None)
#
# if mask is not None and not mask.shape == shape[-2:]:
# # Will likely break before this if collecting statistics
# raise click.BadParameter(
# 'mask variable shape does not match shape of input spatial dimensions',
# param='--mask', param_hint='--mask'
# )
#
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs, dst_crs, src_width, src_height, *coords.bbox.as_list())
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(RESAMPLING, resampling),
'dst_shape': (dst_height, dst_width)
}
if not (dst_crs or src_crs):
raise click.BadParameter(
'must provide valid src_crs to get interactive map',
param='--src-crs',
param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(
BBox.from_affine(dst_transform,
dst_width,
dst_height,
projection=Proj(dst_crs) if dst_crs else None))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing dataset {0}'.format(filename))
for variable in file_vars[filename]:
click.echo('Processing variable {0}'.format(variable))
if not variable in ds.variables:
raise click.ClickException(
'variable {0} was not found in file: {1}'.format(
variable, filename))
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException(
'All datasets must have the same dimensions for {0}'.
format(variable))
data = var_obj[:]
# if mask is not None:
# data = numpy.ma.masked_array(data, mask=mask)
if variable in raw_variables:
palette = DEFAULT_PALETTES['raw']
palette_stretch = '{0},{1}'.format(data.max(), data.min())
renderer = palette_to_stretched_renderer(
palette, palette_stretch)
renderer.get_legend(
image_height=150, max_precision=2)[0].to_image().save(
os.path.join(output_directory,
'{0}_legend.png'.format(variable)))
else:
renderer = fuzzy_renderer
image_filename = os.path.join(
output_directory, '{0}.{1}'.format(variable, format))
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale=scale,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[variable] = local_filename
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(
loader=PackageLoader('clover.cli')).get_template('eems_map.html')
out.write(
template.render(layers=json.dumps(layers),
bounds=str(leaflet_anchors),
tree=[[cmd, depth]
for (cmd, depth) in model.GetCmdTree()],
raw_variables=list(raw_variables)))
webbrowser.open(index_html)
def render_netcdf(filename_pattern, variable, output_directory, renderer_file,
save_file, renderer_type, colormap, fill, colorspace,
palette, palette_stretch, scale, id_variable, lh,
legend_breaks, legend_ticks, legend_precision, format,
src_crs, dst_crs, res, resampling, anchors, interactive_map,
mask_path):
"""
Render netcdf files to images.
colormap is ignored if renderer_file is provided
--dst-crs is ignored if using --map option (always uses EPSG:3857
If no colormap or palette is provided, a default palette may be chosen based on the name of the variable.
If provided, mask must be 1 for areas to be masked out, and 0 otherwise. It
must be in the same CRS as the input datasets, and have the same spatial
dimensions.
"""
# Parameter overrides
if interactive_map:
dst_crs = 'EPSG:3857'
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='filename_pattern',
param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
mask = get_mask(mask_path) if mask_path is not None else None
if renderer_file is not None and not save_file:
if not os.path.exists(renderer_file):
raise click.BadParameter('does not exist',
param='renderer_file',
param_hint='renderer_file')
# see https://bitbucket.org/databasin/ncdjango/wiki/Home for format
renderer_dict = json.loads(open(renderer_file).read())
if variable in renderer_dict and not 'colors' in renderer_dict:
renderer_dict = renderer_dict[variable]
renderer_type = renderer_dict['type']
if renderer_type == 'stretched':
colors = ','.join([str(c[0]) for c in renderer_dict['colors']])
if 'min' in colors or 'max' in colors or 'mean' in colors:
statistics = collect_statistics(filenames, (variable, ),
mask=mask)[variable]
for entry in renderer_dict['colors']:
if isinstance(entry[0], basestring):
if entry[0] in ('min', 'max', 'mean'):
entry[0] = statistics[entry[0]]
elif '*' in entry[0]:
rel_value, statistic = entry[0].split('*')
entry[0] = float(rel_value) * statistics[statistic]
renderer = renderer_from_dict(renderer_dict)
else:
if renderer_type == 'stretched':
if palette is not None:
renderer = palette_to_stretched_renderer(palette,
palette_stretch,
filenames,
variable,
fill_value=fill,
mask=mask)
elif colormap is None and variable in DEFAULT_PALETTES:
palette, palette_stretch = DEFAULT_PALETTES[variable]
renderer = palette_to_stretched_renderer(palette,
palette_stretch,
filenames,
variable,
fill_value=fill,
mask=mask)
else:
if colormap is None:
colormap = 'min:#000000,max:#FFFFFF'
renderer = colormap_to_stretched_renderer(colormap,
colorspace,
filenames,
variable,
fill_value=fill,
mask=mask)
elif renderer_type == 'classified':
if not palette:
raise click.BadParameter(
'palette required for classified (for now)',
param='--palette',
param_hint='--palette')
renderer = palette_to_classified_renderer(
palette,
filenames,
variable,
method='equal',
fill_value=fill,
mask=mask) # TODO: other methods
if save_file:
if os.path.exists(save_file):
with open(save_file, 'r+') as output_file:
data = json.loads(output_file.read())
output_file.seek(0)
output_file.truncate()
data[variable] = renderer.serialize()
output_file.write(json.dumps(data, indent=4))
else:
with open(save_file, 'w') as output_file:
output_file.write(json.dumps({variable: renderer.serialize()}))
if renderer_type == 'stretched':
if legend_ticks is not None and not legend_breaks:
legend_ticks = [float(v) for v in legend_ticks.split(',')]
legend = renderer.get_legend(
image_height=lh,
breaks=legend_breaks,
ticks=legend_ticks,
max_precision=legend_precision)[0].to_image()
elif renderer_type == 'classified':
legend = composite_elements(renderer.get_legend())
legend.save(
os.path.join(output_directory, '{0}_legend.png'.format(variable)))
with Dataset(filenames[0]) as ds:
var_obj = ds.variables[variable]
dimensions = var_obj.dimensions
shape = var_obj.shape
num_dimensions = len(shape)
if num_dimensions == 3:
if id_variable:
if shape[0] != ds.variables[id_variable][:].shape[0]:
raise click.BadParameter(
'must be same dimensionality as 3rd dimension of {0}'.
format(variable),
param='--id_variable',
param_hint='--id_variable')
else:
# Guess from the 3rd dimension
guess = dimensions[0]
if guess in ds.variables and ds.variables[guess][:].shape[
0] == shape[0]:
id_variable = guess
ds_crs = get_crs(ds, variable)
if not ds_crs and is_geographic(ds, variable):
ds_crs = 'EPSG:4326' # Assume all geographic data is WGS84
src_crs = CRS.from_string(ds_crs) if ds_crs else CRS(
{'init': src_crs}) if src_crs else None
# get transforms, assume last 2 dimensions on variable are spatial in row, col order
y_dim, x_dim = dimensions[-2:]
coords = SpatialCoordinateVariables.from_dataset(
ds,
x_dim,
y_dim,
projection=Proj(src_crs.to_dict()) if src_crs else None)
if mask is not None and not mask.shape == shape[-2:]:
# Will likely break before this if collecting statistics
raise click.BadParameter(
'mask variable shape does not match shape of input spatial dimensions',
param='--mask',
param_hint='--mask')
flip_y = False
reproject_kwargs = None
if dst_crs is not None:
if not src_crs:
raise click.BadParameter('must provide src_crs to reproject',
param='--src-crs',
param_hint='--src-crs')
dst_crs = CRS.from_string(dst_crs)
src_height, src_width = coords.shape
dst_transform, dst_width, dst_height = calculate_default_transform(
src_crs,
dst_crs,
src_width,
src_height,
*coords.bbox.as_list(),
resolution=res)
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': coords.affine,
'dst_crs': dst_crs,
'dst_transform': dst_transform,
'resampling': getattr(RESAMPLING, resampling),
'dst_shape': (dst_height, dst_width)
}
else:
dst_transform = coords.affine
dst_height, dst_width = coords.shape
dst_crs = src_crs
if coords.y.is_ascending_order():
# Only needed if we are not already reprojecting the data, since that will flip it automatically
flip_y = True
if anchors or interactive_map:
if not (dst_crs or src_crs):
raise click.BadParameter(
'must provide at least src_crs to get Leaflet anchors or interactive map',
param='--src-crs',
param_hint='--src-crs')
leaflet_anchors = get_leaflet_anchors(
BBox.from_affine(
dst_transform,
dst_width,
dst_height,
projection=Proj(dst_crs) if dst_crs else None))
if anchors:
click.echo('Anchors: {0}'.format(leaflet_anchors))
layers = {}
for filename in filenames:
with Dataset(filename) as ds:
click.echo('Processing {0}'.format(filename))
filename_root = os.path.split(filename)[1].replace('.nc', '')
if not variable in ds.variables:
raise click.BadParameter(
'variable {0} was not found in file: {1}'.format(
variable, filename),
param='variable',
param_hint='VARIABLE')
var_obj = ds.variables[variable]
if not var_obj.dimensions == dimensions:
raise click.ClickException(
'All datasets must have the same dimensions for {0}'.
format(variable))
if num_dimensions == 2:
data = var_obj[:]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
image_filename = os.path.join(
output_directory,
'{0}_{1}.{2}'.format(filename_root, variable, format))
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale,
flip_y=flip_y,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)[0]] = local_filename
elif num_dimensions == 3:
for index in range(shape[0]):
id = ds.variables[id_variable][
index] if id_variable is not None else index
image_filename = os.path.join(
output_directory,
'{0}_{1}__{2}.{3}'.format(filename_root, variable, id,
format))
data = var_obj[index]
if mask is not None:
data = numpy.ma.masked_array(data, mask=mask)
if reproject_kwargs:
data = warp_array(data, **reproject_kwargs)
render_image(renderer,
data,
image_filename,
scale,
flip_y=flip_y,
format=format)
local_filename = os.path.split(image_filename)[1]
layers[os.path.splitext(local_filename)
[0]] = local_filename
# TODO: not tested recently. Make sure still correct
# else:
# # Assume last 2 components of shape are lat & lon, rest are iterated over
# id_variables = None
# if id_variable is not None:
# id_variables = id_variable.split(',')
# for index, name in enumerate(id_variables):
# if name:
# assert data.shape[index] == ds.variables[name][:].shape[0]
#
# ranges = []
# for dim in data.shape[:-2]:
# ranges.append(range(0, dim))
# for combined_index in product(*ranges):
# id_parts = []
# for index, dim_index in enumerate(combined_index):
# if id_variables is not None and index < len(id_variables) and id_variables[index]:
# id = ds.variables[id_variables[index]][dim_index]
#
# if not isinstance(id, basestring):
# if isinstance(id, Iterable):
# id = '_'.join((str(i) for i in id))
# else:
# id = str(id)
#
# id_parts.append(id)
#
# else:
# id_parts.append(str(dim_index))
#
# combined_id = '_'.join(id_parts)
# image_filename = os.path.join(output_directory, '{0}__{1}.{2}'.format(filename_root, combined_id, format))
# if reproject_kwargs:
# data = warp_array(data, **reproject_kwargs) # NOTE: lack of index will break this
# render_image(renderer, data[combined_index], image_filename, scale, flip_y=flip_y, format=format)
#
# local_filename = os.path.split(image_filename)[1]
# layers[os.path.splitext(local_filename)[0]] = local_filename
if interactive_map:
index_html = os.path.join(output_directory, 'index.html')
with open(index_html, 'w') as out:
template = Environment(
loader=PackageLoader('clover.cli')).get_template('map.html')
out.write(
template.render(layers=json.dumps(layers),
bounds=str(leaflet_anchors),
variable=variable))
webbrowser.open(index_html)
def render_tif(filename_pattern, output_directory, renderer_file, save,
renderer_type, colormap, colorspace, palette, scale,
id_variable, lh, legend_breaks, legend_ticks, src_crs, dst_crs,
res, resampling, anchors):
"""
Render single-band GeoTIFF files to images.
colormap is ignored if renderer_file is provided
"""
filenames = glob.glob(filename_pattern)
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='filename_pattern',
param_hint='FILENAME_PATTERN')
if not os.path.exists(output_directory):
os.makedirs(output_directory)
if renderer_file is not None and not save:
if not os.path.exists(renderer_file):
raise click.BadParameter('does not exist',
param='renderer_file',
param_hint='renderer_file')
# see https://bitbucket.org/databasin/ncdjango/wiki/Home for format
renderer_dict = json.loads(open(renderer_file).read())
# if renderer_dict['type'] == 'stretched':
# colors = ','.join([str(c[0]) for c in renderer_dict['colors']])
# if 'min' in colors or 'max' in colors or 'mean' in colors:
# statistics = collect_statistics(filenames, (variable,))[variable]
# for entry in renderer_dict['colors']:
# if isinstance(entry[0], basestring):
# if entry[0] in ('min', 'max', 'mean'):
# entry[0] = statistics[entry[0]]
# elif '*' in entry[0]:
# rel_value, statistic = entry[0].split('*')
# entry[0] = float(rel_value) * statistics[statistic]
renderer = renderer_from_dict(renderer_dict)
else:
if renderer_type == 'stretched':
# if palette is not None:
# renderer = _palette_to_stretched_renderer(palette, 'min,max', filenames, variable)
#
# else:
renderer = _colormap_to_stretched_renderer(colormap, colorspace,
filenames)
elif renderer_type == 'unique':
renderer = UniqueValuesRenderer(_parse_colormap(colormap),
colorspace)
else:
raise NotImplementedError('other renderers not yet built')
# if save:
# if not renderer_file:
# raise click.BadParameter('must be provided to save', param='renderer_file', param_hint='renderer_file')
#
# if os.path.exists(renderer_file):
# with open(renderer_file, 'r+') as output_file:
# data = json.loads(output_file.read())
# output_file.seek(0)
# output_file.truncate()
# data[variable] = renderer.serialize()
# output_file.write(json.dumps(data, indent=4))
# else:
# with open(renderer_file, 'w') as output_file:
# output_file.write(json.dumps({variable: renderer.serialize()}))
if renderer_type == 'streteched':
if legend_ticks is not None and not legend_breaks:
legend_ticks = [float(v) for v in legend_ticks.split(',')]
legend = renderer.get_legend(image_height=lh,
breaks=legend_breaks,
ticks=legend_ticks,
max_precision=2)[0].to_image()
elif renderer_type == 'unique':
legend = renderer.get_legend(image_height=lh)[0].to_image()
legend.save(os.path.join(output_directory, 'legend.png'))
for filename in filenames:
with rasterio.open(filename) as ds:
print('Processing', filename)
filename_root = os.path.split(filename)[1].replace('.nc', '')
data = ds.read(1, masked=True)
# # get transforms, assume last 2 dimensions on variable are spatial in row, col order
# y_dim, x_dim = ds.variables[variable].dimensions[-2:]
# y_len, x_len = data.shape[-2:]
# coords = SpatialCoordinateVariables.from_dataset(ds, x_dim, y_dim)#, projection=Proj(src_crs))
#
# if coords.y.is_ascending_order():
# data = data[::-1]
#
reproject_kwargs = None
if dst_crs is not None:
# TODO: extract this out into a general clover reprojection function
ds_crs = ds.crs
if not (src_crs or ds_crs):
raise click.BadParameter(
'must provide src_crs to reproject',
param='src_crs',
param_hint='src_crs')
dst_crs = {'init': dst_crs}
src_crs = ds_crs if ds_crs else {'init': src_crs}
left, bottom, top, right = ds.bounds
dst_affine, dst_width, dst_height = calculate_default_transform(
left, bottom, right, top, ds.width, ds.height, src_crs,
dst_crs)
dst_shape = (dst_height, dst_width)
# proj_bbox = coords.bbox.project(Proj(dst_crs))
#
# x_dif = proj_bbox.xmax - proj_bbox.xmin
# y_dif = proj_bbox.ymax - proj_bbox.ymin
#
# total_len = float(x_len + y_len)
# # Cellsize is dimension weighted average of x and y dimensions per projected pixel, unless otherwise provided
# avg_cellsize = ((x_dif / float(x_len)) * (float(x_len) / total_len)) + ((y_dif / float(y_len)) * (float(y_len) / total_len))
#
# cellsize = res or avg_cellsize
# dst_affine = Affine(cellsize, 0, proj_bbox.xmin, 0, -cellsize, proj_bbox.ymax)
# dst_shape = (
# max(int(ceil((y_dif) / cellsize)), 1), # height
# max(int(ceil(x_dif / cellsize)), 1) # width
# )
# TODO: replace with method in rasterio
reproject_kwargs = {
'src_crs': src_crs,
'src_transform': ds.affine,
'dst_crs': dst_crs,
'dst_transform': dst_affine,
'resampling': getattr(RESAMPLING, resampling),
'dst_shape': dst_shape
}
if anchors:
# Reproject the bbox of the output to WGS84
full_bbox = BBox(
(dst_affine.c, dst_affine.f +
dst_affine.e * dst_shape[0], dst_affine.c +
dst_affine.a * dst_shape[1], dst_affine.f),
projection=Proj(dst_crs))
wgs84_bbox = full_bbox.project(Proj(init='EPSG:4326'))
print('WGS84 Anchors: {0}'.format(
[[wgs84_bbox.ymin, wgs84_bbox.xmin],
[wgs84_bbox.ymax, wgs84_bbox.xmax]]))
elif anchors:
# Reproject the bbox of the output to WGS84
full_bbox = BBox(ds.bounds, projection=Proj(ds.crs))
wgs84_bbox = full_bbox.project(Proj(init='EPSG:4326'))
print('WGS84 Anchors: {0}'.format(
[[wgs84_bbox.ymin, wgs84_bbox.xmin],
[wgs84_bbox.ymax, wgs84_bbox.xmax]]))
image_filename = os.path.join(output_directory,
'{0}.png'.format(filename_root))
render_image(renderer,
data,
image_filename,
scale,
reproject_kwargs=reproject_kwargs)
def test_SpatialCoordinateVariables_bbox():
proj = Proj(init='EPSG:4326')
bbox = BBox((10.5, 5, 110.5, 55), projection=proj)
coords = SpatialCoordinateVariables.from_bbox(bbox, 10, 5)
assert coords.bbox.as_list() == bbox.as_list()
def raster_to_netcdf(filename_or_raster,
outfilename=None,
variable_name='data',
format='NETCDF4',
**kwargs):
"""
Parameters
----------
filename_or_raster: name of file to open with rasterio, or opened rasterio raster dataset
outfilename: name of output file. If blank, will be same name as input with *.nc extension added
variable_name: output format for netCDF file: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, NETCDF4
format
kwargs: arguments passed to variable creation: zlib
Note: only rasters with descending y coordinates are currently supported
"""
start = time.time()
if isinstance(filename_or_raster, string_types):
if not os.path.exists(filename_or_raster):
raise ValueError(
'File does not exist: {0}'.format(filename_or_raster))
src = rasterio.open(filename_or_raster)
managed_raster = True
else:
src = filename_or_raster
managed_raster = False
if not src.count == 1:
raise NotImplementedError(
'ERROR: multi-band rasters not yet supported for this operation')
prj = pyproj.Proj(**src.crs)
outfilename = outfilename or src.name + '.nc'
with Dataset(outfilename, 'w', format=format) as target:
if prj.is_latlong():
x_varname = 'longitude'
y_varname = 'latitude'
else:
x_varname = 'x'
y_varname = 'y'
# TODO: may need to do this in blocks if source is big
data = src.read(1, masked=True)
coords = SpatialCoordinateVariables.from_bbox(BBox(src.bounds, prj),
src.width, src.height)
coords.add_to_dataset(target, x_varname, y_varname, **kwargs)
out_var = target.createVariable(variable_name,
data.dtype,
dimensions=(y_varname, x_varname),
**kwargs)
out_var[:] = data
set_crs(target, variable_name, prj, set_proj4_att=False)
if managed_raster:
src.close()
print('Elapsed {0:.3f} seconds'.format(time.time() - start))
def generate_service(
self,
filename_or_pattern,
variable_name,
unique=True,
has_colormap=True,
model_set=None,
model_id_lookup=None,
):
"""
Creates an ncdjango service from a geotiff (stack).
:param filename_or_pattern: A local filename or glob pattern.
:param variable_name: The variable name of the data to assign to the ncdjango service.
:param unique: Indicate whether to use a UniqueValuesRenderer or a StretchedRenderer
:param has_colormap: Indicate whether the service has a colormap or not.
:param model_set: The Queryset to filter over.
:param model_id_lookup: A unique identifier used for specific datasets.
:return: One (1) ncdjango service.
"""
has_time = '*' in filename_or_pattern # pattern if true, filename otherwise
# Construct relative path for new netcdf file, relative to NC_ROOT. Used as 'data_path' in ncdjango.Service
nc_rel_path = os.path.join(self.scenario.project.library.name,
self.scenario.project.name,
'Scenario-' + str(self.scenario.sid))
if has_time:
nc_rel_path = os.path.join(nc_rel_path, 'output',
variable_name + '.nc')
else:
nc_rel_path = os.path.join(
nc_rel_path, filename_or_pattern.replace('tif', 'nc'))
# Absolute path where we want the new netcdf to live.
nc_full_path = os.path.join(NC_ROOT, nc_rel_path)
if not os.path.exists(os.path.dirname(nc_full_path)):
os.makedirs(os.path.dirname(nc_full_path))
variable_names = []
# No patterns, so create a simple input raster
if not has_time:
self.convert_to_netcdf(
os.path.join(self.scenario.input_directory,
filename_or_pattern), nc_full_path, variable_name)
# Time series output pattern, convert to timeseries netcdf
else:
ctype = filename_or_pattern[:-4].split('-')[2:][0]
assert ctype == CTYPE_HASH[variable_name] # sanity check
# collect all information to make valid patterns
iterations = []
timesteps = []
ssim_ids = []
glob_pattern = glob.glob(
os.path.join(self.scenario.output_directory,
filename_or_pattern))
glob_pattern.sort()
for f in glob_pattern:
it, ts, *ctype_id = f[:-4].split(os.sep)[-1].split('-')
if it not in iterations:
iterations.append(it) # E.g. ['It0001','It0002', ...]
if ts not in timesteps: # T.g. ['Ts0000','Ts0001', ...]
timesteps.append(ts)
if len(ctype_id) > 1:
ssim_id = int(ctype_id[1])
if ssim_id not in ssim_ids:
ssim_ids.append(
ssim_id) # E.g. ['tg', '93'], ['ta', '234'], etc.
assert ctype == ctype_id[
0] # pattern matching is way off (not sure this would even happen)
# ssim_ids are internal, have to match with our system
# service variables are <variable_name>-<inner_id>-<iteration>
if len(ssim_ids):
filename_patterns = []
ssim_result = ssim_query(
'select * from ' + SSIM_TABLE[variable_name],
self.scenario.project.library)
# Create valid hash map
ssim_hash = {}
for ssim_id in ssim_ids:
inner_id = None
for row in ssim_result:
# match primary key id and project id, and only create filename_patterns if a match if found
if ssim_id == row[0] and str(
self.scenario.project.pid) == str(row[1]):
name = row[2]
inner_id = INNER_TABLE[
variable_name].objects.filter(
name__exact=name,
project=self.scenario.project).first().id
break
if inner_id:
ssim_hash[ssim_id] = inner_id
# Now build proper filename_patterns
for it in iterations:
for ssim_id in ssim_ids:
filename_patterns.append(
os.path.join(
self.scenario.output_directory,
'{}-Ts*-{}-{}.tif'.format(it, ctype, ssim_id)))
for pattern in filename_patterns:
pattern_id = ssim_hash[int(
pattern.split(os.sep)[-1].split('-')[-1][:-4])]
iteration_num = int(
pattern.split(os.sep)[-1].split('-')[0][2:])
iteration_var_name = '{variable_name}-{id}-{iteration}'.format(
variable_name=variable_name,
id=pattern_id,
iteration=iteration_num)
variable_names.append(iteration_var_name)
iteration_nc_file = os.path.join(
self.scenario.output_directory,
iteration_var_name + '.nc')
self.convert_to_netcdf(pattern, iteration_nc_file,
iteration_var_name)
merge_nc_pattern = os.path.join(self.scenario.output_directory,
variable_name + '-*-*.nc')
# no ids
# service variables are <variable_name>-<iteration>
else:
filename_patterns = [
os.path.join(self.scenario.output_directory,
'{}-Ts*-{}.tif'.format(it, ctype))
for it in iterations
]
merge_nc_pattern = os.path.join(self.scenario.output_directory,
variable_name + '-*.nc')
for pattern in filename_patterns:
iteration_num = int(
pattern.split(os.sep)[-1].split('-')[0][2:])
iteration_var_name = '{variable_name}-{iteration}'.format(
variable_name=variable_name, iteration=iteration_num)
variable_names.append(iteration_var_name)
iteration_nc_file = os.path.join(
self.scenario.output_directory,
iteration_var_name + '.nc')
self.convert_to_netcdf(pattern, iteration_nc_file,
iteration_var_name)
self.merge_netcdf(merge_nc_pattern, nc_full_path)
info = describe(nc_full_path)
grid = info['variables'][variable_names[0] if len(variable_names) else
variable_name]['spatial_grid']['extent']
extent = BBox((grid['xmin'], grid['ymin'], grid['xmax'], grid['ymax']),
projection=pyproj.Proj(grid['proj4']))
steps_per_variable = None
t = None
t_start = None
t_end = None
dimensions = list(info['dimensions'].keys())
if 'x' in dimensions:
x, y = ('x', 'y')
else:
x, y = ('lon', 'lat')
if has_time:
t = 'time'
steps_per_variable = info['dimensions'][t]['length']
t_start = datetime.datetime(2000, 1, 1)
t_end = t_start + datetime.timedelta(1) * steps_per_variable
try:
service = Service.objects.create(name=uuid.uuid4(),
data_path=nc_rel_path,
projection=grid['proj4'],
full_extent=extent,
initial_extent=extent)
if has_time and len(variable_names) and steps_per_variable:
# Set required time fields
service.supports_time = True
service.time_start = t_start
service.time_end = t_end
service.time_interval = 1
service.time_interval_units = 'days'
service.calendar = 'standard'
service.save()
if unique:
model = model_set or getattr(self.scenario.project,
variable_name)
unique_id_lookup = model_id_lookup or NAME_HASH[
variable_name] + '_id'
try:
if has_colormap:
queryset = model.values_list('color', unique_id_lookup,
'name')
renderer = self.generate_unique_renderer(queryset)
else:
queryset = model.values_list(unique_id_lookup, 'name')
renderer = self.generate_unique_renderer(
queryset, randomize_colors=True)
except:
raise AssertionError(
CREATE_RENDERER_ERROR_MSG.format(vname=variable_name))
else:
renderer = self.generate_stretched_renderer(info)
if has_time and len(variable_names):
for name in variable_names:
Variable.objects.create(service=service,
index=variable_names.index(name),
variable=name,
projection=grid['proj4'],
x_dimension=x,
y_dimension=y,
name=name,
renderer=renderer,
full_extent=extent,
supports_time=True,
time_dimension=t,
time_start=t_start,
time_end=t_end,
time_steps=steps_per_variable)
else:
Variable.objects.create(service=service,
index=0,
variable=variable_name,
projection=grid['proj4'],
x_dimension=x,
y_dimension=y,
name=variable_name,
renderer=renderer,
full_extent=extent)
return service
except:
raise Error(
CREATE_SERVICE_ERROR_MSG.format(variable_name,
self.scenario.sid))