def get_results_image(self, bounds, size, single_color, kept_colors, gained_colors, species, historic, futures):
kept_colors = self.get_colors(kept_colors, len(futures)+1)
gained_colors = self.get_colors(gained_colors, len(futures)+1)
extent = BBox(bounds, projection=WGS84)
self.service = Service.objects.get(name='{}_p{}_800m_pa'.format(species, historic))
variable = self.service.variable_set.all().first()
native_extent = extent.project(Proj(str(variable.projection)))
coords = SpatialCoordinateVariables.from_bbox(variable.full_extent, *self.get_grid_spatial_dimensions(variable))
x_slice = coords.x.indices_for_range(native_extent.xmin, native_extent.xmax)
y_slice = coords.y.indices_for_range(native_extent.ymin, native_extent.ymax)
historic_data = self.get_grid_for_variable(variable, x_slice=x_slice, y_slice=y_slice)
self.close_dataset()
if not futures:
data = historic_data
renderer = UniqueValuesRenderer([(1, Color.from_hex(single_color))], fill_value=0)
else:
future_grids = []
for future in futures:
self.service = Service.objects.get(name='{}_15gcm_{}_pa'.format(species, future))
variable = self.service.variable_set.all().first()
future_grids.append(self.get_grid_for_variable(variable, x_slice=x_slice, y_slice=y_slice))
self.close_dataset()
future_data = sum(future_grids)
del future_grids
data = numpy.zeros_like(historic_data, numpy.uint8)
data[historic_data == 1] = 1
kept_idx = (historic_data == 1) & (future_data > 0)
data[kept_idx] = future_data[kept_idx] + 1
gained_idx = (historic_data == 0) & (future_data > 0)
data[gained_idx] = future_data[gained_idx] + len(kept_colors) + 1
data[data.mask == 1] = 0
values = numpy.unique(data)
renderer = UniqueValuesRenderer(
[
(i+1, Color.from_hex(c))
for (i, c) in enumerate(kept_colors)
if i+1 in values
] +
[
(i+len(kept_colors)+1, Color.from_hex(c))
for (i, c) in enumerate(gained_colors)
if i+len(kept_colors)+1 in values
],
fill_value=0
)
image = renderer.render_image(data.data).convert('RGBA')
return GeoImage(image, native_extent).warp(extent, size).image
def raster_to_netcdf(filename_or_raster, outfilename=None, variable_name='data', format='NETCDF4', **kwargs):
"""
Parameters
----------
filename_or_raster: name of file to open with rasterio, or opened rasterio raster dataset
outfilename: name of output file. If blank, will be same name as input with *.nc extension added
variable_name: output format for netCDF file: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, NETCDF4
format
kwargs: arguments passed to variable creation: zlib
Note: only rasters with descending y coordinates are currently supported
"""
start = time.time()
if isinstance(filename_or_raster, string_types):
if not os.path.exists(filename_or_raster):
raise ValueError('File does not exist: {0}'.format(filename_or_raster))
src = rasterio.open(filename_or_raster)
managed_raster = True
else:
src = filename_or_raster
managed_raster = False
if not src.count == 1:
raise NotImplementedError('ERROR: multi-band rasters not yet supported for this operation')
prj = pyproj.Proj(**src.crs)
outfilename = outfilename or src.name + '.nc'
with Dataset(outfilename, 'w', format=format) as target:
if prj.is_latlong():
x_varname = 'longitude'
y_varname = 'latitude'
else:
x_varname = 'x'
y_varname = 'y'
# TODO: may need to do this in blocks if source is big
data = src.read(1, masked=True)
coords = SpatialCoordinateVariables.from_bbox(BBox(src.bounds, prj), src.width, src.height)
coords.add_to_dataset(target, x_varname, y_varname, **kwargs)
out_var = target.createVariable(variable_name, data.dtype, dimensions=(y_varname, x_varname), **kwargs)
out_var[:] = data
set_crs(target, variable_name, prj, set_proj4_att=False)
if managed_raster:
src.close()
print('Elapsed {0:.3f} seconds'.format(time.time() - start))
def get_mask(self, **kwargs):
try:
geoJSON = kwargs['geoJSON']
except ValueError:
raise ValueError('Missing constraint arguments')
features = geoJSON['features']
geometries = [f['geometry'] for f in features]
coords = SpatialCoordinateVariables.from_bbox(self.data.extent, *reversed(self.data.shape))
return rasterize(
geometries, out_shape=self.data.shape, fill=1, transform=coords.affine, all_touched=True, default_value=0,
dtype=numpy.uint8
)
def warp_to_grid(self, path):
with rasterio.open(path) as dataset:
bbox = self.data.extent
vrt_options = {
'resampling': Resampling.nearest,
'dst_crs': CRS.from_string(bbox.projection.srs),
'dst_transform': SpatialCoordinateVariables.from_bbox(
bbox, self.data.shape[1], self.data.shape[0]
).affine,
'dst_height': self.data.shape[self.data.y_dim],
'dst_width': self.data.shape[self.data.x_dim]
}
with WarpedVRT(dataset, **vrt_options) as vrt:
return vrt.read(1, masked=True)
def get_mask(self, **kwargs):
try:
min_lon = kwargs['min']
max_lon = kwargs['max']
except KeyError:
raise ValueError('Missing constraint arguments')
min_lon, max_lon = sorted((min_lon, max_lon))
coords = SpatialCoordinateVariables.from_bbox(self.data.extent, *reversed(self.data.shape))
half_pixel_size = float(coords.x.pixel_size) / 2
start, stop = coords.x.indices_for_range(min_lon + half_pixel_size, max_lon - half_pixel_size)
mask = numpy.zeros_like(self.data, 'bool')
mask[:,:start] = True
mask[:,stop+1:] = True
return mask
def get_mask(self, **kwargs):
try:
min_elevation = kwargs['min']
max_elevation = kwargs['max']
except KeyError:
raise ValueError('Missing constraint arguments')
service = Service.objects.get(name='{}_dem'.format(self.region))
with Dataset(os.path.join(settings.NC_SERVICE_DATA_ROOT, service.data_path)) as ds:
v = service.variable_set.first()
coords = SpatialCoordinateVariables.from_bbox(
v.full_extent, ds.variables[v.x_dimension].size, ds.variables[v.y_dimension].size, dtype='float64'
)
window = coords.get_window_for_bbox(self.data.extent)
elevation = ds.variables['elevation'][window.y_slice, window.x_slice]
mask = elevation < min_elevation
mask |= elevation > max_elevation
return mask
def get_mask(self, hours, lat, lon, year, month, day):
date = datetime.date(year, month, day)
daylight = self.daylight(date, lat, lon)
service = Service.objects.get(name='{}_dem'.format(self.region))
with Dataset(os.path.join(settings.NC_SERVICE_DATA_ROOT, service.data_path)) as ds:
lat_arr = ds['lat'][:]
lon_arr = ds['lon'][:]
coords = SpatialCoordinateVariables.from_bbox(
service.full_extent, ds.variables['lon'].size, ds.variables['lat'].size, dtype='float64'
)
window = coords.get_window_for_bbox(self.data.extent)
daylight_arr = self.daylight_array(date, lat_arr[window.y_slice], lon_arr[window.x_slice])
mask = daylight_arr < (daylight - hours)
mask |= daylight_arr > (daylight + hours)
return mask
def get_mask(self, lat, lon, distance):
wgs84 = pyproj.Proj('+proj=longlat +ellps=WGS84 +datum=WGS84 +no_defs')
p = pyproj.Proj({
'proj': 'tmerc',
'lat_0': lat,
'lon_0': lon,
'k': 1,
'x_0': 0,
'y_0': 0,
'ellps': 'WGS84',
'towgs84': '0,0,0,0,0,0,0',
'units': 'm'
})
# Snap point to nearest cell center
coords = SpatialCoordinateVariables.from_bbox(self.data.extent, *reversed(self.data.shape))
lat_pixel_size = coords.y.pixel_size if coords.y.is_ascending_order() else -1 * coords.y.pixel_size
lat = (
sorted(coords.y.values)[
int((lat - coords.y.pixel_size * 1.5 - self.data.extent.ymin) / coords.y.pixel_size)
] - lat_pixel_size/2
)
lon = (
sorted(coords.x.values)[
int((lon - coords.x.pixel_size * 1.5 - self.data.extent.xmin) / coords.x.pixel_size)
] - coords.x.pixel_size/2
)
project_to_custom = partial(pyproj.transform, wgs84, p)
project_to_data = partial(pyproj.transform, p, self.data.extent.projection)
shape = transform(
project_to_data, transform(project_to_custom, Point(lon, lat)).buffer(distance * 1000, resolution=64)
)
return rasterize(
[shape], out_shape=self.data.shape, fill=1, transform=coords.affine, all_touched=True, default_value=0,
dtype=numpy.uint8
)
def handle(self, *args, **options):
message = (
"WARNING: This will update all service data, casting each to it's smallest possible data type. Do you want "
"to continue? [y/n]")
if input(message).lower() not in {'y', 'yes'}:
return
for service in Service.objects.all():
if service.variable_set.all().count() > 1:
print("Skipping service '{}' with more than one variable...".
format(service.name))
continue
variable = service.variable_set.all().get()
path = os.path.join(SERVICE_DATA_ROOT, service.data_path)
tmp_dir = mkdtemp()
tmp_path = os.path.join(tmp_dir,
os.path.basename(service.data_path))
try:
with Dataset(path, 'r') as ds:
data = ds.variables[variable.variable][:]
coords = SpatialCoordinateVariables.from_bbox(
service.full_extent, *reversed(data.shape))
if data.dtype.kind != 'i':
print("Ignoring service '{}' with non-int type".format(
service.name))
continue
# The fill value will be the minimum value of the chosen type, so we want to make sure it's not
# included in the actual data range
min_value = data.min() - 1
max_value = data.max()
# Determine the most suitable data type by finding the minimum type for the min/max values and then
# using the type that will accurately represent both
min_type = str(numpy.min_scalar_type(min_value))
max_type = str(numpy.min_scalar_type(max_value))
min_unsigned, min_size = min_type.split('int')
max_unsigned, max_size = max_type.split('int')
dtype = '{}int{}'.format(min_unsigned and max_unsigned,
max(int(min_size), int(max_size)))
if data.dtype == dtype:
print(
"Service '{}' already has the smallest possible type: {}"
.format(service.name, dtype))
continue
print("Converting service '{}' to type: {}".format(
service.name, dtype))
with Dataset(tmp_path, 'w', format='NETCDF4') as ds:
coords.add_to_dataset(ds, variable.x_dimension,
variable.y_dimension)
data = data.astype(dtype)
fill_value = numpy.ma.maximum_fill_value(
numpy.dtype(dtype))
numpy.ma.set_fill_value(data, fill_value)
data_var = ds.createVariable(
variable.variable,
dtype,
dimensions=(variable.y_dimension,
variable.x_dimension),
fill_value=fill_value)
data_var[:] = data
set_crs(ds, variable.variable,
service.full_extent.projection)
os.unlink(path)
shutil.copy2(tmp_path, path)
finally:
try:
shutil.rmtree(tmp_dir)
except OSError:
pass
def test_SpatialCoordinateVariables_bbox():
proj = Proj(init='EPSG:4326')
bbox = BBox((10.5, 5, 110.5, 55), projection=proj)
coords = SpatialCoordinateVariables.from_bbox(bbox, 10, 5)
assert coords.bbox.as_list() == bbox.as_list()
def test_window_for_bbox():
coords = SpatialCoordinateVariables.from_bbox(BBox([-124, 82, -122, 90], Proj(init='epsg:4326')), 20, 20)
window = coords.get_window_for_bbox(BBox([-123.9, 82.4, -122.1, 89.6]))
assert window.x_slice == slice(1, 19)
assert window.y_slice == slice(1, 19)
def to_netcdf(
files,
output,
variable,
dtype,
src_crs,
x_name,
y_name,
z_name,
datetime_pattern,
netcdf3,
compress,
packed,
xy_dtype,
# z_dtype,
calendar,
autocrop):
"""
Convert rasters to NetCDF and stack them according to a dimension.
X and Y dimension names will be named according to the source projection (lon, lat if geographic projection, x, y
otherwise) unless specified.
Will overwrite an existing NetCDF file.
Only the first band of the input will be turned into a NetCDF file.
"""
# TODO: add format string template to this to parse out components
filenames = list(glob.glob(files))
if not filenames:
raise click.BadParameter('No files found matching that pattern',
param='files',
param_hint='FILES')
z_values = []
if datetime_pattern is not None:
datetimes = (datetime.strptime(x, datetime_pattern) for x in filenames)
# Sort both datimes and filenames by datetimes
z_values, filenames = [
list(x)
for x in zip(*sorted(zip(datetimes, filenames), key=itemgetter(0)))
]
items = tuple(enumerate(filenames))
has_z = len(filenames) > 1
if has_z and not z_name:
raise click.BadParameter('Required when > 1 input file',
param='--z',
param_hint='--z')
if src_crs:
src_crs = CRS.from_string(src_crs)
template_ds = rasterio.open(filenames[0])
src_crs = template_ds.crs or src_crs
if not src_crs:
raise click.BadParameter(
'Required when no CRS information available in source files',
param='--src-crs',
param_hint='--src-crs')
prj = Proj(**src_crs.to_dict())
bounds = template_ds.bounds
width = template_ds.width
height = template_ds.height
window = None
src_dtype = numpy.dtype(template_ds.dtypes[0])
dtype = numpy.dtype(dtype) if dtype else src_dtype
if dtype == src_dtype:
fill_value = template_ds.nodata
if src_dtype.kind in ('u', 'i'):
# nodata always comes from rasterio as floating point
fill_value = int(fill_value)
else:
fill_value = get_fill_value(dtype)
x_name = x_name or ('lon' if src_crs.is_geographic else 'x')
y_name = y_name or ('lat' if src_crs.is_geographic else 'y')
var_kwargs = {'fill_value': fill_value}
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
with Dataset(output, 'w', format=format) as out:
if packed or autocrop:
mins = []
maxs = []
windows = []
click.echo('Inspecting input datasets...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True)
if packed:
mins.append(data.min())
maxs.append(data.max())
if autocrop:
data_window = get_data_window(data)
if data_window != ((0, height), (0, width)):
windows.append(data_window)
if packed:
min_value = min(mins)
max_value = max(maxs)
scale, offset = get_pack_atts(dtype, min_value, max_value)
if autocrop and windows:
window = union(windows)
bounds = template_ds.window_bounds(window)
height = window[0][1] - window[0][0]
width = window[1][1] - window[1][0]
coords = SpatialCoordinateVariables.from_bbox(BBox(bounds, prj), width,
height, xy_dtype)
coords.add_to_dataset(out, x_name, y_name, zlib=compress)
var_dimensions = [y_name, x_name]
shape = list(coords.shape)
if has_z:
shape.insert(0, len(filenames))
out.createDimension(z_name, shape[0])
var_dimensions.insert(0, z_name)
if z_values:
dates = DateVariable(numpy.array(z_values),
units_start_date=z_values[0],
calendar=calendar)
dates.add_to_dataset(out, z_name)
click.echo('Creating {0}:{1} with shape {2}'.format(
output, variable, shape))
out_var = out.createVariable(variable,
dtype,
dimensions=var_dimensions,
zlib=compress,
**var_kwargs)
set_crs(out, variable, prj, set_proj4_att=True)
if packed:
out_var.setncattr('scale_factor', scale)
out_var.setncattr('add_offset', offset)
click.echo('Copying data from input files...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True, window=window)
if has_z:
out_var[index, :] = data
else:
out_var[:] = data
out.sync()
def raster_to_netcdf(filename_or_raster,
outfilename=None,
variable_name='data',
format='NETCDF4',
**kwargs):
"""
Parameters
----------
filename_or_raster: name of file to open with rasterio, or opened rasterio raster dataset
outfilename: name of output file. If blank, will be same name as input with *.nc extension added
variable_name: output format for netCDF file: NETCDF3_CLASSIC, NETCDF3_64BIT, NETCDF4_CLASSIC, NETCDF4
format
kwargs: arguments passed to variable creation: zlib
Note: only rasters with descending y coordinates are currently supported
"""
start = time.time()
if isinstance(filename_or_raster, string_types):
if not os.path.exists(filename_or_raster):
raise ValueError(
'File does not exist: {0}'.format(filename_or_raster))
src = rasterio.open(filename_or_raster)
managed_raster = True
else:
src = filename_or_raster
managed_raster = False
if not src.count == 1:
raise NotImplementedError(
'ERROR: multi-band rasters not yet supported for this operation')
prj = pyproj.Proj(**src.crs)
outfilename = outfilename or src.name + '.nc'
with Dataset(outfilename, 'w', format=format) as target:
if is_latlong(prj):
x_varname = 'longitude'
y_varname = 'latitude'
else:
x_varname = 'x'
y_varname = 'y'
# TODO: may need to do this in blocks if source is big
data = src.read(1, masked=True)
coords = SpatialCoordinateVariables.from_bbox(BBox(src.bounds, prj),
src.width, src.height)
coords.add_to_dataset(target, x_varname, y_varname, **kwargs)
out_var = target.createVariable(variable_name,
data.dtype,
dimensions=(y_varname, x_varname),
**kwargs)
out_var[:] = data
set_crs(target, variable_name, prj, set_proj4_att=False)
if managed_raster:
src.close()
print('Elapsed {0:.3f} seconds'.format(time.time() - start))
def to_netcdf(
files,
output,
variable,
dtype,
src_crs,
x_name,
y_name,
z_name,
datetime_pattern,
netcdf3,
compress,
packed,
xy_dtype,
# z_dtype,
calendar,
autocrop):
"""
Convert rasters to NetCDF and stack them according to a dimension.
X and Y dimension names will be named according to the source projection (lon, lat if geographic projection, x, y
otherwise) unless specified.
Will overwrite an existing NetCDF file.
Only the first band of the input will be turned into a NetCDF file.
"""
# TODO: add format string template to this to parse out components
filenames = list(glob.glob(files))
if not filenames:
raise click.BadParameter('No files found matching that pattern', param='files', param_hint='FILES')
z_values = []
if datetime_pattern is not None:
datetimes = (datetime.strptime(x, datetime_pattern) for x in filenames)
# Sort both datimes and filenames by datetimes
z_values, filenames = [list(x) for x in zip(*sorted(zip(datetimes, filenames), key=itemgetter(0)))]
items = tuple(enumerate(filenames))
has_z = len(filenames) > 1
if has_z and not z_name:
raise click.BadParameter('Required when > 1 input file', param='--z', param_hint='--z')
if src_crs:
src_crs = CRS.from_string(src_crs)
template_ds = rasterio.open(filenames[0])
src_crs = template_ds.crs or src_crs
if not src_crs:
raise click.BadParameter('Required when no CRS information available in source files', param='--src-crs',
param_hint='--src-crs')
prj = Proj(**src_crs.to_dict())
bounds = template_ds.bounds
width = template_ds.width
height = template_ds.height
window = None
src_dtype = numpy.dtype(template_ds.dtypes[0])
dtype = numpy.dtype(dtype) if dtype else src_dtype
if dtype == src_dtype:
fill_value = template_ds.nodata
if src_dtype.kind in ('u', 'i'):
# nodata always comes from rasterio as floating point
fill_value = int(fill_value)
else:
fill_value = get_fill_value(dtype)
x_name = x_name or ('lon' if src_crs.is_geographic else 'x')
y_name = y_name or ('lat' if src_crs.is_geographic else 'y')
var_kwargs = {
'fill_value': fill_value
}
format = 'NETCDF3_CLASSIC' if netcdf3 else 'NETCDF4'
with Dataset(output, 'w', format=format) as out:
if packed or autocrop:
mins = []
maxs = []
windows = []
click.echo('Inspecting input datasets...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True)
if packed:
mins.append(data.min())
maxs.append(data.max())
if autocrop:
data_window = get_data_window(data)
if data_window != ((0, height), (0, width)):
windows.append(data_window)
if packed:
min_value = min(mins)
max_value = max(maxs)
scale, offset = get_pack_atts(dtype, min_value, max_value)
if autocrop and windows:
window = union(windows)
bounds = template_ds.window_bounds(window)
height = window[0][1] - window[0][0]
width = window[1][1] - window[1][0]
coords = SpatialCoordinateVariables.from_bbox(BBox(bounds, prj), width, height, xy_dtype)
coords.add_to_dataset(out, x_name, y_name, zlib=compress)
var_dimensions = [y_name, x_name]
shape = list(coords.shape)
if has_z:
shape.insert(0, len(filenames))
out.createDimension(z_name, shape[0])
var_dimensions.insert(0, z_name)
if z_values:
dates = DateVariable(numpy.array(z_values),
units_start_date=z_values[0], calendar=calendar)
dates.add_to_dataset(out, z_name)
click.echo('Creating {0}:{1} with shape {2}'.format(output, variable, shape))
out_var = out.createVariable(variable, dtype, dimensions=var_dimensions,
zlib=compress, **var_kwargs)
set_crs(out, variable, prj, set_proj4_att=True)
if packed:
out_var.setncattr('scale_factor', scale)
out_var.setncattr('add_offset', offset)
click.echo('Copying data from input files...')
with click.progressbar(items) as iter:
for index, filename in iter:
with rasterio.open(filename) as src:
data = src.read(1, masked=True, window=window)
if has_z:
out_var[index, :] = data
else:
out_var[:] = data
out.sync()
def process_web_outputs(results,
job,
publish_raster_results=False,
renderer_or_fn=None):
outputs = results.format_args()
for k, v in iter(outputs.items()):
if is_raster(v) and publish_raster_results:
service_name = '{0}/{1}'.format(job.uuid, k)
rel_path = '{}.nc'.format(service_name)
abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path)
os.makedirs(os.path.dirname(abs_path))
with Dataset(abs_path, 'w', format='NETCDF4') as ds:
if is_latlong(v.extent.projection):
x_var = 'longitude'
y_var = 'latitude'
else:
x_var = 'x'
y_var = 'y'
coord_vars = SpatialCoordinateVariables.from_bbox(
v.extent, *reversed(v.shape))
coord_vars.add_to_dataset(ds, x_var, y_var)
fill_value = v.fill_value if numpy.ma.core.is_masked(
v) else None
data_var = ds.createVariable('data',
v.dtype,
dimensions=(y_var, x_var),
fill_value=fill_value)
data_var[:] = v
set_crs(ds, 'data', v.extent.projection)
if callable(renderer_or_fn):
renderer = renderer_or_fn(v)
elif renderer_or_fn is None:
renderer = StretchedRenderer([
(numpy.min(v).item(), Color(0, 0, 0)),
(numpy.max(v).item(), Color(255, 255, 255))
])
else:
renderer = renderer_or_fn
with transaction.atomic():
service = Service.objects.create(
name=service_name,
description=
('This service has been automatically generated from the result of a geoprocessing job.'
),
data_path=rel_path,
projection=v.extent.projection.srs,
full_extent=v.extent,
initial_extent=v.extent,
)
Variable.objects.create(service=service,
index=0,
variable='data',
projection=v.extent.projection.srs,
x_dimension=x_var,
y_dimension=y_var,
name='data',
renderer=renderer,
full_extent=v.extent)
ProcessingResultService.objects.create(job=job,
service=service)
outputs[k] = service_name
elif is_ndarray(v):
if v.size < numpy.get_printoptions()['threshold']:
outputs[k] = v.tolist()
else:
outputs[k] = str(v)
return outputs
def main(original_file, climatena_file, out_dir, valid_variables):
with rasterio.open(original_file) as ds:
bounds = ds.bounds
affine = ds.transform
shape = ds.shape
with open(climatena_file, 'r') as f_in:
headers = csv.DictReader(f_in).fieldnames
variables = [
x for x in headers
if x not in ('ID1', 'ID2', 'Latitude', 'Longitude', 'Elevation')
]
if valid_variables:
valid = [x.strip().lower() for x in valid_variables.split(',')]
else:
valid = variables
print('Creating datasets...')
grid = numpy.zeros(shape, dtype='int32')
grid = numpy.ma.masked_where(grid == 0, grid)
for var in (x for x in variables if x.lower() in valid):
out_path = os.path.join(
out_dir, '{}_{}.nc'.format(
os.path.splitext(os.path.basename(climatena_file))[0], var))
if os.path.exists(out_path):
continue
with Dataset(out_path, 'w', format='NETCDF4') as ds:
projection = Proj('EPSG:4326')
coord_vars = SpatialCoordinateVariables.from_bbox(
BBox(bounds, projection=projection), *reversed(grid.shape))
coord_vars.add_to_dataset(ds, 'longitude', 'latitude')
data_var = ds.createVariable(var,
grid.dtype,
dimensions=('latitude', 'longitude'),
fill_value=grid.fill_value)
data_var[:] = grid
set_crs(ds, var, projection)
print('Copying from ClimateNA data... (0%)', end='\r')
with open(climatena_file, 'r') as f_in:
f_in.seek(0, os.SEEK_END)
end = f_in.tell()
f_in.seek(0)
f_in.readline() # Skip headers
while f_in.tell() < end:
lines = ''.join(f_in.readline() for _ in range(1000000))
arr = numpy.loadtxt(StringIO(lines),
delimiter=',',
usecols=[
headers.index(x)
for x in ['Latitude', 'Longitude'] +
variables
])
arr = numpy.moveaxis(arr, 1, 0)
latitudes = arr[0]
longitudes = arr[1]
for i, var in enumerate(variables):
if var.lower() in valid:
out_path = os.path.join(
out_dir, '{}_{}.nc'.format(
os.path.splitext(
os.path.basename(climatena_file))[0], var))
variable = arr[i + 2]
with Dataset(out_path, 'a') as ds:
grid = ds.variables[var][:]
fill_value = grid.fill_value
grid = grid.data
for j, value in enumerate(variable):
if value == -9999:
continue
col, row = [
int(round(x)) for x in ~affine *
(longitudes[j], latitudes[j])
]
if var in MULTIPLIERS:
value *= MULTIPLIERS[var]
grid[row][col] = value
ds.variables[var][:] = numpy.ma.masked_where(
grid == fill_value, grid)
print('Copying from ClimateNA data... ({}%)'.format(
round(f_in.tell() / end * 100)),
end='\r')
print('Copying from ClimateNA data... (100%)')
print('Done.')