def get_crs(ds: nc.Dataset) -> CRS:
attrs = ds.__dict__ # type: dict
proj_id = attrs['MAP_PROJ']
if proj_id == ProjectionTypes.LAT_LON:
pole_lat = attrs['POLE_LAT']
pole_lon = attrs['POLE_LON']
if pole_lat != 90.0 or pole_lon != 0.0:
raise UnsupportedError(
'Geographic coordinate system with rotated pole is not supported'
)
crs = CRS.create_lonlat()
elif proj_id == ProjectionTypes.LAMBERT_CONFORMAL:
crs = CRS.create_lambert(truelat1=attrs['TRUELAT1'],
truelat2=attrs['TRUELAT2'],
origin=LonLat(lon=attrs['STAND_LON'],
lat=attrs['MOAD_CEN_LAT']))
elif proj_id == ProjectionTypes.MERCATOR:
crs = CRS.create_mercator(truelat1=attrs['TRUELAT1'],
origin_lon=attrs['STAND_LON'])
elif proj_id == ProjectionTypes.POLAR_STEREOGRAPHIC:
crs = CRS.create_polar(truelat1=attrs['TRUELAT1'],
origin_lon=attrs['STAND_LON'])
else:
raise UnsupportedError(
'Projection {} is not supported'.format(proj_id))
return crs
def projection(self) -> CRS:
domain = self.data['domains'][0]
map_proj = domain['map_proj']
if map_proj == 'lambert':
return CRS.create_lambert(domain['truelat1'], domain['truelat2'],
LonLat(*domain['center_lonlat']))
elif map_proj == 'mercator':
return CRS.create_mercator(domain['truelat1'],
domain['center_lonlat'][0])
elif map_proj == 'polar':
return CRS.create_polar(domain['truelat1'],
domain['center_lonlat'][0])
elif map_proj == 'lat-lon':
return CRS.create_lonlat()
else:
raise ValueError('Unknown projection: ' + map_proj)
def projection(self) -> CRS:
domain = self.data['domains'][0]
map_proj = domain['map_proj']
if map_proj == 'lambert':
origin = LonLat(lon=domain['stand_lon'],
lat=domain['center_lonlat'][1])
return CRS.create_lambert(domain['truelat1'], domain['truelat2'],
origin)
elif map_proj == 'mercator':
return CRS.create_mercator(domain['truelat1'],
domain['center_lonlat'][0])
elif map_proj == 'polar':
return CRS.create_polar(domain['truelat1'], domain['stand_lon'])
elif map_proj == 'lat-lon':
return CRS.create_lonlat()
else:
assert False, f'invalid projection: {map_proj}'
def get_geo_transform(
ds: nc.Dataset,
crs: CRS) -> Tuple[float, float, float, float, float, float]:
lons_u = ds.variables['XLONG_U']
lons_v = ds.variables['XLONG_V']
lats_u = ds.variables['XLAT_U']
lats_v = ds.variables['XLAT_V']
dim_x = ds.dimensions['west_east'].size
dim_y = ds.dimensions['south_north'].size
# TODO check that nests are non-moving
# assume lat/lon coordinates are identical each time step
t = 0
lower_left_u = LonLat(lon=lons_u[t, 0, 0], lat=lats_u[t, 0, 0])
lower_right_u = LonLat(lon=lons_u[t, 0, -1], lat=lats_u[t, 0, -1])
lower_left_v = LonLat(lon=lons_v[t, 0, 0], lat=lats_v[t, 0, 0])
upper_left_v = LonLat(lon=lons_v[t, -1, 0], lat=lats_v[t, -1, 0])
proj_id = ds.getncattr('MAP_PROJ')
if proj_id == ProjectionTypes.LAT_LON and lower_left_u.lon == lower_right_u.lon:
# global coverage
# WRF uses either 0,0 or 180,180 here, but it should use 0,360 or -180,180.
# Let's fix it by looking at the center longitude.
# Note that this is only an issue for the U grid as the corner points lie
# exactly at the discontinuity, whereas in V the points are half a cell size away.
cen_lon = ds.getncattr('CEN_LON')
lon_min = cen_lon - 180
lon_max = cen_lon + 180
lower_left_u = LonLat(lon=lon_min, lat=lower_left_u.lat)
lower_right_u = LonLat(lon=lon_max, lat=lower_right_u.lat)
lower_left_u_xy = crs.to_xy(lower_left_u)
lower_right_u_xy = crs.to_xy(lower_right_u)
lower_left_v_xy = crs.to_xy(lower_left_v)
upper_left_v_xy = crs.to_xy(upper_left_v)
dx = (lower_right_u_xy.x - lower_left_u_xy.x) / dim_x
dy = (upper_left_v_xy.y - lower_left_v_xy.y) / dim_y
geo_transform = (lower_left_u_xy.x, dx, 0, lower_left_v_xy.y, 0, dy)
return geo_transform
def convert_nml_to_project_domains(nml: dict) -> List[dict]:
max_dom = nml['share']['max_dom'] # type: int
nml = nml['geogrid']
map_proj = nml['map_proj'] # type: str
parent_id = nml['parent_id'] # type: List[int]
parent_grid_ratio = nml['parent_grid_ratio'] # type: List[int]
i_parent_start = nml['i_parent_start'] # type: List[int]
j_parent_start = nml['j_parent_start'] # type: List[int]
e_we = nml['e_we'] # type: List[int]
e_sn = nml['e_sn'] # type: List[int]
dx = [nml['dx']] # type: List[float]
dy = [nml['dy']] # type: List[float]
ref_lon = nml['ref_lon'] # type: float
ref_lat = nml['ref_lat'] # type: float
truelat1 = nml.get('truelat1')
truelat2 = nml.get('truelat2')
standlon = nml.get('stand_lon', 0.0)
# Check that there are no domains with 2 nests on the same level
if parent_id != [1] + list(range(1, max_dom)):
raise RuntimeError(
'We only support 1 nested domain per parent domain.')
# Check whether ref_x/ref_y is omitted, so that we can assume ref == center.
if 'ref_x' in nml or 'ref_y' in nml:
raise NotImplementedError('ref_x/ref_y not supported in namelist.')
# Create CRS object from projection metadata.
if map_proj == 'lat-lon':
if standlon != 0.0:
raise NotImplementedError(
'Rotated lat-lon projection not supported.')
crs = CRS.create_lonlat()
elif map_proj == 'lambert':
# It doesn't matter what the origin is. See wps_binary_to_gdal.py for details.
origin = LonLat(lon=standlon, lat=(truelat1 + truelat2) / 2)
crs = CRS.create_lambert(truelat1, truelat2, origin)
else:
raise NotImplementedError(
f'Map projection "{map_proj}"" not currently supported.')
ref_xy = crs.to_xy(LonLat(lon=ref_lon, lat=ref_lat))
ref_x = [ref_xy.x] # type: List[float]
ref_y = [ref_xy.y] # type: List[float]
min_x = [] # type: List[float]
min_y = [] # type: List[float]
padding_left = [] # type: List[int]
padding_bottom = [] # type: List[int]
padding_right = [] # type: List[int]
padding_top = [] # type: List[int]
cols = [i - 1 for i in e_we]
rows = [i - 1 for i in e_sn]
for idx in range(max_dom - 1):
# Calculate horizontal grid spacing for inner domain
dx.append(dx[idx] / parent_grid_ratio[idx + 1])
dy.append(dy[idx] / parent_grid_ratio[idx + 1])
if idx == 0:
# Calculate min coordinates for outermost domain
min_x.append(ref_x[idx] - (dx[idx] * (cols[idx] / 2)))
min_y.append(ref_y[idx] - (dy[idx] * (rows[idx] / 2)))
# Calculate min coordinates for outer domain
min_x.append(min_x[idx] + (dx[idx] * (i_parent_start[idx + 1] - 1)))
min_y.append(min_y[idx] + (dy[idx] * (j_parent_start[idx + 1] - 1)))
# Calculate center coordinates for inner domain
ref_x.append(min_x[idx + 1] + (dx[idx + 1] * (cols[idx + 1] / 2)))
ref_y.append(min_y[idx + 1] + (dy[idx + 1] * (rows[idx + 1] / 2)))
padding_left.append(i_parent_start[idx + 1] - 1)
padding_bottom.append(j_parent_start[idx + 1] - 1)
padding_right.append(cols[idx] - padding_left[idx] -
cols[idx + 1] // parent_grid_ratio[idx + 1])
padding_top.append(rows[idx] - padding_bottom[idx] -
rows[idx + 1] // parent_grid_ratio[idx + 1])
ref_lonlat = crs.to_lonlat(Coordinate2D(x=ref_x[-1], y=ref_y[-1]))
first_domain = {
'map_proj': map_proj,
'cell_size': [dx[-1], dy[-1]],
'center_lonlat': [ref_lonlat.lon, ref_lonlat.lat],
'domain_size': [cols[-1], rows[-1]],
'stand_lon': standlon,
}
if truelat1 is not None:
first_domain['truelat1'] = truelat1
if truelat2 is not None:
first_domain['truelat2'] = truelat2
domains = [first_domain]
for i in range(max_dom - 1):
domains.append({
'parent_cell_size_ratio':
parent_grid_ratio[::-1][:-1][i],
"padding_left":
padding_left[::-1][i],
"padding_right":
padding_right[::-1][i],
"padding_bottom":
padding_bottom[::-1][i],
"padding_top":
padding_top[::-1][i]
})
return domains
def fill_domains(self):
''' Updated computed fields in each domain object like cell size. '''
domains = self.data.get('domains')
if domains is None:
raise RuntimeError('Domains not configured yet')
innermost_domain = domains[0]
outermost_domain = domains[-1]
innermost_domain['padding_left'] = 0
innermost_domain['padding_right'] = 0
innermost_domain['padding_bottom'] = 0
innermost_domain['padding_top'] = 0
outermost_domain['parent_start'] = [1, 1]
# compute and adjust domain sizes
for idx, domain in enumerate(domains):
if idx == 0:
continue
child_domain = domains[idx - 1]
# We need to make sure that the number of columns in the child domain is an integer multiple
# of the nest's parent domain. As we calculate the inner most domain before calculating the outermost one,
# we need to amend the value for the number of columns or rows for the inner most domain in the case the
# dividend obtained by dividing the number of inner domain's columns by the user's inner-to-outer resolution ratio
# in the case where is not an integer value.
child_domain_size_padded = (
child_domain['domain_size'][0] + child_domain['padding_left'] +
child_domain['padding_right'],
child_domain['domain_size'][1] +
child_domain['padding_bottom'] + child_domain['padding_top'],
)
if (child_domain_size_padded[0] %
domain['parent_cell_size_ratio']) != 0:
new_cols = int(
ceil(child_domain_size_padded[0] /
domain['parent_cell_size_ratio']))
new_child_domain_padded_x = new_cols * domain[
'parent_cell_size_ratio']
else:
new_child_domain_padded_x = child_domain_size_padded[0]
if (child_domain_size_padded[1] %
domain['parent_cell_size_ratio']) != 0:
new_rows = int(
ceil(child_domain_size_padded[1] /
domain['parent_cell_size_ratio']))
new_child_domain_padded_y = new_rows * domain[
'parent_cell_size_ratio']
else:
new_child_domain_padded_y = child_domain_size_padded[1]
if idx == 1:
child_domain['domain_size'] = [
new_child_domain_padded_x, new_child_domain_padded_y
]
else:
child_domain[
'padding_right'] += new_child_domain_padded_x - child_domain_size_padded[
0]
child_domain[
'padding_top'] += new_child_domain_padded_y - child_domain_size_padded[
1]
assert new_child_domain_padded_x % domain[
'parent_cell_size_ratio'] == 0
assert new_child_domain_padded_y % domain[
'parent_cell_size_ratio'] == 0
domain['domain_size'] = [
new_child_domain_padded_x // domain['parent_cell_size_ratio'],
new_child_domain_padded_y // domain['parent_cell_size_ratio']
]
# compute bounding boxes, cell sizes, center lonlat, parent start
for idx, domain in enumerate(domains):
size_x, size_y = domain['domain_size']
padded_size_x = size_x + domain['padding_left'] + domain[
'padding_right']
padded_size_y = size_y + domain['padding_bottom'] + domain[
'padding_top']
domain['domain_size_padded'] = [padded_size_x, padded_size_y]
if idx == 0:
center_lon, center_lat = domain['center_lonlat']
center_xy = self.projection.to_xy(
LonLat(lon=center_lon, lat=center_lat))
domain['bbox'] = get_bbox_from_grid_spec(
center_xy.x, center_xy.y, domain['cell_size'], size_x,
size_y)
else:
child_domain = domains[idx - 1]
domain['cell_size'] = [
child_domain['cell_size'][0] *
domain['parent_cell_size_ratio'],
child_domain['cell_size'][1] *
domain['parent_cell_size_ratio']
]
child_center_x, child_center_y = get_bbox_center(
child_domain['bbox'])
domain['bbox'] = get_parent_bbox_from_child_grid_spec(
child_center_x=child_center_x,
child_center_y=child_center_y,
child_cell_size=child_domain['cell_size'],
child_cols=child_domain['domain_size'][0] +
child_domain['padding_left'] +
child_domain['padding_right'],
child_rows=child_domain['domain_size'][1] +
child_domain['padding_top'] +
child_domain['padding_bottom'],
child_parent_res_ratio=domain['parent_cell_size_ratio'],
parent_left_padding=domain['padding_left'],
parent_right_padding=domain['padding_right'],
parent_bottom_padding=domain['padding_bottom'],
parent_top_padding=domain['padding_top'])
center_x, center_y = get_bbox_center(domain['bbox'])
center_lonlat = self.projection.to_lonlat(
Coordinate2D(x=center_x, y=center_y))
domain['center_lonlat'] = [
center_lonlat.lon, center_lonlat.lat
]
if idx < len(domains) - 1:
parent_domain = domains[idx + 1]
domain['parent_start'] = [
parent_domain['padding_left'] + 1,
parent_domain['padding_bottom'] + 1
]
def read_wps_binary_index_file(folder: str) -> WPSBinaryIndexMetadata:
index_path = os.path.join(folder, 'index')
if not os.path.exists(index_path):
raise UserError(f'{index_path} is missing, this is not a valid WPS Binary dataset')
with open(index_path) as f:
index = '\n'.join(line.strip() for line in f.readlines())
parser = ConfigParser()
parser.read_string('[root]\n' + index)
meta = parser['root']
def clean_str(s: Optional[str]) -> Optional[str]:
if s is None:
return
else:
return s.strip('"')
m = WPSBinaryIndexMetadata()
# encoding
m.little_endian = meta.get('endian') == 'little'
m.signed = meta.get('signed') == 'yes'
m.top_bottom = meta.get('row_order') == 'top_bottom'
m.word_size = int(meta['wordsize'])
m.scale_factor = float(meta.get('scale_factor', '1'))
m.missing_value = float(meta['missing_value']) if 'missing_value' in meta else None
# tile dimensions
m.tile_x = int(meta['tile_x'])
m.tile_y = int(meta['tile_y'])
if 'tile_z_start' in meta:
m.tile_z_start = int(meta['tile_z_start'])
m.tile_z_end = int(meta['tile_z_end'])
else:
m.tile_z_start = 1
m.tile_z_end = int(meta['tile_z'])
m.tile_bdr = int(meta.get('tile_bdr', '0'))
# projection / geographic coordinate system
m.proj_id = meta['projection']
m.stdlon = float(meta['stdlon']) if 'stdlon' in meta else None
m.truelat1 = float(meta['truelat1']) if 'truelat1' in meta else None
m.truelat2 = float(meta['truelat2']) if 'truelat2' in meta else None
# grid georeferencing
m.dx = float(meta['dx'])
m.dy = float(meta['dy'])
m.known_lonlat = LonLat(lon=float(meta['known_lon']), lat=float(meta['known_lat']))
known_x_idx = float(meta.get('known_x', '1'))
known_y_idx = float(meta.get('known_y', '1'))
m.known_idx = Coordinate2D(known_x_idx, known_y_idx)
# categories
m.categorical = meta['type'] == 'categorical'
m.category_min = int(meta['category_min']) if 'category_min' in meta else None
m.category_max = int(meta['category_max']) if 'category_max' in meta else None
# landuse categories
m.landuse_scheme = clean_str(meta.get('mminlu'))
for field in LANDUSE_FIELDS:
setattr(m, field, int(meta[field]) if field in meta else None)
# other
m.filename_digits = int(meta.get('filename_digits', '5'))
m.units = clean_str(meta.get('units'))
m.description = clean_str(meta.get('description'))
m.validate()
return m
def convert_wps_binary_to_vrt_dataset(
folder: str,
use_vsi: bool = False) -> Tuple[str, str, str, Callable[[], None]]:
"""Converts a WPS Binary format dataset into a mosaic VRT dataset referencing per-tile VRT datasets."""
m = read_wps_binary_index_file(folder)
if m.proj_id == 'regular_ll' and m.stdlon is not None:
raise UnsupportedError('Rotated pole system is not supported')
# scan folder for available tiles
tile_filename_re = re.compile('^({d})-({d})\.({d})-({d})$'.format(
d='\d{' + str(m.filename_digits) + '}'))
tiles = []
for filename in os.listdir(folder):
match = tile_filename_re.match(filename)
if match:
tiles.append({
'filename': filename,
'path': os.path.join(folder, filename),
'start_x': int(match.group(1)),
'end_x': int(match.group(2)),
'start_y': int(match.group(3)),
'end_y': int(match.group(4))
})
if not tiles:
raise UserError(f'No tiles found in {folder}')
# determine raster dimensions
xsize = max(tile['end_x'] for tile in tiles) # type: int
ysize = max(tile['end_y'] for tile in tiles) # type: int
zsize = m.tile_z_end - m.tile_z_start + 1
# convert to GDAL metadata
dtype_mapping = {
(1, False): gdal.GDT_Byte, # GDAL only supports unsigned byte
(2, False): gdal.GDT_UInt16,
(2, True): gdal.GDT_Int16,
(3, False): gdal.GDT_UInt32,
(3, True): gdal.GDT_Int32
}
try:
dtype = dtype_mapping[(m.word_size, m.signed)]
except KeyError:
raise UnsupportedError(
'word_size={} signed={} is not supported'.format(
m.word_size, m.signed))
if m.proj_id == 'regular_ll':
crs = CRS.create_lonlat()
elif m.proj_id == 'lambert':
# The map distortion of a Lambert Conformal projection is fully
# defined by the two true latitudes.
#
# However, the longitude of origin is important for WRF as well,
# since we only deal with upright rectangles (the domains) on the map.
# For that reason, WRF allows the user to define the "standard longitude"
# which is the longitude of origin.
#
# The latitude of origin on the other hand does not have any significance
# here and cannot be specified by the user. The geo transform for a given
# grid is computed based on any arbitrary latitude of origin (see below).
# In QGIS, the only difference are the displayed projected y coordinates,
# but the actual grid georeferencing is unaffected.
# This is possible as WRF's georeferencing metadata is based on geographical
# reference coordinates for a grid cell, not projected coordinates.
arbitrary_latitude_origin = (m.truelat1 + m.truelat2) / 2
origin = LonLat(lon=m.stdlon, lat=arbitrary_latitude_origin)
crs = CRS.create_lambert(m.truelat1, m.truelat2, origin)
elif m.proj_id == 'mercator':
# The map distortion of a Mercator projection is fully
# defined by the true latitude.
# The longitude of origin does not have any significance and
# any arbitrary value is handled when computing the geo transform
# for a given grid (see below). See also the comment above for Lambert.
arbitrary_longitude_origin = 0
crs = CRS.create_mercator(m.truelat1, arbitrary_longitude_origin)
elif m.proj_id == 'albers_nad83':
# See the comment above for Lambert. The same applies here.
arbitrary_latitude_origin = (m.truelat1 + m.truelat2) / 2
origin = LonLat(lon=m.stdlon, lat=arbitrary_latitude_origin)
crs = CRS.create_albers_nad83(m.truelat1, m.truelat2, origin)
# FIXME handle polar vs polar_wgs84 differently
elif m.proj_id == 'polar':
# See the comment above for Lambert. The same applies here.
crs = CRS.create_polar(m.truelat1, m.stdlon)
elif m.proj_id == 'polar_wgs84':
# See the comment above for Lambert. The same applies here.
crs = CRS.create_polar(m.truelat1, m.stdlon)
else:
raise UnsupportedError(f'Projection {m.proj_id} is not supported')
known_x_idx_gdal = m.known_idx.x - 0.5
if m.top_bottom:
known_y_idx_gdal = ysize - m.known_idx.y - 0.5
dy_gdal = -m.dy
else:
known_y_idx_gdal = m.known_idx.y - 0.5
dy_gdal = m.dy
known_xy = crs.to_xy(m.known_lonlat)
upper_left_x = known_xy.x - known_x_idx_gdal * m.dx
upper_left_y = known_xy.y + known_y_idx_gdal * m.dy
geo_transform = (upper_left_x, m.dx, 0, upper_left_y, 0, dy_gdal)
# print('known_x_idx_gdal: {}'.format(known_x_idx_gdal))
# print('known_y_idx_gdal: {}'.format(known_y_idx_gdal))
# print('known_xy: {}'.format(m.known_xy))
# print('upper_left_x: {}'.format(upper_left_x))
# print('upper_left_y: {}'.format(upper_left_y))
# VRTRawRasterBand metadata
line_width = m.word_size * (m.tile_x + m.tile_bdr * 2
) # x size incl. border
tile_size = line_width * (m.tile_y + m.tile_bdr * 2
) # tile size incl. border
line_offset = line_width
image_offset = m.tile_bdr * line_width + m.tile_bdr * m.word_size
pixel_offset = m.word_size
byte_order = 'LSB' if m.little_endian else 'MSB'
# create tile VRTs
if use_vsi:
out_dir = get_temp_vsi_path(ext='')
else:
out_dir = get_temp_dir()
driver = gdal.GetDriverByName('VRT') # type: gdal.Driver
tile_vrt_paths = {}
for tile in tiles:
vsi_path = '{}/{}.vrt'.format(out_dir, tile['filename'])
vrt = driver.Create(vsi_path, m.tile_x, m.tile_y,
0) # type: gdal.Dataset
for z in range(m.tile_z_start - 1, m.tile_z_end):
options = [
'subClass=VRTRawRasterBand',
'SourceFilename={}'.format(tile['path']), 'relativeToVRT=0',
'ImageOffset={}'.format(z * tile_size + image_offset),
'PixelOffset={}'.format(pixel_offset),
'LineOffset={}'.format(line_offset), 'ByteOrder=' + byte_order
]
vrt.AddBand(dtype, options)
vrt.FlushCache()
tile_vrt_paths[tile['filename']] = vsi_path
# create mosaic VRT
mosaic_vrt_path = '{}/mosaic.vrt'.format(out_dir)
vrt = driver.Create(mosaic_vrt_path, xsize, ysize, zsize,
dtype) # type: gdal.Dataset
vrt.SetProjection(crs.proj4)
vrt.SetGeoTransform(geo_transform)
if m.categorical:
color_table, cat_names = get_gdal_categories(m.categories,
m.category_min,
m.category_max)
for band_idx in range(1, zsize + 1):
band = vrt.GetRasterBand(band_idx) # type: gdal.Band
if m.missing_value is not None:
band.SetNoDataValue(m.missing_value)
band.SetScale(m.scale_factor)
if m.categorical:
band.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
band.SetRasterColorTable(color_table)
band.SetRasterCategoryNames(cat_names)
sources = {}
for idx, tile in enumerate(tiles):
tile_vrt_path = tile_vrt_paths[tile['filename']]
if m.top_bottom:
end_y = ysize - tile['start_y'] - 1
start_y = end_y - m.tile_y + 1
else:
start_y = tile['start_y'] - 1
sources['source_{}'.format(idx)] = ('''
<SimpleSource>
<SourceFilename relativeToVRT="0">{path}</SourceFilename>
<SourceBand>{band}</SourceBand>
<SrcRect xOff="0" yOff="0" xSize="{tile_x}" ySize="{tile_y}" />
<DstRect xOff="{offset_x}" yOff="{offset_y}" xSize="{tile_x}" ySize="{tile_y}" />
</SimpleSource>''').format(path=tile_vrt_path,
band=band_idx,
tile_x=m.tile_x,
tile_y=m.tile_y,
offset_x=tile['start_x'] - 1,
offset_y=start_y)
band.SetMetadata(sources, 'vrt_sources')
vrt.FlushCache()
vrt_paths = [mosaic_vrt_path] + list(tile_vrt_paths.values())
if use_vsi:
dispose = partial(remove_vsis, vrt_paths)
else:
dispose = partial(remove_dir, out_dir)
short_name = os.path.basename(folder)
title = short_name
if m.units and m.units != 'category':
title += ' in ' + m.units
if m.description:
title += ' (' + m.description + ')'
# The title is returned as VRT does not support dataset descriptions.
return mosaic_vrt_path, title, short_name, dispose
def convert_wps_binary_to_vrt_dataset(
folder: str,
use_vsi: bool = False) -> Tuple[str, str, str, Callable[[], None]]:
"""Converts a WPS Binary format dataset into a mosaic VRT dataset referencing per-tile VRT datasets."""
m = read_wps_binary_index_file(folder)
if m.proj_id == 'regular_ll' and m.stdlon is not None:
raise NotImplementedError('stdlon not supported for regular_ll')
# scan folder for available tiles
tile_filename_re = re.compile('^({d})-({d})\.({d})-({d})$'.format(
d='\d{' + str(m.filename_digits) + '}'))
tiles = []
for filename in os.listdir(folder):
match = tile_filename_re.match(filename)
if match:
tiles.append({
'filename': filename,
'path': os.path.join(folder, filename),
'start_x': int(match.group(1)),
'end_x': int(match.group(2)),
'start_y': int(match.group(3)),
'end_y': int(match.group(4))
})
if not tiles:
raise ValueError('No tiles found')
# determine raster dimensions
xsize = max(tile['end_x'] for tile in tiles) # type: int
ysize = max(tile['end_y'] for tile in tiles) # type: int
zsize = m.tile_z_end - m.tile_z_start + 1
# convert to GDAL metadata
dtype_mapping = {
(1, False): gdal.GDT_Byte, # GDAL only supports unsigned byte
(2, False): gdal.GDT_UInt16,
(2, True): gdal.GDT_Int16,
(3, False): gdal.GDT_UInt32,
(3, True): gdal.GDT_Int32
}
try:
dtype = dtype_mapping[(m.word_size, m.signed)]
except KeyError:
raise ValueError('word_size/signed combination not supported')
if m.proj_id == 'regular_ll':
crs = CRS.create_lonlat()
elif m.proj_id == 'lambert':
# It doesn't matter what the origin is. This only influences the
# projection coordinates to which the data is anchored to but the
# georeferencing itself does not change. See down below on how
# the geo transform is computed based on the known geographical
# coordinates in the data.
origin = LonLat(lon=m.stdlon, lat=(m.truelat1 + m.truelat2) / 2)
crs = CRS.create_lambert(m.truelat1, m.truelat2, origin)
elif m.proj_id == 'mercator':
# See comment above about origin.
origin_lon = m.stdlon if m.stdlon is not None else 0
crs = CRS.create_mercator(m.truelat1, origin_lon)
elif proj_id == 'albers_nad83':
# See comment above about origin.
origin = LonLat(lon=m.stdlon, lat=(m.truelat1 + m.truelat2) / 2)
crs = CRS.create_albers_nad83(m.truelat1, m.truelat2, origin)
# FIXME handle polar vs polar_wgs84 differently
elif m.proj_id == 'polar':
crs = CRS.create_polar(m.truelat1, m.stdlon)
elif m.proj_id == 'polar_wgs84':
crs = CRS.create_polar(m.truelat1, m.stdlon)
else:
raise NotImplementedError('Unsupported projection')
known_x_idx_gdal = m.known_idx.x - 0.5
if m.top_bottom:
known_y_idx_gdal = ysize - m.known_idx.y - 0.5
dy_gdal = -m.dy
else:
known_y_idx_gdal = m.known_idx.y - 0.5
dy_gdal = m.dy
known_xy = crs.to_xy(m.known_lonlat)
upper_left_x = known_xy.x - known_x_idx_gdal * m.dx
upper_left_y = known_xy.y + known_y_idx_gdal * m.dy
geo_transform = (upper_left_x, m.dx, 0, upper_left_y, 0, dy_gdal)
# print('known_x_idx_gdal: {}'.format(known_x_idx_gdal))
# print('known_y_idx_gdal: {}'.format(known_y_idx_gdal))
# print('known_xy: {}'.format(m.known_xy))
# print('upper_left_x: {}'.format(upper_left_x))
# print('upper_left_y: {}'.format(upper_left_y))
# VRTRawRasterBand metadata
line_width = m.word_size * (m.tile_x + m.tile_bdr * 2
) # x size incl. border
tile_size = line_width * (m.tile_y + m.tile_bdr * 2
) # tile size incl. border
line_offset = line_width
image_offset = m.tile_bdr * line_width + m.tile_bdr * m.word_size
pixel_offset = m.word_size
byte_order = 'LSB' if m.little_endian else 'MSB'
# create tile VRTs
if use_vsi:
out_dir = get_temp_vsi_path(ext='')
else:
out_dir = get_temp_dir()
driver = gdal.GetDriverByName('VRT') # type: gdal.Driver
tile_vrt_paths = {}
for tile in tiles:
vsi_path = '{}/{}.vrt'.format(out_dir, tile['filename'])
vrt = driver.Create(vsi_path, m.tile_x, m.tile_y,
0) # type: gdal.Dataset
for z in range(m.tile_z_start - 1, m.tile_z_end):
options = [
'subClass=VRTRawRasterBand',
'SourceFilename={}'.format(tile['path']), 'relativeToVRT=0',
'ImageOffset={}'.format(z * tile_size + image_offset),
'PixelOffset={}'.format(pixel_offset),
'LineOffset={}'.format(line_offset), 'ByteOrder=' + byte_order
]
vrt.AddBand(dtype, options)
vrt.FlushCache()
tile_vrt_paths[tile['filename']] = vsi_path
# create mosaic VRT
mosaic_vrt_path = '{}/mosaic.vrt'.format(out_dir)
vrt = driver.Create(mosaic_vrt_path, xsize, ysize, zsize,
dtype) # type: gdal.Dataset
vrt.SetProjection(crs.proj4)
vrt.SetGeoTransform(geo_transform)
if m.categorical:
color_table, cat_names = get_gdal_categories(m.categories,
m.category_min,
m.category_max)
for band_idx in range(1, zsize + 1):
band = vrt.GetRasterBand(band_idx) # type: gdal.Band
if m.missing_value is not None:
band.SetNoDataValue(m.missing_value)
band.SetScale(m.scale_factor)
if m.categorical:
band.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)
band.SetRasterColorTable(color_table)
band.SetRasterCategoryNames(cat_names)
sources = {}
for idx, tile in enumerate(tiles):
tile_vrt_path = tile_vrt_paths[tile['filename']]
if m.top_bottom:
end_y = ysize - tile['start_y'] - 1
start_y = end_y - m.tile_y + 1
else:
start_y = tile['start_y'] - 1
sources['source_{}'.format(idx)] = ('''
<SimpleSource>
<SourceFilename relativeToVRT="0">{path}</SourceFilename>
<SourceBand>{band}</SourceBand>
<SrcRect xOff="0" yOff="0" xSize="{tile_x}" ySize="{tile_y}" />
<DstRect xOff="{offset_x}" yOff="{offset_y}" xSize="{tile_x}" ySize="{tile_y}" />
</SimpleSource>''').format(path=tile_vrt_path,
band=band_idx,
tile_x=m.tile_x,
tile_y=m.tile_y,
offset_x=tile['start_x'] - 1,
offset_y=start_y)
band.SetMetadata(sources, 'vrt_sources')
vrt.FlushCache()
vrt_paths = [mosaic_vrt_path] + list(tile_vrt_paths.values())
if use_vsi:
dispose = partial(remove_vsis, vrt_paths)
else:
dispose = partial(remove_dir, out_dir)
short_name = os.path.basename(folder)
title = short_name
if m.units and m.units != 'category':
title += ' in ' + m.units
if m.description:
title += ' (' + m.description + ')'
# The title is returned as VRT does not support dataset descriptions.
return mosaic_vrt_path, title, short_name, dispose