def load_variables(self):
dataset = get_dataset(self.dataset_url)
# For variable names, see:
# http://opendap-dm1.knmi.nl:8080/thredds/dodsC/deltamodel/\
# Deltaportaal/DPRD/199101060440_DPRD_S0v1_2100_SW_RF1p0p3.nc.html
excluded_variables = ['time', 'analysis_time', 'lat', 'lon', 'x', 'y',
'station_id', 'station_names']
var_names = [v for v in dataset.keys() if v not in excluded_variables]
for var_name in var_names:
var, _created = Variable.objects.get_or_create(name=var_name)
if _created:
self.variables.add(var)
else:
if var not in self.variables.all():
self.variables.add(var)
return var_names
def publish_to_geoserver(self, request, queryset):
gs = GeoserverClient(**settings.GEOSERVER_CONFIG)
workspace = settings.NETCDF_WORKSPACE_NAME
for obj in queryset:
# upload to geoserver, steps:
# - create column definitions based on the datasources variables
variable_names = set()
variable_columns = []
datasources = obj.datasources.all()
for datasource in datasources:
variable = datasource.variable
variable_names.add(variable.name)
for variable_name in variable_names:
column_definition = {
'type': 'float', 'name': variable_name,
'nullable': True}
variable_columns.append(column_definition)
# - fill this table with the correct values
raw_rows = defaultdict(dict)
# set for identifiers not in the maplayer's shape file
unavailable_identifiers = set()
for datasource in datasources:
# make fill_value NULL in database
ds = get_dataset(datasource.dataset.dataset_url)
name_params = parse_dataset_name(datasource.dataset.name)
year = name_params['year']
scenario = name_params.get('scenario', '')
variable_name = datasource.variable.name
shape_file = obj.shape_file # pick the map layer's shape file
shape_file_extra_fields = obj.get_shape_file_extra_fields()
secondary_shape_file = obj.connection_shape_file
identifier_data_map = shape_file.get_identifier_data_map()
if secondary_shape_file:
secondary_identifier_data_map = (
secondary_shape_file.get_identifier_data_map())
secondary_identifier = secondary_shape_file.identifier
else:
secondary_identifier_data_map = None
secondary_identifier = None
fill_value = ds[variable_name].attributes.get(
'_FillValue')
x_values = ds['x'][:] # [:] loads the data
y_values = ds['y'][:]
# try to get the identifier value
if 'station_id' in ds.keys():
identifier_values = ds['station_id'][:]
else:
identifier_values = None
values = ds[variable_name][:]
# create insertable rows
for i in range(len(x_values)):
x = x_values[i]
y = y_values[i]
if identifier_values is not None:
identifier = identifier_values[i]
if identifier in unavailable_identifiers:
continue
try:
connection_identifier = None
if secondary_identifier_data_map:
connection_identifier = (
identifier_data_map[identifier][secondary_identifier])
geom = secondary_identifier_data_map[connection_identifier]['geom']
var_name = secondary_identifier.lower()
raw_rows[identifier]['extra::%s' % var_name] = connection_identifier
# TODO: determine type for connection identifier: e.g. int(val) == val => integer
column_definition = {
'type': 'integer', 'name': var_name,
'nullable': True}
variable_columns.append(column_definition)
else:
geom = identifier_data_map[identifier]['geom']
raw_rows[identifier]['geom'] = geom
for sf_ef in shape_file_extra_fields:
ef_value = identifier_data_map[identifier][sf_ef]
var_name = sf_ef.lower()
try:
column_type = get_column_type(ef_value)
except ColumnTypeException, info:
logger.error(info)
else:
raw_rows[identifier]['extra::%s' % var_name] = ef_value
column_definition = {
'type': column_type, 'name': var_name,
'nullable': True}
variable_columns.append(column_definition)
except OGRException:
if not shape_file.identifier:
msg = _("Shapefile instance %s needs an "
"identifier.") % shape_file
else:
msg = _("Field '%s' not found in shapefile %s."
% (shape_file.identifier, shape_file))
logger.error(msg)
except KeyError:
# netcdf identifier not in map layer's shape file
unavailable_identifiers.add(identifier)
msg = _("NetCDF identifier '%s' not in shapefile."
) % identifier
logger.debug(msg)
else:
variables = raw_rows[identifier].get(
'variables', defaultdict(dict))
value = values[i]
if value == fill_value:
value = None
if int(x) == 0 and int(y) == 0:
# skip x = 0.0, y = 0.0 and no value,
continue
if not scenario:
# probably reference dataset
ref_year = settings.NETCDF_REFERENCE_YEAR
if not str(year) == str(ref_year):
# This shouldn't happen. No scenario means year
# should be the same as the reference year.
# Therefore, this warning. Maybe, the reference
# year has changed for certain files.
logger.warning(
_("No scenario and year %s does not equal "
"reference year %s.") % (year, ref_year))
scenarios = SCENARIO_MAP.keys()
elif scenario in ['SW', 'RD']:
# split in 'S', 'W', 'R' and 'D'
scenarios = [scenario[0], scenario[1]]
else:
scenarios = [scenario]
for sc in scenarios:
if sc in SCENARIO_MAP.keys():
# map scenario character to more verbose scenario
# word
sc = SCENARIO_MAP[sc]
var_key = '%s_%s_abs' % (year, sc)
variables[variable_name][var_key] = value
# create relative values
raw_rows[identifier]['variables'] = \
variables
if not raw_rows:
msg = _("No data is generated. Contact your administrator.")
logger.error(msg)
messages.error(request, msg)
return
insertable_rows = []
drop_table(obj.parameter) # first drop table, then create again
Table = create_geo_table(obj.parameter, *variable_columns)
# process raw_rows_new: calculate relative values
ref_year = str(settings.NETCDF_REFERENCE_YEAR)
for identifier, data in raw_rows.items():
geom = data['geom']
extra_keys = [key for key in data.keys() if key.startswith('extra::')]
try:
variables = data['variables']
except KeyError:
logger.debug("No variable data found for identifier '%s'"
% identifier)
continue
for variable_name, year_scenario_data in \
variables.items():
year_scenario_keys = []
for key, value in year_scenario_data.items():
year_scenario_keys.append(key)
for year_scenario_key in year_scenario_keys:
year, scenario, postfix = year_scenario_key.split('_')
ref_key = '%s_%s_abs' % (ref_year, scenario)
try:
ref_value = year_scenario_data[ref_key]
except KeyError:
msg = (_("No reference value found for '%s'.") %
ref_key)
logger.error(msg)
continue
else:
# create the table row definitions
abs_value = year_scenario_data[year_scenario_key]
if abs_value is None:
# skip None values
continue
relative_value = abs_value - ref_value
t = Table()
# need to cast this point to a WKTSpatialElement with the
# RD (28992) srid for GeoAlchemy
geom_type = ('%s' % geom.geom_type).upper()
t.geom = WKTSpatialElement(geom.wkt, 28992,
geometry_type=geom_type)
t.year = year
t.scenario = scenario
# set identifier and variable value
setattr(t, 'identifier', identifier)
# need to cast relative_value (numpy.float32) to
# float
setattr(t, variable_name, float(relative_value))
for extra_key in extra_keys:
val = data[extra_key]
extra_name = extra_key[7:] # remove extra::
setattr(t, extra_name, val)
insertable_rows.append(t)
# commit the rows
session.add_all(insertable_rows)
session.commit()
# upload settings to geoserver:
# - check for workspace 'deltaportaal', if it does not exist,
# create it:
gs.create_workspace(workspace)
# - check for datastore, if it does not exist, create it with the
# correct connection parameters
datastore = settings.NETCDF_DATASTORE_NAME
connection_parameters = settings.GEOSERVER_DELTAPORTAAL_DATASTORE
gs.create_datastore(workspace, datastore, connection_parameters)
# - create feature type (or layer), based on this map layer with
# the correct sql query:
sql_query = obj.sql_query or 'SELECT * FROM %s' % obj.parameter
sql_query = "<![CDATA[%s]]>" % sql_query # to post with xml
view = obj.parameter
# delete the layer (if it exists)
gs.delete_layer(view)
# delete the feature type (if it exists)
gs.delete_feature_type(workspace, datastore, view)
# then create it again
gs.create_feature_type(workspace, datastore, view, sql_query)
# recalculate native and lat/lon bounding boxes
gs.recalculate_bounding_boxes(workspace, datastore, view)
# - create or update style(s) and connect it to this view:
styles = obj.styles.all()
if len(styles):
style = styles[0]
style_name = style.name
style_xml = style.xml.strip()
# first delete style
gs.delete_style(style_name)
# then create it again
gs.create_style(style_name, style_data=style_xml)
gs.set_default_style(workspace, datastore, view, style_name)
else:
# if no styles are given, set default style to point
gs.set_default_style(workspace, datastore, view, 'point')
