def test_vector_variable(self, m):
m.return_value = {
"key": {
"enabled": True,
"variables": {
"magmyvar": {
"name": "my_variable",
"east_vector_component": "u",
"north_vector_component": "v",
},
"magnitudemyvar": {
"name": "my_variable",
"east_vector_component": "u",
"north_vector_component": "v",
},
},
},
}
self.assertEqual(len(DatasetConfig("key").variables), 2)
self.assertEqual(len(DatasetConfig("key").vector_variables), 2)
result = DatasetConfig("key").variable["magmyvar"]
self.assertEqual(result.name, "my_variable")
self.assertEqual(result.east_vector_component, "u")
self.assertEqual(result.north_vector_component, "v")
self.assertEqual(result.unit, "Unknown")
def test_datasetconfig_object(self, m):
m.return_value = {
"key": {
"enabled": True,
"url": "my_url",
"climatology": "my_climatology",
"name": "my_name",
"help": "my_help",
"quantum": "my_quantum",
"attribution": "my_<b>attribution</b>",
"cache": "123",
"variables": {
"var": {
"name": "my_variable",
}
}
},
}
self.assertEqual(len(DatasetConfig.get_datasets()), 1)
result = DatasetConfig("key")
self.assertEqual(result.url, "my_url")
self.assertEqual(result.climatology, "my_climatology")
self.assertEqual(result.name, "my_name")
self.assertEqual(result.help, "my_help")
self.assertEqual(result.quantum, "my_quantum")
self.assertEqual(result.attribution, "my_attribution")
self.assertEqual(result.cache, 123)
self.assertFalse(result.variable[Mock(key="var")].is_hidden)
def datasets_query_v1_0():
"""
API Format: /api/v1.0/datasets/
Optional arguments:
* id : Show only the name and id of the datasets
Returns:
Response -- Response object containing list of available datasets w/ some metadata.
"""
data = []
if 'id' in request.args:
for key in DatasetConfig.get_datasets():
config = DatasetConfig(key)
data.append({'id': key, 'value': config.name})
else:
for key in DatasetConfig.get_datasets():
config = DatasetConfig(key)
data.append({
'id': key,
'value': config.name,
'quantum': config.quantum,
'help': config.help,
'attribution': config.attribution,
})
data = sorted(data, key=lambda k: k['value'])
resp = jsonify(data)
return resp
def datasets_query_v1_0():
"""
API Format: /api/v1.0/datasets/
Optional arguments:
* id: Show only the name and id of the datasets
Returns:
List of available datasets w/ some metadata.
"""
data = []
if "id" in request.args:
for key in DatasetConfig.get_datasets():
config = DatasetConfig(key)
data.append({"id": key, "value": config.name})
else:
for key in DatasetConfig.get_datasets():
config = DatasetConfig(key)
data.append(
{
"id": key,
"value": config.name,
"quantum": config.quantum,
"help": config.help,
"attribution": config.attribution,
}
)
data = sorted(data, key=lambda k: k["value"])
resp = jsonify(data)
return resp
def thorough(self):
self.app = create_app()
with self.app.app_context():
datasets = DatasetConfig.get_datasets()
variables = {}
for dataset in datasets:
variables.update({dataset: DatasetConfig(dataset).variables})
data = {'datasets': datasets, 'variables': variables}
return data
def test_datasetconfig_object(self, m):
m.return_value = {
"key": {
"enabled": True,
"url": "my_url",
"geo_ref": {
"url": "my_geo_ref_url",
"drop_variables": ["bathymetry"],
},
"climatology": "my_climatology",
"name": "my_name",
"help": "my_help",
"quantum": "my_quantum",
"type": "my_type",
"grid_angle_file_url": "my_grid_angle_file_url",
"bathymetry_file_url": "my_bathy_file_url.nc",
"model_class": "my_model_class",
"time_dim_units": "my_time_units",
"attribution": "my_<b>attribution</b>",
"cache": "123",
"lat_var_key": "my_lat",
"lon_var_key": "my_lon",
"variables": {
"var": {
"name": "my_variable",
}
},
}
}
self.assertEqual(len(DatasetConfig.get_datasets()), 1)
result = DatasetConfig("key")
self.assertEqual(result.url, "my_url")
self.assertEqual(result.geo_ref["url"], "my_geo_ref_url")
self.assertEqual(result.geo_ref["drop_variables"], ["bathymetry"])
self.assertEqual(result.key, "key")
self.assertEqual(result.climatology, "my_climatology")
self.assertEqual(result.name, "my_name")
self.assertEqual(result.help, "my_help")
self.assertEqual(result.quantum, "my_quantum")
self.assertEqual(result.grid_angle_file_url, "my_grid_angle_file_url")
self.assertEqual(result.bathymetry_file_url, "my_bathy_file_url.nc")
self.assertEqual(result.model_class, "my_model_class")
self.assertEqual(result.type, "my_type")
self.assertEqual(result.time_dim_units, "my_time_units")
self.assertEqual(result.lat_var_key, "my_lat")
self.assertEqual(result.lon_var_key, "my_lon")
self.assertEqual(result.attribution, "my_attribution")
self.assertEqual(result.cache, 123)
self.assertFalse(result.variable[Mock(key="var")].is_hidden)
def plot_v1_0():
if request.method == 'GET':
args = request.args
else:
args = request.form
query = json.loads(args.get('query'))
config = DatasetConfig(query.get('dataset'))
with open_dataset(config) as dataset:
if 'time' in query:
query['time'] = dataset.convert_to_timestamp(query.get('time'))
else:
query['starttime'] = dataset.convert_to_timestamp(
query.get('starttime'))
query['endtime'] = dataset.convert_to_timestamp(
query.get('endtime'))
resp = routes.routes_impl.plot_impl(args, query)
m = hashlib.md5()
m.update(str(resp).encode())
if 'data' in request.args:
plotData = {
'data': str(resp),
'shape': resp.shape,
'mask': str(resp.mask)
}
plotData = json.dumps(plotData)
return Response(plotData, status=200, mimetype='application/json')
return resp
def test_get_variables(self, m):
m.return_value = {"ds": {"variables": {"k": {}, "key": {}}}}
result = DatasetConfig("ds").variables
self.assertEqual(len(result), 2)
self.assertIn("k", result)
self.assertIn("key", result)
def get_data_v1_0():
"""
Returns a geojson representation of requested model data.
API Format: GET /api/v1.0/data?...
Required params:
* dataset: dataset key (e.g. giops_day)
* variable: variable key (e.g. votemper)
* time: time index (e.g. 0)
* depth: depth index (e.g. 49)
* geometry_type: the "shape" of the data being requested
"""
try:
result = GetDataSchema().load(request.args)
except ValidationError as e:
abort(400, str(e))
config = DatasetConfig(result['dataset'])
with open_dataset(config,
variable=result['variable'],
timestamp=result['time']) as ds:
return jsonify(
geojson.dumps(data_array_to_geojson(
ds.nc_data.get_dataset_variable(
result['variable'])[result['time'], result['depth'], :, :],
config.lat_var_key, config.lon_var_key),
allow_nan=True))
def subset_query_v1_0():
args = None
if request.method == "GET":
args = request.args
else:
args = request.form
working_dir = None
subset_filename = None
if "area" in args.keys():
# Predefined area selected
area = args.get("area")
sp = area.split("/", 1)
data = utils.misc.list_areas(sp[0], simplify=False)
b = [x for x in data if x.get("key") == area]
args = args.to_dict()
args["polygons"] = b[0]["polygons"]
config = DatasetConfig(args.get("dataset_name"))
time_range = args["time"].split(",")
variables = args["variables"].split(",")
with open_dataset(
config,
variable=variables,
timestamp=int(time_range[0]),
endtime=int(time_range[1]),
) as dataset:
working_dir, subset_filename = dataset.nc_data.subset(args)
return send_from_directory(working_dir, subset_filename, as_attachment=True)
def timestamp_outOfBounds(dataset: str, time: int):
config = DatasetConfig(dataset)
length = 0
with open_dataset(config) as ds:
length = len(ds.timestamps)
return not (0 <= time < length)
def get_point_data(dataset, variable, time, depth, location):
variables = variable.split(",")
data = []
names = []
units = []
dsc = DatasetConfig(dataset)
with open_dataset(dsc) as ds:
for v in variables:
d = ds.get_point(
location[0],
location[1],
depth,
v,
time
)
variable_name = dsc.variable[ds.variables[v]].name
variable_unit = dsc.variable[ds.variables[v]].unit
data.append(d)
names.append(variable_name)
units.append(variable_unit)
result = {
'value': [f'{float(f):.4g}' for f in data],
'location': [round(f, 4) for f in location],
'name': names,
'units': units,
}
return result
def __init__(self, dataset_name: str, query: str, **kwargs):
self.dataset_name: str = dataset_name
self.dataset_config: DatasetConfig = DatasetConfig(dataset_name)
self.query: dict = query
self.format: str = kwargs['format']
self.dpi: int = int(kwargs['dpi'])
self.size: str = kwargs['size']
self.plotTitle: str = None
self.compare: bool = False
self.data = None
self.time: int = None
self.variables = None
self.variable_names = None
self.variable_units = None
self.scale = None
self.date_formatter = None
# Init interpolation stuff
self.interp: str = "gaussian"
self.radius: int = 25000 # radius in meters
self.neighbours: int = 10
self.filetype, self.mime = utils.get_mimetype(kwargs['format'])
self.filename: str = utils.get_filename(
self.plottype,
dataset_name,
self.filetype
)
def plot(self, fig=None):
if fig is None:
fig = plt.gcf()
fig.text(
1.0,
0.015,
self.dataset_config.attribution,
ha="right",
size="small",
va="top",
)
if self.compare:
fig.text(
1.0,
0.0,
DatasetConfig(self.compare["dataset"]).attribution,
ha="right",
size="small",
va="top",
)
with contextlib.closing(BytesIO()) as buf:
plt.savefig(
buf,
format=self.filetype,
dpi="figure",
bbox_inches="tight",
pad_inches=0.5,
)
plt.close(fig)
buf.seek(0)
return (buf.getvalue(), self.mime, self.filename)
def get_scale(dataset, variable, depth, timestamp, projection, extent, interp,
radius, neighbours):
"""
Calculates and returns the range (min, max values) of a selected variable,
given the current map extents.
"""
x = np.linspace(extent[0], extent[2], 50)
y = np.linspace(extent[1], extent[3], 50)
xx, yy = np.meshgrid(x, y)
dest = Proj(init=projection)
lon, lat = dest(xx, yy, inverse=True)
variables = variable.split(",")
config = DatasetConfig(dataset)
with open_dataset(config, variable=variables, timestamp=timestamp) as ds:
d = ds.get_area(np.array([lat, lon]), depth, timestamp, variables[0],
interp, radius, neighbours)
if len(variables) > 1:
d0 = d
d1 = ds.get_area(np.array([lat, lon]), depth, timestamp,
variables[1], interp, radius, neighbours)
d = __magnitude(d0,
d1) # Use your dot-product instead of exponents
return normalize_scale(d, config.variable[",".join(variables)])
def subset_query_v1_0():
args = None
if request.method == 'GET':
args = request.args
else:
args = request.form
working_dir = None
subset_filename = None
if 'area' in args.keys():
# Predefined area selected
area = args.get('area')
sp = area.split('/', 1)
data = utils.misc.list_areas(sp[0], simplify=False)
b = [x for x in data if x.get('key') == area]
args = args.to_dict()
args['polygons'] = b[0]['polygons']
config = DatasetConfig(args.get('dataset_name'))
time_range = args['time'].split(',')
variables = args['variables'].split(',')
with open_dataset(config,
variable=variables,
timestamp=int(time_range[0]),
endtime=int(time_range[1])) as dataset:
working_dir, subset_filename = dataset.nc_data.subset(args)
return send_from_directory(working_dir,
subset_filename,
as_attachment=True)
def scale(args):
"""
Draws the variable scale that is placed over the map.
Returns a BytesIO object.
"""
dataset_name = args.get("dataset")
config = DatasetConfig(dataset_name)
scale = args.get("scale")
scale = [float(component) for component in scale.split(",")]
variable = args.get("variable")
variable = variable.split(",")
if len(variable) > 1:
variable_unit = config.variable[",".join(variable)].unit
variable_name = config.variable[",".join(variable)].name
else:
variable_unit = config.variable[variable[0]].unit
variable_name = config.variable[variable[0]].name
cmap = colormap.find_colormap(variable_name)
if len(variable) == 2:
cmap = colormap.colormaps.get("speed")
fig = plt.figure(figsize=(2, 5), dpi=75)
ax = fig.add_axes([0.05, 0.05, 0.25, 0.9])
norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1])
formatter = ScalarFormatter()
formatter.set_powerlimits((-3, 4))
bar = ColorbarBase(ax,
cmap=cmap,
norm=norm,
orientation="vertical",
format=formatter)
if variable_name == "Potential Sub Surface Channel":
bar.set_ticks([0, 1], True)
bar.set_label("%s (%s)" %
(variable_name.title(), utils.mathtext(variable_unit)),
fontsize=12)
# Increase tick font size
bar.ax.tick_params(labelsize=12)
buf = BytesIO()
plt.savefig(
buf,
format="png",
dpi="figure",
transparent=False,
bbox_inches="tight",
pad_inches=0.05,
)
plt.close(fig)
buf.seek(0) # Move buffer back to beginning
return buf
def timestamps():
"""
Returns all timestamps available for a given variable in a dataset. This is variable-dependent
because datasets can have multiple "quantums", as in surface 2D variables may be hourly, while
3D variables may be daily.
API Format: /api/v1.0/timestamps/?dataset=''&variable=''
Required Arguments:
* dataset : Dataset key - Can be found using /api/v1.0/datasets
* variable : Variable key - Can be found using /api/v1.0/variables/?dataset='...'...
Raises:
APIError: if dataset or variable is not specified in the request
Returns:
Response object containing all timestamp pairs (e.g. [raw_timestamp_integer, iso_8601_date_string]) for the given
dataset and variable.
"""
args = request.args
if "dataset" not in args:
raise APIError("Please specify a dataset via ?dataset=dataset_name")
dataset = args.get("dataset")
config = DatasetConfig(dataset)
if "variable" not in args:
raise APIError("Please specify a variable via ?variable=variable_name")
variable = args.get("variable")
# Handle possible list of URLs for staggered grid velocity field datasets
url = config.url if not isinstance(config.url, list) else config.url[0]
if url.endswith(".sqlite3"):
with SQLiteDatabase(url) as db:
if variable in config.calculated_variables:
data_vars = get_data_vars_from_equation(
config.calculated_variables[variable]['equation'],
[v.key for v in db.get_data_variables()])
vals = db.get_timestamps(data_vars[0])
else:
vals = db.get_timestamps(variable)
else:
with open_dataset(config, variable=variable) as ds:
vals = list(map(int, ds.nc_data.time_variable.values))
converted_vals = time_index_to_datetime(vals, config.time_dim_units)
result = []
for idx, date in enumerate(converted_vals):
if config.quantum == 'month' or config.variable[
variable].quantum == 'month':
date = datetime.datetime(date.year, date.month, 15)
result.append({'id': vals[idx], 'value': date})
result = sorted(result, key=lambda k: k['id'])
js = json.dumps(result, cls=DateTimeEncoder)
resp = Response(js, status=200, mimetype='application/json')
return resp
def get_data_v1_0(dataset: str, variable: str, time: str, depth: str,
location: str):
config = DatasetConfig(dataset)
with open_dataset(config) as ds:
date = ds.convert_to_timestamp(time)
#print(date)
return routes.routes_impl.get_data_impl(dataset, variable, date, depth,
location)
def test_vector_variable(self, m):
m.return_value = {
"key": {
"enabled": True,
"variables": {
"var,var2": {
"name": "my_variable",
}
}
},
}
self.assertEqual(len(DatasetConfig("key").variables), 0)
self.assertEqual(len(DatasetConfig("key").vector_variables), 1)
result = DatasetConfig("key").variable["var,var2"]
self.assertEqual(result.name, "my_variable")
self.assertEqual(result.unit, "Unknown")
def range_query_v1_0(dataset: str, variable: str, interp: str, radius: int,
neighbours: int, projection: str, extent: str, depth: str,
time: str):
config = DatasetConfig(dataset)
with open_dataset(config) as ds:
date = ds.convert_to_timestamp(time)
return routes.routes_impl.range_query_impl(interp, radius, neighbours,
dataset, projection, extent,
variable, depth, date)
def test_get_dataset_misc(self, m):
m.return_value = {
"dataset": {
"url": "the_url",
"attribution": "My attribution <b>bold</b>",
"climatology": "climatology_url",
"cache": 5,
}
}
self.assertEqual(DatasetConfig("dataset").url, "the_url")
self.assertEqual(
DatasetConfig("dataset").climatology, "climatology_url")
self.assertEqual(
DatasetConfig("dataset").attribution, "My attribution bold")
self.assertEqual(DatasetConfig("dataset").cache, 5)
m.return_value = {"dataset2": {}}
self.assertEqual(DatasetConfig("dataset2").cache, None)
def tile_v1_0(projection: str, interp: str, radius: int, neighbours: int,
dataset: str, variable: str, time: str, depth: str, scale: str,
zoom: int, x: int, y: int):
config = DatasetConfig(dataset)
with open_dataset(config) as ds:
date = ds.convert_to_timestamp(time)
return routes.routes_impl.tile_impl(projection, interp, radius,
neighbours, dataset, variable,
date, depth, scale, zoom, x, y)
def subtract_other(self, data):
if self.compare:
compare_config = DatasetConfig(self.compare["dataset"])
with Dataset(compare_config.url, "r") as dataset:
cli = self.get_data(dataset, self.compare["variables"],
self.compare["time"])
for idx, _ in enumerate(self.variables):
data[:, idx, :] = data[:, idx, :] - cli[:, idx, :]
return data
def test_open_dataset_with_null_url_raises(self, patch_get_dataset_config):
patch_get_dataset_config.return_value = {
"giops": {
"url": None,
"variables": {}
}
}
config = DatasetConfig("giops")
with self.assertRaises(ValueError):
open_dataset(config)
def test_open_dataset_no_model_class_raises(self,
patch_get_dataset_config):
patch_get_dataset_config.return_value = {
"giops": {
"url": "tests/testdata/mercator_test.nc",
"variables": {}
}
}
config = DatasetConfig("giops")
with self.assertRaises(ValueError):
open_dataset(config)
def timestamps():
"""
Returns all timestamps available for a given variable in a dataset.
This is variable-dependent because datasets can have multiple "quantums",
as in surface 2D variables may be hourly, while 3D variables may be daily.
Required Arguments:
* dataset : Dataset key - Can be found using /api/v1.0/datasets
* variable : Variable key - Can be found using /api/v1.0/variables/?dataset='...'...
Returns:
All timestamp pairs (e.g. [raw_timestamp_integer, iso_8601_date_string])
for the given dataset and variable.
"""
try:
result = TimestampsSchema().load(request.args)
except ValidationError as e:
abort(400, str(e))
dataset = result["dataset"]
variable = result["variable"]
config = DatasetConfig(dataset)
# Handle possible list of URLs for staggered grid velocity field datasets
url = config.url if not isinstance(config.url, list) else config.url[0]
if url.endswith(".sqlite3"):
with SQLiteDatabase(url) as db:
if variable in config.calculated_variables:
data_vars = get_data_vars_from_equation(
config.calculated_variables[variable]["equation"],
[v.key for v in db.get_data_variables()],
)
vals = db.get_timestamps(data_vars[0])
else:
vals = db.get_timestamps(variable)
else:
with open_dataset(config, variable=variable) as ds:
vals = list(map(int, ds.nc_data.time_variable.values))
converted_vals = time_index_to_datetime(vals, config.time_dim_units)
result = []
for idx, date in enumerate(converted_vals):
if config.quantum == "month" or config.variable[variable].quantum == "month":
date = datetime.datetime(date.year, date.month, 15)
result.append({"id": vals[idx], "value": date})
result = sorted(result, key=lambda k: k["id"])
js = json.dumps(result, cls=DateTimeEncoder)
resp = Response(js, status=200, mimetype="application/json")
return resp
def subtract_other(self, data):
if self.compare:
compare_config = DatasetConfig(self.compare['dataset'])
with Dataset(compare_config.url, 'r') as dataset:
cli = self.get_data(dataset, self.compare['variables'],
self.compare['time'])
for idx, _ in enumerate(self.variables):
data[:, idx, :] = \
data[:, idx, :] - cli[:, idx, :]
return data
def test_open_dataset_returns_nemo_object(self, patch_calculated_data,
patch_get_dataset_config):
patch_get_dataset_config.return_value = {
"giops": {
"url": "tests/testdata/nemo_test.nc",
"variables": {}
}
}
config = DatasetConfig("giops")
with open_dataset(config) as ds:
self.assertTrue(isinstance(ds, Nemo))
def test_open_dataset_meta_only_returns_fvcom_object(
self, patch_calculated_data, patch_get_dataset_config):
patch_get_dataset_config.return_value = {
"giops": {
"url": "tests/testdata/fvcom_test.nc",
"variables": {}
}
}
config = DatasetConfig("giops")
with open_dataset(config, meta_only=True) as ds:
self.assertTrue(isinstance(ds, Fvcom))
