def test_vector_variable(self, m): m.return_value = { "key": { "enabled": True, "variables": { "magmyvar": { "name": "my_variable", "east_vector_component": "u", "north_vector_component": "v", }, "magnitudemyvar": { "name": "my_variable", "east_vector_component": "u", "north_vector_component": "v", }, }, }, } self.assertEqual(len(DatasetConfig("key").variables), 2) self.assertEqual(len(DatasetConfig("key").vector_variables), 2) result = DatasetConfig("key").variable["magmyvar"] self.assertEqual(result.name, "my_variable") self.assertEqual(result.east_vector_component, "u") self.assertEqual(result.north_vector_component, "v") self.assertEqual(result.unit, "Unknown")
def test_datasetconfig_object(self, m): m.return_value = { "key": { "enabled": True, "url": "my_url", "climatology": "my_climatology", "name": "my_name", "help": "my_help", "quantum": "my_quantum", "attribution": "my_<b>attribution</b>", "cache": "123", "variables": { "var": { "name": "my_variable", } } }, } self.assertEqual(len(DatasetConfig.get_datasets()), 1) result = DatasetConfig("key") self.assertEqual(result.url, "my_url") self.assertEqual(result.climatology, "my_climatology") self.assertEqual(result.name, "my_name") self.assertEqual(result.help, "my_help") self.assertEqual(result.quantum, "my_quantum") self.assertEqual(result.attribution, "my_attribution") self.assertEqual(result.cache, 123) self.assertFalse(result.variable[Mock(key="var")].is_hidden)
def datasets_query_v1_0(): """ API Format: /api/v1.0/datasets/ Optional arguments: * id : Show only the name and id of the datasets Returns: Response -- Response object containing list of available datasets w/ some metadata. """ data = [] if 'id' in request.args: for key in DatasetConfig.get_datasets(): config = DatasetConfig(key) data.append({'id': key, 'value': config.name}) else: for key in DatasetConfig.get_datasets(): config = DatasetConfig(key) data.append({ 'id': key, 'value': config.name, 'quantum': config.quantum, 'help': config.help, 'attribution': config.attribution, }) data = sorted(data, key=lambda k: k['value']) resp = jsonify(data) return resp
def datasets_query_v1_0(): """ API Format: /api/v1.0/datasets/ Optional arguments: * id: Show only the name and id of the datasets Returns: List of available datasets w/ some metadata. """ data = [] if "id" in request.args: for key in DatasetConfig.get_datasets(): config = DatasetConfig(key) data.append({"id": key, "value": config.name}) else: for key in DatasetConfig.get_datasets(): config = DatasetConfig(key) data.append( { "id": key, "value": config.name, "quantum": config.quantum, "help": config.help, "attribution": config.attribution, } ) data = sorted(data, key=lambda k: k["value"]) resp = jsonify(data) return resp
def thorough(self): self.app = create_app() with self.app.app_context(): datasets = DatasetConfig.get_datasets() variables = {} for dataset in datasets: variables.update({dataset: DatasetConfig(dataset).variables}) data = {'datasets': datasets, 'variables': variables} return data
def test_datasetconfig_object(self, m): m.return_value = { "key": { "enabled": True, "url": "my_url", "geo_ref": { "url": "my_geo_ref_url", "drop_variables": ["bathymetry"], }, "climatology": "my_climatology", "name": "my_name", "help": "my_help", "quantum": "my_quantum", "type": "my_type", "grid_angle_file_url": "my_grid_angle_file_url", "bathymetry_file_url": "my_bathy_file_url.nc", "model_class": "my_model_class", "time_dim_units": "my_time_units", "attribution": "my_<b>attribution</b>", "cache": "123", "lat_var_key": "my_lat", "lon_var_key": "my_lon", "variables": { "var": { "name": "my_variable", } }, } } self.assertEqual(len(DatasetConfig.get_datasets()), 1) result = DatasetConfig("key") self.assertEqual(result.url, "my_url") self.assertEqual(result.geo_ref["url"], "my_geo_ref_url") self.assertEqual(result.geo_ref["drop_variables"], ["bathymetry"]) self.assertEqual(result.key, "key") self.assertEqual(result.climatology, "my_climatology") self.assertEqual(result.name, "my_name") self.assertEqual(result.help, "my_help") self.assertEqual(result.quantum, "my_quantum") self.assertEqual(result.grid_angle_file_url, "my_grid_angle_file_url") self.assertEqual(result.bathymetry_file_url, "my_bathy_file_url.nc") self.assertEqual(result.model_class, "my_model_class") self.assertEqual(result.type, "my_type") self.assertEqual(result.time_dim_units, "my_time_units") self.assertEqual(result.lat_var_key, "my_lat") self.assertEqual(result.lon_var_key, "my_lon") self.assertEqual(result.attribution, "my_attribution") self.assertEqual(result.cache, 123) self.assertFalse(result.variable[Mock(key="var")].is_hidden)
def plot_v1_0(): if request.method == 'GET': args = request.args else: args = request.form query = json.loads(args.get('query')) config = DatasetConfig(query.get('dataset')) with open_dataset(config) as dataset: if 'time' in query: query['time'] = dataset.convert_to_timestamp(query.get('time')) else: query['starttime'] = dataset.convert_to_timestamp( query.get('starttime')) query['endtime'] = dataset.convert_to_timestamp( query.get('endtime')) resp = routes.routes_impl.plot_impl(args, query) m = hashlib.md5() m.update(str(resp).encode()) if 'data' in request.args: plotData = { 'data': str(resp), 'shape': resp.shape, 'mask': str(resp.mask) } plotData = json.dumps(plotData) return Response(plotData, status=200, mimetype='application/json') return resp
def test_get_variables(self, m): m.return_value = {"ds": {"variables": {"k": {}, "key": {}}}} result = DatasetConfig("ds").variables self.assertEqual(len(result), 2) self.assertIn("k", result) self.assertIn("key", result)
def get_data_v1_0(): """ Returns a geojson representation of requested model data. API Format: GET /api/v1.0/data?... Required params: * dataset: dataset key (e.g. giops_day) * variable: variable key (e.g. votemper) * time: time index (e.g. 0) * depth: depth index (e.g. 49) * geometry_type: the "shape" of the data being requested """ try: result = GetDataSchema().load(request.args) except ValidationError as e: abort(400, str(e)) config = DatasetConfig(result['dataset']) with open_dataset(config, variable=result['variable'], timestamp=result['time']) as ds: return jsonify( geojson.dumps(data_array_to_geojson( ds.nc_data.get_dataset_variable( result['variable'])[result['time'], result['depth'], :, :], config.lat_var_key, config.lon_var_key), allow_nan=True))
def subset_query_v1_0(): args = None if request.method == "GET": args = request.args else: args = request.form working_dir = None subset_filename = None if "area" in args.keys(): # Predefined area selected area = args.get("area") sp = area.split("/", 1) data = utils.misc.list_areas(sp[0], simplify=False) b = [x for x in data if x.get("key") == area] args = args.to_dict() args["polygons"] = b[0]["polygons"] config = DatasetConfig(args.get("dataset_name")) time_range = args["time"].split(",") variables = args["variables"].split(",") with open_dataset( config, variable=variables, timestamp=int(time_range[0]), endtime=int(time_range[1]), ) as dataset: working_dir, subset_filename = dataset.nc_data.subset(args) return send_from_directory(working_dir, subset_filename, as_attachment=True)
def timestamp_outOfBounds(dataset: str, time: int): config = DatasetConfig(dataset) length = 0 with open_dataset(config) as ds: length = len(ds.timestamps) return not (0 <= time < length)
def get_point_data(dataset, variable, time, depth, location): variables = variable.split(",") data = [] names = [] units = [] dsc = DatasetConfig(dataset) with open_dataset(dsc) as ds: for v in variables: d = ds.get_point( location[0], location[1], depth, v, time ) variable_name = dsc.variable[ds.variables[v]].name variable_unit = dsc.variable[ds.variables[v]].unit data.append(d) names.append(variable_name) units.append(variable_unit) result = { 'value': [f'{float(f):.4g}' for f in data], 'location': [round(f, 4) for f in location], 'name': names, 'units': units, } return result
def __init__(self, dataset_name: str, query: str, **kwargs): self.dataset_name: str = dataset_name self.dataset_config: DatasetConfig = DatasetConfig(dataset_name) self.query: dict = query self.format: str = kwargs['format'] self.dpi: int = int(kwargs['dpi']) self.size: str = kwargs['size'] self.plotTitle: str = None self.compare: bool = False self.data = None self.time: int = None self.variables = None self.variable_names = None self.variable_units = None self.scale = None self.date_formatter = None # Init interpolation stuff self.interp: str = "gaussian" self.radius: int = 25000 # radius in meters self.neighbours: int = 10 self.filetype, self.mime = utils.get_mimetype(kwargs['format']) self.filename: str = utils.get_filename( self.plottype, dataset_name, self.filetype )
def plot(self, fig=None): if fig is None: fig = plt.gcf() fig.text( 1.0, 0.015, self.dataset_config.attribution, ha="right", size="small", va="top", ) if self.compare: fig.text( 1.0, 0.0, DatasetConfig(self.compare["dataset"]).attribution, ha="right", size="small", va="top", ) with contextlib.closing(BytesIO()) as buf: plt.savefig( buf, format=self.filetype, dpi="figure", bbox_inches="tight", pad_inches=0.5, ) plt.close(fig) buf.seek(0) return (buf.getvalue(), self.mime, self.filename)
def get_scale(dataset, variable, depth, timestamp, projection, extent, interp, radius, neighbours): """ Calculates and returns the range (min, max values) of a selected variable, given the current map extents. """ x = np.linspace(extent[0], extent[2], 50) y = np.linspace(extent[1], extent[3], 50) xx, yy = np.meshgrid(x, y) dest = Proj(init=projection) lon, lat = dest(xx, yy, inverse=True) variables = variable.split(",") config = DatasetConfig(dataset) with open_dataset(config, variable=variables, timestamp=timestamp) as ds: d = ds.get_area(np.array([lat, lon]), depth, timestamp, variables[0], interp, radius, neighbours) if len(variables) > 1: d0 = d d1 = ds.get_area(np.array([lat, lon]), depth, timestamp, variables[1], interp, radius, neighbours) d = __magnitude(d0, d1) # Use your dot-product instead of exponents return normalize_scale(d, config.variable[",".join(variables)])
def subset_query_v1_0(): args = None if request.method == 'GET': args = request.args else: args = request.form working_dir = None subset_filename = None if 'area' in args.keys(): # Predefined area selected area = args.get('area') sp = area.split('/', 1) data = utils.misc.list_areas(sp[0], simplify=False) b = [x for x in data if x.get('key') == area] args = args.to_dict() args['polygons'] = b[0]['polygons'] config = DatasetConfig(args.get('dataset_name')) time_range = args['time'].split(',') variables = args['variables'].split(',') with open_dataset(config, variable=variables, timestamp=int(time_range[0]), endtime=int(time_range[1])) as dataset: working_dir, subset_filename = dataset.nc_data.subset(args) return send_from_directory(working_dir, subset_filename, as_attachment=True)
def scale(args): """ Draws the variable scale that is placed over the map. Returns a BytesIO object. """ dataset_name = args.get("dataset") config = DatasetConfig(dataset_name) scale = args.get("scale") scale = [float(component) for component in scale.split(",")] variable = args.get("variable") variable = variable.split(",") if len(variable) > 1: variable_unit = config.variable[",".join(variable)].unit variable_name = config.variable[",".join(variable)].name else: variable_unit = config.variable[variable[0]].unit variable_name = config.variable[variable[0]].name cmap = colormap.find_colormap(variable_name) if len(variable) == 2: cmap = colormap.colormaps.get("speed") fig = plt.figure(figsize=(2, 5), dpi=75) ax = fig.add_axes([0.05, 0.05, 0.25, 0.9]) norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1]) formatter = ScalarFormatter() formatter.set_powerlimits((-3, 4)) bar = ColorbarBase(ax, cmap=cmap, norm=norm, orientation="vertical", format=formatter) if variable_name == "Potential Sub Surface Channel": bar.set_ticks([0, 1], True) bar.set_label("%s (%s)" % (variable_name.title(), utils.mathtext(variable_unit)), fontsize=12) # Increase tick font size bar.ax.tick_params(labelsize=12) buf = BytesIO() plt.savefig( buf, format="png", dpi="figure", transparent=False, bbox_inches="tight", pad_inches=0.05, ) plt.close(fig) buf.seek(0) # Move buffer back to beginning return buf
def timestamps(): """ Returns all timestamps available for a given variable in a dataset. This is variable-dependent because datasets can have multiple "quantums", as in surface 2D variables may be hourly, while 3D variables may be daily. API Format: /api/v1.0/timestamps/?dataset=''&variable='' Required Arguments: * dataset : Dataset key - Can be found using /api/v1.0/datasets * variable : Variable key - Can be found using /api/v1.0/variables/?dataset='...'... Raises: APIError: if dataset or variable is not specified in the request Returns: Response object containing all timestamp pairs (e.g. [raw_timestamp_integer, iso_8601_date_string]) for the given dataset and variable. """ args = request.args if "dataset" not in args: raise APIError("Please specify a dataset via ?dataset=dataset_name") dataset = args.get("dataset") config = DatasetConfig(dataset) if "variable" not in args: raise APIError("Please specify a variable via ?variable=variable_name") variable = args.get("variable") # Handle possible list of URLs for staggered grid velocity field datasets url = config.url if not isinstance(config.url, list) else config.url[0] if url.endswith(".sqlite3"): with SQLiteDatabase(url) as db: if variable in config.calculated_variables: data_vars = get_data_vars_from_equation( config.calculated_variables[variable]['equation'], [v.key for v in db.get_data_variables()]) vals = db.get_timestamps(data_vars[0]) else: vals = db.get_timestamps(variable) else: with open_dataset(config, variable=variable) as ds: vals = list(map(int, ds.nc_data.time_variable.values)) converted_vals = time_index_to_datetime(vals, config.time_dim_units) result = [] for idx, date in enumerate(converted_vals): if config.quantum == 'month' or config.variable[ variable].quantum == 'month': date = datetime.datetime(date.year, date.month, 15) result.append({'id': vals[idx], 'value': date}) result = sorted(result, key=lambda k: k['id']) js = json.dumps(result, cls=DateTimeEncoder) resp = Response(js, status=200, mimetype='application/json') return resp
def get_data_v1_0(dataset: str, variable: str, time: str, depth: str, location: str): config = DatasetConfig(dataset) with open_dataset(config) as ds: date = ds.convert_to_timestamp(time) #print(date) return routes.routes_impl.get_data_impl(dataset, variable, date, depth, location)
def test_vector_variable(self, m): m.return_value = { "key": { "enabled": True, "variables": { "var,var2": { "name": "my_variable", } } }, } self.assertEqual(len(DatasetConfig("key").variables), 0) self.assertEqual(len(DatasetConfig("key").vector_variables), 1) result = DatasetConfig("key").variable["var,var2"] self.assertEqual(result.name, "my_variable") self.assertEqual(result.unit, "Unknown")
def range_query_v1_0(dataset: str, variable: str, interp: str, radius: int, neighbours: int, projection: str, extent: str, depth: str, time: str): config = DatasetConfig(dataset) with open_dataset(config) as ds: date = ds.convert_to_timestamp(time) return routes.routes_impl.range_query_impl(interp, radius, neighbours, dataset, projection, extent, variable, depth, date)
def test_get_dataset_misc(self, m): m.return_value = { "dataset": { "url": "the_url", "attribution": "My attribution <b>bold</b>", "climatology": "climatology_url", "cache": 5, } } self.assertEqual(DatasetConfig("dataset").url, "the_url") self.assertEqual( DatasetConfig("dataset").climatology, "climatology_url") self.assertEqual( DatasetConfig("dataset").attribution, "My attribution bold") self.assertEqual(DatasetConfig("dataset").cache, 5) m.return_value = {"dataset2": {}} self.assertEqual(DatasetConfig("dataset2").cache, None)
def tile_v1_0(projection: str, interp: str, radius: int, neighbours: int, dataset: str, variable: str, time: str, depth: str, scale: str, zoom: int, x: int, y: int): config = DatasetConfig(dataset) with open_dataset(config) as ds: date = ds.convert_to_timestamp(time) return routes.routes_impl.tile_impl(projection, interp, radius, neighbours, dataset, variable, date, depth, scale, zoom, x, y)
def subtract_other(self, data): if self.compare: compare_config = DatasetConfig(self.compare["dataset"]) with Dataset(compare_config.url, "r") as dataset: cli = self.get_data(dataset, self.compare["variables"], self.compare["time"]) for idx, _ in enumerate(self.variables): data[:, idx, :] = data[:, idx, :] - cli[:, idx, :] return data
def test_open_dataset_with_null_url_raises(self, patch_get_dataset_config): patch_get_dataset_config.return_value = { "giops": { "url": None, "variables": {} } } config = DatasetConfig("giops") with self.assertRaises(ValueError): open_dataset(config)
def test_open_dataset_no_model_class_raises(self, patch_get_dataset_config): patch_get_dataset_config.return_value = { "giops": { "url": "tests/testdata/mercator_test.nc", "variables": {} } } config = DatasetConfig("giops") with self.assertRaises(ValueError): open_dataset(config)
def timestamps(): """ Returns all timestamps available for a given variable in a dataset. This is variable-dependent because datasets can have multiple "quantums", as in surface 2D variables may be hourly, while 3D variables may be daily. Required Arguments: * dataset : Dataset key - Can be found using /api/v1.0/datasets * variable : Variable key - Can be found using /api/v1.0/variables/?dataset='...'... Returns: All timestamp pairs (e.g. [raw_timestamp_integer, iso_8601_date_string]) for the given dataset and variable. """ try: result = TimestampsSchema().load(request.args) except ValidationError as e: abort(400, str(e)) dataset = result["dataset"] variable = result["variable"] config = DatasetConfig(dataset) # Handle possible list of URLs for staggered grid velocity field datasets url = config.url if not isinstance(config.url, list) else config.url[0] if url.endswith(".sqlite3"): with SQLiteDatabase(url) as db: if variable in config.calculated_variables: data_vars = get_data_vars_from_equation( config.calculated_variables[variable]["equation"], [v.key for v in db.get_data_variables()], ) vals = db.get_timestamps(data_vars[0]) else: vals = db.get_timestamps(variable) else: with open_dataset(config, variable=variable) as ds: vals = list(map(int, ds.nc_data.time_variable.values)) converted_vals = time_index_to_datetime(vals, config.time_dim_units) result = [] for idx, date in enumerate(converted_vals): if config.quantum == "month" or config.variable[variable].quantum == "month": date = datetime.datetime(date.year, date.month, 15) result.append({"id": vals[idx], "value": date}) result = sorted(result, key=lambda k: k["id"]) js = json.dumps(result, cls=DateTimeEncoder) resp = Response(js, status=200, mimetype="application/json") return resp
def subtract_other(self, data): if self.compare: compare_config = DatasetConfig(self.compare['dataset']) with Dataset(compare_config.url, 'r') as dataset: cli = self.get_data(dataset, self.compare['variables'], self.compare['time']) for idx, _ in enumerate(self.variables): data[:, idx, :] = \ data[:, idx, :] - cli[:, idx, :] return data
def test_open_dataset_returns_nemo_object(self, patch_calculated_data, patch_get_dataset_config): patch_get_dataset_config.return_value = { "giops": { "url": "tests/testdata/nemo_test.nc", "variables": {} } } config = DatasetConfig("giops") with open_dataset(config) as ds: self.assertTrue(isinstance(ds, Nemo))
def test_open_dataset_meta_only_returns_fvcom_object( self, patch_calculated_data, patch_get_dataset_config): patch_get_dataset_config.return_value = { "giops": { "url": "tests/testdata/fvcom_test.nc", "variables": {} } } config = DatasetConfig("giops") with open_dataset(config, meta_only=True) as ds: self.assertTrue(isinstance(ds, Fvcom))