def main():
# get precomputed file / compute preprocessing & feature
data = [
open_dataset("dev", 1),
open_dataset("train", 1),
open_dataset("test", 1)
]
# log.write("Preprocessing dataset...")
# pre_processed_data = preprocessing_data(flatten(data))
log.write("Feature extraction...")
feature_data = feature_extraction(flatten(data))
for fold in range(1, 6):
log.write("Get fold {} of IndoSum dataset".format(fold))
train_data, val_data, test_data = get_data(fold, feature_data)
for method in methods_ready:
# log.write("Evaluating {}".format(method))
log.write("==================================")
log.write("Prediction using {}".format(method))
log.write("==================================")
predicted_labels = run_experiment(train_data, val_data, test_data,
method)
log.write("Acc/P/R evaluation")
label_evaluation(test_data, predicted_labels)
log.write("ROUGE evaluation")
rouge_evaluation(test_data, predicted_labels)
del predicted_labels
del train_data
del val_data
del test_data
gc.collect()
def get_scale(dataset, variable, depth, time, projection, extent):
x = np.linspace(extent[0], extent[2], 50)
y = np.linspace(extent[1], extent[3], 50)
xx, yy = np.meshgrid(x, y)
dest = Proj(init=projection)
lon, lat = dest(xx, yy, inverse=True)
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
d = ds.get_area(
np.array([lat, lon]),
depth,
time,
variables[0]
)
if len(variables) > 1:
d0 = d
d1 = ds.get_area(
np.array([lat, lon]),
depth,
time,
variables[1]
)
d = np.sqrt(d0 ** 2 + d1 ** 2)
variable_unit = get_variable_unit(dataset,
ds.variables[variables[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset), 'r') as ds:
c = ds.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
variables[0]
)
if len(variables) > 1:
c0 = c
c1 = ds.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
variables[1]
)
c = np.sqrt(c0 ** 2 + c1 ** 2)
d = d - c
m = max(abs(d.min()), abs(d.max()))
return -m, m
return d.min(), d.max()
def demo():
flatten = lambda l: [item for sublist in l for item in sublist]
data = [
open_dataset("dev", 1),
open_dataset("train", 1),
open_dataset("test", 1)
]
data = flatten(data)
save_preprocessed_data(data)
def test_open_dataset_with_null_url_raises(self, patch_get_dataset_config):
patch_get_dataset_config.return_value = {
"giops": {
"url": None,
"variables": {}
}
}
config = DatasetConfig("giops")
with self.assertRaises(ValueError):
open_dataset(config)
def get_data(fold, feature_data=None, preprocessed_data=None):
train_data = open_dataset("train", fold)
val_data = open_dataset("dev", fold)
test_data = open_dataset("test", fold)
for data_split in [train_data, val_data, test_data]:
for doc in data_split:
for attr in feature_attr_name:
doc[attr] = feature_data[doc["id"]][attr]
if preprocessed_data:
for attr in preprocessing_attr:
doc[attr] = preprocessed_data[doc["id"]][attr]
return train_data, val_data, test_data
def test_open_dataset_no_model_class_raises(self,
patch_get_dataset_config):
patch_get_dataset_config.return_value = {
"giops": {
"url": "tests/testdata/mercator_test.nc",
"variables": {}
}
}
config = DatasetConfig("giops")
with self.assertRaises(ValueError):
open_dataset(config)
def timestamp_for_date(old_dataset, date, new_dataset):
with open_dataset(get_dataset_url(old_dataset)) as ds:
timestamp = ds.timestamps[date]
with open_dataset(get_dataset_url(new_dataset)) as ds:
timestamps = ds.timestamps
diffs = np.vectorize(lambda x: x.total_seconds())(timestamps - timestamp)
idx = np.where(diffs <= 0)[0]
res = 0
if len(idx) > 0:
res = idx.max()
return Response(json.dumps(res), status=200, mimetype='application/json')
def get_point_data(dataset, variable, time, depth, location):
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
data = []
names = []
units = []
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
for v in variables:
d = ds.get_point(
location[0],
location[1],
depth,
time,
v
)
variable_name = get_variable_name(dataset, ds.variables[v])
variable_unit = get_variable_unit(dataset, ds.variables[v])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
data.append(d)
names.append(variable_name)
units.append(variable_unit)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset)) as ds:
for idx, v in enumerate(variables):
d = ds.get_point(
location[0],
location[1],
depth,
timestamp.month,
v
)
data[idx] = data[idx] - d
names[idx] = names[idx] + " Anomaly"
result = {
'value': map(lambda f: '%s' % float('%.4g' % f), data),
'location': map(lambda f: round(f, 4), location),
'name': names,
'units': units,
}
return result
def get_scale(dataset, variable, depth, time, projection, extent, interp,
radius, neighbours):
x = np.linspace(extent[0], extent[2], 50)
y = np.linspace(extent[1], extent[3], 50)
xx, yy = np.meshgrid(x, y)
dest = Proj(init=projection)
lon, lat = dest(xx, yy, inverse=True)
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
d = ds.get_area(np.array([lat, lon]), depth, time, variables[0],
interp, radius, neighbours)
if len(variables) > 1:
d0 = d
d1 = ds.get_area(np.array([lat, lon]), depth, time, variables[1],
interp, radius, neighbours)
d = np.sqrt(d0**2 + d1**2)
variable_unit = get_variable_unit(dataset, ds.variables[variables[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset), 'r') as ds:
c = ds.get_area(np.array([lat, lon]), depth, timestamp.month - 1,
variables[0], interp, radius, neighbours)
if len(variables) > 1:
c0 = c
c1 = ds.get_area(np.array([lat,
lon]), depth, timestamp.month - 1,
variables[1], interp, radius, neighbours)
c = np.sqrt(c0**2 + c1**2)
d = d - c
m = max(abs(d.nanmin()), abs(d.nanmax()))
return -m, m
# Return min and max values of selected variable, while ignoring
# nan values
return np.nanmin(d), np.nanmax(d)
def get_data_v1_0():
"""
Returns a geojson representation of requested model data.
API Format: GET /api/v1.0/data?...
Required params:
* dataset: dataset key (e.g. giops_day)
* variable: variable key (e.g. votemper)
* time: time index (e.g. 0)
* depth: depth index (e.g. 49)
* geometry_type: the "shape" of the data being requested
"""
try:
result = GetDataSchema().load(request.args)
except ValidationError as e:
abort(400, str(e))
config = DatasetConfig(result['dataset'])
with open_dataset(config,
variable=result['variable'],
timestamp=result['time']) as ds:
return jsonify(
geojson.dumps(data_array_to_geojson(
ds.nc_data.get_dataset_variable(
result['variable'])[result['time'], result['depth'], :, :],
config.lat_var_key, config.lon_var_key),
allow_nan=True))
def timestamp_outOfBounds(dataset: str, time: int):
config = DatasetConfig(dataset)
length = 0
with open_dataset(config) as ds:
length = len(ds.timestamps)
return not (0 <= time < length)
def get_point_data(dataset, variable, time, depth, location):
variables = variable.split(",")
data = []
names = []
units = []
dsc = DatasetConfig(dataset)
with open_dataset(dsc) as ds:
for v in variables:
d = ds.get_point(
location[0],
location[1],
depth,
v,
time
)
variable_name = dsc.variable[ds.variables[v]].name
variable_unit = dsc.variable[ds.variables[v]].unit
data.append(d)
names.append(variable_name)
units.append(variable_unit)
result = {
'value': [f'{float(f):.4g}' for f in data],
'location': [round(f, 4) for f in location],
'name': names,
'units': units,
}
return result
def time_query():
data = []
if 'dataset' in request.args:
dataset = request.args['dataset']
quantum = request.args.get('quantum')
with open_dataset(get_dataset_url(dataset)) as ds:
for idx, date in enumerate(ds.timestamps):
if quantum == 'month':
date = datetime.datetime(date.year, date.month, 15)
data.append({
'id': idx,
'value': date.replace(tzinfo=pytz.UTC)
})
data = sorted(data, key=lambda k: k['id'])
class DateTimeEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, datetime.datetime):
return o.isoformat()
return json.JSONEncoder.default(self, o)
js = json.dumps(data, cls=DateTimeEncoder)
resp = Response(js, status=200, mimetype='application/json')
return resp
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as d:
if self.time < 0:
self.time += len(d.timestamps)
time = np.clip(self.time, 0, len(d.timestamps) - 1)
timestamp = d.timestamps[time]
try:
self.load_misc(d, self.variables)
except IndexError as e:
raise ClientError(
gettext(
"The selected variable(s) were not found in the dataset. \
Most likely, this variable is a derived product from existing dataset variables. \
Please select another variable. ") + str(e))
point_data, point_depths = self.get_data(d, self.variables, time)
point_data = self.apply_scale_factors(point_data)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units, point_data)
self.data = self.subtract_other(point_data)
self.depths = point_depths
self.timestamp = timestamp
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
latvar, lonvar = utils.get_latlon_vars(dataset)
if self.depth:
if self.depth == 'bottom':
self.depth_value = 'Bottom'
self.depth_unit = ''
else:
self.depth = np.clip(int(self.depth), 0,
len(dataset.depths) - 1)
self.depth_value = np.round(dataset.depths[self.depth])
self.depth_unit = "m"
else:
self.depth_value = 0
self.depth_unit = "m"
self.fix_startend_times(dataset)
time = range(self.starttime, self.endtime + 1)
if len(self.variables) > 1:
v = []
for name in self.variables:
pts, distance, t, value = dataset.get_path(
self.points,
self.depth,
time,
name
)
v.append(value ** 2)
value = np.sqrt(np.ma.sum(v, axis=0))
else:
pts, distance, t, value = dataset.get_path(
self.points,
self.depth,
time,
self.variables[0]
)
self.path_points = pts
self.distance = distance
variable_names = self.get_variable_names(dataset, self.variables)
variable_units = self.get_variable_units(dataset, self.variables)
scale_factors = self.get_variable_scale_factors(dataset,
self.variables)
self.variable_unit, self.data = self.kelvin_to_celsius(
variable_units[0],
value
)
self.data = np.multiply(self.data, scale_factors[0])
self.variable_name = variable_names[0]
self.data = self.data.transpose()
if self.cmap is None:
self.cmap = colormap.find_colormap(self.variable_name)
self.times = dataset.timestamps[self.starttime:self.endtime + 1]
def subset_query_v1_0():
args = None
if request.method == "GET":
args = request.args
else:
args = request.form
working_dir = None
subset_filename = None
if "area" in args.keys():
# Predefined area selected
area = args.get("area")
sp = area.split("/", 1)
data = utils.misc.list_areas(sp[0], simplify=False)
b = [x for x in data if x.get("key") == area]
args = args.to_dict()
args["polygons"] = b[0]["polygons"]
config = DatasetConfig(args.get("dataset_name"))
time_range = args["time"].split(",")
variables = args["variables"].split(",")
with open_dataset(
config,
variable=variables,
timestamp=int(time_range[0]),
endtime=int(time_range[1]),
) as dataset:
working_dir, subset_filename = dataset.nc_data.subset(args)
return send_from_directory(working_dir, subset_filename, as_attachment=True)
def subset_query_v1_0():
args = None
if request.method == 'GET':
args = request.args
else:
args = request.form
working_dir = None
subset_filename = None
if 'area' in args.keys():
# Predefined area selected
area = args.get('area')
sp = area.split('/', 1)
data = utils.misc.list_areas(sp[0], simplify=False)
b = [x for x in data if x.get('key') == area]
args = args.to_dict()
args['polygons'] = b[0]['polygons']
config = DatasetConfig(args.get('dataset_name'))
time_range = args['time'].split(',')
variables = args['variables'].split(',')
with open_dataset(config,
variable=variables,
timestamp=int(time_range[0]),
endtime=int(time_range[1])) as dataset:
working_dir, subset_filename = dataset.nc_data.subset(args)
return send_from_directory(working_dir,
subset_filename,
as_attachment=True)
def plot_v1_0():
if request.method == 'GET':
args = request.args
else:
args = request.form
query = json.loads(args.get('query'))
with open_dataset(get_dataset_url(query.get('dataset'))) as dataset:
if 'time' in query:
query['time'] = dataset.convert_to_timestamp(query.get('time'))
else:
query['starttime'] = dataset.convert_to_timestamp(
query.get('starttime'))
query['endtime'] = dataset.convert_to_timestamp(
query.get('endtime'))
resp = routes.routes_impl.plot_impl(args, query)
m = hashlib.md5()
m.update(str(resp).encode())
if 'data' in request.args:
plotData = {
'data': str(resp),
'shape': resp.shape,
'mask': str(resp.mask)
}
plotData = json.dumps(plotData)
return Response(plotData, status=200, mimetype='application/json')
return resp
def range_query_v1_0(dataset, variable, interp, radius, neighbours, projection,
extent, depth, time):
with open_dataset(get_dataset_url(dataset)) as ds:
date = ds.convert_to_timestamp(time)
return routes.routes_impl.range_query_impl(interp, radius, neighbours,
dataset, projection, extent,
variable, depth, date)
def tile_v1_0(projection, interp, radius, neighbours, dataset, variable, time,
depth, scale, zoom, x, y):
with open_dataset(get_dataset_url(dataset)) as ds:
date = ds.convert_to_timestamp(time)
return routes.routes_impl.tile_impl(projection, interp, radius,
neighbours, dataset, variable,
date, depth, scale, zoom, x, y)
def get_scale(dataset, variable, depth, timestamp, projection, extent, interp,
radius, neighbours):
"""
Calculates and returns the range (min, max values) of a selected variable,
given the current map extents.
"""
x = np.linspace(extent[0], extent[2], 50)
y = np.linspace(extent[1], extent[3], 50)
xx, yy = np.meshgrid(x, y)
dest = Proj(init=projection)
lon, lat = dest(xx, yy, inverse=True)
variables = variable.split(",")
config = DatasetConfig(dataset)
with open_dataset(config, variable=variables, timestamp=timestamp) as ds:
d = ds.get_area(np.array([lat, lon]), depth, timestamp, variables[0],
interp, radius, neighbours)
if len(variables) > 1:
d0 = d
d1 = ds.get_area(np.array([lat, lon]), depth, timestamp,
variables[1], interp, radius, neighbours)
d = __magnitude(d0,
d1) # Use your dot-product instead of exponents
return normalize_scale(d, config.variable[",".join(variables)])
def timestamps():
"""
Returns all timestamps available for a given variable in a dataset. This is variable-dependent
because datasets can have multiple "quantums", as in surface 2D variables may be hourly, while
3D variables may be daily.
API Format: /api/v1.0/timestamps/?dataset=''&variable=''
Required Arguments:
* dataset : Dataset key - Can be found using /api/v1.0/datasets
* variable : Variable key - Can be found using /api/v1.0/variables/?dataset='...'...
Raises:
APIError: if dataset or variable is not specified in the request
Returns:
Response object containing all timestamp pairs (e.g. [raw_timestamp_integer, iso_8601_date_string]) for the given
dataset and variable.
"""
args = request.args
if "dataset" not in args:
raise APIError("Please specify a dataset via ?dataset=dataset_name")
dataset = args.get("dataset")
config = DatasetConfig(dataset)
if "variable" not in args:
raise APIError("Please specify a variable via ?variable=variable_name")
variable = args.get("variable")
# Handle possible list of URLs for staggered grid velocity field datasets
url = config.url if not isinstance(config.url, list) else config.url[0]
if url.endswith(".sqlite3"):
with SQLiteDatabase(url) as db:
if variable in config.calculated_variables:
data_vars = get_data_vars_from_equation(
config.calculated_variables[variable]['equation'],
[v.key for v in db.get_data_variables()])
vals = db.get_timestamps(data_vars[0])
else:
vals = db.get_timestamps(variable)
else:
with open_dataset(config, variable=variable) as ds:
vals = list(map(int, ds.nc_data.time_variable.values))
converted_vals = time_index_to_datetime(vals, config.time_dim_units)
result = []
for idx, date in enumerate(converted_vals):
if config.quantum == 'month' or config.variable[
variable].quantum == 'month':
date = datetime.datetime(date.year, date.month, 15)
result.append({'id': vals[idx], 'value': date})
result = sorted(result, key=lambda k: k['id'])
js = json.dumps(result, cls=DateTimeEncoder)
resp = Response(js, status=200, mimetype='application/json')
return resp
def get_data_v1_0(dataset: str, variable: str, time: str, depth: str,
location: str):
config = DatasetConfig(dataset)
with open_dataset(config) as ds:
date = ds.convert_to_timestamp(time)
#print(date)
return routes.routes_impl.get_data_impl(dataset, variable, date, depth,
location)
def load_data(self):
if not isinstance(self.depth, list):
self.depth = [self.depth]
self.depth = sorted(self.depth)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.starttime < 0:
self.starttime += len(dataset.timestamps)
if self.endtime < 0:
self.endtime += len(dataset.timestamps)
start = np.clip(self.starttime, 0, len(dataset.timestamps) - 1)
end = np.clip(self.endtime, 0, len(dataset.timestamps) - 1)
timestamp = dataset.timestamps[start:end + 1]
self.load_misc(dataset, self.variables)
point_data = []
point_depth = []
for p in self.points:
data = []
depth = []
for v in self.variables:
dd = []
jj = []
for d in self.depth:
da, dp = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
d,
start,
end,
v,
return_depth=True
)
dd.append(da)
jj.append(dp)
data.append(np.ma.array(dd))
depth.append(np.ma.array(jj))
point_data.append(np.ma.array(data))
point_depth.append(np.ma.array(depth))
point_data = np.ma.array(point_data)
point_depth = np.ma.array(point_depth)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units,
point_data
)
self.data = self.subtract_climatology(point_data, timestamp)
self.data_depth = point_depth
self.timestamp = timestamp
def load_data(self):
with open_dataset(self.dataset_config) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.timestamp = dataset.timestamps[time]
self.load_temp_sal(dataset, time)
def range_query_v1_0(dataset: str, variable: str, interp: str, radius: int,
neighbours: int, projection: str, extent: str, depth: str,
time: str):
config = DatasetConfig(dataset)
with open_dataset(config) as ds:
date = ds.convert_to_timestamp(time)
return routes.routes_impl.range_query_impl(interp, radius, neighbours,
dataset, projection, extent,
variable, depth, date)
def scale(args):
dataset_name = args.get('dataset')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
with open_dataset(get_dataset_url(dataset_name)) as dataset:
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
if anom:
cmap = colormap.colormaps['anomaly']
variable_name = gettext("%s Anomaly") % variable_name
else:
cmap = colormap.find_colormap(variable_name)
if len(variable) == 2:
if not anom:
cmap = colormap.colormaps.get('speed')
variable_name = re.sub(
r"(?i)( x | y |zonal |meridional |northward |eastward )", " ",
variable_name)
variable_name = re.sub(r" +", " ", variable_name)
fig = plt.figure(figsize=(2, 5), dpi=75)
ax = fig.add_axes([0.05, 0.05, 0.25, 0.9])
norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1])
formatter = ScalarFormatter()
formatter.set_powerlimits((-3, 4))
bar = ColorbarBase(ax, cmap=cmap, norm=norm, orientation='vertical',
format=formatter)
bar.set_label("%s (%s)" % (variable_name.title(),
utils.mathtext(variable_unit)))
buf = StringIO()
try:
plt.savefig(buf, format='png', dpi='figure', transparent=False,
bbox_inches='tight', pad_inches=0.05)
plt.close(fig)
return buf.getvalue()
finally:
buf.close()
def scale(args):
dataset_name = args.get('dataset')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
with open_dataset(get_dataset_url(dataset_name)) as dataset:
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
if anom:
cmap = colormap.colormaps['anomaly']
variable_name = gettext("%s Anomaly") % variable_name
else:
cmap = colormap.find_colormap(variable_name)
if len(variable) == 2:
if not anom:
cmap = colormap.colormaps.get('speed')
variable_name = re.sub(
r"(?i)( x | y |zonal |meridional |northward |eastward )", " ",
variable_name)
variable_name = re.sub(r" +", " ", variable_name)
fig = plt.figure(figsize=(2, 5), dpi=75)
ax = fig.add_axes([0.05, 0.05, 0.25, 0.9])
norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1])
formatter = ScalarFormatter()
formatter.set_powerlimits((-3, 4))
bar = ColorbarBase(ax, cmap=cmap, norm=norm, orientation='vertical',
format=formatter)
bar.set_label("%s (%s)" % (variable_name.title(),
utils.mathtext(variable_unit)))
buf = StringIO()
try:
plt.savefig(buf, format='png', dpi='figure', transparent=False,
bbox_inches='tight', pad_inches=0.05)
plt.close(fig)
return buf.getvalue()
finally:
buf.close()
def tile_v1_0(projection: str, interp: str, radius: int, neighbours: int,
dataset: str, variable: str, time: str, depth: str, scale: str,
zoom: int, x: int, y: int):
config = DatasetConfig(dataset)
with open_dataset(config) as ds:
date = ds.convert_to_timestamp(time)
return routes.routes_impl.tile_impl(projection, interp, radius,
neighbours, dataset, variable,
date, depth, scale, zoom, x, y)
def load_data(self):
variables = self.dataset_config.variables
temp_var_key = self.__find_var_key(variables, r'^(.*temp.*|thetao.*)$')
sal_var_key = self.__find_var_key(variables, r'^(.*sal.*|so)$')
with open_dataset(self.dataset_config, timestamp=self.time, variable=[temp_var_key, sal_var_key]) as ds:
self.iso_timestamp = ds.nc_data.timestamp_to_iso_8601(self.time)
self.__load_temp_sal(ds, self.time, temp_var_key, sal_var_key)
def load_data(self):
if not isinstance(self.depth, list):
self.depth = [self.depth]
self.depth = sorted(self.depth)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.starttime < 0:
self.starttime += len(dataset.timestamps)
if self.endtime < 0:
self.endtime += len(dataset.timestamps)
start = np.clip(self.starttime, 0, len(dataset.timestamps) - 1)
end = np.clip(self.endtime, 0, len(dataset.timestamps) - 1)
timestamp = dataset.timestamps[start:end + 1]
self.load_misc(dataset, self.variables)
point_data = []
point_depth = []
for p in self.points:
data = []
depth = []
for v in self.variables:
dd = []
jj = []
for d in self.depth:
da, dp = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
d,
start,
end,
v,
return_depth=True)
dd.append(da)
jj.append(dp)
data.append(np.ma.array(dd))
depth.append(np.ma.array(jj))
point_data.append(np.ma.array(data))
point_depth.append(np.ma.array(depth))
point_data = np.ma.array(point_data)
point_depth = np.ma.array(point_depth)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units, point_data)
self.data = self.subtract_other(point_data)
self.data_depth = point_depth
self.timestamp = timestamp
def test_open_dataset_returns_nemo_object(self, patch_calculated_data,
patch_get_dataset_config):
patch_get_dataset_config.return_value = {
"giops": {
"url": "tests/testdata/nemo_test.nc",
"variables": {}
}
}
config = DatasetConfig("giops")
with open_dataset(config) as ds:
self.assertTrue(isinstance(ds, Nemo))
def test_open_dataset_meta_only_returns_fvcom_object(
self, patch_calculated_data, patch_get_dataset_config):
patch_get_dataset_config.return_value = {
"giops": {
"url": "tests/testdata/fvcom_test.nc",
"variables": {}
}
}
config = DatasetConfig("giops")
with open_dataset(config, meta_only=True) as ds:
self.assertTrue(isinstance(ds, Fvcom))
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.timestamp = dataset.timestamps[time]
self.load_temp_sal(dataset, time)
self.variable_units[0], self.temperature = \
super(point.PointPlotter, self).kelvin_to_celsius(
self.variable_units[0], self.temperature
)
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as d:
if self.time < 0:
self.time += len(d.timestamps)
time = np.clip(self.time, 0, len(d.timestamps) - 1)
timestamp = d.timestamps[time]
self.load_misc(d, self.variables)
point_data, point_depths = self.get_data(d, self.variables, time)
point_data = self.apply_scale_factors(point_data)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units,
point_data
)
self.data = self.subtract_climatology(point_data, timestamp)
self.depths = point_depths
self.timestamp = timestamp
def load_data(self):
if self.projection == 'EPSG:32661':
blat = min(self.bounds[0], self.bounds[2])
blat = 5 * np.floor(blat / 5)
self.basemap = basemap.load_map('npstere', (blat, 0), None, None)
elif self.projection == 'EPSG:3031':
blat = max(self.bounds[0], self.bounds[2])
blat = 5 * np.ceil(blat / 5)
self.basemap = basemap.load_map('spstere', (blat, 180), None, None)
else:
distance = VincentyDistance()
height = distance.measure(
(self.bounds[0], self.centroid[1]),
(self.bounds[2], self.centroid[1])
) * 1000 * 1.25
width = distance.measure(
(self.centroid[0], self.bounds[1]),
(self.centroid[0], self.bounds[3])
) * 1000 * 1.25
self.basemap = basemap.load_map(
'lcc', self.centroid, height, width
)
if self.basemap.aspect < 1:
gridx = 500
gridy = int(500 * self.basemap.aspect)
else:
gridy = 500
gridx = int(500 / self.basemap.aspect)
self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.variable_unit = self.get_variable_units(
dataset, self.variables
)[0]
self.variable_name = self.get_variable_names(
dataset,
self.variables
)[0]
scale_factor = self.get_variable_scale_factors(
dataset, self.variables
)[0]
if self.cmap is None:
if len(self.variables) == 1:
self.cmap = colormap.find_colormap(self.variable_name)
else:
self.cmap = colormap.colormaps.get('speed')
if len(self.variables) == 2:
self.variable_name = self.vector_name(self.variable_name)
if self.depth == 'bottom':
depth_value = 'Bottom'
else:
self.depth = np.clip(
int(self.depth), 0, len(dataset.depths) - 1)
depth_value = dataset.depths[self.depth]
data = []
allvars = []
for v in self.variables:
var = dataset.variables[v]
allvars.append(v)
if self.filetype in ['csv', 'odv', 'txt']:
d, depth_value = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
v,
return_depth=True
)
else:
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
v
)
d = np.multiply(d, scale_factor)
self.variable_unit, d = self.kelvin_to_celsius(
self.variable_unit, d)
data.append(d)
if self.filetype not in ['csv', 'odv', 'txt']:
if len(var.dimensions) == 3:
self.depth_label = ""
elif self.depth == 'bottom':
self.depth_label = " at Bottom"
else:
self.depth_label = " at " + \
str(int(np.round(depth_value))) + " m"
if len(data) == 2:
data[0] = np.sqrt(data[0] ** 2 + data[1] ** 2)
self.data = data[0]
quiver_data = []
if self.quiver is not None and \
self.quiver['variable'] != '' and \
self.quiver['variable'] != 'none':
for v in self.quiver['variable'].split(','):
allvars.append(v)
var = dataset.variables[v]
quiver_unit = get_variable_unit(self.dataset_name, var)
quiver_name = get_variable_name(self.dataset_name, var)
quiver_lon, quiver_lat = self.basemap.makegrid(50, 50)
d = dataset.get_area(
np.array([quiver_lat, quiver_lon]),
self.depth,
self.time,
v
)
quiver_data.append(d)
self.quiver_name = self.vector_name(quiver_name)
self.quiver_longitude = quiver_lon
self.quiver_latitude = quiver_lat
self.quiver_unit = quiver_unit
self.quiver_data = quiver_data
if all(map(lambda v: len(dataset.variables[v].dimensions) == 3,
allvars)):
self.depth = 0
contour_data = []
if self.contour is not None and \
self.contour['variable'] != '' and \
self.contour['variable'] != 'none':
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
self.contour['variable']
)
contour_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_name = get_variable_name(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_unit, d = self.kelvin_to_celsius(contour_unit, d)
d = np.multiply(d, contour_factor)
contour_data.append(d)
self.contour_unit = contour_unit
self.contour_name = contour_name
self.contour_data = contour_data
self.timestamp = dataset.timestamps[self.time]
if self.variables != self.variables_anom:
self.variable_name += " Anomaly"
with open_dataset(
get_dataset_climatology(self.dataset_name)
) as dataset:
data = []
for v in self.variables:
var = dataset.variables[v]
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.timestamp.month - 1,
v
)
data.append(d)
if len(data) == 2:
data = np.sqrt(data[0] ** 2 + data[1] ** 2)
else:
data = data[0]
u, data = self.kelvin_to_celsius(
dataset.variables[self.variables[0]].unit,
data)
self.data -= data
# Load bathymetry data
self.bathymetry = overlays.bathymetry(
self.basemap,
self.latitude,
self.longitude,
blur=2
)
if self.depth != 'bottom' and self.depth != 0:
if len(quiver_data) > 0:
quiver_bathymetry = overlays.bathymetry(
self.basemap, quiver_lat, quiver_lon)
self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked
for d in self.quiver_data:
d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked
for d in self.contour_data:
d[np.where(self.bathymetry < depth_value)] = np.ma.masked
else:
mask = maskoceans(self.longitude, self.latitude, self.data).mask
self.data[~mask] = np.ma.masked
for d in self.quiver_data:
mask = maskoceans(
self.quiver_longitude, self.quiver_latitude, d).mask
d[~mask] = np.ma.masked
for d in contour_data:
mask = maskoceans(self.longitude, self.latitude, d).mask
d[~mask] = np.ma.masked
if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']:
area_polys = []
for a in self.area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Poly(r, inners))
area_polys.append(MultiPolygon(polygons))
points = [Point(p) for p in zip(self.latitude.ravel(),
self.longitude.ravel())]
indicies = []
for a in area_polys:
indicies.append(np.where(
map(
lambda p, poly=a: poly.contains(p),
points
)
)[0])
indicies = np.unique(np.array(indicies).ravel())
newmask = np.ones(self.data.shape, dtype=bool)
newmask[np.unravel_index(indicies, newmask.shape)] = False
self.data.mask |= newmask
self.depth_value = depth_value
def plot(projection, x, y, z, args):
lat, lon = get_latlon_coords(projection, x, y, z)
if len(lat.shape) == 1:
lat, lon = np.meshgrid(lat, lon)
dataset_name = args.get('dataset')
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
depth = args.get('depth')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
data = []
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if args.get('time') is None or (type(args.get('time')) == str and
len(args.get('time')) == 0):
time = -1
else:
time = int(args.get('time'))
t_len = len(dataset.timestamps)
while time >= t_len:
time -= t_len
while time < 0:
time += len(dataset.timestamps)
timestamp = dataset.timestamps[time]
for v in variable:
data.append(dataset.get_area(
np.array([lat, lon]),
depth,
time,
v
))
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
scale_factor = get_variable_scale_factor(
dataset_name,
dataset.variables[variable[0]]
)
if anom:
cmap = colormap.colormaps['anomaly']
else:
cmap = colormap.find_colormap(variable_name)
if depth != 'bottom':
depthm = dataset.depths[depth]
else:
depthm = 0
if scale_factor != 1.0:
for idx, val in enumerate(data):
data[idx] = np.multiply(val, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
for idx, val in enumerate(data):
data[idx] = np.add(val, -273.15)
if len(data) == 1:
data = data[0]
if len(data) == 2:
data = np.sqrt(data[0] ** 2 + data[1] ** 2)
if not anom:
cmap = colormap.colormaps.get('speed')
if anom:
with open_dataset(get_dataset_climatology(dataset_name)) as dataset:
a = dataset.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
v
)
data = data - a
f, fname = tempfile.mkstemp()
os.close(f)
data = data.transpose()
xpx = x * 256
ypx = y * 256
with Dataset(ETOPO_FILE % (projection, z), 'r') as dataset:
bathymetry = dataset["z"][ypx:(ypx + 256), xpx:(xpx + 256)]
bathymetry = gaussian_filter(bathymetry, 0.5)
data[np.where(bathymetry > -depthm)] = np.ma.masked
sm = matplotlib.cm.ScalarMappable(
matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1]), cmap=cmap)
img = sm.to_rgba(np.squeeze(data))
im = Image.fromarray((img * 255.0).astype(np.uint8))
im.save(fname, format='png', optimize=True)
with open(fname, 'r') as f:
buf = f.read()
os.remove(fname)
return buf
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
for idx, v in enumerate(self.variables):
var = dataset.variables[v]
if not (set(var.dimensions) & set(dataset.depth_dimensions)):
for potential in dataset.variables:
if potential in self.variables:
continue
pot = dataset.variables[potential]
if (set(pot.dimensions) &
set(dataset.depth_dimensions)):
if len(pot.shape) > 3:
self.variables[idx] = potential
self.variables_anom[idx] = potential
value = parallel = perpendicular = None
variable_names = self.get_variable_names(dataset, self.variables)
variable_units = self.get_variable_units(dataset, self.variables)
scale_factors = self.get_variable_scale_factors(dataset,
self.variables)
if len(self.variables) > 1:
v = []
for name in self.variables:
v.append(dataset.variables[name])
distances, times, lat, lon, bearings = geo.path_to_points(
self.points, 100
)
transect_pts, distance, x, dep = dataset.get_path_profile(
self.points, time, self.variables[0], 100)
transect_pts, distance, y, dep = dataset.get_path_profile(
self.points, time, self.variables[1], 100)
x = np.multiply(x, scale_factors[0])
y = np.multiply(y, scale_factors[1])
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(y, x) - r
mag = np.sqrt(x ** 2 + y ** 2)
parallel = mag * np.cos(theta)
perpendicular = mag * np.sin(theta)
else:
transect_pts, distance, value, dep = dataset.get_path_profile(
self.points, time, self.variables[0])
value = np.multiply(value, scale_factors[0])
variable_units[0], value = self.kelvin_to_celsius(
variable_units[0],
value
)
if len(self.variables) == 2:
variable_names[0] = self.vector_name(variable_names[0])
if self.cmap is None:
self.cmap = colormap.find_colormap(variable_names[0])
self.timestamp = dataset.timestamps[int(time)]
self.depth = dep
self.depth_unit = "m"
self.transect_data = {
"points": transect_pts,
"distance": distance,
"data": value,
"name": variable_names[0],
"unit": variable_units[0],
"parallel": parallel,
"perpendicular": perpendicular,
}
if self.surface is not None:
surface_pts, surface_dist, t, surface_value = \
dataset.get_path(
self.points,
0,
time,
self.surface,
)
surface_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.surface]
)
surface_name = get_variable_name(
self.dataset_name,
dataset.variables[self.surface]
)
surface_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.surface]
)
surface_value = np.multiply(surface_value, surface_factor)
surface_unit, surface_value = self.kelvin_to_celsius(
surface_unit,
surface_value
)
self.surface_data = {
"points": surface_pts,
"distance": surface_dist,
"data": surface_value,
"name": surface_name,
"unit": surface_unit
}
if self.variables != self.variables_anom:
with open_dataset(
get_dataset_climatology(self.dataset_name)
) as dataset:
if self.variables[0] in dataset.variables:
if len(self.variables) == 1:
climate_points, climate_distance, climate_data = \
dataset.get_path_profile(self.points,
self.timestamp.month - 1,
self.variables[0])
u, climate_data = self.kelvin_to_celsius(
dataset.variables[self.variables[0]].unit,
climate_data
)
self.transect_data['data'] -= - climate_data
else:
climate_pts, climate_distance, climate_x, cdep = \
dataset.get_path_profile(
self.points,
self.timestamp.month - 1,
self.variables[0],
100
)
climate_pts, climate_distance, climate_y, cdep = \
dataset.get_path_profile(
self.points,
self.timestamp.month - 1,
self.variables[0],
100
)
climate_distances, ctimes, clat, clon, bearings = \
geo.path_to_points(self.points, 100)
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(y, x) - r
mag = np.sqrt(x ** 2 + y ** 2)
climate_parallel = mag * np.cos(theta)
climate_perpendicular = mag * np.sin(theta)
self.transect_data['parallel'] -= climate_parallel
self.transect_data[
'perpendicular'] -= climate_perpendicular
# Bathymetry
with Dataset(app.config['BATHYMETRY_FILE'], 'r') as dataset:
bath_x, bath_y = bathymetry(
dataset.variables['y'],
dataset.variables['x'],
dataset.variables['z'],
self.points)
self.bathymetry = {
'x': bath_x,
'y': bath_y
}
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
self.load_misc(dataset, self.variables)
self.fix_startend_times(dataset)
self.variable_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.variables[0]]
)
self.variable_name = get_variable_name(
self.dataset_name,
dataset.variables[self.variables[0]]
)
var = self.variables[0]
if self.depth != 'all' and self.depth != 'bottom' and \
(set(dataset.variables[var].dimensions) &
set(dataset.depth_dimensions)):
self.depth_label = " at %d m" % (
np.round(dataset.depths[self.depth])
)
elif self.depth == 'bottom':
self.depth_label = ' at Bottom'
else:
self.depth_label = ''
if not (set(dataset.variables[var].dimensions) &
set(dataset.depth_dimensions)):
self.depth = 0
times = None
point_data = []
for p in self.points:
data = []
for v in self.variables:
if self.depth == 'all':
d, dep = dataset.get_timeseries_profile(
float(p[0]),
float(p[1]),
self.starttime,
self.endtime,
v
)
else:
d, dep = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
self.depth,
self.starttime,
self.endtime,
v,
return_depth=True
)
data.append(d)
point_data.append(np.ma.array(data))
point_data = np.ma.array(point_data)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
times = dataset.timestamps[self.starttime:self.endtime + 1]
if self.query.get('dataset_quantum') == 'month':
times = [datetime.date(x.year, x.month, 1) for x in times]
# depths = dataset.depths
depths = dep
# TODO: pint
if self.variable_unit.startswith("Kelvin"):
self.variable_unit = "Celsius"
for idx, v in enumerate(self.variables):
point_data[:, idx, :] = point_data[:, idx, :] - 273.15
if point_data.shape[1] == 2:
point_data = np.ma.expand_dims(
np.sqrt(
point_data[:, 0, :] ** 2 + point_data[:, 1, :] ** 2
), 1
)
self.times = times
self.data = point_data
self.depths = depths
self.depth_unit = "m"
def load_data(self):
if isinstance(self.observation[0], numbers.Number):
self.observation_variable_names = []
self.observation_variable_units = []
with Dataset(app.config["OBSERVATION_AGG_URL"], 'r') as ds:
t = netcdftime.utime(ds['time'].units)
for idx, o in enumerate(self.observation):
observation = {}
ts = t.num2date(ds['time'][o]).replace(tzinfo=pytz.UTC)
observation['time'] = ts.isoformat()
observation['longitude'] = ds['lon'][o]
observation['latitude'] = ds['lat'][o]
observation['depth'] = ds['z'][:]
observation['depthunit'] = ds['z'].units
observation['datatypes'] = []
data = []
for v in sorted(ds.variables):
if v in ['z', 'lat', 'lon', 'profile', 'time']:
continue
var = ds[v]
if var.datatype == '|S1':
continue
observation['datatypes'].append("%s [%s]" % (
var.long_name,
var.units
))
data.append(var[o, :])
if idx == 0:
self.observation_variable_names.append(
var.long_name)
self.observation_variable_units.append(var.units)
observation['data'] = np.ma.array(data).transpose()
self.observation[idx] = observation
self.points = map(lambda o: [o['latitude'], o['longitude']],
self.observation)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
ts = dataset.timestamps
observation_times = []
timestamps = []
for o in self.observation:
observation_time = dateutil.parser.parse(o['time'])
observation_times.append(observation_time)
deltas = [
(x.replace(tzinfo=pytz.UTC) -
observation_time).total_seconds()
for x in ts]
time = np.abs(deltas).argmin()
timestamp = ts[time]
timestamps.append(timestamp)
self.load_misc(dataset, self.variables)
point_data, self.depths = self.get_data(
dataset, self.variables, time)
point_data = np.ma.array(point_data)
point_data = self.apply_scale_factors(point_data)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units,
point_data
)
self.data = self.subtract_climatology(point_data, timestamp)
self.observation_time = observation_time
self.observation_times = observation_times
self.timestamps = timestamps
self.timestamp = timestamp
def load_data(self):
ds_url = app.config['DRIFTER_URL']
data_names = []
data_units = []
with Dataset(ds_url % self.drifter, 'r') as ds:
self.name = ds.buoyid
self.imei = str(chartostring(ds['imei'][0]))
self.wmo = str(chartostring(ds['wmo'][0]))
t = netcdftime.utime(ds['data_date'].units)
d = []
for v in self.buoyvariables:
d.append(ds[v][:])
if "long_name" in ds[v].ncattrs():
data_names.append(ds[v].long_name)
else:
data_names.append(v)
if "units" in ds[v].ncattrs():
data_units.append(ds[v].units)
else:
data_units.append(None)
self.data = d
self.times = t.num2date(ds['data_date'][:])
self.points = np.array([
ds['latitude'][:],
ds['longitude'][:],
]).transpose()
data_names = data_names[:len(self.buoyvariables)]
data_units = data_units[:len(self.buoyvariables)]
for i, t in enumerate(self.times):
if t.tzinfo is None:
self.times[i] = t.replace(tzinfo=pytz.UTC)
self.data_names = data_names
self.data_units = data_units
if self.starttime is not None:
d = dateutil.parser.parse(self.starttime)
self.start = np.where(self.times >= d)[0].min()
else:
self.start = 0
if self.endtime is not None:
d = dateutil.parser.parse(self.endtime)
self.end = np.where(self.times <= d)[0].max() + 1
else:
self.end = len(self.times) - 1
if self.start < 0:
self.start += len(self.times)
self.start = np.clip(self.start, 0, len(self.times) - 1)
if self.end < 0:
self.end += len(self.times)
self.end = np.clip(self.end, 0, len(self.times) - 1)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
depth = int(self.depth)
try:
model_start = np.where(
dataset.timestamps <= self.times[self.start]
)[0][-1]
except IndexError:
model_start = 0
model_start -= 1
model_start = np.clip(model_start, 0, len(dataset.timestamps) - 1)
try:
model_end = np.where(
dataset.timestamps >= self.times[self.end]
)[0][0]
except IndexError:
model_end = len(dataset.timestamps) - 1
model_end += 1
model_end = np.clip(
model_end,
model_start,
len(dataset.timestamps) - 1
)
model_times = map(
lambda t: time.mktime(t.timetuple()),
dataset.timestamps[model_start:model_end + 1]
)
output_times = map(
lambda t: time.mktime(t.timetuple()),
self.times[self.start:self.end + 1]
)
d = []
for v in self.variables:
pts, dist, mt, md = dataset.get_path(
self.points[self.start:self.end + 1],
depth,
range(model_start, model_end + 1),
v,
times=output_times
)
f = interp1d(
model_times,
md,
assume_sorted=True,
bounds_error=False,
)
d.append(np.diag(f(mt)))
model_data = np.ma.array(d)
variable_names = []
variable_units = []
scale_factors = []
for v in self.variables:
variable_units.append(get_variable_unit(self.dataset_name,
dataset.variables[v]))
variable_names.append(get_variable_name(self.dataset_name,
dataset.variables[v]))
scale_factors.append(
get_variable_scale_factor(self.dataset_name,
dataset.variables[v])
)
for idx, sf in enumerate(scale_factors):
model_data[idx, :] = np.multiply(model_data[idx, :], sf)
for idx, u in enumerate(variable_units):
variable_units[idx], model_data[idx, :] = \
self.kelvin_to_celsius(u, model_data[idx, :])
self.model_data = model_data
self.model_times = map(datetime.datetime.utcfromtimestamp, mt)
self.variable_names = variable_names
self.variable_units = variable_units
def stats(dataset_name, query):
variables = query.get('variable')
if isinstance(variables, str) or isinstance(variables, unicode):
variables = variables.split(',')
variables = [re.sub('_anom$', '', v) for v in variables]
area = query.get('area')
names = None
data = None
names = []
all_rings = []
for idx, a in enumerate(area):
if isinstance(a, str) or isinstance(a, unicode):
a = a.encode("utf-8")
sp = a.split('/', 1)
if data is None:
data = list_areas(sp[0])
b = [x for x in data if x.get('key') == a]
a = b[0]
area[idx] = a
rings = [LinearRing(p) for p in a['polygons']]
if len(rings) > 1:
u = cascaded_union(rings)
else:
u = rings[0]
all_rings.append(u.envelope)
if a.get('name'):
names.append(a.get('name'))
names = sorted(names)
if len(all_rings) > 1:
combined = cascaded_union(all_rings)
else:
combined = all_rings[0]
combined = combined.envelope
bounds = combined.bounds
area_polys = []
output = []
for a in area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Polygon(r, inners))
area_polys.append(MultiPolygon(polygons))
output.append({
'name': a.get('name'),
'variables': [],
})
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if query.get('time') is None or (type(query.get('time')) == str and
len(query.get('time')) == 0):
time = -1
else:
time = int(query.get('time'))
if time < 0:
time += len(dataset.timestamps)
time = np.clip(time, 0, len(dataset.timestamps) - 1)
depth = 0
depthm = 0
if query.get('depth'):
if query.get('depth') == 'bottom':
depth = 'bottom'
depthm = 'Bottom'
if len(query.get('depth')) > 0 and \
query.get('depth') != 'bottom':
depth = int(query.get('depth'))
depth = np.clip(depth, 0, len(dataset.depths) - 1)
depthm = dataset.depths[depth]
lat, lon = np.meshgrid(
np.linspace(bounds[0], bounds[2], 50),
np.linspace(bounds[1], bounds[3], 50)
)
output_fmtstr = "%6.5g"
for v_idx, v in enumerate(variables):
var = dataset.variables[v]
variable_name = get_variable_name(dataset_name, var)
variable_unit = get_variable_unit(dataset_name, var)
scale_factor = get_variable_scale_factor(dataset_name, var)
lat, lon, d = dataset.get_raw_point(
lat.ravel(),
lon.ravel(),
depth,
time,
v
)
if scale_factor != 1.0:
d = np.multiply(d, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = d - 273.15
lon[np.where(lon > 180)] -= 360
if len(var.dimensions) == 3:
variable_depth = ""
elif depth == 'bottom':
variable_depth = "(@ Bottom)"
else:
variable_depth = "(@%d m)" % np.round(depthm)
points = [Point(p) for p in zip(lat.ravel(), lon.ravel())]
for i, a in enumerate(area):
indices = np.where(
map(
lambda p, poly=area_polys[i]: poly.contains(p),
points
)
)
selection = np.ma.array(d.ravel()[indices])
if len(selection) > 0 and not selection.mask.all():
output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': output_fmtstr % (
np.ma.amin(selection).astype(float)
),
'max': output_fmtstr % (
np.ma.amax(selection).astype(float)
),
'mean': output_fmtstr % (
np.ma.mean(selection).astype(float)
),
'median': output_fmtstr % (
np.ma.median(selection).astype(float)
),
'stddev': output_fmtstr % (
np.ma.std(selection).astype(float)
),
'num': "%d" % selection.count(),
})
else:
output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': gettext("No Data"),
'max': gettext("No Data"),
'mean': gettext("No Data"),
'median': gettext("No Data"),
'stddev': gettext("No Data"),
'num': "0",
})
return json.dumps(sorted(output, key=itemgetter('name')))
