def test_get_variable_misc(self, m):
m.return_value = {
"var": {
"name": "the_name",
"unit": "My Unit",
"scale": [0, 10],
},
"var2": {
"hide": True,
}
}
self.assertEqual(
util.get_variable_name("dataset", mock.Mock(key="var")),
"the_name")
variable_mock = mock.Mock()
variable_mock.configure_mock(key="none", name="var_name")
self.assertEqual(util.get_variable_name("dataset", variable_mock),
"var_name")
variable_mock.configure_mock(key="nameless", name=None)
self.assertEqual(util.get_variable_name("dataset", variable_mock),
"Nameless")
self.assertEqual(
util.get_variable_unit("dataset", mock.Mock(key="var")), "My Unit")
variable_mock.configure_mock(key="none", unit="var_unit")
self.assertEqual(util.get_variable_unit("dataset", variable_mock),
"var_unit")
self.assertEqual(
util.get_variable_unit("dataset", mock.Mock(key="varx",
unit=None)), "Unknown")
self.assertEqual(
util.get_variable_scale("dataset", mock.Mock(key="var")), [0, 10])
variable_mock.configure_mock(key="none", valid_min=5, valid_max=50)
self.assertEqual(util.get_variable_scale("dataset", variable_mock),
[5, 50])
self.assertEqual(
util.get_variable_scale(
"dataset",
mock.Mock(key="varx",
scale=None,
valid_min=None,
valid_max=None)), [0, 100])
self.assertFalse(
util.is_variable_hidden("dataset", mock.Mock(key="var")))
self.assertTrue(
util.is_variable_hidden("dataset", mock.Mock(key="var2")))
self.assertFalse(
util.is_variable_hidden("dataset", mock.Mock(key="var3")))
def get_scale(dataset, variable, depth, time, projection, extent):
x = np.linspace(extent[0], extent[2], 50)
y = np.linspace(extent[1], extent[3], 50)
xx, yy = np.meshgrid(x, y)
dest = Proj(init=projection)
lon, lat = dest(xx, yy, inverse=True)
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
d = ds.get_area(
np.array([lat, lon]),
depth,
time,
variables[0]
)
if len(variables) > 1:
d0 = d
d1 = ds.get_area(
np.array([lat, lon]),
depth,
time,
variables[1]
)
d = np.sqrt(d0 ** 2 + d1 ** 2)
variable_unit = get_variable_unit(dataset,
ds.variables[variables[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset), 'r') as ds:
c = ds.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
variables[0]
)
if len(variables) > 1:
c0 = c
c1 = ds.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
variables[1]
)
c = np.sqrt(c0 ** 2 + c1 ** 2)
d = d - c
m = max(abs(d.min()), abs(d.max()))
return -m, m
return d.min(), d.max()
def get_variable_units(self, dataset, variables):
units = []
for idx, v in enumerate(variables):
units.append(get_variable_unit(self.dataset_name,
dataset.variables[v]))
return units
def get_variable_units(self, dataset, variables):
units = []
for idx, v in enumerate(variables):
units.append(get_variable_unit(self.dataset_name,
dataset.variables[v]))
return units
def scale(args):
dataset_name = args.get('dataset')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
with open_dataset(get_dataset_url(dataset_name)) as dataset:
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
if anom:
cmap = colormap.colormaps['anomaly']
variable_name = gettext("%s Anomaly") % variable_name
else:
cmap = colormap.find_colormap(variable_name)
if len(variable) == 2:
if not anom:
cmap = colormap.colormaps.get('speed')
variable_name = re.sub(
r"(?i)( x | y |zonal |meridional |northward |eastward )", " ",
variable_name)
variable_name = re.sub(r" +", " ", variable_name)
fig = plt.figure(figsize=(2, 5), dpi=75)
ax = fig.add_axes([0.05, 0.05, 0.25, 0.9])
norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1])
formatter = ScalarFormatter()
formatter.set_powerlimits((-3, 4))
bar = ColorbarBase(ax, cmap=cmap, norm=norm, orientation='vertical',
format=formatter)
bar.set_label("%s (%s)" % (variable_name.title(),
utils.mathtext(variable_unit)))
buf = StringIO()
try:
plt.savefig(buf, format='png', dpi='figure', transparent=False,
bbox_inches='tight', pad_inches=0.05)
plt.close(fig)
return buf.getvalue()
finally:
buf.close()
def scale(args):
dataset_name = args.get('dataset')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
with open_dataset(get_dataset_url(dataset_name)) as dataset:
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
if anom:
cmap = colormap.colormaps['anomaly']
variable_name = gettext("%s Anomaly") % variable_name
else:
cmap = colormap.find_colormap(variable_name)
if len(variable) == 2:
if not anom:
cmap = colormap.colormaps.get('speed')
variable_name = re.sub(
r"(?i)( x | y |zonal |meridional |northward |eastward )", " ",
variable_name)
variable_name = re.sub(r" +", " ", variable_name)
fig = plt.figure(figsize=(2, 5), dpi=75)
ax = fig.add_axes([0.05, 0.05, 0.25, 0.9])
norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1])
formatter = ScalarFormatter()
formatter.set_powerlimits((-3, 4))
bar = ColorbarBase(ax, cmap=cmap, norm=norm, orientation='vertical',
format=formatter)
bar.set_label("%s (%s)" % (variable_name.title(),
utils.mathtext(variable_unit)))
buf = StringIO()
try:
plt.savefig(buf, format='png', dpi='figure', transparent=False,
bbox_inches='tight', pad_inches=0.05)
plt.close(fig)
return buf.getvalue()
finally:
buf.close()
def get_point_data(dataset, variable, time, depth, location):
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
data = []
names = []
units = []
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
for v in variables:
d = ds.get_point(
location[0],
location[1],
depth,
time,
v
)
variable_name = get_variable_name(dataset, ds.variables[v])
variable_unit = get_variable_unit(dataset, ds.variables[v])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
data.append(d)
names.append(variable_name)
units.append(variable_unit)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset)) as ds:
for idx, v in enumerate(variables):
d = ds.get_point(
location[0],
location[1],
depth,
timestamp.month,
v
)
data[idx] = data[idx] - d
names[idx] = names[idx] + " Anomaly"
result = {
'value': map(lambda f: '%s' % float('%.4g' % f), data),
'location': map(lambda f: round(f, 4), location),
'name': names,
'units': units,
}
return result
def get_scale(dataset, variable, depth, time, projection, extent, interp,
radius, neighbours):
x = np.linspace(extent[0], extent[2], 50)
y = np.linspace(extent[1], extent[3], 50)
xx, yy = np.meshgrid(x, y)
dest = Proj(init=projection)
lon, lat = dest(xx, yy, inverse=True)
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
d = ds.get_area(np.array([lat, lon]), depth, time, variables[0],
interp, radius, neighbours)
if len(variables) > 1:
d0 = d
d1 = ds.get_area(np.array([lat, lon]), depth, time, variables[1],
interp, radius, neighbours)
d = np.sqrt(d0**2 + d1**2)
variable_unit = get_variable_unit(dataset, ds.variables[variables[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset), 'r') as ds:
c = ds.get_area(np.array([lat, lon]), depth, timestamp.month - 1,
variables[0], interp, radius, neighbours)
if len(variables) > 1:
c0 = c
c1 = ds.get_area(np.array([lat,
lon]), depth, timestamp.month - 1,
variables[1], interp, radius, neighbours)
c = np.sqrt(c0**2 + c1**2)
d = d - c
m = max(abs(d.min()), abs(d.max()))
return -m, m
return d.min(), d.max()
def get_values(self, area_info, dataset_name, variables):
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if self.time is None or (type(self.time) == str and
len(self.time) == 0):
time = -1
else:
time = int(self.time)
if time < 0:
time += len(dataset.timestamps)
time = np.clip(time, 0, len(dataset.timestamps) - 1)
depth = 0
depthm = 0
if self.depth:
if self.depth == 'bottom':
depth = 'bottom'
depthm = 'Bottom'
if len(self.depth) > 0 and \
self.depth != 'bottom':
depth = int(self.depth)
depth = np.clip(depth, 0, len(dataset.depths) - 1)
depthm = dataset.depths[depth]
lat, lon = np.meshgrid(
np.linspace(area_info.bounds[0], area_info.bounds[2], 50),
area_info.spaced_points
)
output_fmtstr = "%6.5g"
for v_idx, v in enumerate(variables):
var = dataset.variables[v]
variable_name = get_variable_name(dataset_name, var)
variable_unit = get_variable_unit(dataset_name, var)
scale_factor = get_variable_scale_factor(dataset_name, var)
lat, lon, d = dataset.get_raw_point(
lat.ravel(),
lon.ravel(),
depth,
time,
v
)
if scale_factor != 1.0:
d = np.multiply(d, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = d - 273.15
lon[np.where(lon > 180)] -= 360
if len(var.dimensions) == 3:
variable_depth = ""
elif depth == 'bottom':
variable_depth = "(@ Bottom)"
else:
variable_depth = "(@%d m)" % np.round(depthm)
points = [Point(p) for p in zip(lat.ravel(), lon.ravel())]
for i, a in enumerate(area_info.area_query):
indices = np.where(
map(
lambda p, poly=area_info.area_polys[i]: poly.contains(p),
points
)
)
selection = np.ma.array(d.ravel()[indices])
if len(selection) > 0 and not selection.mask.all():
area_info.output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': output_fmtstr % (
np.ma.amin(selection).astype(float)
),
'max': output_fmtstr % (
np.ma.amax(selection).astype(float)
),
'mean': output_fmtstr % (
np.ma.mean(selection).astype(float)
),
'median': output_fmtstr % (
np.ma.median(selection).astype(float)
),
'stddev': output_fmtstr % (
np.ma.std(selection).astype(float)
),
'num': "%d" % selection.count(),
})
else:
area_info.output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': gettext("No Data"),
'max': gettext("No Data"),
'mean': gettext("No Data"),
'median': gettext("No Data"),
'stddev': gettext("No Data"),
'num': "0",
})
ClientError(gettext("there are no datapoints in the area you selected. \
you may have selected a area on land or you may \
have an ara that is smallenough to fit between \
the datapoints try selection a different area or \
a larger area"))
area_info.stats = area_info.output
return
raise ServerError(gettext("An Error has occurred. When opening the dataset. \
Please try again or try a different dataset. \
If you would like to report this error please \
contact [email protected]"))
def stats(dataset_name, query):
variables = query.get('variable')
if isinstance(variables, str) or isinstance(variables, unicode):
variables = variables.split(',')
variables = [re.sub('_anom$', '', v) for v in variables]
area = query.get('area')
names = None
data = None
names = []
all_rings = []
for idx, a in enumerate(area):
if isinstance(a, str) or isinstance(a, unicode):
a = a.encode("utf-8")
sp = a.split('/', 1)
if data is None:
data = list_areas(sp[0])
b = [x for x in data if x.get('key') == a]
a = b[0]
area[idx] = a
rings = [LinearRing(p) for p in a['polygons']]
if len(rings) > 1:
u = cascaded_union(rings)
else:
u = rings[0]
all_rings.append(u.envelope)
if a.get('name'):
names.append(a.get('name'))
names = sorted(names)
if len(all_rings) > 1:
combined = cascaded_union(all_rings)
else:
combined = all_rings[0]
combined = combined.envelope
bounds = combined.bounds
area_polys = []
output = []
for a in area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Polygon(r, inners))
area_polys.append(MultiPolygon(polygons))
output.append({
'name': a.get('name'),
'variables': [],
})
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if query.get('time') is None or (type(query.get('time')) == str and
len(query.get('time')) == 0):
time = -1
else:
time = int(query.get('time'))
if time < 0:
time += len(dataset.timestamps)
time = np.clip(time, 0, len(dataset.timestamps) - 1)
depth = 0
depthm = 0
if query.get('depth'):
if query.get('depth') == 'bottom':
depth = 'bottom'
depthm = 'Bottom'
if len(query.get('depth')) > 0 and \
query.get('depth') != 'bottom':
depth = int(query.get('depth'))
depth = np.clip(depth, 0, len(dataset.depths) - 1)
depthm = dataset.depths[depth]
lat, lon = np.meshgrid(
np.linspace(bounds[0], bounds[2], 50),
np.linspace(bounds[1], bounds[3], 50)
)
output_fmtstr = "%6.5g"
for v_idx, v in enumerate(variables):
var = dataset.variables[v]
variable_name = get_variable_name(dataset_name, var)
variable_unit = get_variable_unit(dataset_name, var)
scale_factor = get_variable_scale_factor(dataset_name, var)
lat, lon, d = dataset.get_raw_point(
lat.ravel(),
lon.ravel(),
depth,
time,
v
)
if scale_factor != 1.0:
d = np.multiply(d, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = d - 273.15
lon[np.where(lon > 180)] -= 360
if len(var.dimensions) == 3:
variable_depth = ""
elif depth == 'bottom':
variable_depth = "(@ Bottom)"
else:
variable_depth = "(@%d m)" % np.round(depthm)
points = [Point(p) for p in zip(lat.ravel(), lon.ravel())]
for i, a in enumerate(area):
indices = np.where(
map(
lambda p, poly=area_polys[i]: poly.contains(p),
points
)
)
selection = np.ma.array(d.ravel()[indices])
if len(selection) > 0 and not selection.mask.all():
output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': output_fmtstr % (
np.ma.amin(selection).astype(float)
),
'max': output_fmtstr % (
np.ma.amax(selection).astype(float)
),
'mean': output_fmtstr % (
np.ma.mean(selection).astype(float)
),
'median': output_fmtstr % (
np.ma.median(selection).astype(float)
),
'stddev': output_fmtstr % (
np.ma.std(selection).astype(float)
),
'num': "%d" % selection.count(),
})
else:
output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': gettext("No Data"),
'max': gettext("No Data"),
'mean': gettext("No Data"),
'median': gettext("No Data"),
'stddev': gettext("No Data"),
'num': "0",
})
return json.dumps(sorted(output, key=itemgetter('name')))
def load_data(self):
distance = VincentyDistance()
height = distance.measure(
(self.bounds[0], self.centroid[1]),
(self.bounds[2], self.centroid[1])) * 1000 * 1.25
width = distance.measure(
(self.centroid[0], self.bounds[1]),
(self.centroid[0], self.bounds[3])) * 1000 * 1.25
if self.projection == 'EPSG:32661':
near_pole, covers_pole = self.pole_proximity(self.points[0])
blat = min(self.bounds[0], self.bounds[2])
blat = 5 * np.floor(blat / 5)
if self.centroid[0] > 80 or near_pole or covers_pole:
self.basemap = basemap.load_map(
'npstere', self.centroid, height, width,
min(self.bounds[0], self.bounds[2]))
else:
self.basemap = basemap.load_map('lcc', self.centroid, height,
width)
elif self.projection == 'EPSG:3031':
near_pole, covers_pole = self.pole_proximity(self.points[0])
blat = max(self.bounds[0], self.bounds[2])
blat = 5 * np.ceil(blat / 5)
if ((self.centroid[0] < -80 or self.bounds[1] < -80
or self.bounds[3] < -80)
or covers_pole): # is centerered close to the south pole
self.basemap = basemap.load_map('spstere', (blat, 180), height,
width)
else:
self.basemap = basemap.load_map(
'lcc', self.centroid, height, width,
max(self.bounds[0], self.bounds[2]))
elif abs(self.centroid[1] - self.bounds[1]) > 90:
height_bounds = [self.bounds[0], self.bounds[2]]
width_bounds = [self.bounds[1], self.bounds[3]]
height_buffer = (abs(height_bounds[1] - height_bounds[0])) * 0.1
width_buffer = (abs(width_bounds[0] - width_bounds[1])) * 0.1
if abs(width_bounds[1] - width_bounds[0]) > 360:
raise ClientError(
gettext(
"You have requested an area that exceads the width of the world. \
Thinking big is good but plots need to be less the 360 deg wide."
))
if height_bounds[1] < 0:
height_bounds[1] = height_bounds[1] + height_buffer
else:
height_bounds[1] = height_bounds[1] + height_buffer
if height_bounds[0] < 0:
height_bounds[0] = height_bounds[0] - height_buffer
else:
height_bounds[0] = height_bounds[0] - height_buffer
new_width_bounds = []
new_width_bounds.append(width_bounds[0] - width_buffer)
new_width_bounds.append(width_bounds[1] + width_buffer)
if abs(new_width_bounds[1] - new_width_bounds[0]) > 360:
width_buffer = np.floor(
(360 - abs(width_bounds[1] - width_bounds[0])) / 2)
new_width_bounds[0] = width_bounds[0] - width_buffer
new_width_bounds[1] = width_bounds[1] + width_buffer
if new_width_bounds[0] < -360:
new_width_bounds[0] = -360
if new_width_bounds[1] > 720:
new_width_bounds[1] = 720
self.basemap = basemap.load_map(
'merc', self.centroid, (height_bounds[0], height_bounds[1]),
(new_width_bounds[0], new_width_bounds[1]))
else:
self.basemap = basemap.load_map('lcc', self.centroid, height,
width)
if self.basemap.aspect < 1:
gridx = 500
gridy = int(500 * self.basemap.aspect)
else:
gridy = 500
gridx = int(500 / self.basemap.aspect)
self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.variable_unit = self.get_variable_units(
dataset, self.variables)[0]
self.variable_name = self.get_variable_names(
dataset, self.variables)[0]
scale_factor = self.get_variable_scale_factors(
dataset, self.variables)[0]
if self.cmap is None:
if len(self.variables) == 1:
self.cmap = colormap.find_colormap(self.variable_name)
else:
self.cmap = colormap.colormaps.get('speed')
if len(self.variables) == 2:
self.variable_name = self.vector_name(self.variable_name)
if self.depth == 'bottom':
depth_value = 'Bottom'
else:
self.depth = np.clip(int(self.depth), 0,
len(dataset.depths) - 1)
depth_value = dataset.depths[self.depth]
data = []
allvars = []
for v in self.variables:
var = dataset.variables[v]
allvars.append(v)
if self.filetype in ['csv', 'odv', 'txt']:
d, depth_value = dataset.get_area(np.array(
[self.latitude, self.longitude]),
self.depth,
self.time,
v,
self.interp,
self.radius,
self.neighbours,
return_depth=True)
else:
d = dataset.get_area(
np.array([self.latitude,
self.longitude]), self.depth, self.time, v,
self.interp, self.radius, self.neighbours)
d = np.multiply(d, scale_factor)
self.variable_unit, d = self.kelvin_to_celsius(
self.variable_unit, d)
data.append(d)
if self.filetype not in ['csv', 'odv', 'txt']:
if len(var.dimensions) == 3:
self.depth_label = ""
elif self.depth == 'bottom':
self.depth_label = " at Bottom"
else:
self.depth_label = " at " + \
str(int(np.round(depth_value))) + " m"
if len(data) == 2:
data[0] = np.sqrt(data[0]**2 + data[1]**2)
self.data = data[0]
quiver_data = []
# Store the quiver data on the same grid as the main variable. This
# will only be used for CSV export.
quiver_data_fullgrid = []
if self.quiver is not None and \
self.quiver['variable'] != '' and \
self.quiver['variable'] != 'none':
for v in self.quiver['variable'].split(','):
allvars.append(v)
var = dataset.variables[v]
quiver_unit = get_variable_unit(self.dataset_name, var)
quiver_name = get_variable_name(self.dataset_name, var)
quiver_lon, quiver_lat = self.basemap.makegrid(50, 50)
d = dataset.get_area(
np.array([quiver_lat, quiver_lon]),
self.depth,
self.time,
v,
self.interp,
self.radius,
self.neighbours,
)
quiver_data.append(d)
# Get the quiver data on the same grid as the main
# variable.
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
v,
self.interp,
self.radius,
self.neighbours,
)
quiver_data_fullgrid.append(d)
self.quiver_name = self.vector_name(quiver_name)
self.quiver_longitude = quiver_lon
self.quiver_latitude = quiver_lat
self.quiver_unit = quiver_unit
self.quiver_data = quiver_data
self.quiver_data_fullgrid = quiver_data_fullgrid
if all(
map(lambda v: len(dataset.variables[v].dimensions) == 3,
allvars)):
self.depth = 0
contour_data = []
if self.contour is not None and \
self.contour['variable'] != '' and \
self.contour['variable'] != 'none':
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
self.contour['variable'],
self.interp,
self.radius,
self.neighbours,
)
contour_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_name = get_variable_name(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_unit, d = self.kelvin_to_celsius(contour_unit, d)
d = np.multiply(d, contour_factor)
contour_data.append(d)
self.contour_unit = contour_unit
self.contour_name = contour_name
self.contour_data = contour_data
self.timestamp = dataset.timestamps[self.time]
if self.compare:
self.variable_name += " Difference"
with open_dataset(get_dataset_url(
self.compare['dataset'])) as dataset:
data = []
for v in self.compare['variables']:
var = dataset.variables[v]
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.compare['depth'],
self.compare['time'],
v,
self.interp,
self.radius,
self.neighbours,
)
data.append(d)
if len(data) == 2:
data = np.sqrt(data[0]**2 + data[1]**2)
else:
data = data[0]
u, data = self.kelvin_to_celsius(
dataset.variables[self.compare['variables'][0]].unit, data)
self.data -= data
# Load bathymetry data
self.bathymetry = overlays.bathymetry(self.basemap,
self.latitude,
self.longitude,
blur=2)
if self.depth != 'bottom' and self.depth != 0:
if len(quiver_data) > 0:
quiver_bathymetry = overlays.bathymetry(
self.basemap, quiver_lat, quiver_lon)
self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked
for d in self.quiver_data:
d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked
for d in self.contour_data:
d[np.where(self.bathymetry < depth_value)] = np.ma.masked
else:
mask = maskoceans(self.longitude, self.latitude, self.data).mask
self.data[~mask] = np.ma.masked
for d in self.quiver_data:
mask = maskoceans(self.quiver_longitude, self.quiver_latitude,
d).mask
d[~mask] = np.ma.masked
for d in contour_data:
mask = maskoceans(self.longitude, self.latitude, d).mask
d[~mask] = np.ma.masked
if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']:
area_polys = []
for a in self.area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Poly(r, inners))
area_polys.append(MultiPolygon(polygons))
points = [
Point(p)
for p in zip(self.latitude.ravel(), self.longitude.ravel())
]
indicies = []
for a in area_polys:
indicies.append(
np.where(map(lambda p, poly=a: poly.contains(p),
points))[0])
indicies = np.unique(np.array(indicies).ravel())
newmask = np.ones(self.data.shape, dtype=bool)
newmask[np.unravel_index(indicies, newmask.shape)] = False
self.data.mask |= newmask
self.depth_value = depth_value
def load_data(self):
ds_url = app.config['DRIFTER_URL']
data_names = []
data_units = []
with Dataset(ds_url % self.drifter, 'r') as ds:
self.name = ds.buoyid
self.imei = str(chartostring(ds['imei'][0]))
self.wmo = str(chartostring(ds['wmo'][0]))
t = netcdftime.utime(ds['data_date'].units)
d = []
for v in self.buoyvariables:
d.append(ds[v][:])
if "long_name" in ds[v].ncattrs():
data_names.append(ds[v].long_name)
else:
data_names.append(v)
if "units" in ds[v].ncattrs():
data_units.append(ds[v].units)
else:
data_units.append(None)
self.data = d
self.times = t.num2date(ds['data_date'][:])
self.points = np.array([
ds['latitude'][:],
ds['longitude'][:],
]).transpose()
data_names = data_names[:len(self.buoyvariables)]
data_units = data_units[:len(self.buoyvariables)]
for i, t in enumerate(self.times):
if t.tzinfo is None:
self.times[i] = t.replace(tzinfo=pytz.UTC)
self.data_names = data_names
self.data_units = data_units
if self.starttime is not None:
d = dateutil.parser.parse(self.starttime)
self.start = np.where(self.times >= d)[0].min()
else:
self.start = 0
if self.endtime is not None:
d = dateutil.parser.parse(self.endtime)
self.end = np.where(self.times <= d)[0].max() + 1
else:
self.end = len(self.times) - 1
if self.start < 0:
self.start += len(self.times)
self.start = np.clip(self.start, 0, len(self.times) - 1)
if self.end < 0:
self.end += len(self.times)
self.end = np.clip(self.end, 0, len(self.times) - 1)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
depth = int(self.depth)
try:
model_start = np.where(
dataset.timestamps <= self.times[self.start]
)[0][-1]
except IndexError:
model_start = 0
model_start -= 1
model_start = np.clip(model_start, 0, len(dataset.timestamps) - 1)
try:
model_end = np.where(
dataset.timestamps >= self.times[self.end]
)[0][0]
except IndexError:
model_end = len(dataset.timestamps) - 1
model_end += 1
model_end = np.clip(
model_end,
model_start,
len(dataset.timestamps) - 1
)
model_times = map(
lambda t: time.mktime(t.timetuple()),
dataset.timestamps[model_start:model_end + 1]
)
output_times = map(
lambda t: time.mktime(t.timetuple()),
self.times[self.start:self.end + 1]
)
d = []
for v in self.variables:
pts, dist, mt, md = dataset.get_path(
self.points[self.start:self.end + 1],
depth,
range(model_start, model_end + 1),
v,
times=output_times
)
f = interp1d(
model_times,
md,
assume_sorted=True,
bounds_error=False,
)
d.append(np.diag(f(mt)))
model_data = np.ma.array(d)
variable_names = []
variable_units = []
scale_factors = []
for v in self.variables:
variable_units.append(get_variable_unit(self.dataset_name,
dataset.variables[v]))
variable_names.append(get_variable_name(self.dataset_name,
dataset.variables[v]))
scale_factors.append(
get_variable_scale_factor(self.dataset_name,
dataset.variables[v])
)
for idx, sf in enumerate(scale_factors):
model_data[idx, :] = np.multiply(model_data[idx, :], sf)
for idx, u in enumerate(variable_units):
variable_units[idx], model_data[idx, :] = \
self.kelvin_to_celsius(u, model_data[idx, :])
self.model_data = model_data
self.model_times = map(datetime.datetime.utcfromtimestamp, mt)
self.variable_names = variable_names
self.variable_units = variable_units
def load_data(self):
ds_url = app.config['DRIFTER_URL']
data_names = []
data_units = []
with Dataset(ds_url % self.drifter, 'r') as ds:
self.name = ds.buoyid
self.imei = str(chartostring(ds['imei'][0]))
self.wmo = str(chartostring(ds['wmo'][0]))
t = netcdftime.utime(ds['data_date'].units)
d = []
for v in self.buoyvariables:
d.append(ds[v][:])
if "long_name" in ds[v].ncattrs():
data_names.append(ds[v].long_name)
else:
data_names.append(v)
if "units" in ds[v].ncattrs():
data_units.append(ds[v].units)
else:
data_units.append(None)
self.data = d
self.times = t.num2date(ds['data_date'][:])
self.points = np.array([
ds['latitude'][:],
ds['longitude'][:],
]).transpose()
data_names = data_names[:len(self.buoyvariables)]
data_units = data_units[:len(self.buoyvariables)]
for i, t in enumerate(self.times):
if t.tzinfo is None:
self.times[i] = t.replace(tzinfo=pytz.UTC)
self.data_names = data_names
self.data_units = data_units
if self.starttime is not None:
d = dateutil.parser.parse(self.starttime)
self.start = np.where(self.times >= d)[0].min()
else:
self.start = 0
if self.endtime is not None:
d = dateutil.parser.parse(self.endtime)
self.end = np.where(self.times <= d)[0].max() + 1
else:
self.end = len(self.times) - 1
if self.start < 0:
self.start += len(self.times)
self.start = np.clip(self.start, 0, len(self.times) - 1)
if self.end < 0:
self.end += len(self.times)
self.end = np.clip(self.end, 0, len(self.times) - 1)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
depth = int(self.depth)
try:
model_start = np.where(
dataset.timestamps <= self.times[self.start])[0][-1]
except IndexError:
model_start = 0
model_start -= 1
model_start = np.clip(model_start, 0, len(dataset.timestamps) - 1)
try:
model_end = np.where(
dataset.timestamps >= self.times[self.end])[0][0]
except IndexError:
model_end = len(dataset.timestamps) - 1
model_end += 1
model_end = np.clip(model_end, model_start,
len(dataset.timestamps) - 1)
model_times = map(lambda t: time.mktime(t.timetuple()),
dataset.timestamps[model_start:model_end + 1])
output_times = map(lambda t: time.mktime(t.timetuple()),
self.times[self.start:self.end + 1])
d = []
for v in self.variables:
pts, dist, mt, md = dataset.get_path(
self.points[self.start:self.end + 1],
depth,
range(model_start, model_end + 1),
v,
times=output_times)
f = interp1d(
model_times,
md,
assume_sorted=True,
bounds_error=False,
)
d.append(np.diag(f(mt)))
model_data = np.ma.array(d)
variable_names = []
variable_units = []
scale_factors = []
for v in self.variables:
variable_units.append(
get_variable_unit(self.dataset_name, dataset.variables[v]))
variable_names.append(
get_variable_name(self.dataset_name, dataset.variables[v]))
scale_factors.append(
get_variable_scale_factor(self.dataset_name,
dataset.variables[v]))
for idx, sf in enumerate(scale_factors):
model_data[idx, :] = np.multiply(model_data[idx, :], sf)
for idx, u in enumerate(variable_units):
variable_units[idx], model_data[idx, :] = \
self.kelvin_to_celsius(u, model_data[idx, :])
self.model_data = model_data
self.model_times = map(datetime.datetime.utcfromtimestamp, mt)
self.variable_names = variable_names
self.variable_units = variable_units
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
self.load_misc(dataset, self.variables)
self.fix_startend_times(dataset)
self.variable_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.variables[0]]
)
self.variable_name = get_variable_name(
self.dataset_name,
dataset.variables[self.variables[0]]
)
var = self.variables[0]
if self.depth != 'all' and self.depth != 'bottom' and \
(set(dataset.variables[var].dimensions) &
set(dataset.depth_dimensions)):
self.depth_label = " at %d m" % (
np.round(dataset.depths[self.depth])
)
elif self.depth == 'bottom':
self.depth_label = ' at Bottom'
else:
self.depth_label = ''
if not (set(dataset.variables[var].dimensions) &
set(dataset.depth_dimensions)):
self.depth = 0
times = None
point_data = []
for p in self.points:
data = []
for v in self.variables:
if self.depth == 'all':
d, dep = dataset.get_timeseries_profile(
float(p[0]),
float(p[1]),
self.starttime,
self.endtime,
v
)
else:
d, dep = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
self.depth,
self.starttime,
self.endtime,
v,
return_depth=True
)
data.append(d)
point_data.append(np.ma.array(data))
point_data = np.ma.array(point_data)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
times = dataset.timestamps[self.starttime:self.endtime + 1]
if self.query.get('dataset_quantum') == 'month':
times = [datetime.date(x.year, x.month, 1) for x in times]
# depths = dataset.depths
depths = dep
# TODO: pint
if self.variable_unit.startswith("Kelvin"):
self.variable_unit = "Celsius"
for idx, v in enumerate(self.variables):
point_data[:, idx, :] = point_data[:, idx, :] - 273.15
if point_data.shape[1] == 2:
point_data = np.ma.expand_dims(
np.sqrt(
point_data[:, 0, :] ** 2 + point_data[:, 1, :] ** 2
), 1
)
self.times = times
self.data = point_data
self.depths = depths
self.depth_unit = "m"
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
for idx, v in enumerate(self.variables):
var = dataset.variables[v]
if not (set(var.dimensions) & set(dataset.depth_dimensions)):
for potential in dataset.variables:
if potential in self.variables:
continue
pot = dataset.variables[potential]
if (set(pot.dimensions) &
set(dataset.depth_dimensions)):
if len(pot.shape) > 3:
self.variables[idx] = potential
self.variables_anom[idx] = potential
value = parallel = perpendicular = None
variable_names = self.get_variable_names(dataset, self.variables)
variable_units = self.get_variable_units(dataset, self.variables)
scale_factors = self.get_variable_scale_factors(dataset,
self.variables)
if len(self.variables) > 1:
v = []
for name in self.variables:
v.append(dataset.variables[name])
distances, times, lat, lon, bearings = geo.path_to_points(
self.points, 100
)
transect_pts, distance, x, dep = dataset.get_path_profile(
self.points, time, self.variables[0], 100)
transect_pts, distance, y, dep = dataset.get_path_profile(
self.points, time, self.variables[1], 100)
x = np.multiply(x, scale_factors[0])
y = np.multiply(y, scale_factors[1])
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(y, x) - r
mag = np.sqrt(x ** 2 + y ** 2)
parallel = mag * np.cos(theta)
perpendicular = mag * np.sin(theta)
else:
transect_pts, distance, value, dep = dataset.get_path_profile(
self.points, time, self.variables[0])
value = np.multiply(value, scale_factors[0])
variable_units[0], value = self.kelvin_to_celsius(
variable_units[0],
value
)
if len(self.variables) == 2:
variable_names[0] = self.vector_name(variable_names[0])
if self.cmap is None:
self.cmap = colormap.find_colormap(variable_names[0])
self.timestamp = dataset.timestamps[int(time)]
self.depth = dep
self.depth_unit = "m"
self.transect_data = {
"points": transect_pts,
"distance": distance,
"data": value,
"name": variable_names[0],
"unit": variable_units[0],
"parallel": parallel,
"perpendicular": perpendicular,
}
if self.surface is not None:
surface_pts, surface_dist, t, surface_value = \
dataset.get_path(
self.points,
0,
time,
self.surface,
)
surface_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.surface]
)
surface_name = get_variable_name(
self.dataset_name,
dataset.variables[self.surface]
)
surface_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.surface]
)
surface_value = np.multiply(surface_value, surface_factor)
surface_unit, surface_value = self.kelvin_to_celsius(
surface_unit,
surface_value
)
self.surface_data = {
"points": surface_pts,
"distance": surface_dist,
"data": surface_value,
"name": surface_name,
"unit": surface_unit
}
if self.variables != self.variables_anom:
with open_dataset(
get_dataset_climatology(self.dataset_name)
) as dataset:
if self.variables[0] in dataset.variables:
if len(self.variables) == 1:
climate_points, climate_distance, climate_data = \
dataset.get_path_profile(self.points,
self.timestamp.month - 1,
self.variables[0])
u, climate_data = self.kelvin_to_celsius(
dataset.variables[self.variables[0]].unit,
climate_data
)
self.transect_data['data'] -= - climate_data
else:
climate_pts, climate_distance, climate_x, cdep = \
dataset.get_path_profile(
self.points,
self.timestamp.month - 1,
self.variables[0],
100
)
climate_pts, climate_distance, climate_y, cdep = \
dataset.get_path_profile(
self.points,
self.timestamp.month - 1,
self.variables[0],
100
)
climate_distances, ctimes, clat, clon, bearings = \
geo.path_to_points(self.points, 100)
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(y, x) - r
mag = np.sqrt(x ** 2 + y ** 2)
climate_parallel = mag * np.cos(theta)
climate_perpendicular = mag * np.sin(theta)
self.transect_data['parallel'] -= climate_parallel
self.transect_data[
'perpendicular'] -= climate_perpendicular
# Bathymetry
with Dataset(app.config['BATHYMETRY_FILE'], 'r') as dataset:
bath_x, bath_y = bathymetry(
dataset.variables['y'],
dataset.variables['x'],
dataset.variables['z'],
self.points)
self.bathymetry = {
'x': bath_x,
'y': bath_y
}
def plot(projection, x, y, z, args):
lat, lon = get_latlon_coords(projection, x, y, z)
if len(lat.shape) == 1:
lat, lon = np.meshgrid(lat, lon)
dataset_name = args.get('dataset')
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
depth = args.get('depth')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
data = []
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if args.get('time') is None or (type(args.get('time')) == str and
len(args.get('time')) == 0):
time = -1
else:
time = int(args.get('time'))
t_len = len(dataset.timestamps)
while time >= t_len:
time -= t_len
while time < 0:
time += len(dataset.timestamps)
timestamp = dataset.timestamps[time]
for v in variable:
data.append(dataset.get_area(
np.array([lat, lon]),
depth,
time,
v
))
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
scale_factor = get_variable_scale_factor(
dataset_name,
dataset.variables[variable[0]]
)
if anom:
cmap = colormap.colormaps['anomaly']
else:
cmap = colormap.find_colormap(variable_name)
if depth != 'bottom':
depthm = dataset.depths[depth]
else:
depthm = 0
if scale_factor != 1.0:
for idx, val in enumerate(data):
data[idx] = np.multiply(val, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
for idx, val in enumerate(data):
data[idx] = np.add(val, -273.15)
if len(data) == 1:
data = data[0]
if len(data) == 2:
data = np.sqrt(data[0] ** 2 + data[1] ** 2)
if not anom:
cmap = colormap.colormaps.get('speed')
if anom:
with open_dataset(get_dataset_climatology(dataset_name)) as dataset:
a = dataset.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
v
)
data = data - a
f, fname = tempfile.mkstemp()
os.close(f)
data = data.transpose()
xpx = x * 256
ypx = y * 256
with Dataset(ETOPO_FILE % (projection, z), 'r') as dataset:
bathymetry = dataset["z"][ypx:(ypx + 256), xpx:(xpx + 256)]
bathymetry = gaussian_filter(bathymetry, 0.5)
data[np.where(bathymetry > -depthm)] = np.ma.masked
sm = matplotlib.cm.ScalarMappable(
matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1]), cmap=cmap)
img = sm.to_rgba(np.squeeze(data))
im = Image.fromarray((img * 255.0).astype(np.uint8))
im.save(fname, format='png', optimize=True)
with open(fname, 'r') as f:
buf = f.read()
os.remove(fname)
return buf
def test_get_variable_misc(self, m):
m.return_value = {
"var": {
"name": "the_name",
"unit": "My Unit",
"scale": [0, 10],
},
"var2": {
"hide": True,
}
}
self.assertEqual(
util.get_variable_name("dataset", mock.Mock(key="var")),
"the_name"
)
variable_mock = mock.Mock()
variable_mock.configure_mock(key="none", name="var_name")
self.assertEqual(
util.get_variable_name("dataset", variable_mock),
"var_name"
)
variable_mock.configure_mock(key="nameless", name=None)
self.assertEqual(
util.get_variable_name("dataset", variable_mock),
"Nameless"
)
self.assertEqual(
util.get_variable_unit("dataset", mock.Mock(key="var")),
"My Unit"
)
variable_mock.configure_mock(key="none", unit="var_unit")
self.assertEqual(
util.get_variable_unit("dataset", variable_mock),
"var_unit"
)
self.assertEqual(
util.get_variable_unit(
"dataset", mock.Mock(key="varx", unit=None)),
"Unknown"
)
self.assertEqual(
util.get_variable_scale("dataset", mock.Mock(key="var")),
[0, 10]
)
variable_mock.configure_mock(key="none", valid_min=5, valid_max=50)
self.assertEqual(
util.get_variable_scale(
"dataset", variable_mock),
[5, 50]
)
self.assertEqual(
util.get_variable_scale(
"dataset", mock.Mock(key="varx", scale=None, valid_min=None,
valid_max=None)),
[0, 100]
)
self.assertFalse(
util.is_variable_hidden("dataset", mock.Mock(key="var"))
)
self.assertTrue(
util.is_variable_hidden("dataset", mock.Mock(key="var2"))
)
self.assertFalse(
util.is_variable_hidden("dataset", mock.Mock(key="var3"))
)
def load_data(self):
if self.projection == 'EPSG:32661':
blat = min(self.bounds[0], self.bounds[2])
blat = 5 * np.floor(blat / 5)
self.basemap = basemap.load_map('npstere', (blat, 0), None, None)
elif self.projection == 'EPSG:3031':
blat = max(self.bounds[0], self.bounds[2])
blat = 5 * np.ceil(blat / 5)
self.basemap = basemap.load_map('spstere', (blat, 180), None, None)
else:
distance = VincentyDistance()
height = distance.measure(
(self.bounds[0], self.centroid[1]),
(self.bounds[2], self.centroid[1])
) * 1000 * 1.25
width = distance.measure(
(self.centroid[0], self.bounds[1]),
(self.centroid[0], self.bounds[3])
) * 1000 * 1.25
self.basemap = basemap.load_map(
'lcc', self.centroid, height, width
)
if self.basemap.aspect < 1:
gridx = 500
gridy = int(500 * self.basemap.aspect)
else:
gridy = 500
gridx = int(500 / self.basemap.aspect)
self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.variable_unit = self.get_variable_units(
dataset, self.variables
)[0]
self.variable_name = self.get_variable_names(
dataset,
self.variables
)[0]
scale_factor = self.get_variable_scale_factors(
dataset, self.variables
)[0]
if self.cmap is None:
if len(self.variables) == 1:
self.cmap = colormap.find_colormap(self.variable_name)
else:
self.cmap = colormap.colormaps.get('speed')
if len(self.variables) == 2:
self.variable_name = self.vector_name(self.variable_name)
if self.depth == 'bottom':
depth_value = 'Bottom'
else:
self.depth = np.clip(
int(self.depth), 0, len(dataset.depths) - 1)
depth_value = dataset.depths[self.depth]
data = []
allvars = []
for v in self.variables:
var = dataset.variables[v]
allvars.append(v)
if self.filetype in ['csv', 'odv', 'txt']:
d, depth_value = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
v,
return_depth=True
)
else:
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
v
)
d = np.multiply(d, scale_factor)
self.variable_unit, d = self.kelvin_to_celsius(
self.variable_unit, d)
data.append(d)
if self.filetype not in ['csv', 'odv', 'txt']:
if len(var.dimensions) == 3:
self.depth_label = ""
elif self.depth == 'bottom':
self.depth_label = " at Bottom"
else:
self.depth_label = " at " + \
str(int(np.round(depth_value))) + " m"
if len(data) == 2:
data[0] = np.sqrt(data[0] ** 2 + data[1] ** 2)
self.data = data[0]
quiver_data = []
if self.quiver is not None and \
self.quiver['variable'] != '' and \
self.quiver['variable'] != 'none':
for v in self.quiver['variable'].split(','):
allvars.append(v)
var = dataset.variables[v]
quiver_unit = get_variable_unit(self.dataset_name, var)
quiver_name = get_variable_name(self.dataset_name, var)
quiver_lon, quiver_lat = self.basemap.makegrid(50, 50)
d = dataset.get_area(
np.array([quiver_lat, quiver_lon]),
self.depth,
self.time,
v
)
quiver_data.append(d)
self.quiver_name = self.vector_name(quiver_name)
self.quiver_longitude = quiver_lon
self.quiver_latitude = quiver_lat
self.quiver_unit = quiver_unit
self.quiver_data = quiver_data
if all(map(lambda v: len(dataset.variables[v].dimensions) == 3,
allvars)):
self.depth = 0
contour_data = []
if self.contour is not None and \
self.contour['variable'] != '' and \
self.contour['variable'] != 'none':
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
self.contour['variable']
)
contour_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_name = get_variable_name(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_unit, d = self.kelvin_to_celsius(contour_unit, d)
d = np.multiply(d, contour_factor)
contour_data.append(d)
self.contour_unit = contour_unit
self.contour_name = contour_name
self.contour_data = contour_data
self.timestamp = dataset.timestamps[self.time]
if self.variables != self.variables_anom:
self.variable_name += " Anomaly"
with open_dataset(
get_dataset_climatology(self.dataset_name)
) as dataset:
data = []
for v in self.variables:
var = dataset.variables[v]
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.timestamp.month - 1,
v
)
data.append(d)
if len(data) == 2:
data = np.sqrt(data[0] ** 2 + data[1] ** 2)
else:
data = data[0]
u, data = self.kelvin_to_celsius(
dataset.variables[self.variables[0]].unit,
data)
self.data -= data
# Load bathymetry data
self.bathymetry = overlays.bathymetry(
self.basemap,
self.latitude,
self.longitude,
blur=2
)
if self.depth != 'bottom' and self.depth != 0:
if len(quiver_data) > 0:
quiver_bathymetry = overlays.bathymetry(
self.basemap, quiver_lat, quiver_lon)
self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked
for d in self.quiver_data:
d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked
for d in self.contour_data:
d[np.where(self.bathymetry < depth_value)] = np.ma.masked
else:
mask = maskoceans(self.longitude, self.latitude, self.data).mask
self.data[~mask] = np.ma.masked
for d in self.quiver_data:
mask = maskoceans(
self.quiver_longitude, self.quiver_latitude, d).mask
d[~mask] = np.ma.masked
for d in contour_data:
mask = maskoceans(self.longitude, self.latitude, d).mask
d[~mask] = np.ma.masked
if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']:
area_polys = []
for a in self.area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Poly(r, inners))
area_polys.append(MultiPolygon(polygons))
points = [Point(p) for p in zip(self.latitude.ravel(),
self.longitude.ravel())]
indicies = []
for a in area_polys:
indicies.append(np.where(
map(
lambda p, poly=a: poly.contains(p),
points
)
)[0])
indicies = np.unique(np.array(indicies).ravel())
newmask = np.ones(self.data.shape, dtype=bool)
newmask[np.unravel_index(indicies, newmask.shape)] = False
self.data.mask |= newmask
self.depth_value = depth_value
def load_data(self):
if self.projection == 'EPSG:32661':
blat = min(self.bounds[0], self.bounds[2])
blat = 5 * np.floor(blat / 5)
self.basemap = basemap.load_map('npstere', (blat, 0), None, None)
elif self.projection == 'EPSG:3031':
blat = max(self.bounds[0], self.bounds[2])
blat = 5 * np.ceil(blat / 5)
self.basemap = basemap.load_map('spstere', (blat, 180), None, None)
else:
distance = VincentyDistance()
height = distance.measure(
(self.bounds[0], self.centroid[1]),
(self.bounds[2], self.centroid[1])) * 1000 * 1.25
width = distance.measure(
(self.centroid[0], self.bounds[1]),
(self.centroid[0], self.bounds[3])) * 1000 * 1.25
self.basemap = basemap.load_map('lcc', self.centroid, height,
width)
if self.basemap.aspect < 1:
gridx = 500
gridy = int(500 * self.basemap.aspect)
else:
gridy = 500
gridx = int(500 / self.basemap.aspect)
self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.variable_unit = self.get_variable_units(
dataset, self.variables)[0]
self.variable_name = self.get_variable_names(
dataset, self.variables)[0]
scale_factor = self.get_variable_scale_factors(
dataset, self.variables)[0]
if self.cmap is None:
if len(self.variables) == 1:
self.cmap = colormap.find_colormap(self.variable_name)
else:
self.cmap = colormap.colormaps.get('speed')
if len(self.variables) == 2:
self.variable_name = self.vector_name(self.variable_name)
if self.depth == 'bottom':
depth_value = 'Bottom'
else:
self.depth = np.clip(int(self.depth), 0,
len(dataset.depths) - 1)
depth_value = dataset.depths[self.depth]
data = []
allvars = []
for v in self.variables:
var = dataset.variables[v]
allvars.append(v)
if self.filetype in ['csv', 'odv', 'txt']:
d, depth_value = dataset.get_area(np.array(
[self.latitude, self.longitude]),
self.depth,
self.time,
v,
return_depth=True)
else:
d = dataset.get_area(
np.array([self.latitude, self.longitude]), self.depth,
self.time, v)
d = np.multiply(d, scale_factor)
self.variable_unit, d = self.kelvin_to_celsius(
self.variable_unit, d)
data.append(d)
if self.filetype not in ['csv', 'odv', 'txt']:
if len(var.dimensions) == 3:
self.depth_label = ""
elif self.depth == 'bottom':
self.depth_label = " at Bottom"
else:
self.depth_label = " at " + \
str(int(np.round(depth_value))) + " m"
if len(data) == 2:
data[0] = np.sqrt(data[0]**2 + data[1]**2)
self.data = data[0]
quiver_data = []
if self.quiver is not None and \
self.quiver['variable'] != '' and \
self.quiver['variable'] != 'none':
for v in self.quiver['variable'].split(','):
allvars.append(v)
var = dataset.variables[v]
quiver_unit = get_variable_unit(self.dataset_name, var)
quiver_name = get_variable_name(self.dataset_name, var)
quiver_lon, quiver_lat = self.basemap.makegrid(50, 50)
d = dataset.get_area(np.array([quiver_lat, quiver_lon]),
self.depth, self.time, v)
quiver_data.append(d)
self.quiver_name = self.vector_name(quiver_name)
self.quiver_longitude = quiver_lon
self.quiver_latitude = quiver_lat
self.quiver_unit = quiver_unit
self.quiver_data = quiver_data
if all(
map(lambda v: len(dataset.variables[v].dimensions) == 3,
allvars)):
self.depth = 0
contour_data = []
if self.contour is not None and \
self.contour['variable'] != '' and \
self.contour['variable'] != 'none':
d = dataset.get_area(np.array([self.latitude,
self.longitude]), self.depth,
self.time, self.contour['variable'])
contour_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_name = get_variable_name(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_unit, d = self.kelvin_to_celsius(contour_unit, d)
d = np.multiply(d, contour_factor)
contour_data.append(d)
self.contour_unit = contour_unit
self.contour_name = contour_name
self.contour_data = contour_data
self.timestamp = dataset.timestamps[self.time]
if self.variables != self.variables_anom:
self.variable_name += " Anomaly"
with open_dataset(get_dataset_climatology(
self.dataset_name)) as dataset:
data = []
for v in self.variables:
var = dataset.variables[v]
d = dataset.get_area(
np.array([self.latitude, self.longitude]), self.depth,
self.timestamp.month - 1, v)
data.append(d)
if len(data) == 2:
data = np.sqrt(data[0]**2 + data[1]**2)
else:
data = data[0]
u, data = self.kelvin_to_celsius(
dataset.variables[self.variables[0]].unit, data)
self.data -= data
# Load bathymetry data
self.bathymetry = overlays.bathymetry(self.basemap,
self.latitude,
self.longitude,
blur=2)
if self.depth != 'bottom' and self.depth != 0:
if len(quiver_data) > 0:
quiver_bathymetry = overlays.bathymetry(
self.basemap, quiver_lat, quiver_lon)
self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked
for d in self.quiver_data:
d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked
for d in self.contour_data:
d[np.where(self.bathymetry < depth_value)] = np.ma.masked
else:
mask = maskoceans(self.longitude, self.latitude, self.data).mask
self.data[~mask] = np.ma.masked
for d in self.quiver_data:
mask = maskoceans(self.quiver_longitude, self.quiver_latitude,
d).mask
d[~mask] = np.ma.masked
for d in contour_data:
mask = maskoceans(self.longitude, self.latitude, d).mask
d[~mask] = np.ma.masked
if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']:
area_polys = []
for a in self.area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Poly(r, inners))
area_polys.append(MultiPolygon(polygons))
points = [
Point(p)
for p in zip(self.latitude.ravel(), self.longitude.ravel())
]
indicies = []
for a in area_polys:
indicies.append(
np.where(map(lambda p, poly=a: poly.contains(p),
points))[0])
indicies = np.unique(np.array(indicies).ravel())
newmask = np.ones(self.data.shape, dtype=bool)
newmask[np.unravel_index(indicies, newmask.shape)] = False
self.data.mask |= newmask
self.depth_value = depth_value
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
self.load_misc(dataset, self.variables)
self.fix_startend_times(dataset)
self.variable_unit = get_variable_unit(
self.dataset_name, dataset.variables[self.variables[0]])
self.variable_name = get_variable_name(
self.dataset_name, dataset.variables[self.variables[0]])
var = self.variables[0]
if self.depth != 'all' and self.depth != 'bottom' and \
(set(dataset.variables[var].dimensions) &
set(dataset.depth_dimensions)):
self.depth_label = " at %d m" % (np.round(
dataset.depths[self.depth]))
elif self.depth == 'bottom':
self.depth_label = ' at Bottom'
else:
self.depth_label = ''
if not (set(dataset.variables[var].dimensions)
& set(dataset.depth_dimensions)):
self.depth = 0
times = None
point_data = []
for p in self.points:
data = []
for v in self.variables:
if self.depth == 'all':
d, dep = dataset.get_timeseries_profile(
float(p[0]), float(p[1]), self.starttime,
self.endtime, v)
else:
d, dep = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
self.depth,
self.starttime,
self.endtime,
v,
return_depth=True)
data.append(d)
point_data.append(np.ma.array(data))
point_data = np.ma.array(point_data)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
times = dataset.timestamps[self.starttime:self.endtime + 1]
if self.query.get('dataset_quantum') == 'month':
times = [datetime.date(x.year, x.month, 1) for x in times]
# depths = dataset.depths
depths = dep
# TODO: pint
if self.variable_unit.startswith("Kelvin"):
self.variable_unit = "Celsius"
for idx, v in enumerate(self.variables):
point_data[:, idx, :] = point_data[:, idx, :] - 273.15
if point_data.shape[1] == 2:
point_data = np.ma.expand_dims(
np.sqrt(point_data[:, 0, :]**2 + point_data[:, 1, :]**2), 1)
self.times = times
self.data = point_data
self.depths = depths
self.depth_unit = "m"
def plot(projection, x, y, z, args):
lat, lon = get_latlon_coords(projection, x, y, z)
if len(lat.shape) == 1:
lat, lon = np.meshgrid(lat, lon)
dataset_name = args.get('dataset')
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
depth = args.get('depth')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
data = []
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if args.get('time') is None or (type(args.get('time')) == str
and len(args.get('time')) == 0):
time = -1
else:
time = int(args.get('time'))
t_len = len(dataset.timestamps)
while time >= t_len:
time -= t_len
while time < 0:
time += len(dataset.timestamps)
timestamp = dataset.timestamps[time]
for v in variable:
data.append(dataset.get_area(np.array([lat, lon]), depth, time, v))
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
scale_factor = get_variable_scale_factor(
dataset_name, dataset.variables[variable[0]])
if anom:
cmap = colormap.colormaps['anomaly']
else:
cmap = colormap.find_colormap(variable_name)
if depth != 'bottom':
depthm = dataset.depths[depth]
else:
depthm = 0
if scale_factor != 1.0:
for idx, val in enumerate(data):
data[idx] = np.multiply(val, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
for idx, val in enumerate(data):
data[idx] = np.add(val, -273.15)
if len(data) == 1:
data = data[0]
if len(data) == 2:
data = np.sqrt(data[0]**2 + data[1]**2)
if not anom:
cmap = colormap.colormaps.get('speed')
if anom:
with open_dataset(get_dataset_climatology(dataset_name)) as dataset:
a = dataset.get_area(np.array([lat, lon]), depth,
timestamp.month - 1, v)
data = data - a
f, fname = tempfile.mkstemp()
os.close(f)
data = data.transpose()
xpx = x * 256
ypx = y * 256
with Dataset(ETOPO_FILE % (projection, z), 'r') as dataset:
bathymetry = dataset["z"][ypx:(ypx + 256), xpx:(xpx + 256)]
bathymetry = gaussian_filter(bathymetry, 0.5)
data[np.where(bathymetry > -depthm)] = np.ma.masked
sm = matplotlib.cm.ScalarMappable(matplotlib.colors.Normalize(
vmin=scale[0], vmax=scale[1]),
cmap=cmap)
img = sm.to_rgba(np.squeeze(data))
im = Image.fromarray((img * 255.0).astype(np.uint8))
im.save(fname, format='png', optimize=True)
with open(fname, 'r') as f:
buf = f.read()
os.remove(fname)
return buf
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
for idx, v in enumerate(self.variables):
var = dataset.variables[v]
if not (set(var.dimensions) & set(dataset.depth_dimensions)):
for potential in dataset.variables:
if potential in self.variables:
continue
pot = dataset.variables[potential]
if (set(pot.dimensions)
& set(dataset.depth_dimensions)):
if len(pot.dimensions) > 3:
self.variables[idx] = potential.key
value = parallel = perpendicular = None
variable_names = self.get_variable_names(dataset, self.variables)
variable_units = self.get_variable_units(dataset, self.variables)
scale_factors = self.get_variable_scale_factors(
dataset, self.variables)
# Load data sent from primary/Left Map
if len(self.variables) > 1:
# Only velocity has 2 variables
v = []
for name in self.variables:
v.append(dataset.variables[name])
distances, times, lat, lon, bearings = geo.path_to_points(
self.points, 100)
transect_pts, distance, x, dep = dataset.get_path_profile(
self.points, time, self.variables[0], 100)
transect_pts, distance, y, dep = dataset.get_path_profile(
self.points, time, self.variables[1], 100)
# Calculate vector components
x = np.multiply(x, scale_factors[0])
y = np.multiply(y, scale_factors[1])
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(y, x) - r
mag = np.sqrt(x**2 + y**2)
parallel = mag * np.cos(theta)
perpendicular = mag * np.sin(theta)
else:
# Get data for one variable
transect_pts, distance, value, dep = dataset.get_path_profile(
self.points, time, self.variables[0])
value = np.multiply(value, scale_factors[0])
# Get variable units and convert to Celsius if needed
variable_units[0], value = self.kelvin_to_celsius(
variable_units[0], value)
if len(self.variables) == 2:
variable_names[0] = self.vector_name(variable_names[0])
# If a colourmap has not been manually specified by the
# Navigator...
if self.cmap is None:
self.cmap = colormap.find_colormap(variable_names[0])
self.timestamp = dataset.timestamps[int(time)]
self.depth = dep
self.depth_unit = "m"
self.transect_data = {
"points": transect_pts,
"distance": distance,
"data": value,
"name": variable_names[0],
"unit": variable_units[0],
"parallel": parallel,
"perpendicular": perpendicular,
}
if self.surface is not None:
surface_pts, surface_dist, t, surface_value = \
dataset.get_path(
self.points,
0,
time,
self.surface,
)
surface_unit = get_variable_unit(
self.dataset_name, dataset.variables[self.surface])
surface_name = get_variable_name(
self.dataset_name, dataset.variables[self.surface])
surface_factor = get_variable_scale_factor(
self.dataset_name, dataset.variables[self.surface])
surface_value = np.multiply(surface_value, surface_factor)
surface_unit, surface_value = self.kelvin_to_celsius(
surface_unit, surface_value)
self.surface_data = {
"points": surface_pts,
"distance": surface_dist,
"data": surface_value,
"name": surface_name,
"unit": surface_unit
}
# Load data sent from Right Map (if in compare mode)
if self.compare:
def interpolate_depths(data, depth_in, depth_out):
output = []
for i in range(0, depth_in.shape[0]):
f = interp1d(
depth_in[i],
data[:, i],
bounds_error=False,
assume_sorted=True,
)
output.append(
f(depth_out[i].view(np.ma.MaskedArray).filled()))
return np.ma.masked_invalid(output).transpose()
with open_dataset(get_dataset_url(
self.compare['dataset'])) as dataset:
# Get and format date
self.compare['date'] = np.clip(self.compare['time'], 0,
len(dataset.timestamps) - 1)
self.compare['date'] = dataset.timestamps[int(
self.compare['date'])]
# 1 variable
if len(self.compare['variables']) == 1:
# Get and store the "nicely formatted" string for the variable name
self.compare['name'] = self.get_variable_names(
dataset, self.compare['variables'])[0]
# Find correct colourmap
if (self.compare['colormap'] == 'default'):
self.compare['colormap'] = colormap.find_colormap(
self.compare['name'])
else:
self.compare['colormap'] = colormap.find_colormap(
self.compare['colormap'])
climate_points, climate_distance, climate_data, cdep = \
dataset.get_path_profile(self.points,
self.compare['time'],
self.compare['variables'][0])
# Get variable units and convert to Celsius if needed
self.compare['unit'], climate_data = self.kelvin_to_celsius(
dataset.variables[self.compare['variables'][0]].unit,
climate_data)
self.__fill_invalid_shift(climate_data)
if (self.depth.shape != cdep.shape) or \
(self.depth != cdep).any():
# Need to interpolate the depths
climate_data = interpolate_depths(
climate_data, cdep, self.depth)
if self.transect_data['data'] is None:
self.transect_data['parallel'] -= climate_data
self.transect_data['perpendicular'] -= climate_data
else:
self.transect_data['compare_data'] = climate_data
# Velocity variables
else:
# Get and store the "nicely formatted" string for the variable name
self.compare['name'] = self.get_variable_names(
dataset, self.compare['variables'])[0]
climate_pts, climate_distance, climate_x, cdep = \
dataset.get_path_profile(
self.points,
self.compare['time'],
self.compare['variables'][0],
100
)
climate_pts, climate_distance, climate_y, cdep = \
dataset.get_path_profile(
self.points,
self.compare['time'],
self.compare['variables'][0],
100
)
climate_distances, ctimes, clat, clon, bearings = \
geo.path_to_points(self.points, 100)
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(climate_y, climate_x) - r
mag = np.sqrt(climate_x**2 + climate_y**2)
if np.all(self.depth != cdep):
theta = interpolate_depths(theta, cdep, self.depth)
self.__fill_invalid_shift(theta)
mag = interpolate_depths(mag, cdep, self.depth)
self.__fill_invalid_shift(mag)
self.compare['parallel'] = mag * np.cos(theta)
self.compare['perpendicular'] = mag * np.sin(theta)
"""
if self.transect_data['parallel'] is None:
self.transect_data['data'] -= mag
else:
self.transect_data['parallel'] -= climate_parallel
self.transect_data['perpendicular'] -= climate_perpendicular
"""
# Bathymetry
with Dataset(app.config['BATHYMETRY_FILE'], 'r') as dataset:
bath_x, bath_y = bathymetry(dataset.variables['y'],
dataset.variables['x'],
dataset.variables['z'], self.points)
self.bathymetry = {'x': bath_x, 'y': bath_y}