def timestamp_for_date(old_dataset, date, new_dataset):
with open_dataset(get_dataset_url(old_dataset)) as ds:
timestamp = ds.timestamps[date]
with open_dataset(get_dataset_url(new_dataset)) as ds:
timestamps = ds.timestamps
diffs = np.vectorize(lambda x: x.total_seconds())(timestamps - timestamp)
idx = np.where(diffs <= 0)[0]
res = 0
if len(idx) > 0:
res = idx.max()
return Response(json.dumps(res), status=200, mimetype='application/json')
def time_query():
data = []
if 'dataset' in request.args:
dataset = request.args['dataset']
quantum = request.args.get('quantum')
with open_dataset(get_dataset_url(dataset)) as ds:
for idx, date in enumerate(ds.timestamps):
if quantum == 'month':
date = datetime.datetime(date.year, date.month, 15)
data.append({
'id': idx,
'value': date.replace(tzinfo=pytz.UTC)
})
data = sorted(data, key=lambda k: k['id'])
class DateTimeEncoder(json.JSONEncoder):
def default(self, o):
if isinstance(o, datetime.datetime):
return o.isoformat()
return json.JSONEncoder.default(self, o)
js = json.dumps(data, cls=DateTimeEncoder)
resp = Response(js, status=200, mimetype='application/json')
return resp
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as d:
if self.time < 0:
self.time += len(d.timestamps)
time = np.clip(self.time, 0, len(d.timestamps) - 1)
timestamp = d.timestamps[time]
try:
self.load_misc(d, self.variables)
except IndexError as e:
raise ClientError(
gettext(
"The selected variable(s) were not found in the dataset. \
Most likely, this variable is a derived product from existing dataset variables. \
Please select another variable. ") + str(e))
point_data, point_depths = self.get_data(d, self.variables, time)
point_data = self.apply_scale_factors(point_data)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units, point_data)
self.data = self.subtract_other(point_data)
self.depths = point_depths
self.timestamp = timestamp
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
latvar, lonvar = utils.get_latlon_vars(dataset)
if self.depth:
if self.depth == 'bottom':
self.depth_value = 'Bottom'
self.depth_unit = ''
else:
self.depth = np.clip(int(self.depth), 0,
len(dataset.depths) - 1)
self.depth_value = np.round(dataset.depths[self.depth])
self.depth_unit = "m"
else:
self.depth_value = 0
self.depth_unit = "m"
self.fix_startend_times(dataset)
time = range(self.starttime, self.endtime + 1)
if len(self.variables) > 1:
v = []
for name in self.variables:
pts, distance, t, value = dataset.get_path(
self.points,
self.depth,
time,
name
)
v.append(value ** 2)
value = np.sqrt(np.ma.sum(v, axis=0))
else:
pts, distance, t, value = dataset.get_path(
self.points,
self.depth,
time,
self.variables[0]
)
self.path_points = pts
self.distance = distance
variable_names = self.get_variable_names(dataset, self.variables)
variable_units = self.get_variable_units(dataset, self.variables)
scale_factors = self.get_variable_scale_factors(dataset,
self.variables)
self.variable_unit, self.data = self.kelvin_to_celsius(
variable_units[0],
value
)
self.data = np.multiply(self.data, scale_factors[0])
self.variable_name = variable_names[0]
self.data = self.data.transpose()
if self.cmap is None:
self.cmap = colormap.find_colormap(self.variable_name)
self.times = dataset.timestamps[self.starttime:self.endtime + 1]
def get_scale(dataset, variable, depth, time, projection, extent):
x = np.linspace(extent[0], extent[2], 50)
y = np.linspace(extent[1], extent[3], 50)
xx, yy = np.meshgrid(x, y)
dest = Proj(init=projection)
lon, lat = dest(xx, yy, inverse=True)
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
d = ds.get_area(
np.array([lat, lon]),
depth,
time,
variables[0]
)
if len(variables) > 1:
d0 = d
d1 = ds.get_area(
np.array([lat, lon]),
depth,
time,
variables[1]
)
d = np.sqrt(d0 ** 2 + d1 ** 2)
variable_unit = get_variable_unit(dataset,
ds.variables[variables[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset), 'r') as ds:
c = ds.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
variables[0]
)
if len(variables) > 1:
c0 = c
c1 = ds.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
variables[1]
)
c = np.sqrt(c0 ** 2 + c1 ** 2)
d = d - c
m = max(abs(d.min()), abs(d.max()))
return -m, m
return d.min(), d.max()
def load_data(self):
if not isinstance(self.depth, list):
self.depth = [self.depth]
self.depth = sorted(self.depth)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.starttime < 0:
self.starttime += len(dataset.timestamps)
if self.endtime < 0:
self.endtime += len(dataset.timestamps)
start = np.clip(self.starttime, 0, len(dataset.timestamps) - 1)
end = np.clip(self.endtime, 0, len(dataset.timestamps) - 1)
timestamp = dataset.timestamps[start:end + 1]
self.load_misc(dataset, self.variables)
point_data = []
point_depth = []
for p in self.points:
data = []
depth = []
for v in self.variables:
dd = []
jj = []
for d in self.depth:
da, dp = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
d,
start,
end,
v,
return_depth=True
)
dd.append(da)
jj.append(dp)
data.append(np.ma.array(dd))
depth.append(np.ma.array(jj))
point_data.append(np.ma.array(data))
point_depth.append(np.ma.array(depth))
point_data = np.ma.array(point_data)
point_depth = np.ma.array(point_depth)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units,
point_data
)
self.data = self.subtract_climatology(point_data, timestamp)
self.data_depth = point_depth
self.timestamp = timestamp
def scale(args):
dataset_name = args.get('dataset')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
with open_dataset(get_dataset_url(dataset_name)) as dataset:
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
if anom:
cmap = colormap.colormaps['anomaly']
variable_name = gettext("%s Anomaly") % variable_name
else:
cmap = colormap.find_colormap(variable_name)
if len(variable) == 2:
if not anom:
cmap = colormap.colormaps.get('speed')
variable_name = re.sub(
r"(?i)( x | y |zonal |meridional |northward |eastward )", " ",
variable_name)
variable_name = re.sub(r" +", " ", variable_name)
fig = plt.figure(figsize=(2, 5), dpi=75)
ax = fig.add_axes([0.05, 0.05, 0.25, 0.9])
norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1])
formatter = ScalarFormatter()
formatter.set_powerlimits((-3, 4))
bar = ColorbarBase(ax, cmap=cmap, norm=norm, orientation='vertical',
format=formatter)
bar.set_label("%s (%s)" % (variable_name.title(),
utils.mathtext(variable_unit)))
buf = StringIO()
try:
plt.savefig(buf, format='png', dpi='figure', transparent=False,
bbox_inches='tight', pad_inches=0.05)
plt.close(fig)
return buf.getvalue()
finally:
buf.close()
def scale(args):
dataset_name = args.get('dataset')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
with open_dataset(get_dataset_url(dataset_name)) as dataset:
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
if anom:
cmap = colormap.colormaps['anomaly']
variable_name = gettext("%s Anomaly") % variable_name
else:
cmap = colormap.find_colormap(variable_name)
if len(variable) == 2:
if not anom:
cmap = colormap.colormaps.get('speed')
variable_name = re.sub(
r"(?i)( x | y |zonal |meridional |northward |eastward )", " ",
variable_name)
variable_name = re.sub(r" +", " ", variable_name)
fig = plt.figure(figsize=(2, 5), dpi=75)
ax = fig.add_axes([0.05, 0.05, 0.25, 0.9])
norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1])
formatter = ScalarFormatter()
formatter.set_powerlimits((-3, 4))
bar = ColorbarBase(ax, cmap=cmap, norm=norm, orientation='vertical',
format=formatter)
bar.set_label("%s (%s)" % (variable_name.title(),
utils.mathtext(variable_unit)))
buf = StringIO()
try:
plt.savefig(buf, format='png', dpi='figure', transparent=False,
bbox_inches='tight', pad_inches=0.05)
plt.close(fig)
return buf.getvalue()
finally:
buf.close()
def load_data(self):
if not isinstance(self.depth, list):
self.depth = [self.depth]
self.depth = sorted(self.depth)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.starttime < 0:
self.starttime += len(dataset.timestamps)
if self.endtime < 0:
self.endtime += len(dataset.timestamps)
start = np.clip(self.starttime, 0, len(dataset.timestamps) - 1)
end = np.clip(self.endtime, 0, len(dataset.timestamps) - 1)
timestamp = dataset.timestamps[start:end + 1]
self.load_misc(dataset, self.variables)
point_data = []
point_depth = []
for p in self.points:
data = []
depth = []
for v in self.variables:
dd = []
jj = []
for d in self.depth:
da, dp = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
d,
start,
end,
v,
return_depth=True)
dd.append(da)
jj.append(dp)
data.append(np.ma.array(dd))
depth.append(np.ma.array(jj))
point_data.append(np.ma.array(data))
point_depth.append(np.ma.array(depth))
point_data = np.ma.array(point_data)
point_depth = np.ma.array(point_depth)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units, point_data)
self.data = self.subtract_other(point_data)
self.data_depth = point_depth
self.timestamp = timestamp
def subtract_other(self, data):
if self.compare:
with Dataset(get_dataset_url(self.compare['dataset']),
'r') as dataset:
cli = self.get_data(dataset, self.compare['variables'],
self.compare['time'])
for idx, v in enumerate(self.variables):
data[:, idx, :] = \
data[:, idx, :] - cli[:, idx, :]
return data
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.timestamp = dataset.timestamps[time]
self.load_temp_sal(dataset, time)
self.variable_units[0], self.temperature = \
super(point.PointPlotter, self).kelvin_to_celsius(
self.variable_units[0], self.temperature
)
def get_point_data(dataset, variable, time, depth, location):
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
data = []
names = []
units = []
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
for v in variables:
d = ds.get_point(
location[0],
location[1],
depth,
time,
v
)
variable_name = get_variable_name(dataset, ds.variables[v])
variable_unit = get_variable_unit(dataset, ds.variables[v])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
data.append(d)
names.append(variable_name)
units.append(variable_unit)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset)) as ds:
for idx, v in enumerate(variables):
d = ds.get_point(
location[0],
location[1],
depth,
timestamp.month,
v
)
data[idx] = data[idx] - d
names[idx] = names[idx] + " Anomaly"
result = {
'value': map(lambda f: '%s' % float('%.4g' % f), data),
'location': map(lambda f: round(f, 4), location),
'name': names,
'units': units,
}
return result
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as d:
if self.time < 0:
self.time += len(d.timestamps)
time = np.clip(self.time, 0, len(d.timestamps) - 1)
timestamp = d.timestamps[time]
self.load_misc(d, self.variables)
point_data, point_depths = self.get_data(d, self.variables, time)
point_data = self.apply_scale_factors(point_data)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units, point_data)
self.data = self.subtract_climatology(point_data, timestamp)
self.depths = point_depths
self.timestamp = timestamp
def get_scale(dataset, variable, depth, time, projection, extent, interp,
radius, neighbours):
x = np.linspace(extent[0], extent[2], 50)
y = np.linspace(extent[1], extent[3], 50)
xx, yy = np.meshgrid(x, y)
dest = Proj(init=projection)
lon, lat = dest(xx, yy, inverse=True)
variables_anom = variable.split(",")
variables = [re.sub('_anom$', '', v) for v in variables_anom]
with open_dataset(get_dataset_url(dataset)) as ds:
timestamp = ds.timestamps[time]
d = ds.get_area(np.array([lat, lon]), depth, time, variables[0],
interp, radius, neighbours)
if len(variables) > 1:
d0 = d
d1 = ds.get_area(np.array([lat, lon]), depth, time, variables[1],
interp, radius, neighbours)
d = np.sqrt(d0**2 + d1**2)
variable_unit = get_variable_unit(dataset, ds.variables[variables[0]])
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = np.add(d, -273.15)
if variables != variables_anom:
with open_dataset(get_dataset_climatology(dataset), 'r') as ds:
c = ds.get_area(np.array([lat, lon]), depth, timestamp.month - 1,
variables[0], interp, radius, neighbours)
if len(variables) > 1:
c0 = c
c1 = ds.get_area(np.array([lat,
lon]), depth, timestamp.month - 1,
variables[1], interp, radius, neighbours)
c = np.sqrt(c0**2 + c1**2)
d = d - c
m = max(abs(d.min()), abs(d.max()))
return -m, m
return d.min(), d.max()
def test_get_dataset_misc(self, m):
m.return_value = {
"dataset": {
"url": "the_url",
"attribution": "My attribution <b>bold</b>",
"climatology": "climatology_url",
"cache": 5,
}
}
self.assertEqual(util.get_dataset_url("dataset"), "the_url")
self.assertEqual(util.get_dataset_climatology("dataset"),
"climatology_url")
self.assertEqual(util.get_dataset_attribution("dataset"),
"My attribution bold")
self.assertEqual(util.get_dataset_cache("dataset"), 5)
m.return_value = {"dataset2": {}}
self.assertEqual(util.get_dataset_cache("dataset2"), None)
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as d:
if self.time < 0:
self.time += len(d.timestamps)
time = np.clip(self.time, 0, len(d.timestamps) - 1)
timestamp = d.timestamps[time]
self.load_misc(d, self.variables)
point_data, point_depths = self.get_data(d, self.variables, time)
point_data = self.apply_scale_factors(point_data)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units,
point_data
)
self.data = self.subtract_climatology(point_data, timestamp)
self.depths = point_depths
self.timestamp = timestamp
def depth():
var = request.args.get('variable')
if var is None:
raise FAILURE
variables = var.split(',')
variables = [re.sub('_anom$', '', v) for v in variables]
data = []
if 'dataset' in request.args:
dataset = request.args['dataset']
with open_dataset(get_dataset_url(dataset)) as ds:
for variable in variables:
if variable and \
variable in ds.variables and \
set(ds.depth_dimensions) & \
set(ds.variables[variable].dimensions):
if str(request.args.get('all')).lower() in [
'true', 'yes', 'on'
]:
data.append({
'id': 'all',
'value': gettext('All Depths')
})
for idx, value in enumerate(np.round(ds.depths)):
data.append({'id': idx, 'value': "%d m" % (value)})
if len(data) > 0:
data.insert(0, {
'id': 'bottom',
'value': gettext('Bottom')
})
data = [e for i, e in enumerate(data) if data.index(e) == i]
resp = jsonify(data)
return resp
def test_get_dataset_misc(self, m):
m.return_value = {
"dataset": {
"url": "the_url",
"attribution": "My attribution <b>bold</b>",
"climatology": "climatology_url",
"cache": 5,
}
}
self.assertEqual(util.get_dataset_url("dataset"), "the_url")
self.assertEqual(
util.get_dataset_climatology("dataset"), "climatology_url")
self.assertEqual(
util.get_dataset_attribution("dataset"), "My attribution bold")
self.assertEqual(util.get_dataset_cache("dataset"), 5)
m.return_value = {
"dataset2": {
}
}
self.assertEqual(util.get_dataset_cache("dataset2"), None)
def get_values(self, area_info, dataset_name, variables):
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if self.time is None or (type(self.time) == str and
len(self.time) == 0):
time = -1
else:
time = int(self.time)
if time < 0:
time += len(dataset.timestamps)
time = np.clip(time, 0, len(dataset.timestamps) - 1)
depth = 0
depthm = 0
if self.depth:
if self.depth == 'bottom':
depth = 'bottom'
depthm = 'Bottom'
if len(self.depth) > 0 and \
self.depth != 'bottom':
depth = int(self.depth)
depth = np.clip(depth, 0, len(dataset.depths) - 1)
depthm = dataset.depths[depth]
lat, lon = np.meshgrid(
np.linspace(area_info.bounds[0], area_info.bounds[2], 50),
area_info.spaced_points
)
output_fmtstr = "%6.5g"
for v_idx, v in enumerate(variables):
var = dataset.variables[v]
variable_name = get_variable_name(dataset_name, var)
variable_unit = get_variable_unit(dataset_name, var)
scale_factor = get_variable_scale_factor(dataset_name, var)
lat, lon, d = dataset.get_raw_point(
lat.ravel(),
lon.ravel(),
depth,
time,
v
)
if scale_factor != 1.0:
d = np.multiply(d, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = d - 273.15
lon[np.where(lon > 180)] -= 360
if len(var.dimensions) == 3:
variable_depth = ""
elif depth == 'bottom':
variable_depth = "(@ Bottom)"
else:
variable_depth = "(@%d m)" % np.round(depthm)
points = [Point(p) for p in zip(lat.ravel(), lon.ravel())]
for i, a in enumerate(area_info.area_query):
indices = np.where(
map(
lambda p, poly=area_info.area_polys[i]: poly.contains(p),
points
)
)
selection = np.ma.array(d.ravel()[indices])
if len(selection) > 0 and not selection.mask.all():
area_info.output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': output_fmtstr % (
np.ma.amin(selection).astype(float)
),
'max': output_fmtstr % (
np.ma.amax(selection).astype(float)
),
'mean': output_fmtstr % (
np.ma.mean(selection).astype(float)
),
'median': output_fmtstr % (
np.ma.median(selection).astype(float)
),
'stddev': output_fmtstr % (
np.ma.std(selection).astype(float)
),
'num': "%d" % selection.count(),
})
else:
area_info.output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': gettext("No Data"),
'max': gettext("No Data"),
'mean': gettext("No Data"),
'median': gettext("No Data"),
'stddev': gettext("No Data"),
'num': "0",
})
ClientError(gettext("there are no datapoints in the area you selected. \
you may have selected a area on land or you may \
have an ara that is smallenough to fit between \
the datapoints try selection a different area or \
a larger area"))
area_info.stats = area_info.output
return
raise ServerError(gettext("An Error has occurred. When opening the dataset. \
Please try again or try a different dataset. \
If you would like to report this error please \
contact [email protected]"))
def stats(dataset_name, query):
variables = query.get('variable')
if isinstance(variables, str) or isinstance(variables, unicode):
variables = variables.split(',')
variables = [re.sub('_anom$', '', v) for v in variables]
area = query.get('area')
names = None
data = None
names = []
all_rings = []
for idx, a in enumerate(area):
if isinstance(a, str) or isinstance(a, unicode):
a = a.encode("utf-8")
sp = a.split('/', 1)
if data is None:
data = list_areas(sp[0])
b = [x for x in data if x.get('key') == a]
a = b[0]
area[idx] = a
rings = [LinearRing(p) for p in a['polygons']]
if len(rings) > 1:
u = cascaded_union(rings)
else:
u = rings[0]
all_rings.append(u.envelope)
if a.get('name'):
names.append(a.get('name'))
names = sorted(names)
if len(all_rings) > 1:
combined = cascaded_union(all_rings)
else:
combined = all_rings[0]
combined = combined.envelope
bounds = combined.bounds
area_polys = []
output = []
for a in area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Polygon(r, inners))
area_polys.append(MultiPolygon(polygons))
output.append({
'name': a.get('name'),
'variables': [],
})
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if query.get('time') is None or (type(query.get('time')) == str and
len(query.get('time')) == 0):
time = -1
else:
time = int(query.get('time'))
if time < 0:
time += len(dataset.timestamps)
time = np.clip(time, 0, len(dataset.timestamps) - 1)
depth = 0
depthm = 0
if query.get('depth'):
if query.get('depth') == 'bottom':
depth = 'bottom'
depthm = 'Bottom'
if len(query.get('depth')) > 0 and \
query.get('depth') != 'bottom':
depth = int(query.get('depth'))
depth = np.clip(depth, 0, len(dataset.depths) - 1)
depthm = dataset.depths[depth]
lat, lon = np.meshgrid(
np.linspace(bounds[0], bounds[2], 50),
np.linspace(bounds[1], bounds[3], 50)
)
output_fmtstr = "%6.5g"
for v_idx, v in enumerate(variables):
var = dataset.variables[v]
variable_name = get_variable_name(dataset_name, var)
variable_unit = get_variable_unit(dataset_name, var)
scale_factor = get_variable_scale_factor(dataset_name, var)
lat, lon, d = dataset.get_raw_point(
lat.ravel(),
lon.ravel(),
depth,
time,
v
)
if scale_factor != 1.0:
d = np.multiply(d, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
d = d - 273.15
lon[np.where(lon > 180)] -= 360
if len(var.dimensions) == 3:
variable_depth = ""
elif depth == 'bottom':
variable_depth = "(@ Bottom)"
else:
variable_depth = "(@%d m)" % np.round(depthm)
points = [Point(p) for p in zip(lat.ravel(), lon.ravel())]
for i, a in enumerate(area):
indices = np.where(
map(
lambda p, poly=area_polys[i]: poly.contains(p),
points
)
)
selection = np.ma.array(d.ravel()[indices])
if len(selection) > 0 and not selection.mask.all():
output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': output_fmtstr % (
np.ma.amin(selection).astype(float)
),
'max': output_fmtstr % (
np.ma.amax(selection).astype(float)
),
'mean': output_fmtstr % (
np.ma.mean(selection).astype(float)
),
'median': output_fmtstr % (
np.ma.median(selection).astype(float)
),
'stddev': output_fmtstr % (
np.ma.std(selection).astype(float)
),
'num': "%d" % selection.count(),
})
else:
output[i]['variables'].append({
'name': ("%s %s" % (variable_name,
variable_depth)).strip(),
'unit': variable_unit,
'min': gettext("No Data"),
'max': gettext("No Data"),
'mean': gettext("No Data"),
'median': gettext("No Data"),
'stddev': gettext("No Data"),
'num': "0",
})
return json.dumps(sorted(output, key=itemgetter('name')))
def subset_query(output_format, dataset_name, variables, min_range, max_range,
time, should_zip):
# Bounding box extents
bottom_left = [float(x) for x in min_range.split(',')]
top_right = [float(x) for x in max_range.split(',')]
# Time range
time_range = [int(x) for x in time.split(',')]
apply_time_range = False
if time_range[0] != time_range[1]:
apply_time_range = True
# Ensure we have an output folder that will be cleaned by tmpreaper
if not os.path.isdir("/tmp/subset"):
os.makedirs("/tmp/subset")
working_dir = "/tmp/subset/"
with xr.open_dataset(get_dataset_url(dataset_name)) as dataset:
# Finds a variable in a dictionary given a substring containing common characters
# Not a fool-proof method but I want to avoid regex because I hate it.
#
# dataset: opened xarray dataset to search
# substring: common characters to find
def find_variable(dataset, substring):
variable_list = dataset.variables.keys()
for key in variable_list:
if substring in key:
return key
# Get lat/lon variable names from dataset (since they all differ >.>)
lat = find_variable(dataset, "lat")
lon = find_variable(dataset, "lon")
# Find closest indices in dataset corresponding to each calculated point
# riops used "latitude" and "longitude"
ymin_index, xmin_index = find_nearest_grid_point(
bottom_left[0], bottom_left[1], dataset, lat, lon)
ymax_index, xmax_index = find_nearest_grid_point(
top_right[0], top_right[1], dataset, lat, lon)
# Get nicely formatted bearings
p0 = geopy.Point(bottom_left)
p1 = geopy.Point(top_right)
# Get timestamp
time_variable = find_variable(dataset, "time")
timestamp = str(
format_date(
pandas.to_datetime(dataset[time_variable][int(
time_range[0])].values), "yyyyMMdd"))
if apply_time_range:
endtimestamp = "-" + str(
format_date(
pandas.to_datetime(dataset[time_variable][int(
time_range[1])].values), "yyyyMMdd"))
else:
endtimestamp = ""
# Do subsetting
if "riops" in dataset_name:
# Riops has different coordinate names...why? ¯\_(ツ)_/¯
subset = dataset.isel(yc=slice(ymin_index, ymax_index),
xc=slice(xmin_index, xmax_index))
elif dataset_name == "giops_forecast":
subset = dataset.isel(latitude=slice(ymin_index, ymax_index),
longitude=slice(xmin_index, xmax_index))
else:
subset = dataset.isel(y=slice(ymin_index, ymax_index),
x=slice(xmin_index, xmax_index))
# Select requested time (time range if applicable)
if apply_time_range:
subset = subset.isel(**{
time_variable:
slice(time_range[0], time_range[1] + 1)
}) # slice doesn't include the last element
else:
subset = subset.isel(**{
time_variable:
slice(int(time_range[0]),
int(time_range[0]) + 1)
})
# Filter out unwanted variables
output_vars = variables.split(',')
output_vars.extend(
[find_variable(dataset, 'depth'), time_variable, lat,
lon]) # Keep the coordinate variables
for variable in subset.data_vars:
if variable not in output_vars:
subset = subset.drop(variable)
filename = dataset_name + "_" + "%dN%dW-%dN%dW" % (p0.latitude, p0.longitude, p1.latitude, p1.longitude) \
+ "_" + timestamp + endtimestamp + "_" + output_format
# Export to NetCDF
if output_format == "NETCDF3_NC":
# "Special" netcdf export (┛ಠ_ಠ)┛彡┻━┻
# This part only ever runs on giops files so no need to worry about variable
# names changing
subset.to_netcdf(working_dir + filename + ".nc",
format="NETCDF3_CLASSIC")
# Open the GIOPS NCOM file
giops_file = netCDF4.Dataset(working_dir + filename + ".nc")
giops_variables = giops_file.variables
# Create converted ncdf file
ds = netCDF4.Dataset(working_dir + filename + "_converted.nc",
'w',
format="NETCDF3_CLASSIC")
ds.description = "Converted GIOPS " + filename
ds.history = "Created: " + str(datetime.datetime.now())
ds.source = "www.navigator.oceansdata.ca | GIOPS source: dd.weather.gc.ca"
# Find correct variable names in subset
lat_var = find_variable(subset, 'lat')
lon_var = find_variable(subset, 'lon')
time_var = find_variable(subset, 'time')
depth_var = find_variable(subset, 'depth')
# Create the netcdf dimensions
ds.createDimension('lat', len(giops_variables[lat_var][:]))
ds.createDimension('lon', len(giops_variables[lon_var][:]))
ds.createDimension('time', len(giops_variables[time_var][:]))
ds.CreateDimension('depth', len(giops_variables[depth_var][:]))
# Create the netcdf variables and assign the values
latitudes = ds.createVariable('lat', 'f', ('lat', ))
longitudes = ds.createVariable('lon', 'f', ('lon', ))
latitudes = giops_variables[lat_var][:]
longitudes = giops_variables[lon_var][:]
times = ds.createVariable('time', 'i', ('time', ))
# Convert time from seconds to hours
for i in range(0, len(giops_variables[time_var])):
times[i] = giops_variables[time_var][i] / 3600
# Variable Attributes, mimicking HYCOM headers
latitudes.long_name = "Latitude"
latitudes.units = "degrees_north"
latitudes.NAVO_code = 1
longitudes.long_name = "Longitude"
longitudes.units = "degrees_east"
longitudes.NAVO_code = 2
times.long_name = "Validity time"
times.units = "hours since 1950-01-01 00:00:00"
times.time_origin = "1950-01-01 00:00:00"
levels = ds.createVariable('depth', 'i', ('depth', ))
levels = giops_variables[depth_var][:]
levels.long_name = "Depth"
levels.units = "meter"
levels.positive = "down"
levels.NAVO_code = 5
for variable in giops_variables:
if variable == "vosaline":
salinity = ds.createVariable('salinity',
'f', (
'time',
'depth',
'lat',
'lon',
),
fill_value=-30000.0)
salinity = giops_variables['vosaline'][:]
salinity.long_name = "Salinity"
salinity.units = "psu"
salinity.valid_min = 0.0
salinity.valid_max = 45.0
salinity.NAVO_code = 16
if variable == "votemper":
temp = ds.createVariable('water_temp',
'f', (
'time',
'depth',
'lat',
'lon',
),
fill_value=-30000.0)
# Convert from Kelvin to Celcius
for i in range(0, len(giops_variables['depth'][:])):
temp[:, i, :, :] = giops_variables[
'votemper'][:, i, :, :] - 273.15
temp.valid_min = -100.0
temp.valid_max = 100.0
temp.NAVO_code = 15
if variable == "sossheig":
height = ds.createVariable('surf_el',
float, ('time', 'lat', 'lon'),
fill_value=-30000.0)
height = giops_variables['sossheig'][:]
heights.long_name = "Water Surface Elevation"
heights.units = "meter"
heights.NAVO_code = 32
if variable == "vozocrtx":
x_velo = ds.createVariable('water_u',
float,
('time', 'depth', 'lat', 'lon'),
fill_value=-30000.0)
x_velo = giops_variables['vozocrtx'][:]
x_velo.long_name = "Eastward Water Velocity"
x_velo.units = "meter/sec"
x_velo.NAVO_code = 17
if variable == "vomecrty":
y_velo = ds.createVariable('water_v',
float,
('time', 'depth', 'lat', 'lon'),
fill_value=-30000.0)
y_velo = giops_variables['vomecrty'][:]
y_velo.long_name = "Northward Water Velocity"
y_velo.units = "meter/sec"
y_velo.NAVO_code = 18
ds.close()
giops_file.close()
# (┛ಠ_ಠ)┛彡┻━┻
return send_from_directory(working_dir,
'%s_converted.nc' % filename,
as_attachment=True)
else:
# Save subset normally
subset.to_netcdf(working_dir + filename + ".nc",
format=output_format)
if should_zip == 1:
myzip = zipfile.ZipFile('%s%s.zip' % (working_dir, filename), mode='w')
myzip.write('%s%s.nc' % (working_dir, filename),
os.path.basename('%s%s.nc' % (working_dir, filename)))
myzip.comment = 'Generated from www.navigator.oceansdata.ca'
myzip.close() # Must be called to actually create zip
return send_from_directory(working_dir,
'%s.zip' % filename,
as_attachment=True)
return send_from_directory(working_dir,
'%s.nc' % filename,
as_attachment=True)
def vars_query():
data = []
if 'dataset' in request.args:
dataset = request.args['dataset']
if get_dataset_climatology(dataset) != "" and 'anom' in request.args:
with open_dataset(get_dataset_climatology(dataset)) as ds:
climatology_variables = map(str, ds.variables)
else:
climatology_variables = []
three_d = '3d_only' in request.args
with open_dataset(get_dataset_url(dataset)) as ds:
if 'vectors_only' not in request.args:
for v in ds.variables:
if ('time_counter' in v.dimensions or
'time' in v.dimensions) \
and ('y' in v.dimensions or
'yc' in v.dimensions or
'node' in v.dimensions or
'nele' in v.dimensions or
'latitude' in v.dimensions or
'lat' in v.dimensions):
if three_d and not (set(ds.depth_dimensions)
& set(v.dimensions)):
continue
else:
if not is_variable_hidden(dataset, v):
data.append({
'id':
v.key,
'value':
get_variable_name(dataset, v),
'scale':
get_variable_scale(dataset, v)
})
if v.key in climatology_variables:
data.append({
'id':
v.key + "_anom",
'value':
get_variable_name(dataset, v) +
" Anomaly",
'scale': [-10, 10]
})
VECTOR_MAP = {
'vozocrtx':
'vozocrtx,vomecrty',
'itzocrtx':
'itzocrtx,itmecrty',
'iicevelu':
'iicevelu,iicevelv',
'u_wind':
'u_wind,v_wind',
'u':
'u,v',
'ua':
'ua,va',
'u-component_of_wind_height_above_ground':
'u-component_of_wind_height_above_ground,v-component_of_wind_height_above_ground'
}
if 'vectors' in request.args or 'vectors_only' in request.args:
rxp = r"(?i)( x | y |zonal |meridional |northward |eastward)"
for key, value in VECTOR_MAP.iteritems():
if key in ds.variables:
n = get_variable_name(dataset, ds.variables[key])
data.append({
'id':
value,
'value':
re.sub(r" +", " ", re.sub(rxp, " ", n)),
'scale': [
0,
get_variable_scale(dataset,
ds.variables[key])[1]
]
})
data = sorted(data, key=lambda k: k['value'])
resp = jsonify(data)
return resp
def load_data(self):
if self.projection == 'EPSG:32661':
blat = min(self.bounds[0], self.bounds[2])
blat = 5 * np.floor(blat / 5)
self.basemap = basemap.load_map('npstere', (blat, 0), None, None)
elif self.projection == 'EPSG:3031':
blat = max(self.bounds[0], self.bounds[2])
blat = 5 * np.ceil(blat / 5)
self.basemap = basemap.load_map('spstere', (blat, 180), None, None)
else:
distance = VincentyDistance()
height = distance.measure(
(self.bounds[0], self.centroid[1]),
(self.bounds[2], self.centroid[1])) * 1000 * 1.25
width = distance.measure(
(self.centroid[0], self.bounds[1]),
(self.centroid[0], self.bounds[3])) * 1000 * 1.25
self.basemap = basemap.load_map('lcc', self.centroid, height,
width)
if self.basemap.aspect < 1:
gridx = 500
gridy = int(500 * self.basemap.aspect)
else:
gridy = 500
gridx = int(500 / self.basemap.aspect)
self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.variable_unit = self.get_variable_units(
dataset, self.variables)[0]
self.variable_name = self.get_variable_names(
dataset, self.variables)[0]
scale_factor = self.get_variable_scale_factors(
dataset, self.variables)[0]
if self.cmap is None:
if len(self.variables) == 1:
self.cmap = colormap.find_colormap(self.variable_name)
else:
self.cmap = colormap.colormaps.get('speed')
if len(self.variables) == 2:
self.variable_name = self.vector_name(self.variable_name)
if self.depth == 'bottom':
depth_value = 'Bottom'
else:
self.depth = np.clip(int(self.depth), 0,
len(dataset.depths) - 1)
depth_value = dataset.depths[self.depth]
data = []
allvars = []
for v in self.variables:
var = dataset.variables[v]
allvars.append(v)
if self.filetype in ['csv', 'odv', 'txt']:
d, depth_value = dataset.get_area(np.array(
[self.latitude, self.longitude]),
self.depth,
self.time,
v,
return_depth=True)
else:
d = dataset.get_area(
np.array([self.latitude, self.longitude]), self.depth,
self.time, v)
d = np.multiply(d, scale_factor)
self.variable_unit, d = self.kelvin_to_celsius(
self.variable_unit, d)
data.append(d)
if self.filetype not in ['csv', 'odv', 'txt']:
if len(var.dimensions) == 3:
self.depth_label = ""
elif self.depth == 'bottom':
self.depth_label = " at Bottom"
else:
self.depth_label = " at " + \
str(int(np.round(depth_value))) + " m"
if len(data) == 2:
data[0] = np.sqrt(data[0]**2 + data[1]**2)
self.data = data[0]
quiver_data = []
if self.quiver is not None and \
self.quiver['variable'] != '' and \
self.quiver['variable'] != 'none':
for v in self.quiver['variable'].split(','):
allvars.append(v)
var = dataset.variables[v]
quiver_unit = get_variable_unit(self.dataset_name, var)
quiver_name = get_variable_name(self.dataset_name, var)
quiver_lon, quiver_lat = self.basemap.makegrid(50, 50)
d = dataset.get_area(np.array([quiver_lat, quiver_lon]),
self.depth, self.time, v)
quiver_data.append(d)
self.quiver_name = self.vector_name(quiver_name)
self.quiver_longitude = quiver_lon
self.quiver_latitude = quiver_lat
self.quiver_unit = quiver_unit
self.quiver_data = quiver_data
if all(
map(lambda v: len(dataset.variables[v].dimensions) == 3,
allvars)):
self.depth = 0
contour_data = []
if self.contour is not None and \
self.contour['variable'] != '' and \
self.contour['variable'] != 'none':
d = dataset.get_area(np.array([self.latitude,
self.longitude]), self.depth,
self.time, self.contour['variable'])
contour_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_name = get_variable_name(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_unit, d = self.kelvin_to_celsius(contour_unit, d)
d = np.multiply(d, contour_factor)
contour_data.append(d)
self.contour_unit = contour_unit
self.contour_name = contour_name
self.contour_data = contour_data
self.timestamp = dataset.timestamps[self.time]
if self.variables != self.variables_anom:
self.variable_name += " Anomaly"
with open_dataset(get_dataset_climatology(
self.dataset_name)) as dataset:
data = []
for v in self.variables:
var = dataset.variables[v]
d = dataset.get_area(
np.array([self.latitude, self.longitude]), self.depth,
self.timestamp.month - 1, v)
data.append(d)
if len(data) == 2:
data = np.sqrt(data[0]**2 + data[1]**2)
else:
data = data[0]
u, data = self.kelvin_to_celsius(
dataset.variables[self.variables[0]].unit, data)
self.data -= data
# Load bathymetry data
self.bathymetry = overlays.bathymetry(self.basemap,
self.latitude,
self.longitude,
blur=2)
if self.depth != 'bottom' and self.depth != 0:
if len(quiver_data) > 0:
quiver_bathymetry = overlays.bathymetry(
self.basemap, quiver_lat, quiver_lon)
self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked
for d in self.quiver_data:
d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked
for d in self.contour_data:
d[np.where(self.bathymetry < depth_value)] = np.ma.masked
else:
mask = maskoceans(self.longitude, self.latitude, self.data).mask
self.data[~mask] = np.ma.masked
for d in self.quiver_data:
mask = maskoceans(self.quiver_longitude, self.quiver_latitude,
d).mask
d[~mask] = np.ma.masked
for d in contour_data:
mask = maskoceans(self.longitude, self.latitude, d).mask
d[~mask] = np.ma.masked
if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']:
area_polys = []
for a in self.area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Poly(r, inners))
area_polys.append(MultiPolygon(polygons))
points = [
Point(p)
for p in zip(self.latitude.ravel(), self.longitude.ravel())
]
indicies = []
for a in area_polys:
indicies.append(
np.where(map(lambda p, poly=a: poly.contains(p),
points))[0])
indicies = np.unique(np.array(indicies).ravel())
newmask = np.ones(self.data.shape, dtype=bool)
newmask[np.unravel_index(indicies, newmask.shape)] = False
self.data.mask |= newmask
self.depth_value = depth_value
def load_data(self):
if self.projection == 'EPSG:32661':
blat = min(self.bounds[0], self.bounds[2])
blat = 5 * np.floor(blat / 5)
self.basemap = basemap.load_map('npstere', (blat, 0), None, None)
elif self.projection == 'EPSG:3031':
blat = max(self.bounds[0], self.bounds[2])
blat = 5 * np.ceil(blat / 5)
self.basemap = basemap.load_map('spstere', (blat, 180), None, None)
else:
distance = VincentyDistance()
height = distance.measure(
(self.bounds[0], self.centroid[1]),
(self.bounds[2], self.centroid[1])
) * 1000 * 1.25
width = distance.measure(
(self.centroid[0], self.bounds[1]),
(self.centroid[0], self.bounds[3])
) * 1000 * 1.25
self.basemap = basemap.load_map(
'lcc', self.centroid, height, width
)
if self.basemap.aspect < 1:
gridx = 500
gridy = int(500 * self.basemap.aspect)
else:
gridy = 500
gridx = int(500 / self.basemap.aspect)
self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.variable_unit = self.get_variable_units(
dataset, self.variables
)[0]
self.variable_name = self.get_variable_names(
dataset,
self.variables
)[0]
scale_factor = self.get_variable_scale_factors(
dataset, self.variables
)[0]
if self.cmap is None:
if len(self.variables) == 1:
self.cmap = colormap.find_colormap(self.variable_name)
else:
self.cmap = colormap.colormaps.get('speed')
if len(self.variables) == 2:
self.variable_name = self.vector_name(self.variable_name)
if self.depth == 'bottom':
depth_value = 'Bottom'
else:
self.depth = np.clip(
int(self.depth), 0, len(dataset.depths) - 1)
depth_value = dataset.depths[self.depth]
data = []
allvars = []
for v in self.variables:
var = dataset.variables[v]
allvars.append(v)
if self.filetype in ['csv', 'odv', 'txt']:
d, depth_value = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
v,
return_depth=True
)
else:
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
v
)
d = np.multiply(d, scale_factor)
self.variable_unit, d = self.kelvin_to_celsius(
self.variable_unit, d)
data.append(d)
if self.filetype not in ['csv', 'odv', 'txt']:
if len(var.dimensions) == 3:
self.depth_label = ""
elif self.depth == 'bottom':
self.depth_label = " at Bottom"
else:
self.depth_label = " at " + \
str(int(np.round(depth_value))) + " m"
if len(data) == 2:
data[0] = np.sqrt(data[0] ** 2 + data[1] ** 2)
self.data = data[0]
quiver_data = []
if self.quiver is not None and \
self.quiver['variable'] != '' and \
self.quiver['variable'] != 'none':
for v in self.quiver['variable'].split(','):
allvars.append(v)
var = dataset.variables[v]
quiver_unit = get_variable_unit(self.dataset_name, var)
quiver_name = get_variable_name(self.dataset_name, var)
quiver_lon, quiver_lat = self.basemap.makegrid(50, 50)
d = dataset.get_area(
np.array([quiver_lat, quiver_lon]),
self.depth,
self.time,
v
)
quiver_data.append(d)
self.quiver_name = self.vector_name(quiver_name)
self.quiver_longitude = quiver_lon
self.quiver_latitude = quiver_lat
self.quiver_unit = quiver_unit
self.quiver_data = quiver_data
if all(map(lambda v: len(dataset.variables[v].dimensions) == 3,
allvars)):
self.depth = 0
contour_data = []
if self.contour is not None and \
self.contour['variable'] != '' and \
self.contour['variable'] != 'none':
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
self.contour['variable']
)
contour_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_name = get_variable_name(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_unit, d = self.kelvin_to_celsius(contour_unit, d)
d = np.multiply(d, contour_factor)
contour_data.append(d)
self.contour_unit = contour_unit
self.contour_name = contour_name
self.contour_data = contour_data
self.timestamp = dataset.timestamps[self.time]
if self.variables != self.variables_anom:
self.variable_name += " Anomaly"
with open_dataset(
get_dataset_climatology(self.dataset_name)
) as dataset:
data = []
for v in self.variables:
var = dataset.variables[v]
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.timestamp.month - 1,
v
)
data.append(d)
if len(data) == 2:
data = np.sqrt(data[0] ** 2 + data[1] ** 2)
else:
data = data[0]
u, data = self.kelvin_to_celsius(
dataset.variables[self.variables[0]].unit,
data)
self.data -= data
# Load bathymetry data
self.bathymetry = overlays.bathymetry(
self.basemap,
self.latitude,
self.longitude,
blur=2
)
if self.depth != 'bottom' and self.depth != 0:
if len(quiver_data) > 0:
quiver_bathymetry = overlays.bathymetry(
self.basemap, quiver_lat, quiver_lon)
self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked
for d in self.quiver_data:
d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked
for d in self.contour_data:
d[np.where(self.bathymetry < depth_value)] = np.ma.masked
else:
mask = maskoceans(self.longitude, self.latitude, self.data).mask
self.data[~mask] = np.ma.masked
for d in self.quiver_data:
mask = maskoceans(
self.quiver_longitude, self.quiver_latitude, d).mask
d[~mask] = np.ma.masked
for d in contour_data:
mask = maskoceans(self.longitude, self.latitude, d).mask
d[~mask] = np.ma.masked
if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']:
area_polys = []
for a in self.area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Poly(r, inners))
area_polys.append(MultiPolygon(polygons))
points = [Point(p) for p in zip(self.latitude.ravel(),
self.longitude.ravel())]
indicies = []
for a in area_polys:
indicies.append(np.where(
map(
lambda p, poly=a: poly.contains(p),
points
)
)[0])
indicies = np.unique(np.array(indicies).ravel())
newmask = np.ones(self.data.shape, dtype=bool)
newmask[np.unravel_index(indicies, newmask.shape)] = False
self.data.mask |= newmask
self.depth_value = depth_value
def load_data(self):
ds_url = app.config['DRIFTER_URL']
data_names = []
data_units = []
with Dataset(ds_url % self.drifter, 'r') as ds:
self.name = ds.buoyid
self.imei = str(chartostring(ds['imei'][0]))
self.wmo = str(chartostring(ds['wmo'][0]))
t = netcdftime.utime(ds['data_date'].units)
d = []
for v in self.buoyvariables:
d.append(ds[v][:])
if "long_name" in ds[v].ncattrs():
data_names.append(ds[v].long_name)
else:
data_names.append(v)
if "units" in ds[v].ncattrs():
data_units.append(ds[v].units)
else:
data_units.append(None)
self.data = d
self.times = t.num2date(ds['data_date'][:])
self.points = np.array([
ds['latitude'][:],
ds['longitude'][:],
]).transpose()
data_names = data_names[:len(self.buoyvariables)]
data_units = data_units[:len(self.buoyvariables)]
for i, t in enumerate(self.times):
if t.tzinfo is None:
self.times[i] = t.replace(tzinfo=pytz.UTC)
self.data_names = data_names
self.data_units = data_units
if self.starttime is not None:
d = dateutil.parser.parse(self.starttime)
self.start = np.where(self.times >= d)[0].min()
else:
self.start = 0
if self.endtime is not None:
d = dateutil.parser.parse(self.endtime)
self.end = np.where(self.times <= d)[0].max() + 1
else:
self.end = len(self.times) - 1
if self.start < 0:
self.start += len(self.times)
self.start = np.clip(self.start, 0, len(self.times) - 1)
if self.end < 0:
self.end += len(self.times)
self.end = np.clip(self.end, 0, len(self.times) - 1)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
depth = int(self.depth)
try:
model_start = np.where(
dataset.timestamps <= self.times[self.start]
)[0][-1]
except IndexError:
model_start = 0
model_start -= 1
model_start = np.clip(model_start, 0, len(dataset.timestamps) - 1)
try:
model_end = np.where(
dataset.timestamps >= self.times[self.end]
)[0][0]
except IndexError:
model_end = len(dataset.timestamps) - 1
model_end += 1
model_end = np.clip(
model_end,
model_start,
len(dataset.timestamps) - 1
)
model_times = map(
lambda t: time.mktime(t.timetuple()),
dataset.timestamps[model_start:model_end + 1]
)
output_times = map(
lambda t: time.mktime(t.timetuple()),
self.times[self.start:self.end + 1]
)
d = []
for v in self.variables:
pts, dist, mt, md = dataset.get_path(
self.points[self.start:self.end + 1],
depth,
range(model_start, model_end + 1),
v,
times=output_times
)
f = interp1d(
model_times,
md,
assume_sorted=True,
bounds_error=False,
)
d.append(np.diag(f(mt)))
model_data = np.ma.array(d)
variable_names = []
variable_units = []
scale_factors = []
for v in self.variables:
variable_units.append(get_variable_unit(self.dataset_name,
dataset.variables[v]))
variable_names.append(get_variable_name(self.dataset_name,
dataset.variables[v]))
scale_factors.append(
get_variable_scale_factor(self.dataset_name,
dataset.variables[v])
)
for idx, sf in enumerate(scale_factors):
model_data[idx, :] = np.multiply(model_data[idx, :], sf)
for idx, u in enumerate(variable_units):
variable_units[idx], model_data[idx, :] = \
self.kelvin_to_celsius(u, model_data[idx, :])
self.model_data = model_data
self.model_times = map(datetime.datetime.utcfromtimestamp, mt)
self.variable_names = variable_names
self.variable_units = variable_units
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
for idx, v in enumerate(self.variables):
var = dataset.variables[v]
if not (set(var.dimensions) & set(dataset.depth_dimensions)):
for potential in dataset.variables:
if potential in self.variables:
continue
pot = dataset.variables[potential]
if (set(pot.dimensions)
& set(dataset.depth_dimensions)):
if len(pot.dimensions) > 3:
self.variables[idx] = potential.key
value = parallel = perpendicular = None
variable_names = self.get_variable_names(dataset, self.variables)
variable_units = self.get_variable_units(dataset, self.variables)
scale_factors = self.get_variable_scale_factors(
dataset, self.variables)
# Load data sent from primary/Left Map
if len(self.variables) > 1:
# Only velocity has 2 variables
v = []
for name in self.variables:
v.append(dataset.variables[name])
distances, times, lat, lon, bearings = geo.path_to_points(
self.points, 100)
transect_pts, distance, x, dep = dataset.get_path_profile(
self.points, time, self.variables[0], 100)
transect_pts, distance, y, dep = dataset.get_path_profile(
self.points, time, self.variables[1], 100)
# Calculate vector components
x = np.multiply(x, scale_factors[0])
y = np.multiply(y, scale_factors[1])
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(y, x) - r
mag = np.sqrt(x**2 + y**2)
parallel = mag * np.cos(theta)
perpendicular = mag * np.sin(theta)
else:
# Get data for one variable
transect_pts, distance, value, dep = dataset.get_path_profile(
self.points, time, self.variables[0])
value = np.multiply(value, scale_factors[0])
# Get variable units and convert to Celsius if needed
variable_units[0], value = self.kelvin_to_celsius(
variable_units[0], value)
if len(self.variables) == 2:
variable_names[0] = self.vector_name(variable_names[0])
# If a colourmap has not been manually specified by the
# Navigator...
if self.cmap is None:
self.cmap = colormap.find_colormap(variable_names[0])
self.timestamp = dataset.timestamps[int(time)]
self.depth = dep
self.depth_unit = "m"
self.transect_data = {
"points": transect_pts,
"distance": distance,
"data": value,
"name": variable_names[0],
"unit": variable_units[0],
"parallel": parallel,
"perpendicular": perpendicular,
}
if self.surface is not None:
surface_pts, surface_dist, t, surface_value = \
dataset.get_path(
self.points,
0,
time,
self.surface,
)
surface_unit = get_variable_unit(
self.dataset_name, dataset.variables[self.surface])
surface_name = get_variable_name(
self.dataset_name, dataset.variables[self.surface])
surface_factor = get_variable_scale_factor(
self.dataset_name, dataset.variables[self.surface])
surface_value = np.multiply(surface_value, surface_factor)
surface_unit, surface_value = self.kelvin_to_celsius(
surface_unit, surface_value)
self.surface_data = {
"points": surface_pts,
"distance": surface_dist,
"data": surface_value,
"name": surface_name,
"unit": surface_unit
}
# Load data sent from Right Map (if in compare mode)
if self.compare:
def interpolate_depths(data, depth_in, depth_out):
output = []
for i in range(0, depth_in.shape[0]):
f = interp1d(
depth_in[i],
data[:, i],
bounds_error=False,
assume_sorted=True,
)
output.append(
f(depth_out[i].view(np.ma.MaskedArray).filled()))
return np.ma.masked_invalid(output).transpose()
with open_dataset(get_dataset_url(
self.compare['dataset'])) as dataset:
# Get and format date
self.compare['date'] = np.clip(self.compare['time'], 0,
len(dataset.timestamps) - 1)
self.compare['date'] = dataset.timestamps[int(
self.compare['date'])]
# 1 variable
if len(self.compare['variables']) == 1:
# Get and store the "nicely formatted" string for the variable name
self.compare['name'] = self.get_variable_names(
dataset, self.compare['variables'])[0]
# Find correct colourmap
if (self.compare['colormap'] == 'default'):
self.compare['colormap'] = colormap.find_colormap(
self.compare['name'])
else:
self.compare['colormap'] = colormap.find_colormap(
self.compare['colormap'])
climate_points, climate_distance, climate_data, cdep = \
dataset.get_path_profile(self.points,
self.compare['time'],
self.compare['variables'][0])
# Get variable units and convert to Celsius if needed
self.compare['unit'], climate_data = self.kelvin_to_celsius(
dataset.variables[self.compare['variables'][0]].unit,
climate_data)
self.__fill_invalid_shift(climate_data)
if (self.depth.shape != cdep.shape) or \
(self.depth != cdep).any():
# Need to interpolate the depths
climate_data = interpolate_depths(
climate_data, cdep, self.depth)
if self.transect_data['data'] is None:
self.transect_data['parallel'] -= climate_data
self.transect_data['perpendicular'] -= climate_data
else:
self.transect_data['compare_data'] = climate_data
# Velocity variables
else:
# Get and store the "nicely formatted" string for the variable name
self.compare['name'] = self.get_variable_names(
dataset, self.compare['variables'])[0]
climate_pts, climate_distance, climate_x, cdep = \
dataset.get_path_profile(
self.points,
self.compare['time'],
self.compare['variables'][0],
100
)
climate_pts, climate_distance, climate_y, cdep = \
dataset.get_path_profile(
self.points,
self.compare['time'],
self.compare['variables'][0],
100
)
climate_distances, ctimes, clat, clon, bearings = \
geo.path_to_points(self.points, 100)
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(climate_y, climate_x) - r
mag = np.sqrt(climate_x**2 + climate_y**2)
if np.all(self.depth != cdep):
theta = interpolate_depths(theta, cdep, self.depth)
self.__fill_invalid_shift(theta)
mag = interpolate_depths(mag, cdep, self.depth)
self.__fill_invalid_shift(mag)
self.compare['parallel'] = mag * np.cos(theta)
self.compare['perpendicular'] = mag * np.sin(theta)
"""
if self.transect_data['parallel'] is None:
self.transect_data['data'] -= mag
else:
self.transect_data['parallel'] -= climate_parallel
self.transect_data['perpendicular'] -= climate_perpendicular
"""
# Bathymetry
with Dataset(app.config['BATHYMETRY_FILE'], 'r') as dataset:
bath_x, bath_y = bathymetry(dataset.variables['y'],
dataset.variables['x'],
dataset.variables['z'], self.points)
self.bathymetry = {'x': bath_x, 'y': bath_y}
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
for idx, v in enumerate(self.variables):
var = dataset.variables[v]
if not (set(var.dimensions) & set(dataset.depth_dimensions)):
for potential in dataset.variables:
if potential in self.variables:
continue
pot = dataset.variables[potential]
if (set(pot.dimensions) &
set(dataset.depth_dimensions)):
if len(pot.shape) > 3:
self.variables[idx] = potential
self.variables_anom[idx] = potential
value = parallel = perpendicular = None
variable_names = self.get_variable_names(dataset, self.variables)
variable_units = self.get_variable_units(dataset, self.variables)
scale_factors = self.get_variable_scale_factors(dataset,
self.variables)
if len(self.variables) > 1:
v = []
for name in self.variables:
v.append(dataset.variables[name])
distances, times, lat, lon, bearings = geo.path_to_points(
self.points, 100
)
transect_pts, distance, x, dep = dataset.get_path_profile(
self.points, time, self.variables[0], 100)
transect_pts, distance, y, dep = dataset.get_path_profile(
self.points, time, self.variables[1], 100)
x = np.multiply(x, scale_factors[0])
y = np.multiply(y, scale_factors[1])
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(y, x) - r
mag = np.sqrt(x ** 2 + y ** 2)
parallel = mag * np.cos(theta)
perpendicular = mag * np.sin(theta)
else:
transect_pts, distance, value, dep = dataset.get_path_profile(
self.points, time, self.variables[0])
value = np.multiply(value, scale_factors[0])
variable_units[0], value = self.kelvin_to_celsius(
variable_units[0],
value
)
if len(self.variables) == 2:
variable_names[0] = self.vector_name(variable_names[0])
if self.cmap is None:
self.cmap = colormap.find_colormap(variable_names[0])
self.timestamp = dataset.timestamps[int(time)]
self.depth = dep
self.depth_unit = "m"
self.transect_data = {
"points": transect_pts,
"distance": distance,
"data": value,
"name": variable_names[0],
"unit": variable_units[0],
"parallel": parallel,
"perpendicular": perpendicular,
}
if self.surface is not None:
surface_pts, surface_dist, t, surface_value = \
dataset.get_path(
self.points,
0,
time,
self.surface,
)
surface_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.surface]
)
surface_name = get_variable_name(
self.dataset_name,
dataset.variables[self.surface]
)
surface_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.surface]
)
surface_value = np.multiply(surface_value, surface_factor)
surface_unit, surface_value = self.kelvin_to_celsius(
surface_unit,
surface_value
)
self.surface_data = {
"points": surface_pts,
"distance": surface_dist,
"data": surface_value,
"name": surface_name,
"unit": surface_unit
}
if self.variables != self.variables_anom:
with open_dataset(
get_dataset_climatology(self.dataset_name)
) as dataset:
if self.variables[0] in dataset.variables:
if len(self.variables) == 1:
climate_points, climate_distance, climate_data = \
dataset.get_path_profile(self.points,
self.timestamp.month - 1,
self.variables[0])
u, climate_data = self.kelvin_to_celsius(
dataset.variables[self.variables[0]].unit,
climate_data
)
self.transect_data['data'] -= - climate_data
else:
climate_pts, climate_distance, climate_x, cdep = \
dataset.get_path_profile(
self.points,
self.timestamp.month - 1,
self.variables[0],
100
)
climate_pts, climate_distance, climate_y, cdep = \
dataset.get_path_profile(
self.points,
self.timestamp.month - 1,
self.variables[0],
100
)
climate_distances, ctimes, clat, clon, bearings = \
geo.path_to_points(self.points, 100)
r = np.radians(np.subtract(90, bearings))
theta = np.arctan2(y, x) - r
mag = np.sqrt(x ** 2 + y ** 2)
climate_parallel = mag * np.cos(theta)
climate_perpendicular = mag * np.sin(theta)
self.transect_data['parallel'] -= climate_parallel
self.transect_data[
'perpendicular'] -= climate_perpendicular
# Bathymetry
with Dataset(app.config['BATHYMETRY_FILE'], 'r') as dataset:
bath_x, bath_y = bathymetry(
dataset.variables['y'],
dataset.variables['x'],
dataset.variables['z'],
self.points)
self.bathymetry = {
'x': bath_x,
'y': bath_y
}
def load_data(self):
ds_url = app.config['DRIFTER_URL']
data_names = []
data_units = []
with Dataset(ds_url % self.drifter, 'r') as ds:
self.name = ds.buoyid
self.imei = str(chartostring(ds['imei'][0]))
self.wmo = str(chartostring(ds['wmo'][0]))
t = netcdftime.utime(ds['data_date'].units)
d = []
for v in self.buoyvariables:
d.append(ds[v][:])
if "long_name" in ds[v].ncattrs():
data_names.append(ds[v].long_name)
else:
data_names.append(v)
if "units" in ds[v].ncattrs():
data_units.append(ds[v].units)
else:
data_units.append(None)
self.data = d
self.times = t.num2date(ds['data_date'][:])
self.points = np.array([
ds['latitude'][:],
ds['longitude'][:],
]).transpose()
data_names = data_names[:len(self.buoyvariables)]
data_units = data_units[:len(self.buoyvariables)]
for i, t in enumerate(self.times):
if t.tzinfo is None:
self.times[i] = t.replace(tzinfo=pytz.UTC)
self.data_names = data_names
self.data_units = data_units
if self.starttime is not None:
d = dateutil.parser.parse(self.starttime)
self.start = np.where(self.times >= d)[0].min()
else:
self.start = 0
if self.endtime is not None:
d = dateutil.parser.parse(self.endtime)
self.end = np.where(self.times <= d)[0].max() + 1
else:
self.end = len(self.times) - 1
if self.start < 0:
self.start += len(self.times)
self.start = np.clip(self.start, 0, len(self.times) - 1)
if self.end < 0:
self.end += len(self.times)
self.end = np.clip(self.end, 0, len(self.times) - 1)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
depth = int(self.depth)
try:
model_start = np.where(
dataset.timestamps <= self.times[self.start])[0][-1]
except IndexError:
model_start = 0
model_start -= 1
model_start = np.clip(model_start, 0, len(dataset.timestamps) - 1)
try:
model_end = np.where(
dataset.timestamps >= self.times[self.end])[0][0]
except IndexError:
model_end = len(dataset.timestamps) - 1
model_end += 1
model_end = np.clip(model_end, model_start,
len(dataset.timestamps) - 1)
model_times = map(lambda t: time.mktime(t.timetuple()),
dataset.timestamps[model_start:model_end + 1])
output_times = map(lambda t: time.mktime(t.timetuple()),
self.times[self.start:self.end + 1])
d = []
for v in self.variables:
pts, dist, mt, md = dataset.get_path(
self.points[self.start:self.end + 1],
depth,
range(model_start, model_end + 1),
v,
times=output_times)
f = interp1d(
model_times,
md,
assume_sorted=True,
bounds_error=False,
)
d.append(np.diag(f(mt)))
model_data = np.ma.array(d)
variable_names = []
variable_units = []
scale_factors = []
for v in self.variables:
variable_units.append(
get_variable_unit(self.dataset_name, dataset.variables[v]))
variable_names.append(
get_variable_name(self.dataset_name, dataset.variables[v]))
scale_factors.append(
get_variable_scale_factor(self.dataset_name,
dataset.variables[v]))
for idx, sf in enumerate(scale_factors):
model_data[idx, :] = np.multiply(model_data[idx, :], sf)
for idx, u in enumerate(variable_units):
variable_units[idx], model_data[idx, :] = \
self.kelvin_to_celsius(u, model_data[idx, :])
self.model_data = model_data
self.model_times = map(datetime.datetime.utcfromtimestamp, mt)
self.variable_names = variable_names
self.variable_units = variable_units
def load_data(self):
distance = VincentyDistance()
height = distance.measure(
(self.bounds[0], self.centroid[1]),
(self.bounds[2], self.centroid[1])) * 1000 * 1.25
width = distance.measure(
(self.centroid[0], self.bounds[1]),
(self.centroid[0], self.bounds[3])) * 1000 * 1.25
if self.projection == 'EPSG:32661':
near_pole, covers_pole = self.pole_proximity(self.points[0])
blat = min(self.bounds[0], self.bounds[2])
blat = 5 * np.floor(blat / 5)
if self.centroid[0] > 80 or near_pole or covers_pole:
self.basemap = basemap.load_map(
'npstere', self.centroid, height, width,
min(self.bounds[0], self.bounds[2]))
else:
self.basemap = basemap.load_map('lcc', self.centroid, height,
width)
elif self.projection == 'EPSG:3031':
near_pole, covers_pole = self.pole_proximity(self.points[0])
blat = max(self.bounds[0], self.bounds[2])
blat = 5 * np.ceil(blat / 5)
if ((self.centroid[0] < -80 or self.bounds[1] < -80
or self.bounds[3] < -80)
or covers_pole): # is centerered close to the south pole
self.basemap = basemap.load_map('spstere', (blat, 180), height,
width)
else:
self.basemap = basemap.load_map(
'lcc', self.centroid, height, width,
max(self.bounds[0], self.bounds[2]))
elif abs(self.centroid[1] - self.bounds[1]) > 90:
height_bounds = [self.bounds[0], self.bounds[2]]
width_bounds = [self.bounds[1], self.bounds[3]]
height_buffer = (abs(height_bounds[1] - height_bounds[0])) * 0.1
width_buffer = (abs(width_bounds[0] - width_bounds[1])) * 0.1
if abs(width_bounds[1] - width_bounds[0]) > 360:
raise ClientError(
gettext(
"You have requested an area that exceads the width of the world. \
Thinking big is good but plots need to be less the 360 deg wide."
))
if height_bounds[1] < 0:
height_bounds[1] = height_bounds[1] + height_buffer
else:
height_bounds[1] = height_bounds[1] + height_buffer
if height_bounds[0] < 0:
height_bounds[0] = height_bounds[0] - height_buffer
else:
height_bounds[0] = height_bounds[0] - height_buffer
new_width_bounds = []
new_width_bounds.append(width_bounds[0] - width_buffer)
new_width_bounds.append(width_bounds[1] + width_buffer)
if abs(new_width_bounds[1] - new_width_bounds[0]) > 360:
width_buffer = np.floor(
(360 - abs(width_bounds[1] - width_bounds[0])) / 2)
new_width_bounds[0] = width_bounds[0] - width_buffer
new_width_bounds[1] = width_bounds[1] + width_buffer
if new_width_bounds[0] < -360:
new_width_bounds[0] = -360
if new_width_bounds[1] > 720:
new_width_bounds[1] = 720
self.basemap = basemap.load_map(
'merc', self.centroid, (height_bounds[0], height_bounds[1]),
(new_width_bounds[0], new_width_bounds[1]))
else:
self.basemap = basemap.load_map('lcc', self.centroid, height,
width)
if self.basemap.aspect < 1:
gridx = 500
gridy = int(500 * self.basemap.aspect)
else:
gridy = 500
gridx = int(500 / self.basemap.aspect)
self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
if self.time < 0:
self.time += len(dataset.timestamps)
self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1)
self.variable_unit = self.get_variable_units(
dataset, self.variables)[0]
self.variable_name = self.get_variable_names(
dataset, self.variables)[0]
scale_factor = self.get_variable_scale_factors(
dataset, self.variables)[0]
if self.cmap is None:
if len(self.variables) == 1:
self.cmap = colormap.find_colormap(self.variable_name)
else:
self.cmap = colormap.colormaps.get('speed')
if len(self.variables) == 2:
self.variable_name = self.vector_name(self.variable_name)
if self.depth == 'bottom':
depth_value = 'Bottom'
else:
self.depth = np.clip(int(self.depth), 0,
len(dataset.depths) - 1)
depth_value = dataset.depths[self.depth]
data = []
allvars = []
for v in self.variables:
var = dataset.variables[v]
allvars.append(v)
if self.filetype in ['csv', 'odv', 'txt']:
d, depth_value = dataset.get_area(np.array(
[self.latitude, self.longitude]),
self.depth,
self.time,
v,
self.interp,
self.radius,
self.neighbours,
return_depth=True)
else:
d = dataset.get_area(
np.array([self.latitude,
self.longitude]), self.depth, self.time, v,
self.interp, self.radius, self.neighbours)
d = np.multiply(d, scale_factor)
self.variable_unit, d = self.kelvin_to_celsius(
self.variable_unit, d)
data.append(d)
if self.filetype not in ['csv', 'odv', 'txt']:
if len(var.dimensions) == 3:
self.depth_label = ""
elif self.depth == 'bottom':
self.depth_label = " at Bottom"
else:
self.depth_label = " at " + \
str(int(np.round(depth_value))) + " m"
if len(data) == 2:
data[0] = np.sqrt(data[0]**2 + data[1]**2)
self.data = data[0]
quiver_data = []
# Store the quiver data on the same grid as the main variable. This
# will only be used for CSV export.
quiver_data_fullgrid = []
if self.quiver is not None and \
self.quiver['variable'] != '' and \
self.quiver['variable'] != 'none':
for v in self.quiver['variable'].split(','):
allvars.append(v)
var = dataset.variables[v]
quiver_unit = get_variable_unit(self.dataset_name, var)
quiver_name = get_variable_name(self.dataset_name, var)
quiver_lon, quiver_lat = self.basemap.makegrid(50, 50)
d = dataset.get_area(
np.array([quiver_lat, quiver_lon]),
self.depth,
self.time,
v,
self.interp,
self.radius,
self.neighbours,
)
quiver_data.append(d)
# Get the quiver data on the same grid as the main
# variable.
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
v,
self.interp,
self.radius,
self.neighbours,
)
quiver_data_fullgrid.append(d)
self.quiver_name = self.vector_name(quiver_name)
self.quiver_longitude = quiver_lon
self.quiver_latitude = quiver_lat
self.quiver_unit = quiver_unit
self.quiver_data = quiver_data
self.quiver_data_fullgrid = quiver_data_fullgrid
if all(
map(lambda v: len(dataset.variables[v].dimensions) == 3,
allvars)):
self.depth = 0
contour_data = []
if self.contour is not None and \
self.contour['variable'] != '' and \
self.contour['variable'] != 'none':
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.depth,
self.time,
self.contour['variable'],
self.interp,
self.radius,
self.neighbours,
)
contour_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_name = get_variable_name(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_factor = get_variable_scale_factor(
self.dataset_name,
dataset.variables[self.contour['variable']])
contour_unit, d = self.kelvin_to_celsius(contour_unit, d)
d = np.multiply(d, contour_factor)
contour_data.append(d)
self.contour_unit = contour_unit
self.contour_name = contour_name
self.contour_data = contour_data
self.timestamp = dataset.timestamps[self.time]
if self.compare:
self.variable_name += " Difference"
with open_dataset(get_dataset_url(
self.compare['dataset'])) as dataset:
data = []
for v in self.compare['variables']:
var = dataset.variables[v]
d = dataset.get_area(
np.array([self.latitude, self.longitude]),
self.compare['depth'],
self.compare['time'],
v,
self.interp,
self.radius,
self.neighbours,
)
data.append(d)
if len(data) == 2:
data = np.sqrt(data[0]**2 + data[1]**2)
else:
data = data[0]
u, data = self.kelvin_to_celsius(
dataset.variables[self.compare['variables'][0]].unit, data)
self.data -= data
# Load bathymetry data
self.bathymetry = overlays.bathymetry(self.basemap,
self.latitude,
self.longitude,
blur=2)
if self.depth != 'bottom' and self.depth != 0:
if len(quiver_data) > 0:
quiver_bathymetry = overlays.bathymetry(
self.basemap, quiver_lat, quiver_lon)
self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked
for d in self.quiver_data:
d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked
for d in self.contour_data:
d[np.where(self.bathymetry < depth_value)] = np.ma.masked
else:
mask = maskoceans(self.longitude, self.latitude, self.data).mask
self.data[~mask] = np.ma.masked
for d in self.quiver_data:
mask = maskoceans(self.quiver_longitude, self.quiver_latitude,
d).mask
d[~mask] = np.ma.masked
for d in contour_data:
mask = maskoceans(self.longitude, self.latitude, d).mask
d[~mask] = np.ma.masked
if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']:
area_polys = []
for a in self.area:
rings = [LinearRing(p) for p in a['polygons']]
innerrings = [LinearRing(p) for p in a['innerrings']]
polygons = []
for r in rings:
inners = []
for ir in innerrings:
if r.contains(ir):
inners.append(ir)
polygons.append(Poly(r, inners))
area_polys.append(MultiPolygon(polygons))
points = [
Point(p)
for p in zip(self.latitude.ravel(), self.longitude.ravel())
]
indicies = []
for a in area_polys:
indicies.append(
np.where(map(lambda p, poly=a: poly.contains(p),
points))[0])
indicies = np.unique(np.array(indicies).ravel())
newmask = np.ones(self.data.shape, dtype=bool)
newmask[np.unravel_index(indicies, newmask.shape)] = False
self.data.mask |= newmask
self.depth_value = depth_value
def plot(projection, x, y, z, args):
lat, lon = get_latlon_coords(projection, x, y, z)
if len(lat.shape) == 1:
lat, lon = np.meshgrid(lat, lon)
dataset_name = args.get('dataset')
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
depth = args.get('depth')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
data = []
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if args.get('time') is None or (type(args.get('time')) == str
and len(args.get('time')) == 0):
time = -1
else:
time = int(args.get('time'))
t_len = len(dataset.timestamps)
while time >= t_len:
time -= t_len
while time < 0:
time += len(dataset.timestamps)
timestamp = dataset.timestamps[time]
for v in variable:
data.append(dataset.get_area(np.array([lat, lon]), depth, time, v))
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
scale_factor = get_variable_scale_factor(
dataset_name, dataset.variables[variable[0]])
if anom:
cmap = colormap.colormaps['anomaly']
else:
cmap = colormap.find_colormap(variable_name)
if depth != 'bottom':
depthm = dataset.depths[depth]
else:
depthm = 0
if scale_factor != 1.0:
for idx, val in enumerate(data):
data[idx] = np.multiply(val, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
for idx, val in enumerate(data):
data[idx] = np.add(val, -273.15)
if len(data) == 1:
data = data[0]
if len(data) == 2:
data = np.sqrt(data[0]**2 + data[1]**2)
if not anom:
cmap = colormap.colormaps.get('speed')
if anom:
with open_dataset(get_dataset_climatology(dataset_name)) as dataset:
a = dataset.get_area(np.array([lat, lon]), depth,
timestamp.month - 1, v)
data = data - a
f, fname = tempfile.mkstemp()
os.close(f)
data = data.transpose()
xpx = x * 256
ypx = y * 256
with Dataset(ETOPO_FILE % (projection, z), 'r') as dataset:
bathymetry = dataset["z"][ypx:(ypx + 256), xpx:(xpx + 256)]
bathymetry = gaussian_filter(bathymetry, 0.5)
data[np.where(bathymetry > -depthm)] = np.ma.masked
sm = matplotlib.cm.ScalarMappable(matplotlib.colors.Normalize(
vmin=scale[0], vmax=scale[1]),
cmap=cmap)
img = sm.to_rgba(np.squeeze(data))
im = Image.fromarray((img * 255.0).astype(np.uint8))
im.save(fname, format='png', optimize=True)
with open(fname, 'r') as f:
buf = f.read()
os.remove(fname)
return buf
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
self.load_misc(dataset, self.variables)
self.fix_startend_times(dataset)
self.variable_unit = get_variable_unit(
self.dataset_name, dataset.variables[self.variables[0]])
self.variable_name = get_variable_name(
self.dataset_name, dataset.variables[self.variables[0]])
var = self.variables[0]
if self.depth != 'all' and self.depth != 'bottom' and \
(set(dataset.variables[var].dimensions) &
set(dataset.depth_dimensions)):
self.depth_label = " at %d m" % (np.round(
dataset.depths[self.depth]))
elif self.depth == 'bottom':
self.depth_label = ' at Bottom'
else:
self.depth_label = ''
if not (set(dataset.variables[var].dimensions)
& set(dataset.depth_dimensions)):
self.depth = 0
times = None
point_data = []
for p in self.points:
data = []
for v in self.variables:
if self.depth == 'all':
d, dep = dataset.get_timeseries_profile(
float(p[0]), float(p[1]), self.starttime,
self.endtime, v)
else:
d, dep = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
self.depth,
self.starttime,
self.endtime,
v,
return_depth=True)
data.append(d)
point_data.append(np.ma.array(data))
point_data = np.ma.array(point_data)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
times = dataset.timestamps[self.starttime:self.endtime + 1]
if self.query.get('dataset_quantum') == 'month':
times = [datetime.date(x.year, x.month, 1) for x in times]
# depths = dataset.depths
depths = dep
# TODO: pint
if self.variable_unit.startswith("Kelvin"):
self.variable_unit = "Celsius"
for idx, v in enumerate(self.variables):
point_data[:, idx, :] = point_data[:, idx, :] - 273.15
if point_data.shape[1] == 2:
point_data = np.ma.expand_dims(
np.sqrt(point_data[:, 0, :]**2 + point_data[:, 1, :]**2), 1)
self.times = times
self.data = point_data
self.depths = depths
self.depth_unit = "m"
def load_data(self):
if isinstance(self.observation[0], numbers.Number):
self.observation_variable_names = []
self.observation_variable_units = []
with Dataset(app.config["OBSERVATION_AGG_URL"], 'r') as ds:
t = netcdftime.utime(ds['time'].units)
for idx, o in enumerate(self.observation):
observation = {}
ts = t.num2date(ds['time'][o]).replace(tzinfo=pytz.UTC)
observation['time'] = ts.isoformat()
observation['longitude'] = ds['lon'][o]
observation['latitude'] = ds['lat'][o]
observation['depth'] = ds['z'][:]
observation['depthunit'] = ds['z'].units
observation['datatypes'] = []
data = []
for v in sorted(ds.variables):
if v in ['z', 'lat', 'lon', 'profile', 'time']:
continue
var = ds[v]
if var.datatype == '|S1':
continue
observation['datatypes'].append("%s [%s]" % (
var.long_name,
var.units
))
data.append(var[o, :])
if idx == 0:
self.observation_variable_names.append(
var.long_name)
self.observation_variable_units.append(var.units)
observation['data'] = np.ma.array(data).transpose()
self.observation[idx] = observation
self.points = map(lambda o: [o['latitude'], o['longitude']],
self.observation)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
ts = dataset.timestamps
observation_times = []
timestamps = []
for o in self.observation:
observation_time = dateutil.parser.parse(o['time'])
observation_times.append(observation_time)
deltas = [
(x.replace(tzinfo=pytz.UTC) -
observation_time).total_seconds()
for x in ts]
time = np.abs(deltas).argmin()
timestamp = ts[time]
timestamps.append(timestamp)
self.load_misc(dataset, self.variables)
point_data, self.depths = self.get_data(
dataset, self.variables, time)
point_data = np.ma.array(point_data)
point_data = self.apply_scale_factors(point_data)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units,
point_data
)
self.data = self.subtract_climatology(point_data, timestamp)
self.observation_time = observation_time
self.observation_times = observation_times
self.timestamps = timestamps
self.timestamp = timestamp
def load_data(self):
"""
Calculates and returns the depth, depth-value, and depth unit from a given dataset
Args:
depth: Stored depth information (self.depth or self.compare['depth'])
clip_length: How many depth values to clip (usually len(dataset.depths) - 1)
dataset: Opened dataset
Returns:
(depth, depth_value, depth_unit)
"""
def find_depth(depth, clip_length, dataset):
depth_value = 0
depth_unit = "m"
if depth:
if depth == 'bottom':
depth_value = 'Bottom'
depth_unit = ''
return (depth, depth_value, depth_unit)
else:
depth = np.clip(int(depth), 0, clip_length)
depth_value = np.round(dataset.depths[depth])
depth_unit = "m"
return (depth, depth_value, depth_unit)
return (depth, depth_value, depth_unit)
# Load left/Main Map
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
latvar, lonvar = utils.get_latlon_vars(dataset)
self.depth, self.depth_value, self.depth_unit = find_depth(
self.depth,
len(dataset.depths) - 1, dataset)
self.fix_startend_times(dataset, self.starttime, self.endtime)
time = range(self.starttime, self.endtime + 1)
if len(self.variables) > 1:
v = []
for name in self.variables:
self.path_points, self.distance, t, value = dataset.get_path(
self.points, self.depth, time, name)
v.append(value**2)
value = np.sqrt(np.ma.sum(v, axis=0))
self.variable_name = self.get_variable_names(
dataset, self.variables)[0]
self.variable_name = self.vector_name(self.variable_name)
else:
self.path_points, self.distance, t, value = dataset.get_path(
self.points, self.depth, time, self.variables[0])
self.variable_name = self.get_variable_names(
dataset, self.variables)[0]
variable_units = self.get_variable_units(dataset, self.variables)
scale_factors = self.get_variable_scale_factors(
dataset, self.variables)
self.variable_unit, self.data = self.kelvin_to_celsius(
variable_units[0], value)
self.times = dataset.timestamps[self.starttime:self.endtime + 1]
self.data = np.multiply(self.data, scale_factors[0])
self.data = self.data.transpose()
# Get colourmap
if self.cmap is None:
self.cmap = colormap.find_colormap(self.variable_name)
# Load data sent from Right Map (if in compare mode)
if self.compare:
with open_dataset(get_dataset_url(
self.compare['dataset'])) as dataset:
latvar, lonvar = utils.get_latlon_vars(dataset)
self.compare['depth'], self.compare[
'depth_value'], self.compare['depth_unit'] = find_depth(
self.compare['depth'],
len(dataset.depths) - 1, dataset)
self.fix_startend_times(dataset, self.compare['starttime'],
self.compare['endtime'])
time = range(self.compare['starttime'],
self.compare['endtime'] + 1)
if len(self.compare['variables']) > 1:
v = []
for name in self.compare['variables']:
path, distance, t, value = dataset.get_path(
self.points, self.compare['depth'], time, name)
v.append(value**2)
value = np.sqrt(np.ma.sum(v, axis=0))
self.compare['variable_name'] = self.get_variable_names(
dataset, self.compare['variables'])[0]
self.compare['variable_name'] = self.vector_name(
self.compare['variable_name'])
else:
path, distance, t, value = dataset.get_path(
self.points, self.compare['depth'], time,
self.compare['variables'][0])
self.compare['variable_name'] = self.get_variable_names(
dataset, self.compare['variables'])[0]
# Colourmap
if (self.compare['colormap'] == 'default'):
self.compare['colormap'] = colormap.find_colormap(
self.compare['variable_name'])
else:
self.compare['colormap'] = colormap.find_colormap(
self.compare['colormap'])
variable_units = self.get_variable_units(
dataset, self.compare['variables'])
scale_factors = self.get_variable_scale_factors(
dataset, self.compare['variables'])
self.compare['variable_unit'], self.compare[
'data'] = self.kelvin_to_celsius(variable_units[0], value)
self.compare['times'] = dataset.timestamps[
self.compare['starttime']:self.compare['endtime'] + 1]
self.compare['data'] = np.multiply(self.compare['data'],
scale_factors[0])
self.compare['data'] = self.compare['data'].transpose()
# Comparison over different time ranges makes no sense
if self.starttime != self.compare['starttime'] or\
self.endtime != self.compare['endtime']:
raise ClientError(
gettext(
"Please ensure the Start Time and End Time for the Left and Right maps are identical."
))
def plot(projection, x, y, z, args):
lat, lon = get_latlon_coords(projection, x, y, z)
if len(lat.shape) == 1:
lat, lon = np.meshgrid(lat, lon)
dataset_name = args.get('dataset')
variable = args.get('variable')
if variable.endswith('_anom'):
variable = variable[0:-5]
anom = True
else:
anom = False
variable = variable.split(',')
depth = args.get('depth')
scale = args.get('scale')
scale = [float(component) for component in scale.split(',')]
data = []
with open_dataset(get_dataset_url(dataset_name)) as dataset:
if args.get('time') is None or (type(args.get('time')) == str and
len(args.get('time')) == 0):
time = -1
else:
time = int(args.get('time'))
t_len = len(dataset.timestamps)
while time >= t_len:
time -= t_len
while time < 0:
time += len(dataset.timestamps)
timestamp = dataset.timestamps[time]
for v in variable:
data.append(dataset.get_area(
np.array([lat, lon]),
depth,
time,
v
))
variable_name = get_variable_name(dataset_name,
dataset.variables[variable[0]])
variable_unit = get_variable_unit(dataset_name,
dataset.variables[variable[0]])
scale_factor = get_variable_scale_factor(
dataset_name,
dataset.variables[variable[0]]
)
if anom:
cmap = colormap.colormaps['anomaly']
else:
cmap = colormap.find_colormap(variable_name)
if depth != 'bottom':
depthm = dataset.depths[depth]
else:
depthm = 0
if scale_factor != 1.0:
for idx, val in enumerate(data):
data[idx] = np.multiply(val, scale_factor)
if variable_unit.startswith("Kelvin"):
variable_unit = "Celsius"
for idx, val in enumerate(data):
data[idx] = np.add(val, -273.15)
if len(data) == 1:
data = data[0]
if len(data) == 2:
data = np.sqrt(data[0] ** 2 + data[1] ** 2)
if not anom:
cmap = colormap.colormaps.get('speed')
if anom:
with open_dataset(get_dataset_climatology(dataset_name)) as dataset:
a = dataset.get_area(
np.array([lat, lon]),
depth,
timestamp.month - 1,
v
)
data = data - a
f, fname = tempfile.mkstemp()
os.close(f)
data = data.transpose()
xpx = x * 256
ypx = y * 256
with Dataset(ETOPO_FILE % (projection, z), 'r') as dataset:
bathymetry = dataset["z"][ypx:(ypx + 256), xpx:(xpx + 256)]
bathymetry = gaussian_filter(bathymetry, 0.5)
data[np.where(bathymetry > -depthm)] = np.ma.masked
sm = matplotlib.cm.ScalarMappable(
matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1]), cmap=cmap)
img = sm.to_rgba(np.squeeze(data))
im = Image.fromarray((img * 255.0).astype(np.uint8))
im.save(fname, format='png', optimize=True)
with open(fname, 'r') as f:
buf = f.read()
os.remove(fname)
return buf
def load_data(self):
if isinstance(self.observation[0], numbers.Number):
self.observation_variable_names = []
self.observation_variable_units = []
with Dataset(app.config["OBSERVATION_AGG_URL"], 'r') as ds:
t = netcdftime.utime(ds['time'].units)
for idx, o in enumerate(self.observation):
observation = {}
ts = t.num2date(ds['time'][o]).replace(tzinfo=pytz.UTC)
observation['time'] = ts.isoformat()
observation['longitude'] = ds['lon'][o]
observation['latitude'] = ds['lat'][o]
observation['depth'] = ds['z'][:]
observation['depthunit'] = ds['z'].units
observation['datatypes'] = []
data = []
for v in sorted(ds.variables):
if v in ['z', 'lat', 'lon', 'profile', 'time']:
continue
var = ds[v]
if var.datatype == '|S1':
continue
observation['datatypes'].append("%s [%s]" % (
var.long_name,
var.units
))
data.append(var[o, :])
if idx == 0:
self.observation_variable_names.append(
var.long_name)
self.observation_variable_units.append(var.units)
observation['data'] = np.ma.array(data).transpose()
self.observation[idx] = observation
self.points = map(lambda o: [o['latitude'], o['longitude']],
self.observation)
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
ts = dataset.timestamps
observation_times = []
timestamps = []
for o in self.observation:
observation_time = dateutil.parser.parse(o['time'])
observation_times.append(observation_time)
deltas = [
(x.replace(tzinfo=pytz.UTC) -
observation_time).total_seconds()
for x in ts]
time = np.abs(deltas).argmin()
timestamp = ts[time]
timestamps.append(timestamp)
try:
self.load_misc(dataset, self.variables)
except IndexError as e:
raise ClientError(gettext("The selected variable(s) were not found in the dataset. \
Most likely, this variable is a derived product from existing dataset variables. \
Please select another variable.") + str(e))
point_data, self.depths = self.get_data(dataset, self.variables, time)
point_data = np.ma.array(point_data)
point_data = self.apply_scale_factors(point_data)
self.variable_units, point_data = self.kelvin_to_celsius(
self.variable_units,
point_data
)
self.data = point_data
self.observation_time = observation_time
self.observation_times = observation_times
self.timestamps = timestamps
self.timestamp = timestamp
def load_data(self):
with open_dataset(get_dataset_url(self.dataset_name)) as dataset:
self.load_misc(dataset, self.variables)
self.fix_startend_times(dataset)
self.variable_unit = get_variable_unit(
self.dataset_name,
dataset.variables[self.variables[0]]
)
self.variable_name = get_variable_name(
self.dataset_name,
dataset.variables[self.variables[0]]
)
var = self.variables[0]
if self.depth != 'all' and self.depth != 'bottom' and \
(set(dataset.variables[var].dimensions) &
set(dataset.depth_dimensions)):
self.depth_label = " at %d m" % (
np.round(dataset.depths[self.depth])
)
elif self.depth == 'bottom':
self.depth_label = ' at Bottom'
else:
self.depth_label = ''
if not (set(dataset.variables[var].dimensions) &
set(dataset.depth_dimensions)):
self.depth = 0
times = None
point_data = []
for p in self.points:
data = []
for v in self.variables:
if self.depth == 'all':
d, dep = dataset.get_timeseries_profile(
float(p[0]),
float(p[1]),
self.starttime,
self.endtime,
v
)
else:
d, dep = dataset.get_timeseries_point(
float(p[0]),
float(p[1]),
self.depth,
self.starttime,
self.endtime,
v,
return_depth=True
)
data.append(d)
point_data.append(np.ma.array(data))
point_data = np.ma.array(point_data)
for idx, factor in enumerate(self.scale_factors):
if factor != 1.0:
point_data[idx] = np.multiply(point_data[idx], factor)
times = dataset.timestamps[self.starttime:self.endtime + 1]
if self.query.get('dataset_quantum') == 'month':
times = [datetime.date(x.year, x.month, 1) for x in times]
# depths = dataset.depths
depths = dep
# TODO: pint
if self.variable_unit.startswith("Kelvin"):
self.variable_unit = "Celsius"
for idx, v in enumerate(self.variables):
point_data[:, idx, :] = point_data[:, idx, :] - 273.15
if point_data.shape[1] == 2:
point_data = np.ma.expand_dims(
np.sqrt(
point_data[:, 0, :] ** 2 + point_data[:, 1, :] ** 2
), 1
)
self.times = times
self.data = point_data
self.depths = depths
self.depth_unit = "m"
