def timestamp_for_date(old_dataset, date, new_dataset): with open_dataset(get_dataset_url(old_dataset)) as ds: timestamp = ds.timestamps[date] with open_dataset(get_dataset_url(new_dataset)) as ds: timestamps = ds.timestamps diffs = np.vectorize(lambda x: x.total_seconds())(timestamps - timestamp) idx = np.where(diffs <= 0)[0] res = 0 if len(idx) > 0: res = idx.max() return Response(json.dumps(res), status=200, mimetype='application/json')
def time_query(): data = [] if 'dataset' in request.args: dataset = request.args['dataset'] quantum = request.args.get('quantum') with open_dataset(get_dataset_url(dataset)) as ds: for idx, date in enumerate(ds.timestamps): if quantum == 'month': date = datetime.datetime(date.year, date.month, 15) data.append({ 'id': idx, 'value': date.replace(tzinfo=pytz.UTC) }) data = sorted(data, key=lambda k: k['id']) class DateTimeEncoder(json.JSONEncoder): def default(self, o): if isinstance(o, datetime.datetime): return o.isoformat() return json.JSONEncoder.default(self, o) js = json.dumps(data, cls=DateTimeEncoder) resp = Response(js, status=200, mimetype='application/json') return resp
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as d: if self.time < 0: self.time += len(d.timestamps) time = np.clip(self.time, 0, len(d.timestamps) - 1) timestamp = d.timestamps[time] try: self.load_misc(d, self.variables) except IndexError as e: raise ClientError( gettext( "The selected variable(s) were not found in the dataset. \ Most likely, this variable is a derived product from existing dataset variables. \ Please select another variable. ") + str(e)) point_data, point_depths = self.get_data(d, self.variables, time) point_data = self.apply_scale_factors(point_data) self.variable_units, point_data = self.kelvin_to_celsius( self.variable_units, point_data) self.data = self.subtract_other(point_data) self.depths = point_depths self.timestamp = timestamp
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as dataset: latvar, lonvar = utils.get_latlon_vars(dataset) if self.depth: if self.depth == 'bottom': self.depth_value = 'Bottom' self.depth_unit = '' else: self.depth = np.clip(int(self.depth), 0, len(dataset.depths) - 1) self.depth_value = np.round(dataset.depths[self.depth]) self.depth_unit = "m" else: self.depth_value = 0 self.depth_unit = "m" self.fix_startend_times(dataset) time = range(self.starttime, self.endtime + 1) if len(self.variables) > 1: v = [] for name in self.variables: pts, distance, t, value = dataset.get_path( self.points, self.depth, time, name ) v.append(value ** 2) value = np.sqrt(np.ma.sum(v, axis=0)) else: pts, distance, t, value = dataset.get_path( self.points, self.depth, time, self.variables[0] ) self.path_points = pts self.distance = distance variable_names = self.get_variable_names(dataset, self.variables) variable_units = self.get_variable_units(dataset, self.variables) scale_factors = self.get_variable_scale_factors(dataset, self.variables) self.variable_unit, self.data = self.kelvin_to_celsius( variable_units[0], value ) self.data = np.multiply(self.data, scale_factors[0]) self.variable_name = variable_names[0] self.data = self.data.transpose() if self.cmap is None: self.cmap = colormap.find_colormap(self.variable_name) self.times = dataset.timestamps[self.starttime:self.endtime + 1]
def get_scale(dataset, variable, depth, time, projection, extent): x = np.linspace(extent[0], extent[2], 50) y = np.linspace(extent[1], extent[3], 50) xx, yy = np.meshgrid(x, y) dest = Proj(init=projection) lon, lat = dest(xx, yy, inverse=True) variables_anom = variable.split(",") variables = [re.sub('_anom$', '', v) for v in variables_anom] with open_dataset(get_dataset_url(dataset)) as ds: timestamp = ds.timestamps[time] d = ds.get_area( np.array([lat, lon]), depth, time, variables[0] ) if len(variables) > 1: d0 = d d1 = ds.get_area( np.array([lat, lon]), depth, time, variables[1] ) d = np.sqrt(d0 ** 2 + d1 ** 2) variable_unit = get_variable_unit(dataset, ds.variables[variables[0]]) if variable_unit.startswith("Kelvin"): variable_unit = "Celsius" d = np.add(d, -273.15) if variables != variables_anom: with open_dataset(get_dataset_climatology(dataset), 'r') as ds: c = ds.get_area( np.array([lat, lon]), depth, timestamp.month - 1, variables[0] ) if len(variables) > 1: c0 = c c1 = ds.get_area( np.array([lat, lon]), depth, timestamp.month - 1, variables[1] ) c = np.sqrt(c0 ** 2 + c1 ** 2) d = d - c m = max(abs(d.min()), abs(d.max())) return -m, m return d.min(), d.max()
def load_data(self): if not isinstance(self.depth, list): self.depth = [self.depth] self.depth = sorted(self.depth) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: if self.starttime < 0: self.starttime += len(dataset.timestamps) if self.endtime < 0: self.endtime += len(dataset.timestamps) start = np.clip(self.starttime, 0, len(dataset.timestamps) - 1) end = np.clip(self.endtime, 0, len(dataset.timestamps) - 1) timestamp = dataset.timestamps[start:end + 1] self.load_misc(dataset, self.variables) point_data = [] point_depth = [] for p in self.points: data = [] depth = [] for v in self.variables: dd = [] jj = [] for d in self.depth: da, dp = dataset.get_timeseries_point( float(p[0]), float(p[1]), d, start, end, v, return_depth=True ) dd.append(da) jj.append(dp) data.append(np.ma.array(dd)) depth.append(np.ma.array(jj)) point_data.append(np.ma.array(data)) point_depth.append(np.ma.array(depth)) point_data = np.ma.array(point_data) point_depth = np.ma.array(point_depth) for idx, factor in enumerate(self.scale_factors): if factor != 1.0: point_data[idx] = np.multiply(point_data[idx], factor) self.variable_units, point_data = self.kelvin_to_celsius( self.variable_units, point_data ) self.data = self.subtract_climatology(point_data, timestamp) self.data_depth = point_depth self.timestamp = timestamp
def scale(args): dataset_name = args.get('dataset') scale = args.get('scale') scale = [float(component) for component in scale.split(',')] variable = args.get('variable') if variable.endswith('_anom'): variable = variable[0:-5] anom = True else: anom = False variable = variable.split(',') with open_dataset(get_dataset_url(dataset_name)) as dataset: variable_unit = get_variable_unit(dataset_name, dataset.variables[variable[0]]) variable_name = get_variable_name(dataset_name, dataset.variables[variable[0]]) if variable_unit.startswith("Kelvin"): variable_unit = "Celsius" if anom: cmap = colormap.colormaps['anomaly'] variable_name = gettext("%s Anomaly") % variable_name else: cmap = colormap.find_colormap(variable_name) if len(variable) == 2: if not anom: cmap = colormap.colormaps.get('speed') variable_name = re.sub( r"(?i)( x | y |zonal |meridional |northward |eastward )", " ", variable_name) variable_name = re.sub(r" +", " ", variable_name) fig = plt.figure(figsize=(2, 5), dpi=75) ax = fig.add_axes([0.05, 0.05, 0.25, 0.9]) norm = matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1]) formatter = ScalarFormatter() formatter.set_powerlimits((-3, 4)) bar = ColorbarBase(ax, cmap=cmap, norm=norm, orientation='vertical', format=formatter) bar.set_label("%s (%s)" % (variable_name.title(), utils.mathtext(variable_unit))) buf = StringIO() try: plt.savefig(buf, format='png', dpi='figure', transparent=False, bbox_inches='tight', pad_inches=0.05) plt.close(fig) return buf.getvalue() finally: buf.close()
def load_data(self): if not isinstance(self.depth, list): self.depth = [self.depth] self.depth = sorted(self.depth) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: if self.starttime < 0: self.starttime += len(dataset.timestamps) if self.endtime < 0: self.endtime += len(dataset.timestamps) start = np.clip(self.starttime, 0, len(dataset.timestamps) - 1) end = np.clip(self.endtime, 0, len(dataset.timestamps) - 1) timestamp = dataset.timestamps[start:end + 1] self.load_misc(dataset, self.variables) point_data = [] point_depth = [] for p in self.points: data = [] depth = [] for v in self.variables: dd = [] jj = [] for d in self.depth: da, dp = dataset.get_timeseries_point( float(p[0]), float(p[1]), d, start, end, v, return_depth=True) dd.append(da) jj.append(dp) data.append(np.ma.array(dd)) depth.append(np.ma.array(jj)) point_data.append(np.ma.array(data)) point_depth.append(np.ma.array(depth)) point_data = np.ma.array(point_data) point_depth = np.ma.array(point_depth) for idx, factor in enumerate(self.scale_factors): if factor != 1.0: point_data[idx] = np.multiply(point_data[idx], factor) self.variable_units, point_data = self.kelvin_to_celsius( self.variable_units, point_data) self.data = self.subtract_other(point_data) self.data_depth = point_depth self.timestamp = timestamp
def subtract_other(self, data): if self.compare: with Dataset(get_dataset_url(self.compare['dataset']), 'r') as dataset: cli = self.get_data(dataset, self.compare['variables'], self.compare['time']) for idx, v in enumerate(self.variables): data[:, idx, :] = \ data[:, idx, :] - cli[:, idx, :] return data
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as dataset: if self.time < 0: self.time += len(dataset.timestamps) time = np.clip(self.time, 0, len(dataset.timestamps) - 1) self.timestamp = dataset.timestamps[time] self.load_temp_sal(dataset, time) self.variable_units[0], self.temperature = \ super(point.PointPlotter, self).kelvin_to_celsius( self.variable_units[0], self.temperature )
def get_point_data(dataset, variable, time, depth, location): variables_anom = variable.split(",") variables = [re.sub('_anom$', '', v) for v in variables_anom] data = [] names = [] units = [] with open_dataset(get_dataset_url(dataset)) as ds: timestamp = ds.timestamps[time] for v in variables: d = ds.get_point( location[0], location[1], depth, time, v ) variable_name = get_variable_name(dataset, ds.variables[v]) variable_unit = get_variable_unit(dataset, ds.variables[v]) if variable_unit.startswith("Kelvin"): variable_unit = "Celsius" d = np.add(d, -273.15) data.append(d) names.append(variable_name) units.append(variable_unit) if variables != variables_anom: with open_dataset(get_dataset_climatology(dataset)) as ds: for idx, v in enumerate(variables): d = ds.get_point( location[0], location[1], depth, timestamp.month, v ) data[idx] = data[idx] - d names[idx] = names[idx] + " Anomaly" result = { 'value': map(lambda f: '%s' % float('%.4g' % f), data), 'location': map(lambda f: round(f, 4), location), 'name': names, 'units': units, } return result
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as d: if self.time < 0: self.time += len(d.timestamps) time = np.clip(self.time, 0, len(d.timestamps) - 1) timestamp = d.timestamps[time] self.load_misc(d, self.variables) point_data, point_depths = self.get_data(d, self.variables, time) point_data = self.apply_scale_factors(point_data) self.variable_units, point_data = self.kelvin_to_celsius( self.variable_units, point_data) self.data = self.subtract_climatology(point_data, timestamp) self.depths = point_depths self.timestamp = timestamp
def get_scale(dataset, variable, depth, time, projection, extent, interp, radius, neighbours): x = np.linspace(extent[0], extent[2], 50) y = np.linspace(extent[1], extent[3], 50) xx, yy = np.meshgrid(x, y) dest = Proj(init=projection) lon, lat = dest(xx, yy, inverse=True) variables_anom = variable.split(",") variables = [re.sub('_anom$', '', v) for v in variables_anom] with open_dataset(get_dataset_url(dataset)) as ds: timestamp = ds.timestamps[time] d = ds.get_area(np.array([lat, lon]), depth, time, variables[0], interp, radius, neighbours) if len(variables) > 1: d0 = d d1 = ds.get_area(np.array([lat, lon]), depth, time, variables[1], interp, radius, neighbours) d = np.sqrt(d0**2 + d1**2) variable_unit = get_variable_unit(dataset, ds.variables[variables[0]]) if variable_unit.startswith("Kelvin"): variable_unit = "Celsius" d = np.add(d, -273.15) if variables != variables_anom: with open_dataset(get_dataset_climatology(dataset), 'r') as ds: c = ds.get_area(np.array([lat, lon]), depth, timestamp.month - 1, variables[0], interp, radius, neighbours) if len(variables) > 1: c0 = c c1 = ds.get_area(np.array([lat, lon]), depth, timestamp.month - 1, variables[1], interp, radius, neighbours) c = np.sqrt(c0**2 + c1**2) d = d - c m = max(abs(d.min()), abs(d.max())) return -m, m return d.min(), d.max()
def test_get_dataset_misc(self, m): m.return_value = { "dataset": { "url": "the_url", "attribution": "My attribution <b>bold</b>", "climatology": "climatology_url", "cache": 5, } } self.assertEqual(util.get_dataset_url("dataset"), "the_url") self.assertEqual(util.get_dataset_climatology("dataset"), "climatology_url") self.assertEqual(util.get_dataset_attribution("dataset"), "My attribution bold") self.assertEqual(util.get_dataset_cache("dataset"), 5) m.return_value = {"dataset2": {}} self.assertEqual(util.get_dataset_cache("dataset2"), None)
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as d: if self.time < 0: self.time += len(d.timestamps) time = np.clip(self.time, 0, len(d.timestamps) - 1) timestamp = d.timestamps[time] self.load_misc(d, self.variables) point_data, point_depths = self.get_data(d, self.variables, time) point_data = self.apply_scale_factors(point_data) self.variable_units, point_data = self.kelvin_to_celsius( self.variable_units, point_data ) self.data = self.subtract_climatology(point_data, timestamp) self.depths = point_depths self.timestamp = timestamp
def depth(): var = request.args.get('variable') if var is None: raise FAILURE variables = var.split(',') variables = [re.sub('_anom$', '', v) for v in variables] data = [] if 'dataset' in request.args: dataset = request.args['dataset'] with open_dataset(get_dataset_url(dataset)) as ds: for variable in variables: if variable and \ variable in ds.variables and \ set(ds.depth_dimensions) & \ set(ds.variables[variable].dimensions): if str(request.args.get('all')).lower() in [ 'true', 'yes', 'on' ]: data.append({ 'id': 'all', 'value': gettext('All Depths') }) for idx, value in enumerate(np.round(ds.depths)): data.append({'id': idx, 'value': "%d m" % (value)}) if len(data) > 0: data.insert(0, { 'id': 'bottom', 'value': gettext('Bottom') }) data = [e for i, e in enumerate(data) if data.index(e) == i] resp = jsonify(data) return resp
def test_get_dataset_misc(self, m): m.return_value = { "dataset": { "url": "the_url", "attribution": "My attribution <b>bold</b>", "climatology": "climatology_url", "cache": 5, } } self.assertEqual(util.get_dataset_url("dataset"), "the_url") self.assertEqual( util.get_dataset_climatology("dataset"), "climatology_url") self.assertEqual( util.get_dataset_attribution("dataset"), "My attribution bold") self.assertEqual(util.get_dataset_cache("dataset"), 5) m.return_value = { "dataset2": { } } self.assertEqual(util.get_dataset_cache("dataset2"), None)
def get_values(self, area_info, dataset_name, variables): with open_dataset(get_dataset_url(dataset_name)) as dataset: if self.time is None or (type(self.time) == str and len(self.time) == 0): time = -1 else: time = int(self.time) if time < 0: time += len(dataset.timestamps) time = np.clip(time, 0, len(dataset.timestamps) - 1) depth = 0 depthm = 0 if self.depth: if self.depth == 'bottom': depth = 'bottom' depthm = 'Bottom' if len(self.depth) > 0 and \ self.depth != 'bottom': depth = int(self.depth) depth = np.clip(depth, 0, len(dataset.depths) - 1) depthm = dataset.depths[depth] lat, lon = np.meshgrid( np.linspace(area_info.bounds[0], area_info.bounds[2], 50), area_info.spaced_points ) output_fmtstr = "%6.5g" for v_idx, v in enumerate(variables): var = dataset.variables[v] variable_name = get_variable_name(dataset_name, var) variable_unit = get_variable_unit(dataset_name, var) scale_factor = get_variable_scale_factor(dataset_name, var) lat, lon, d = dataset.get_raw_point( lat.ravel(), lon.ravel(), depth, time, v ) if scale_factor != 1.0: d = np.multiply(d, scale_factor) if variable_unit.startswith("Kelvin"): variable_unit = "Celsius" d = d - 273.15 lon[np.where(lon > 180)] -= 360 if len(var.dimensions) == 3: variable_depth = "" elif depth == 'bottom': variable_depth = "(@ Bottom)" else: variable_depth = "(@%d m)" % np.round(depthm) points = [Point(p) for p in zip(lat.ravel(), lon.ravel())] for i, a in enumerate(area_info.area_query): indices = np.where( map( lambda p, poly=area_info.area_polys[i]: poly.contains(p), points ) ) selection = np.ma.array(d.ravel()[indices]) if len(selection) > 0 and not selection.mask.all(): area_info.output[i]['variables'].append({ 'name': ("%s %s" % (variable_name, variable_depth)).strip(), 'unit': variable_unit, 'min': output_fmtstr % ( np.ma.amin(selection).astype(float) ), 'max': output_fmtstr % ( np.ma.amax(selection).astype(float) ), 'mean': output_fmtstr % ( np.ma.mean(selection).astype(float) ), 'median': output_fmtstr % ( np.ma.median(selection).astype(float) ), 'stddev': output_fmtstr % ( np.ma.std(selection).astype(float) ), 'num': "%d" % selection.count(), }) else: area_info.output[i]['variables'].append({ 'name': ("%s %s" % (variable_name, variable_depth)).strip(), 'unit': variable_unit, 'min': gettext("No Data"), 'max': gettext("No Data"), 'mean': gettext("No Data"), 'median': gettext("No Data"), 'stddev': gettext("No Data"), 'num': "0", }) ClientError(gettext("there are no datapoints in the area you selected. \ you may have selected a area on land or you may \ have an ara that is smallenough to fit between \ the datapoints try selection a different area or \ a larger area")) area_info.stats = area_info.output return raise ServerError(gettext("An Error has occurred. When opening the dataset. \ Please try again or try a different dataset. \ If you would like to report this error please \ contact [email protected]"))
def stats(dataset_name, query): variables = query.get('variable') if isinstance(variables, str) or isinstance(variables, unicode): variables = variables.split(',') variables = [re.sub('_anom$', '', v) for v in variables] area = query.get('area') names = None data = None names = [] all_rings = [] for idx, a in enumerate(area): if isinstance(a, str) or isinstance(a, unicode): a = a.encode("utf-8") sp = a.split('/', 1) if data is None: data = list_areas(sp[0]) b = [x for x in data if x.get('key') == a] a = b[0] area[idx] = a rings = [LinearRing(p) for p in a['polygons']] if len(rings) > 1: u = cascaded_union(rings) else: u = rings[0] all_rings.append(u.envelope) if a.get('name'): names.append(a.get('name')) names = sorted(names) if len(all_rings) > 1: combined = cascaded_union(all_rings) else: combined = all_rings[0] combined = combined.envelope bounds = combined.bounds area_polys = [] output = [] for a in area: rings = [LinearRing(p) for p in a['polygons']] innerrings = [LinearRing(p) for p in a['innerrings']] polygons = [] for r in rings: inners = [] for ir in innerrings: if r.contains(ir): inners.append(ir) polygons.append(Polygon(r, inners)) area_polys.append(MultiPolygon(polygons)) output.append({ 'name': a.get('name'), 'variables': [], }) with open_dataset(get_dataset_url(dataset_name)) as dataset: if query.get('time') is None or (type(query.get('time')) == str and len(query.get('time')) == 0): time = -1 else: time = int(query.get('time')) if time < 0: time += len(dataset.timestamps) time = np.clip(time, 0, len(dataset.timestamps) - 1) depth = 0 depthm = 0 if query.get('depth'): if query.get('depth') == 'bottom': depth = 'bottom' depthm = 'Bottom' if len(query.get('depth')) > 0 and \ query.get('depth') != 'bottom': depth = int(query.get('depth')) depth = np.clip(depth, 0, len(dataset.depths) - 1) depthm = dataset.depths[depth] lat, lon = np.meshgrid( np.linspace(bounds[0], bounds[2], 50), np.linspace(bounds[1], bounds[3], 50) ) output_fmtstr = "%6.5g" for v_idx, v in enumerate(variables): var = dataset.variables[v] variable_name = get_variable_name(dataset_name, var) variable_unit = get_variable_unit(dataset_name, var) scale_factor = get_variable_scale_factor(dataset_name, var) lat, lon, d = dataset.get_raw_point( lat.ravel(), lon.ravel(), depth, time, v ) if scale_factor != 1.0: d = np.multiply(d, scale_factor) if variable_unit.startswith("Kelvin"): variable_unit = "Celsius" d = d - 273.15 lon[np.where(lon > 180)] -= 360 if len(var.dimensions) == 3: variable_depth = "" elif depth == 'bottom': variable_depth = "(@ Bottom)" else: variable_depth = "(@%d m)" % np.round(depthm) points = [Point(p) for p in zip(lat.ravel(), lon.ravel())] for i, a in enumerate(area): indices = np.where( map( lambda p, poly=area_polys[i]: poly.contains(p), points ) ) selection = np.ma.array(d.ravel()[indices]) if len(selection) > 0 and not selection.mask.all(): output[i]['variables'].append({ 'name': ("%s %s" % (variable_name, variable_depth)).strip(), 'unit': variable_unit, 'min': output_fmtstr % ( np.ma.amin(selection).astype(float) ), 'max': output_fmtstr % ( np.ma.amax(selection).astype(float) ), 'mean': output_fmtstr % ( np.ma.mean(selection).astype(float) ), 'median': output_fmtstr % ( np.ma.median(selection).astype(float) ), 'stddev': output_fmtstr % ( np.ma.std(selection).astype(float) ), 'num': "%d" % selection.count(), }) else: output[i]['variables'].append({ 'name': ("%s %s" % (variable_name, variable_depth)).strip(), 'unit': variable_unit, 'min': gettext("No Data"), 'max': gettext("No Data"), 'mean': gettext("No Data"), 'median': gettext("No Data"), 'stddev': gettext("No Data"), 'num': "0", }) return json.dumps(sorted(output, key=itemgetter('name')))
def subset_query(output_format, dataset_name, variables, min_range, max_range, time, should_zip): # Bounding box extents bottom_left = [float(x) for x in min_range.split(',')] top_right = [float(x) for x in max_range.split(',')] # Time range time_range = [int(x) for x in time.split(',')] apply_time_range = False if time_range[0] != time_range[1]: apply_time_range = True # Ensure we have an output folder that will be cleaned by tmpreaper if not os.path.isdir("/tmp/subset"): os.makedirs("/tmp/subset") working_dir = "/tmp/subset/" with xr.open_dataset(get_dataset_url(dataset_name)) as dataset: # Finds a variable in a dictionary given a substring containing common characters # Not a fool-proof method but I want to avoid regex because I hate it. # # dataset: opened xarray dataset to search # substring: common characters to find def find_variable(dataset, substring): variable_list = dataset.variables.keys() for key in variable_list: if substring in key: return key # Get lat/lon variable names from dataset (since they all differ >.>) lat = find_variable(dataset, "lat") lon = find_variable(dataset, "lon") # Find closest indices in dataset corresponding to each calculated point # riops used "latitude" and "longitude" ymin_index, xmin_index = find_nearest_grid_point( bottom_left[0], bottom_left[1], dataset, lat, lon) ymax_index, xmax_index = find_nearest_grid_point( top_right[0], top_right[1], dataset, lat, lon) # Get nicely formatted bearings p0 = geopy.Point(bottom_left) p1 = geopy.Point(top_right) # Get timestamp time_variable = find_variable(dataset, "time") timestamp = str( format_date( pandas.to_datetime(dataset[time_variable][int( time_range[0])].values), "yyyyMMdd")) if apply_time_range: endtimestamp = "-" + str( format_date( pandas.to_datetime(dataset[time_variable][int( time_range[1])].values), "yyyyMMdd")) else: endtimestamp = "" # Do subsetting if "riops" in dataset_name: # Riops has different coordinate names...why? ¯\_(ツ)_/¯ subset = dataset.isel(yc=slice(ymin_index, ymax_index), xc=slice(xmin_index, xmax_index)) elif dataset_name == "giops_forecast": subset = dataset.isel(latitude=slice(ymin_index, ymax_index), longitude=slice(xmin_index, xmax_index)) else: subset = dataset.isel(y=slice(ymin_index, ymax_index), x=slice(xmin_index, xmax_index)) # Select requested time (time range if applicable) if apply_time_range: subset = subset.isel(**{ time_variable: slice(time_range[0], time_range[1] + 1) }) # slice doesn't include the last element else: subset = subset.isel(**{ time_variable: slice(int(time_range[0]), int(time_range[0]) + 1) }) # Filter out unwanted variables output_vars = variables.split(',') output_vars.extend( [find_variable(dataset, 'depth'), time_variable, lat, lon]) # Keep the coordinate variables for variable in subset.data_vars: if variable not in output_vars: subset = subset.drop(variable) filename = dataset_name + "_" + "%dN%dW-%dN%dW" % (p0.latitude, p0.longitude, p1.latitude, p1.longitude) \ + "_" + timestamp + endtimestamp + "_" + output_format # Export to NetCDF if output_format == "NETCDF3_NC": # "Special" netcdf export (┛ಠ_ಠ)┛彡┻━┻ # This part only ever runs on giops files so no need to worry about variable # names changing subset.to_netcdf(working_dir + filename + ".nc", format="NETCDF3_CLASSIC") # Open the GIOPS NCOM file giops_file = netCDF4.Dataset(working_dir + filename + ".nc") giops_variables = giops_file.variables # Create converted ncdf file ds = netCDF4.Dataset(working_dir + filename + "_converted.nc", 'w', format="NETCDF3_CLASSIC") ds.description = "Converted GIOPS " + filename ds.history = "Created: " + str(datetime.datetime.now()) ds.source = "www.navigator.oceansdata.ca | GIOPS source: dd.weather.gc.ca" # Find correct variable names in subset lat_var = find_variable(subset, 'lat') lon_var = find_variable(subset, 'lon') time_var = find_variable(subset, 'time') depth_var = find_variable(subset, 'depth') # Create the netcdf dimensions ds.createDimension('lat', len(giops_variables[lat_var][:])) ds.createDimension('lon', len(giops_variables[lon_var][:])) ds.createDimension('time', len(giops_variables[time_var][:])) ds.CreateDimension('depth', len(giops_variables[depth_var][:])) # Create the netcdf variables and assign the values latitudes = ds.createVariable('lat', 'f', ('lat', )) longitudes = ds.createVariable('lon', 'f', ('lon', )) latitudes = giops_variables[lat_var][:] longitudes = giops_variables[lon_var][:] times = ds.createVariable('time', 'i', ('time', )) # Convert time from seconds to hours for i in range(0, len(giops_variables[time_var])): times[i] = giops_variables[time_var][i] / 3600 # Variable Attributes, mimicking HYCOM headers latitudes.long_name = "Latitude" latitudes.units = "degrees_north" latitudes.NAVO_code = 1 longitudes.long_name = "Longitude" longitudes.units = "degrees_east" longitudes.NAVO_code = 2 times.long_name = "Validity time" times.units = "hours since 1950-01-01 00:00:00" times.time_origin = "1950-01-01 00:00:00" levels = ds.createVariable('depth', 'i', ('depth', )) levels = giops_variables[depth_var][:] levels.long_name = "Depth" levels.units = "meter" levels.positive = "down" levels.NAVO_code = 5 for variable in giops_variables: if variable == "vosaline": salinity = ds.createVariable('salinity', 'f', ( 'time', 'depth', 'lat', 'lon', ), fill_value=-30000.0) salinity = giops_variables['vosaline'][:] salinity.long_name = "Salinity" salinity.units = "psu" salinity.valid_min = 0.0 salinity.valid_max = 45.0 salinity.NAVO_code = 16 if variable == "votemper": temp = ds.createVariable('water_temp', 'f', ( 'time', 'depth', 'lat', 'lon', ), fill_value=-30000.0) # Convert from Kelvin to Celcius for i in range(0, len(giops_variables['depth'][:])): temp[:, i, :, :] = giops_variables[ 'votemper'][:, i, :, :] - 273.15 temp.valid_min = -100.0 temp.valid_max = 100.0 temp.NAVO_code = 15 if variable == "sossheig": height = ds.createVariable('surf_el', float, ('time', 'lat', 'lon'), fill_value=-30000.0) height = giops_variables['sossheig'][:] heights.long_name = "Water Surface Elevation" heights.units = "meter" heights.NAVO_code = 32 if variable == "vozocrtx": x_velo = ds.createVariable('water_u', float, ('time', 'depth', 'lat', 'lon'), fill_value=-30000.0) x_velo = giops_variables['vozocrtx'][:] x_velo.long_name = "Eastward Water Velocity" x_velo.units = "meter/sec" x_velo.NAVO_code = 17 if variable == "vomecrty": y_velo = ds.createVariable('water_v', float, ('time', 'depth', 'lat', 'lon'), fill_value=-30000.0) y_velo = giops_variables['vomecrty'][:] y_velo.long_name = "Northward Water Velocity" y_velo.units = "meter/sec" y_velo.NAVO_code = 18 ds.close() giops_file.close() # (┛ಠ_ಠ)┛彡┻━┻ return send_from_directory(working_dir, '%s_converted.nc' % filename, as_attachment=True) else: # Save subset normally subset.to_netcdf(working_dir + filename + ".nc", format=output_format) if should_zip == 1: myzip = zipfile.ZipFile('%s%s.zip' % (working_dir, filename), mode='w') myzip.write('%s%s.nc' % (working_dir, filename), os.path.basename('%s%s.nc' % (working_dir, filename))) myzip.comment = 'Generated from www.navigator.oceansdata.ca' myzip.close() # Must be called to actually create zip return send_from_directory(working_dir, '%s.zip' % filename, as_attachment=True) return send_from_directory(working_dir, '%s.nc' % filename, as_attachment=True)
def vars_query(): data = [] if 'dataset' in request.args: dataset = request.args['dataset'] if get_dataset_climatology(dataset) != "" and 'anom' in request.args: with open_dataset(get_dataset_climatology(dataset)) as ds: climatology_variables = map(str, ds.variables) else: climatology_variables = [] three_d = '3d_only' in request.args with open_dataset(get_dataset_url(dataset)) as ds: if 'vectors_only' not in request.args: for v in ds.variables: if ('time_counter' in v.dimensions or 'time' in v.dimensions) \ and ('y' in v.dimensions or 'yc' in v.dimensions or 'node' in v.dimensions or 'nele' in v.dimensions or 'latitude' in v.dimensions or 'lat' in v.dimensions): if three_d and not (set(ds.depth_dimensions) & set(v.dimensions)): continue else: if not is_variable_hidden(dataset, v): data.append({ 'id': v.key, 'value': get_variable_name(dataset, v), 'scale': get_variable_scale(dataset, v) }) if v.key in climatology_variables: data.append({ 'id': v.key + "_anom", 'value': get_variable_name(dataset, v) + " Anomaly", 'scale': [-10, 10] }) VECTOR_MAP = { 'vozocrtx': 'vozocrtx,vomecrty', 'itzocrtx': 'itzocrtx,itmecrty', 'iicevelu': 'iicevelu,iicevelv', 'u_wind': 'u_wind,v_wind', 'u': 'u,v', 'ua': 'ua,va', 'u-component_of_wind_height_above_ground': 'u-component_of_wind_height_above_ground,v-component_of_wind_height_above_ground' } if 'vectors' in request.args or 'vectors_only' in request.args: rxp = r"(?i)( x | y |zonal |meridional |northward |eastward)" for key, value in VECTOR_MAP.iteritems(): if key in ds.variables: n = get_variable_name(dataset, ds.variables[key]) data.append({ 'id': value, 'value': re.sub(r" +", " ", re.sub(rxp, " ", n)), 'scale': [ 0, get_variable_scale(dataset, ds.variables[key])[1] ] }) data = sorted(data, key=lambda k: k['value']) resp = jsonify(data) return resp
def load_data(self): if self.projection == 'EPSG:32661': blat = min(self.bounds[0], self.bounds[2]) blat = 5 * np.floor(blat / 5) self.basemap = basemap.load_map('npstere', (blat, 0), None, None) elif self.projection == 'EPSG:3031': blat = max(self.bounds[0], self.bounds[2]) blat = 5 * np.ceil(blat / 5) self.basemap = basemap.load_map('spstere', (blat, 180), None, None) else: distance = VincentyDistance() height = distance.measure( (self.bounds[0], self.centroid[1]), (self.bounds[2], self.centroid[1])) * 1000 * 1.25 width = distance.measure( (self.centroid[0], self.bounds[1]), (self.centroid[0], self.bounds[3])) * 1000 * 1.25 self.basemap = basemap.load_map('lcc', self.centroid, height, width) if self.basemap.aspect < 1: gridx = 500 gridy = int(500 * self.basemap.aspect) else: gridy = 500 gridx = int(500 / self.basemap.aspect) self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: if self.time < 0: self.time += len(dataset.timestamps) self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1) self.variable_unit = self.get_variable_units( dataset, self.variables)[0] self.variable_name = self.get_variable_names( dataset, self.variables)[0] scale_factor = self.get_variable_scale_factors( dataset, self.variables)[0] if self.cmap is None: if len(self.variables) == 1: self.cmap = colormap.find_colormap(self.variable_name) else: self.cmap = colormap.colormaps.get('speed') if len(self.variables) == 2: self.variable_name = self.vector_name(self.variable_name) if self.depth == 'bottom': depth_value = 'Bottom' else: self.depth = np.clip(int(self.depth), 0, len(dataset.depths) - 1) depth_value = dataset.depths[self.depth] data = [] allvars = [] for v in self.variables: var = dataset.variables[v] allvars.append(v) if self.filetype in ['csv', 'odv', 'txt']: d, depth_value = dataset.get_area(np.array( [self.latitude, self.longitude]), self.depth, self.time, v, return_depth=True) else: d = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.time, v) d = np.multiply(d, scale_factor) self.variable_unit, d = self.kelvin_to_celsius( self.variable_unit, d) data.append(d) if self.filetype not in ['csv', 'odv', 'txt']: if len(var.dimensions) == 3: self.depth_label = "" elif self.depth == 'bottom': self.depth_label = " at Bottom" else: self.depth_label = " at " + \ str(int(np.round(depth_value))) + " m" if len(data) == 2: data[0] = np.sqrt(data[0]**2 + data[1]**2) self.data = data[0] quiver_data = [] if self.quiver is not None and \ self.quiver['variable'] != '' and \ self.quiver['variable'] != 'none': for v in self.quiver['variable'].split(','): allvars.append(v) var = dataset.variables[v] quiver_unit = get_variable_unit(self.dataset_name, var) quiver_name = get_variable_name(self.dataset_name, var) quiver_lon, quiver_lat = self.basemap.makegrid(50, 50) d = dataset.get_area(np.array([quiver_lat, quiver_lon]), self.depth, self.time, v) quiver_data.append(d) self.quiver_name = self.vector_name(quiver_name) self.quiver_longitude = quiver_lon self.quiver_latitude = quiver_lat self.quiver_unit = quiver_unit self.quiver_data = quiver_data if all( map(lambda v: len(dataset.variables[v].dimensions) == 3, allvars)): self.depth = 0 contour_data = [] if self.contour is not None and \ self.contour['variable'] != '' and \ self.contour['variable'] != 'none': d = dataset.get_area(np.array([self.latitude, self.longitude]), self.depth, self.time, self.contour['variable']) contour_unit = get_variable_unit( self.dataset_name, dataset.variables[self.contour['variable']]) contour_name = get_variable_name( self.dataset_name, dataset.variables[self.contour['variable']]) contour_factor = get_variable_scale_factor( self.dataset_name, dataset.variables[self.contour['variable']]) contour_unit, d = self.kelvin_to_celsius(contour_unit, d) d = np.multiply(d, contour_factor) contour_data.append(d) self.contour_unit = contour_unit self.contour_name = contour_name self.contour_data = contour_data self.timestamp = dataset.timestamps[self.time] if self.variables != self.variables_anom: self.variable_name += " Anomaly" with open_dataset(get_dataset_climatology( self.dataset_name)) as dataset: data = [] for v in self.variables: var = dataset.variables[v] d = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.timestamp.month - 1, v) data.append(d) if len(data) == 2: data = np.sqrt(data[0]**2 + data[1]**2) else: data = data[0] u, data = self.kelvin_to_celsius( dataset.variables[self.variables[0]].unit, data) self.data -= data # Load bathymetry data self.bathymetry = overlays.bathymetry(self.basemap, self.latitude, self.longitude, blur=2) if self.depth != 'bottom' and self.depth != 0: if len(quiver_data) > 0: quiver_bathymetry = overlays.bathymetry( self.basemap, quiver_lat, quiver_lon) self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked for d in self.quiver_data: d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked for d in self.contour_data: d[np.where(self.bathymetry < depth_value)] = np.ma.masked else: mask = maskoceans(self.longitude, self.latitude, self.data).mask self.data[~mask] = np.ma.masked for d in self.quiver_data: mask = maskoceans(self.quiver_longitude, self.quiver_latitude, d).mask d[~mask] = np.ma.masked for d in contour_data: mask = maskoceans(self.longitude, self.latitude, d).mask d[~mask] = np.ma.masked if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']: area_polys = [] for a in self.area: rings = [LinearRing(p) for p in a['polygons']] innerrings = [LinearRing(p) for p in a['innerrings']] polygons = [] for r in rings: inners = [] for ir in innerrings: if r.contains(ir): inners.append(ir) polygons.append(Poly(r, inners)) area_polys.append(MultiPolygon(polygons)) points = [ Point(p) for p in zip(self.latitude.ravel(), self.longitude.ravel()) ] indicies = [] for a in area_polys: indicies.append( np.where(map(lambda p, poly=a: poly.contains(p), points))[0]) indicies = np.unique(np.array(indicies).ravel()) newmask = np.ones(self.data.shape, dtype=bool) newmask[np.unravel_index(indicies, newmask.shape)] = False self.data.mask |= newmask self.depth_value = depth_value
def load_data(self): if self.projection == 'EPSG:32661': blat = min(self.bounds[0], self.bounds[2]) blat = 5 * np.floor(blat / 5) self.basemap = basemap.load_map('npstere', (blat, 0), None, None) elif self.projection == 'EPSG:3031': blat = max(self.bounds[0], self.bounds[2]) blat = 5 * np.ceil(blat / 5) self.basemap = basemap.load_map('spstere', (blat, 180), None, None) else: distance = VincentyDistance() height = distance.measure( (self.bounds[0], self.centroid[1]), (self.bounds[2], self.centroid[1]) ) * 1000 * 1.25 width = distance.measure( (self.centroid[0], self.bounds[1]), (self.centroid[0], self.bounds[3]) ) * 1000 * 1.25 self.basemap = basemap.load_map( 'lcc', self.centroid, height, width ) if self.basemap.aspect < 1: gridx = 500 gridy = int(500 * self.basemap.aspect) else: gridy = 500 gridx = int(500 / self.basemap.aspect) self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: if self.time < 0: self.time += len(dataset.timestamps) self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1) self.variable_unit = self.get_variable_units( dataset, self.variables )[0] self.variable_name = self.get_variable_names( dataset, self.variables )[0] scale_factor = self.get_variable_scale_factors( dataset, self.variables )[0] if self.cmap is None: if len(self.variables) == 1: self.cmap = colormap.find_colormap(self.variable_name) else: self.cmap = colormap.colormaps.get('speed') if len(self.variables) == 2: self.variable_name = self.vector_name(self.variable_name) if self.depth == 'bottom': depth_value = 'Bottom' else: self.depth = np.clip( int(self.depth), 0, len(dataset.depths) - 1) depth_value = dataset.depths[self.depth] data = [] allvars = [] for v in self.variables: var = dataset.variables[v] allvars.append(v) if self.filetype in ['csv', 'odv', 'txt']: d, depth_value = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.time, v, return_depth=True ) else: d = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.time, v ) d = np.multiply(d, scale_factor) self.variable_unit, d = self.kelvin_to_celsius( self.variable_unit, d) data.append(d) if self.filetype not in ['csv', 'odv', 'txt']: if len(var.dimensions) == 3: self.depth_label = "" elif self.depth == 'bottom': self.depth_label = " at Bottom" else: self.depth_label = " at " + \ str(int(np.round(depth_value))) + " m" if len(data) == 2: data[0] = np.sqrt(data[0] ** 2 + data[1] ** 2) self.data = data[0] quiver_data = [] if self.quiver is not None and \ self.quiver['variable'] != '' and \ self.quiver['variable'] != 'none': for v in self.quiver['variable'].split(','): allvars.append(v) var = dataset.variables[v] quiver_unit = get_variable_unit(self.dataset_name, var) quiver_name = get_variable_name(self.dataset_name, var) quiver_lon, quiver_lat = self.basemap.makegrid(50, 50) d = dataset.get_area( np.array([quiver_lat, quiver_lon]), self.depth, self.time, v ) quiver_data.append(d) self.quiver_name = self.vector_name(quiver_name) self.quiver_longitude = quiver_lon self.quiver_latitude = quiver_lat self.quiver_unit = quiver_unit self.quiver_data = quiver_data if all(map(lambda v: len(dataset.variables[v].dimensions) == 3, allvars)): self.depth = 0 contour_data = [] if self.contour is not None and \ self.contour['variable'] != '' and \ self.contour['variable'] != 'none': d = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.time, self.contour['variable'] ) contour_unit = get_variable_unit( self.dataset_name, dataset.variables[self.contour['variable']]) contour_name = get_variable_name( self.dataset_name, dataset.variables[self.contour['variable']]) contour_factor = get_variable_scale_factor( self.dataset_name, dataset.variables[self.contour['variable']]) contour_unit, d = self.kelvin_to_celsius(contour_unit, d) d = np.multiply(d, contour_factor) contour_data.append(d) self.contour_unit = contour_unit self.contour_name = contour_name self.contour_data = contour_data self.timestamp = dataset.timestamps[self.time] if self.variables != self.variables_anom: self.variable_name += " Anomaly" with open_dataset( get_dataset_climatology(self.dataset_name) ) as dataset: data = [] for v in self.variables: var = dataset.variables[v] d = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.timestamp.month - 1, v ) data.append(d) if len(data) == 2: data = np.sqrt(data[0] ** 2 + data[1] ** 2) else: data = data[0] u, data = self.kelvin_to_celsius( dataset.variables[self.variables[0]].unit, data) self.data -= data # Load bathymetry data self.bathymetry = overlays.bathymetry( self.basemap, self.latitude, self.longitude, blur=2 ) if self.depth != 'bottom' and self.depth != 0: if len(quiver_data) > 0: quiver_bathymetry = overlays.bathymetry( self.basemap, quiver_lat, quiver_lon) self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked for d in self.quiver_data: d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked for d in self.contour_data: d[np.where(self.bathymetry < depth_value)] = np.ma.masked else: mask = maskoceans(self.longitude, self.latitude, self.data).mask self.data[~mask] = np.ma.masked for d in self.quiver_data: mask = maskoceans( self.quiver_longitude, self.quiver_latitude, d).mask d[~mask] = np.ma.masked for d in contour_data: mask = maskoceans(self.longitude, self.latitude, d).mask d[~mask] = np.ma.masked if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']: area_polys = [] for a in self.area: rings = [LinearRing(p) for p in a['polygons']] innerrings = [LinearRing(p) for p in a['innerrings']] polygons = [] for r in rings: inners = [] for ir in innerrings: if r.contains(ir): inners.append(ir) polygons.append(Poly(r, inners)) area_polys.append(MultiPolygon(polygons)) points = [Point(p) for p in zip(self.latitude.ravel(), self.longitude.ravel())] indicies = [] for a in area_polys: indicies.append(np.where( map( lambda p, poly=a: poly.contains(p), points ) )[0]) indicies = np.unique(np.array(indicies).ravel()) newmask = np.ones(self.data.shape, dtype=bool) newmask[np.unravel_index(indicies, newmask.shape)] = False self.data.mask |= newmask self.depth_value = depth_value
def load_data(self): ds_url = app.config['DRIFTER_URL'] data_names = [] data_units = [] with Dataset(ds_url % self.drifter, 'r') as ds: self.name = ds.buoyid self.imei = str(chartostring(ds['imei'][0])) self.wmo = str(chartostring(ds['wmo'][0])) t = netcdftime.utime(ds['data_date'].units) d = [] for v in self.buoyvariables: d.append(ds[v][:]) if "long_name" in ds[v].ncattrs(): data_names.append(ds[v].long_name) else: data_names.append(v) if "units" in ds[v].ncattrs(): data_units.append(ds[v].units) else: data_units.append(None) self.data = d self.times = t.num2date(ds['data_date'][:]) self.points = np.array([ ds['latitude'][:], ds['longitude'][:], ]).transpose() data_names = data_names[:len(self.buoyvariables)] data_units = data_units[:len(self.buoyvariables)] for i, t in enumerate(self.times): if t.tzinfo is None: self.times[i] = t.replace(tzinfo=pytz.UTC) self.data_names = data_names self.data_units = data_units if self.starttime is not None: d = dateutil.parser.parse(self.starttime) self.start = np.where(self.times >= d)[0].min() else: self.start = 0 if self.endtime is not None: d = dateutil.parser.parse(self.endtime) self.end = np.where(self.times <= d)[0].max() + 1 else: self.end = len(self.times) - 1 if self.start < 0: self.start += len(self.times) self.start = np.clip(self.start, 0, len(self.times) - 1) if self.end < 0: self.end += len(self.times) self.end = np.clip(self.end, 0, len(self.times) - 1) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: depth = int(self.depth) try: model_start = np.where( dataset.timestamps <= self.times[self.start] )[0][-1] except IndexError: model_start = 0 model_start -= 1 model_start = np.clip(model_start, 0, len(dataset.timestamps) - 1) try: model_end = np.where( dataset.timestamps >= self.times[self.end] )[0][0] except IndexError: model_end = len(dataset.timestamps) - 1 model_end += 1 model_end = np.clip( model_end, model_start, len(dataset.timestamps) - 1 ) model_times = map( lambda t: time.mktime(t.timetuple()), dataset.timestamps[model_start:model_end + 1] ) output_times = map( lambda t: time.mktime(t.timetuple()), self.times[self.start:self.end + 1] ) d = [] for v in self.variables: pts, dist, mt, md = dataset.get_path( self.points[self.start:self.end + 1], depth, range(model_start, model_end + 1), v, times=output_times ) f = interp1d( model_times, md, assume_sorted=True, bounds_error=False, ) d.append(np.diag(f(mt))) model_data = np.ma.array(d) variable_names = [] variable_units = [] scale_factors = [] for v in self.variables: variable_units.append(get_variable_unit(self.dataset_name, dataset.variables[v])) variable_names.append(get_variable_name(self.dataset_name, dataset.variables[v])) scale_factors.append( get_variable_scale_factor(self.dataset_name, dataset.variables[v]) ) for idx, sf in enumerate(scale_factors): model_data[idx, :] = np.multiply(model_data[idx, :], sf) for idx, u in enumerate(variable_units): variable_units[idx], model_data[idx, :] = \ self.kelvin_to_celsius(u, model_data[idx, :]) self.model_data = model_data self.model_times = map(datetime.datetime.utcfromtimestamp, mt) self.variable_names = variable_names self.variable_units = variable_units
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as dataset: if self.time < 0: self.time += len(dataset.timestamps) time = np.clip(self.time, 0, len(dataset.timestamps) - 1) for idx, v in enumerate(self.variables): var = dataset.variables[v] if not (set(var.dimensions) & set(dataset.depth_dimensions)): for potential in dataset.variables: if potential in self.variables: continue pot = dataset.variables[potential] if (set(pot.dimensions) & set(dataset.depth_dimensions)): if len(pot.dimensions) > 3: self.variables[idx] = potential.key value = parallel = perpendicular = None variable_names = self.get_variable_names(dataset, self.variables) variable_units = self.get_variable_units(dataset, self.variables) scale_factors = self.get_variable_scale_factors( dataset, self.variables) # Load data sent from primary/Left Map if len(self.variables) > 1: # Only velocity has 2 variables v = [] for name in self.variables: v.append(dataset.variables[name]) distances, times, lat, lon, bearings = geo.path_to_points( self.points, 100) transect_pts, distance, x, dep = dataset.get_path_profile( self.points, time, self.variables[0], 100) transect_pts, distance, y, dep = dataset.get_path_profile( self.points, time, self.variables[1], 100) # Calculate vector components x = np.multiply(x, scale_factors[0]) y = np.multiply(y, scale_factors[1]) r = np.radians(np.subtract(90, bearings)) theta = np.arctan2(y, x) - r mag = np.sqrt(x**2 + y**2) parallel = mag * np.cos(theta) perpendicular = mag * np.sin(theta) else: # Get data for one variable transect_pts, distance, value, dep = dataset.get_path_profile( self.points, time, self.variables[0]) value = np.multiply(value, scale_factors[0]) # Get variable units and convert to Celsius if needed variable_units[0], value = self.kelvin_to_celsius( variable_units[0], value) if len(self.variables) == 2: variable_names[0] = self.vector_name(variable_names[0]) # If a colourmap has not been manually specified by the # Navigator... if self.cmap is None: self.cmap = colormap.find_colormap(variable_names[0]) self.timestamp = dataset.timestamps[int(time)] self.depth = dep self.depth_unit = "m" self.transect_data = { "points": transect_pts, "distance": distance, "data": value, "name": variable_names[0], "unit": variable_units[0], "parallel": parallel, "perpendicular": perpendicular, } if self.surface is not None: surface_pts, surface_dist, t, surface_value = \ dataset.get_path( self.points, 0, time, self.surface, ) surface_unit = get_variable_unit( self.dataset_name, dataset.variables[self.surface]) surface_name = get_variable_name( self.dataset_name, dataset.variables[self.surface]) surface_factor = get_variable_scale_factor( self.dataset_name, dataset.variables[self.surface]) surface_value = np.multiply(surface_value, surface_factor) surface_unit, surface_value = self.kelvin_to_celsius( surface_unit, surface_value) self.surface_data = { "points": surface_pts, "distance": surface_dist, "data": surface_value, "name": surface_name, "unit": surface_unit } # Load data sent from Right Map (if in compare mode) if self.compare: def interpolate_depths(data, depth_in, depth_out): output = [] for i in range(0, depth_in.shape[0]): f = interp1d( depth_in[i], data[:, i], bounds_error=False, assume_sorted=True, ) output.append( f(depth_out[i].view(np.ma.MaskedArray).filled())) return np.ma.masked_invalid(output).transpose() with open_dataset(get_dataset_url( self.compare['dataset'])) as dataset: # Get and format date self.compare['date'] = np.clip(self.compare['time'], 0, len(dataset.timestamps) - 1) self.compare['date'] = dataset.timestamps[int( self.compare['date'])] # 1 variable if len(self.compare['variables']) == 1: # Get and store the "nicely formatted" string for the variable name self.compare['name'] = self.get_variable_names( dataset, self.compare['variables'])[0] # Find correct colourmap if (self.compare['colormap'] == 'default'): self.compare['colormap'] = colormap.find_colormap( self.compare['name']) else: self.compare['colormap'] = colormap.find_colormap( self.compare['colormap']) climate_points, climate_distance, climate_data, cdep = \ dataset.get_path_profile(self.points, self.compare['time'], self.compare['variables'][0]) # Get variable units and convert to Celsius if needed self.compare['unit'], climate_data = self.kelvin_to_celsius( dataset.variables[self.compare['variables'][0]].unit, climate_data) self.__fill_invalid_shift(climate_data) if (self.depth.shape != cdep.shape) or \ (self.depth != cdep).any(): # Need to interpolate the depths climate_data = interpolate_depths( climate_data, cdep, self.depth) if self.transect_data['data'] is None: self.transect_data['parallel'] -= climate_data self.transect_data['perpendicular'] -= climate_data else: self.transect_data['compare_data'] = climate_data # Velocity variables else: # Get and store the "nicely formatted" string for the variable name self.compare['name'] = self.get_variable_names( dataset, self.compare['variables'])[0] climate_pts, climate_distance, climate_x, cdep = \ dataset.get_path_profile( self.points, self.compare['time'], self.compare['variables'][0], 100 ) climate_pts, climate_distance, climate_y, cdep = \ dataset.get_path_profile( self.points, self.compare['time'], self.compare['variables'][0], 100 ) climate_distances, ctimes, clat, clon, bearings = \ geo.path_to_points(self.points, 100) r = np.radians(np.subtract(90, bearings)) theta = np.arctan2(climate_y, climate_x) - r mag = np.sqrt(climate_x**2 + climate_y**2) if np.all(self.depth != cdep): theta = interpolate_depths(theta, cdep, self.depth) self.__fill_invalid_shift(theta) mag = interpolate_depths(mag, cdep, self.depth) self.__fill_invalid_shift(mag) self.compare['parallel'] = mag * np.cos(theta) self.compare['perpendicular'] = mag * np.sin(theta) """ if self.transect_data['parallel'] is None: self.transect_data['data'] -= mag else: self.transect_data['parallel'] -= climate_parallel self.transect_data['perpendicular'] -= climate_perpendicular """ # Bathymetry with Dataset(app.config['BATHYMETRY_FILE'], 'r') as dataset: bath_x, bath_y = bathymetry(dataset.variables['y'], dataset.variables['x'], dataset.variables['z'], self.points) self.bathymetry = {'x': bath_x, 'y': bath_y}
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as dataset: if self.time < 0: self.time += len(dataset.timestamps) time = np.clip(self.time, 0, len(dataset.timestamps) - 1) for idx, v in enumerate(self.variables): var = dataset.variables[v] if not (set(var.dimensions) & set(dataset.depth_dimensions)): for potential in dataset.variables: if potential in self.variables: continue pot = dataset.variables[potential] if (set(pot.dimensions) & set(dataset.depth_dimensions)): if len(pot.shape) > 3: self.variables[idx] = potential self.variables_anom[idx] = potential value = parallel = perpendicular = None variable_names = self.get_variable_names(dataset, self.variables) variable_units = self.get_variable_units(dataset, self.variables) scale_factors = self.get_variable_scale_factors(dataset, self.variables) if len(self.variables) > 1: v = [] for name in self.variables: v.append(dataset.variables[name]) distances, times, lat, lon, bearings = geo.path_to_points( self.points, 100 ) transect_pts, distance, x, dep = dataset.get_path_profile( self.points, time, self.variables[0], 100) transect_pts, distance, y, dep = dataset.get_path_profile( self.points, time, self.variables[1], 100) x = np.multiply(x, scale_factors[0]) y = np.multiply(y, scale_factors[1]) r = np.radians(np.subtract(90, bearings)) theta = np.arctan2(y, x) - r mag = np.sqrt(x ** 2 + y ** 2) parallel = mag * np.cos(theta) perpendicular = mag * np.sin(theta) else: transect_pts, distance, value, dep = dataset.get_path_profile( self.points, time, self.variables[0]) value = np.multiply(value, scale_factors[0]) variable_units[0], value = self.kelvin_to_celsius( variable_units[0], value ) if len(self.variables) == 2: variable_names[0] = self.vector_name(variable_names[0]) if self.cmap is None: self.cmap = colormap.find_colormap(variable_names[0]) self.timestamp = dataset.timestamps[int(time)] self.depth = dep self.depth_unit = "m" self.transect_data = { "points": transect_pts, "distance": distance, "data": value, "name": variable_names[0], "unit": variable_units[0], "parallel": parallel, "perpendicular": perpendicular, } if self.surface is not None: surface_pts, surface_dist, t, surface_value = \ dataset.get_path( self.points, 0, time, self.surface, ) surface_unit = get_variable_unit( self.dataset_name, dataset.variables[self.surface] ) surface_name = get_variable_name( self.dataset_name, dataset.variables[self.surface] ) surface_factor = get_variable_scale_factor( self.dataset_name, dataset.variables[self.surface] ) surface_value = np.multiply(surface_value, surface_factor) surface_unit, surface_value = self.kelvin_to_celsius( surface_unit, surface_value ) self.surface_data = { "points": surface_pts, "distance": surface_dist, "data": surface_value, "name": surface_name, "unit": surface_unit } if self.variables != self.variables_anom: with open_dataset( get_dataset_climatology(self.dataset_name) ) as dataset: if self.variables[0] in dataset.variables: if len(self.variables) == 1: climate_points, climate_distance, climate_data = \ dataset.get_path_profile(self.points, self.timestamp.month - 1, self.variables[0]) u, climate_data = self.kelvin_to_celsius( dataset.variables[self.variables[0]].unit, climate_data ) self.transect_data['data'] -= - climate_data else: climate_pts, climate_distance, climate_x, cdep = \ dataset.get_path_profile( self.points, self.timestamp.month - 1, self.variables[0], 100 ) climate_pts, climate_distance, climate_y, cdep = \ dataset.get_path_profile( self.points, self.timestamp.month - 1, self.variables[0], 100 ) climate_distances, ctimes, clat, clon, bearings = \ geo.path_to_points(self.points, 100) r = np.radians(np.subtract(90, bearings)) theta = np.arctan2(y, x) - r mag = np.sqrt(x ** 2 + y ** 2) climate_parallel = mag * np.cos(theta) climate_perpendicular = mag * np.sin(theta) self.transect_data['parallel'] -= climate_parallel self.transect_data[ 'perpendicular'] -= climate_perpendicular # Bathymetry with Dataset(app.config['BATHYMETRY_FILE'], 'r') as dataset: bath_x, bath_y = bathymetry( dataset.variables['y'], dataset.variables['x'], dataset.variables['z'], self.points) self.bathymetry = { 'x': bath_x, 'y': bath_y }
def load_data(self): ds_url = app.config['DRIFTER_URL'] data_names = [] data_units = [] with Dataset(ds_url % self.drifter, 'r') as ds: self.name = ds.buoyid self.imei = str(chartostring(ds['imei'][0])) self.wmo = str(chartostring(ds['wmo'][0])) t = netcdftime.utime(ds['data_date'].units) d = [] for v in self.buoyvariables: d.append(ds[v][:]) if "long_name" in ds[v].ncattrs(): data_names.append(ds[v].long_name) else: data_names.append(v) if "units" in ds[v].ncattrs(): data_units.append(ds[v].units) else: data_units.append(None) self.data = d self.times = t.num2date(ds['data_date'][:]) self.points = np.array([ ds['latitude'][:], ds['longitude'][:], ]).transpose() data_names = data_names[:len(self.buoyvariables)] data_units = data_units[:len(self.buoyvariables)] for i, t in enumerate(self.times): if t.tzinfo is None: self.times[i] = t.replace(tzinfo=pytz.UTC) self.data_names = data_names self.data_units = data_units if self.starttime is not None: d = dateutil.parser.parse(self.starttime) self.start = np.where(self.times >= d)[0].min() else: self.start = 0 if self.endtime is not None: d = dateutil.parser.parse(self.endtime) self.end = np.where(self.times <= d)[0].max() + 1 else: self.end = len(self.times) - 1 if self.start < 0: self.start += len(self.times) self.start = np.clip(self.start, 0, len(self.times) - 1) if self.end < 0: self.end += len(self.times) self.end = np.clip(self.end, 0, len(self.times) - 1) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: depth = int(self.depth) try: model_start = np.where( dataset.timestamps <= self.times[self.start])[0][-1] except IndexError: model_start = 0 model_start -= 1 model_start = np.clip(model_start, 0, len(dataset.timestamps) - 1) try: model_end = np.where( dataset.timestamps >= self.times[self.end])[0][0] except IndexError: model_end = len(dataset.timestamps) - 1 model_end += 1 model_end = np.clip(model_end, model_start, len(dataset.timestamps) - 1) model_times = map(lambda t: time.mktime(t.timetuple()), dataset.timestamps[model_start:model_end + 1]) output_times = map(lambda t: time.mktime(t.timetuple()), self.times[self.start:self.end + 1]) d = [] for v in self.variables: pts, dist, mt, md = dataset.get_path( self.points[self.start:self.end + 1], depth, range(model_start, model_end + 1), v, times=output_times) f = interp1d( model_times, md, assume_sorted=True, bounds_error=False, ) d.append(np.diag(f(mt))) model_data = np.ma.array(d) variable_names = [] variable_units = [] scale_factors = [] for v in self.variables: variable_units.append( get_variable_unit(self.dataset_name, dataset.variables[v])) variable_names.append( get_variable_name(self.dataset_name, dataset.variables[v])) scale_factors.append( get_variable_scale_factor(self.dataset_name, dataset.variables[v])) for idx, sf in enumerate(scale_factors): model_data[idx, :] = np.multiply(model_data[idx, :], sf) for idx, u in enumerate(variable_units): variable_units[idx], model_data[idx, :] = \ self.kelvin_to_celsius(u, model_data[idx, :]) self.model_data = model_data self.model_times = map(datetime.datetime.utcfromtimestamp, mt) self.variable_names = variable_names self.variable_units = variable_units
def load_data(self): distance = VincentyDistance() height = distance.measure( (self.bounds[0], self.centroid[1]), (self.bounds[2], self.centroid[1])) * 1000 * 1.25 width = distance.measure( (self.centroid[0], self.bounds[1]), (self.centroid[0], self.bounds[3])) * 1000 * 1.25 if self.projection == 'EPSG:32661': near_pole, covers_pole = self.pole_proximity(self.points[0]) blat = min(self.bounds[0], self.bounds[2]) blat = 5 * np.floor(blat / 5) if self.centroid[0] > 80 or near_pole or covers_pole: self.basemap = basemap.load_map( 'npstere', self.centroid, height, width, min(self.bounds[0], self.bounds[2])) else: self.basemap = basemap.load_map('lcc', self.centroid, height, width) elif self.projection == 'EPSG:3031': near_pole, covers_pole = self.pole_proximity(self.points[0]) blat = max(self.bounds[0], self.bounds[2]) blat = 5 * np.ceil(blat / 5) if ((self.centroid[0] < -80 or self.bounds[1] < -80 or self.bounds[3] < -80) or covers_pole): # is centerered close to the south pole self.basemap = basemap.load_map('spstere', (blat, 180), height, width) else: self.basemap = basemap.load_map( 'lcc', self.centroid, height, width, max(self.bounds[0], self.bounds[2])) elif abs(self.centroid[1] - self.bounds[1]) > 90: height_bounds = [self.bounds[0], self.bounds[2]] width_bounds = [self.bounds[1], self.bounds[3]] height_buffer = (abs(height_bounds[1] - height_bounds[0])) * 0.1 width_buffer = (abs(width_bounds[0] - width_bounds[1])) * 0.1 if abs(width_bounds[1] - width_bounds[0]) > 360: raise ClientError( gettext( "You have requested an area that exceads the width of the world. \ Thinking big is good but plots need to be less the 360 deg wide." )) if height_bounds[1] < 0: height_bounds[1] = height_bounds[1] + height_buffer else: height_bounds[1] = height_bounds[1] + height_buffer if height_bounds[0] < 0: height_bounds[0] = height_bounds[0] - height_buffer else: height_bounds[0] = height_bounds[0] - height_buffer new_width_bounds = [] new_width_bounds.append(width_bounds[0] - width_buffer) new_width_bounds.append(width_bounds[1] + width_buffer) if abs(new_width_bounds[1] - new_width_bounds[0]) > 360: width_buffer = np.floor( (360 - abs(width_bounds[1] - width_bounds[0])) / 2) new_width_bounds[0] = width_bounds[0] - width_buffer new_width_bounds[1] = width_bounds[1] + width_buffer if new_width_bounds[0] < -360: new_width_bounds[0] = -360 if new_width_bounds[1] > 720: new_width_bounds[1] = 720 self.basemap = basemap.load_map( 'merc', self.centroid, (height_bounds[0], height_bounds[1]), (new_width_bounds[0], new_width_bounds[1])) else: self.basemap = basemap.load_map('lcc', self.centroid, height, width) if self.basemap.aspect < 1: gridx = 500 gridy = int(500 * self.basemap.aspect) else: gridy = 500 gridx = int(500 / self.basemap.aspect) self.longitude, self.latitude = self.basemap.makegrid(gridx, gridy) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: if self.time < 0: self.time += len(dataset.timestamps) self.time = np.clip(self.time, 0, len(dataset.timestamps) - 1) self.variable_unit = self.get_variable_units( dataset, self.variables)[0] self.variable_name = self.get_variable_names( dataset, self.variables)[0] scale_factor = self.get_variable_scale_factors( dataset, self.variables)[0] if self.cmap is None: if len(self.variables) == 1: self.cmap = colormap.find_colormap(self.variable_name) else: self.cmap = colormap.colormaps.get('speed') if len(self.variables) == 2: self.variable_name = self.vector_name(self.variable_name) if self.depth == 'bottom': depth_value = 'Bottom' else: self.depth = np.clip(int(self.depth), 0, len(dataset.depths) - 1) depth_value = dataset.depths[self.depth] data = [] allvars = [] for v in self.variables: var = dataset.variables[v] allvars.append(v) if self.filetype in ['csv', 'odv', 'txt']: d, depth_value = dataset.get_area(np.array( [self.latitude, self.longitude]), self.depth, self.time, v, self.interp, self.radius, self.neighbours, return_depth=True) else: d = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.time, v, self.interp, self.radius, self.neighbours) d = np.multiply(d, scale_factor) self.variable_unit, d = self.kelvin_to_celsius( self.variable_unit, d) data.append(d) if self.filetype not in ['csv', 'odv', 'txt']: if len(var.dimensions) == 3: self.depth_label = "" elif self.depth == 'bottom': self.depth_label = " at Bottom" else: self.depth_label = " at " + \ str(int(np.round(depth_value))) + " m" if len(data) == 2: data[0] = np.sqrt(data[0]**2 + data[1]**2) self.data = data[0] quiver_data = [] # Store the quiver data on the same grid as the main variable. This # will only be used for CSV export. quiver_data_fullgrid = [] if self.quiver is not None and \ self.quiver['variable'] != '' and \ self.quiver['variable'] != 'none': for v in self.quiver['variable'].split(','): allvars.append(v) var = dataset.variables[v] quiver_unit = get_variable_unit(self.dataset_name, var) quiver_name = get_variable_name(self.dataset_name, var) quiver_lon, quiver_lat = self.basemap.makegrid(50, 50) d = dataset.get_area( np.array([quiver_lat, quiver_lon]), self.depth, self.time, v, self.interp, self.radius, self.neighbours, ) quiver_data.append(d) # Get the quiver data on the same grid as the main # variable. d = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.time, v, self.interp, self.radius, self.neighbours, ) quiver_data_fullgrid.append(d) self.quiver_name = self.vector_name(quiver_name) self.quiver_longitude = quiver_lon self.quiver_latitude = quiver_lat self.quiver_unit = quiver_unit self.quiver_data = quiver_data self.quiver_data_fullgrid = quiver_data_fullgrid if all( map(lambda v: len(dataset.variables[v].dimensions) == 3, allvars)): self.depth = 0 contour_data = [] if self.contour is not None and \ self.contour['variable'] != '' and \ self.contour['variable'] != 'none': d = dataset.get_area( np.array([self.latitude, self.longitude]), self.depth, self.time, self.contour['variable'], self.interp, self.radius, self.neighbours, ) contour_unit = get_variable_unit( self.dataset_name, dataset.variables[self.contour['variable']]) contour_name = get_variable_name( self.dataset_name, dataset.variables[self.contour['variable']]) contour_factor = get_variable_scale_factor( self.dataset_name, dataset.variables[self.contour['variable']]) contour_unit, d = self.kelvin_to_celsius(contour_unit, d) d = np.multiply(d, contour_factor) contour_data.append(d) self.contour_unit = contour_unit self.contour_name = contour_name self.contour_data = contour_data self.timestamp = dataset.timestamps[self.time] if self.compare: self.variable_name += " Difference" with open_dataset(get_dataset_url( self.compare['dataset'])) as dataset: data = [] for v in self.compare['variables']: var = dataset.variables[v] d = dataset.get_area( np.array([self.latitude, self.longitude]), self.compare['depth'], self.compare['time'], v, self.interp, self.radius, self.neighbours, ) data.append(d) if len(data) == 2: data = np.sqrt(data[0]**2 + data[1]**2) else: data = data[0] u, data = self.kelvin_to_celsius( dataset.variables[self.compare['variables'][0]].unit, data) self.data -= data # Load bathymetry data self.bathymetry = overlays.bathymetry(self.basemap, self.latitude, self.longitude, blur=2) if self.depth != 'bottom' and self.depth != 0: if len(quiver_data) > 0: quiver_bathymetry = overlays.bathymetry( self.basemap, quiver_lat, quiver_lon) self.data[np.where(self.bathymetry < depth_value)] = np.ma.masked for d in self.quiver_data: d[np.where(quiver_bathymetry < depth_value)] = np.ma.masked for d in self.contour_data: d[np.where(self.bathymetry < depth_value)] = np.ma.masked else: mask = maskoceans(self.longitude, self.latitude, self.data).mask self.data[~mask] = np.ma.masked for d in self.quiver_data: mask = maskoceans(self.quiver_longitude, self.quiver_latitude, d).mask d[~mask] = np.ma.masked for d in contour_data: mask = maskoceans(self.longitude, self.latitude, d).mask d[~mask] = np.ma.masked if self.area and self.filetype in ['csv', 'odv', 'txt', 'geotiff']: area_polys = [] for a in self.area: rings = [LinearRing(p) for p in a['polygons']] innerrings = [LinearRing(p) for p in a['innerrings']] polygons = [] for r in rings: inners = [] for ir in innerrings: if r.contains(ir): inners.append(ir) polygons.append(Poly(r, inners)) area_polys.append(MultiPolygon(polygons)) points = [ Point(p) for p in zip(self.latitude.ravel(), self.longitude.ravel()) ] indicies = [] for a in area_polys: indicies.append( np.where(map(lambda p, poly=a: poly.contains(p), points))[0]) indicies = np.unique(np.array(indicies).ravel()) newmask = np.ones(self.data.shape, dtype=bool) newmask[np.unravel_index(indicies, newmask.shape)] = False self.data.mask |= newmask self.depth_value = depth_value
def plot(projection, x, y, z, args): lat, lon = get_latlon_coords(projection, x, y, z) if len(lat.shape) == 1: lat, lon = np.meshgrid(lat, lon) dataset_name = args.get('dataset') variable = args.get('variable') if variable.endswith('_anom'): variable = variable[0:-5] anom = True else: anom = False variable = variable.split(',') depth = args.get('depth') scale = args.get('scale') scale = [float(component) for component in scale.split(',')] data = [] with open_dataset(get_dataset_url(dataset_name)) as dataset: if args.get('time') is None or (type(args.get('time')) == str and len(args.get('time')) == 0): time = -1 else: time = int(args.get('time')) t_len = len(dataset.timestamps) while time >= t_len: time -= t_len while time < 0: time += len(dataset.timestamps) timestamp = dataset.timestamps[time] for v in variable: data.append(dataset.get_area(np.array([lat, lon]), depth, time, v)) variable_name = get_variable_name(dataset_name, dataset.variables[variable[0]]) variable_unit = get_variable_unit(dataset_name, dataset.variables[variable[0]]) scale_factor = get_variable_scale_factor( dataset_name, dataset.variables[variable[0]]) if anom: cmap = colormap.colormaps['anomaly'] else: cmap = colormap.find_colormap(variable_name) if depth != 'bottom': depthm = dataset.depths[depth] else: depthm = 0 if scale_factor != 1.0: for idx, val in enumerate(data): data[idx] = np.multiply(val, scale_factor) if variable_unit.startswith("Kelvin"): variable_unit = "Celsius" for idx, val in enumerate(data): data[idx] = np.add(val, -273.15) if len(data) == 1: data = data[0] if len(data) == 2: data = np.sqrt(data[0]**2 + data[1]**2) if not anom: cmap = colormap.colormaps.get('speed') if anom: with open_dataset(get_dataset_climatology(dataset_name)) as dataset: a = dataset.get_area(np.array([lat, lon]), depth, timestamp.month - 1, v) data = data - a f, fname = tempfile.mkstemp() os.close(f) data = data.transpose() xpx = x * 256 ypx = y * 256 with Dataset(ETOPO_FILE % (projection, z), 'r') as dataset: bathymetry = dataset["z"][ypx:(ypx + 256), xpx:(xpx + 256)] bathymetry = gaussian_filter(bathymetry, 0.5) data[np.where(bathymetry > -depthm)] = np.ma.masked sm = matplotlib.cm.ScalarMappable(matplotlib.colors.Normalize( vmin=scale[0], vmax=scale[1]), cmap=cmap) img = sm.to_rgba(np.squeeze(data)) im = Image.fromarray((img * 255.0).astype(np.uint8)) im.save(fname, format='png', optimize=True) with open(fname, 'r') as f: buf = f.read() os.remove(fname) return buf
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as dataset: self.load_misc(dataset, self.variables) self.fix_startend_times(dataset) self.variable_unit = get_variable_unit( self.dataset_name, dataset.variables[self.variables[0]]) self.variable_name = get_variable_name( self.dataset_name, dataset.variables[self.variables[0]]) var = self.variables[0] if self.depth != 'all' and self.depth != 'bottom' and \ (set(dataset.variables[var].dimensions) & set(dataset.depth_dimensions)): self.depth_label = " at %d m" % (np.round( dataset.depths[self.depth])) elif self.depth == 'bottom': self.depth_label = ' at Bottom' else: self.depth_label = '' if not (set(dataset.variables[var].dimensions) & set(dataset.depth_dimensions)): self.depth = 0 times = None point_data = [] for p in self.points: data = [] for v in self.variables: if self.depth == 'all': d, dep = dataset.get_timeseries_profile( float(p[0]), float(p[1]), self.starttime, self.endtime, v) else: d, dep = dataset.get_timeseries_point( float(p[0]), float(p[1]), self.depth, self.starttime, self.endtime, v, return_depth=True) data.append(d) point_data.append(np.ma.array(data)) point_data = np.ma.array(point_data) for idx, factor in enumerate(self.scale_factors): if factor != 1.0: point_data[idx] = np.multiply(point_data[idx], factor) times = dataset.timestamps[self.starttime:self.endtime + 1] if self.query.get('dataset_quantum') == 'month': times = [datetime.date(x.year, x.month, 1) for x in times] # depths = dataset.depths depths = dep # TODO: pint if self.variable_unit.startswith("Kelvin"): self.variable_unit = "Celsius" for idx, v in enumerate(self.variables): point_data[:, idx, :] = point_data[:, idx, :] - 273.15 if point_data.shape[1] == 2: point_data = np.ma.expand_dims( np.sqrt(point_data[:, 0, :]**2 + point_data[:, 1, :]**2), 1) self.times = times self.data = point_data self.depths = depths self.depth_unit = "m"
def load_data(self): if isinstance(self.observation[0], numbers.Number): self.observation_variable_names = [] self.observation_variable_units = [] with Dataset(app.config["OBSERVATION_AGG_URL"], 'r') as ds: t = netcdftime.utime(ds['time'].units) for idx, o in enumerate(self.observation): observation = {} ts = t.num2date(ds['time'][o]).replace(tzinfo=pytz.UTC) observation['time'] = ts.isoformat() observation['longitude'] = ds['lon'][o] observation['latitude'] = ds['lat'][o] observation['depth'] = ds['z'][:] observation['depthunit'] = ds['z'].units observation['datatypes'] = [] data = [] for v in sorted(ds.variables): if v in ['z', 'lat', 'lon', 'profile', 'time']: continue var = ds[v] if var.datatype == '|S1': continue observation['datatypes'].append("%s [%s]" % ( var.long_name, var.units )) data.append(var[o, :]) if idx == 0: self.observation_variable_names.append( var.long_name) self.observation_variable_units.append(var.units) observation['data'] = np.ma.array(data).transpose() self.observation[idx] = observation self.points = map(lambda o: [o['latitude'], o['longitude']], self.observation) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: ts = dataset.timestamps observation_times = [] timestamps = [] for o in self.observation: observation_time = dateutil.parser.parse(o['time']) observation_times.append(observation_time) deltas = [ (x.replace(tzinfo=pytz.UTC) - observation_time).total_seconds() for x in ts] time = np.abs(deltas).argmin() timestamp = ts[time] timestamps.append(timestamp) self.load_misc(dataset, self.variables) point_data, self.depths = self.get_data( dataset, self.variables, time) point_data = np.ma.array(point_data) point_data = self.apply_scale_factors(point_data) self.variable_units, point_data = self.kelvin_to_celsius( self.variable_units, point_data ) self.data = self.subtract_climatology(point_data, timestamp) self.observation_time = observation_time self.observation_times = observation_times self.timestamps = timestamps self.timestamp = timestamp
def load_data(self): """ Calculates and returns the depth, depth-value, and depth unit from a given dataset Args: depth: Stored depth information (self.depth or self.compare['depth']) clip_length: How many depth values to clip (usually len(dataset.depths) - 1) dataset: Opened dataset Returns: (depth, depth_value, depth_unit) """ def find_depth(depth, clip_length, dataset): depth_value = 0 depth_unit = "m" if depth: if depth == 'bottom': depth_value = 'Bottom' depth_unit = '' return (depth, depth_value, depth_unit) else: depth = np.clip(int(depth), 0, clip_length) depth_value = np.round(dataset.depths[depth]) depth_unit = "m" return (depth, depth_value, depth_unit) return (depth, depth_value, depth_unit) # Load left/Main Map with open_dataset(get_dataset_url(self.dataset_name)) as dataset: latvar, lonvar = utils.get_latlon_vars(dataset) self.depth, self.depth_value, self.depth_unit = find_depth( self.depth, len(dataset.depths) - 1, dataset) self.fix_startend_times(dataset, self.starttime, self.endtime) time = range(self.starttime, self.endtime + 1) if len(self.variables) > 1: v = [] for name in self.variables: self.path_points, self.distance, t, value = dataset.get_path( self.points, self.depth, time, name) v.append(value**2) value = np.sqrt(np.ma.sum(v, axis=0)) self.variable_name = self.get_variable_names( dataset, self.variables)[0] self.variable_name = self.vector_name(self.variable_name) else: self.path_points, self.distance, t, value = dataset.get_path( self.points, self.depth, time, self.variables[0]) self.variable_name = self.get_variable_names( dataset, self.variables)[0] variable_units = self.get_variable_units(dataset, self.variables) scale_factors = self.get_variable_scale_factors( dataset, self.variables) self.variable_unit, self.data = self.kelvin_to_celsius( variable_units[0], value) self.times = dataset.timestamps[self.starttime:self.endtime + 1] self.data = np.multiply(self.data, scale_factors[0]) self.data = self.data.transpose() # Get colourmap if self.cmap is None: self.cmap = colormap.find_colormap(self.variable_name) # Load data sent from Right Map (if in compare mode) if self.compare: with open_dataset(get_dataset_url( self.compare['dataset'])) as dataset: latvar, lonvar = utils.get_latlon_vars(dataset) self.compare['depth'], self.compare[ 'depth_value'], self.compare['depth_unit'] = find_depth( self.compare['depth'], len(dataset.depths) - 1, dataset) self.fix_startend_times(dataset, self.compare['starttime'], self.compare['endtime']) time = range(self.compare['starttime'], self.compare['endtime'] + 1) if len(self.compare['variables']) > 1: v = [] for name in self.compare['variables']: path, distance, t, value = dataset.get_path( self.points, self.compare['depth'], time, name) v.append(value**2) value = np.sqrt(np.ma.sum(v, axis=0)) self.compare['variable_name'] = self.get_variable_names( dataset, self.compare['variables'])[0] self.compare['variable_name'] = self.vector_name( self.compare['variable_name']) else: path, distance, t, value = dataset.get_path( self.points, self.compare['depth'], time, self.compare['variables'][0]) self.compare['variable_name'] = self.get_variable_names( dataset, self.compare['variables'])[0] # Colourmap if (self.compare['colormap'] == 'default'): self.compare['colormap'] = colormap.find_colormap( self.compare['variable_name']) else: self.compare['colormap'] = colormap.find_colormap( self.compare['colormap']) variable_units = self.get_variable_units( dataset, self.compare['variables']) scale_factors = self.get_variable_scale_factors( dataset, self.compare['variables']) self.compare['variable_unit'], self.compare[ 'data'] = self.kelvin_to_celsius(variable_units[0], value) self.compare['times'] = dataset.timestamps[ self.compare['starttime']:self.compare['endtime'] + 1] self.compare['data'] = np.multiply(self.compare['data'], scale_factors[0]) self.compare['data'] = self.compare['data'].transpose() # Comparison over different time ranges makes no sense if self.starttime != self.compare['starttime'] or\ self.endtime != self.compare['endtime']: raise ClientError( gettext( "Please ensure the Start Time and End Time for the Left and Right maps are identical." ))
def plot(projection, x, y, z, args): lat, lon = get_latlon_coords(projection, x, y, z) if len(lat.shape) == 1: lat, lon = np.meshgrid(lat, lon) dataset_name = args.get('dataset') variable = args.get('variable') if variable.endswith('_anom'): variable = variable[0:-5] anom = True else: anom = False variable = variable.split(',') depth = args.get('depth') scale = args.get('scale') scale = [float(component) for component in scale.split(',')] data = [] with open_dataset(get_dataset_url(dataset_name)) as dataset: if args.get('time') is None or (type(args.get('time')) == str and len(args.get('time')) == 0): time = -1 else: time = int(args.get('time')) t_len = len(dataset.timestamps) while time >= t_len: time -= t_len while time < 0: time += len(dataset.timestamps) timestamp = dataset.timestamps[time] for v in variable: data.append(dataset.get_area( np.array([lat, lon]), depth, time, v )) variable_name = get_variable_name(dataset_name, dataset.variables[variable[0]]) variable_unit = get_variable_unit(dataset_name, dataset.variables[variable[0]]) scale_factor = get_variable_scale_factor( dataset_name, dataset.variables[variable[0]] ) if anom: cmap = colormap.colormaps['anomaly'] else: cmap = colormap.find_colormap(variable_name) if depth != 'bottom': depthm = dataset.depths[depth] else: depthm = 0 if scale_factor != 1.0: for idx, val in enumerate(data): data[idx] = np.multiply(val, scale_factor) if variable_unit.startswith("Kelvin"): variable_unit = "Celsius" for idx, val in enumerate(data): data[idx] = np.add(val, -273.15) if len(data) == 1: data = data[0] if len(data) == 2: data = np.sqrt(data[0] ** 2 + data[1] ** 2) if not anom: cmap = colormap.colormaps.get('speed') if anom: with open_dataset(get_dataset_climatology(dataset_name)) as dataset: a = dataset.get_area( np.array([lat, lon]), depth, timestamp.month - 1, v ) data = data - a f, fname = tempfile.mkstemp() os.close(f) data = data.transpose() xpx = x * 256 ypx = y * 256 with Dataset(ETOPO_FILE % (projection, z), 'r') as dataset: bathymetry = dataset["z"][ypx:(ypx + 256), xpx:(xpx + 256)] bathymetry = gaussian_filter(bathymetry, 0.5) data[np.where(bathymetry > -depthm)] = np.ma.masked sm = matplotlib.cm.ScalarMappable( matplotlib.colors.Normalize(vmin=scale[0], vmax=scale[1]), cmap=cmap) img = sm.to_rgba(np.squeeze(data)) im = Image.fromarray((img * 255.0).astype(np.uint8)) im.save(fname, format='png', optimize=True) with open(fname, 'r') as f: buf = f.read() os.remove(fname) return buf
def load_data(self): if isinstance(self.observation[0], numbers.Number): self.observation_variable_names = [] self.observation_variable_units = [] with Dataset(app.config["OBSERVATION_AGG_URL"], 'r') as ds: t = netcdftime.utime(ds['time'].units) for idx, o in enumerate(self.observation): observation = {} ts = t.num2date(ds['time'][o]).replace(tzinfo=pytz.UTC) observation['time'] = ts.isoformat() observation['longitude'] = ds['lon'][o] observation['latitude'] = ds['lat'][o] observation['depth'] = ds['z'][:] observation['depthunit'] = ds['z'].units observation['datatypes'] = [] data = [] for v in sorted(ds.variables): if v in ['z', 'lat', 'lon', 'profile', 'time']: continue var = ds[v] if var.datatype == '|S1': continue observation['datatypes'].append("%s [%s]" % ( var.long_name, var.units )) data.append(var[o, :]) if idx == 0: self.observation_variable_names.append( var.long_name) self.observation_variable_units.append(var.units) observation['data'] = np.ma.array(data).transpose() self.observation[idx] = observation self.points = map(lambda o: [o['latitude'], o['longitude']], self.observation) with open_dataset(get_dataset_url(self.dataset_name)) as dataset: ts = dataset.timestamps observation_times = [] timestamps = [] for o in self.observation: observation_time = dateutil.parser.parse(o['time']) observation_times.append(observation_time) deltas = [ (x.replace(tzinfo=pytz.UTC) - observation_time).total_seconds() for x in ts] time = np.abs(deltas).argmin() timestamp = ts[time] timestamps.append(timestamp) try: self.load_misc(dataset, self.variables) except IndexError as e: raise ClientError(gettext("The selected variable(s) were not found in the dataset. \ Most likely, this variable is a derived product from existing dataset variables. \ Please select another variable.") + str(e)) point_data, self.depths = self.get_data(dataset, self.variables, time) point_data = np.ma.array(point_data) point_data = self.apply_scale_factors(point_data) self.variable_units, point_data = self.kelvin_to_celsius( self.variable_units, point_data ) self.data = point_data self.observation_time = observation_time self.observation_times = observation_times self.timestamps = timestamps self.timestamp = timestamp
def load_data(self): with open_dataset(get_dataset_url(self.dataset_name)) as dataset: self.load_misc(dataset, self.variables) self.fix_startend_times(dataset) self.variable_unit = get_variable_unit( self.dataset_name, dataset.variables[self.variables[0]] ) self.variable_name = get_variable_name( self.dataset_name, dataset.variables[self.variables[0]] ) var = self.variables[0] if self.depth != 'all' and self.depth != 'bottom' and \ (set(dataset.variables[var].dimensions) & set(dataset.depth_dimensions)): self.depth_label = " at %d m" % ( np.round(dataset.depths[self.depth]) ) elif self.depth == 'bottom': self.depth_label = ' at Bottom' else: self.depth_label = '' if not (set(dataset.variables[var].dimensions) & set(dataset.depth_dimensions)): self.depth = 0 times = None point_data = [] for p in self.points: data = [] for v in self.variables: if self.depth == 'all': d, dep = dataset.get_timeseries_profile( float(p[0]), float(p[1]), self.starttime, self.endtime, v ) else: d, dep = dataset.get_timeseries_point( float(p[0]), float(p[1]), self.depth, self.starttime, self.endtime, v, return_depth=True ) data.append(d) point_data.append(np.ma.array(data)) point_data = np.ma.array(point_data) for idx, factor in enumerate(self.scale_factors): if factor != 1.0: point_data[idx] = np.multiply(point_data[idx], factor) times = dataset.timestamps[self.starttime:self.endtime + 1] if self.query.get('dataset_quantum') == 'month': times = [datetime.date(x.year, x.month, 1) for x in times] # depths = dataset.depths depths = dep # TODO: pint if self.variable_unit.startswith("Kelvin"): self.variable_unit = "Celsius" for idx, v in enumerate(self.variables): point_data[:, idx, :] = point_data[:, idx, :] - 273.15 if point_data.shape[1] == 2: point_data = np.ma.expand_dims( np.sqrt( point_data[:, 0, :] ** 2 + point_data[:, 1, :] ** 2 ), 1 ) self.times = times self.data = point_data self.depths = depths self.depth_unit = "m"