def rasterize(self, eid, fid, uvws, grid_spacing=0.1): (u1, v1), (u2, v2) = self.face_bounds[(eid, fid)] minu, minv, maxu, maxv = min(uvws[:, 0]), min(uvws[:, 1]), max(uvws[:, 0]), max(uvws[:, 1]) numu, numv = (maxu - minu) * (u2 - u1) / grid_spacing, (maxv - minv) * (v2 - v1) / grid_spacing numu, numv = map(lambda f: int(math.ceil(f)), (numu, numv)) numu, numv = numu + 1, numv + 1 grid_sample_points = np.meshgrid(np.linspace(minu, maxu, numu), np.linspace(minv, maxv, numv)) uspacing = (maxu - minu) / (numu - 1.0) vspacing = (maxv - minv) / (numv - 1.0) placement = minu, minv try: values = scipy.interpolate.griddata(uvws[:, 0:2], uvws[:, 2], tuple(grid_sample_points), "linear") except: values = scipy.interpolate.griddata(uvws[:, 0:2], uvws[:, 2], tuple(grid_sample_points), "nearest") area_in_wcs = (v2 - v1) * (u2 - u1) threshold = area_in_wcs / math.sqrt(len(uvws)) / 2.0 # ** 2 # Keep these UV's are normalized here, so for distances # a non-uniformly scaled copy needs to be made. uvs = uvws[:, 0:2].copy() uvs *= (u2 - u1), (v2 - v1) grid_sample_points[0] *= u2 - u1 grid_sample_points[1] *= v2 - v1 tree = cKDTree(uvs) gridpoints = _ndim_coords_from_arrays(tuple(grid_sample_points), ndim=uvs.shape[1]) dists = tree.query(gridpoints)[0] values[dists > threshold] = np.nan return grid_values(grid(placement, numu, numv, uspacing, vspacing), values)
def getFuchsInterp( Temperature, name='density', Uset = [4,6,8,10,12]): U_, mu_, qty_ = getFuchsPoints( Temperature, name=name, Uset = Uset) points = _ndim_coords_from_arrays((U_, mu_)) if len ( Uset) > 1: return CloughTocher2DInterpolator(points, qty_) else: return None
def maskMeshByDistance(x,y,d,grid): """Filters all (x,y) coordinates that are more than d in meshgrid given some actual coordinates (x,y). Args: x (numpy.ndarray): x-coordinates. y (numpy.ndarray): y-coordinates. d (float): Maximum distance. grid (numpy.ndarray): Numpy meshgrid. Returns: idxs (list): List of booleans. """ #Convert x/y into useful array xy=np.vstack((x,y)) #Compute distances to nearest neighbors tree = cKDTree(xy.T) xi = _ndim_coords_from_arrays(tuple(grid)) dists, indexes = tree.query(xi) #Get indices of distances that are further apart than d idxs = (dists > d) return idxs,
def __call__(self, xi, outp=None): """Do the interpolation. Modified for in-place evaluation support and without method choice. Evaluation points are to be given as an array with shape (n, dim), where n is the number of points. """ ndim = len(self.grid) if xi.ndim != 2: raise ValueError('`xi` has {} axes instead of 2.'.format(xi.ndim)) if xi.shape[1] != ndim: raise ValueError('`xi` has axis 1 with length {} instead ' 'of the grid dimension {}.'.format(xi.shape[1], ndim)) if outp is not None: if not isinstance(outp, np.ndarray): raise TypeError('`outp` {!r} not a `numpy.ndarray` ' 'instance.'.format(outp)) if outp.shape != (xi.shape[0],): raise ValueError('Output shape {} not equal to (n,), where ' 'n={} is the total number of evaluation ' 'points.'.format(outp.shape, xi.shape[0])) xi = _ndim_coords_from_arrays(xi, ndim=ndim) if xi.shape[-1] != ndim: raise ValueError('The requested sample points xi have dimension ' '{}, but this _NearestInterpolator has ' 'dimension {}.'.format(xi.shape[-1], ndim)) indices, norm_distances = self._find_indices(xi.T) return self._evaluate_nearest(indices, norm_distances, outp)
def interpolate_xarray_linear(xpoints, ypoints, values, shape, chunks=CHUNK_SIZE): """Interpolate linearly, generating a dask array.""" from scipy.interpolate.interpnd import (LinearNDInterpolator, _ndim_coords_from_arrays) if isinstance(chunks, (list, tuple)): vchunks, hchunks = chunks else: vchunks, hchunks = chunks, chunks points = _ndim_coords_from_arrays( np.vstack((np.asarray(ypoints), np.asarray(xpoints))).T) interpolator = LinearNDInterpolator(points, values) grid_x, grid_y = da.meshgrid(da.arange(shape[1], chunks=hchunks), da.arange(shape[0], chunks=vchunks)) # workaround for non-thread-safe first call of the interpolator: interpolator((0, 0)) res = da.map_blocks(intp, grid_x, grid_y, interpolator=interpolator) return DataArray(res, dims=('y', 'x'))
def __call__(self, *args): """ Evaluate interpolator at given points. Parameters ---------- xi : ndarray of float, shape (..., ndim) Points where to interpolate data at. """ from scipy.interpolate.interpnd import _ndim_coords_from_arrays xi = _ndim_coords_from_arrays(args, ndim=self.points.shape[1]) xi = self._check_call_shape(xi) xi = self._scale_x(xi) r, idxs = self.tree.query(xi, self.k_nearest, eps=self.query_eps, distance_upper_bound=self.max_distance or np.inf) w = getattr(self, '_' + self.kernel)(r).reshape((-1, self.k_nearest, 1)) + self.kernel_eps w /= np.sum(w, axis=1).reshape((-1, 1, 1)) yt = np.vstack((self.values, [0])) # if idxs[i, j] == len(values), then i:th point doesnt have j:th match yi = np.sum(yt[idxs, :] * w, axis=1) return yi
def gradient(self, *args): """Evaluate gradient of interpolator at the given points. """ ps = self.points xi = _ndim_coords_from_arrays(args, ndim=ps.shape[1]) xi = self._check_call_shape(xi) kdtree = scipy.spatial.cKDTree(xi) dmaps_kdtree = self.diffusion_maps._kdtree radius = self.diffusion_maps._cut_off grad = np.zeros_like(xi) distance_matrix \ = kdtree.sparse_distance_matrix(dmaps_kdtree, radius, output_type='coo_matrix') kernel_matrix \ = self.diffusion_maps.compute_kernel_matrix(distance_matrix) I, J = xi.shape[0], ps.shape[0] V = np.zeros((ps.shape[1], I, J)) for i in range(I): for j in range(J): V[:, i, j] = xi[i, :] - ps[j, :] for i in range(I): ri = np.squeeze(kernel_matrix.getrow(i).toarray()) ki = scipy.sparse.spdiags(ri, 0, J, J) ki_psi = ki @ np.squeeze(self.values) grad[i, :] = V[:, i, :] @ ki_psi return grad
def detrendedInterpolationLocal(self, data, flag=0, grid_method='linear'): """ Interpolate using a detrended approach Args: data: data to interpolate grid_method: scipy.interpolate.griddata interpolation method """ # take the new full_df and fill a data column df = self.full_df.copy() df['data'] = data[df['cell_local']].values df = df.set_index('cell_id') df['fit'] = df.groupby('cell_id').apply( lambda x: np.polyfit(x.elevation, x.data, 1)) # drop all the duplicates df.reset_index(inplace=True) df.drop_duplicates(subset=['cell_id'], keep='first', inplace=True) df.set_index('cell_id', inplace=True) df[['slope', 'intercept']] = df.fit.apply(pd.Series) # df = df.drop(columns='fit').reset_index() # apply trend constraints if flag == 1: df.loc[df['slope'] < 0, ['slope', 'intercept']] = 0 elif flag == -1: df.loc[df['slope'] > 0, ['slope', 'intercept']] = 0 # get triangulation if self.tri is None: xy = _ndim_coords_from_arrays( (self.metadata.utm_x, self.metadata.utm_y)) self.tri = qhull.Delaunay(xy) # interpolate the slope/intercept grid_slope = grid_interpolate_deconstructed(self.tri, df.slope.values[:], (self.GridX, self.GridY), method=grid_method) grid_intercept = grid_interpolate_deconstructed( self.tri, df.intercept.values[:], (self.GridX, self.GridY), method=grid_method) # remove the elevation trend from the HRRR precip el_trend = df.elevation * df.slope + df.intercept dtrend = df.data - el_trend # interpolate the residuals over the DEM idtrend = grid_interpolate_deconstructed(self.tri, dtrend, (self.GridX, self.GridY), method=grid_method) # reinterpolate rtrend = idtrend + grid_slope * self.GridZ + grid_intercept return rtrend
def griddata(points, values, xi, method='linear', fill_value=np.nan, tol = 1e-6): """modified griddata plus tol arg""" points = _ndim_coords_from_arrays(points) if points.ndim < 2: ndim = points.ndim else: ndim = points.shape[-1] if ndim == 1 and method in ('nearest', 'linear', 'cubic'): from interpolate import interp1d points = points.ravel() if isinstance(xi, tuple): if len(xi) != 1: raise ValueError("invalid number of dimensions in xi") xi, = xi # Sort points/values together, necessary as input for interp1d idx = np.argsort(points) points = points[idx] values = values[idx] ip = interp1d(points, values, kind=method, axis=0, bounds_error=False, fill_value=fill_value) return ip(xi) elif method == 'nearest': ip = NearestNDInterpolator(points, values) return ip(xi) elif method == 'linear': ip = LinearNDInterpolator(points, values, fill_value=fill_value) return ip(xi) elif method == 'cubic' and ndim == 2: ip = CloughTocher2DInterpolator(points, values, fill_value=fill_value, tol = tol) return ip(xi) else: raise ValueError("Unknown interpolation method %r for " "%d dimensional data" % (method, ndim))
def getFuchsInterp( Temperature, name='density', Uset = [4,6,8,10,12]): U_, mu_, qty_ = getFuchsPoints( Temperature, name=name, Uset = Uset) points = _ndim_coords_from_arrays((U_, mu_)) if len ( Uset) > 1: interp = CloughTocher2DInterpolator(points, qty_) return lambda u,mu : interp( np.column_stack(( np.ravel(u),np.ravel(mu) )) ).reshape(u.shape) else: return None
def __call__(self, xi, method=None): """ Interpolation at coordinates Parameters ---------- xi : ndarray of shape (..., ndim) The coordinates to sample the gridded data at method : str The method of interpolation to perform. Supported are "linear" and "nearest". """ method = self.method if method is None else method if method not in ["linear", "nearest", "kNN"]: raise ValueError("Method '%s' is not defined" % method) ndim = len(self.grid) xi = _ndim_coords_from_arrays(xi, ndim=ndim) if xi.shape[-1] != len(self.grid): raise ValueError("The requested sample points xi have dimension " "%d, but this RegularGridInterpolator has " "dimension %d" % (xi.shape[1], ndim)) xi_shape = xi.shape xi = xi.reshape(-1, xi_shape[-1]) if self.bounds_error: for i, p in enumerate(xi.T): if not np.logical_and(np.all(self.grid[i][0] <= p), np.all(p <= self.grid[i][-1])): raise ValueError( "One of the requested xi is out of bounds " "in dimension %d" % i) indices, norm_distances, out_of_bounds = self._find_indices(xi.T) if method == "linear": result = self._evaluate_linear(indices, norm_distances, out_of_bounds) elif method == "nearest": result = self._evaluate_nearest(indices, norm_distances, out_of_bounds) elif method == "kNN": result = self._evaluate_NN(indices, norm_distances) if not self.bounds_error and self.fill_value is not None: result[out_of_bounds] = self.fill_value return result.reshape(xi_shape[:-1] + self.values.shape[ndim:])
def interpolate_xarray_linear(xpoints, ypoints, values, shape): """Interpolate linearly, generating a dask array.""" from scipy.interpolate.interpnd import (LinearNDInterpolator, _ndim_coords_from_arrays) points = _ndim_coords_from_arrays(np.vstack((np.asarray(ypoints), np.asarray(xpoints))).T) interpolator = LinearNDInterpolator(points, values) def intp(grid_x, grid_y, interpolator): return interpolator((grid_y, grid_x)) grid_x, grid_y = da.meshgrid(da.arange(shape[1], chunks=CHUNK_SIZE), da.arange(shape[0], chunks=CHUNK_SIZE)) # workaround for non-thread-safe first call of the interpolator: interpolator((0, 0)) res = da.map_blocks(intp, grid_x, grid_y, interpolator=interpolator) return DataArray(res, dims=('y', 'x'))
def griddata(points, values, xi, method='linear', fill_value=np.nan, tol=1e-6): """modified griddata plus tol arg""" points = _ndim_coords_from_arrays(points) if points.ndim < 2: ndim = points.ndim else: ndim = points.shape[-1] if ndim == 1 and method in ('nearest', 'linear', 'cubic'): from interpolate import interp1d points = points.ravel() if isinstance(xi, tuple): if len(xi) != 1: raise ValueError("invalid number of dimensions in xi") xi, = xi # Sort points/values together, necessary as input for interp1d idx = np.argsort(points) points = points[idx] values = values[idx] ip = interp1d(points, values, kind=method, axis=0, bounds_error=False, fill_value=fill_value) return ip(xi) elif method == 'nearest': ip = NearestNDInterpolator(points, values) return ip(xi) elif method == 'linear': ip = LinearNDInterpolator(points, values, fill_value=fill_value) return ip(xi) elif method == 'cubic' and ndim == 2: ip = CloughTocher2DInterpolator(points, values, fill_value=fill_value, tol=tol) return ip(xi) else: raise ValueError("Unknown interpolation method %r for " "%d dimensional data" % (method, ndim))
def __call__(self, *args): """Evaluate interpolator at the given points. """ xi = _ndim_coords_from_arrays(args, ndim=self.points.shape[1]) xi = self._check_call_shape(xi) kdtree = scipy.spatial.cKDTree(xi) dmaps_kdtree = self.diffusion_maps._kdtree radius = self.diffusion_maps._cut_off ew = self.diffusion_maps.eigenvalues ev = self.diffusion_maps.eigenvectors aux = ev.T @ np.diag(1.0 / ew) @ ev @ self.values distance_matrix \ = kdtree.sparse_distance_matrix(dmaps_kdtree, radius, output_type='coo_matrix') kernel_matrix \ = self.diffusion_maps.compute_kernel_matrix(distance_matrix) return np.squeeze(kernel_matrix @ aux)
def xyz2grd(point_lons, point_lats, point_zvals, grid_lons, grid_lats): # https://stackoverflow.com/questions/30655749/how-to-set-a-maximum-distance-between-points-for-interpolation-when-using-scipy if grid_lons.ndim == 1: grid_lons, grid_lats = np.meshgrid(grid_lons, grid_lats) grid_sampling = np.abs(grid_lats[1] - grid_lats[0]) xy = np.vstack((point_lons, point_lats)).T # Construct kd-tree, functionality copied from scipy.interpolate tree = cKDTree(xy) xi = _ndim_coords_from_arrays((grid_lons, grid_lats), ndim=xy.shape[1]) dists, indexes = tree.query(xi) grid_interp = spi.griddata(xy, point_zvals, (grid_lons, grid_lats), method='nearest') # Copy original result but mask missing values with NaNs result = grid_interp[:] result[dists > grid_sampling / 2.] = np.nan return result
def _remap_scene_nearest(self, swath_scene, grid_def, share_dynamic_grids=True, share_remap_mask=True, **kwargs): # TODO: Make methods more flexible than just a function call gridded_scene = GriddedScene() grid_name = grid_def["grid_name"] # Group products together that shared the same geolocation product_groups = defaultdict(list) for product_name, swath_product in swath_scene.items(): swath_def = swath_product["swath_definition"] geo_id = swath_def["swath_name"] product_groups[geo_id].append(product_name) grid_coverage = kwargs.get("grid_coverage", GRID_COVERAGE) orig_grid_def = grid_def for geo_id, product_names in product_groups.items(): pp_names = "\n\t".join(product_names) LOG.debug( "Running ll2cr on the geolocation data for the following products:\n\t%s", pp_names) LOG.debug("Swath name: %s", geo_id) # TODO: Move into it's own function if this gets complicated # TODO: Add some multiprocessing try: swath_def = swath_scene[product_names[0]]["swath_definition"] if not share_dynamic_grids: grid_def = orig_grid_def.copy() cols_fn, rows_fn = self.run_ll2cr(swath_def, grid_def) except StandardError: LOG.error("Remapping error") if self.exit_on_error: raise continue LOG.debug( "Running nearest neighbor for the following products:\n\t%s", "\n\t".join(product_names)) edge_res = swath_def.get("limb_resolution", None) if kwargs.get("distance_upper_bound", None) is None: if edge_res is not None: if grid_def.is_latlong: distance_upper_bound = (edge_res / 2) / grid_def.cell_width_meters else: distance_upper_bound = (edge_res / 2) / grid_def["cell_width"] LOG.debug("Distance upper bound dynamically set to %f", distance_upper_bound) else: distance_upper_bound = 3.0 kwargs["distance_upper_bound"] = distance_upper_bound try: grid_x, grid_y = numpy.mgrid[:grid_def["height"], : grid_def["width"]] # we need flattened versions of these shape = (swath_def["swath_rows"] * swath_def["swath_columns"], ) cols_array = numpy.memmap(cols_fn, shape=shape, dtype=swath_def["data_type"]) rows_array = numpy.memmap(rows_fn, shape=shape, dtype=swath_def["data_type"]) good_mask = ~mask_helper(cols_array, swath_def["fill_value"]) if share_remap_mask: for product_name in product_names: LOG.debug( "Combining data masks before building KDTree for nearest neighbor: %s", product_name) good_mask &= ~swath_scene[product_name].get_data_mask( ).ravel() x = _ndim_coords_from_arrays( (cols_array[good_mask], rows_array[good_mask])) xi = _ndim_coords_from_arrays((grid_y, grid_x)) dist, i = cKDTree(x).query( xi, distance_upper_bound=kwargs["distance_upper_bound"]) except StandardError: LOG.debug("Remapping exception: ", exc_info=True) LOG.error("Remapping error") if self.exit_on_error: self._clear_ll2cr_cache() raise continue product_filepaths = swath_scene.get_data_filepaths(product_names) output_filepaths = self._add_prefix("grid_%s_" % (grid_name, ), *product_filepaths) # Prepare the products fill_value = numpy.nan for product_name, output_fn in izip(product_names, output_filepaths): LOG.debug( "Running nearest neighbor on '%s' with search distance %f", product_name, kwargs["distance_upper_bound"]) if os.path.isfile(output_fn): if not self.overwrite_existing: LOG.error( "Intermediate remapping file already exists: %s" % (output_fn, )) raise RuntimeError( "Intermediate remapping file already exists: %s" % (output_fn, )) else: LOG.warning( "Intermediate remapping file already exists, will overwrite: %s", output_fn) try: image_array = swath_scene[product_name].get_data_array( ).ravel() values = numpy.append(image_array[good_mask], image_array.dtype.type(fill_value)) output_array = values[i] output_array.tofile(output_fn) # Give the gridded product ownership of the remapped data swath_product = swath_scene[product_name] gridded_product = GriddedProduct() gridded_product.from_swath_product(swath_product) gridded_product["grid_definition"] = grid_def gridded_product["fill_value"] = fill_value gridded_product["grid_data"] = output_fn # Check grid coverage valid_points = numpy.count_nonzero( ~gridded_product.get_data_mask()) grid_covered_ratio = valid_points / float( grid_def["width"] * grid_def["height"]) grid_covered = grid_covered_ratio > grid_coverage if not grid_covered: msg = "Nearest neighbor resampling only found %f%% of the grid covered (need %f%%) for %s" % ( grid_covered_ratio * 100, grid_coverage * 100, product_name) LOG.warning(msg) continue LOG.debug( "Nearest neighbor resampling found %f%% of the grid covered for %s" % (grid_covered_ratio * 100, product_name)) gridded_scene[product_name] = gridded_product # hopefully force garbage collection del output_array except StandardError: LOG.debug("Remapping exception: ", exc_info=True) LOG.error("Remapping error") self._safe_remove(output_fn) if self.exit_on_error: self._clear_ll2cr_cache() raise continue LOG.debug("Done running nearest neighbor on '%s'", product_name) # Remove ll2cr files now that we are done with them self._clear_ll2cr_cache() if not gridded_scene: raise RuntimeError( "Nearest neighbor resampling could not remap any of the data to grid '%s'" % (grid_name, )) return gridded_scene
def plot_map(df, data_type='AOD', lat=(-90, 90), lon=(-180, 180), plot_type='pcolormesh', show=True, grid_size=0.5, vmin=None, vmax=None, show_limits=True): ''' For a (list of) DataFrame(s) this function will plot the average of the total or dust AOD onto a map. For a (list of) MatchFrame(s) several different values can be plotted on a map. These are the average difference in AOD or time (data_set[1] - data_set[0]), the local RMS values, or the number of matched-up data points. If AERONET data is included then the individual sites will be plotted, otherwise it will be plotted on a grid. Parameters ---------- df : AeroCT DataFrame / MatchFrame, or list of DataFrames / MatchFrames data_type : str, optional (Default: 'AOD') This describes which type of data to plot. If df is a DataFrame(s) then it describes the type of AOD data to plot. It can take any one of the values 'AOD', 'dust AOD', or 'total AOD'. If 'AOD' is chosen then the total AOD will be plotted, unless the data frame has only dust AOD in which case that will be plotted. If df is a MatchFrame(s) then it can take the any of the values: 'AOD' - Plot the mean AOD bias between the two data sets for each location. 'RMS' - Plot the local RMS values. 'time diff' - Plot the average time difference between the two data sets. 'heatmap' - Plot the number of matched-up data points within each location. lat : tuple, optional (Default: (-90, 90)) A tuple of the latitude bounds of the plot in degrees. lon : tuple, optional (Default: (-180, 180)) A tuple of the longitude bounds of the plot in degrees. plot_type : str, optional (Default: 'pcolormesh') The type of plot to produce if it does not contain AERONET data. 'scatter' to plot a scatter grid of all AOD data, and 'pcolormesh' or 'contourf' for griddded plots. The grid cell values for gridded plots are found by using a cubic interpolation scheme with scipy.interpolate.griddata(). show : bool, optional (Default: True) If True the figure is shown, otherwise it is returned grid_size : float, optional (Default: 0.5) The size of the grid squares in degrees if not using indivdual sites vmin, vmax : scalar, optional (Default: None) vmin and vmax are used to set limits for the color map. If either is None, it is autoscaled to the respective min or max of the plotted data. show_limits : bool, optional (Default: True) If True, the minimum and maximum of the data is shown under the plot. ''' # Convert a list of match frames to a single match frame that may be plotted if isinstance(df, list): df = aeroct.concatenate_data_frames(df) date = '{0} to {1}'.format(df.date[0].date(), df.date[-1].date()) elif isinstance(df.date, list): date = '{0} to {1}'.format(df.date[0].date(), df.date[-1].date()) else: date = df.date.date() fig = plt.figure(figsize=(8, 6)) ax = plt.axes(projection=ccrs.PlateCarree()) plt.xlim(lon) plt.ylim(lat) mon_cmap = cm.get_cmap('inferno_r') div_cmap = cm.get_cmap('RdYlBu_r') # USE IRIS PLOT IF THERE IS A CUBE IN THE DATA FRAME if df.cube is not None: # Suppress warnings and plot with warnings.catch_warnings(): warnings.simplefilter("ignore", category=UserWarning) day_avg_cube = df.cube.collapsed('time', analysis.MEAN) if df.__class__.__name__ == 'DataFrame': plt.title('{0}: Dust AOD Mean For {1}'.format(df.name, date)) cmap = mon_cmap data = day_avg_cube.aod else: plt.title('Dust AOD Difference (Mean) ({0} - {1}) for {2}'\ .format(df.names[1], df.names[0], date)) cmap = shiftedColorMap(div_cmap, day_avg_cube.data, vmin, vmax) data = day_avg_cube.data if plot_type == 'pcolormesh': iplt.pcolormesh(day_avg_cube, cmap=cmap, vmin=vmin, vmax=vmax) elif plot_type == 'contourf': iplt.contourf(day_avg_cube, cmap=cmap, vmin=vmin, vmax=vmax) elif df.__class__.__name__ == 'DataFrame': # Select the total or dust AOD data which is in the given bounds if data_type == 'AOD': aod, lons, lats = df.get_data(None)[:3] aod_type = df.get_data(None, return_type=True)[-1] elif data_type in ['total', 'total AOD']: aod, lons, lats = df.get_data('total')[:3] aod_type = 'total' elif data_type in ['dust', 'dust AOD']: aod, lons, lats = df.get_data('dust')[:3] aod_type = 'dust' plt.title('{0}: {1} AOD Mean For {2}'.format(df.name, aod_type.title(), date)) cmap = mon_cmap in_bounds = (lon[0] < lons) & (lons < lon[1]) & (lat[0] < lats) & ( lats < lat[1]) data = aod[in_bounds] lons = lons[in_bounds] lats = lats[in_bounds] # PLOT AOD AT AERONET SITES AS A SCATTER PLOT if df.data_set == 'aeronet': # Get the list of data points at each location site_lons, i_site = np.unique(lons, return_index=True) site_lats = lats[i_site] in_sites = site_lons[:, np.newaxis] == lons # Average the AOD at each site and take std data = np.mean(data * in_sites, axis=1) # aod_site_std = np.sqrt(np.mean(data**2 * in_sites, axis=1) - aod_site_avg**2) plt.scatter(site_lons, site_lats, c=data, cmap=cmap, edgecolors='k', linewidths=0.2, s=100, vmin=vmin, vmax=vmax) # PLOT THE AOD AT EVERY DATA POINT elif plot_type == 'scatter': plt.scatter(lons, lats, c=data, marker='o', s=(50. / fig.dpi)**2, vmin=vmin, vmax=vmax) # OTHERWISE PLOT A GRID else: # Using scipy.interpolate.griddata # First get the axes grid = np.mgrid[(lon[0] + grid_size / 2):lon[1]:grid_size, (lat[0] + grid_size / 2):lat[1]:grid_size] ll = zip(lons, lats) data = griddata(ll, data, tuple(grid), method='linear') # Mask grid data where there are no nearby points. Firstly create kd-tree THRESHOLD = 2 * grid_size # Maximum distance to look for nearby points tree = cKDTree(ll) xi = _ndim_coords_from_arrays(tuple(grid)) dists = tree.query(xi)[0] # Copy original result but mask missing values with NaNs data[dists > THRESHOLD] = np.nan if plot_type == 'pcolormesh': plt.pcolormesh(grid[0], grid[1], data, cmap=cmap, vmin=vmin, vmax=vmax) elif plot_type == 'contourf': plt.contourf(grid[0], grid[1], data, cmap=cmap, vmin=vmin, vmax=vmax) elif df.__class__.__name__ == 'MatchFrame': # Find the data within the given longitude and latitude bounds in_bounds = (df.longitudes > lon[0]) & (df.longitudes < lon[1]) & \ (df.latitudes > lat[0]) & (df.latitudes < lat[1]) lons = df.longitudes[in_bounds] lats = df.latitudes[in_bounds] # Get the data for which to find the area average and the title if data_type == 'time diff': data = df.time_diff[in_bounds] plt.title('Mean Time Difference ({0} - {1})\n for {2}'\ .format(df.names[1], df.names[0], date)) elif data_type == 'RMS': data = (df.data[1, in_bounds] - df.data[0, in_bounds])**2 plt.title('Root Mean Square ({0}, {1})\n for {2}'\ .format(df.names[0], df.names[1], date)) elif data_type == 'heatmap': data = np.ones_like(df.data[1, in_bounds]) plt.title('Number Of Matched Data Points ({0}, {1})\n for {2}'\ .format(df.names[0], df.names[1], date)) else: data = df.data[1, in_bounds] - df.data[0, in_bounds] plt.title('Mean AOD Difference ({0} - {1})\n for {2}'\ .format(df.names[1], df.names[0], date)) # If AERONET is included plot the sites on a map if np.any([df.data_sets[i] == 'aeronet' for i in [0, 1]]): # Get the list of data points at each location site_lons, i_site = np.unique(lons, return_index=True) site_lats = lats[i_site] in_sites = site_lons[:, np.newaxis] == lons # Average the AOD at each site and take std data = np.mean(data * in_sites, axis=1) if data_type == 'RMS': data = np.sqrt(data) cmap = mon_cmap elif data_type == 'heatmap': data = np.sum(in_sites, axis=1) cmap = mon_cmap else: # Shift colour map to have a midpoint of zero cmap = shiftedColorMap(div_cmap, data, vmin, vmax) plt.scatter(site_lons, site_lats, c=data, s=(500 / fig.dpi)**2, edgecolors='k', linewidths=0.2, cmap=cmap, vmin=vmin, vmax=vmax) # OTHERWISE PLOT A GRID else: # Using scipy.interpolate.griddata # First get the axes grid = np.mgrid[(lon[0] + grid_size / 2):lon[1]:grid_size, (lat[0] + grid_size / 2):lat[1]:grid_size] ll = zip(lons, lats) if data_type == 'heatmap': lon_edges = np.arange(lon[0], lon[1] + 1e-5, grid_size) lat_edges = np.arange(lat[0], lat[1] + 1e-5, grid_size) data = np.histogram2d(lons, lats, [lon_edges, lat_edges])[0] else: data = griddata(ll, data, tuple(grid), method='cubic') # Mask grid data where there are no nearby points. Firstly create kd-tree THRESHOLD = grid_size # Maximum distance to look for nearby points tree = cKDTree(ll) xi = _ndim_coords_from_arrays(tuple(grid)) dists = tree.query(xi)[0] # Copy original result but mask missing values with NaNs data[dists > THRESHOLD] = np.nan if data_type == 'RMS': data = np.sqrt(data) if data_type in ['RMS', 'heatmap']: cmap = mon_cmap else: # Shift colour map to have a midpoint of zero cmap = shiftedColorMap(div_cmap, data, vmin, vmax) if plot_type == 'pcolormesh': plt.pcolormesh(grid[0], grid[1], data, cmap=cmap, vmin=vmin, vmax=vmax) elif plot_type == 'contourf': plt.contourf(grid[0], grid[1], data, cmap=cmap, vmin=vmin, vmax=vmax) if show_limits: limits = (np.nanmin(data), np.nanmax(data)) fig.text(0.5, 0.22, 'Min: {0:.04f} Max: {1:.04f}'.format(limits[0], limits[1]), horizontalalignment='center', verticalalignment='center', fontsize=12) ax.coastlines() plt.colorbar(orientation='horizontal') fig.tight_layout() if show == True: plt.show() return else: return fig
def _remap_scene_nearest(self, swath_scene, grid_def, share_dynamic_grids=True, share_remap_mask=True, **kwargs): # TODO: Make methods more flexible than just a function call gridded_scene = GriddedScene() grid_name = grid_def["grid_name"] # Group products together that shared the same geolocation product_groups = defaultdict(list) for product_name, swath_product in swath_scene.items(): swath_def = swath_product["swath_definition"] geo_id = swath_def["swath_name"] product_groups[geo_id].append(product_name) grid_coverage = kwargs.get("grid_coverage", GRID_COVERAGE) orig_grid_def = grid_def for geo_id, product_names in product_groups.items(): pp_names = "\n\t".join(product_names) LOG.debug("Running ll2cr on the geolocation data for the following products:\n\t%s", pp_names) LOG.debug("Swath name: %s", geo_id) # TODO: Move into it's own function if this gets complicated # TODO: Add some multiprocessing try: swath_def = swath_scene[product_names[0]]["swath_definition"] if not share_dynamic_grids: grid_def = orig_grid_def.copy() cols_fn, rows_fn = self.run_ll2cr(swath_def, grid_def) except StandardError: LOG.error("Remapping error") if self.exit_on_error: raise continue LOG.debug("Running nearest neighbor for the following products:\n\t%s", "\n\t".join(product_names)) edge_res = swath_def.get("limb_resolution", None) if kwargs.get("distance_upper_bound", None) is None: if edge_res is not None: if grid_def.is_latlong: distance_upper_bound = (edge_res / 2) / grid_def.cell_width_meters else: distance_upper_bound = (edge_res / 2) / grid_def["cell_width"] LOG.debug("Distance upper bound dynamically set to %f", distance_upper_bound) else: distance_upper_bound = 3.0 kwargs["distance_upper_bound"] = distance_upper_bound try: grid_x, grid_y = numpy.mgrid[:grid_def["height"], :grid_def["width"]] # we need flattened versions of these shape = (swath_def["swath_rows"] * swath_def["swath_columns"],) cols_array = numpy.memmap(cols_fn, shape=shape, dtype=swath_def["data_type"]) rows_array = numpy.memmap(rows_fn, shape=shape, dtype=swath_def["data_type"]) good_mask = ~mask_helper(cols_array, swath_def["fill_value"]) if share_remap_mask: for product_name in product_names: LOG.debug("Combining data masks before building KDTree for nearest neighbor: %s", product_name) good_mask &= ~swath_scene[product_name].get_data_mask().ravel() x = _ndim_coords_from_arrays((cols_array[good_mask], rows_array[good_mask])) xi = _ndim_coords_from_arrays((grid_y, grid_x)) dist, i = cKDTree(x).query(xi, distance_upper_bound=kwargs["distance_upper_bound"]) except StandardError: LOG.debug("Remapping exception: ", exc_info=True) LOG.error("Remapping error") if self.exit_on_error: self._clear_ll2cr_cache() raise continue product_filepaths = swath_scene.get_data_filepaths(product_names) output_filepaths = self._add_prefix("grid_%s_" % (grid_name,), *product_filepaths) # Prepare the products fill_value = numpy.nan for product_name, output_fn in izip(product_names, output_filepaths): LOG.debug("Running nearest neighbor on '%s' with search distance %f", product_name, kwargs["distance_upper_bound"]) if os.path.isfile(output_fn): if not self.overwrite_existing: LOG.error("Intermediate remapping file already exists: %s" % (output_fn,)) raise RuntimeError("Intermediate remapping file already exists: %s" % (output_fn,)) else: LOG.warning("Intermediate remapping file already exists, will overwrite: %s", output_fn) try: image_array = swath_scene[product_name].get_data_array().ravel() values = numpy.append(image_array[good_mask], image_array.dtype.type(fill_value)) output_array = values[i] output_array.tofile(output_fn) # Give the gridded product ownership of the remapped data swath_product = swath_scene[product_name] gridded_product = GriddedProduct() gridded_product.from_swath_product(swath_product) gridded_product["grid_definition"] = grid_def gridded_product["fill_value"] = fill_value gridded_product["grid_data"] = output_fn # Check grid coverage valid_points = numpy.count_nonzero(~gridded_product.get_data_mask()) grid_covered_ratio = valid_points / float(grid_def["width"] * grid_def["height"]) grid_covered = grid_covered_ratio > grid_coverage if not grid_covered: msg = "Nearest neighbor resampling only found %f%% of the grid covered (need %f%%) for %s" % (grid_covered_ratio * 100, grid_coverage * 100, product_name) LOG.warning(msg) continue LOG.debug("Nearest neighbor resampling found %f%% of the grid covered for %s" % (grid_covered_ratio * 100, product_name)) gridded_scene[product_name] = gridded_product # hopefully force garbage collection del output_array except StandardError: LOG.debug("Remapping exception: ", exc_info=True) LOG.error("Remapping error") self._safe_remove(output_fn) if self.exit_on_error: self._clear_ll2cr_cache() raise continue LOG.debug("Done running nearest neighbor on '%s'", product_name) # Remove ll2cr files now that we are done with them self._clear_ll2cr_cache() if not gridded_scene: raise RuntimeError("Nearest neighbor resampling could not remap any of the data to grid '%s'" % (grid_name,)) return gridded_scene
def find_closest_nlce( U=8, T=0.67, mu=4., qty='dens', **kwargs): """ This function finds the closest values of U and T in the NLCE data that straddle the values U and T given as arguments. """ msg0 = "U={:0.2f}, T={:0.2f}".format( U, T) nUs = 3 us = [ float(u.split('/U')[-1]) for u in \ glob.glob( ldaconf.basedir + 'NLCE8_FinalK/U*' ) ] du = [ np.abs( U-u ) for u in us ] index = np.argsort( du ) Ulist0 = range( nUs ) Upts = [ us[index[i]] for i in Ulist0 ] # The T points are not uniformly spaced so we find the two closest ones # We start with a list of available T points: Ts = np.array( sorted( [ float(g.split('/T')[1].split('.dat')[0]) for g in \ glob.glob(ldaconf.basedir + 'NLCE8_FinalK/U00/T*') ] ) ) diff = T-Ts error = False if np.all( diff < 0 ): print "Available temperatures do not make it this low:" print " T = ", T error = True if not error: order_pos = np.argsort(np.abs( diff[diff>0] )) order_neg = np.argsort(np.abs( diff[diff<0] )) Tpts = sorted( [ Ts[diff>0][ order_pos[0] ] , Ts[diff<0][ order_neg[0]] ] ) else: order = np.argsort( np.abs( diff ) ) Tpts = sorted( [ Ts[order[0]], Ts[order[1]], Ts[order[2]] ] ) #Ta = min(Ts[order[0]], Ts[order[1]]) #Tb = max(Ts[order[0]], Ts[order[1]]) #print "T in ", Ta, Tb datadir = ldaconf.basedir datfiles = [] for Uval in Upts: for Tval in Tpts: fname = datadir + \ 'NLCE8_FinalK/U{:02d}/T{:0.2f}.dat'.format(int(Uval),Tval) datfiles.append([ fname, Uval, Tval ]) if qty == 'dens': COL = 1 elif qty == 'entr': COL = 2 elif qty == 'spi': COL = 3 elif qty == 'kappa': COL = 4 else: raise "Qty not defined:", qty MUCOL = 0 basedat = [] qtyinterp = kwargs.get( 'qtyinterp', 'nearest' ) for f in datfiles: msg = 'U={:0.2f}, T={:0.2f}'.format(U,T) + \ ' mu={:0.2f}, Upt={:0.3f}, Tpt={:0.3f}'.\ format(mu, f[1], f[2]) try: dat = np.loadtxt(f[0]) # careful = False, for NLCE data it is ok to not worry about # chemical potentials that are outside the data range qtyresult = get_qty_mu( dat, mu, MUCOL, COL, msg=msg, careful=False) if qtyresult == 'out-of-bounds': print msg print "out-of-bounds" continue basedat.append( [f[1], f[2], qtyresult] ) except Exception as e: print "Failed to get data from file = ", f print msg fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.plot( dat[:,MUCOL], dat[:,COL], '.-') ax.axvline( mu ) ax.text( 0.5, 1.05, msg, ha='center', va='bottom', \ transform=ax.transAxes) if matplotlib.get_backend() == 'agg': fig.savefig('err_mu.png', dpi=200) else: plt.show() raise e basedat = np.array(basedat) #print "Closest dat = ", basedat points = _ndim_coords_from_arrays(( basedat[:,0] , basedat[:,1])) #finterp = CloughTocher2DInterpolator(points, basedat[:,2]) finterp = LinearNDInterpolator( points, basedat[:,2] ) try: result = finterp( U,T ) if np.isnan(result): if U >= 30 and U<=32.5: result = finterp( 29.99, T ) logger.warning(" nlce: U={:0.1f} replaced to U=29.99 ".format(U) ) if np.isnan(result): raise Exception("!!!! nlce: Invalid result !!!!\n" + msg0) except Exception as e: print e error = True if error or kwargs.get('showinterp',False): #print "Interp points:" #print basedat tri = Delaunay(points) fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.triplot(points[:,0], points[:,1], tri.simplices.copy()) ax.plot(points[:,0], points[:,1], 'o') ax.plot( U, T, 'o', ms=6., color='red') xlim = ax.get_xlim() dx = (xlim[1]-xlim[0])/10. ax.set_xlim( xlim[0]-dx, xlim[1]+dx ) ylim = ax.get_ylim() dy = (ylim[1]-ylim[0])/10. ax.set_ylim( ylim[0]-dy, ylim[1]+dy ) ax.set_xlabel('$U/t$') ax.set_ylabel('$T/t$',rotation=0,labelpad=8) tt = kwargs.get('title_text','') ax.set_title( tt + '$U/t={:.2f}$'.format(U) + ',\ \ ' + '$T/t={:.2f}$'.format(T), \ ha='center', va='bottom', fontsize=10) save_err = kwargs.get('save_err',None) if save_err is not None: print "saving png to ", save_err fig.savefig( save_err, dpi=300) if matplotlib.get_backend() == 'agg': print "saving png to err.png" fig.savefig('err.png', dpi=200) else: plt.show() raise return result
def find_closest_qmc( U=8, T=0.67, mu=4.0, **kwargs): """ This function finds the closest values of U and T in the QMC data that straddle the values U and T given as arguments. """ nUs = 4 nTs = 3 ALLPTS = kwargs.get('ALLPTS', False) # select which quantity will be returned, options are # spi and entropy QTY = kwargs.get('QTY', 'spi' ) #datadir = '/home/pmd/sandbox/hubbard-lda/QMC_Final/' #datadir = '/home/pmd/sandbox/hubbard-lda/COMBINED_Final/' if QTY == 'spi': datadir = '/home/pmd/sandbox/hubbard-lda/COMB_Final_Spi/' elif QTY == 'entropy': datadir = '/home/pmd/sandbox/hubbard-lda/COMB_Final_Entr/' elif QTY == 'density': datadir = '/home/pmd/sandbox/hubbard-lda/COMB_Final_Spi/' else: raise ValueError('Quantity not defined:' + str(QTY) ) fname = datadir + 'U*' us = [ float(u.split('/U')[-1]) for u in glob.glob(fname) ] du = [ np.abs(U-u) for u in us ] index = np.argsort(du) if ALLPTS: Ulist0 = range(len(index)) else: Ulist0 = range( nUs ) us = [ us[index[i]] for i in Ulist0] #print us #print du #print index #print "Closest Us = ", us datfiles = [] for u in us: # For the Spi and Stheta data if QTY == 'spi' or QTY == 'density': fname = datadir + 'U{U:02d}/T*dat'.format(U=int(u)) fs = sorted(glob.glob(fname)) Ts = [ float(f.split('T')[1].split('.dat')[0]) for f in fs ] elif QTY=='entropy': fname = datadir + 'U{U:02d}/S*dat'.format(U=int(u)) fs = sorted(glob.glob(fname)) Ts = [ float(f.split('S')[1].split('.dat')[0]) for f in fs ] Ts_g = [] ; Ts_l = []; for t in Ts: if t > T: Ts_g.append(t) else: Ts_l.append(t) order_g = np.argsort( [ np.abs( T -t ) for t in Ts_g ] ) order_l = np.argsort( [ np.abs( T -t ) for t in Ts_l ] ) try: Tpts = [ Ts_g[ order_g[0]] , Ts_l[ order_l[0]] ] except: #print #print "problem adding U=",u, "T=",Ts #print "available T data does not stride the point" #print "T =", T #print "Ts =", Ts #print "will add nearest Ts nevertheless" Tpts = [ ] #raise ValueError("QMC data not available.") dT = [ np.abs( T - t) for t in Ts ] index = np.argsort(dT) if ALLPTS: Tlist0 = range(len(Ts)) else: Tlist0 = range( min(nTs , len(Ts))) for i in Tlist0: Tnew = Ts[index[i]] if Tnew not in Tpts: Tpts.append(Tnew) for Tpt in Tpts: index = Ts.index( Tpt ) try: datfiles.append( [ fs[ index ], u, Ts[index] ] ) except: print "problem adding U=",u, "T=",Ts raise # Need to make sure that selected T values stride both # sides of the point #print #print u #print Ts #print dT #print index #print fs # for i in range(min(3, len(Ts))): # try: # datfiles.append( [ fs[index[i]], u, Ts[index[i]] ] ) # except: # print "problem adding U=",u, "T=",Ts # raise # #datfiles.append( [ fs[index[1]], u, Ts[index[1]] ] ) #print datfiles MUCOL = 0 DENSCOL = 1 ENTRCOL = 2 SPICOL = 3 if QTY == 'spi': COL = SPICOL elif QTY == 'entropy': COL = ENTRCOL elif QTY == 'density': COL = DENSCOL basedat = [] for f in datfiles: # f[0] is the datafile name # f[1] is U # f[2] is T radius = kwargs.get('radius', np.nan ) msg = 'U={:0.2f}, T={:0.2f}'.format(U,T) + \ ' mu={:0.2f}, r={:0.2f}, Upt={:0.3f}, Tpt={:0.3f}'.\ format(mu, radius, f[1], f[2]) try: dat = np.loadtxt(f[0]) spival = get_qty_mu( dat, mu, MUCOL, COL, msg=msg ) if spival == 'out-of-bounds': continue #return 'out-of-bounds' basedat.append( [f[1], f[2], spival] ) except Exception as e : print "Failed to get data from file = ", f # toggle plotting, not implemented yet: #if showqtyinterp: fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.plot( dat[:,MUCOL], dat[:,COL], '.-') ax.axvline( mu ) ax.text( 0.5, 1.05, msg, ha='center', va='bottom', \ transform=ax.transAxes) plt.show() raise e error = False try: basedat = np.array(basedat) Us = np.unique(basedat[:,0] ) Ts = np.unique(basedat[:,1] ) validTriang = not ( len(Us) ==1 or len(Ts) == 1 ) #print "#Us={:d}, #Ts={:d}".format( len(Us), len(Ts) ) #print msg if validTriang: points = _ndim_coords_from_arrays(( basedat[:,0] , basedat[:,1])) #print "Closest dat = ", basedat #finterp = CloughTocher2DInterpolator(points, basedat[:,2]) finterp = LinearNDInterpolator( points, basedat[:,2] ) else: raise ValueError('not enough finterp points') except Exception as e : print "QTY=%s -> Interp Error:"%QTY, msg print e error = True if error == False: try: result = finterp( U,T ) if np.isnan(result): raise Exception("\n!!!! Invalid result, QTY:%s!!!!\n"%QTY) except Exception as e: if kwargs.get('error_nan', False): return np.nan else: print e error = True if error == False: if result >= 8. and QTY == 'spi' : print " Obtained Spi > 8. : U={:0.2f}, T={:0.2f}, mu={:0.2f}".\ format( U, T, mu ), print " ==> Spi={:0.2f}".format(float(result)) error = True elif result >=4. and QTY == 'entropy': print " Obtained Ent > 4. : U={:0.2f}, T={:0.2f}, mu={:0.2f}".\ format( U, T, mu ), print " ==> Result={:0.2f}".format(float(result)) error = True if error or kwargs.get('showinterp',False): if kwargs.get('error_nan', False): return np.nan print "Interp points:" print basedat tri = Delaunay(points) fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.triplot(points[:,0], points[:,1], tri.simplices.copy()) ax.plot(points[:,0], points[:,1], 'o') ax.plot( U, T, 'o', ms=6., color='red') xlim = ax.get_xlim() dx = (xlim[1]-xlim[0])/10. ax.set_xlim( xlim[0]-dx, xlim[1]+dx ) ylim = ax.get_ylim() dy = (ylim[1]-ylim[0])/10. ax.set_ylim( ylim[0]-dy, ylim[1]+dy ) ax.set_xlabel('$U/t$') ax.set_ylabel('$T/t$',rotation=0,labelpad=8) tt = kwargs.get('title_text','') ax.set_title( tt + '$U/t={:.2f}$'.format(U) + \ ',\ \ ' + '$T/t={:.2f}$'.format(T), \ ha='center', va='bottom', fontsize=10) save_err = kwargs.get('save_err',None) if save_err is not None: print "Saving png." fig.savefig( save_err, dpi=300) plt.show() if not kwargs.get('single', False): raise ValueError("Could not interpolate using QMC data.") return result
def getHTSEInterp( Temperature, name='density'): U_, mu_, qty_ = getHTSEPoints( Temperature, name=name) points = _ndim_coords_from_arrays((U_, mu_)) return CloughTocher2DInterpolator(points, qty_)
def find_closest_nlce(U=8, T=0.67, mu=4., qty='dens', **kwargs): """ This function finds the closest values of U and T in the NLCE data that straddle the values U and T given as arguments. """ msg0 = "U={:0.2f}, T={:0.2f}".format(U, T) nUs = 3 us = [ float(u.split('/U')[-1]) for u in \ glob.glob( ldaconf.basedir + 'NLCE8_FinalK/U*' ) ] du = [np.abs(U - u) for u in us] index = np.argsort(du) Ulist0 = range(nUs) Upts = [us[index[i]] for i in Ulist0] # The T points are not uniformly spaced so we find the two closest ones # We start with a list of available T points: Ts = np.array( sorted( [ float(g.split('/T')[1].split('.dat')[0]) for g in \ glob.glob(ldaconf.basedir + 'NLCE8_FinalK/U00/T*') ] ) ) diff = T - Ts error = False if np.all(diff < 0): print "Available temperatures do not make it this low:" print " T = ", T error = True if not error: order_pos = np.argsort(np.abs(diff[diff > 0])) order_neg = np.argsort(np.abs(diff[diff < 0])) Tpts = sorted([Ts[diff > 0][order_pos[0]], Ts[diff < 0][order_neg[0]]]) else: order = np.argsort(np.abs(diff)) Tpts = sorted([Ts[order[0]], Ts[order[1]], Ts[order[2]]]) #Ta = min(Ts[order[0]], Ts[order[1]]) #Tb = max(Ts[order[0]], Ts[order[1]]) #print "T in ", Ta, Tb datadir = ldaconf.basedir datfiles = [] for Uval in Upts: for Tval in Tpts: fname = datadir + \ 'NLCE8_FinalK/U{:02d}/T{:0.2f}.dat'.format(int(Uval),Tval) datfiles.append([fname, Uval, Tval]) if qty == 'dens': COL = 1 elif qty == 'entr': COL = 2 elif qty == 'spi': COL = 3 elif qty == 'kappa': COL = 4 else: raise "Qty not defined:", qty MUCOL = 0 basedat = [] qtyinterp = kwargs.get('qtyinterp', 'nearest') for f in datfiles: msg = 'U={:0.2f}, T={:0.2f}'.format(U,T) + \ ' mu={:0.2f}, Upt={:0.3f}, Tpt={:0.3f}'.\ format(mu, f[1], f[2]) try: dat = np.loadtxt(f[0]) # careful = False, for NLCE data it is ok to not worry about # chemical potentials that are outside the data range qtyresult = get_qty_mu(dat, mu, MUCOL, COL, msg=msg, careful=False) if qtyresult == 'out-of-bounds': print msg print "out-of-bounds" continue basedat.append([f[1], f[2], qtyresult]) except Exception as e: print "Failed to get data from file = ", f print msg fig = plt.figure(figsize=(3.5, 3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot(gs[0]) ax.grid(alpha=0.5) ax.plot(dat[:, MUCOL], dat[:, COL], '.-') ax.axvline(mu) ax.text( 0.5, 1.05, msg, ha='center', va='bottom', \ transform=ax.transAxes) if matplotlib.get_backend() == 'agg': fig.savefig('err_mu.png', dpi=200) else: plt.show() raise e basedat = np.array(basedat) #print "Closest dat = ", basedat points = _ndim_coords_from_arrays((basedat[:, 0], basedat[:, 1])) #finterp = CloughTocher2DInterpolator(points, basedat[:,2]) finterp = LinearNDInterpolator(points, basedat[:, 2]) try: result = finterp(U, T) if np.isnan(result): if U >= 30 and U <= 32.5: result = finterp(29.99, T) logger.warning( " nlce: U={:0.1f} replaced to U=29.99 ".format(U)) if np.isnan(result): raise Exception("!!!! nlce: Invalid result !!!!\n" + msg0) except Exception as e: print e error = True if error or kwargs.get('showinterp', False): #print "Interp points:" #print basedat tri = Delaunay(points) fig = plt.figure(figsize=(3.5, 3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot(gs[0]) ax.grid(alpha=0.5) ax.triplot(points[:, 0], points[:, 1], tri.simplices.copy()) ax.plot(points[:, 0], points[:, 1], 'o') ax.plot(U, T, 'o', ms=6., color='red') xlim = ax.get_xlim() dx = (xlim[1] - xlim[0]) / 10. ax.set_xlim(xlim[0] - dx, xlim[1] + dx) ylim = ax.get_ylim() dy = (ylim[1] - ylim[0]) / 10. ax.set_ylim(ylim[0] - dy, ylim[1] + dy) ax.set_xlabel('$U/t$') ax.set_ylabel('$T/t$', rotation=0, labelpad=8) tt = kwargs.get('title_text', '') ax.set_title( tt + '$U/t={:.2f}$'.format(U) + ',\ \ ' + '$T/t={:.2f}$'.format(T), \ ha='center', va='bottom', fontsize=10) save_err = kwargs.get('save_err', None) if save_err is not None: print "saving png to ", save_err fig.savefig(save_err, dpi=300) if matplotlib.get_backend() == 'agg': print "saving png to err.png" fig.savefig('err.png', dpi=200) else: plt.show() raise return result
def __call__(self, xi, method=None, compute_gradients=True): """ Interpolate at the sample coordinates. Parameters ---------- xi : ndarray of shape (..., ndim) The coordinates to sample the gridded data at method : str, optional The method of interpolation to perform. Supported are 'slinear', 'cubic', and 'quintic'. Default is None, which will use the method defined at the construction of the interpolation object instance. compute_gradients : bool, optional If a spline interpolation method is chosen, this determines whether gradient calculations should be made and cached. Default is True. Returns ------- array_like Value of interpolant at all sample points. """ # cache latest evaluation point for gradient method's use later self._xi = xi method = self.method if method is None else method if method not in self._all_methods: all_m = ', '.join(['"' + m + '"' for m in self._all_methods]) raise ValueError('Method "%s" is not defined. Valid methods are ' '%s.' % (method, all_m)) ndim = len(self.grid) self.ndim = ndim xi = _ndim_coords_from_arrays(xi, ndim=ndim) if xi.shape[-1] != len(self.grid): raise ValueError("The requested sample points xi have dimension " "%d, but this RegularGridInterp has " "dimension %d" % (xi.shape[1], ndim)) xi_shape = xi.shape xi = xi.reshape(-1, xi_shape[-1]) if self.bounds_error: for i, p in enumerate(xi.T): if not np.logical_and(np.all(self.grid[i][0] <= p), np.all(p <= self.grid[i][-1])): raise OutOfBoundsError("One of the requested xi is out of bounds", i, p[0], self.grid[i][0], self.grid[i][-1]) indices, norm_distances, out_of_bounds = self._find_indices(xi.T) if np.iscomplexobj(self.values[:]): raise ValueError("method '%s' does not support complex values.") ki = self._ki if method != self.method: # re-validate dimensions vs spline order ki = [] for i, p in enumerate(self.grid): n_p = len(p) k = self._interp_config[method] ki.append(k) if n_p <= k: if not self._spline_dim_error: ki[-1] = n_p - 1 else: raise ValueError("There are %d points in dimension" " %d, but method %s requires at " "least % d points per dimension." "" % (n_p, i, method, k + 1)) interpolator = make_interp_spline result = self._evaluate_splines(self.values[:].T, xi, indices, interpolator, method, ki, compute_gradients=compute_gradients) if not self.bounds_error and self.fill_value is not None: result[out_of_bounds] = self.fill_value return result.reshape(xi_shape[:-1] + self.values.shape[ndim:])
def modified_ip_call(ip, *args): xi = _ndim_coords_from_arrays(args, ndim=ip.points.shape[1]) xi = ip._check_call_shape(xi) xi = ip._scale_x(xi) dist, i = ip.tree.query(xi, n_jobs=-1) return dist, ip.values[i]
def CT(dx, dy, bounds, XY, values, mask=False, kdtree=None, clip=False, extrapolate=None): """Use Scipy's CloughTocher C1 continuous interpolation using unstructured meshes to interpolate arbitrary locations to a grid Parameters ---------- dx : float The required spacing between grid nodes bounds : ndarray of floats Length 4 array with the minimum and maximum in two directions. [Xmin, Xmax, Ymin, Ymax] XY : 2D ndarray of floats Two columns, each column contains the co-ordinate in a dimension values : ndarray The values to interpolate to the grid mask : float, optional Force interpolated values that are greater than a distance of mask from any known point to be NaN kdtree : scipy.spatial.ckdtree.cKDTree, optional If no kdtree is given for the set of points, one is created. To speed up multiple interpolations, the user can pass their own fixed kdtree and prevent the generation of one every time. clip : bool, optional Interpolation can overshoot the known value. clip = True ensures that the min max of the grid is the same as the known data points. extrapolate : bool, optional Extrapolate the grid past the convex hull of the known points using nearest neighbour interpolation. Returns ------- x : array of floats The unique grid node along the first dimension y : array of floats The unique grid node along the second dimension vals : array of floats The interpolated values on a grid, represented by a 2D array """ values[values == np.inf] = np.nan mn = np.nanmin(values) mx = np.nanmax(values) if (mx - mn) == 0.0: values = values - mn else: values = (values - mn) / (mx - mn) # Create the CT function for interpolation f = CloughTocher2DInterpolator(XY, values) xc, yc, intPoints = getGridLocations2D(bounds, dx, dy) # Interpolate to the grid vals = f(intPoints) # Reshape to a 2D array vals = StatArray.StatArray(vals.reshape(yc.size, xc.size), name=cf.getName(values), units=cf.getUnits(values)) if mask or extrapolate: if (kdtree is None): kdt = cKDTree(XY) else: kdt = kdtree # Use distance masking if mask: g = np.meshgrid(xc, yc) xi = _ndim_coords_from_arrays(tuple(g), ndim=XY.shape[1]) dists, indexes = kdt.query(xi) vals[dists > mask] = np.nan # Truncate values to the observed values if (clip): minV = np.nanmin(values) maxV = np.nanmax(values) mask = ~np.isnan(vals) mask[mask] &= vals[mask] < minV vals[mask] = minV mask = ~np.isnan(vals) mask[mask] &= vals[mask] > maxV vals[mask] = maxV if (not extrapolate is None): assert isinstance( extrapolate, str), 'extrapolate must be a string. Choose [Nearest]' extrapolate = extrapolate.lower() if (extrapolate == 'nearest'): # Get the indices of the nans iNan = np.argwhere(np.isnan(vals)) # Create Query locations from the nans xi = np.zeros([iNan.shape[0], 2]) xi[:, 0] = x[iNan[:, 1]] xi[:, 1] = y[iNan[:, 0]] # Get the nearest neighbours dists, indexes = kdt.query(xi) # Assign the nearest value to the Nan locations vals[iNan[:, 0], iNan[:, 1]] = values[indexes] else: assert False, 'Extrapolation option not available. Choose [Nearest]' if (mx - mn) == 0.0: values = values + mn else: vals = (vals * (mx - mn)) + mn return xc, yc, vals
def find_closest_nlce( U=8, T=0.67, mu=4., qty='dens', **kwargs): """ This function finds the closest values of U and T in the NLCE data that straddle the values U and T given as arguments. This case is a little bit easier than the QMC data because here pretty much everything is available, so we can directly get the closest four (U,T) points to get the triangulation """ Ustep = 2. Ua = Ustep*(U/Ustep - (U%Ustep)/Ustep) Ub = Ua + 2. #print "U in ", Ua, Ub # The T points are not uniformly spaced so we find the two closest ones Ts = np.array([0.4, 0.5, 0.64, 0.72, 0.84, 0.9, 1.0, 1.2, \ 1.4, 1.6, 1.8, 2.0, 2.2, 2.4]) diff = T-Ts error = False if np.all( diff < 0 ): print "Available temperatures do not make it this low:" print " T = ", T error = True if not error: order_pos = np.argsort(np.abs( diff[diff>0] )) order_neg = np.argsort(np.abs( diff[diff<0] )) Tpts = sorted( [ Ts[diff>0][ order_pos[0] ] , Ts[diff<0][ order_neg[0]] ] ) else: order = np.argsort( np.abs( diff ) ) Tpts = sorted( [ Ts[order[0]], Ts[order[1]], Ts[order[2]] ] ) #Ta = min(Ts[order[0]], Ts[order[1]]) #Tb = max(Ts[order[0]], Ts[order[1]]) #print "T in ", Ta, Tb datadir = '/home/pmd/sandbox/hubbard-lda/' datfiles = [] for Uval in [Ua,Ub]: for Tval in Tpts: fname = datadir + \ 'NLCE8_Final/U{:02d}/T{:0.2f}.dat'.format(int(Uval),Tval) datfiles.append([ fname, Uval, Tval ]) if qty == 'dens': COL = 1 elif qty == 'entr': COL = 2 elif qty == 'spi': COL = 3 else: raise "Qty not defined:", qty MUCOL = 0 basedat = [] qtyinterp = kwargs.get( 'qtyinterp', 'nearest' ) for f in datfiles: try: dat = np.loadtxt(f[0]) if qtyinterp == 'nearest': index = np.argmin( np.abs(dat[:, MUCOL] - mu )) basedat.append( [f[1], f[2], dat[index, COL]] ) else: # find the two closest chemical potentials that # stride the point mudat = dat[:,MUCOL] # since the mu's are ordered we can do: index0 = np.where( mudat <=mu )[0][-1] index1 = np.where( mudat > mu )[0][0] qty0 = dat[ index0, COL ] qty1 = dat[ index1, COL ] mu0 = dat[ index0, MUCOL ] mu1 = dat[ index1, MUCOL ] qtyresult = qty0 + (mu-mu0) * (qty1-qty0) / (mu1-mu0) basedat.append( [f[1], f[2], qtyresult] ) # not implemented yet: #if showqtyinterp: # fig = plt.figure( figsize=(3.5,3.5)) # gs = matplotlib.gridspec.GridSpec( 1,1 ,\ # left=0.15, right=0.96, bottom=0.12, top=0.88) # ax = fig.add_subplot( gs[0] ) # ax.grid(alpha=0.5) #print #print " mu = ", mu #print "index0 = ", index0 #print "index1 = ", index1 #print "Doing linear interpolation for the qty" #print " mu0 = ", mu0 #print " mu1 = ", mu1 #print "qty0 = ", qty0 #print "qty1 = ", qty1 #print "qtyresult = ", qtyresult except Exception as e: print "Failed to get data from file = ", f raise e basedat = np.array(basedat) #print "Closest dat = ", basedat points = _ndim_coords_from_arrays(( basedat[:,0] , basedat[:,1])) #finterp = CloughTocher2DInterpolator(points, basedat[:,2]) finterp = LinearNDInterpolator( points, basedat[:,2] ) try: result = finterp( U,T ) if np.isnan(result): raise Exception("!!!! Invalid result !!!!\n") except Exception as e: print e error = True if error or kwargs.get('showinterp',False): #print "Interp points:" #print basedat tri = Delaunay(points) fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.triplot(points[:,0], points[:,1], tri.simplices.copy()) ax.plot(points[:,0], points[:,1], 'o') ax.plot( U, T, 'o', ms=6., color='red') xlim = ax.get_xlim() dx = (xlim[1]-xlim[0])/10. ax.set_xlim( xlim[0]-dx, xlim[1]+dx ) ylim = ax.get_ylim() dy = (ylim[1]-ylim[0])/10. ax.set_ylim( ylim[0]-dy, ylim[1]+dy ) ax.set_xlabel('$U/t$') ax.set_ylabel('$T/t$',rotation=0,labelpad=8) tt = kwargs.get('title_text','') ax.set_title( tt + '$U/t={:.2f}$'.format(U) + ',\ \ ' + '$T/t={:.2f}$'.format(T), \ ha='center', va='bottom', fontsize=10) save_err = kwargs.get('save_err',None) if save_err is not None: print "Saving png." fig.savefig( save_err, dpi=300) plt.show() return result
def find_closest_qmc( U=8, T=0.67, mu=4.0, **kwargs): """ This function finds the closest values of U and T in the QMC data that straddle the values U and T given as arguments. """ nUs = 4 nTs = 3 ALLPTS = kwargs.get('ALLPTS', False) # select which quantity will be returned, options are # spi and entropy QTY = kwargs.get('QTY', 'spi' ) if QTY == 'spi': datadir = basedir + 'COMB_Final_Spi/' elif QTY == 'entropy': datadir = basedir + 'COMB_Final_Entr/' elif QTY == 'density': datadir = basedir + 'COMB_Final_Spi/' elif QTY == 'kappa': datadir = basedir + 'COMB_Final_Spi/' else: raise ValueError('Quantity not defined:' + str(QTY) ) fname = datadir + 'U*' us = [ float(u.split('/U')[-1]) for u in glob.glob(fname) ] du = [ np.abs(U-u) for u in us ] index = np.argsort(du) if ALLPTS: Ulist0 = range(len(index)) else: Ulist0 = range( nUs ) us = [ us[index[i]] for i in Ulist0] #print us #print du #print index #print "Closest Us = ", us datfiles = [] for u in us: # For the Spi and Stheta data if QTY == 'spi' or QTY == 'density' or QTY == 'kappa': fname = datadir + 'U{U:02d}/T*dat'.format(U=int(u)) fs = sorted(glob.glob(fname)) Ts = [ float(f.split('T')[1].split('.dat')[0]) for f in fs ] elif QTY=='entropy': fname = datadir + 'U{U:02d}/S*dat'.format(U=int(u)) fs = sorted(glob.glob(fname)) Ts = [ float(f.split('S')[1].split('.dat')[0]) for f in fs ] Ts_g = [] ; Ts_l = []; for t in Ts: if t > T: Ts_g.append(t) else: Ts_l.append(t) order_g = np.argsort( [ np.abs( T -t ) for t in Ts_g ] ) order_l = np.argsort( [ np.abs( T -t ) for t in Ts_l ] ) try: Tpts = [ Ts_g[ order_g[0]] , Ts_l[ order_l[0]] ] except: #print #print "problem adding U=",u, "T=",Ts #print "available T data does not stride the point" #print "T =", T #print "Ts =", Ts #print "will add nearest Ts nevertheless" Tpts = [ ] #raise ValueError("QMC data not available.") dT = [ np.abs( T - t) for t in Ts ] index = np.argsort(dT) if ALLPTS: Tlist0 = range(len(Ts)) else: Tlist0 = range( min(nTs , len(Ts))) for i in Tlist0: Tnew = Ts[index[i]] if Tnew not in Tpts: Tpts.append(Tnew) for Tpt in Tpts: index = Ts.index( Tpt ) try: datfiles.append( [ fs[ index ], u, Ts[index] ] ) except: print "problem adding U=",u, "T=",Ts raise # Need to make sure that selected T values stride both # sides of the point #print #print u #print Ts #print dT #print index #print fs # for i in range(min(3, len(Ts))): # try: # datfiles.append( [ fs[index[i]], u, Ts[index[i]] ] ) # except: # print "problem adding U=",u, "T=",Ts # raise # #datfiles.append( [ fs[index[1]], u, Ts[index[1]] ] ) #print datfiles MUCOL = 0 DENSCOL = 1 ENTRCOL = 2 SPICOL = 3 CMPRCOL = 4 if QTY == 'spi': COL = SPICOL elif QTY == 'entropy': COL = ENTRCOL elif QTY == 'density': COL = DENSCOL elif QTY == 'kappa': COL = CMPRCOL msg0 = 'U={:0.2f}, T={:0.2f}'.format(U,T) logger.debug("number of nearby points = " + str(len(datfiles))) basedat = [] basedaterr = [] datserr = [] for mm, f in enumerate(datfiles): # f[0] is the datafile name # f[1] is U # f[2] is T radius = kwargs.get('radius', np.nan ) msg = 'U={:0.2f}, T={:0.2f}'.format(U,T) + \ ' mu={:0.2f}, r={:0.2f}, Upt={:0.3f}, Tpt={:0.3f}'.\ format(mu, radius, f[1], f[2]) try: dat = np.loadtxt(f[0]) spival = get_qty_mu( dat, mu, MUCOL, COL, msg=msg ) # Toggle the false here to plot all of the out of bounds if spival == 'out-of-bounds': #spival_symmetry = logger.info('qty is out of bounds') basedaterr.append( [f[1], f[2], np.nan] ) datserr.append( dat ) if False: fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.plot( dat[:,MUCOL], dat[:,COL], '.-') ax.axvline( mu ) ax.text( 0.5, 1.05, msg, ha='center', va='bottom', \ transform=ax.transAxes, fontsize=6.) if matplotlib.get_backend() == 'agg': fig.savefig('err_mu_%02d.png'%mm, dpi=200) plt.close(fig) else: plt.show() plt.close(fig) continue else: basedat.append( [f[1], f[2], spival] ) except Exception as e : print "Failed to get data from file = ", f # toggle plotting, not implemented yet: if True: fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.plot( dat[:,MUCOL], dat[:,COL], '.-') ax.axvline( mu ) ax.text( 0.5, 1.05, msg, ha='center', va='bottom', \ transform=ax.transAxes) if matplotlib.get_backend() == 'agg': fig.savefig('err_mu_%02d.png'%mm, dpi=200) else: plt.show() raise e logger.debug("number of nearby valid points = " + str(len(basedat))) error = False points = None # MAKE THE TRIANGULATION basedat = np.array(basedat) Us = np.unique(basedat[:,0] ) Ts = np.unique(basedat[:,1] ) validTriang = not ( len(Us) ==1 or len(Ts) == 1 ) #print "#Us={:d}, #Ts={:d}".format( len(Us), len(Ts) ) #print msg if validTriang: points = _ndim_coords_from_arrays(( basedat[:,0] , basedat[:,1])) #print "Closest dat = ", basedat #finterp = CloughTocher2DInterpolator(points, basedat[:,2]) finterp = LinearNDInterpolator( points, basedat[:,2] ) else: logerr = 'not enough finterp points, QTY=%s'%QTY + '\n' + msg + '\n' \ + "number of basedat pts = " + str(len(basedat)) print basedat print "len Us = ", len(Us) print "len Ts = ", len(Ts) print "len 'out-of-bounds' = ", len( basedaterr ) if len( basedaterr ) > 0: for bb, bdaterr in enumerate(basedaterr): msgbb = 'U={:0.2f}, T={:0.2f}'.format(U,T) +\ ' mu={:0.2f}, r={:0.2f}, Upt={:0.3f}, Tpt={:0.3f}'.\ format(mu, radius, basedaterr[bb][0], basedaterr[bb][1] ) daterr = datserr[bb] fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.plot( daterr[:,MUCOL], daterr[:,COL], '.-') ax.axvline( mu ) ax.text( 0.5, 1.05, msgbb, ha='center', va='bottom', \ transform=ax.transAxes, fontsize=6.) if matplotlib.get_backend() == 'agg': fig.savefig('err_mu_%02d.png'%bb, dpi=200) plt.close(fig) else: plt.show() plt.close(fig) logger.exception(logerr) raise ValueError('finterp') if points == None: logger.warning( "points object is None" ) if error == False: try: result = finterp( U,T ) if np.isnan(result): if U >= 30.0 and U <=32.5: result = finterp( 29.99, T ) logger.warning(" qmc: U={:0.1f} replaced to U=29.99 ".\ format(U) ) if np.isnan(result): raise Exception("\n!!!! qmc: Invalid result, QTY:%s!!!!\n"%QTY \ + msg0) except Exception as e: if kwargs.get('error_nan', False): return np.nan else: error = True logger.exception("Invalid QTY result!") if error == False: if result >= 8. and QTY == 'spi' : print " Obtained Spi > 8. : U={:0.2f}, T={:0.2f}, mu={:0.2f}".\ format( U, T, mu ), print " ==> Spi={:0.2f}".format(float(result)) error = True elif result >=4. and QTY == 'entropy': print " Obtained Ent > 4. : U={:0.2f}, T={:0.2f}, mu={:0.2f}".\ format( U, T, mu ), print " ==> Result={:0.2f}".format(float(result)) error = True logger.debug("error status = " + str(error)) if error or kwargs.get('showinterp',False): logger.debug("Inside error if statement...") if kwargs.get('error_nan', False): pass #return np.nan #print "Interp points:" #print basedat if len(basedat) == 0 and len(basedaterr) > 0 : basedaterr = np.array(basedaterr) Userr = np.unique(basedaterr[:,0] ) Tserr = np.unique(basedaterr[:,1] ) validTriangerr = not ( len(Userr) ==1 or len(Tserr) == 1 ) points = _ndim_coords_from_arrays(( basedaterr[:,0] , basedaterr[:,1])) tri = Delaunay(points) else: tri = Delaunay(points) fig = plt.figure( figsize=(3.5,3.5)) gs = matplotlib.gridspec.GridSpec( 1,1 ,\ left=0.15, right=0.96, bottom=0.12, top=0.88) ax = fig.add_subplot( gs[0] ) ax.grid(alpha=0.5) ax.triplot(points[:,0], points[:,1], tri.simplices.copy()) ax.plot(points[:,0], points[:,1], 'o') ax.plot( U, T, 'o', ms=6., color='red') xlim = ax.get_xlim() dx = (xlim[1]-xlim[0])/10. ax.set_xlim( xlim[0]-dx, xlim[1]+dx ) ylim = ax.get_ylim() dy = (ylim[1]-ylim[0])/10. ax.set_ylim( ylim[0]-dy, ylim[1]+dy ) ax.set_xlabel('$U/t$') ax.set_ylabel('$T/t$',rotation=0,labelpad=8) tt = kwargs.get('title_text','') ax.set_title( tt + '$U/t={:.2f}$'.format(U) + \ ',\ \ ' + '$T/t={:.2f}$'.format(T), \ ha='center', va='bottom', fontsize=10) save_err = kwargs.get('save_err',None) if save_err is not None: print "Saving png." fig.savefig( save_err, dpi=300) if matplotlib.get_backend() == 'agg': fig.savefig('err.png', dpi=200) print "Saved error to err.png" else: plt.show() if not kwargs.get('single', False): raise ValueError("Could not interpolate using QMC data.") if ALLPTS: if 'savepath' in kwargs.keys(): fig.savefig( kwargs.get('savepath',None) , dpi=300) if error: raise return result
def get_hi(self, dates_u, verbos=False): # does dates_u cover one year or more #dmonthly reload = [False, False] d_no = np.shape(dates_u)[0] if d_no > 2: print('Warning CPOM hi not compatible with tw over 1 month') return None ### first check if any of the dates in dates_u ### match the preloaded, ddL ### if the first date matches the second data_u = np.zeros([d_no, self.G.m, self.G.n]) for n, d in enumerate(dates_u): if d == self.dl1: ### return buffered data_u[n] = self.hi1 elif d == self.dl2: ### return buffered data_u[n] = self.hi2 else: ### load and bin ### check dates ### normal month, data on the first dload = d.replace(day=1) file = self.path + d.strftime('%Y%m_') + 'Thick.map' if verbos: print(file) f = np.genfromtxt(file) hi = f[:, 2] lon = f[:, 1] lat = f[:, 0] xpts, ypts = self.G.mplot(lon, lat) # print('Binning awkward CPOM data') # # # ret = binned_statistic_2d( xpts, ypts, hi, # statistic='mean', bins=[self.edges_x,self.edges_y]) # data_u[n] = ret.statistic # ret = binned_statistic_2d( xpts, ypts, [], # statistic='count', bins=[self.edges_x,self.edges_y]) # data_u[n][ret.statistic<4] = np.nan print('Regridding awkward CPOM data') xpts, ypts = self.G.mplot(lon, lat) xy = np.vstack([xpts, ypts]).T data_u[n] = griddata(xy, hi, (self.G.xpts, self.G.ypts), method='nearest') # Construct kd-tree, functionality copied from scipy.interpolate ### this finds empty values on the grid tree = cKDTree(xy) xi = _ndim_coords_from_arrays((self.G.xpts, self.G.ypts)) dists, indexes = tree.query(xi) # Copy original result but mask missing values with NaNs data_u[n][dists > self.THRESH] = np.nan reload[n] = True ### update buffered thickness if d_no == 2: self.dl1 = dates_u[0] self.hi1 = data_u[0] self.dl2 = dates_u[1] self.hi2 = data_u[1] else: self.dl2 = dates_u[0] self.hi2 = data_u[0] return data_u
def __getMean3D_CloughTocher(self, dx, dy, mask=None, clip=False, force=False): # Get the discretization if (dx is None): tmp = self.points.bounds[1]-self.points.bounds[0] dx = 0.01 * tmp assert dx > 0.0, "dx must be positive!" # Get the discretization if (dy is None): tmp = self.points.bounds[3]-self.points.bounds[2] dy = 0.01 * tmp assert dy > 0.0, "dy must be positive!" tmp = np.column_stack((self.points.x, self.points.y)) # Get the points to interpolate to x,y,intPoints = interpolation.getGridLocations2D(self.points.bounds, dx, dy) # Create a distance mask if mask: self.points.setKdTree(nDims=2) # Set the KdTree on the data points g = np.meshgrid(x,y) xi = _ndim_coords_from_arrays(tuple(g), ndim=tmp.shape[1]) dists, indexes = self.points.kdtree.query(xi) iMask = np.where(dists > mask) # Get the value bounds minV = np.nanmin(self.mean) maxV = np.nanmax(self.mean) # Initialize 3D volume mean3D = StatArray(np.zeros([self.zGrid.size, y.size, x.size], order = 'F'),name = 'Conductivity', units = '$Sm^{-1}$') # Triangulate the data locations dTri = Delaunay(tmp) # Interpolate for each depth print('Interpolating using clough tocher') Bar=progressbar.ProgressBar() for i in Bar(range(self.zGrid.size)): # Get the model values for the current depth vals1D = self.mean[i,:] # Create the interpolant f=CloughTocher2DInterpolator(dTri,vals1D) # Interpolate to the grid vals = f(intPoints) # Reshape to a 2D array vals = vals.reshape(y.size,x.size) # clip values to the observed values if (clip): vals.clip(minV, maxV) # Mask based on distance if (mask): vals[iMask] = np.nan # Add values to the 3D array mean3D[i,:,:] = vals self.mean3D = mean3D #.reshape(self.zGrid.size*y.size*x.size)
def getHTSEInterp( Temperature, name='density'): U_, mu_, qty_ = getHTSEPoints( Temperature, name=name) points = _ndim_coords_from_arrays((U_, mu_)) interp = CloughTocher2DInterpolator(points, qty_) return lambda u,mu : interp( np.column_stack(( np.ravel(u),np.ravel(mu) )) ).reshape(u.shape)
def __call__(self, xi, method=None, compute_gradients=True): """ Interpolate at the sample coordinates. Parameters ---------- xi : ndarray of shape (..., ndim) The coordinates to sample the gridded data at method : str, optional The method of interpolation to perform. Supported are 'slinear', 'cubic', and 'quintic'. Default is None, which will use the method defined at the construction of the interpolation object instance. compute_gradients : bool, optional If a spline interpolation method is chosen, this determines whether gradient calculations should be made and cached. Default is True. Returns ------- array_like Value of interpolant at all sample points. """ # cache latest evaluation point for gradient method's use later self._xi = xi method = self.method if method is None else method if method not in self._all_methods: all_m = ', '.join(['"' + m + '"' for m in self._all_methods]) raise ValueError('Method "%s" is not defined. Valid methods are ' '%s.' % (method, all_m)) ndim = len(self.grid) self.ndim = ndim xi = _ndim_coords_from_arrays(xi, ndim=ndim) if xi.shape[-1] != len(self.grid): raise ValueError("The requested sample points xi have dimension " "%d, but this RegularGridInterp has " "dimension %d" % (xi.shape[1], ndim)) xi_shape = xi.shape xi = xi.reshape(-1, xi_shape[-1]) if self.bounds_error: for i, p in enumerate(xi.T): if not np.logical_and(np.all(self.grid[i][0] <= p), np.all(p <= self.grid[i][-1])): raise OutOfBoundsError( "One of the requested xi is out of bounds", i, p[0], self.grid[i][0], self.grid[i][-1]) indices, norm_distances, out_of_bounds = self._find_indices(xi.T) if np.iscomplexobj(self.values[:]): raise ValueError("method '%s' does not support complex values.") ki = self._ki if method != self.method: # re-validate dimensions vs spline order ki = [] for i, p in enumerate(self.grid): n_p = len(p) k = self._interp_config[method] ki.append(k) if n_p <= k: if not self._spline_dim_error: ki[-1] = n_p - 1 else: raise ValueError("There are %d points in dimension" " %d, but method %s requires at " "least % d points per dimension." "" % (n_p, i, method, k + 1)) interpolator = make_interp_spline result = self._evaluate_splines(self.values[:].T, xi, indices, interpolator, method, ki, compute_gradients=compute_gradients) if not self.bounds_error and self.fill_value is not None: result[out_of_bounds] = self.fill_value return result.reshape(xi_shape[:-1] + self.values.shape[ndim:])
def _points_from_grids(grid, values, xi, method): """Return `points` and `new_points` from original grid and new grid/points. Returns ``points``, ``new_points``, and ``shape`` to use with :func:`emg3d.maps.interp_volume_average`, :func:`emg3d.maps.interp_spline_3d`, and :class:`scipy.interpolate.RegularGridInterpolator`. For the input parameters, see :func:`emg3d.maps.interpolate`. Returns ------- points : (ndarray, ndarray, ndarray) Tuple containing the x-, y-, and z-coordinates of the input values. new_points : {(ndarray, ndarray, ndarray); ndarray} Depends on the ``method``: - If ``method='volume'``: (ndarray, ndarray, ndarray) Tuple containing the x-, y-, and z-coordinates of the output values. - Else: ndarray Coordinates in an ndarray of shape (..., 3): ``array([[x1, y1, z1], ..., [xn, yn, zn]])``. shape : tuple Final shape of the output values. """ # Specific checks for method='volume'. if method == 'volume': msg = "``method='volume'`` is only implemented for " # 'xi' must be a TensorMesh. if not hasattr(xi, 'nodes_x'): msg += "TensorMesh instances as input for ``xi``." raise ValueError(msg) # Shape of the values must correspond to shape of cells. if grid.shape_cells != values.shape: msg += "cell-centered properties; required shape = " raise ValueError(msg + f"{grid.shape_cells}.") # General dimensionality check. else: electric = [grid.shape_edges_x, grid.shape_faces_y, grid.shape_edges_z] magnetic = [grid.shape_faces_x, grid.shape_edges_y, grid.shape_faces_z] centered = [ grid.shape_cells, ] if values.shape not in np.r_[electric, magnetic, centered]: msg = ("``values`` must be a 3D ndarray living on cell centers, " "edges, or faces of the ``grid``.") raise ValueError(msg) # Get electric flag (living on edges vs living on faces). electric = values.shape not in [ grid.shape_faces_x, grid.shape_edges_y, grid.shape_faces_z ] # Check if 'xi' is a TensorMesh. xi_is_grid = hasattr(xi, 'nodes_x') # # Get points from input # # # 1. Get required tuples from input grids. points = tuple() if xi_is_grid: new_points = tuple() shape = tuple() # Loop over dimensions to get the vectors corresponding to input data. for i, coord in enumerate(['x', 'y', 'z']): # Cell nodes. comp_shape = [grid.shape_cells[i], grid.shape_nodes[i]][electric] if method == 'volume' or values.shape[i] == comp_shape: prop = ['cell_centers_', 'nodes_'][electric] pts = getattr(grid, prop + coord) if xi_is_grid: new_pts = getattr(xi, prop + coord) # Cell centers. else: prop = ['nodes_', 'cell_centers_'][electric] pts = getattr(grid, prop + coord) if xi_is_grid: new_pts = getattr(xi, prop + coord) # Add to points. points += (pts, ) if xi_is_grid: new_points += (new_pts, ) shape += (len(new_pts), ) # After this step the points/new_points are: # points: (x-points, y-points, z-points) # new_points: (new-x-points, new-y-points, new-z-points) # if xi_is_grid # 'volume' takes new_points as tuples. However, the other methods take an # (..., 3) ndarray of the coordinates. if method != 'volume': # # Convert points to correct format # # if xi_is_grid: xx, yy, zz = np.broadcast_arrays(new_points[0][:, None, None], new_points[1][:, None], new_points[2]) new_points = np.r_[xx.ravel('F'), yy.ravel('F'), zz.ravel('F')] new_points = new_points.reshape(-1, 3, order='F') else: # Replicate the same expansion of xi as used in # RegularGridInterpolator, so the input xi can be quite flexible. new_points = interpnd._ndim_coords_from_arrays(xi, ndim=3) shape = new_points.shape[:-1] new_points = new_points.reshape(-1, 3, order='F') # After this step the new_points are: # new_points: array([[x1, y1, z1], ..., [xn, yn, zn]]) else: shape = xi.shape_cells return points, new_points, shape
def calc_data_map(data, x_pos, y_pos, grid_size, interp_method='nearest', threshold_dist=None): ''' Turns scatter data into grid data by interpolating up to a given threshold distance. Example: Make a plot of a 10 x 10 grid of increasing values with some missing data. :: data = np.linspace(-1, 1, 100) x_pos, y_pos = np.meshgrid(np.arange(0.5,10.5),np.arange(0.5, 10.5)) missing = [0, 5, 25] data_missing = np.delete(data, missing) x_missing = np.reshape(np.delete(x_pos, missing),-1) y_missing = np.reshape(np.delete(y_pos, missing),-1) interp_map, xy = calc_data_map(data_missing, x_missing, y_missing, [10, 10], threshold_dist=1.5) plot_spatial_map(interp_map, xy[0], xy[1]) .. image:: _images/posmap_calcmap.png Args: data (nch): list of values x_pos (nch): list of x positions y_pos (nch): list of y positions grid_size (tuple): number of points along each axis interp_method (str): method used for interpolation threshold_dist (float): distance to neighbors before disregarding a point on the image Returns: tuple: tuple containing: | *data_map (2,n array):* map of the data on the given grid | *xy (2,n array):* new grid positions to use with this map ''' extent = [np.min(x_pos), np.max(x_pos), np.min(y_pos), np.max(y_pos)] x_spacing = (extent[1] - extent[0]) / (grid_size[0] - 1) y_spacing = (extent[3] - extent[2]) / (grid_size[1] - 1) xy = np.vstack((x_pos, y_pos)).T xq, yq = np.meshgrid( np.arange(extent[0], extent[0] + x_spacing * grid_size[0], x_spacing), np.arange(extent[2], extent[2] + y_spacing * grid_size[1], y_spacing)) # Remove nan values non_nan = np.logical_not(np.isnan(data)) data = data[non_nan] xy = xy[non_nan] # Interpolate new_xy = (np.reshape(xq, -1), np.reshape(yq, -1)) X = griddata(xy, data, new_xy, method=interp_method, rescale=False) # Construct kd-tree, functionality copied from scipy.interpolate tree = cKDTree(xy) xi = _ndim_coords_from_arrays((np.reshape(xq, -1), np.reshape(yq, -1))) dists, indexes = tree.query(xi) # Mask values with distances over the threshold with NaNs if threshold_dist: X[dists > threshold_dist] = np.nan data_map = np.reshape(X, grid_size) return data_map, new_xy