def _run_tool(self): dataset = self.dataset orig_metadata = get_metadata(dataset)[dataset] src_path = orig_metadata['file_path'] #run filter with rasterio.open(src_path) as src: out_image = self._run(src, orig_metadata) out_meta = src.profile # save the resulting raster out_meta.update({"dtype": out_image.dtype, "height": out_image.shape[0], "width": out_image.shape[1], "transform": None}) new_metadata = { 'parameter': orig_metadata['parameter'], 'datatype': orig_metadata['datatype'], 'file_format': orig_metadata['file_format'], 'unit': orig_metadata['unit'] } new_dset, file_path, catalog_entry = self._create_new_dataset( old_dataset=dataset, ext='.tif', dataset_metadata=new_metadata, ) with rasterio.open(file_path, "w", **out_meta) as dest: dest.write(out_image) return {'datasets': new_dset, 'catalog_entries': catalog_entry}
def _run_tool(self): dataset = self.dataset io = load_plugins('io', 'timeseries-hdf5')['timeseries-hdf5'] orig_metadata = get_metadata(dataset)[dataset] if orig_metadata['file_path'] is None: raise IOError('No data file available for this dataset') df = io.read(orig_metadata['file_path']) # run filter new_df = self._run(df) # setup new dataset new_metadata = { 'parameter': new_df.metadata.get('parameter'), 'unit': new_df.metadata.get('unit'), 'datatype': orig_metadata['datatype'], 'file_format': orig_metadata['file_format'], } new_dset, file_path, catalog_entry = self._create_new_dataset( old_dataset=dataset, ext='.h5', dataset_metadata=new_metadata, ) # save dataframe io.write(file_path, new_df, new_metadata) return {'datasets': new_dset}
def _run_tool(self): dataset = self.dataset # get metadata, path etc from dataset orig_metadata = get_metadata(dataset)[dataset] elev_file = orig_metadata['file_path'] new_dset, file_path, catalog_entry = self._create_new_dataset( old_dataset=dataset, ext='.tif', dataset_metadata={ 'parameter': 'streams', 'datatype': orig_metadata['datatype'], 'file_format': orig_metadata['file_format'], }) fa = self.flow_accumulation if self.flow_accumulation is not None else wbt.d_inf_flow_accumulation( elev_file) # fa = wbt.d8_flow_accumulation(fill) wbt.extract_streams( flow_accum=fa, threshold=self.stream_threshold, output=file_path, ) return {'datasets': new_dset}
def set_threshold_bounds(cls): if cls.dataset: orig_metadata = get_metadata(cls.dataset)[cls.dataset] elev_file = orig_metadata['file_path'] fa = cls.flow_accumulation = wbt.d_inf_flow_accumulation(elev_file) amax = np.nanmax(fa) * .5 amin = np.nanmean(fa) threshold = cls.params()['stream_threshold'] threshold.bounds = (amin, amax) threshold.default = amax * 0.1
def _run_tool(self): dataset = self.dataset input_ts = load_plugins('io', 'timeseries-hdf5')['timeseries-hdf5'] orig_metadata = get_metadata(dataset)[dataset] parameter = orig_metadata['parameter'] if orig_metadata['file_path'] is None: raise IOError('No data file available for this dataset') df = input_ts.read(orig_metadata['file_path']) # apply transformation # run filter # new_df = self._run(df, options) metadata = df.metadata if 'file_path' in metadata: del metadata['file_path'] df.sort_values([parameter], ascending=False, na_position='last', inplace=True) df['Rank'] = df[parameter].rank(method='min', ascending=False) df.dropna(inplace=True) df['Percent Exceeded'] = (df['Rank'] / (df[parameter].count() + 1)) * 100 df.index = df['Percent Exceeded'] setattr_on_dataframe(df, 'metadata', metadata) new_df = df # setup new dataset new_metadata = { 'parameter': new_df.metadata.get('parameter'), 'datatype': orig_metadata['datatype'], 'options': self.set_options, 'file_format': orig_metadata['file_format'], 'unit': new_df.metadata.get('unit'), } new_dset, file_path, catalog_entry = self._create_new_dataset( old_dataset=dataset, ext='.h5', dataset_metadata=new_metadata, ) # save dataframe output = load_plugins('io', 'xy-hdf5')['xy-hdf5'] output.write(file_path, new_df, new_metadata) return {'datasets': new_dset, 'catalog_entries': catalog_entry}
def _run_tool(self): dataset = self.dataset # get metadata, path etc from first dataset, i.e. assume all datasets # are in same folder. This will break if you try and combine datasets # from different providers orig_metadata = get_metadata(dataset)[dataset] src_path = orig_metadata['file_path'] if self.new_crs is None: raise ValueError( "A new coordinated reference system MUST be provided") dst_crs = self.new_crs new_metadata = { 'parameter': orig_metadata['parameter'], 'datatype': orig_metadata['datatype'], 'file_format': orig_metadata['file_format'], } new_dset, file_path, catalog_entry = self._create_new_dataset( old_dataset=dataset, ext='.tif', dataset_metadata=new_metadata, ) # run filter with rasterio.open(src_path) as src: # write out tif file subprocess.check_output([ 'gdalwarp', src_path, file_path, '-s_srs', src.crs.to_string(), '-t_srs', dst_crs ]) with rasterio.open(file_path) as f: geometry = util.bbox2poly(f.bounds.left, f.bounds.bottom, f.bounds.right, f.bounds.top, as_shapely=True) update_metadata(catalog_entry, quest_metadata={'geometry': geometry.to_wkt()}) return {'datasets': new_dset, 'catalog_entries': catalog_entry}
def _run_tool(self): dataset = self.dataset orig_metadata = get_metadata(dataset)[dataset] elev_file = orig_metadata['file_path'] new_dset, file_path, catalog_entry = self._create_new_dataset( old_dataset=dataset, ext='.tif') wbt.fill_depressions(elev_file, output=file_path) quest_metadata = { 'parameter': 'streams', 'datatype': orig_metadata['datatype'], 'file_format': orig_metadata['file_format'], } update_metadata(new_dset, quest_metadata=quest_metadata) return {'datasets': new_dset}
def _run_tool(self): dataset = self.elevation_dataset # get metadata, path etc from dataset orig_metadata = get_metadata(dataset)[dataset] elev_file = orig_metadata['file_path'] try: original_outlets = [ f['geometry'] for f in get_metadata(self.outlets) ] except: original_outlets = self.outlets new_dset, file_path, catalog_entry = self._create_new_dataset( old_dataset=dataset, ext='.tif', dataset_metadata={ 'parameter': 'watershed_boundary', 'datatype': orig_metadata['datatype'], 'file_format': orig_metadata['file_format'], }) d8 = wbt.d8_pointer(elev_file) point_shp = points_to_shp(original_outlets) if self.snap_distance > 0: pp = wbt.vector_points_to_raster(point_shp, base=elev_file) snap_options = { 'pour_pts': pp, 'snap_dist': self.snap_distance, } fa = None if self.algorithm == 'nearest-stream': st = self.streams_dataset if st: st = open_dataset(st, with_nodata=True, isel_band=0) else: fa = wbt.d_inf_flow_accumulation(elev_file) st = wbt.extract_streams(fa, threshold=.1) snap_options.update(streams=st) else: fa = fa or wbt.d_inf_flow_accumulation(elev_file) # fa = wbt.d8_flow_accumulation(elev_file) snap_options.update(flow_accum=fa) snap_function = self.SNAP_DISTANCE_ALGORITHMS[self.algorithm] snapped = snap_function(**snap_options) indices = np.nonzero(np.nan_to_num(snapped)) snapped_outlets = [(snapped.x.values[row], snapped.y.values[col]) for col, row in zip(*indices)] point_shp = points_to_shp(snapped_outlets) wbt.watershed( d8_pntr=d8, pour_pts=point_shp, output=file_path, ) new_catalog_entries = raster_to_polygons(file_path) quest_metadata = { 'parameter': 'streams', 'datatype': orig_metadata['datatype'], 'file_format': orig_metadata['file_format'], 'file_path': file_path, } update_metadata(new_dset, quest_metadata=quest_metadata) return { 'datasets': new_dset, 'catalog_entries': [new_catalog_entries, snapped_outlets, catalog_entry] }
def _run_tool(self): # if len(datasets) < 2: # raise ValueError('There must be at LEAST two datasets for this filter') datasets = self.datasets orig_metadata = get_metadata(datasets[0])[datasets[0]] raster_files = [get_metadata(dataset)[dataset]['file_path'] for dataset in datasets] for dataset in datasets: if get_metadata(dataset)[dataset]['parameter'] != orig_metadata['parameter']: raise ValueError('Parameters must match for all datasets') if get_metadata(dataset)[dataset]['unit'] != orig_metadata['unit']: raise ValueError('Units must match for all datasets') new_metadata = { 'parameter': orig_metadata['parameter'], 'datatype': orig_metadata['datatype'], 'file_format': orig_metadata['file_format'], 'unit': orig_metadata['unit'], } new_dset, file_path, catalog_entry = self._create_new_dataset( old_dataset=datasets[0], ext='.tif', dataset_metadata=new_metadata, ) open_datasets = [rasterio.open(d) for d in raster_files] profile = open_datasets[0].profile # hack to avoid nodata out of range of dtype error for NED datasets profile['nodata'] = -32768.0 if profile['nodata'] == -3.4028234663853e+38 else profile['nodata'] new_data, transform = rasterio.merge.merge(open_datasets, nodata=profile['nodata']) for d in open_datasets: d.close() profile.pop('tiled', None) profile.update( height=new_data.shape[1], width=new_data.shape[2], transform=transform, driver='GTiff' ) with rasterio.open(file_path, 'w', **profile) as output: output.write(new_data.astype(profile['dtype'])) bbox = self.bbox if bbox is not None: bbox = box(*bbox) geo = gpd.GeoDataFrame({'geometry': bbox}, index=[0], crs=from_epsg(4326)) geo = geo.to_crs(crs=profile['crs']) bbox = geo.geometry with rasterio.open(file_path, 'r') as merged: new_data, transform = rasterio.mask.mask(dataset=merged, shapes=bbox, all_touched=True, crop=True) # profile.pop('tiled', None) profile.update( height=new_data.shape[1], width=new_data.shape[2], transform=transform, ) with rasterio.open(file_path, 'w', **profile) as clipped: clipped.write(new_data) with rasterio.open(file_path) as f: geometry = util.bbox2poly(f.bounds.left, f.bounds.bottom, f.bounds.right, f.bounds.top, as_shapely=True) update_metadata(catalog_entry, quest_metadata={'geometry': geometry.to_wkt()}) return {'datasets': new_dset, 'catalog_entries': catalog_entry}