def __init__(self, database, report_path, rs_project): super().__init__(rs_project, report_path) self.log = Logger('GNAT Report') self.database = database self.images_dir = os.path.join(os.path.dirname(report_path), 'images') safe_makedirs(self.images_dir)
def download_unzip_nhd(huc, download_folder, unzip_folder, force_download): try: nhd_url = get_nhdhr_url(huc[:4]) except Exception: # Fallback to guess at an address nhd_url = 'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/NHDPlusHR/Beta/GDB/NHDPLUS_H_{}_HU4_GDB.zip'.format( huc[:4]) safe_makedirs(download_folder) safe_makedirs(unzip_folder) nhd_unzip_folder = download_unzip(nhd_url, download_folder, unzip_folder, force_download) # get the gdb folder def matchGdb(fname): fpath = os.path.join(nhd_unzip_folder, fname) return os.path.isdir(fpath) and re.match(r"^.*\.gdb", fpath) try: filegdb = os.path.join( nhd_unzip_folder, next(filter(matchGdb, os.listdir(nhd_unzip_folder)))) except Exception: raise Exception("Could not find the GDB folder inside: {}".format( nhd_unzip_folder)) return filegdb, nhd_url
def _create_ds(self): """Note: this wipes any existing Datasets (files). It also creates and opens the new dataset. This makes heavy use of the DatasetRegistry Raises: VectorBaseException: [description] """ # Make a folder if we need to ds_dir = os.path.dirname(self.filepath) if not os.path.isdir(ds_dir): self.log.debug('Creating directory: {}'.format(ds_dir)) safe_makedirs(ds_dir) # Wipe the existing dataset if it exists. if os.path.exists(self.filepath): self.log.info('Deleting existing dataset: {}'.format( self.filepath)) self.__ds_reg.delete_dataset(self.filepath, self.driver) else: self.log.info('Dataset not found. Creating: {}'.format( self.filepath)) self.allow_write = True self.ogr_ds = self.__ds_reg.create(self.filepath, self.ogr_layer_name, self.driver) self.log.debug('Dataset created: {}'.format(self.filepath))
def gnat(huc, output_folder): """[summary] Args: huc ([type]): [description] Raises: Exception: [description] Exception: [description] Exception: [description] Exception: [description] Returns: [type]: [description] """ log = Logger("GNAT") log.info('GNAT v.{}'.format(cfg.version)) try: int(huc) except ValueError: raise Exception( 'Invalid HUC identifier "{}". Must be an integer'.format(huc)) if not (len(huc) == 4 or len(huc) == 8): raise Exception('Invalid HUC identifier. Must be four digit integer') safe_makedirs(output_folder)
def process_lst(lst_xml_folder): """This is a slightly hack-y script to create some XMLS for the land_surface_temp script It's a bit of an afterthought so it just plunks down the XMLS all alone in a folder Args: lst_xml_folder ([type]): [description] """ log = Logger("Generate XMLS for LST") hucs = [str(1700 + x) for x in range(1, 13)] for huc in hucs: hucdir = os.path.join(lst_xml_folder, huc) xml_file = os.path.join(hucdir, 'project.rs.xml') safe_makedirs(hucdir) if os.path.exists(xml_file): safe_remove_file(xml_file) project_name = f'Land Surface Temperature for HUC {huc}' project = RSProject(cfg, xml_file) project.create(project_name, 'LST') project.add_metadata({ 'ModelVersion': cfg.version, 'HUC': huc, 'dateCreated': datetime.datetime.now().isoformat(), 'HUC{}'.format(len(huc)): huc }) realizations = project.XMLBuilder.add_sub_element( project.XMLBuilder.root, 'Realizations') realization = project.XMLBuilder.add_sub_element( realizations, 'LST', None, { 'id': 'LST1', 'dateCreated': datetime.datetime.now().isoformat(), 'guid': str(uuid.uuid4()), 'productVersion': cfg.version }) project.XMLBuilder.add_sub_element(realization, 'Name', project_name) output_node = project.XMLBuilder.add_sub_element( realization, 'Outputs') zipfile_node = project.add_dataset(output_node, f'{huc}.zip', RSLayer(f'LST Result for {huc}', 'LST_ZIP', 'ZipFile', '1706.zip'), 'ZipFile', replace=True, rel_path=True) project.XMLBuilder.write() log.info('done')
def extract_canals(flowlines, epsg, boundary, outpath): log = Logger('Canals') driver = ogr.GetDriverByName("ESRI Shapefile") inDataSource = driver.Open(flowlines, 0) inLayer = inDataSource.GetLayer() inSpatialRef = inLayer.GetSpatialRef() inLayer.SetAttributeFilter("FCode IN ({0})".format(','.join( [str(fcode) for fcode in canal_fcodes]))) log.info('{:,} canal features identified in NHD flow lines.'.format( inLayer.GetFeatureCount())) if os.path.isfile(outpath): log.info( 'Skipping extracting canals from NHD flow lines because file exists.' ) return outpath extract_path = os.path.dirname(outpath) safe_makedirs(extract_path) # Create the output shapefile outSpatialRef, transform = get_transform_from_epsg(inSpatialRef, epsg) outDataSource = driver.CreateDataSource(outpath) outLayer = outDataSource.CreateLayer('network', outSpatialRef, geom_type=ogr.wkbMultiLineString) outLayerDefn = outLayer.GetLayerDefn() for feature in inLayer: geom = feature.GetGeometryRef() geom.Transform(transform) outFeature = ogr.Feature(outLayerDefn) outFeature.SetGeometry(geom) outLayer.CreateFeature(outFeature) outFeature = None feature = None log.info('{:,} features written to canals shapefile.'.format( outLayer.GetFeatureCount())) inDataSource = None outDataSource = None return outpath
def __init__(self, database, report_path, rs_project): super().__init__(rs_project, report_path) self.log = Logger('Confinement Report') self.database = database self.images_dir = os.path.join(os.path.dirname(report_path), 'images') safe_makedirs(self.images_dir) self.report_intro('Confinement') # reach_attribute_summary(database, images_dir, inner_div) self.raw_confinement() [ self.confinement_ratio(field, field) for field in ['Confinement_Ratio', 'Constriction_Ratio'] ]
def main(): # TODO Add transportation networks to vbet inputs # TODO Prepare clipped NHD Catchments as vbet polygons input parser = argparse.ArgumentParser( description='Floodplain Connectivity (BETA)', # epilog="This is an epilog" ) parser.add_argument('vbet_network', help='Vector line network', type=str) parser.add_argument('vbet_polygon', help='Vector polygon layer', type=str) parser.add_argument('roads', help='Vector line network', type=str) parser.add_argument('railroads', help='Vector line network', type=str) parser.add_argument('output_dir', help='Folder where output project will be created', type=str) parser.add_argument('--debug_gpkg', help='Debug geopackage', type=str) parser.add_argument('--verbose', help='(optional) a little extra logging ', action='store_true', default=False) args = dotenv.parse_args_env(parser) # make sure the output folder exists safe_makedirs(args.output_dir) # Initiate the log file log = Logger('FLOOD_CONN') log.setup(logPath=os.path.join(args.output_dir, 'floodplain_connectivity.log'), verbose=args.verbose) log.title('Floodplain Connectivity (BETA)') try: floodplain_connectivity(args.vbet_network, args.vbet_polygon, args.roads, args.railroads, args.output_dir, args.debug_gpkg) except Exception as e: log.error(e) traceback.print_exc(file=sys.stdout) sys.exit(1) sys.exit(0)
def download_hand(huc6, epsg, download_folder, boundary, raster_path, force_download=False, warp_options: dict = {}): """ Identify rasters within HUC8, download them and mosaic into single GeoTIF :param vector_path: Path to bounding polygon ShapeFile :param epsg: Output spatial reference :param buffer_dist: Distance in DEGREES to buffer the bounding polygon :param output_folder: Temporary folder where downloaded rasters will be saved :param force_download: The download will always be performed if this is true. :param warp_options: Extra options to pass to raster warp :return: """ log = Logger('HAND') file_path = os.path.join(download_folder, huc6 + 'hand.tif') raster_url = 'unknown' # First check the cache of 6 digit HUC rasters (if it exists) if os.path.isfile(file_path): temp_path = file_path log.info('Hand download file found. Skipping download: {}'.format( file_path)) else: safe_makedirs(download_folder) # Download the HAND raster raster_url = '/'.join( s.strip('/') for s in [base_url, huc6, huc6 + 'hand.tif']) log.info('HAND URL {}'.format(raster_url)) temp_path = download_file(raster_url, download_folder) # Reproject to the desired SRS and clip to the watershed boundary raster_warp(temp_path, raster_path, epsg, clip=boundary, warp_options=warp_options) return raster_path, raster_url
def __init__(self, database, report_path, rs_project): # Need to call the constructor of the inherited class: super().__init__(rs_project, report_path) self.log = Logger('BratReport') self.database = database # The report has a core CSS file but we can extend it with our own if we want: css_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'brat_report.css') self.add_css(css_path) self.images_dir = os.path.join(os.path.dirname(report_path), 'images') safe_makedirs(self.images_dir) # Now we just need to write the sections we want in the report in order self.report_intro() self.reach_attribute_summary() self.dam_capacity() self.hydrology_plots() self.ownership() self.vegetation() self.conservation()
def download_unzip_national(download_folder, unzip_folder, force_download): nhd_url = 'https://prd-tnm.s3.amazonaws.com/StagedProducts/Hydrography/WBD/National/GDB/WBD_National_GDB.zip' safe_makedirs(download_folder) safe_makedirs(unzip_folder) nhd_unzip_folder = download_unzip(nhd_url, download_folder, unzip_folder, force_download) # get the gdb folder def matchGdb(fname): fpath = os.path.join(nhd_unzip_folder, fname) return os.path.isdir(fpath) and re.match(r"^.*\.gdb", fpath) try: filegdb = os.path.join( nhd_unzip_folder, next(filter(matchGdb, os.listdir(nhd_unzip_folder)))) except Exception: raise Exception("Could not find the GDB folder inside: {}".format( nhd_unzip_folder)) return filegdb
def create_project(huc, output_dir, realization_meta): project_name = 'Confinement for HUC {}'.format(huc) project = RSProject(cfg, output_dir) project.create(project_name, 'Confinement') project.add_metadata({'HUC{}'.format(len(huc)): str(huc)}) project.add_metadata({'HUC': str(huc)}) realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root, 'Realizations') realization = project.XMLBuilder.add_sub_element( realizations, 'Confinement', None, { 'id': 'Confinement1', 'dateCreated': datetime.datetime.now().isoformat(), 'guid': str(uuid.uuid4()), 'productVersion': cfg.version }) project.XMLBuilder.add_sub_element(realization, 'Name', project_name) project.add_metadata(realization_meta) proj_nodes = { 'Inputs': project.XMLBuilder.add_sub_element(realization, 'Inputs'), 'Intermediates': project.XMLBuilder.add_sub_element(realization, 'Intermediates'), 'Outputs': project.XMLBuilder.add_sub_element(realization, 'Outputs') } proj_dir = os.path.dirname(project.xml_path) safe_makedirs(os.path.join(proj_dir, 'inputs')) safe_makedirs(os.path.join(proj_dir, 'intermediates')) safe_makedirs(os.path.join(proj_dir, 'outputs')) report_path = os.path.join(project.project_dir, LayerTypes['CONFINEMENT_RUN_REPORT'].rel_path) project.add_report(proj_nodes['Outputs'], LayerTypes['CONFINEMENT_RUN_REPORT'], replace=True) project.XMLBuilder.write() return project, realization, proj_nodes, report_path
def create_project(huc, output_dir): project_name = 'VBET for HUC {}'.format(huc) project = RSProject(cfg, output_dir) project.create(project_name, 'VBET') project.add_metadata({ 'HUC{}'.format(len(huc)): str(huc), 'HUC': str(huc), 'VBETVersion': cfg.version, 'VBETTimestamp': str(int(time.time())) }) realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root, 'Realizations') realization = project.XMLBuilder.add_sub_element( realizations, 'VBET', None, { 'id': 'VBET', 'dateCreated': datetime.datetime.now().isoformat(), 'guid': str(uuid.uuid1()), 'productVersion': cfg.version }) project.XMLBuilder.add_sub_element(realization, 'Name', project_name) proj_nodes = { 'Inputs': project.XMLBuilder.add_sub_element(realization, 'Inputs'), 'Intermediates': project.XMLBuilder.add_sub_element(realization, 'Intermediates'), 'Outputs': project.XMLBuilder.add_sub_element(realization, 'Outputs') } # Make sure we have these folders proj_dir = os.path.dirname(project.xml_path) safe_makedirs(os.path.join(proj_dir, 'inputs')) safe_makedirs(os.path.join(proj_dir, 'intermediates')) safe_makedirs(os.path.join(proj_dir, 'outputs')) project.XMLBuilder.write() return project, realization, proj_nodes
def unzip(file_path, destination_folder, force_overwrite=False, retries=3): """[summary] Args: file_path: Full path to an existing zip archive destination_folder: Path where the zip archive will be unzipped force_overwrite (bool, optional): Force overwrite of a file if it's already there. Defaults to False. retries (int, optional): Number of retries on a single file. Defaults to 3. Raises: Exception: [description] Exception: [description] Exception: [description] """ log = Logger('Unzipper') if not os.path.isfile(file_path): raise Exception('Unzip error: file not found: {}'.format(file_path)) try: log.info('Attempting unzip: {} ==> {}'.format(file_path, destination_folder)) zip_ref = zipfile.ZipFile(file_path, 'r') # only unzip files we don't already have safe_makedirs(destination_folder) log.info('Extracting: {}'.format(file_path)) # Only unzip things we haven't already unzipped for fitem in zip_ref.filelist: uz_success = False uz_retry = 0 while not uz_success and uz_retry < retries: try: outfile = os.path.join(destination_folder, fitem.filename) if fitem.is_dir(): if not os.path.isdir(outfile): zip_ref.extract(fitem, destination_folder) log.debug(' (creating) {}'.format(fitem.filename)) else: log.debug(' (skipping) {}'.format(fitem.filename)) else: if force_overwrite or (fitem.file_size > 0 and not os.path.isfile(outfile)) or (os.path.getsize(outfile) / fitem.file_size) < 0.99999: log.debug(' (unzipping) {}'.format(fitem.filename)) zip_ref.extract(fitem, destination_folder) else: log.debug(' (skipping) {}'.format(fitem.filename)) uz_success = True except Exception as e: log.debug(e) log.warning('unzipping file failed. waiting 3 seconds and retrying...') time.sleep(3) uz_retry += 1 if (not uz_success): raise Exception('Unzipping of file {} failed after {} attempts'.format(fitem.filename, retries)) zip_ref.close() log.info('Done') except zipfile.BadZipFile as e: # If the zip file is bad then we have to remove it. log.error('BadZipFile. Cleaning up zip file and output folder') safe_remove_file(file_path) safe_remove_dir(destination_folder) raise Exception('Unzip error: BadZipFile') except Exception as e: log.error('Error unzipping. Cleaning up output folder') safe_remove_dir(destination_folder) raise Exception('Unzip error: file could not be unzipped')
def cleaner(inpath, outpath): driver = ogr.GetDriverByName("ESRI Shapefile") inDataSource = driver.Open(inpath, 0) inLayer = inDataSource.GetLayer() geom_type = inLayer.GetGeomType() inSpatialRef = inLayer.GetSpatialRef() safe_makedirs(os.path.dirname(outpath)) if os.path.exists(outpath) and os.path.isfile(outpath): driver.DeleteDataSource(outpath) # Create the output shapefile outSpatialRef = osr.SpatialReference() outSpatialRef.ImportFromWkt(inSpatialRef.ExportToWkt()) outDataSource = driver.CreateDataSource(outpath) outLayer = outDataSource.CreateLayer('network', outSpatialRef, geom_type=geom_type) outLayerDefn = outLayer.GetLayerDefn() # Create new fields in the output shp and get a list of field names for feature creation fieldNames = [] for i in range(inLayer.GetLayerDefn().GetFieldCount()): fieldDefn = inLayer.GetLayerDefn().GetFieldDefn(i) outLayer.CreateField(fieldDefn) fieldNames.append(fieldDefn.name) total = inLayer.GetFeatureCount() counter = 0 nongeom = 0 progbar = ProgressBar(inLayer.GetFeatureCount(), 50, "Unioning features") for feature in inLayer: progbar.update(counter) ingeom = feature.GetGeometryRef() fieldVals = [] # make list of field values for feature for f in fieldNames: fieldVals.append(feature.GetField(f)) if ingeom: outFeature = ogr.Feature(outLayerDefn) sys.stdout.write( "\nProcessing feature: {}/{} {}: nongeometrics found. ". format(counter, total, nongeom)) # sys.stdout.flush() # Buffer by 0 cleans everything up and makes it nice sys.stdout.write(' BUFFERING...') geomBuffer = ingeom.Buffer(0) sys.stdout.write('SETTING...') outFeature.SetGeometry(geomBuffer) sys.stdout.write('SETTING...') for v, val in enumerate( fieldVals): # Set output feature attributes sys.stdout.write('FIELDS...') outFeature.SetField(fieldNames[v], val) sys.stdout.write('WRITING...') outLayer.CreateFeature(outFeature) outFeature = None sys.stdout.write('DONE!\n') else: # DISCARD features with no geometry nongeom += 1 counter += 1 progbar.finish() inDataSource = None outDataSource = None
def intersect_geometry_with_feature_class(geometry, feature_class, epsg, out_path, output_geom_type): if output_geom_type not in [ogr.wkbMultiPoint, ogr.wkbMultiLineString]: raise Exception('Unsupported ogr type: "{}"'.format(output_geom_type)) # Remove output shapefile if it already exists driver = ogr.GetDriverByName("ESRI Shapefile") if os.path.exists(out_path): driver.DeleteDataSource(out_path) else: # Make sure the output folder exists safe_makedirs(os.path.dirname(out_path)) # Create the output shapefile data_source = driver.CreateDataSource(out_path) spatial_ref = osr.SpatialReference() spatial_ref.ImportFromEPSG(epsg) layer = data_source.CreateLayer('intersection', spatial_ref, geom_type=output_geom_type) geom_union = get_geometry_unary_union(feature_class, epsg) # Nothing to do if there were no features in the feature class if not geom_union: data_source = None return geom_inter = geometry.intersection(geom_union) # Nothing to do if the intersection is empty if geom_inter.is_empty: data_source = None return # Single features and collections need to be converted into Multi-features if output_geom_type == ogr.wkbMultiPoint and not isinstance( geom_inter, MultiPoint): if isinstance(geom_inter, Point): geom_inter = MultiPoint([(geom_inter)]) elif isinstance(geom_inter, LineString): # Break this linestring down into vertices as points geom_inter = MultiPoint(list(geom_inter.coords)) elif isinstance(geom_inter, MultiLineString): # Break this linestring down into vertices as points geom_inter = MultiPoint( reduce(lambda acc, ls: acc + list(ls.coords), list(geom_inter.geoms), [])) elif isinstance(geom_inter, GeometryCollection): geom_inter = MultiPoint( [geom for geom in geom_inter.geoms if isinstance(geom, Point)]) elif output_geom_type == ogr.wkbMultiLineString and not isinstance( geom_inter, MultiLineString): if isinstance(geom_inter, LineString): geom_inter = MultiLineString([(geom_inter)]) else: raise Exception( 'Unsupported ogr type: "{}" does not match shapely type of "{}"' .format(output_geom_type, geom_inter.type)) out_layer_def = layer.GetLayerDefn() feature = ogr.Feature(out_layer_def) feature.SetGeometry(ogr.CreateGeometryFromWkb(geom_inter.wkb)) layer.CreateFeature(feature) data_source = None
def download_file(s3_url, download_folder, force_download=False): """ Download a file given a HTTPS URL that points to a file on S3 :param s3_url: HTTPS URL for a file on S3 :param download_folder: Folder where the file will be downloaded. :param force_download: :return: Local file path where the file was downloaded """ log = Logger('Download') safe_makedirs(download_folder) # Retrieve the S3 bucket and path from the HTTPS URL result = re.match(r'https://([^.]+)[^/]+/(.*)', s3_url) # If file already exists and forcing download then ensure unique file name file_path = os.path.join(download_folder, os.path.basename(result.group(2))) file_path_pending = os.path.join(download_folder, os.path.basename(result.group(2)) + '.pending') if os.path.isfile(file_path) and force_download: safe_remove_file(file_path) # If there is a pending path and the pending path is fairly new # then wait for it. while pending_check(file_path_pending, PENDING_TIMEOUT): log.debug('Waiting for .pending file. Another process is working on this.') time.sleep(30) log.info('Waiting done. Proceeding.') # Skip the download if the file exists if os.path.isfile(file_path) and os.path.getsize(file_path) > 0: log.info('Skipping download because file exists.') else: _file, tmpfilepath = tempfile.mkstemp(suffix=".temp", prefix="rstools_download") # Write our pending file. No matter what we must clean this file up!!! def refresh_pending(init=False): with open(file_path_pending, 'w') as f: f.write(str(datetime.datetime.now())) # Cleaning up the commone areas is really important def download_cleanup(): os.close(_file) safe_remove_file(tmpfilepath) safe_remove_file(file_path_pending) refresh_pending() pending_timer = Timer() log.info('Downloading {}'.format(s3_url)) # Actual file download for download_retries in range(MAX_ATTEMPTS): if download_retries > 0: log.warning('Download file retry: {}'.format(download_retries)) try: dl = 0 _file, tmpfilepath = tempfile.mkstemp(suffix=".temp", prefix="rstools_download") with requests.get(s3_url, stream=True) as r: r.raise_for_status() byte_total = int(r.headers.get('content-length')) progbar = ProgressBar(byte_total, 50, s3_url, byteFormat=True) # Binary write to file with open(tmpfilepath, 'wb') as tempf: for chunk in r.iter_content(chunk_size=8192): # Periodically refreshing our .pending file # so other processes will be aware we are still working on it. if pending_timer.ellapsed() > 10: refresh_pending() if chunk: # filter out keep-alive new chunks dl += len(chunk) tempf.write(chunk) progbar.update(dl) # Close the temporary file. It will be removed if (not os.path.isfile(tmpfilepath)): raise Exception('Error writing to temporary file: {}'.format(tmpfilepath)) progbar.finish() break except Exception as e: log.debug('Error downloading file from s3 {}: \n{}'.format(s3_url, str(e))) # if this is our last chance then the function must fail [0,1,2] if download_retries == MAX_ATTEMPTS - 1: download_cleanup() # Always clean up raise e # Now copy the temporary file (retry 3 times) for copy_retries in range(MAX_ATTEMPTS): if copy_retries > 0: log.warning('Copy file retry: {}'.format(copy_retries)) try: shutil.copy(tmpfilepath, file_path) # Make sure to clean up so the next process doesn't encounter a broken file if not file_compare(file_path, tmpfilepath): raise Exception('Error copying temporary download to final path') break except Exception as e: log.debug('Error copying file from temporary location {}: \n{}'.format(tmpfilepath, str(e))) # if this is our last chance then the function must fail [0,1,2] if copy_retries == MAX_ATTEMPTS - 1: download_cleanup() # Always clean up raise e download_cleanup() # Always clean up return file_path
def make_photopoints(folder_path, photos, photo_pts): log = Logger('Shapefile') outpath = os.path.join(folder_path, LayerTypes['Photos'].rel_path) safe_makedirs(os.path.dirname(outpath)) driver = ogr.GetDriverByName("ESRI Shapefile") if os.path.exists(outpath): driver.DeleteDataSource(outpath) # Create the output shapefile outSpatialRef = osr.SpatialReference() outSpatialRef.ImportFromEPSG(int(cfg.OUTPUT_EPSG)) outDataSource = driver.CreateDataSource(outpath) outLayer = outDataSource.CreateLayer('photos', outSpatialRef, geom_type=ogr.wkbPoint) # Add the fields we're interested in for field in PHOTO_FIELDS: csv_name = field[0] shp_name = field[1] ogr_type = field[2] fld = ogr.FieldDefn(shp_name, ogr_type) if ogr_type == ogr.OFTString and len(field) > 3: fld.SetWidth(field[3]) outLayer.CreateField(fld) for p in photos: photo_point = [ phpt for phpt in photo_pts if phpt['pk'] == p['fk_PhotoPoint'] ][0].copy() # Pull some fields from the other record photo_point['pk'] = p['pk'] photo_point['PhotoDate'] = p['PhotoDate'] photo_point['PhotoNotes'] = p['PhotoNotes'] photo_point['Path'] = 'Photo/{}.jpg'.format(photo_point['pk']) if not os.path.isfile(os.path.join(folder_path, photo_point['Path'])): raise Exception('file not found: {}'.format(photo_point['Path'])) feature = ogr.Feature(outLayer.GetLayerDefn()) for field in PHOTO_FIELDS: if photo_point[csv_name]: # Set the attributes using the values from the delimited text file if ogr_type == ogr.OFTInteger: feature.SetField(shp_name, int(photo_point[csv_name])) elif ogr_type == ogr.OFTReal: feature.SetField(shp_name, float(photo_point[csv_name])) elif ogr_type == ogr.OFTString: feature.SetField(shp_name, photo_point[csv_name]) # create the WKT for the feature using Python string formatting pt = ogr.Geometry(ogr.wkbPoint) pt.AddPoint(float(photo_point['Longitude']), float(photo_point['Latitude'])) # Set the feature geometry using the point feature.SetGeometry(pt) # Create the feature in the layer (shapefile) outLayer.CreateFeature(feature) # Dereference the feature feature = None
huc4 = row[0][0:4] if huc4 not in hucs: hucs[huc4] = {} hucs[huc4][row[0]] = {} locations = [] with open(usu_csv, 'r') as csvfile: reader = csv.DictReader(csvfile) for row in reader: huc8 = row['HUC8Dir'].split('_')[1] huc4 = huc8[0:4] if huc4 in hucs: if huc8 in hucs[huc4]: hucs[huc4][huc8]['USU'] = float(row['PRECIP_IN']) * 25.4 safe_makedirs(temp_folder) driver = ogr.GetDriverByName("OpenFileGDB") chart_values = [] for huc4, huc8s in hucs.items(): print('Processing', huc4) unzip_folder = os.path.join(temp_folder, huc4) try: nhd_url = get_nhdhr_url(huc4) nhd_unzip_folder = download_unzip(nhd_url, temp_folder, unzip_folder, False) except Exception as e: print('WARNING: Failed to download', huc4) continue folder_name = os.path.basename(nhd_url).split('.')[0]
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('projectpath', help='NHD flow line ShapeFile path', type=str) parser.add_argument('--verbose', help='(optional) a little extra logging ', action='store_true', default=False) args = dotenv.parse_args_env(parser) if args.projectpath is None or len(args.projectpath) < 10: raise Exception('projectpath has invalid value') safe_makedirs(args.projectpath) # Initiate the log file log = Logger('Inundation XML') log.setup(logPath=os.path.join(args.projectpath, 'Inundation.log'), verbose=args.verbose) try: log.info('Starting') build_xml(args.projectpath) edit_xml(args.projectpath) log.info('Exiting') except Exception as e: log.error(e) traceback.print_exc(file=sys.stdout) sys.exit(1)
def vbet(huc, flowlines_orig, flowareas_orig, orig_slope, json_transforms, orig_dem, hillshade, max_hand, min_hole_area_m, project_folder, reach_codes: List[str], meta: Dict[str, str]): """[summary] Args: huc ([type]): [description] flowlines_orig ([type]): [description] flowareas_orig ([type]): [description] orig_slope ([type]): [description] json_transforms ([type]): [description] orig_dem ([type]): [description] hillshade ([type]): [description] max_hand ([type]): [description] min_hole_area_m ([type]): [description] project_folder ([type]): [description] reach_codes (List[int]): NHD reach codes for features to include in outputs meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs """ log = Logger('VBET') log.info('Starting VBET v.{}'.format(cfg.version)) project, _realization, proj_nodes = create_project(huc, project_folder) # Incorporate project metadata to the riverscapes project if meta is not None: project.add_metadata(meta) # Copy the inp _proj_slope_node, proj_slope = project.add_project_raster( proj_nodes['Inputs'], LayerTypes['SLOPE_RASTER'], orig_slope) _proj_dem_node, proj_dem = project.add_project_raster( proj_nodes['Inputs'], LayerTypes['DEM'], orig_dem) _hillshade_node, hillshade = project.add_project_raster( proj_nodes['Inputs'], LayerTypes['HILLSHADE'], hillshade) # Copy input shapes to a geopackage inputs_gpkg_path = os.path.join(project_folder, LayerTypes['INPUTS'].rel_path) intermediates_gpkg_path = os.path.join( project_folder, LayerTypes['INTERMEDIATES'].rel_path) flowlines_path = os.path.join( inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path) flowareas_path = os.path.join( inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path) # Make sure we're starting with a fresh slate of new geopackages GeopackageLayer.delete(inputs_gpkg_path) GeopackageLayer.delete(intermediates_gpkg_path) copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG) copy_feature_class(flowareas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG) project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS']) # Create a copy of the flow lines with just the perennial and also connectors inside flow areas network_path = os.path.join( intermediates_gpkg_path, LayerTypes['INTERMEDIATES'].sub_layers['VBET_NETWORK'].rel_path) vbet_network(flowlines_path, flowareas_path, network_path, cfg.OUTPUT_EPSG, reach_codes) # Generate HAND from dem and vbet_network # TODO make a place for this temporary folder. it can be removed after hand is generated. temp_hand_dir = os.path.join(project_folder, "intermediates", "hand_processing") safe_makedirs(temp_hand_dir) hand_raster = os.path.join(project_folder, LayerTypes['HAND_RASTER'].rel_path) create_hand_raster(proj_dem, network_path, temp_hand_dir, hand_raster) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['HAND_RASTER']) # Build Transformation Tables with sqlite3.connect(intermediates_gpkg_path) as conn: cursor = conn.cursor() # Build tables with open( os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'vbet_schema.sql')) as sqlfile: sql_commands = sqlfile.read() cursor.executescript(sql_commands) conn.commit() # Load tables for sqldata in glob.glob(os.path.join( os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'data', '**', '*.sql'), recursive=True): with open(sqldata) as sqlfile: sql_commands = sqlfile.read() cursor.executescript(sql_commands) conn.commit() # Load transforms from table transforms = load_transform_functions(json_transforms, intermediates_gpkg_path) # Get raster resolution as min buffer and apply bankfull width buffer to reaches with rasterio.open(proj_slope) as raster: t = raster.transform min_buffer = (t[0] + abs(t[4])) / 2 log.info("Buffering Polyine by bankfull width buffers") network_path_buffered = os.path.join( intermediates_gpkg_path, LayerTypes['INTERMEDIATES']. sub_layers['VBET_NETWORK_BUFFERED'].rel_path) buffer_by_field(network_path, network_path_buffered, "BFwidth", cfg.OUTPUT_EPSG, min_buffer) # Rasterize the channel polygon and write to raster log.info('Writing channel raster using slope as a template') flow_area_raster = os.path.join(project_folder, LayerTypes['FLOW_AREA_RASTER'].rel_path) channel_buffer_raster = os.path.join( project_folder, LayerTypes['CHANNEL_BUFFER_RASTER'].rel_path) rasterize(network_path_buffered, channel_buffer_raster, proj_slope) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['CHANNEL_BUFFER_RASTER']) rasterize(flowareas_path, flow_area_raster, proj_slope) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['FLOW_AREA_RASTER']) channel_dist_raster = os.path.join(project_folder, LayerTypes['CHANNEL_DISTANCE'].rel_path) fa_dist_raster = os.path.join(project_folder, LayerTypes['FLOW_AREA_DISTANCE'].rel_path) proximity_raster(channel_buffer_raster, channel_dist_raster) proximity_raster(flow_area_raster, fa_dist_raster) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['CHANNEL_DISTANCE']) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['FLOW_AREA_DISTANCE']) slope_transform_raster = os.path.join( project_folder, LayerTypes['NORMALIZED_SLOPE'].rel_path) hand_transform_raster = os.path.join( project_folder, LayerTypes['NORMALIZED_HAND'].rel_path) chan_dist_transform_raster = os.path.join( project_folder, LayerTypes['NORMALIZED_CHANNEL_DISTANCE'].rel_path) fa_dist_transform_raster = os.path.join( project_folder, LayerTypes['NORMALIZED_FLOWAREA_DISTANCE'].rel_path) topo_evidence_raster = os.path.join(project_folder, LayerTypes['EVIDENCE_TOPO'].rel_path) channel_evidence_raster = os.path.join( project_folder, LayerTypes['EVIDENCE_CHANNEL'].rel_path) evidence_raster = os.path.join(project_folder, LayerTypes['VBET_EVIDENCE'].rel_path) # Open evidence rasters concurrently. We're looping over windows so this shouldn't affect # memory consumption too much with rasterio.open(proj_slope) as slp_src, \ rasterio.open(hand_raster) as hand_src, \ rasterio.open(channel_dist_raster) as cdist_src, \ rasterio.open(fa_dist_raster) as fadist_src: # All 3 rasters should have the same extent and properties. They differ only in dtype out_meta = slp_src.meta # Rasterio can't write back to a VRT so rest the driver and number of bands for the output out_meta['driver'] = 'GTiff' out_meta['count'] = 1 out_meta['compress'] = 'deflate' # out_meta['dtype'] = rasterio.uint8 # We use this to buffer the output cell_size = abs(slp_src.get_transform()[1]) with rasterio.open(evidence_raster, 'w', **out_meta) as dest_evidence, \ rasterio.open(topo_evidence_raster, "w", **out_meta) as dest, \ rasterio.open(channel_evidence_raster, 'w', **out_meta) as dest_channel, \ rasterio.open(slope_transform_raster, "w", **out_meta) as slope_ev_out, \ rasterio.open(hand_transform_raster, 'w', **out_meta) as hand_ev_out, \ rasterio.open(chan_dist_transform_raster, 'w', **out_meta) as chan_dist_ev_out, \ rasterio.open(fa_dist_transform_raster, 'w', **out_meta) as fa_dist_ev_out: progbar = ProgressBar(len(list(slp_src.block_windows(1))), 50, "Calculating evidence layer") counter = 0 # Again, these rasters should be orthogonal so their windows should also line up for _ji, window in slp_src.block_windows(1): progbar.update(counter) counter += 1 slope_data = slp_src.read(1, window=window, masked=True) hand_data = hand_src.read(1, window=window, masked=True) cdist_data = cdist_src.read(1, window=window, masked=True) fadist_data = fadist_src.read(1, window=window, masked=True) slope_transform = np.ma.MaskedArray(transforms["Slope"]( slope_data.data), mask=slope_data.mask) hand_transform = np.ma.MaskedArray(transforms["HAND"]( hand_data.data), mask=hand_data.mask) channel_dist_transform = np.ma.MaskedArray( transforms["Channel"](cdist_data.data), mask=cdist_data.mask) fa_dist_transform = np.ma.MaskedArray(transforms["Flow Areas"]( fadist_data.data), mask=fadist_data.mask) fvals_topo = slope_transform * hand_transform fvals_channel = np.maximum(channel_dist_transform, fa_dist_transform) fvals_evidence = np.maximum(fvals_topo, fvals_channel) # Fill the masked values with the appropriate nodata vals # Unthresholded in the base band (mostly for debugging) dest.write(np.ma.filled(np.float32(fvals_topo), out_meta['nodata']), window=window, indexes=1) slope_ev_out.write(slope_transform.astype('float32').filled( out_meta['nodata']), window=window, indexes=1) hand_ev_out.write(hand_transform.astype('float32').filled( out_meta['nodata']), window=window, indexes=1) chan_dist_ev_out.write( channel_dist_transform.astype('float32').filled( out_meta['nodata']), window=window, indexes=1) fa_dist_ev_out.write( fa_dist_transform.astype('float32').filled( out_meta['nodata']), window=window, indexes=1) dest_channel.write(np.ma.filled(np.float32(fvals_channel), out_meta['nodata']), window=window, indexes=1) dest_evidence.write(np.ma.filled(np.float32(fvals_evidence), out_meta['nodata']), window=window, indexes=1) progbar.finish() # The remaining rasters get added to the project project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['NORMALIZED_SLOPE']) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['NORMALIZED_HAND']) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['NORMALIZED_CHANNEL_DISTANCE']) project.add_project_raster(proj_nodes["Intermediates"], LayerTypes['NORMALIZED_FLOWAREA_DISTANCE']) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['EVIDENCE_TOPO']) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['EVIDENCE_CHANNEL']) project.add_project_raster(proj_nodes['Outputs'], LayerTypes['VBET_EVIDENCE']) # Get the length of a meter (roughly) degree_factor = GeopackageLayer.rough_convert_metres_to_raster_units( proj_slope, 1) buff_dist = cell_size min_hole_degrees = min_hole_area_m * (degree_factor**2) # Get the full paths to the geopackages intermed_gpkg_path = os.path.join(project_folder, LayerTypes['INTERMEDIATES'].rel_path) vbet_path = os.path.join(project_folder, LayerTypes['VBET_OUTPUTS'].rel_path) for str_val, thr_val in thresh_vals.items(): plgnize_id = 'THRESH_{}'.format(str_val) with TempRaster('vbet_raw_thresh_{}'.format(plgnize_id)) as tmp_raw_thresh, \ TempRaster('vbet_cleaned_thresh_{}'.format(plgnize_id)) as tmp_cleaned_thresh: log.debug('Temporary threshold raster: {}'.format( tmp_raw_thresh.filepath)) threshold(evidence_raster, thr_val, tmp_raw_thresh.filepath) raster_clean(tmp_raw_thresh.filepath, tmp_cleaned_thresh.filepath, buffer_pixels=1) plgnize_lyr = RSLayer('Raw Threshold at {}%'.format(str_val), plgnize_id, 'Vector', plgnize_id.lower()) # Add a project node for this thresholded vector LayerTypes['INTERMEDIATES'].add_sub_layer(plgnize_id, plgnize_lyr) vbet_id = 'VBET_{}'.format(str_val) vbet_lyr = RSLayer('Threshold at {}%'.format(str_val), vbet_id, 'Vector', vbet_id.lower()) # Add a project node for this thresholded vector LayerTypes['VBET_OUTPUTS'].add_sub_layer(vbet_id, vbet_lyr) # Now polygonize the raster log.info('Polygonizing') polygonize( tmp_cleaned_thresh.filepath, 1, '{}/{}'.format(intermed_gpkg_path, plgnize_lyr.rel_path), cfg.OUTPUT_EPSG) log.info('Done') # Now the final sanitization sanitize(str_val, '{}/{}'.format(intermed_gpkg_path, plgnize_lyr.rel_path), '{}/{}'.format(vbet_path, vbet_lyr.rel_path), buff_dist, network_path) log.info('Completed thresholding at {}'.format(thr_val)) # Now add our Geopackages to the project XML project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES']) project.add_project_geopackage(proj_nodes['Outputs'], LayerTypes['VBET_OUTPUTS']) report_path = os.path.join(project.project_dir, LayerTypes['REPORT'].rel_path) project.add_report(proj_nodes['Outputs'], LayerTypes['REPORT'], replace=True) report = VBETReport(report_path, project) report.write() log.info('VBET Completed Successfully')
def main(): parser = argparse.ArgumentParser(description='Riverscapes Context Tool', # epilog="This is an epilog" ) parser.add_argument('huc', help='HUC identifier', type=str) parser.add_argument('existing', help='National existing vegetation raster', type=str) parser.add_argument('historic', help='National historic vegetation raster', type=str) parser.add_argument('ownership', help='National land ownership shapefile', type=str) parser.add_argument('fairmarket', help='National fair market value raster', type=str) parser.add_argument('ecoregions', help='National EcoRegions shapefile', type=str) parser.add_argument('prism', help='Folder containing PRISM rasters in BIL format', type=str) parser.add_argument('output', help='Path to the output folder', type=str) parser.add_argument( 'download', help= 'Temporary folder for downloading data. Different HUCs may share this', type=str) parser.add_argument('--force', help='(optional) download existing files ', action='store_true', default=False) parser.add_argument( '--parallel', help= '(optional) for running multiple instances of this at the same time', action='store_true', default=False) parser.add_argument('--temp_folder', help='(optional) cache folder for downloading files ', type=str) parser.add_argument( '--meta', help='riverscapes project metadata as comma separated key=value pairs', type=str) parser.add_argument('--verbose', help='(optional) a little extra logging ', action='store_true', default=False) parser.add_argument( '--debug', help= '(optional) more output about things like memory usage. There is a performance cost', action='store_true', default=False) args = dotenv.parse_args_env(parser) # Initiate the log file log = Logger("RS Context") log.setup(logPath=os.path.join(args.output, "rs_context.log"), verbose=args.verbose) log.title('Riverscapes Context For HUC: {}'.format(args.huc)) log.info('HUC: {}'.format(args.huc)) log.info('EPSG: {}'.format(cfg.OUTPUT_EPSG)) log.info('Existing veg: {}'.format(args.existing)) log.info('Historical veg: {}'.format(args.historic)) log.info('Ownership: {}'.format(args.ownership)) log.info('Fair Market Value Raster: {}'.format(args.fairmarket)) log.info('Output folder: {}'.format(args.output)) log.info('Download folder: {}'.format(args.download)) log.info('Force download: {}'.format(args.force)) # This is a general place for unzipping downloaded files and other temporary work. # We use GUIDS to make it specific to a particular run of the tool to avoid unzip collisions parallel_code = "-" + str(uuid.uuid4()) if args.parallel is True else "" scratch_dir = args.temp_folder if args.temp_folder else os.path.join( args.download, 'scratch', 'rs_context{}'.format(parallel_code)) safe_makedirs(scratch_dir) meta = parse_metadata(args.meta) try: if args.debug is True: from rscommons.debug import ThreadRun memfile = os.path.join(args.output, 'rs_context_memusage.log') retcode, max_obj = ThreadRun( rs_context, memfile, args.huc, args.existing, args.historic, args.ownership, args.fairmarket, args.ecoregions, args.prism, args.output, args.download, scratch_dir, args.parallel, args.force, meta) log.debug('Return code: {}, [Max process usage] {}'.format( retcode, max_obj)) else: rs_context(args.huc, args.existing, args.historic, args.ownership, args.fairmarket, args.ecoregions, args.prism, args.output, args.download, scratch_dir, args.parallel, args.force, meta) except Exception as e: log.error(e) traceback.print_exc(file=sys.stdout) # Cleaning up the scratch folder is essential safe_remove_dir(scratch_dir) sys.exit(1) # Cleaning up the scratch folder is essential safe_remove_dir(scratch_dir) sys.exit(0)
def merge_feature_classes(feature_classes, epsg, boundary, outpath): log = Logger('Shapefile') if os.path.isfile(outpath): log.info('Skipping merging feature classes because file exists.') return safe_makedirs(os.path.dirname(outpath)) log.info('Merging {} feature classes.'.format(len(feature_classes))) driver = ogr.GetDriverByName("ESRI Shapefile") # Create the output shapefile outDataSource = driver.CreateDataSource(outpath) outLayer = None outSpatialRef = None transform = None fccount = 0 for inpath in feature_classes: fccount += 1 log.info("Merging feature class {}/{}".format(fccount, len(feature_classes))) inDataSource = driver.Open(inpath, 0) inLayer = inDataSource.GetLayer() inSpatialRef = inLayer.GetSpatialRef() inLayer.SetSpatialFilter(ogr.CreateGeometryFromWkb(boundary.wkb)) # First input spatial ref sets the SRS for the output file outSpatialRefTmp, transform = get_transform_from_epsg( inSpatialRef, epsg) if outLayer is None: outSpatialRef = outSpatialRefTmp outLayer = outDataSource.CreateLayer( 'network', outSpatialRef, geom_type=ogr.wkbMultiLineString) outLayerDefn = outLayer.GetLayerDefn() # Transfer fields over inLayerDef = inLayer.GetLayerDefn() for i in range(inLayerDef.GetFieldCount()): inFieldDef = inLayerDef.GetFieldDefn(i) # Only create fields if we really don't have them # NOTE: THIS ASSUMES ALL FIELDS OF THE SAME NAME HAVE THE SAME TYPE if outLayerDefn.GetFieldIndex(inFieldDef.GetName()) == -1: outLayer.CreateField(inFieldDef) progbar = ProgressBar(inLayer.GetFeatureCount(), 50, "Processing features") outLayerDefn = outLayer.GetLayerDefn() counter = 0 for feature in inLayer: counter += 1 progbar.update(counter) geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning( 'Feature with FID={} has no geometry. Skipping'.format( feature.GetFID())) continue geom.Transform(transform) # get a Shapely representation of the line # featobj = json.loads(geom.ExportToJson()) # polyline = shape(featobj) # if boundary.intersects(polyline): # clipped = boundary.intersection(polyline) outFeature = ogr.Feature(outLayerDefn) for i in range(inLayerDef.GetFieldCount()): outFeature.SetField( outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) outFeature.SetGeometry(geom) outLayer.CreateFeature(outFeature) feature = None outFeature = None progbar.finish() inDataSource = None outDataSource = None log.info('Merge complete.')
def rs_context(huc, existing_veg, historic_veg, ownership, fair_market, ecoregions, prism_folder, output_folder, download_folder, scratch_dir, parallel, force_download, meta: Dict[str, str]): """ Download riverscapes context layers for the specified HUC and organize them as a Riverscapes project :param huc: Eight, 10 or 12 digit HUC identification number :param existing_veg: Path to the existing vegetation conditions raster :param historic_veg: Path to the historical vegetation conditions raster :param ownership: Path to the national land ownership Shapefile :param output_folder: Output location for the riverscapes context project :param download_folder: Temporary folder where downloads are cached. This can be shared between rs_context processes :param force_download: If false then downloads can be skipped if the files already exist :param prism_folder: folder containing PRISM rasters in *.bil format :param meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs :return: """ log = Logger("RS Context") log.info('Starting RSContext v.{}'.format(cfg.version)) try: int(huc) except ValueError: raise Exception( 'Invalid HUC identifier "{}". Must be an integer'.format(huc)) if not (len(huc) in [4, 8, 10, 12]): raise Exception( 'Invalid HUC identifier. Must be 4, 8, 10 or 12 digit integer') safe_makedirs(output_folder) safe_makedirs(download_folder) # We need a temporary folder for slope rasters, Stitching inputs, intermeditary products, etc. scratch_dem_folder = os.path.join(scratch_dir, 'rs_context', huc) safe_makedirs(scratch_dem_folder) project, realization = create_project(huc, output_folder) hydrology_gpkg_path = os.path.join(output_folder, LayerTypes['HYDROLOGY'].rel_path) dem_node, dem_raster = project.add_project_raster(realization, LayerTypes['DEM']) _node, hill_raster = project.add_project_raster(realization, LayerTypes['HILLSHADE']) _node, flow_accum = project.add_project_raster(realization, LayerTypes['FA']) _node, drain_area = project.add_project_raster(realization, LayerTypes['DA']) hand_node, hand_raster = project.add_project_raster( realization, LayerTypes['HAND']) _node, slope_raster = project.add_project_raster(realization, LayerTypes['SLOPE']) _node, existing_clip = project.add_project_raster(realization, LayerTypes['EXVEG']) _node, historic_clip = project.add_project_raster(realization, LayerTypes['HISTVEG']) _node, fair_market_clip = project.add_project_raster( realization, LayerTypes['FAIR_MARKET']) # Download the four digit NHD archive containing the flow lines and watershed boundaries log.info('Processing NHD') # Incorporate project metadata to the riverscapes project if meta is not None: project.add_metadata(meta) nhd_download_folder = os.path.join(download_folder, 'nhd', huc[:4]) nhd_unzip_folder = os.path.join(scratch_dir, 'nhd', huc[:4]) nhd, db_path, huc_name, nhd_url = clean_nhd_data( huc, nhd_download_folder, nhd_unzip_folder, os.path.join(output_folder, 'hydrology'), cfg.OUTPUT_EPSG, False) # Clean up the unzipped files. We won't need them again if parallel: safe_remove_dir(nhd_unzip_folder) project.add_metadata({'Watershed': huc_name}) boundary = 'WBDHU{}'.format(len(huc)) # For coarser rasters than the DEM we need to buffer our clip polygon to include enough pixels # This shouldn't be too much more data because these are usually integer rasters that are much lower res. buffered_clip_path100 = os.path.join( hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP100'].rel_path) copy_feature_class(nhd[boundary], buffered_clip_path100, epsg=cfg.OUTPUT_EPSG, buffer=100) buffered_clip_path500 = os.path.join( hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path) copy_feature_class(nhd[boundary], buffered_clip_path500, epsg=cfg.OUTPUT_EPSG, buffer=500) # PRISM climate rasters mean_annual_precip = None bil_files = glob.glob(os.path.join(prism_folder, '**', '*.bil')) if (len(bil_files) == 0): raise Exception('Could not find any .bil files in the prism folder') for ptype in PrismTypes: try: # Next should always be guarded source_raster_path = next( x for x in bil_files if ptype.lower() in os.path.basename(x).lower()) except StopIteration: raise Exception( 'Could not find .bil file corresponding to "{}"'.format(ptype)) _node, project_raster_path = project.add_project_raster( realization, LayerTypes[ptype]) raster_warp(source_raster_path, project_raster_path, cfg.OUTPUT_EPSG, buffered_clip_path500, {"cutlineBlend": 1}) # Use the mean annual precipitation to calculate bankfull width if ptype.lower() == 'ppt': polygon = get_geometry_unary_union(nhd[boundary], epsg=cfg.OUTPUT_EPSG) mean_annual_precip = raster_buffer_stats2( {1: polygon}, project_raster_path)[1]['Mean'] log.info('Mean annual precipitation for HUC {} is {} mm'.format( huc, mean_annual_precip)) project.add_metadata( {'mean_annual_precipitation_mm': str(mean_annual_precip)}) calculate_bankfull_width(nhd['NHDFlowline'], mean_annual_precip) # Add the DB record to the Project XML db_lyr = RSLayer('NHD Tables', 'NHDTABLES', 'SQLiteDB', os.path.relpath(db_path, output_folder)) sqlite_el = project.add_dataset(realization, db_path, db_lyr, 'SQLiteDB') project.add_metadata({'origin_url': nhd_url}, sqlite_el) # Add any results to project XML for name, file_path in nhd.items(): lyr_obj = RSLayer(name, name, 'Vector', os.path.relpath(file_path, output_folder)) vector_nod, _fpath = project.add_project_vector(realization, lyr_obj) project.add_metadata({'origin_url': nhd_url}, vector_nod) states = get_nhd_states(nhd[boundary]) # Download the NTD archive containing roads and rail log.info('Processing NTD') ntd_raw = {} ntd_unzip_folders = [] ntd_urls = get_ntd_urls(states) for state, ntd_url in ntd_urls.items(): ntd_download_folder = os.path.join(download_folder, 'ntd', state.lower()) ntd_unzip_folder = os.path.join( scratch_dir, 'ntd', state.lower(), 'unzipped' ) # a little awkward but I need a folder for this and this was the best name I could find ntd_raw[state] = download_shapefile_collection(ntd_url, ntd_download_folder, ntd_unzip_folder, force_download) ntd_unzip_folders.append(ntd_unzip_folder) ntd_clean = clean_ntd_data(ntd_raw, nhd['NHDFlowline'], nhd[boundary], os.path.join(output_folder, 'transportation'), cfg.OUTPUT_EPSG) # clean up the NTD Unzip folder. We won't need it again if parallel: for unzip_path in ntd_unzip_folders: safe_remove_dir(unzip_path) # Write transportation layers to project file log.info('Write transportation layers to project file') # Add any results to project XML for name, file_path in ntd_clean.items(): lyr_obj = RSLayer(name, name, 'Vector', os.path.relpath(file_path, output_folder)) ntd_node, _fpath = project.add_project_vector(realization, lyr_obj) project.add_metadata({**ntd_urls}, ntd_node) # Download the HAND raster huc6 = huc[0:6] hand_download_folder = os.path.join(download_folder, 'hand') _hpath, hand_url = download_hand(huc6, cfg.OUTPUT_EPSG, hand_download_folder, nhd[boundary], hand_raster, warp_options={"cutlineBlend": 1}) project.add_metadata({'origin_url': hand_url}, hand_node) # download contributing DEM rasters, mosaic and reproject into compressed GeoTIF ned_download_folder = os.path.join(download_folder, 'ned') ned_unzip_folder = os.path.join(scratch_dir, 'ned') dem_rasters, urls = download_dem(nhd[boundary], cfg.OUTPUT_EPSG, 0.01, ned_download_folder, ned_unzip_folder, force_download) need_dem_rebuild = force_download or not os.path.exists(dem_raster) if need_dem_rebuild: raster_vrt_stitch(dem_rasters, dem_raster, cfg.OUTPUT_EPSG, clip=nhd[boundary], warp_options={"cutlineBlend": 1}) verify_areas(dem_raster, nhd[boundary]) # Calculate slope rasters seperately and then stitch them slope_parts = [] hillshade_parts = [] need_slope_build = need_dem_rebuild or not os.path.isfile(slope_raster) need_hs_build = need_dem_rebuild or not os.path.isfile(hill_raster) project.add_metadata( { 'num_rasters': str(len(urls)), 'origin_urls': json.dumps(urls) }, dem_node) for dem_r in dem_rasters: slope_part_path = os.path.join( scratch_dem_folder, 'SLOPE__' + os.path.basename(dem_r).split('.')[0] + '.tif') hs_part_path = os.path.join( scratch_dem_folder, 'HS__' + os.path.basename(dem_r).split('.')[0] + '.tif') slope_parts.append(slope_part_path) hillshade_parts.append(hs_part_path) if force_download or need_dem_rebuild or not os.path.exists( slope_part_path): gdal_dem_geographic(dem_r, slope_part_path, 'slope') need_slope_build = True if force_download or need_dem_rebuild or not os.path.exists( hs_part_path): gdal_dem_geographic(dem_r, hs_part_path, 'hillshade') need_hs_build = True if need_slope_build: raster_vrt_stitch(slope_parts, slope_raster, cfg.OUTPUT_EPSG, clip=nhd[boundary], clean=parallel, warp_options={"cutlineBlend": 1}) verify_areas(slope_raster, nhd[boundary]) else: log.info('Skipping slope build because nothing has changed.') if need_hs_build: raster_vrt_stitch(hillshade_parts, hill_raster, cfg.OUTPUT_EPSG, clip=nhd[boundary], clean=parallel, warp_options={"cutlineBlend": 1}) verify_areas(hill_raster, nhd[boundary]) else: log.info('Skipping hillshade build because nothing has changed.') # Remove the unzipped rasters. We won't need them anymore if parallel: safe_remove_dir(ned_unzip_folder) # Calculate flow accumulation raster based on the DEM log.info('Running flow accumulation and converting to drainage area.') flow_accumulation(dem_raster, flow_accum, dinfinity=False, pitfill=True) flow_accum_to_drainage_area(flow_accum, drain_area) # Clip and re-project the existing and historic vegetation log.info('Processing existing and historic vegetation rasters.') clip_vegetation(buffered_clip_path100, existing_veg, existing_clip, historic_veg, historic_clip, cfg.OUTPUT_EPSG) log.info('Process the Fair Market Value Raster.') raster_warp(fair_market, fair_market_clip, cfg.OUTPUT_EPSG, clip=buffered_clip_path500, warp_options={"cutlineBlend": 1}) # Clip the landownership Shapefile to a 10km buffer around the watershed boundary own_path = os.path.join(output_folder, LayerTypes['OWNERSHIP'].rel_path) project.add_dataset(realization, own_path, LayerTypes['OWNERSHIP'], 'Vector') clip_ownership(nhd[boundary], ownership, own_path, cfg.OUTPUT_EPSG, 10000) ####################################################### # Segmentation ####################################################### # For now let's just make a copy of the NHD FLowlines tmr = Timer() rs_segmentation(nhd['NHDFlowline'], ntd_clean['Roads'], ntd_clean['Rail'], own_path, hydrology_gpkg_path, SEGMENTATION['Max'], SEGMENTATION['Min'], huc) log.debug('Segmentation done in {:.1f} seconds'.format(tmr.ellapsed())) project.add_project_geopackage(realization, LayerTypes['HYDROLOGY']) # Add Bankfull Buffer Polygons bankfull_path = os.path.join( hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BANKFULL_CHANNEL'].rel_path) bankfull_buffer( os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['NETWORK'].rel_path), cfg.OUTPUT_EPSG, bankfull_path, ) # TODO Add nhd/bankfull union when merge feature classes in vector.ops works with Geopackage layers # bankfull_nhd_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path) # clip_path = os.path.join(hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['BUFFEREDCLIP500'].rel_path) # bankfull_nhd_area(bankfull_path, nhd['NHDArea'], clip_path, cfg.OUTPUT_EPSG, hydrology_gpkg_path, LayerTypes['HYDROLOGY'].sub_layers['COMPOSITE_CHANNEL_AREA'].rel_path) # Filter the ecoregions Shapefile to only include attributes that intersect with our HUC eco_path = os.path.join(output_folder, 'ecoregions', 'ecoregions.shp') project.add_dataset(realization, eco_path, LayerTypes['ECOREGIONS'], 'Vector') filter_ecoregions(nhd[boundary], ecoregions, eco_path, cfg.OUTPUT_EPSG, 10000) report_path = os.path.join(project.project_dir, LayerTypes['REPORT'].rel_path) project.add_report(realization, LayerTypes['REPORT'], replace=True) report = RSContextReport(report_path, project, output_folder) report.write() log.info('Process completed successfully.') return { 'DEM': dem_raster, 'Slope': slope_raster, 'ExistingVeg': existing_veg, 'HistoricVeg': historic_veg, 'NHD': nhd }
def main(): parser = argparse.ArgumentParser(description='Riverscapes VBET Tool', # epilog="This is an epilog" ) parser.add_argument('huc', help='NHD flow line ShapeFile path', type=str) parser.add_argument('flowlines', help='NHD flow line ShapeFile path', type=str) parser.add_argument('flowareas', help='NHD flow areas ShapeFile path', type=str) parser.add_argument('slope', help='Slope raster path', type=str) parser.add_argument('dem', help='DEM raster path', type=str) parser.add_argument('hillshade', help='Hillshade raster path', type=str) parser.add_argument( 'output_dir', help='Folder where output VBET project will be created', type=str) parser.add_argument( '--reach_codes', help= 'Comma delimited reach codes (FCode) to retain when filtering features. Omitting this option retains all features.', type=str) parser.add_argument('--max_slope', help='Maximum slope to be considered', type=float, default=12) parser.add_argument('--max_hand', help='Maximum HAND to be considered', type=float, default=50) parser.add_argument('--min_hole_area', help='Minimum hole retained in valley bottom (sq m)', type=float, default=50000) parser.add_argument( '--meta', help='riverscapes project metadata as comma separated key=value pairs', type=str) parser.add_argument('--verbose', help='(optional) a little extra logging ', action='store_true', default=False) parser.add_argument( '--debug', help= 'Add debug tools for tracing things like memory usage at a performance cost.', action='store_true', default=False) args = dotenv.parse_args_env(parser) # make sure the output folder exists safe_makedirs(args.output_dir) # Initiate the log file log = Logger('VBET') log.setup(logPath=os.path.join(args.output_dir, 'vbet.log'), verbose=args.verbose) log.title('Riverscapes VBET For HUC: {}'.format(args.huc)) meta = parse_metadata(args.meta) json_transform = json.dumps({ "Slope": 1, "HAND": 2, "Channel": 3, "Flow Areas": 4 }) reach_codes = args.reach_codes.split(',') if args.reach_codes else None try: if args.debug is True: from rscommons.debug import ThreadRun memfile = os.path.join(args.output_dir, 'vbet_mem.log') retcode, max_obj = ThreadRun(vbet, memfile, args.huc, args.flowlines, args.flowareas, args.slope, json_transform, args.dem, args.hillshade, args.max_hand, args.min_hole_area, args.output_dir, reach_codes, meta) log.debug('Return code: {}, [Max process usage] {}'.format( retcode, max_obj)) else: vbet(args.huc, args.flowlines, args.flowareas, args.slope, json_transform, args.dem, args.hillshade, args.max_hand, args.min_hole_area, args.output_dir, reach_codes, meta) except Exception as e: log.error(e) traceback.print_exc(file=sys.stdout) sys.exit(1) sys.exit(0)
def load_vegetation_raster(rasterpath: str, gpkg: str, existing=False, output_folder=None): """[summary] Args: rasterpath ([type]): [description] existing (bool, optional): [description]. Defaults to False. output_folder ([type], optional): [description]. Defaults to None. Returns: [type]: [description] """ conversion_lookup = { "Open Water": 500, "Riparian": 100, "Hardwood": 100, "Grassland": 50, "Shrubland": 50, "Non-vegetated": 40, "Snow-Ice": 40, "Sparsely Vegetated": 40, "Barren": 40, "Hardwood-Conifer": 20, # New for LANDFIRE 200 "Conifer-Hardwood": 20, "Conifer": 20, "Developed": 2, "Developed-Low Intensity": 2, "Developed-Medium Intensity": 2, "Developed-High Intensity": 2, "Developed-Roads": 2, "Quarries-Strip Mines-Gravel Pits-Well and Wind Pads": 2, # Updated for LANDFIRE 200 "Exotic Tree-Shrub": 3, "Exotic Herbaceous": 3, "Ruderal Wet Meadow and Marsh": 3, # New for LANDFIRE 200 "Agricultural": 1 } if existing else { "Open Water": 500, "Riparian": 100, "Hardwood": 100, "Shrubland": 50, "Grassland": 50, "Perennial Ice/Snow": 40, "Barren-Rock/Sand/Clay": 40, "Sparse": 40, "Conifer": 20, "Hardwood-Conifer": 20, "Conifer-Hardwood": 20} vegetated_classes = [ "Riparian", "Hardwood", "Hardwood-Conifer", "Grassland", "Shrubland", "Sparsely Vegetated", "Conifer-Hardwood", "Conifer", "Ruderal Wet Meadow and Marsh", "Agricultural" ] if existing else [ "Riparian", "Hardwood", "Conifer", "Shrubland", "Hardwood-Conifer", "Conifer-Hardwood", "Grassland"] lui_lookup = { # TODO check for Landfire 200 updates "Agricultural-Aquaculture": 0.66, "Agricultural-Bush fruit and berries": 0.66, "Agricultural-Close Grown Crop": 0.66, "Agricultural-Fallow/Idle Cropland": 0.33, "Agricultural-Orchard": 0.66, "Agricultural-Pasture and Hayland": 0.33, "Agricultural-Row Crop": 0.66, "Agricultural-Row Crop-Close Grown Crop": 0.66, "Agricultural-Vineyard": 0.66, "Agricultural-Wheat": 0.66, "Developed-High Intensity": 1.0, "Developed-Medium Intensity": 1.0, "Developed-Low Intensity": 1.0, "Developed-Roads": 1.0, "Developed-Upland Deciduous Forest": 1.0, "Developed-Upland Evergreen Forest": 1.0, "Developed-Upland Herbaceous": 1.0, "Developed-Upland Mixed Forest": 1.0, "Developed-Upland Shrubland": 1.0, "Managed Tree Plantation - Northern and Central Hardwood and Conifer Plantation Group": 0.66, "Managed Tree Plantation - Southeast Conifer and Hardwood Plantation Group": 0.66, "Quarries-Strip Mines-Gravel Pits": 1.0} # Read xml for reclass - arcgis tends to overwrite this file. use csv instead and make sure to ship with rasters # root = ET.parse(f"{rasterpath}.aux.xml").getroot() # ifield_value = int(root.findall(".//FieldDefn/[Name='VALUE']")[0].attrib['index']) # ifield_conversion_source = int(root.findall(".//FieldDefn/[Name='EVT_PHYS']")[0].attrib['index']) if existing else int(root.findall(".//FieldDefn/[Name='GROUPVEG']")[0].attrib['index']) # ifield_group_name = int(root.findall(".//FieldDefn/[Name='EVT_GP_N']")[0].attrib['index']) if existing else int(root.findall(".//FieldDefn/[Name='GROUPNAME']")[0].attrib['index']) # conversion_values = {int(n[ifield_value].text): conversion_lookup.setdefault(n[ifield_conversion_source].text, 0) for n in root.findall(".//Row")} # riparian_values = {int(n[ifield_value].text): 1 if n[ifield_conversion_source].text == "Riparian" else 0 for n in root.findall(".//Row")} # native_riparian_values = {int(n[ifield_value].text): 1 if n[ifield_conversion_source].text == "Riparian" and not ("Introduced" in n[ifield_group_name].text) else 0 for n in root.findall(".//Row")} # vegetation_values = {int(n[ifield_value].text): 1 if n[ifield_conversion_source].text in vegetated_classes else 0 for n in root.findall(".//Row")} # lui_values = {int(n[ifield_value].text): lui_lookup.setdefault(n[ifield_group_name].text, 0) for n in root.findall(".//Row")} if existing is True else {} # Load reclass values with sqlite3.connect(gpkg) as conn: c = conn.cursor() valid_values = [v[0] for v in c.execute("SELECT VegetationID FROM VegetationTypes").fetchall()] conversion_values = {row[0]: conversion_lookup.setdefault(row[1], 0) for row in c.execute('SELECT VegetationID, Physiognomy FROM VegetationTypes').fetchall()} riparian_values = {row[0]: 1 if row[1] == "Riparian" else 0 for row in c.execute('SELECT VegetationID, Physiognomy FROM VegetationTypes').fetchall()} native_riparian_values = {row[0]: 1 if row[1] == "Riparian" and not("Introduced" in row[2]) else 0 for row in c.execute('SELECT VegetationID, Physiognomy, LandUseGroup FROM VegetationTypes').fetchall()} vegetation_values = {row[0]: 1 if row[1] in vegetated_classes else 0 for row in c.execute('SELECT VegetationID, Physiognomy FROM VegetationTypes').fetchall()} lui_values = {row[0]: lui_lookup.setdefault(row[1], 0) for row in c.execute('SELECT VegetationID, LandUseGroup FROM VegetationTypes').fetchall()} if existing else {} # Read array with rasterio.open(rasterpath) as raster: no_data = int(raster.nodatavals[0]) conversion_values[no_data] = 0 riparian_values[no_data] = 0 native_riparian_values[no_data] = 0 vegetation_values[no_data] = 0 if existing: lui_values[no_data] = 0.0 valid_values.append(no_data) raw_array = raster.read(1, masked=True) for value in np.unique(raw_array): if not isinstance(value, type(np.ma.masked)) and value not in valid_values: raise Exception(f"Vegetation raster value {value} not found in current data dictionary") # Reclass array https://stackoverflow.com/questions/16992713/translate-every-element-in-numpy-array-according-to-key riparian_array = np.vectorize(riparian_values.get)(raw_array) native_riparian_array = np.vectorize(native_riparian_values.get)(raw_array) vegetated_array = np.vectorize(vegetation_values.get)(raw_array) conversion_array = np.vectorize(conversion_values.get)(raw_array) lui_array = np.vectorize(lui_values.get)(raw_array) if existing else None output = {"RAW": raw_array, "RIPARIAN": riparian_array, "NATIVE_RIPARIAN": native_riparian_array, "VEGETATED": vegetated_array, "CONVERSION": conversion_array, "LUI": lui_array} if output_folder: for raster_name, raster_array in output.items(): if raster_array is not None: safe_makedirs(output_folder) with rasterio.open(os.path.join(output_folder, f"{'EXISTING' if existing else 'HISTORIC'}_{raster_name}.tif"), 'w', driver='GTiff', height=raster.height, width=raster.width, count=1, dtype=np.int16, crs=raster.crs, transform=raster.transform) as dataset: dataset.write(raster_array.astype(np.int16), 1) return output
def export_feature_class(filegdb, featureclass, output_dir, output_epsg, retained_fields, attribute_filter, spatial_filter): log = Logger('Export FC') log.info('Exporting geodatabase feature class {}'.format(featureclass)) # Get the input layer in_driver = ogr.GetDriverByName("OpenFileGDB") in_datasource = in_driver.Open(filegdb, 0) in_layer = in_datasource.GetLayer(featureclass) inSpatialRef = in_layer.GetSpatialRef() if attribute_filter: log.info('Export attribute filter: {}'.format(attribute_filter)) in_layer.SetAttributeFilter(attribute_filter) if spatial_filter: log.info('Export spatial filter area: {}'.format(spatial_filter.area)) in_layer.SetSpatialFilter( ogr.CreateGeometryFromJson(json.dumps(mapping(spatial_filter)))) safe_makedirs(output_dir) # Create the output layer out_shapefile = os.path.join(output_dir, featureclass + '.shp') out_driver = ogr.GetDriverByName("ESRI Shapefile") outSpatialRef, transform = get_transform_from_epsg(inSpatialRef, output_epsg) # Remove output shapefile if it already exists if os.path.exists(out_shapefile): out_driver.DeleteDataSource(out_shapefile) # Create the output shapefile out_datasource = out_driver.CreateDataSource(out_shapefile) out_layer = out_datasource.CreateLayer(featureclass, outSpatialRef, geom_type=in_layer.GetGeomType()) # Add input Layer Fields to the output Layer if it is the one we want in_layer_def = in_layer.GetLayerDefn() for i in range(0, in_layer_def.GetFieldCount()): field_def = in_layer_def.GetFieldDefn(i) field_name = field_def.GetName() if retained_fields and field_name not in retained_fields: continue fieldTypeCode = field_def.GetType() if field_name.lower() == 'nhdplusid' and fieldTypeCode == ogr.OFTReal: field_def.SetWidth(32) field_def.SetPrecision(0) out_layer.CreateField(field_def) # Get the output Layer's Feature Definition out_layer_def = out_layer.GetLayerDefn() # Add features to the ouput Layer progbar = ProgressBar(in_layer.GetFeatureCount(), 50, "Adding features to output layer") counter = 0 for in_feature in in_layer: counter += 1 progbar.update(counter) # Create output Feature out_feature = ogr.Feature(out_layer_def) geom = in_feature.GetGeometryRef() geom.Transform(transform) # Add field values from input Layer for i in range(0, out_layer_def.GetFieldCount()): field_def = out_layer_def.GetFieldDefn(i) field_name = field_def.GetName() if retained_fields and field_name not in retained_fields: continue out_feature.SetField( out_layer_def.GetFieldDefn(i).GetNameRef(), in_feature.GetField(i)) # Set geometry as centroid out_feature.SetGeometry(geom) # Add new feature to output Layer out_layer.CreateFeature(out_feature) out_feature = None progbar.finish() # Save and close DataSources in_datasource = None out_datasource = None return out_shapefile
def copy_feature_class(inpath, epsg, outpath, clip_shape=None, attribute_filter=None): """Copy a Shapefile from one location to another This method is capable of reprojecting the geometries as they are copied. It is also possible to filter features by both attributes and also clip the features to another geometryNone Arguments: inpath {str} -- File path to input Shapefile that will be copied. epsg {int} -- Output coordinate system outpath {str} -- File path where the output Shapefile will be generated. Keyword Arguments: clip_shape {shape} -- Shapely polygon geometry in the output EPSG used to clip the input geometries (default: {None}) attribute_filter {str} -- Attribute filter used to limit the input features that will be copied. (default: {None}) """ log = Logger('Shapefile') # if os.path.isfile(outpath): # log.info('Skipping copy of feature classes because output file exists.') # return driver = ogr.GetDriverByName("ESRI Shapefile") inDataSource = driver.Open(inpath, 0) inLayer = inDataSource.GetLayer() inSpatialRef = inLayer.GetSpatialRef() geom_type = inLayer.GetGeomType() # Optionally limit which features are copied by using an attribute filter if attribute_filter: inLayer.SetAttributeFilter(attribute_filter) # If there's a clip geometry provided then limit the features copied to # those that intersect (partially or entirely) by this clip feature. # Note that this makes the subsequent intersection process a lot more # performant because the SetSaptialFilter() uses the ShapeFile's spatial # index which is much faster than manually checking if all pairs of features intersect. clip_geom = None if clip_shape: clip_geom = ogr.CreateGeometryFromWkb(clip_shape.wkb) inLayer.SetSpatialFilter(clip_geom) outpath_dir = os.path.dirname(outpath) safe_makedirs(outpath_dir) # Create the output shapefile outSpatialRef, transform = get_transform_from_epsg(inSpatialRef, epsg) outDataSource = driver.CreateDataSource(outpath) outLayer = outDataSource.CreateLayer('network', outSpatialRef, geom_type=geom_type) outLayerDefn = outLayer.GetLayerDefn() # Add input Layer Fields to the output Layer if it is the one we want inLayerDefn = inLayer.GetLayerDefn() for i in range(0, inLayerDefn.GetFieldCount()): fieldDefn = inLayerDefn.GetFieldDefn(i) outLayer.CreateField(fieldDefn) # Get the output Layer's Feature Definition outLayerDefn = outLayer.GetLayerDefn() progbar = ProgressBar(inLayer.GetFeatureCount(), 50, "Copying features") counter = 0 for feature in inLayer: counter += 1 progbar.update(counter) geom = feature.GetGeometryRef() if geom is None: progbar.erase() # get around the progressbar log.warning('Feature with FID={} has no geometry. Skipping'.format( feature.GetFID())) continue geom.Transform(transform) # if clip_shape: # raw = shape(json.loads(geom.ExportToJson())) # try: # clip = raw.intersection(clip_shape) # geom = ogr.CreateGeometryFromJson(json.dumps(mapping(clip))) # except Exception as e: # progbar.erase() # get around the progressbar # log.warning('Invalid shape with FID={} cannot be intersected'.format(feature.GetFID())) # Create output Feature outFeature = ogr.Feature(outLayerDefn) outFeature.SetGeometry(geom) # Add field values from input Layer for i in range(0, outLayerDefn.GetFieldCount()): outFeature.SetField( outLayerDefn.GetFieldDefn(i).GetNameRef(), feature.GetField(i)) outLayer.CreateFeature(outFeature) outFeature = None progbar.finish() inDataSource = None outDataSource = None
def distance_from_features(polygons, tmp_folder, bounds, cell_size_meters, cell_size_degrees, output, features, statistic, field): """[summary] Feature class rasterization https://gdal.org/programs/gdal_rasterize.html Euclidean distance documentation https://www.pydoc.io/pypi/pygeoprocessing-1.0.0/autoapi/geoprocessing/index.html#geoprocessing.distance_transform_edt Arguments: polygons {[type]} -- [description] features_shapefile {[type]} -- [description] temp_folder {[type]} -- [description] """ log = Logger('Conflict') if features is None: log.warning( 'Skipping distance calculation for {} because feature class does not exist.' .format(field)) return with get_shp_or_gpkg(features) as lyr: if lyr.ogr_layer.GetFeatureCount() < 1: log.warning( 'Skipping distance calculation for {} because feature class is empty.' .format(field)) return safe_makedirs(tmp_folder) root_path = os.path.join(tmp_folder, os.path.splitext(os.path.basename(features))[0]) features_raster = root_path + '_features.tif' distance_raster = root_path + '_euclidean.tif' if os.path.isfile(features_raster): rasterio.shutil.delete(features_raster) if os.path.isfile(distance_raster): rasterio.shutil.delete(distance_raster) progbar = ProgressBar(100, 50, "Rasterizing ") def poly_progress(progress, _msg, _data): progbar.update(int(progress * 100)) # Rasterize the features (roads, rail etc) and calculate a raster of Euclidean distance from these features log.info( 'Rasterizing {:,} features at {}m cell size for generating {} field using {} distance.' .format(len(polygons), cell_size_meters, field, statistic)) progbar.update(0) gdal.Rasterize(features_raster, os.path.dirname(features), layers=os.path.basename(features), xRes=cell_size_degrees, yRes=cell_size_degrees, burnValues=1, outputType=gdal.GDT_Int16, creationOptions=['COMPRESS=LZW'], outputBounds=bounds, callback=poly_progress) progbar.finish() log.info('Calculating Euclidean distance for {}'.format(field)) geoprocessing.distance_transform_edt((features_raster, 1), distance_raster) # Calculate the Euclidean distance statistics (mean, min, max etc) for each polygon values = raster_buffer_stats2(polygons, distance_raster) # Retrieve the desired statistic and store it as the specified output field log.info('Extracting {} statistic for {}.'.format(statistic, field)) progbar = ProgressBar(len(values), 50, "Extracting Statistics") counter = 0 for reach_id, statistics in values.items(): counter += 1 progbar.update(counter) if reach_id not in output: output[reach_id] = {} output[reach_id][field] = round( statistics[statistic] * cell_size_meters, 0) progbar.finish() # Cleanup the temporary datasets rasterio.shutil.delete(features_raster) rasterio.shutil.delete(distance_raster) log.info('{} distance calculation complete'.format(field))
def rvd(huc: int, flowlines_orig: Path, existing_veg_orig: Path, historic_veg_orig: Path, valley_bottom_orig: Path, output_folder: Path, reach_codes: List[str], flow_areas_orig: Path, waterbodies_orig: Path, meta=None): """[Generate segmented reaches on flowline network and calculate RVD from historic and existing vegetation rasters Args: huc (integer): Watershed ID flowlines_orig (Path): Segmented flowlines feature layer existing_veg_orig (Path): LANDFIRE version 2.00 evt raster, with adjacent xml metadata file historic_veg_orig (Path): LANDFIRE version 2.00 bps raster, with adjacent xml metadata file valley_bottom_orig (Path): Vbet polygon feature layer output_folder (Path): destination folder for project output reach_codes (List[int]): NHD reach codes for features to include in outputs flow_areas_orig (Path): NHD flow area polygon feature layer waterbodies (Path): NHD waterbodies polygon feature layer meta (Dict[str,str]): dictionary of riverscapes metadata key: value pairs """ log = Logger("RVD") log.info('RVD v.{}'.format(cfg.version)) try: int(huc) except ValueError: raise Exception('Invalid HUC identifier "{}". Must be an integer'.format(huc)) if not (len(huc) == 4 or len(huc) == 8): raise Exception('Invalid HUC identifier. Must be four digit integer') safe_makedirs(output_folder) project, _realization, proj_nodes = create_project(huc, output_folder) # Incorporate project metadata to the riverscapes project if meta is not None: project.add_metadata(meta) log.info('Adding inputs to project') _prj_existing_path_node, prj_existing_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['EXVEG'], existing_veg_orig) _prj_historic_path_node, prj_historic_path = project.add_project_raster(proj_nodes['Inputs'], LayerTypes['HISTVEG'], historic_veg_orig) # TODO: Don't forget the att_filter # _prj_flowlines_node, prj_flowlines = project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS'], flowlines, att_filter="\"ReachCode\" Like '{}%'".format(huc)) # Copy in the vectors we need inputs_gpkg_path = os.path.join(output_folder, LayerTypes['INPUTS'].rel_path) intermediates_gpkg_path = os.path.join(output_folder, LayerTypes['INTERMEDIATES'].rel_path) outputs_gpkg_path = os.path.join(output_folder, LayerTypes['OUTPUTS'].rel_path) # Make sure we're starting with empty/fresh geopackages GeopackageLayer.delete(inputs_gpkg_path) GeopackageLayer.delete(intermediates_gpkg_path) GeopackageLayer.delete(outputs_gpkg_path) # Copy our input layers and also find the difference in the geometry for the valley bottom flowlines_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOWLINES'].rel_path) vbottom_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['VALLEY_BOTTOM'].rel_path) copy_feature_class(flowlines_orig, flowlines_path, epsg=cfg.OUTPUT_EPSG) copy_feature_class(valley_bottom_orig, vbottom_path, epsg=cfg.OUTPUT_EPSG) with GeopackageLayer(flowlines_path) as flow_lyr: # Set the output spatial ref as this for the whole project out_srs = flow_lyr.spatial_ref meter_conversion = flow_lyr.rough_convert_metres_to_vector_units(1) distance_buffer = flow_lyr.rough_convert_metres_to_vector_units(1) # Transform issues reading 102003 as espg id. Using sr wkt seems to work, however arcgis has problems loading feature classes with this method... raster_srs = ogr.osr.SpatialReference() ds = gdal.Open(prj_existing_path, 0) raster_srs.ImportFromWkt(ds.GetProjectionRef()) raster_srs.SetAxisMappingStrategy(osr.OAMS_TRADITIONAL_GIS_ORDER) transform_shp_to_raster = VectorBase.get_transform(out_srs, raster_srs) gt = ds.GetGeoTransform() cell_area = ((gt[1] / meter_conversion) * (-gt[5] / meter_conversion)) # Create the output feature class fields with GeopackageLayer(outputs_gpkg_path, layer_name='ReachGeometry', delete_dataset=True) as out_lyr: out_lyr.create_layer(ogr.wkbMultiLineString, spatial_ref=out_srs, options=['FID=ReachID'], fields={ 'GNIS_NAME': ogr.OFTString, 'ReachCode': ogr.OFTString, 'TotDASqKm': ogr.OFTReal, 'NHDPlusID': ogr.OFTReal, 'WatershedID': ogr.OFTInteger }) metadata = { 'RVD_DateTime': datetime.datetime.now().isoformat(), 'Reach_Codes': reach_codes } # Execute the SQL to create the lookup tables in the RVD geopackage SQLite database watershed_name = create_database(huc, outputs_gpkg_path, metadata, cfg.OUTPUT_EPSG, os.path.join(os.path.abspath(os.path.dirname(__file__)), '..', 'database', 'rvd_schema.sql')) project.add_metadata({'Watershed': watershed_name}) geom_vbottom = get_geometry_unary_union(vbottom_path, spatial_ref=raster_srs) flowareas_path = None if flow_areas_orig: flowareas_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['FLOW_AREA'].rel_path) copy_feature_class(flow_areas_orig, flowareas_path, epsg=cfg.OUTPUT_EPSG) geom_flow_areas = get_geometry_unary_union(flowareas_path) # Difference with existing vbottom geom_vbottom = geom_vbottom.difference(geom_flow_areas) else: del LayerTypes['INPUTS'].sub_layers['FLOW_AREA'] waterbodies_path = None if waterbodies_orig: waterbodies_path = os.path.join(inputs_gpkg_path, LayerTypes['INPUTS'].sub_layers['WATERBODIES'].rel_path) copy_feature_class(waterbodies_orig, waterbodies_path, epsg=cfg.OUTPUT_EPSG) geom_waterbodies = get_geometry_unary_union(waterbodies_path) # Difference with existing vbottom geom_vbottom = geom_vbottom.difference(geom_waterbodies) else: del LayerTypes['INPUTS'].sub_layers['WATERBODIES'] # Add the inputs to the XML _nd, _in_gpkg_path, _sublayers = project.add_project_geopackage(proj_nodes['Inputs'], LayerTypes['INPUTS']) # Filter the flow lines to just the required features and then segment to desired length # TODO: These are brat methods that need to be refactored to use VectorBase layers cleaned_path = os.path.join(outputs_gpkg_path, 'ReachGeometry') build_network(flowlines_path, flowareas_path, cleaned_path, waterbodies_path=waterbodies_path, epsg=cfg.OUTPUT_EPSG, reach_codes=reach_codes, create_layer=False) # Generate Voroni polygons log.info("Calculating Voronoi Polygons...") # Add all the points (including islands) to the list flowline_thiessen_points_groups = centerline_points(cleaned_path, distance_buffer, transform_shp_to_raster) flowline_thiessen_points = [pt for group in flowline_thiessen_points_groups.values() for pt in group] simple_save([pt.point for pt in flowline_thiessen_points], ogr.wkbPoint, raster_srs, "Thiessen_Points", intermediates_gpkg_path) # Exterior is the shell and there is only ever 1 myVorL = NARVoronoi(flowline_thiessen_points) # Generate Thiessen Polys myVorL.createshapes() # Dissolve by flowlines log.info("Dissolving Thiessen Polygons") dissolved_polys = myVorL.dissolve_by_property('fid') # Clip Thiessen Polys log.info("Clipping Thiessen Polygons to Valley Bottom") clipped_thiessen = clip_polygons(geom_vbottom, dissolved_polys) # Save Intermediates simple_save(clipped_thiessen.values(), ogr.wkbPolygon, raster_srs, "Thiessen", intermediates_gpkg_path) simple_save(dissolved_polys.values(), ogr.wkbPolygon, raster_srs, "ThiessenPolygonsDissolved", intermediates_gpkg_path) simple_save(myVorL.polys, ogr.wkbPolygon, raster_srs, "ThiessenPolygonsRaw", intermediates_gpkg_path) _nd, _inter_gpkg_path, _sublayers = project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES']) # OLD METHOD FOR AUDIT # dissolved_polys2 = dissolve_by_points(flowline_thiessen_points_groups, myVorL.polys) # simple_save(dissolved_polys2.values(), ogr.wkbPolygon, out_srs, "ThiessenPolygonsDissolved_OLD", intermediates_gpkg_path) # Load Vegetation Rasters log.info(f"Loading Existing and Historic Vegetation Rasters") vegetation = {} vegetation["EXISTING"] = load_vegetation_raster(prj_existing_path, outputs_gpkg_path, True, output_folder=os.path.join(output_folder, 'Intermediates')) vegetation["HISTORIC"] = load_vegetation_raster(prj_historic_path, outputs_gpkg_path, False, output_folder=os.path.join(output_folder, 'Intermediates')) for epoch in vegetation.keys(): for name in vegetation[epoch].keys(): if not f"{epoch}_{name}" == "HISTORIC_LUI": project.add_project_raster(proj_nodes['Intermediates'], LayerTypes[f"{epoch}_{name}"]) if vegetation["EXISTING"]["RAW"].shape != vegetation["HISTORIC"]["RAW"].shape: raise Exception('Vegetation raster shapes are not equal Existing={} Historic={}. Cannot continue'.format(vegetation["EXISTING"]["RAW"].shape, vegetation["HISTORIC"]["RAW"].shape)) # Vegetation zone calculations riparian_zone_arrays = {} riparian_zone_arrays["RIPARIAN_ZONES"] = ((vegetation["EXISTING"]["RIPARIAN"] + vegetation["HISTORIC"]["RIPARIAN"]) > 0) * 1 riparian_zone_arrays["NATIVE_RIPARIAN_ZONES"] = ((vegetation["EXISTING"]["NATIVE_RIPARIAN"] + vegetation["HISTORIC"]["NATIVE_RIPARIAN"]) > 0) * 1 riparian_zone_arrays["VEGETATION_ZONES"] = ((vegetation["EXISTING"]["VEGETATED"] + vegetation["HISTORIC"]["VEGETATED"]) > 0) * 1 # Save Intermediate Rasters for name, raster in riparian_zone_arrays.items(): save_intarr_to_geotiff(raster, os.path.join(output_folder, "Intermediates", f"{name}.tif"), prj_existing_path) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes[name]) # Calculate Riparian Departure per Reach riparian_arrays = {f"{epoch.capitalize()}{(name.capitalize()).replace('Native_riparian', 'NativeRiparian')}Mean": array for epoch, arrays in vegetation.items() for name, array in arrays.items() if name in ["RIPARIAN", "NATIVE_RIPARIAN"]} # Vegetation Cell Counts raw_arrays = {f"{epoch}": array for epoch, arrays in vegetation.items() for name, array in arrays.items() if name == "RAW"} # Generate Vegetation Conversions vegetation_change = (vegetation["HISTORIC"]["CONVERSION"] - vegetation["EXISTING"]["CONVERSION"]) save_intarr_to_geotiff(vegetation_change, os.path.join(output_folder, "Intermediates", "Conversion_Raster.tif"), prj_existing_path) project.add_project_raster(proj_nodes['Intermediates'], LayerTypes['VEGETATION_CONVERSION']) # load conversion types dictionary from database conn = sqlite3.connect(outputs_gpkg_path) conn.row_factory = dict_factory curs = conn.cursor() curs.execute('SELECT * FROM ConversionTypes') conversion_classifications = curs.fetchall() curs.execute('SELECT * FROM vwConversions') conversion_ids = curs.fetchall() # Split vegetation change classes into binary arrays vegetation_change_arrays = { c['FieldName']: (vegetation_change == int(c["TypeValue"])) * 1 if int(c["TypeValue"]) in np.unique(vegetation_change) else None for c in conversion_classifications } # Calcuate average and unique cell counts per reach progbar = ProgressBar(len(clipped_thiessen.keys()), 50, "Extracting array values by reach...") counter = 0 discarded = 0 with rasterio.open(prj_existing_path) as dataset: unique_vegetation_counts = {} reach_average_riparian = {} reach_average_change = {} for reachid, poly in clipped_thiessen.items(): counter += 1 progbar.update(counter) # we can discount a lot of shapes here. if not poly.is_valid or poly.is_empty or poly.area == 0 or poly.geom_type not in ["Polygon", "MultiPolygon"]: discarded += 1 continue raw_values_unique = {} change_values_mean = {} riparian_values_mean = {} reach_raster = np.ma.masked_invalid( features.rasterize( [poly], out_shape=dataset.shape, transform=dataset.transform, all_touched=True, fill=np.nan)) for raster_name, raster in raw_arrays.items(): if raster is not None: current_raster = np.ma.masked_array(raster, mask=reach_raster.mask) raw_values_unique[raster_name] = np.unique(np.ma.filled(current_raster, fill_value=0), return_counts=True) else: raw_values_unique[raster_name] = [] for raster_name, raster in riparian_arrays.items(): if raster is not None: current_raster = np.ma.masked_array(raster, mask=reach_raster.mask) riparian_values_mean[raster_name] = np.ma.mean(current_raster) else: riparian_values_mean[raster_name] = 0.0 for raster_name, raster in vegetation_change_arrays.items(): if raster is not None: current_raster = np.ma.masked_array(raster, mask=reach_raster.mask) change_values_mean[raster_name] = np.ma.mean(current_raster) else: change_values_mean[raster_name] = 0.0 unique_vegetation_counts[reachid] = raw_values_unique reach_average_riparian[reachid] = riparian_values_mean reach_average_change[reachid] = change_values_mean progbar.finish() with SQLiteCon(outputs_gpkg_path) as gpkg: # Ensure all reaches are present in the ReachAttributes table before storing RVD output values gpkg.curs.execute('INSERT INTO ReachAttributes (ReachID) SELECT ReachID FROM ReachGeometry;') errs = 0 for reachid, epochs in unique_vegetation_counts.items(): for epoch in epochs.values(): insert_values = [[reachid, int(vegetationid), float(count * cell_area), int(count)] for vegetationid, count in zip(epoch[0], epoch[1]) if vegetationid != 0] try: gpkg.curs.executemany('''INSERT INTO ReachVegetation ( ReachID, VegetationID, Area, CellCount) VALUES (?,?,?,?)''', insert_values) # Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is except sqlite3.IntegrityError as err: # THis is likely a constraint error. errstr = "Integrity Error when inserting records: ReachID: {} VegetationIDs: {}".format(reachid, str(list(epoch[0]))) log.error(errstr) errs += 1 except sqlite3.Error as err: # This is any other kind of error errstr = "SQL Error when inserting records: ReachID: {} VegetationIDs: {} ERROR: {}".format(reachid, str(list(epoch[0])), str(err)) log.error(errstr) errs += 1 if errs > 0: raise Exception('Errors were found inserting records into the database. Cannot continue.') gpkg.conn.commit() # load RVD departure levels from DepartureLevels database table with SQLiteCon(outputs_gpkg_path) as gpkg: gpkg.curs.execute('SELECT LevelID, MaxRVD FROM DepartureLevels ORDER BY MaxRVD ASC') departure_levels = gpkg.curs.fetchall() # Calcuate Average Departure for Riparian and Native Riparian riparian_departure_values = riparian_departure(reach_average_riparian, departure_levels) write_db_attributes(outputs_gpkg_path, riparian_departure_values, departure_type_columns) # Add Conversion Code, Type to Vegetation Conversion with SQLiteCon(outputs_gpkg_path) as gpkg: gpkg.curs.execute('SELECT LevelID, MaxValue, NAME FROM ConversionLevels ORDER BY MaxValue ASC') conversion_levels = gpkg.curs.fetchall() reach_values_with_conversion_codes = classify_conversions(reach_average_change, conversion_ids, conversion_levels) write_db_attributes(outputs_gpkg_path, reach_values_with_conversion_codes, rvd_columns) # # Write Output to GPKG table # log.info('Insert values to GPKG tables') # # TODO move this to write_attirubtes method # with get_shp_or_gpkg(outputs_gpkg_path, layer_name='ReachAttributes', write=True, ) as in_layer: # # Create each field and store the name and index in a list of tuples # field_indices = [(field, in_layer.create_field(field, field_type)) for field, field_type in { # "FromConifer": ogr.OFTReal, # "FromDevegetated": ogr.OFTReal, # "FromGrassShrubland": ogr.OFTReal, # "FromDeciduous": ogr.OFTReal, # "NoChange": ogr.OFTReal, # "Deciduous": ogr.OFTReal, # "GrassShrubland": ogr.OFTReal, # "Devegetation": ogr.OFTReal, # "Conifer": ogr.OFTReal, # "Invasive": ogr.OFTReal, # "Development": ogr.OFTReal, # "Agriculture": ogr.OFTReal, # "ConversionCode": ogr.OFTInteger, # "ConversionType": ogr.OFTString}.items()] # for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]): # reach = feature.GetFID() # if reach not in reach_values_with_conversion_codes: # continue # # Set all the field values and then store the feature # for field, _idx in field_indices: # if field in reach_values_with_conversion_codes[reach]: # if not reach_values_with_conversion_codes[reach][field]: # feature.SetField(field, None) # else: # feature.SetField(field, reach_values_with_conversion_codes[reach][field]) # in_layer.ogr_layer.SetFeature(feature) # # Create each field and store the name and index in a list of tuples # field_indices = [(field, in_layer.create_field(field, field_type)) for field, field_type in { # "EXISTING_RIPARIAN_MEAN": ogr.OFTReal, # "HISTORIC_RIPARIAN_MEAN": ogr.OFTReal, # "RIPARIAN_DEPARTURE": ogr.OFTReal, # "EXISTING_NATIVE_RIPARIAN_MEAN": ogr.OFTReal, # "HISTORIC_NATIVE_RIPARIAN_MEAN": ogr.OFTReal, # "NATIVE_RIPARIAN_DEPARTURE": ogr.OFTReal, }.items()] # for feature, _counter, _progbar in in_layer.iterate_features("Writing Attributes", write_layers=[in_layer]): # reach = feature.GetFID() # if reach not in riparian_departure_values: # continue # # Set all the field values and then store the feature # for field, _idx in field_indices: # if field in riparian_departure_values[reach]: # if not riparian_departure_values[reach][field]: # feature.SetField(field, None) # else: # feature.SetField(field, riparian_departure_values[reach][field]) # in_layer.ogr_layer.SetFeature(feature) # with sqlite3.connect(outputs_gpkg_path) as conn: # cursor = conn.cursor() # errs = 0 # for reachid, epochs in unique_vegetation_counts.items(): # for epoch in epochs.values(): # insert_values = [[reachid, int(vegetationid), float(count * cell_area), int(count)] for vegetationid, count in zip(epoch[0], epoch[1]) if vegetationid != 0] # try: # cursor.executemany('''INSERT INTO ReachVegetation ( # ReachID, # VegetationID, # Area, # CellCount) # VALUES (?,?,?,?)''', insert_values) # # Sqlite can't report on SQL errors so we have to print good log messages to help intuit what the problem is # except sqlite3.IntegrityError as err: # # THis is likely a constraint error. # errstr = "Integrity Error when inserting records: ReachID: {} VegetationIDs: {}".format(reachid, str(list(epoch[0]))) # log.error(errstr) # errs += 1 # except sqlite3.Error as err: # # This is any other kind of error # errstr = "SQL Error when inserting records: ReachID: {} VegetationIDs: {} ERROR: {}".format(reachid, str(list(epoch[0])), str(err)) # log.error(errstr) # errs += 1 # if errs > 0: # raise Exception('Errors were found inserting records into the database. Cannot continue.') # conn.commit() # Add intermediates and the report to the XML # project.add_project_geopackage(proj_nodes['Intermediates'], LayerTypes['INTERMEDIATES']) already # added above project.add_project_geopackage(proj_nodes['Outputs'], LayerTypes['OUTPUTS']) # Add the report to the XML report_path = os.path.join(project.project_dir, LayerTypes['REPORT'].rel_path) project.add_report(proj_nodes['Outputs'], LayerTypes['REPORT'], replace=True) report = RVDReport(report_path, project) report.write() log.info('RVD complete')