def main():
parser = argparse.ArgumentParser(
description='RVD XML Augmenter',
# epilog="This is an epilog"
)
parser.add_argument('out_project_xml', help='Input XML file', type=str)
parser.add_argument('in_xmls', help='Comma-separated list of XMLs in decreasing priority', type=str)
parser.add_argument('--verbose', help='(optional) a little extra logging ', action='store_true', default=False)
args = dotenv.parse_args_env(parser)
# Initiate the log file
log = Logger('XML Augmenter')
log.setup(verbose=args.verbose)
log.title('XML Augmenter: {}'.format(args.out_project_xml))
try:
out_prj = RSProject(None, args.out_project_xml)
out_prj.rs_meta_augment(
args.in_xmls.split(','),
lyrs_in_out
)
out_prj.XMLBuilder.write()
report_path = out_prj.XMLBuilder.find('.//HTMLFile[@id="RVD_REPORT"]/Path').text
report = RVDReport(os.path.join(out_prj.project_dir, report_path), out_prj)
report.write()
except Exception as e:
log.error(e)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
sys.exit(0)
def edit_xml(projectpath):
"""Here's an example of how to edit a pre-existing project.rs.xml file
Args:
projectpath ([type]): [description]
"""
log = Logger('edit_xml')
log.info('Loading the XML to make edits...')
# Load up a new RSProject class
project = RSProject(cfg, projectpath)
# Now, instead of creating nodes we can just find them
r1_node = project.XMLBuilder.find_by_id('INN_CTX01')
# Now we can add new metadata values to this node
# Note that we specify r1_node. If you don't do this then it writes to the project metadata
project.add_metadata({'EditedVal': 'Some Realization Value here'}, r1_node)
# Same is true for Rasters if we want
r1_input_raster_node = project.XMLBuilder.find_by_id('AP_01')
project.add_metadata({'EditedVal Raster': 'Some Raster Value here'},
r1_input_raster_node)
# Don't forget to write back to the file
log.info('Writing file')
project.XMLBuilder.write()
log.info('Done')
def create_project(huc, output_dir):
project_name = 'RVD for HUC {}'.format(huc)
project = RSProject(cfg, output_dir)
project.create(project_name, 'RVD')
project.add_metadata({
'HUC{}'.format(len(huc)): str(huc),
'HUC': str(huc),
'RVDVersion': cfg.version,
'RVDTimestamp': str(int(time.time()))
})
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root, 'Realizations')
realization = project.XMLBuilder.add_sub_element(realizations, 'RVD', None, {
'id': 'RVD1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid1()),
'productVersion': cfg.version
})
proj_nodes = {
'Name': project.XMLBuilder.add_sub_element(realization, 'Name', project_name),
'Inputs': project.XMLBuilder.add_sub_element(realization, 'Inputs'),
'Intermediates': project.XMLBuilder.add_sub_element(realization, 'Intermediates'),
'Outputs': project.XMLBuilder.add_sub_element(realization, 'Outputs')
}
project.XMLBuilder.write()
return project, realization, proj_nodes
def process_lst(lst_xml_folder):
"""This is a slightly hack-y script to create some XMLS for the land_surface_temp script
It's a bit of an afterthought so it just plunks down the XMLS all alone in a folder
Args:
lst_xml_folder ([type]): [description]
"""
log = Logger("Generate XMLS for LST")
hucs = [str(1700 + x) for x in range(1, 13)]
for huc in hucs:
hucdir = os.path.join(lst_xml_folder, huc)
xml_file = os.path.join(hucdir, 'project.rs.xml')
safe_makedirs(hucdir)
if os.path.exists(xml_file):
safe_remove_file(xml_file)
project_name = f'Land Surface Temperature for HUC {huc}'
project = RSProject(cfg, xml_file)
project.create(project_name, 'LST')
project.add_metadata({
'ModelVersion': cfg.version,
'HUC': huc,
'dateCreated': datetime.datetime.now().isoformat(),
'HUC{}'.format(len(huc)): huc
})
realizations = project.XMLBuilder.add_sub_element(
project.XMLBuilder.root, 'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'LST', None, {
'id': 'LST1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid4()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
output_node = project.XMLBuilder.add_sub_element(
realization, 'Outputs')
zipfile_node = project.add_dataset(output_node,
f'{huc}.zip',
RSLayer(f'LST Result for {huc}',
'LST_ZIP', 'ZipFile',
'1706.zip'),
'ZipFile',
replace=True,
rel_path=True)
project.XMLBuilder.write()
log.info('done')
def create_project(huc, output_dir):
project_name = 'Riverscapes Context for HUC {}'.format(huc)
project = RSProject(cfg, output_dir)
project.create(project_name, 'RSContext')
project.add_metadata({'HUC{}'.format(len(huc)): str(huc)})
project.add_metadata({'HUC': str(huc)})
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root,
'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'RSContext', None, {
'id': 'RSContext1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid4()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
project.XMLBuilder.write()
return project, realization
def create_project(project_csv, output_dir):
project_name = project_csv['Name']
project = RSProject(cfg, output_dir)
project.create(project_name, 'fmLTPBR')
project.add_metadata(project_csv)
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root,
'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'FMLTPBR', None, {
'id': 'fmLTPBR1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid1()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
project.XMLBuilder.write()
return project, realization
def create_project(huc, output_dir):
project_name = 'VBET for HUC {}'.format(huc)
project = RSProject(cfg, output_dir)
project.create(project_name, 'VBET')
project.add_metadata({
'HUC{}'.format(len(huc)): str(huc),
'HUC': str(huc),
'VBETVersion': cfg.version,
'VBETTimestamp': str(int(time.time()))
})
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root,
'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'VBET', None, {
'id': 'VBET',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid1()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
proj_nodes = {
'Inputs':
project.XMLBuilder.add_sub_element(realization, 'Inputs'),
'Intermediates':
project.XMLBuilder.add_sub_element(realization, 'Intermediates'),
'Outputs':
project.XMLBuilder.add_sub_element(realization, 'Outputs')
}
# Make sure we have these folders
proj_dir = os.path.dirname(project.xml_path)
safe_makedirs(os.path.join(proj_dir, 'inputs'))
safe_makedirs(os.path.join(proj_dir, 'intermediates'))
safe_makedirs(os.path.join(proj_dir, 'outputs'))
project.XMLBuilder.write()
return project, realization, proj_nodes
def create_project(huc, output_dir):
""" Create riverscapes project XML
Args:
huc (str): Watershed HUC code
output_dir (str): Full absolute path to output folder
Returns:
tuple: (project XML object, realization node, dictionary of other nodes)
"""
project_name = 'BRAT for HUC {}'.format(huc)
project = RSProject(cfg, output_dir)
project.create(project_name, 'BRAT')
project.add_metadata({
'HUC{}'.format(len(huc)): str(huc),
'HUC': str(huc),
'BRATBuildVersion': cfg.version,
'BRATBuildTimestamp': str(int(time.time()))
})
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root, 'Realizations')
realization = project.XMLBuilder.add_sub_element(realizations, 'BRAT', None, {
'id': 'BRAT1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid1()),
'productVersion': cfg.version
})
proj_nodes = {
'Name': project.XMLBuilder.add_sub_element(realization, 'Name', project_name),
'Inputs': project.XMLBuilder.add_sub_element(realization, 'Inputs'),
'Intermediates': project.XMLBuilder.add_sub_element(realization, 'Intermediates'),
'Outputs': project.XMLBuilder.add_sub_element(realization, 'Outputs')
}
project.XMLBuilder.write()
return project, realization, proj_nodes
def create_project(huc, output_dir, realization_meta):
project_name = 'Confinement for HUC {}'.format(huc)
project = RSProject(cfg, output_dir)
project.create(project_name, 'Confinement')
project.add_metadata({'HUC{}'.format(len(huc)): str(huc)})
project.add_metadata({'HUC': str(huc)})
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root,
'Realizations')
realization = project.XMLBuilder.add_sub_element(
realizations, 'Confinement', None, {
'id': 'Confinement1',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid4()),
'productVersion': cfg.version
})
project.XMLBuilder.add_sub_element(realization, 'Name', project_name)
project.add_metadata(realization_meta)
proj_nodes = {
'Inputs':
project.XMLBuilder.add_sub_element(realization, 'Inputs'),
'Intermediates':
project.XMLBuilder.add_sub_element(realization, 'Intermediates'),
'Outputs':
project.XMLBuilder.add_sub_element(realization, 'Outputs')
}
proj_dir = os.path.dirname(project.xml_path)
safe_makedirs(os.path.join(proj_dir, 'inputs'))
safe_makedirs(os.path.join(proj_dir, 'intermediates'))
safe_makedirs(os.path.join(proj_dir, 'outputs'))
report_path = os.path.join(project.project_dir,
LayerTypes['CONFINEMENT_RUN_REPORT'].rel_path)
project.add_report(proj_nodes['Outputs'],
LayerTypes['CONFINEMENT_RUN_REPORT'],
replace=True)
project.XMLBuilder.write()
return project, realization, proj_nodes, report_path
'RSContext').getchildren()
[
self.layerprint(lyr, section, self.project_root) for lyr in layers
if lyr.tag in ['DEM', 'Raster', 'Vector', 'Geopackage']
]
[
self.layerprint(lyr, section, self.project_root) for lyr in layers
if lyr.tag in ['SQLiteDB']
]
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('projectxml',
help='Path to the project.rs.xml',
type=str)
parser.add_argument('report_path',
help='Output path where report will be generated',
type=str)
args = dotenv.parse_args_env(parser)
cfg = ModelConfig(
'http://xml.riverscapes.xyz/Projects/XSD/V1/RSContext.xsd',
__version__)
project = RSProject(cfg, args.projectxml)
report = RSContextReport(args.report_path, project,
os.path.dirname(args.projectxml))
report.write()
def build_xml(projectpath):
"""Here's an example of how to build a project.rs.xml file
Args:
projectpath ([type]): [description]
"""
# Create the top-level nodes
log = Logger('build_xml')
log.info('Starting the build of the XML')
project_name = 'Inundation Mapper'
project = RSProject(cfg, projectpath)
project.create(project_name, 'Inundation')
# Add the root metadata
project.add_metadata({
'ModelVersion': cfg.version,
'date_created': str(int(time.time())),
'HUC8': '16010201',
'InundationVersion': cfg.version,
'watershed': 'Upper Bear',
'site_name': 'Mill Creek',
})
# Create the realizations container node
realizations = project.XMLBuilder.add_sub_element(project.XMLBuilder.root,
'Realizations')
# Example InundationContext Realization
# ================================================================================================
r1_node = project.XMLBuilder.add_sub_element(
realizations, 'InundationContext', None, {
'id': 'INN_CTX01',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid1()),
'productVersion': cfg.version
})
# add a <Name> node
project.XMLBuilder.add_sub_element(r1_node, 'Name', project_name)
# Realization <MetaData>
project.add_metadata(
{
'mapper': 'Karen Bartelt',
'date_mapped': '02042020',
'year1': 'estimated pre beaver',
'year2': '2019',
'RS_used': 'RS_01'
}, r1_node)
# Add an <Input> and <Output> nodes
r1_inputs = project.XMLBuilder.add_sub_element(r1_node, 'Inputs')
r1_outputs = project.XMLBuilder.add_sub_element(r1_node, 'Outputs')
# Now we can add inputs to the context raster
# Note the return is an HTML node and a raster path we can use for other things
raster_node, raster_path = project.add_project_raster(r1_inputs,
LayerTypes['AP_01'],
replace=False)
# Here we add a vector node
vector_node, vector_path = project.add_project_vector(
r1_inputs, LayerTypes['APSTR_01'], replace=False)
# Example DCE Realization
# ================================================================================================
r2_node = project.XMLBuilder.add_sub_element(
realizations, 'InundationDCE', None, {
'id': 'DCE_01',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid1()),
'productVersion': cfg.version
})
r2_name_node = project.XMLBuilder.add_sub_element(r2_node, 'Name',
'August 2019')
# Add an <Input> and <Output> nodes
r2_inputs = project.XMLBuilder.add_sub_element(r2_node, 'Inputs')
r2_outputs = project.XMLBuilder.add_sub_element(r2_node, 'Outputs')
# Example CD Realization
# ================================================================================================
r3_node = project.XMLBuilder.add_sub_element(
realizations, 'InundationCD', None, {
'id': '',
'dateCreated': datetime.datetime.now().isoformat(),
'guid': str(uuid.uuid1()),
'productVersion': cfg.version
})
r3_name_node = project.XMLBuilder.add_sub_element(
r3_node, 'Name', '2019 vs estimated pre beaver')
# Add an <Input> and <Output> nodes
r3_inputs = project.XMLBuilder.add_sub_element(r3_node, 'Inputs')
r3_outputs = project.XMLBuilder.add_sub_element(r3_node, 'Outputs')
# Finally write the file
log.info('Writing file')
project.XMLBuilder.write()
log.info('Done')
def brat_run(project_root, csv_dir):
"""Run the BRAT model and calculat dam capacity
as well as conservation and restoration.
Arguments:
database {str} -- Path to existing BRAT SQLite database
csv_dir {str} -- Path to the directory containing the BRAT lookup CSV data files
shapefile {str} -- Path to the existing BRAT reach segment ShapeFile
project_root {str} -- (Optional) path to Riverscapes project directory
"""
log = Logger('BRAT Run')
log.info('Starting BRAT run')
project = RSProject(cfg, project_root)
project.add_metadata({
'BRATRunVersion': cfg.version,
'BRATRunTimestamp': str(int(time.time()))
})
realizations = project.XMLBuilder.find('Realizations').findall('BRAT')
if len(realizations) != 1:
raise Exception(
'Could not find a valid realization inside the existing brat project'
)
# Fetch some XML nodes we'll need to aleter
r_node = realizations[0]
input_node = r_node.find('Inputs')
intermediate_node = r_node.find('Intermediates')
outputs_node = r_node.find('Outputs')
# Get the filepaths for the DB and shapefile
gpkg_path = os.path.join(
project.project_dir,
r_node.find('Outputs/Geopackage[@id="OUTPUTS"]/Path').text)
if not os.path.isfile(gpkg_path):
raise Exception(
'BRAT geopackage file missing at {}. You must run Brat Build first.'
.format(gpkg_path))
# Update any of the lookup tables we need
update_database(gpkg_path, csv_dir)
# Store the BRAT Run date time to the database (for reporting)
store_metadata(gpkg_path, 'BRAT_Run_DateTime',
datetime.datetime.now().isoformat())
watershed, max_drainage_area, ecoregion = get_watershed_info(gpkg_path)
# Set database output columns to NULL before processing (note omission of string lookup fields from view)
set_reach_fields_null(gpkg_path, output_fields[ogr.OFTReal])
set_reach_fields_null(gpkg_path, output_fields[ogr.OFTInteger])
# Calculate the low and high flow using regional discharge equations
hydrology(gpkg_path, 'Low', watershed)
hydrology(gpkg_path, '2', watershed)
# Calculate the vegetation and combined FIS for the existing and historical vegetation epochs
for epoch, prefix, ltype, orig_id in Epochs:
# Calculate the vegetation suitability for each buffer
[
vegetation_suitability(gpkg_path, buffer, prefix, ecoregion)
for buffer in get_stream_buffers(gpkg_path)
]
# Run the vegetation and then combined FIS for this epoch
vegetation_fis(gpkg_path, epoch, prefix)
combined_fis(gpkg_path, epoch, prefix, max_drainage_area)
orig_raster = os.path.join(
project.project_dir,
input_node.find('Raster[@id="{}"]/Path'.format(orig_id)).text)
_veg_suit_raster_node, veg_suit_raster = project.add_project_raster(
intermediate_node, LayerTypes[ltype], None, True)
output_vegetation_raster(gpkg_path, orig_raster, veg_suit_raster,
epoch, prefix, ecoregion)
# Calculate departure from historical conditions
with SQLiteCon(gpkg_path) as database:
log.info('Calculating departure from historic conditions')
database.curs.execute(
'UPDATE ReachAttributes SET mCC_HisDep = mCC_HPE_CT - mCC_EX_CT WHERE (mCC_EX_CT IS NOT NULL) AND (mCC_HPE_CT IS NOT NULL)'
)
database.conn.commit()
# Land use intesity, conservation and restoration
land_use(gpkg_path, 100.0)
conservation(gpkg_path)
report_path = os.path.join(project.project_dir,
LayerTypes['BRAT_RUN_REPORT'].rel_path)
project.add_report(outputs_node,
LayerTypes['BRAT_RUN_REPORT'],
replace=True)
report = BratReport(gpkg_path, report_path, project)
report.write()
log.info('BRAT run complete')
