def main():
# parse command line options
parser = argparse.ArgumentParser()
parser.add_argument('visitID', help='Visit ID', type=int)
parser.add_argument('outputfolder', help='Path to output folder', type=str)
parser.add_argument('--datafolder', help='(optional) Top level folder containing TopoMetrics Riverscapes projects', type=str)
parser.add_argument('--verbose', help='Get more information in your logs.', action='store_true', default=False )
args = parser.parse_args()
# Make sure the output folder exists
resultsFolder = os.path.join(args.outputfolder, "outputs")
# Initiate the log file
logg = Logger("Program")
logfile = os.path.join(resultsFolder, "validation.log")
xmlfile = os.path.join(resultsFolder, "validation.xml")
logg.setup(logPath=logfile, verbose=args.verbose)
# Initiate the log file
log = Logger("Program")
log.setup(logPath=logfile, verbose=args.verbose)
try:
# Make some folders if we need to:
if not os.path.isdir(args.outputfolder):
os.makedirs(args.outputfolder)
if not os.path.isdir(resultsFolder):
os.makedirs(resultsFolder)
# If we need to go get our own topodata.zip file and unzip it we do this
if args.datafolder is None:
topoDataFolder = os.path.join(args.outputfolder, "inputs")
fileJSON, projectFolder = downloadUnzipTopo(args.visitID, topoDataFolder)
# otherwise just pass in a path to existing data
else:
projectFolder = args.datafolder
finalResult = myMainMethod(projectFolder, xmlfile, args.visitID)
sys.exit(finalResult)
except (DataException, MissingException, NetworkException) as e:
# Exception class prints the relevant information
traceback.print_exc(file=sys.stdout)
sys.exit(e.returncode)
except AssertionError as e:
log.error(e.message)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
except Exception as e:
log.error(e.message)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
sys.exit(0)
def main():
# parse command line options
parser = argparse.ArgumentParser()
parser.add_argument('visitID', help='the id of the site to use (no spaces)',type=str)
parser.add_argument('outputfolder', help='Output folder')
parser.add_argument('--datafolder', help='(optional) Top level folder containing TopoMetrics Riverscapes projects', type=str)
parser.add_argument('--logfile', help='output log file.', default="", type=str)
parser.add_argument('--verbose', help = 'Get more information in your logs.', action='store_true', default=False)
args = parser.parse_args()
# Make sure the output folder exists
resultsFolder = os.path.join(args.outputfolder, "outputs")
# Initiate the log file
if args.logfile == "":
logfile = os.path.join(resultsFolder, "cad_export.log")
else:
logfile = args.logfile
logg = Logger("CADExport")
logg.setup(logPath=logfile, verbose=args.verbose)
try:
# Make some folders if we need to:
if not os.path.isdir(args.outputfolder):
os.makedirs(args.outputfolder)
if not os.path.isdir(resultsFolder):
os.makedirs(resultsFolder)
# If we need to go get our own topodata.zip file and unzip it we do this
if args.datafolder is None:
topoDataFolder = os.path.join(args.outputfolder, "inputs")
fileJSON, projectFolder = downloadUnzipTopo(args.visitID, topoDataFolder)
# otherwise just pass in a path to existing data
else:
projectFolder = args.datafolder
projectxml = os.path.join(projectFolder, "project.rs.xml")
finalResult = export_cad_files(projectxml, resultsFolder)
except (MissingException, NetworkException, DataException) as e:
traceback.print_exc(file=sys.stdout)
sys.exit(e.returncode)
except AssertionError as e:
traceback.print_exc(file=sys.stdout)
sys.exit(1)
except Exception as e:
traceback.print_exc(file=sys.stdout)
sys.exit(1)
sys.exit(0)
def downloadExtractParseVisits(visits, outputFolder):
log = Logger('Downloading')
log.info("Downloading all visits from the API")
projects = []
for visit in visits:
try:
extractpath = os.path.join(outputFolder, 'VISIT_{}'.format(visit))
projpath = os.path.join(extractpath, 'project.rs.xml')
downloadUnzipTopo(visit, extractpath)
proj = TopoProject(extractpath)
if proj.isrsproject:
projects.append({"project": proj, "visit": visit})
else:
log.error("File not found: {}".format(projpath))
raise DataException("Missing Project File")
# Just move on if something fails
except Exception, e:
pass
def processZipFile(invVisit, APIVisit):
dirpath = tempfile.mkdtemp()
try:
doupload = False
# Copy some useful things from the API:
for k in ['status', 'lastMeasurementChange', 'lastUpdated', 'name']:
invVisit[k] = APIVisit[k]
file, projpath = downloadUnzipTopo(APIVisit['id'], dirpath)
topo = TopoProject(projpath)
invVisit["topozip"] = True
latestRealizationdate = latestRealizationDate(topo)
if "latestRealization" not in invVisit or APIstrtodate(invVisit["latestRealization"]) >= latestRealizationdate:
invVisit["latestRealization"] = APIdatetostr(latestRealizationdate)
invVisit["size"] = file['size']
uploadProjectToRiverscapes(dirpath, invVisit)
except MissingException, e:
invVisit["topozip"] = False
invVisit["error"] = e.message
def main():
# parse command line options
parser = argparse.ArgumentParser()
parser.add_argument('visitID', help='Visit ID', type=int)
parser.add_argument('outputfolder', help='Path to output folder', type=str)
parser.add_argument(
'--datafolder',
help=
'(optional) Top level folder containing TopoMetrics Riverscapes projects',
type=str)
parser.add_argument('--verbose',
help='Get more information in your logs.',
action='store_true',
default=False)
args = parser.parse_args()
# Make sure the output folder exists
resultsFolder = os.path.join(args.outputfolder, "outputs")
# Initiate the log file
logg = Logger("Program")
logfile = os.path.join(resultsFolder, "bankfull_metrics.log")
xmlfile = os.path.join(resultsFolder, "bankfull_metrics.xml")
logg.setup(logPath=logfile, verbose=args.verbose)
# Initiate the log file
log = Logger("Program")
log.setup(logPath=logfile, verbose=args.verbose)
try:
# Make some folders if we need to:
if not os.path.isdir(args.outputfolder):
os.makedirs(args.outputfolder)
if not os.path.isdir(resultsFolder):
os.makedirs(resultsFolder)
# If we need to go get our own topodata.zip file and unzip it we do this
if args.datafolder is None:
topoDataFolder = os.path.join(args.outputfolder, "inputs")
fileJSON, projectFolder = downloadUnzipTopo(
args.visitID, topoDataFolder)
# otherwise just pass in a path to existing data
else:
projectFolder = args.datafolder
from lib.topoproject import TopoProject
topo_project = TopoProject(
os.path.join(projectFolder, "project.rs.xml"))
tree = ET.parse(os.path.join(projectFolder, "project.rs.xml"))
root = tree.getroot()
visitid = root.findtext(
"./MetaData/Meta[@name='Visit']") if root.findtext(
"./MetaData/Meta[@name='Visit']"
) is not None else root.findtext(
"./MetaData/Meta[@name='VisitID']")
finalResult = bankfull_metrics(topo_project.getpath("DEM"),
topo_project.getpath("DetrendedDEM"),
topo_project.getpath("Topo_Points"))
write_bfmetrics_xml(finalResult, visitid, xmlfile)
sys.exit(0)
except (DataException, MissingException, NetworkException) as e:
# Exception class prints the relevant information
traceback.print_exc(file=sys.stdout)
sys.exit(e.returncode)
except AssertionError as e:
log.error(e.message)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
except Exception as e:
log.error(e.message)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
def main():
# parse command line options
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(
dest='mode', help='For help type `hydroprep.py manual -h`')
# The manual subparser is for when we know explicit paths
manual = subparsers.add_parser('manual', help='manual help')
manual.add_argument('dem',
help='DEM raster path',
type=argparse.FileType('r'))
manual.add_argument('wsdem',
help='Water surface raster path',
type=argparse.FileType('r'))
manual.add_argument('thalweg',
help='Thalweg ShapeFile path',
type=argparse.FileType('r'))
manual.add_argument('outputfolder', help='Output folder')
manual.add_argument('--verbose',
help='Get more information in your logs.',
action='store_true',
default=False)
# The project subparser is when we want to pass in a project.rs.xml file
project = subparsers.add_parser('project', help='project help')
project.add_argument('visitID', help='Visit ID', type=int)
project.add_argument('outputfolder',
help='Path to output folder',
type=str)
project.add_argument(
'--datafolder',
help=
'(optional) Top level folder containing TopoMetrics Riverscapes projects',
type=str)
project.add_argument('--verbose',
help='Get more information in your logs.',
action='store_true',
default=False)
args = parser.parse_args()
try:
if args.mode == "project":
resultsFolder = os.path.join(args.outputfolder, "outputs")
if not os.path.isdir(args.outputfolder):
os.makedirs(args.outputfolder)
if not os.path.isdir(resultsFolder):
os.makedirs(resultsFolder)
# If we need to go get our own topodata.zip file and unzip it we do this
if args.datafolder is None:
topoDataFolder = os.path.join(args.outputfolder, "inputs")
fileJSON, projectFolder = downloadUnzipTopo(
args.visitID, topoDataFolder)
# otherwise just pass in a path to existing data
else:
projectFolder = args.datafolder
tp = TopoProject(projectFolder)
funcargs = (tp.getpath("DEM"), tp.getpath("WaterSurfaceDEM"),
tp.getpath("Thalweg"), resultsFolder, args.verbose)
else:
funcargs = (args.dem.name, args.wsdem.name, args.thalweg.name,
args.outputfolder, args.verbose)
hydroPrep(*funcargs)
except (DataException, MissingException, NetworkException) as e:
# Exception class prints the relevant information
traceback.print_exc(file=sys.stdout)
sys.exit(e.returncode)
except AssertionError as e:
traceback.print_exc(file=sys.stdout)
sys.exit(1)
except Exception as e:
traceback.print_exc(file=sys.stdout)
sys.exit(1)
sys.exit(0)
def channelUnitScraper(outputDir, watersheds):
visitData = {}
for watershed, apiName in watersheds.iteritems():
visitData[watershed] = {}
loadVisitData(watershed, apiName, visitData[watershed])
for watershed, visits in visitData.iteritems():
outPath = os.path.join(outputDir, watershed.replace(' ', '') + ".shp")
outShape = None
featureID = 0
for visitID, visit in visits.iteritems():
if len(visit['ChannelUnits']) < 1:
continue
try:
dirpath = tempfile.mkdtemp()
# Open the visit channel unit shapefile.
# Need the spatial reference from one of the visits to create the output watershed shapefile
try:
fileJSON, projPath = downloadUnzipTopo(visitID, dirpath)
topo = TopoProject(projPath)
cuPath = topo.getpath('ChannelUnits')
except (DataException, MissingException), e:
print "Error retrieving channel units ShapeFile for visit {0}".format(
visitID)
continue
try:
shpCU = Shapefile(cuPath)
except Exception, e:
print "Error OGR opening channel unit ShapeFile for visit {0}".format(
visitID)
continue
if not outShape:
# Create new ShapeFile for this watershed
outShape = Shapefile()
outShape.create(outPath,
shpCU.spatialRef,
geoType=ogr.wkbPolygon)
outShape.createField("ID", ogr.OFTInteger)
outShape.createField("Watershed", ogr.OFTString)
outShape.createField("Site", ogr.OFTString)
outShape.createField("VisitID", ogr.OFTInteger)
outShape.createField("SampleYear", ogr.OFTInteger)
outShape.createField("Org", ogr.OFTString)
outShape.createField("UnitNumber", ogr.OFTInteger)
outShape.createField("UnitArea", ogr.OFTReal)
outShape.createField("Tier1", ogr.OFTString)
outShape.createField("Tier2", ogr.OFTString)
outShape.createField("AvgSiteWid", ogr.OFTReal)
outShape.createField("ReachLen", ogr.OFTReal)
# Loop over all channel unit polygons for this visit
feats = shpCU.featuresToShapely()
for aFeat in feats:
featureID += 1
cuNumber = aFeat['fields']['UnitNumber']
featureDefn = outShape.layer.GetLayerDefn()
outFeature = ogr.Feature(featureDefn)
outFeature.SetField('ID', featureID)
outFeature.SetField('Watershed', visit['Watershed'])
outFeature.SetField('Site', visit['Site'])
outFeature.SetField('VisitID', visitID)
outFeature.SetField('SampleYear', visit['SampleYear'])
outFeature.SetField('Org', visit['Organization'])
outFeature.SetField('AvgSiteWid',
visit['AverageSiteWidth'])
outFeature.SetField('ReachLen', visit['TotalReachLength'])
outFeature.SetField('UnitNumber', cuNumber)
outFeature.SetField(
'Tier1', visit['ChannelUnits'][cuNumber]['Tier1'])
outFeature.SetField(
'Tier2', visit['ChannelUnits'][cuNumber]['Tier2'])
outFeature.SetField('UnitArea', aFeat['geometry'].area)
outFeature.SetGeometry(
ogr.CreateGeometryFromJson(
json.dumps(mapping(aFeat['geometry']))))
outShape.layer.CreateFeature(outFeature)
finally:
def main():
# parse command line options
parser = argparse.ArgumentParser()
parser.add_argument('visitID',
help='the visit id of the site to use (no spaces)',
type=str)
parser.add_argument('outputfolder', help='Output folder', type=str)
parser.add_argument(
'substrate_values',
nargs='+',
help=
"one or more percentiles of grain size to calculate. 50 for D50, 84 for D84, etc",
type=int)
parser.add_argument('--out_channel_roughness_value',
help="i.e. 4000.0",
type=float,
default=4000.0)
parser.add_argument('--ocular_estimates',
help="(optional) local json file of ocular estimates")
parser.add_argument(
'--datafolder',
help=
'(optional) local folder containing TopoMetrics Riverscapes projects',
type=str)
parser.add_argument('--env',
"-e",
help="(optional) local env file",
type=str)
parser.add_argument('--verbose',
help='Get more information in your logs.',
action='store_true',
default=False)
args = parser.parse_args()
if not all([
args.visitID, args.outputfolder, args.substrate_values,
args.out_channel_roughness_value
]):
print "ERROR: Missing arguments"
parser.print_help()
if args.env:
setEnvFromFile(args.env)
# Make sure the output folder exists
resultsFolder = os.path.join(args.outputfolder, "outputs")
# Initiate the log file
logg = Logger("Program")
logfile = os.path.join(resultsFolder, "substrate_raster.log")
logg.setup(logPath=logfile, verbose=args.verbose)
# Fiona debug-level loggers can cause problems
logging.getLogger("Fiona").setLevel(logging.ERROR)
logging.getLogger("fiona").setLevel(logging.ERROR)
logging.getLogger("fiona.collection").setLevel(logging.ERROR)
logging.getLogger("shapely.geos").setLevel(logging.ERROR)
logging.getLogger("rasterio").setLevel(logging.ERROR)
try:
# Make some folders if we need to:
if not os.path.isdir(args.outputfolder):
os.makedirs(args.outputfolder)
if not os.path.isdir(resultsFolder):
os.makedirs(resultsFolder)
# If we need to go get our own topodata.zip file and unzip it we do this
if args.datafolder is None:
topoDataFolder = os.path.join(args.outputfolder, "inputs")
if not os.path.isdir(topoDataFolder):
os.makedirs(topoDataFolder)
fileJSON, projectFolder = downloadUnzipTopo(
args.visitID, topoDataFolder)
# otherwise just pass in a path to existing data
else:
projectFolder = args.datafolder
if args.ocular_estimates is None:
dict_ocular = APIGet(
"visits/{}/measurements/Substrate%20Cover".format(
str(args.visitID)))
dict_units = APIGet("visits/{}/measurements/Channel%20Unit".format(
str(args.visitID)))
dict_unitkey = {
x['value']['ChannelUnitID']: x['value']['ChannelUnitNumber']
for x in dict_units['values']
}
for i in range(len(dict_ocular['values'])):
dict_ocular['values'][i]['value'][
'ChannelUnitNumber'] = dict_unitkey[
dict_ocular['values'][i]['value']['ChannelUnitID']]
else:
dict_ocular = json.load(open(args.ocular_estimates, 'rt'))
dict_units = APIGet("visits/{}/measurements/Channel%20Unit".format(
str(args.visitID)))
dict_unitkey = {
x['value']['ChannelUnitID']: x['value']['ChannelUnitNumber']
for x in dict_units['values']
}
for i in range(len(dict_ocular['values'])):
dict_ocular['values'][i]['value'][
'ChannelUnitNumber'] = dict_unitkey[
dict_ocular['values'][i]['value']['ChannelUnitID']]
generate_substrate_raster(projectFolder, resultsFolder,
args.substrate_values, dict_ocular,
args.out_channel_roughness_value)
except (DataException, MissingException, NetworkException) as e:
# Exception class prints the relevant information
traceback.print_exc(file=sys.stdout)
sys.exit(e.returncode)
except AssertionError as e:
logg.error(e.message)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
except Exception as e:
logg.error(e.message)
traceback.print_exc(file=sys.stdout)
sys.exit(1)
sys.exit(0)