def test_siteUpstreamOnSegment(self):
streamGraph = StreamGraph()
node1 = streamGraph.addNode((0, 0))
node2 = streamGraph.addNode((0, -1))
segment = streamGraph.addSegment(node1, node2, "1", 1, 1, 1)
streamGraph.addSite("site1", "1", 0.1)
streamGraph.addSite("site2", "1", 0.9)
streamGraph.addSite("site3", "1", 1)
navigator = StreamGraphNavigator(streamGraph)
downstreamSite = navigator.getNextUpstreamSite(segment, 0.5)
foundSiteID = downstreamSite[0]
foundSiteDist = downstreamSite[1]
self.assertEqual(foundSiteID, "site1")
self.assertEqual(foundSiteDist, 0.4)
def test_findUpstreamSiteWithBacktrack(self):
streamGraph = StreamGraph()
node1 = streamGraph.addNode((0, 0))
node2 = streamGraph.addNode((0, -1))
node3 = streamGraph.addNode((1, 0))
segment1 = streamGraph.addSegment(node1, node2, "1", 1, 2,
1) #trib of segment2 path
segment2 = streamGraph.addSegment(node3, node2, "2", 1, 1, 1)
streamGraph.addSite("site1", "2", 0.2)
dataBoundary = DataBoundary(point=(0, 0), radius=10)
streamGraph.safeDataBoundary.append(dataBoundary)
navigator = StreamGraphNavigator(streamGraph)
downstreamSite = navigator.getNextUpstreamSite(segment1, 0.5)
foundSiteID = downstreamSite[0]
foundSiteDist = downstreamSite[1]
self.assertEqual(foundSiteID, "site1")
self.assertEqual(foundSiteDist, 1.3)
def getSiteID(self, useBadSites=True, logWarnings=False):
""" Get the siteID. Lat and Lng are provided to the constructor.
:param useBadSites: When false, any site that has warnings associated with it will be ignored in calculating the new ID.
:param logWarnings: Should this request log warnings into the SiteInfoCreator's WarningLog instance?
:return: A dict {"id": the_id, "story": the_story}"""
lat = self.lat
lng = self.lng
warningLog = self.warningLog
streamGraph = self.streamGraph
siteIDManager = self.siteIDManager
streamGraph.setAssignBadSitesStatus(useBadSites)
#typically lat/long are switched to fit the x/y order paradigm
point = (lng, lat)
story = ""
newID = ""
huc = ""
#create the json that gets resturned
def getResults(siteID="unknown",
story="See warning log",
failed=False):
results = dict()
results["id"] = Helpers.formatID(siteID)
snapLatFormatted = Helpers.getFloatTruncated(lat, 7)
snapLngFormatted = Helpers.getFloatTruncated(lng, 7)
storyHeader = "Requested site info at " + str(
snapLatFormatted) + ", " + str(snapLngFormatted) + ". "
useBadSitesStory = (
"" if useBadSites else
"ADONNIS ignored sites with incorrect ID's when calculating the new ID. "
)
results["story"] = storyHeader + useBadSitesStory + story
return results
if Failures.isFailureCode(self.baseData):
return getResults(failed=True)
#snap query point to a segment
snapablePoint = SnapablePoint(point=point, name="", id="")
snapInfo = snapPoint(snapablePoint, self.baseData,
snapCutoff=1) #get the most likely snap
if Failures.isFailureCode(snapInfo):
if __debug__:
print("could not snap")
if logWarnings:
warningLog.addWarning(WarningLog.HIGH_PRIORITY, snapInfo)
return getResults(failed=True)
feature = snapInfo[0].feature
segmentID = str(feature["properties"]["OBJECTID"])
distAlongSegment = snapInfo[0].distAlongFeature
#get the segment ID of the snapped segment
graphSegment = streamGraph.getCleanedSegment(segmentID)
snappedPoint = streamGraph.segments[segmentID].getPointOnSegment(
distAlongSegment)
if __debug__:
SnapSites.visualize(self.baseData, [])
streamGraph.visualize(customPoints=[snappedPoint],
showSegInfo=True)
streamGraph.visualize(customPoints=[snappedPoint],
showSegInfo=False)
#build a navigator object
#we want to terminate the search each time a query happens
#this allows us to stagger upstream and downstream searches
#although this means repeating parts of the search multiple times, searching a already constructed
#graph takes practically no time at all
navigator = StreamGraphNavigator(streamGraph,
terminateSearchOnQuery=True)
upstreamSite = None
downstreamSite = None
endOfUpstreamNetwork = False
endOfDownstreamNetwork = False
secondaryQueries = 0
primaryQueries = 0
#each iteration extends the graph by one query worth of data
# in this step we try to find an upstream and downstream site
while (
upstreamSite is None or downstreamSite is None
) and secondaryQueries < MAX_SECONDARY_SITE_QUERIES and primaryQueries < MAX_PRIMARY_QUERIES and (
endOfUpstreamNetwork is False
or endOfDownstreamNetwork is False):
if upstreamSite is None and endOfUpstreamNetwork is False:
#we haven't found upstream yet
upstreamReturn = navigator.getNextUpstreamSite(
graphSegment, distAlongSegment)
if upstreamReturn == Failures.END_OF_BASIN_CODE:
endOfUpstreamNetwork = True
if Failures.isFailureCode(
upstreamReturn
) is not True and upstreamReturn is not None:
upstreamSite = upstreamReturn
if downstreamSite is None and endOfDownstreamNetwork is False:
#we haven't found downstream yet
downstreamReturn = navigator.getNextDownstreamSite(
graphSegment, distAlongSegment)
if downstreamReturn == Failures.END_OF_BASIN_CODE:
endOfDownstreamNetwork = True
if Failures.isFailureCode(
downstreamReturn
) is not True and downstreamReturn is not None:
downstreamSite = downstreamReturn
if upstreamSite is not None or downstreamSite is not None:
#we've found at least one site
secondaryQueries += 1
else:
primaryQueries += 1
#add warnings from found sites, collect HUC
if upstreamSite is not None:
siteAssignment = upstreamSite[0]
if logWarnings:
for warning in siteAssignment.generalWarnings:
warningLog.addWarningTuple(warning)
for warning in siteAssignment.assignmentWarnings:
warningLog.addWarningTuple(warning)
huc = siteAssignment.huc
if downstreamSite is not None:
siteAssignment = downstreamSite[0]
if logWarnings:
for warning in siteAssignment.generalWarnings:
warningLog.addWarningTuple(warning)
for warning in siteAssignment.assignmentWarnings:
warningLog.addWarningTuple(warning)
huc = siteAssignment.huc
if logWarnings:
for warning in streamGraph.currentAssignmentWarnings: #apply warnings from streamGraph
warningLog.addWarningTuple(warning)
#handle all combinations of having an upstream site and/or a downstream site (also having neither)
#~~~~~~~~~~~~~~~~~~~UPSTREAM AND DOWNSTREAM SITES FOUND CASE~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
if upstreamSite is not None and downstreamSite is not None:
#we have an upstream and downstream
upstreamSiteID = upstreamSite[0].siteID
downstreamSiteID = downstreamSite[0].siteID
partCode = upstreamSiteID[0:2]
if Helpers.siteIDCompare(downstreamSiteID, upstreamSiteID) < 0:
message = "The found upstream site is larger than found downstream site. ADONNIS output almost certainly incorrect."
if logWarnings:
warningLog.addWarning(WarningLog.HIGH_PRIORITY, message)
fullUpstreamSiteID = Helpers.getFullID(upstreamSiteID)
fullDownstreamSiteID = Helpers.getFullID(downstreamSiteID)
upstreamSiteIdDsnStr = fullUpstreamSiteID[2:]
downstreamSiteIdDsnStr = fullDownstreamSiteID[2:]
#get the downstream number portion of the ID
upstreamSiteIdDsn = int(upstreamSiteIdDsnStr)
downstreamSiteIdDsn = int(downstreamSiteIdDsnStr)
totalAddressSpaceDistance = upstreamSite[1] + downstreamSite[1]
newSitePercentage = downstreamSite[1] / totalAddressSpaceDistance
newDon = int(downstreamSiteIdDsn * (1 - newSitePercentage) +
upstreamSiteIdDsn * newSitePercentage)
newID = Helpers.buildFullID(partCode, newDon)
newID = self.beautifyID(newID,
downstreamSiteID,
upstreamSiteID,
logWarnings=logWarnings)
story = "Found a upstream site " + Helpers.formatID(
upstreamSiteID) + " and a downstream site " + Helpers.formatID(
downstreamSiteID
) + ". New site is the weighted average of these two sites."
if __debug__:
print("found upstream is " + upstreamSiteID)
print("found downstream is " + downstreamSiteID)
streamGraph.visualize(customPoints=[snappedPoint])
SnapSites.visualize(self.baseData, [])
#~~~~~~~~~~~~~~~~~~~UPSTREAM SITE FOUND ONLY CASE~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
elif upstreamSite is not None:
upstreamSiteID = upstreamSite[0].siteID
partCode = upstreamSiteID[:2]
fullUpstreamID = Helpers.getFullID(upstreamSiteID)
foundSiteNeighbors = siteIDManager.getNeighborIDs(
upstreamSiteID, huc)
if Failures.isFailureCode(foundSiteNeighbors):
nextSequentialDownstreamSite = None
else:
nextSequentialDownstreamSite = foundSiteNeighbors[1]
upstreamSiteDSN = int(fullUpstreamID[2:])
upstreamSiteDistance = upstreamSite[1]
#calculate offset. If we have a sequential downstream use that as a bound
siteIDOffset = math.ceil(upstreamSiteDistance / MIN_SITE_DISTANCE)
if nextSequentialDownstreamSite is not None:
#if we have the sequential downstream bound, don't let the new site get added any closer than halfway between
siteIDOffset = min(
siteIDOffset,
Helpers.getSiteIDOffset(upstreamSiteID,
nextSequentialDownstreamSite) / 2)
newSiteIDDSN = upstreamSiteDSN + siteIDOffset
#allowed wiggle room in the new site. Depending on how much distance is between the found site
#we allow for a larger range in the final ID. Has to be at least 10% within the rule of min_site_distance
#at most 5 digits up or down. At least, 0
allowedError = math.floor(max(1, min(siteIDOffset / 10, 5)))
upperBound = Helpers.buildFullID(
partCode, upstreamSiteDSN + siteIDOffset + allowedError)
lowerBound = Helpers.buildFullID(
partCode, upstreamSiteDSN + siteIDOffset - allowedError)
newID = Helpers.buildFullID(partCode, newSiteIDDSN)
newID = self.beautifyID(newID,
lowerBound,
upperBound,
logWarnings=logWarnings)
offsetAfterBeautify = Helpers.getSiteIDOffset(
newID, fullUpstreamID)
if nextSequentialDownstreamSite is None:
story = "Only found a upstream site (" + upstreamSiteID + "). New site ID is based on upstream site while allowing space for " + str(
offsetAfterBeautify
) + " sites between upstream site and new site"
if logWarnings:
warningLog.addWarning(
WarningLog.HIGH_PRIORITY,
"No downstream bound on result. Needs verification!")
else:
story = "Found an upstream site " + Helpers.formatID(
upstreamSiteID
) + ". Based on list of all sites, assume that " + Helpers.formatID(
nextSequentialDownstreamSite
) + " is the nearest sequential downstream site. New ID is based on the upstream site and bounded by the sequential downstream site"
if logWarnings:
warningLog.addWarning(
WarningLog.LOW_PRIORITY,
"Found upstream and downstream bound. But, downstream bound is based on list of sequential sites and may not be the true downstream bound. This could result in site ID clustering."
)
if __debug__:
print("found upstream, but not downstream")
print("upstream siteID is " + str(upstreamSiteID))
streamGraph.visualize(customPoints=[snappedPoint])
SnapSites.visualize(self.baseData, [])
#~~~~~~~~~~~~~~~~~~~DOWNSTREAM SITE ONLY CASE~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
elif downstreamSite is not None:
downstreamSiteID = downstreamSite[0].siteID
partCode = downstreamSiteID[:2]
fullDownstreamID = Helpers.getFullID(downstreamSiteID)
foundSiteNeighbors = siteIDManager.getNeighborIDs(
downstreamSiteID, huc)
if Failures.isFailureCode(foundSiteNeighbors):
nextSequentialUpstreamSite = None
else:
nextSequentialUpstreamSite = foundSiteNeighbors[0]
downstreamSiteDSN = int(fullDownstreamID[2:])
downstreamSiteDistance = downstreamSite[1]
siteIDOffset = math.ceil(downstreamSiteDistance /
MIN_SITE_DISTANCE)
if nextSequentialUpstreamSite is not None:
#if we have the sequential upstream bound, don't let the new site get added any closer than halfway between
siteIDOffset = min(
siteIDOffset,
Helpers.getSiteIDOffset(downstreamSiteID,
nextSequentialUpstreamSite) / 2)
newSiteIDDSN = downstreamSiteDSN - siteIDOffset
allowedError = math.floor(max(1, min(siteIDOffset / 10, 5)))
upperBound = Helpers.buildFullID(
partCode, downstreamSiteDSN - siteIDOffset + allowedError)
lowerBound = Helpers.buildFullID(
partCode, downstreamSiteDSN - siteIDOffset - allowedError)
newID = Helpers.buildFullID(partCode, newSiteIDDSN)
newID = self.beautifyID(newID,
lowerBound,
upperBound,
logWarnings=logWarnings)
offsetAfterBeautify = Helpers.getSiteIDOffset(
newID, fullDownstreamID)
if nextSequentialUpstreamSite is None:
story = "Only found a downstream site " + Helpers.formatID(
downstreamSiteID
) + ". New site ID is based on downstream site while allowing space for " + str(
offsetAfterBeautify
) + " sites between downstream site and new site"
if logWarnings:
warningLog.addWarning(
WarningLog.HIGH_PRIORITY,
"No upstream bound on result. Needs verification!")
else:
story = "Found a downstream site " + Helpers.formatID(
downstreamSiteID
) + ". Based on list of all sites, assume that " + Helpers.formatID(
nextSequentialUpstreamSite
) + " is the nearest sequential upstream site. New ID is based on the downstream site and bounded by the sequential upstream site"
if logWarnings:
warningLog.addWarning(
WarningLog.LOW_PRIORITY,
"Found upstream and downstream bound. But, upstream bound is based on list of sequential sites and may not be the true upstream bound. This could result in site ID clustering."
)
if __debug__:
print("found downstream, but not upstream")
print("downstream siteID is " + str(downstreamSiteID))
streamGraph.visualize(customPoints=[snappedPoint])
SnapSites.visualize(self.baseData, [])
#~~~~~~~~~~~~~~~~~~~NO SITES FOUND CASE~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
else:
# get huge radius of sites:
sitesInfo = GDALData.loadSitesFromQuery(lat, lng, 30)
if Failures.isFailureCode(sitesInfo):
if logWarnings:
warningLog.addWarning(WarningLog.HIGH_PRIORITY, sitesInfo)
return getResults(failed=True)
sites = []
for site in sitesInfo:
siteNumber = site["properties"]["site_no"]
siteHUC = site["properties"]["huc_cd"]
sitePoint = site["geometry"]["coordinates"]
fastDistance = Helpers.fastMagDist(sitePoint[0], sitePoint[1],
point[0], point[1])
sites.append((siteNumber, sitePoint, fastDistance, siteHUC))
sortedSites = sorted(sites, key=lambda site: site[2])
huc = sortedSites[0][3]
oppositePairA = None
oppositePairB = None
foundOppositePair = False
i = 1
while foundOppositePair is False:
curSite = sortedSites[i]
curPartCode = curSite[0][:2]
curSitePoint = curSite[1]
curDirection = Helpers.normalize(curSitePoint[0] - point[0],
curSitePoint[1] - point[1])
for cmpSite in sortedSites[:i]:
cmpSitePoint = cmpSite[1]
cmpDirection = Helpers.normalize(
cmpSitePoint[0] - point[0], cmpSitePoint[1] - point[1])
cmpPartCode = cmpSite[0][:2]
dot = Helpers.dot(curDirection[0], curDirection[1],
cmpDirection[0], cmpDirection[1])
#check if these two directions are mostly opposite
# dot < 0 means they are at least perpendicular
if dot < 0.4 and curPartCode == cmpPartCode:
foundOppositePair = True
oppositePairA = cmpSite
oppositePairB = curSite
i += 1
partCode = oppositePairA[0][:2]
fullIDA = Helpers.getFullID(oppositePairA[0])
fullIDB = Helpers.getFullID(oppositePairB[0])
dsnA = int(fullIDA[2:])
dsnB = int(fullIDB[2:])
distA = oppositePairA[2]
distB = oppositePairB[2]
totalAddressSpaceDistance = distA + distB
newSitePercentage = distA / totalAddressSpaceDistance
newDsn = int(dsnA * (1 - newSitePercentage) +
dsnB * newSitePercentage)
newID = Helpers.buildFullID(partCode, newDsn)
newID = self.beautifyID(newID,
fullIDA,
fullIDB,
logWarnings=logWarnings)
story = "Could not find any sites on the network. Estimating based on " + Helpers.formatID(
oppositePairA[0]) + " and " + Helpers.formatID(
oppositePairB[0]) + "."
if __debug__:
print(
"no sites found nearby. Estimating new ID based on nearby sites"
)
print("new estimate based on " + oppositePairA[0] + " and " +
oppositePairB[0])
print("estimation is " + newID)
streamGraph.visualize()
SnapSites.visualize(self.baseData, [])
return getResults(siteID=newID, story=story)