def test_logger(self):
outputFolder = self.dir + '%src%test%data%outputFolder'.replace("%", os.sep)
Logger.configureLogger(outputFolder=outputFolder,
prefix="prefix")
Logger.getInstance().info("MY LOG")
def createCostSummaryWithAdditionalProperties(self, costAttribute,
startPointFeature,
endPointFeature, costSummaryMap):
startVertexID = startPointFeature["properties"]["vertex_id"]
endVertexID = endPointFeature["properties"]["vertex_id"]
# if startVertexID == endVertexID:
# return None
if (startVertexID
not in costSummaryMap) or (endVertexID
not in costSummaryMap[startVertexID]):
Logger.getInstance().warning(
"Not contained into the costSummaryMap: %s %s" %
(startVertexID, endVertexID))
return None
summaryFeature = costSummaryMap[startVertexID][endVertexID]
summaryFeature = copy.deepcopy(summaryFeature)
total_cost = summaryFeature["properties"]["total_cost"]
del summaryFeature["properties"]["start_vertex_id"]
del summaryFeature["properties"]["end_vertex_id"]
del summaryFeature["properties"]["total_cost"]
# pointIdentifierKey = getConfigurationProperties(section="WFS_CONFIG")["point_identifier"]
# startPointId = startPointFeature["properties"][pointIdentifierKey]
# endPointId = endPointFeature["properties"][pointIdentifierKey]
# newPropertiesId = startPointId + "-" + endPointId
# if newPropertiesId in self.additionalFeaturePropertiesCache:
# newProperties = self.additionalFeaturePropertiesCache[newPropertiesId]
# else:
newProperties = self.insertAdditionalProperties(startPointFeature,
endPointFeature)
# self.additionalFeaturePropertiesCache[newPropertiesId] = newProperties
# coordinates = summaryFeature["geometry"]["coordinates"]
# coordinates[0] = startPointFeature["properties"]["selectedPointCoordinates"]
# coordinates[1] = endPointFeature["properties"]["selectedPointCoordinates"]
newProperties["costAttribute"] = getEnglishMeaning(costAttribute)
newProperties[getEnglishMeaning(costAttribute)] = total_cost
newProperties["startVertexId"] = startVertexID
newProperties["endVertexId"] = endVertexID
newProperties["selectedStartCoordinates"] = startPointFeature[
"properties"]["selectedPointCoordinates"]
newProperties["selectedEndCoordinates"] = endPointFeature["properties"][
"selectedPointCoordinates"]
newProperties["nearestStartCoordinates"] = startPointFeature["properties"][
"nearestVertexCoordinates"]
newProperties["nearestEndCoordinates"] = endPointFeature["properties"][
"nearestVertexCoordinates"]
for key in newProperties:
summaryFeature["properties"][key] = newProperties[key]
return summaryFeature
def getTotalShortestPathCostOneToMany(self,
startVertexID=None,
endVerticesID=[],
costAttribute=None):
"""
Using the power of pgr_Dijsktra algorithm this function calculate the total routing cost from a set of point to a single point.
:param startVertexID: Initial vertexes to calculate the shortest path.
:param endVerticesID: Set of ending vertexes to calculate the shortest path.
:param costAttribute: Impedance/cost to measure the weight of the route.
:return: Shortest path summary json.
"""
Logger.getInstance().info("Start getTotalShortestPathCostOneToMany")
sql = "SELECT " \
"s.id AS start_vertex_id," \
"e.id AS end_vertex_id," \
"r.agg_cost as total_cost," \
"ST_MakeLine(s.the_geom, e.the_geom) AS geom " \
"FROM(" \
"SELECT * " \
"FROM pgr_dijkstraCost(" \
"\'SELECT " \
"id::integer," \
"source::integer," \
"target::integer," \
"(CASE " \
"WHEN luokka <> 0 AND (liikennevi = 0 OR liikennevi = 2 OR liikennevi = 5 OR liikennevi = 4) " \
"THEN %s " \
"ELSE -1 " \
"END)::double precision AS cost," \
"(CASE " \
"WHEN luokka <> 0 AND (liikennevi = 0 OR liikennevi = 2 OR liikennevi = 5 OR liikennevi = 3) " \
"THEN %s " \
"ELSE -1 " \
"END)::double precision AS reverse_cost " \
"FROM table_name', %s, ARRAY[%s], true)) as r," \
"table_name_vertices_pgr AS s," \
"table_name_vertices_pgr AS e " \
"WHERE " \
"s.id = r.start_vid " \
"and e.id = r.end_vid ".replace("table_name", self.tableName) \
% (costAttribute, costAttribute, startVertexID, ",".join(map(str, endVerticesID)))
# "GROUP BY " \
# "s.id, e.id, r.agg_cost" \
geojson = self.serviceProvider.execute(sql)
Logger.getInstance().info("End getTotalShortestPathCostOneToMany")
return geojson
def calculateTravelTimeFromDataframe(self, travelTimeSummaryDF):
Logger.getInstance().info("Dataframe length=%s" %
len(travelTimeSummaryDF))
if len(travelTimeSummaryDF["costAttribute"]) > 0:
costAttribute = travelTimeSummaryDF["costAttribute"][0]
travelTimeSummaryDF["total_travel_time"] = travelTimeSummaryDF.startPoint_EuclideanDistanceWalkingTime + \
travelTimeSummaryDF.startPoint_AVGWalkingDistanceWalkingTime + \
travelTimeSummaryDF[costAttribute] + \
travelTimeSummaryDF.endPoint_ParkingTime + \
travelTimeSummaryDF.endPoint_AVGWalkingDistanceWalkingTime + \
travelTimeSummaryDF.endPoint_EuclideanDistanceWalkingTime
return travelTimeSummaryDF
def calculateSmallSummary(self, shortestPath, costAttribute):
"""
Given a Geojson (Geometry type: LineString) files, read all the files from the given ``folderPath`` and
sum cost attribute and distance values (distance, and any of: speed_limit_time, day_avg_delay_time, midday_delay_time and
rush_hour_delay_time) and return the sum up of those values.
:param shortestPath: Shortest path geojson features.
:param costAttribute: The sum up will be based on the given cost impedance value.
:return: start point centroid id, end point centroid id, total distance and total travel time.
"""
Logger.getInstance().info("Start calculateSmallSummary for: %s" %
costAttribute)
pointIdentifierKey = getConfigurationProperties(
section="WFS_CONFIG")["point_identifier"]
startPointId = shortestPath["overallProperties"]["startPoint_" +
pointIdentifierKey]
endPointId = shortestPath["overallProperties"]["endPoint_" +
pointIdentifierKey]
travelTime = 0.0
distance = 0.0
for segmentFeature in shortestPath["features"]:
for key in segmentFeature["properties"]:
if costAttribute == key:
travelTime += segmentFeature["properties"][costAttribute]
if getEnglishMeaning(CostAttributes.DISTANCE) == key:
distance += segmentFeature["properties"][getEnglishMeaning(
CostAttributes.DISTANCE)]
totalDistance = shortestPath["overallProperties"]["startPoint_" + PostfixAttribute.EUCLIDEAN_DISTANCE] + \
shortestPath["overallProperties"]["startPoint_" + PostfixAttribute.AVG_WALKING_DISTANCE] + \
distance + \
shortestPath["overallProperties"]["endPoint_" + PostfixAttribute.AVG_WALKING_DISTANCE] + \
shortestPath["overallProperties"]["endPoint_" + PostfixAttribute.EUCLIDEAN_DISTANCE]
totalTravelTime = shortestPath["overallProperties"]["startPoint_" + PostfixAttribute.EUCLIDEAN_DISTANCE + PostfixAttribute.WALKING_TIME] + \
shortestPath["overallProperties"]["startPoint_" + PostfixAttribute.AVG_WALKING_DISTANCE + PostfixAttribute.WALKING_TIME] + \
travelTime + \
shortestPath["overallProperties"]["endPoint_" + PostfixAttribute.PARKING_TIME] + \
shortestPath["overallProperties"]["endPoint_" + PostfixAttribute.AVG_WALKING_DISTANCE + PostfixAttribute.WALKING_TIME] + \
shortestPath["overallProperties"]["endPoint_" + PostfixAttribute.EUCLIDEAN_DISTANCE + PostfixAttribute.WALKING_TIME]
return startPointId, endPointId, totalDistance, totalTravelTime
def getTotalShortestPathCostOneToOne(self, startVertexID, endVertexID,
costAttribute):
"""
Using the power of pgr_Dijsktra algorithm this function calculate the total routing cost for a pair of points.
:param startVertexID: Initial Vertex to calculate the shortest path.
:param endVertexID: Last Vertex to calculate the shortest path.
:param costAttribute: Impedance/cost to measure the weight of the route.
:return: Shortest path summary json.
"""
Logger.getInstance().info("Start getTotalShortestPathCostOneToOne")
sql = "SELECT " \
"s.id AS start_vertex_id," \
"e.id AS end_vertex_id," \
"r.agg_cost as total_cost," \
"ST_MakeLine(s.the_geom, e.the_geom) AS geom " \
"FROM(" \
"SELECT * " \
"FROM pgr_dijkstraCost(" \
"\'SELECT id::integer, source::integer, target::integer, " \
"(CASE " \
"WHEN (oneway = 1 OR oneway = 0) " \
"THEN %s " \
"ELSE -1 " \
"END)::double precision AS cost, " \
"(CASE " \
"WHEN (oneway = 0) " \
"THEN %s " \
"ELSE -1 " \
"END)::double precision AS reverse_cost " \
"FROM table_name', %s, %s, true)) as r," \
"table_name_vertices_pgr AS s," \
"table_name_vertices_pgr AS e " \
"WHERE " \
"s.id = r.start_vid " \
"and e.id = r.end_vid ".replace("table_name", self.tableName) \
% (costAttribute, costAttribute, startVertexID, endVertexID)
# "GROUP BY " \
# "s.id, e.id, r.agg_cost" \
geojson = self.serviceProvider.execute(sql)
Logger.getInstance().info("End getTotalShortestPathCostOneToOne")
return geojson
def execute(self, url):
"""
Request a JSON from an URL.
:param url: URL.
:return: Downloaded Json.
"""
startTime = time.time()
Logger.getInstance().info("requestFeatures Start Time: %s" %
getFormattedDatetime(timemilis=startTime))
u = openURL(url)
geojson = json.loads(u.read().decode('utf-8'))
endTime = time.time()
Logger.getInstance().info("requestFeatures End Time: %s" %
getFormattedDatetime(timemilis=endTime))
totalTime = timeDifference(startTime, endTime)
Logger.getInstance().info("requestFeatures Total Time: %s m" %
totalTime)
return geojson
import traceback
import sys
sys.path.append('/dgl/codes/DORA/')
import src.main.doraInit as dora
from src.main.util import Logger
try:
dora.main()
except Exception as err:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
Logger.getInstance().exception(''.join('>> ' + line for line in lines))
def createGeneralSummary(self, startCoordinatesGeojsonFilename,
endCoordinatesGeojsonFilename, costAttribute,
outputFolderPath, outputFilename):
"""
Using the power of pgr_Dijsktra algorithm this function calculate the total routing cost for a pair of set of points.
It differentiate if must to use one-to-one, one-to-many, many-to-one or many-to-many specific stored procedures from the pgrouting extension.
:param startCoordinatesGeojsonFilename: Geojson file (Geometry type: MultiPoint) containing pair of points.
:param outputFolderPath: URL to store the shortest path geojson features of each pair of points.
:param costAttribute: Attribute to calculate the impedance of the Shortest Path algorithm.
:param outputFolderPath: Folder containing the shortest path geojson features.
:param outputFilename: Filename to give to the summary file.
:return: None. Store the information in the ``outputFolderPath``.
"""
Logger.getInstance().info("Start createGeneralSummary for: %s" %
costAttribute)
Logger.getInstance().info("Start merge additional layers")
inputStartCoordinates = self.operations.mergeAdditionalLayers(
originalJsonURL=startCoordinatesGeojsonFilename,
outputFolderPath=outputFolderPath)
inputEndCoordinates = self.operations.mergeAdditionalLayers(
originalJsonURL=endCoordinatesGeojsonFilename,
outputFolderPath=outputFolderPath)
Logger.getInstance().info("End merge additional layers")
Logger.getInstance().info("Start nearest vertices finding")
epsgCode = self.operations.extractCRSWithGeopandas(
startCoordinatesGeojsonFilename)
endEpsgCode = self.operations.extractCRSWithGeopandas(
endCoordinatesGeojsonFilename)
startVerticesID, startPointsFeaturesList = self.getVerticesID(
inputStartCoordinates, epsgCode)
endVerticesID, endPointsFeaturesList = self.getVerticesID(
inputEndCoordinates, endEpsgCode)
Logger.getInstance().info("End nearest vertices finding")
totals = None
Logger.getInstance().info("Start cost summary calculation")
if len(startVerticesID) == 1 and len(endVerticesID) == 1:
totals = self.transportMode.getTotalShortestPathCostOneToOne(
startVertexID=startVerticesID[0],
endVertexID=endVerticesID[0],
costAttribute=costAttribute)
elif len(startVerticesID) == 1 and len(endVerticesID) > 1:
totals = self.transportMode.getTotalShortestPathCostOneToMany(
startVertexID=startVerticesID[0],
endVerticesID=endVerticesID,
costAttribute=costAttribute)
elif len(startVerticesID) > 1 and len(endVerticesID) == 1:
totals = self.transportMode.getTotalShortestPathCostManyToOne(
startVerticesID=startVerticesID,
endVertexID=endVerticesID[0],
costAttribute=costAttribute)
elif len(startVerticesID) > 1 and len(endVerticesID) > 1:
totals = self.transportMode.getTotalShortestPathCostManyToMany(
startVerticesID=startVerticesID,
endVerticesID=endVerticesID,
costAttribute=costAttribute)
Logger.getInstance().info("End cost summary calculation")
costSummaryMap = self.createCostSummaryMap(totals)
# summaryFeature = costSummaryMap[startVertexID][endVertexID]
# KeyError: 125736
counterStartPoints = 0
counterEndPoints = 0
################################################################################################################
# for featureShortPathSummary in totals["features"]:
# startVertexID = featureShortPathSummary["properties"]["start_vertex_id"]
# endVertexID = featureShortPathSummary["properties"]["end_vertex_id"]
# total_cost = featureShortPathSummary["properties"]["total_cost"]
# del featureShortPathSummary["properties"]["start_vertex_id"]
# del featureShortPathSummary["properties"]["end_vertex_id"]
# del featureShortPathSummary["properties"]["total_cost"]
#
# startPointFeature = startPointsFeaturesList[counterStartPoints]
# endPointFeature = endPointsFeaturesList[counterEndPoints]
#
# self.createCostSummaryWithAdditionalProperties(costAttribute, endPointFeature, endVertexID,
# featureShortPathSummary, startPointFeature,
# startVertexID,
# total_cost)
# counterEndPoints += 1
# if counterEndPoints == len(endPointsFeaturesList):
# counterStartPoints += 1
# counterEndPoints = 0
################################################################################################################
features = []
################################################################################################################
# for startPointFeature in startPointsFeaturesList:
# for endPointFeature in endPointsFeaturesList:
# newFeature = createCostSummaryWithAdditionalProperties(costAttribute,
# startPointFeature,
# endPointFeature,
# costSummaryMap)
# if newFeature:
# features.append(newFeature)
################################################################################################################
Logger.getInstance().info(
"Start createCostSummaryWithAdditionalProperties")
with Parallel(
n_jobs=int(
getConfigurationProperties(
section="PARALLELIZATION")["jobs"]),
backend="threading",
verbose=int(
getConfigurationProperties(
section="PARALLELIZATION")["verbose"])) as parallel:
# while len(verticesID) <= len(geojson["features"]):
parallel._print = parallel_job_print
returns = parallel(
delayed(createCostSummaryWithAdditionalProperties)(
self, costAttribute, copy.deepcopy(startPointFeature),
copy.deepcopy(endPointFeature), costSummaryMap)
for startPointFeature in startPointsFeaturesList
for endPointFeature in endPointsFeaturesList)
for newFeature in returns:
if newFeature:
features.append(newFeature)
# print(returns)
Logger.getInstance().info(
"End createCostSummaryWithAdditionalProperties")
################################################################################################################
totals["features"] = features
if not outputFolderPath.endswith(os.sep):
summaryFolderPath = outputFolderPath + os.sep + "summary" + os.sep
else:
summaryFolderPath = outputFolderPath + "summary" + os.sep
pointIdentifierKey = getConfigurationProperties(
section="WFS_CONFIG")["point_identifier"]
columns = {
"startPoint_" + pointIdentifierKey: "ykr_from_id",
"endPoint_" + pointIdentifierKey: "ykr_to_id",
"total_travel_time": "travel_time"
}
filepath = self.fileActions.writeFile(folderPath=summaryFolderPath,
filename=outputFilename +
".geojson",
data=totals)
del totals
dataframeSummary = self.operations.calculateTravelTimeFromGeojsonFile(
travelTimeSummaryURL=filepath)
dataframeSummary = self.operations.renameColumnsAndExtractSubSet(
travelTimeMatrix=dataframeSummary, columns=columns)
outputFilename = getEnglishMeaning(
costAttribute) + "_" + outputFilename
csv_separator = getConfigurationProperties(
section="WFS_CONFIG")["csv_separator"]
csv_path = os.path.join(summaryFolderPath, outputFilename + ".csv")
if not os.path.exists(summaryFolderPath):
os.makedirs(summaryFolderPath)
dataframeSummary.to_csv(csv_path, sep=csv_separator, index=False)
self.fileActions.compressOutputFile(folderPath=summaryFolderPath,
zip_filename="summary.zip",
filepath=filepath)
self.fileActions.compressOutputFile(folderPath=summaryFolderPath,
zip_filename="summary_csv.zip",
filepath=csv_path)
debug = False
if "debug" in getConfigurationProperties(section="WFS_CONFIG"):
debug = "True".__eq__(
getConfigurationProperties(section="WFS_CONFIG")["debug"])
if not debug:
self.fileActions.deleteFile(folderPath=summaryFolderPath,
filename=outputFilename + ".geojson")
self.fileActions.deleteFile(folderPath=summaryFolderPath,
filename=outputFilename + ".csv")
def createDetailedSummary(self, folderPath, costAttribute, outputFilename):
"""
Given a set of Geojson (Geometry type: LineString) files, read all the files from the given ``folderPath`` and
sum all the attribute values (distance, speed_limit_time, day_avg_delay_time, midday_delay_time and
rush_hour_delay_time) and create a simple features Geojson (Geometry type: LineString)
with the summary information.
:param folderPath: Folder containing the shortest path geojson features.
:param outputFilename: Filename to give to the summary file.
:return: None. Store the summary information in the folderPath with the name given in outputFilename.
"""
Logger.getInstance().info("Start createDetailedSummary for: %s" %
costAttribute)
if not folderPath.endswith(os.sep):
attributeFolderPath = folderPath + os.sep + "geoms" + os.sep + getEnglishMeaning(
costAttribute) + os.sep
summaryFolderPath = folderPath + os.sep + "summary" + os.sep
else:
attributeFolderPath = folderPath + "geoms" + os.sep + getEnglishMeaning(
costAttribute) + os.sep
summaryFolderPath = folderPath + "summary" + os.sep
totals = {
"features": [],
"totalFeatures": 0,
"type": "FeatureCollection"
}
for file in os.listdir(attributeFolderPath):
if file.endswith(
".geojson"
) and file != "metroAccessDigiroadSummary.geojson":
filemetadata = file.split("-")
if len(filemetadata) < 2:
Logger.getInstance().info(filemetadata)
shortestPath = self.fileActions.readJson(
url=attributeFolderPath + file)
if "crs" not in totals:
totals["crs"] = shortestPath["crs"]
newSummaryFeature = {
"geometry": {
"coordinates": [],
"type": GeometryType.LINE_STRING
},
"properties": {
# "startVertexId": int(filemetadata[2]),
# "endVertexId": int(filemetadata[3].replace(".geojson", "")),
"costAttribute": filemetadata[1]
}
}
for property in shortestPath["overallProperties"]:
newSummaryFeature["properties"][property] = shortestPath[
"overallProperties"][property]
startPoints = None
endPoints = None
lastSequence = 1
for segmentFeature in shortestPath["features"]:
for key in segmentFeature["properties"]:
if key == "seq":
if segmentFeature["properties"][key] == 1:
# Sequence one is the first linestring geometry in the path
startPoints = segmentFeature["geometry"][
"coordinates"]
if segmentFeature["properties"][key] > lastSequence:
# The last sequence is the last linestring geometry in the path
endPoints = segmentFeature["geometry"][
"coordinates"]
lastSequence = segmentFeature["properties"][
key]
if key not in ["id", "direction", "seq"]:
if key not in newSummaryFeature["properties"]:
newSummaryFeature["properties"][key] = 0
newSummaryFeature["properties"][key] = newSummaryFeature["properties"][key] + \
segmentFeature["properties"][key]
try:
newSummaryFeature["geometry"]["coordinates"] = newSummaryFeature["geometry"]["coordinates"] + \
startPoints
startAndEndPointAreDifferent = lastSequence > 1
if startAndEndPointAreDifferent:
newSummaryFeature["geometry"]["coordinates"] = newSummaryFeature["geometry"]["coordinates"] + \
endPoints
totals["features"].append(newSummaryFeature)
except Exception as err:
Logger.getInstance().exception(err)
raise err
totals["totalFeatures"] = len(totals["features"])
outputFilename = getEnglishMeaning(
costAttribute) + "_" + outputFilename
self.fileActions.writeFile(folderPath=summaryFolderPath,
filename=outputFilename,
data=totals)
def main():
"""
Read the arguments written in the command line to read the input coordinates from a
Geojson file (a set of pair points) and the location (URL) to store the Shortest Path geojson features for each
pair of points.
Call the ``calculateTotalTimeTravel`` from the WFSServiceProvider configured
with the parameters in './resources/configuration.properties' and calculate the shortest path for each
pair of points and store a Geojson file per each of them.
After that, call the function ``createSummary`` to summarize the total time expend to go from one point to another
for each of the different impedance attribute (cost).
:return: None. All the information is stored in the ``shortestPathOutput`` URL.
"""
argv = sys.argv[1:]
opts, args = getopt.getopt(argv, "s:e:o:c:t:", [
"start_point=", "end_point=", "outputFolder=", "costAttributes=",
"transportMode", "is_entry_list", "routes", "summary", "all", "help"
])
startPointsGeojsonFilename = None
outputFolder = None
# impedance = CostAttributes.DISTANCE
# impedance = None
impedanceList = []
car_impedances = {
"DISTANCE": CostAttributes.DISTANCE,
"SPEED_LIMIT_TIME": CostAttributes.SPEED_LIMIT_TIME,
"DAY_AVG_DELAY_TIME": CostAttributes.DAY_AVG_DELAY_TIME,
"MIDDAY_DELAY_TIME": CostAttributes.MIDDAY_DELAY_TIME,
"RUSH_HOUR_DELAY": CostAttributes.RUSH_HOUR_DELAY
}
bicycle_impedances = {
"DISTANCE": CostAttributes.DISTANCE,
"BICYCLE_FAST_TIME": CostAttributes.BICYCLE_FAST_TIME,
"BICYCLE_SLOW_TIME": CostAttributes.BICYCLE_SLOW_TIME
}
allImpedanceAttribute = False
summaryOnly = False
routesOnly = False
isEntryList = False
impedanceErrorMessage = "Use the paramenter -c or --cost.\nValues allowed: DISTANCE, SPEED_LIMIT_TIME, DAY_AVG_DELAY_TIME, MIDDAY_DELAY_TIME, RUSH_HOUR_DELAY.\nThe parameter --all enable the analysis for all the impedance attributes."
transportModeErrorMessage = "Use the paramenter -t or --transportMode.\nValues allowed: PRIVATE_CAR, BICYCLE."
for opt, arg in opts:
if opt in "--help":
printHelp()
return
# print("options: %s, arg: %s" % (opt, arg))
if opt in ("-s", "--start_point"):
startPointsGeojsonFilename = arg
if opt in ("-e", "--end_point"):
endPointsGeojsonFilename = arg
if opt in ("-o", "--outputFolder"):
outputFolder = arg
if opt in ("-t", "--transportMode"):
transportModeSelected = arg
if opt in "--summary":
summaryOnly = True
if opt in "--routes":
routesOnly = True
if opt in "--is_entry_list":
isEntryList = True
if opt in "--all":
allImpedanceAttribute = True
else:
if opt in ("-c", "--costAttributes"):
impedanceListTemp = arg.split(",")
for impedanceArg in impedanceListTemp:
if (impedanceArg not in car_impedances) and (
impedanceArg not in bicycle_impedances):
raise ImpedanceAttributeNotDefinedException(
impedanceErrorMessage)
if impedanceArg in car_impedances:
impedance = car_impedances[impedanceArg]
elif impedanceArg in bicycle_impedances:
impedance = bicycle_impedances[impedanceArg]
impedanceList.append(impedance)
if not startPointsGeojsonFilename or not endPointsGeojsonFilename or not outputFolder:
raise NotParameterGivenException("Type --help for more information.")
if not transportModeSelected:
raise TransportModeNotDefinedException(transportModeErrorMessage)
if not allImpedanceAttribute and not impedance:
raise ImpedanceAttributeNotDefinedException(impedanceErrorMessage)
generalLogger = GeneralLogger(loggerName="GENERAL",
outputFolder=outputFolder,
prefix="General")
MAX_TRIES = 2
RECOVERY_WAIT_TIME = 10
RECOVERY_WAIT_TIME_8_MIN = 480
postgisServiceProvider = PostgisServiceProvider()
transportMode = None
impedances = None
if transportModeSelected == TransportModes.BICYCLE:
transportMode = BicycleTransportMode(postgisServiceProvider)
impedances = bicycle_impedances
elif transportModeSelected == TransportModes.PRIVATE_CAR:
transportMode = PrivateCarTransportMode(postgisServiceProvider)
elif transportModeSelected == TransportModes.OSM_PRIVATE_CAR:
transportMode = OSMPrivateCarTransportMode(postgisServiceProvider)
impedances = car_impedances
starter = DORARouterAnalyst(transportMode=transportMode)
startTime = time.time()
functionName = "Routing Data Analysis"
generalLogger.getLogger().info(
"%s Start Time: %s" %
(functionName, getFormattedDatetime(timemilis=startTime)))
if not isEntryList:
prefix = os.path.basename(
startPointsGeojsonFilename) + "_" + os.path.basename(
endPointsGeojsonFilename)
error_counts = 0
executed = False
while not executed:
try:
generalLogger.getLogger().info("Analyzing %s" % prefix)
executeSpatialDataAnalysis(outputFolder,
startPointsGeojsonFilename,
endPointsGeojsonFilename, starter,
impedanceList, impedances,
allImpedanceAttribute, summaryOnly,
routesOnly, prefix)
error_counts = 0
executed = True
gc.collect()
except Exception as err:
error_counts += 1
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value,
exc_traceback)
generalLogger.getLogger().exception(''.join('>> ' + line
for line in lines))
memory = psutil.virtual_memory()
generalLogger.getLogger().warning(
"MEMORY USAGE: total=%s, available=%s, percent=%s, used=%s, free=%s"
% (memory.total, memory.available, memory.percent,
memory.used, memory.free))
Logger.getInstance().exception(''.join('>> ' + line
for line in lines))
time.sleep(RECOVERY_WAIT_TIME)
generalLogger.getLogger().warning(
"Calling garbage collector...")
gc.collect()
time.sleep(RECOVERY_WAIT_TIME_8_MIN)
memory = psutil.virtual_memory()
generalLogger.getLogger().warning(
"MEMORY USAGE: total=%s, available=%s, percent=%s, used=%s, free=%s"
% (memory.total, memory.available, memory.percent,
memory.used, memory.free))
if error_counts < (MAX_TRIES + 1):
message = "Error recovery for the %s time%s" % (
error_counts, ("" if error_counts < 2 else "s"))
generalLogger.getLogger().warning(message)
Logger.getInstance().warning(message)
else:
message = "Recurrent error, skipping analysis for: %s" % prefix
generalLogger.getLogger().warning(message)
Logger.getInstance().warning(message)
executed = True
else:
for startRoot, startDirs, startFiles in os.walk(
startPointsGeojsonFilename):
for startPointsFilename in startFiles:
if startPointsFilename.endswith("geojson"):
for endRoot, endDirs, endFiles in os.walk(
endPointsGeojsonFilename):
for endPointsFilename in endFiles:
if endPointsFilename.endswith("geojson"):
prefix = startPointsFilename + "_" + endPointsFilename
error_counts = 0
executed = False
while not executed:
try:
generalLogger.getLogger().info(
"Analyzing %s" % prefix)
executeSpatialDataAnalysis(
outputFolder,
os.path.join(
startRoot,
startPointsFilename),
os.path.join(
endRoot, endPointsFilename),
starter, impedanceList, impedances,
allImpedanceAttribute, summaryOnly,
routesOnly, prefix + "-")
error_counts = 0
executed = True
gc.collect()
except Exception as err:
error_counts += 1
exc_type, exc_value, exc_traceback = sys.exc_info(
)
lines = traceback.format_exception(
exc_type, exc_value, exc_traceback)
generalLogger.getLogger().exception(
''.join('>> ' + line
for line in lines))
memory = psutil.virtual_memory()
generalLogger.getLogger().warning(
"MEMORY USAGE: total=%s, available=%s, percent=%s, used=%s, free=%s"
% (memory.total, memory.available,
memory.percent, memory.used,
memory.free))
Logger.getInstance().exception(''.join(
'>> ' + line for line in lines))
time.sleep(RECOVERY_WAIT_TIME)
generalLogger.getLogger().warning(
"Calling garbage collector...")
gc.collect()
time.sleep(RECOVERY_WAIT_TIME_8_MIN)
memory = psutil.virtual_memory()
generalLogger.getLogger().warning(
"MEMORY USAGE: total=%s, available=%s, percent=%s, used=%s, free=%s"
% (memory.total, memory.available,
memory.percent, memory.used,
memory.free))
if error_counts < (MAX_TRIES + 1):
message = "Error recovery for the %s time%s" % (
error_counts,
(""
if error_counts < 2 else "s"))
generalLogger.getLogger().warning(
message)
Logger.getInstance().warning(
message)
else:
message = "Recurrent error, skipping analysis for: %s" % prefix
generalLogger.getLogger().warning(
message)
Logger.getInstance().warning(
message)
executed = True
endTime = time.time()
generalLogger.getLogger().info(
"%s End Time: %s" %
(functionName, getFormattedDatetime(timemilis=endTime)))
totalTime = timeDifference(startTime, endTime)
generalLogger.getLogger().info("%s Total Time: %s m" %
(functionName, totalTime))