def setUp(self):
self.dir = os.getcwd()
self.comparison = Comparison()
self.postGISServiceProvider = PostGISServiceProvider()
self.fileActions = FileActions()
self.testTableName = "test_travel_time_matrix"
self.rushHourTableName = "rush_hour_time_car"
self.middayTableName = "midday_time_car"
self.rushHourTableNameDetailed = "rush_hour_time_car_detailed"
self.bicycleTravelTimeMatrix = "bicycle_travel_time_matrix"
class SpatialPatternsTest(unittest.TestCase):
def setUp(self):
self.dir = os.getcwd()
self.spatialPatter = SpatialPatterns(
comparison=Comparison(),
postGISServiceProvider=PostGISServiceProvider())
self.fileActions = FileActions()
self.testTableName = "test_travel_time_matrix"
self.rushHourTableName = "rush_hour_time_car"
self.middayTableName = "midday_time_car"
self.rushHourTableNameDetailed = "rush_hour_time_car_detailed"
self.bicycleTravelTimeMatrix = "bicycle_travel_time_matrix"
def test_givenAGeojson_then_storedItInATheCorrectTable(self):
travelTimeMatrixURL = self.dir + "%data%testData%rush_hour_delay_time_YKRCostSummary-13000-5-oldRoadNetwork.geojson".replace(
"%", os.sep)
columns = {
"startPoint_YKR_ID": "ykr_from_id",
"endPoint_YKR_ID": "ykr_to_id",
"total_travel_time": "travel_time"
}
isExecuted = self.spatialPatter.insertData(travelTimeMatrixURL,
columns)
self.assertTrue(isExecuted)
def test_givenAGeojsonWithtravelTimeDifference_then_storedItInATheCorrectTable(
self):
travelTimeMatrixURL = self.dir + "%data%outputData%rush_hour_travelTimeMatrixDifference.geojson".replace(
"%", os.sep)
columns = {
"startPoint_id": "ykr_from_id",
"endPoint_id": "ykr_to_id",
"total_travel_time": "travel_time",
"travel_time_difference": "travel_time_difference"
}
isExecuted = self.spatialPatter.insertData(
travelTimeMatrixURL, self.rushHourTableNameDetailed, columns)
self.assertTrue(isExecuted)
def test_givenSlowBicycleCostSummaryGeojson_then_storedItInBicycleCostSummaryTable(
self):
travelTimeMatrixURL = self.dir + "%data%outputData%slow_time_BicycleRoadNetwork-13000-5.geojson".replace(
"%", os.sep)
columns = {
"startPoint_YKR_ID": "ykr_from_id",
"endPoint_YKR_ID": "ykr_to_id",
"total_travel_time": "travel_time",
}
isExecuted = self.spatialPatter.insertData(
travelTimeMatrixURL, "slow_" + self.bicycleTravelTimeMatrix,
columns)
self.assertTrue(isExecuted)
def test_givenFastBicycleCostSummaryGeojson_then_storedItInBicycleCostSummaryTable(
self):
travelTimeMatrixURL = self.dir + "%data%outputData%cardat2%fast_time_dijsktraCostMetroAccessDigiroadSummary.geojson".replace(
"%", os.sep)
columns = {
"startPoint_YKR_ID": "ykr_from_id",
"endPoint_YKR_ID": "ykr_to_id",
"total_travel_time": "travel_time",
}
tableName = "cardat_two_fast_" + self.bicycleTravelTimeMatrix + ""
isExecuted = self.spatialPatter.insertData(travelTimeMatrixURL,
tableName, columns)
self.assertTrue(isExecuted)
def test_decompressZip_to_uploadGeojsonFilesToPostgresTable(self):
zippath = "C:/Users/jeisonle/repository/Car-Routing/DigiroadPreDataAnalysis/digiroad/test/data/geojson/Subsets/subset1/summary1.zip"
outputFolder = self.dir + "%data%outputData%summaries".replace(
"%", os.sep)
outputFolder = os.path.join(outputFolder,
os.path.basename(zippath).split(".")[-2])
self.fileActions.decompressZipfile(zippath, outputFolder)
columns = {
"startPoint_YKR_ID": "ykr_from_id",
"endPoint_YKR_ID": "ykr_to_id",
"total_travel_time": "travel_time",
}
tableName = "cardat_fast_" + self.bicycleTravelTimeMatrix + ""
for root, dirs, files in os.walk(outputFolder):
for filename in files:
if filename.endswith("geojson"):
f = os.path.join(root, filename)
print("Loading: %s" % filename)
isExecuted = self.spatialPatter.insertData(
f, tableName, columns)
if isExecuted:
os.remove(f)
self.assertTrue(isExecuted)
def runTravelTimeMatrixOperations(querying, uploading, outputFolder, zippath,
directionality, targets):
try:
comparison = Comparison()
postGISServiceProvider = PostGISServiceProvider()
spatialPatterns = SpatialPatterns(
comparison=comparison,
postGISServiceProvider=postGISServiceProvider)
fileActions = FileActions()
attributes = getConfigurationProperties(section="ATTRIBUTES_MAPPING")
columns = {}
columnList = []
for attribute_key in attributes:
attribute_splitted = attributes[attribute_key].split(",")
key = attribute_splitted[0]
value = attribute_splitted[1]
columns[key] = value
columnList.append(value)
tableName = getConfigurationProperties(
section="DATABASE_CONFIG")["travel_time_table_name"]
if uploading:
log_filename = "uploading_" + os.path.basename(zippath).split(
".")[-2]
Logger.configureLogger(outputFolder, log_filename)
try:
zip_ref = zipfile.ZipFile(zippath, 'r')
members = zip_ref.namelist()
Logger.getInstance().info("%s geojson files to be uploaded." %
(len(members)))
for member in members:
extractZipFile(zip_ref, member, outputFolder)
f = os.path.join(outputFolder, member)
isExecuted = spatialPatterns.insertData(
f, tableName, tuple(columnList), outputFolder)
os.remove(f)
if not isExecuted:
raise NotUploadedTravelTimeMatrixException(member)
finally:
zip_ref.close()
Logger.getInstance().info("Uploaded: %s" % zippath)
if querying:
targetList = targets.split(",")
for target in targetList:
log_filename = "querying_travel_time_matrix_%s_%s" % (
directionality, target)
Logger.configureLogger(outputFolder, log_filename)
traveltimeMatrixFilename = "travel_time_matrix_%s_%s.geojson" % (
directionality, target)
Logger.getInstance().info("Querying %s: %s" %
(directionality, target))
if "TO".__eq__(directionality):
geodataframe = postGISServiceProvider.getTravelTimeMatrixTo(
ykrid=target, tableName=tableName)
else:
geodataframe = postGISServiceProvider.getTravelTimeMatrixFrom(
ykrid=target, tableName=tableName)
geojson = postGISServiceProvider.convertToGeojson(geodataframe)
fileActions.writeFile(folderPath=outputFolder,
filename=traveltimeMatrixFilename,
data=geojson)
Logger.getInstance().info(
"Find the the travel time matrix geojson file in this path: %s"
% (os.path.join(outputFolder, traveltimeMatrixFilename)))
except Exception as err:
exc_type, exc_value, exc_traceback = sys.exc_info()
lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
Logger.getInstance().exception(''.join('>> ' + line for line in lines))
class PostGISServiceProviderTest(unittest.TestCase):
def setUp(self):
self.dir = os.getcwd()
self.comparison = Comparison()
self.postGISServiceProvider = PostGISServiceProvider()
self.fileActions = FileActions()
self.testTableName = "test_travel_time_matrix"
self.rushHourTableName = "rush_hour_time_car"
self.middayTableName = "midday_time_car"
self.rushHourTableNameDetailed = "rush_hour_time_car_detailed"
self.bicycleTravelTimeMatrix = "bicycle_travel_time_matrix"
def test_insertTestData(self):
columns = {
"startPoint_YKR_ID": "ykr_from_id",
"endPoint_YKR_ID": "ykr_to_id",
"total_travel_time": "travel_time"
}
travelTimeMatrix = self.dummyTravelTimeMatrix()
travelTimeMatrix = self.postGISServiceProvider.renameColumnsAndExtractSubSet(
travelTimeMatrix=travelTimeMatrix, columns=columns)
travelTimeMatrix = self.postGISServiceProvider.insertableTravelTimeMatrixGeoDataFrame(
travelTimeMatrix=travelTimeMatrix,
tableName=self.testTableName,
column1="ykr_from_id",
column2="ykr_to_id")
isExecuted = self.postGISServiceProvider.insert(
dataFrame=travelTimeMatrix,
tableName=self.testTableName,
isTableExist="append")
self.assertTrue(isExecuted)
def test_insertGridData(self):
gridcellsURL = self.dir + "%data%testData%YKRGridCells.geojson".replace(
"%", os.sep)
tableName = "ykr_gridcells"
gridcellsURLDF = gpd.GeoDataFrame.from_file(gridcellsURL)
columns = {"X": "x", "Y": "y", "YKR_ID": "ykr_id"}
gridcellsURLDF = gridcellsURLDF.rename(index=str, columns=columns)
gridcellsURLDF = gridcellsURLDF.to_crs(GPD_CRS.PSEUDO_MERCATOR)
isExecuted = self.postGISServiceProvider.insert(
gridcellsURLDF,
tableName=tableName,
isTableExist="replace",
geometryType=GeometryType.POLYGON)
self.assertTrue(isExecuted)
def dummyTravelTimeMatrix(self):
data = {
'startPoint_YKR_ID':
pd.Series([1., 2., 3., 1.], index=[1, 2, 3, 4]),
'endPoint_YKR_ID':
pd.Series([4., 5., 6., 5.], index=[1, 2, 3, 4]),
'total_travel_time':
pd.Series([20., 30., 40., 50.], index=[1, 2, 3, 4])
}
lineString1 = LineString([[2793620.4483544305, 8460782.78014875],
[2748951.962723606, 8424743.938335517]])
geometries = [lineString1, lineString1, lineString1, lineString1]
dataframe = pd.DataFrame(data)
crs = {"init": 'epsg:3857'}
travelTimeMatrix = gpd.GeoDataFrame(dataframe,
crs=crs,
geometry=geometries)
return travelTimeMatrix
def test_givenARowThatAlreadyExist_then_returnTrue(self):
travelTimeMatrix = self.dummyTravelTimeMatrix()
columns = {
"startPoint_YKR_ID": "ykr_from_id",
"endPoint_YKR_ID": "ykr_to_id",
"total_travel_time": "travel_time"
}
travelTimeMatrix = travelTimeMatrix.rename(index=str, columns=columns)
self.assertEqual(
0,
len(
self.postGISServiceProvider.
insertableTravelTimeMatrixGeoDataFrame(
travelTimeMatrix=travelTimeMatrix,
tableName=self.testTableName,
column1="ykr_from_id",
column2="ykr_to_id")))
def test_givenAYKRID_then_getItsTravelTimeMatrix(self):
# 5973738 rautatientori
# 5878018
# 5870644
# 5963599
# 5920413
ykrid = 5793265
tableName = "cardat_two_fast_" + self.bicycleTravelTimeMatrix + ""
outputFolder = self.dir + "%data%outputDataBicycle%".replace(
"%", os.sep)
filename = tableName + "_%s.geojson" % ykrid
geodataframe = self.postGISServiceProvider.getTravelTimeMatrixTo(
ykrid=ykrid, tableName=tableName)
geojson = self.postGISServiceProvider.convertToGeojson(geodataframe)
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
self.assertGreater(len(geojson["features"]), 0)
def test_givenAYKRID_then_getItsTravelTimeMatrixDifferences(self):
ykrid = 5973738
tableName = "cardat_two_fast_" + self.bicycleTravelTimeMatrix + ""
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = tableName + "_%s.geojson" % ykrid
geodataframe = self.postGISServiceProvider.getTravelTimeMatrixDifferences(
ykrid=ykrid, tableName=tableName)
geojson = self.postGISServiceProvider.convertToGeojson(geodataframe)
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
self.assertGreater(len(geojson["features"]), 0)
def setUp(self):
self.fileActions = FileActions()
self.dir = os.getcwd()
self.comparison = Comparison()
class ComparisonTest(unittest.TestCase):
def setUp(self):
self.fileActions = FileActions()
self.dir = os.getcwd()
self.comparison = Comparison()
def test_getGridCellSamples(self):
gridCellsURL = self.dir + "%data%testData%YKRGridCells.geojson".replace(
"%", os.sep)
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = "sampleYKRGridCells-5.geojson"
gridCellSamples = self.comparison.getGridSamples(
gridCellsURL=gridCellsURL, sampleSie=5)
geojson = self.comparison.convertToGeojson(gridCellSamples)
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
self.assertIsNotNone(gridCellSamples)
def test_givenGridCellsPolygons_then_transformCentroidsInPoints(self):
gridCellsURL = self.dir + "%data%testData%points.geojson".replace(
"%", os.sep)
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = "points.geojson"
points = self.comparison.createPointsFromGrigCells(
gridCellsURL=gridCellsURL)
geojson = self.comparison.convertToGeojson(points)
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
self.assertIsNotNone(points)
def test_loadTravelTimeMatrixDataFrameSubset(self):
travelTimeMatrixURL = "C:%Users%jeisonle%Downloads%HelsinkiRegion_TravelTimeMatrix2015".replace(
"%", os.sep)
# gridCellsURL = self.dir + "%data%testData%Test_Points_Reititin_with_MatrixID.geojson".replace("%", os.sep)
originGridCellsURL = self.dir + "%data%testData%sampleYKRGridPoints-5.geojson".replace(
"%", os.sep)
destinationGridCellsURL = self.dir + "%data%testData%sampleYKRGridPoints-13000.geojson".replace(
"%", os.sep)
travelTimeMatrixSubset = self.comparison.loadTravelTimeMatrixDataFrameSubset(
travelTimeMatrixURL=travelTimeMatrixURL,
originGridCellsURL=originGridCellsURL,
destinationGridCellsURL=destinationGridCellsURL,
gridID="YKR_ID")
# out = r"...\Travel_times_from_chosen_originIDs_to_selected_destinationIDs.txt"
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = "travelTime5-13000PointsSubset.csv"
travelTimeMatrixSubset.to_csv(outputFolder + filename,
sep=';',
index=False)
self.assertIsNotNone(travelTimeMatrixSubset)
def test_givenCarRoutingCostSummary_then_mergeItWithMetropAccessData(self):
travelTimeMatrixURL = self.dir + "%data%outputData%travelTime5-13000PointsSubset.csv".replace(
"%", os.sep)
carRoutingCostSummaryURL = self.dir + "%data%testData%rush_hour_delay_time_YKRCostSummary-5-13000.geojson".replace(
"%", os.sep)
# carRoutingCostSummaryURL = self.dir + "%data%testData%midday_delay_time_YKRCostSummary-100Points.geojson".replace(
# "%", os.sep)
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = "comp_rush_hour_delay_time_mergedCostSummaryWithMetropAccessData.geojson"
# filename = "comp_midday_delay_time_mergedCostSummaryWithMetropAccessData.geojson"
mergeResult = self.comparison.mergeMetropAccessData(
travelTimeMatrixURL=travelTimeMatrixURL,
carRoutingCostSummaryURL=carRoutingCostSummaryURL)
geojson = self.comparison.convertToGeojson(mergeResult)
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
self.assertIsNotNone(mergeResult)
def test_givenTravelTimeMergedDataLayer_then_calculateTheDifferenceBetweenOldAndNewData(
self):
travelTimeSummaryURL = self.dir + "%data%outputData%comp_rush_hour_delay_time_mergedCostSummaryWithMetropAccessData.geojson".replace(
"%", os.sep)
# travelTimeSummaryURL = self.dir + "%data%outputData%comp_midday_delay_time_mergedCostSummaryWithMetropAccessData.geojson".replace(
# "%", os.sep)
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = "rush_hour_travelTimeMatrixDifference.geojson"
# filename = "midday_delay_travelTimeMatrixDifference.geojson"
travelTimeDifferenceLayer = self.comparison.calculateDifferenceBetweenOldAndNewTravelTimes(
travelTimeSummaryURL=travelTimeSummaryURL)
geojson = self.comparison.convertToGeojson(travelTimeDifferenceLayer)
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
self.assertIsNotNone(travelTimeDifferenceLayer)
def test_getProblematicSetOfPoints(self):
rush_hour_travel_time_filename = self.dir + "%data%outputData%rush_hour_travelTimeMatrixDifference.geojson".replace(
"%", os.sep)
midday_travel_time_filename = self.dir + "%data%outputData%midday_delay_travelTimeMatrixDifference.geojson".replace(
"%", os.sep)
threshold = 10
travelTimeProblematicPoints = self.comparison.extractSummaryThatExceedAThreshold(
travelTimeSummaryURL=rush_hour_travel_time_filename,
threshold=threshold)
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = "problematicPoints.txt"
travelTimeProblematicPoints.to_csv(outputFolder + filename,
sep=';',
index=False)
print(len(travelTimeProblematicPoints[0:]))
self.assertIsNotNone(travelTimeProblematicPoints)
def test_givenYKRPoints_then_createHeatLayer(self):
gridCellsURL = self.dir + "%data%testData%Test_Points_MetropA_Digiroad.geojson".replace(
"%", os.sep)
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
gridCellSamples = self.comparison.getGridSamples(
gridCellsURL=gridCellsURL, sampleSie=4, YKR_ID="ID")
geojson = self.comparison.convertToGeojson(gridCellSamples)
filename = "sampleYKRGridCellsForHeatMap.geojson"
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
selectedGridCellsURL = outputFolder + filename
travelTimeMatrixDifferenceURL = outputFolder + "midday_delay_travelTimeMatrixDifference.geojson"
heatMapLayer = self.comparison.createMultiPointHeatMapLayer(
selectedGridCellsURL=selectedGridCellsURL,
travelTimeMatrixDifferenceURL=travelTimeMatrixDifferenceURL)
self.assertIsNotNone(heatMapLayer)
def test_createWalkingDistanceLayer(self):
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = "walking_distance_regions.geojson"
p1 = Polygon([(387678.024778, 6675360.99039),
(387891.53396, 6670403.35286),
(383453.380944, 6670212.21613),
(383239.871737, 6675169.85373),
(387678.024778, 6675360.99039)])
g = [p1]
d = {
'region': pd.Series(["Helsinki City Center"], index=['a']),
'walking_distance': pd.Series([180], index=['a'])
}
df = pd.DataFrame(d)
crs = {"init": 'epsg:3067'}
walking_distance = gpd.GeoDataFrame(df, crs=crs, geometry=g)
geojson = self.comparison.convertToGeojson(walking_distance)
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
def test_readShapefileAsADataFrame(self):
# 5973738 rautatientori
# 5870644
# 5963599
# 5920413
# 5878018
ykr_id = 5878018
ykrGridGeojson = self.dir + "%data%testData%YKRGridCells.geojson".replace(
"%", os.sep)
shapefile = self.dir + (
"&data&outputData&data-13000-5&Car_Rush_hour_2015_to_%s.shp" %
ykr_id).replace("&", os.sep)
pgGeojson = self.dir + "%data%outputData%rush_hour_delay_time_YKRCostSummary-13000-5.geojson".replace(
"%", os.sep)
outputFolder = self.dir + "%data%outputData%".replace("%", os.sep)
filename = "detailedAttributeDifferences_newRoeadNetwork_%s.geojson" % ykr_id
oldMetropAccessDataFrame = gpd.read_file(shapefile)
newPgRoutingDataFrame = gpd.read_file(pgGeojson)
ykrDataFrame = gpd.read_file(ykrGridGeojson)
ykrDataFrame = ykrDataFrame[["YKR_ID", "x", "y", "geometry"]]
ykrDataFrame = ykrDataFrame.to_crs(GPD_CRS.PSEUDO_MERCATOR)
newPgRoutingDataFrame.startPoint_id = [
int(x) for x in newPgRoutingDataFrame.startPoint_YKR_ID.values
]
newPgRoutingDataFrame.endPoint_id = [
int(y) for y in newPgRoutingDataFrame.endPoint_YKR_ID.values
]
newPgRoutingDataFrame = newPgRoutingDataFrame.drop(['geometry'],
axis=1)
columns = {
"Kavely_O_T": "old_startPoint_EuclideanDistanceWalkingTime",
"Kavely_T_P": "old_startPoint_AVGWalkingDistanceWalkingTime",
"Kavely_T_D": "old_endPoint_EuclideanDistanceWalkingTime",
"Kavely_P_T": "old_endPoint_AVGWalkingDistanceWalkingTime",
"Parkkiaika": "old_endPoint_ParkingTime",
"Ruuhka_aa": "old_rush_hour_delay_time",
"Keskpva_aa": "old_midday_delay_time",
"Kokopva_aa": "old_day_average_delay_time",
"Digiroa_aa": "old_speed_limit_time",
}
oldMetropAccessDataFrame = oldMetropAccessDataFrame.rename(
index=str, columns=columns)
oldMetropAccessDataFrame = oldMetropAccessDataFrame.to_crs(
GPD_CRS.PSEUDO_MERCATOR)
oldMetropAccessDataFrame = oldMetropAccessDataFrame.drop(['geometry'],
axis=1)
mergedData = oldMetropAccessDataFrame.merge(ykrDataFrame,
left_on="from_id",
right_on="YKR_ID")
mergedData = mergedData.merge(newPgRoutingDataFrame,
left_on=("from_id", "to_id"),
right_on=("startPoint_id",
"endPoint_id"))
costAttribute = newPgRoutingDataFrame["costAttribute"][0]
mergedData["dif_start_EuclideanDistanceWalkingTime"] = mergedData.startPoint_EuclideanDistanceWalkingTime - \
mergedData.old_startPoint_EuclideanDistanceWalkingTime
mergedData[
"dif_startPoint_AVGWalkingDistanceWalkingTime"] = mergedData.startPoint_AVGWalkingDistanceWalkingTime - \
mergedData.old_startPoint_AVGWalkingDistanceWalkingTime
mergedData["dif_endPoint_EuclideanDistanceWalkingTime"] = mergedData.endPoint_EuclideanDistanceWalkingTime - \
mergedData.old_endPoint_EuclideanDistanceWalkingTime
mergedData["dif_endPoint_AVGWalkingDistanceWalkingTime"] = mergedData.endPoint_AVGWalkingDistanceWalkingTime - \
mergedData.old_endPoint_AVGWalkingDistanceWalkingTime
mergedData["dif_endPoint_ParkingTime"] = mergedData.endPoint_ParkingTime - \
mergedData.old_endPoint_ParkingTime
difCostAttribute = "dif_" + costAttribute
mergedData[difCostAttribute] = mergedData["old_%s" % costAttribute] - \
mergedData[costAttribute]
# mergedData = mergedData[["from_id", "to_id"]]
# self.assertIsNotNone(oldMetropAccessDataFrame)
# self.assertIsNotNone(newPgRoutingDataFrame)
geojson = self.fileActions.convertDataFrameToGeojson(mergedData)
self.fileActions.writeFile(folderPath=outputFolder,
filename=filename,
data=geojson)
self.assertIsNotNone(geojson)