def __load_data(self, fileName=None):
print("--------------Start loading--------------")
print("Load from {}".format(fileName))
ls = LoadSave(fileName)
data = ls.load_data()
print("--------------loading successed !--------------\n")
return data
def concat_different_files():
PATH = [
"..//Data//Original//trajAll08001000.pkl",
"..//Data//Original//trajAll10001300.pkl",
"..//Data//Original//trajAll17001906.pkl"
]
trajData = {}
trajDataIndex = 0
ls = LoadSave(None)
# trajDataIndex is the new index of each trajectory.
for ind, path in enumerate(PATH):
ls._fileName = path
trajDataTmp = ls.load_data()
trajDataTmpIndex = list(trajDataTmp.keys())
for i in trajDataTmpIndex:
trajData[trajDataIndex] = trajDataTmp[i]
print("Trajectory data index: {}".format(trajDataIndex))
trajDataIndex += 1
gc.collect()
ls._fileName = "..//Data//Original//trajDataAll.pkl"
ls.save_data(trajData)
return
def reshape_traj_data():
PATH = "..//Data//Original//trajDataAll.pkl"
ls = LoadSave(PATH)
trajData = ls.load_data()
trajIndex = list(trajData.keys())
trajNewData = {}
print("-----------------------------------")
print("Processing trajAll begins:")
for ind, item in enumerate(trajIndex):
print("Now is {}, total is {}.".format(ind + 1, len(trajIndex)))
traj = trajData[item]
traj["boundingBoxSize"] = traj["W"] * traj["H"]
traj.drop(["carNum", "maskTraj", "maskCrossTraj", "W", "H"],
axis=1,
inplace=True)
traj.rename(columns={
"inCrossTraj": "IN",
"unixTime": "globalTime"
},
inplace=True)
traj["TIME"] = traj["globalTime"] - traj["globalTime"].iloc[0]
traj["TIME"] = traj[
"TIME"].dt.seconds + traj["TIME"].dt.microseconds / 1000000
trajNewData[ind] = traj
ls._fileName = "..//Data//Original//trajDataOriginal.pkl"
ls.save_data(trajNewData)
return
def visualizing_before_filtering():
# Loading original data
FEATURE_PATH = "..//Data//Original//trajDataOriginalFeatures.pkl"
ls = LoadSave(FEATURE_PATH)
trajDataFeatures = ls.load_data()
###################################################
###################################################
# Basic feature plot
f, ax = plt.subplots(figsize=(8, 6))
sns.distplot(trajDataFeatures["pointNums"], bins=50)
plt.title("Distribution of the number of points".format(
trajDataFeatures["pointNums"].quantile(0.1)))
plt.savefig("..//Plots//NumberOfPointsDistribution.pdf",
bbox_inches='tight')
f, ax = plt.subplots(figsize=(8, 6))
sns.distplot(trajDataFeatures["totalTime"], bins=50)
plt.title("Distribution of the total time".format(
trajDataFeatures["totalTime"].quantile(0.1)))
plt.savefig("..//Plots//TotalTimeDistribution.pdf", bbox_inches='tight')
f, ax = plt.subplots(figsize=(8, 6))
sns.distplot(trajDataFeatures["headTailDist"], bins=50)
plt.title("Distribution of the head-tail-distance".format(
trajDataFeatures["headTailDist"].quantile(0.1)))
plt.savefig("..//Plots//HeadTailDistanceDistribution.pdf",
bbox_inches='tight')
basicReport = trajDataFeatures.describe(
[0.01, 0.05, 0.1, 0.15, 0.2, 0.25, 0.5, 0.9, 0.999])
nf = FeatureBasedNoiseFilter(trajDataFeatures=trajDataFeatures)
nf.set_feature_filter_param_lower_bound(headTailDist=20,
pointNums=20,
totalTime=6,
stopPointsPrecent=0)
nf.set_feature_filter_param_upper_bound(headTailDist=2000,
pointNums=20000,
totalTime=20000,
stopPointsPrecent=0.99)
noiseIndex, noiseCond = nf._feature_noisy_filter()
trajDataFeatures.drop(noiseIndex, axis=0, inplace=True)
###################################################
###################################################
print("Nums is {}".format(len(trajDataFeatures)))
f, ax = plt.subplots(figsize=(8, 6))
sns.distplot(trajDataFeatures["pointNums"], bins=50)
plt.title("Distribution of the number of points after filtering".format(
trajDataFeatures["pointNums"].quantile(0.1)))
plt.savefig("..//Plots//NumberOfPointsDistributionAfterFiltering.pdf",
bbox_inches='tight')
f, ax = plt.subplots(figsize=(8, 6))
sns.distplot(trajDataFeatures["totalTime"], bins=50)
plt.title("Distribution of the total time after filtering".format(
trajDataFeatures["totalTime"].quantile(0.1)))
plt.savefig("..//Plots//TotalTimeDistributionAfterFiltering.pdf",
bbox_inches='tight')
f, ax = plt.subplots(figsize=(8, 6))
sns.distplot(trajDataFeatures["headTailDist"], bins=50)
plt.title("Distribution of the head-tail-distance after filtering".format(
trajDataFeatures["headTailDist"].quantile(0.1)))
plt.savefig("..//Plots//HeadTailDistanceDistributionAfterFiltering.pdf",
bbox_inches='tight')
plt.close("all")
del trajDataFeatures
gc.collect()
return basicReport
def train_test_split(trajData=None, trajDataFeatures=None):
trainDataDay = 2
testDataDaya = 3
trajDataTrainFeatures = trajDataFeatures[
trajDataFeatures["enterSceneTime"].dt.day == trainDataDay]
trajDataTestFeatures = trajDataFeatures[
trajDataFeatures["enterSceneTime"].dt.day == testDataDaya]
print("\nData befor spliting :{}".format(trajDataFeatures.shape))
print("Train data shape :{}".format(trajDataTrainFeatures.shape))
print("Test data shape :{}".format(trajDataTestFeatures.shape))
trajDataTrainFeatures.reset_index(inplace=True, drop=True)
trajDataTestFeatures.reset_index(inplace=True, drop=True)
trainIndex = trajDataTrainFeatures["trajIndex"]
testIndex = trajDataTestFeatures["trajIndex"]
trajDataTrain = dict([(int(index), trajData[index])
for index in trainIndex])
trajDataTest = dict([(int(index), trajData[index]) for index in testIndex])
ls = LoadSave(None)
ls._fileName = "..//Data//TrainData//AllData//trajDataTrain.pkl"
ls.save_data(trajDataTrain)
ls._fileName = "..//Data//TrainData//AllData//trajDataTrainFeatures.pkl"
ls.save_data(trajDataTrainFeatures)
ls._fileName = "..//Data//TestData//AllData//trajDataTest.pkl"
ls.save_data(trajDataTest)
ls._fileName = "..//Data//TestData//AllData//trajDataTestFeatures.pkl"
ls.save_data(trajDataTestFeatures)
return
def __save_data(self, data=None, fileName=None):
print("--------------Start saving--------------")
print("Saving file to {}".format(fileName))
ls = LoadSave(fileName)
ls.save_data(data)
print("--------------Saving successed !--------------\n")
ls = LoadSave(None)
ls._fileName = "..//Data//TrainData//AllData//trajDataTrain.pkl"
ls.save_data(trajDataTrain)
ls._fileName = "..//Data//TrainData//AllData//trajDataTrainFeatures.pkl"
ls.save_data(trajDataTrainFeatures)
ls._fileName = "..//Data//TestData//AllData//trajDataTest.pkl"
ls.save_data(trajDataTest)
ls._fileName = "..//Data//TestData//AllData//trajDataTestFeatures.pkl"
ls.save_data(trajDataTestFeatures)
return
if __name__ == "__main__":
ls = LoadSave()
ls._fileName = "..//Data//Original//trajDataOriginal.pkl"
trajData = ls.load_data()
ls._fileName = "..//Data//Original//trajDataOriginalFeatures.pkl"
trajDataFeatures = ls.load_data()
# visualizing_before_filtering()
nf = FeatureBasedNoiseFilter(trajDataFeatures=trajDataFeatures)
nf.set_feature_filter_param_lower_bound(headTailDist=20,
pointNums=20,
totalTime=6,
stopPointsPrecent=0)
nf.set_feature_filter_param_upper_bound(headTailDist=2000,
pointNums=20000,
totalTime=20000,
stopPointsPrecent=0.99)
def extract_non_stop_traj_data():
# Load the completed trajectory data
ls = LoadSave("..//Data//TrainData//Completed//trajDataCompleted.pkl")
trajData = ls.load_data()
ls._fileName = "..//Data//TrainData//Completed//trajDataFeaturesCompleted.pkl"
trajDataFeatures = ls.load_data()
trajDataKeys = list(trajData.keys())
trajDataNew = {}
for ind, item in enumerate(trajDataKeys):
trajGroupby = trajData[item].groupby(["stopIndex"])
tmp = trajGroupby.mean().reset_index().rename(
columns={"index": "stopIndex"})
tmp = tmp[tmp["stopIndex"] != -1]
# This step is in order to append the average stop points on the new trajectory.
trajDataNew[item] = trajData[item][trajData[item]["stopIndex"] ==
-1].reset_index(drop=True)
trajDataNew[item] = pd.concat([trajDataNew[item], tmp],
ignore_index=True)
trajDataNew[item].sort_values(by="TIME", inplace=True)
print("Now is {}, total is {}.".format(ind + 1, len(trajData)))
# Save the completed trajectory data without stop points to the cache
ls._fileName = "..//Data//TrainData//ClusteringCache//trajDataWithoutStop.pkl"
ls.save_data(trajDataNew)
ls._fileName = "..//Data//TrainData//ClusteringCache//trajDataWithoutStopFeatures.pkl"
ls.save_data(trajDataFeatures)
return trajDataNew
axis=1))
traj = self._trajData[ind][["X", "Y"]]
trajCompressedTmp, flag = self.traj_window_compression(traj, dist)
if flag == -1:
print("Fatal error on {}".format(ind))
self._trajDataCompressed[ind] = trajCompressedTmp
else:
self._trajDataCompressed[ind] = trajCompressedTmp
return self._trajDataCompressed
###############################################################################
if __name__ == "__main__":
extract_non_stop_traj_data()
ls = LoadSave(
"..//Data//TrainData//ClusteringCache//trajDataWithoutStop.pkl")
trajData = ls.load_data()
ls._fileName = "..//Data//TrainData//ClusteringCache//trajDataWithoutStopFeatures.pkl"
trajDataFeatures = ls.load_data()
ptsNumList = [55]
for ptsUsedNum in ptsNumList:
compressor = WindowCompression(trajData)
compressor.set_compression_param(samplingNums=ptsUsedNum)
trajDataCompressed = compressor.compress_trajectory()
PATH = "..//Data//TrainData//ClusteringCache//" + "trajDataCompressed_" + str(
ptsUsedNum) + ".pkl"
ls._fileName = PATH
ls.save_data(trajDataCompressed)
self._mergedRoutes = DataFrame(self._mergedRoutes.T, columns=["originalLabel", "mergedLabel"])
self._trajDataFeatures = pd.merge(self._trajDataFeatures, self._mergedRoutes, on='originalLabel', how='left')
uniqueLabel = self._trajDataFeatures["mergedLabel"].unique()
uniqueLabel = np.sort(uniqueLabel)
for ind, item in enumerate(uniqueLabel):
self._mergedRoutes["mergedLabel"].replace(item, ind, inplace=True)
self._trajDataFeatures["mergedLabel"].replace(item, ind, inplace=True)
self.calc_average_route()
self.save_data(self._trajDataFeatures, "..//Data//TrainData//ClusteringCache//trajDataFeatures.pkl")
self.save_data(self._averageRoute, "..//Data//TrainData//ClusteringCache//trajDataAverageRoutes.pkl")
###############################################################################
if __name__ == "__main__":
ls = LoadSave(None)
ls._fileName = "..//Data//TrainData//ClusteringCache//trajDataFeaturesBeforeMerging.pkl"
trajDataFeatures = ls.load_data()
ls._fileName = "..//Data//TrainData//ClusteringCache//trajDataBeforeMerging.pkl"
trajData = ls.load_data()
rm = RouteMerging(trajDataWindowCompressed=trajData.copy(),
trajDataFeatures=trajDataFeatures.copy(),
matchingPointsDist=40)
rm.route_merging()
mergedMat = rm._mergedMatrix
mergedRoutes = rm._mergedRoutes
rm.plot_merged_routes()
features = rm._trajDataFeatures
completed = features["trajIndex"].values
if self._save == True:
self.save_data(
self._trajDataFeatures[self._trajDataFeatures["completeFlag"]
== True],
"..//Data//TrajectoryDataFeaturesComplete.pkl")
self.save_data(
self._trajDataFeatures[self._trajDataFeatures["completeFlag"]
== False],
"..//Data//TrajectoryDataFeaturesBroken.pkl")
return self._trajDataFeatures, startStatus, endStatus
if __name__ == "__main__":
# Load data from training file.
ls = LoadSave("..//Data//TrainData//AllData//trajDataTrainFeatures.pkl")
trajDataFeatures = ls.load_data()
ls._fileName = "..//Data//TrainData//AllData//trajDataTrain.pkl"
trajData = ls.load_data()
nf = InterestedRegionDiscovery(trajDataFeatures.copy())
nf.set_dbscan_start_param(alpha=0.80, eps=30, minSamples=95)
nf.set_dbscan_end_param(alpha=0.80, eps=55, minSamples=95)
trajDataFeaturesNew, startStatus, endStatus = nf.filtering()
ls = LoadSave("..//Data//TrainData//AllData//trajDataTrainFeatures.pkl")
ls.save_data(trajDataFeaturesNew)
# Extracting and saving the completed trajectory data
trajDataFeaturesCompleted = trajDataFeaturesNew[
def load_traj_data():
PATH = "..//Data//Original//trajDataOriginal.pkl"
ls = LoadSave(PATH)
trajData = ls.load_data()
return trajData