def pushButtonPlotNewWindowClick(self):
d = QtWidgets.QFileDialog.getOpenFileNames(
self, 'Open file',
'C:/Users/aqua_/source/repos/PythonTensor/PythonTensor/generated')
if (len(d[0]) != 0):
res = []
for fname in d[0]:
with open(fname, 'r') as f:
obj = json.load(f, object_hook=CentralPoint.deserialize)
res.append(obj)
Plotter.plot(res, self.Controller.locationsCount,
self.Controller.minTime, self.Controller.maxTime,
self.Controller.minPauseTime,
self.Controller.maxPauseTime,
self.Controller.levyCoeff, 'softmax')
def tabWidgetGraphChanged(self, num):
Plotter.clearGraph()
state = self.checkBoxPlotTrained.checkState()
if state == 2:
Plotter.plotT(self.tabWidgetGraph.currentIndex())
Plotter.plotG(self.tabWidgetGraph.currentIndex(),
self.Controller.locationsCount, self.Controller.minTime,
self.Controller.maxTime, self.Controller.minPauseTime,
self.Controller.maxPauseTime, self.Controller.levyCoeff)
def pushButtonAddGraphClick(self):
d = QtWidgets.QFileDialog.getOpenFileNames(
self, 'Open file',
'C:/Users/aqua_/source/repos/PythonTensor/PythonTensor/generated')
if (len(d[0]) != 0):
Plotter.clearGraph()
res = []
for fname in d[0]:
with open(fname, 'r') as f:
obj = json.load(f, object_hook=CentralPoint.deserialize)
res.append(obj)
Plotter.appendData(res)
state = self.checkBoxPlotTrained.checkState()
if state == 2:
Plotter.plotT(self.tabWidgetGraph.currentIndex())
Plotter.plotG(self.tabWidgetGraph.currentIndex(),
self.Controller.locationsCount,
self.Controller.minTime, self.Controller.maxTime,
self.Controller.minPauseTime,
self.Controller.maxPauseTime,
self.Controller.levyCoeff)
def __init__(self):
os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
super().__init__()
self.setupUi(self)
self.pushButtonConvert.clicked.connect(self.convert_click)
self.pushButtonDeserialize.clicked.connect(self.deserialize)
self.pushButtonSerialize.clicked.connect(self.serialize)
self.pushButtonPlotTrain.clicked.connect(self.plotTrain)
self.pushButtonInit.clicked.connect(self.netInit)
self.pushButtonTrain.clicked.connect(self.train_click)
#self.pushButtonTesting.clicked.connect(self.testing_click)
self.pushButtonGenerate.clicked.connect(self.testing_click)
self.pushButtonNetSerialize.clicked.connect(
self.pushButtonNetSerializeClick)
self.pushButtonNetDeserialize.clicked.connect(
self.pushButtonNetDeserializeClick)
self.pushButtonAddGraph.clicked.connect(self.pushButtonAddGraphClick)
self.pushButtonClearGraph.clicked.connect(
self.pushButtonClearGraphClick)
self.pushButtonPlotNewWindow.clicked.connect(
self.pushButtonPlotNewWindowClick)
self.model = SetListModel()
#self.listView_6.setModel(self.model)
#self.listView_6.selectionModel().selectionChanged.connect(self.listChangedEvent)
#graphWidget = pg.PlotWidget()
#self.verticalLayoutVisualize.addWidget(graphWidget)
#self.sourceVisualizer = Visualizer(graphWidget)
self.tabWidget.currentChanged.connect(self.tabChanged)
self.tabWidgetGraph.currentChanged.connect(self.tabWidgetGraphChanged)
self.checkBoxPlotTrained.clicked.connect(self.checkBoxPlotTrainedClick)
fig = Plotter.initFigure()
self.plotWidget = FigureCanvas(Plotter.fig)
lay = QtWidgets.QVBoxLayout(self.verticalWidget)
lay.setContentsMargins(0, 0, 0, 0)
lay.addWidget(self.plotWidget)
self.bar = None
def pushButtonClearGraphClick(self):
Plotter.clear()
state = self.checkBoxPlotTrained.checkState()
if state == 2:
Plotter.plotT(self.tabWidgetGraph.currentIndex())
def setTrainData(self, data, locations, window):
self.trainPoints = data
self.locationsCount = len(locations)
self.distancesInLocation = []
self.distancesBetweenLocation = []
self.firstPoints = []
self.trainStartTimes = []
self.lastLocations = []
self.lastLocationsWindow = window
self.window = window
self.mostPopularStartLocations = []
d = []
num = 0
for i in data:
countMostPopular = 0
locs = [0] * self.locationsCount
d1 = []
a = [0] * self.locationsCount
self.distancesInLocation.append(a.copy())
self.distancesBetweenLocation.append(a.copy())
self.firstPoints.append([])
if not len(self.firstPoints[num]) == self.window:
self.firstPoints[num].append(i[0].departurePoint)
for j in i:
dep = Location.FindLocation(j.departurePoint, locations)
dest = Location.FindLocation(j.destination, locations)
d1.append(LocationJump(dep, dest, j.time, j.pauseTime))
if not len(self.firstPoints[num]) == self.window:
self.firstPoints[num].append(j.destination)
if dep.Number == dest.Number:
self.distancesInLocation[len(
self.distancesInLocation
) - 1][dep.Number] = self.distancesInLocation[len(
self.distancesInLocation
) - 1][
dep.
Number] + PathPoint.distanceInMetersBetweenEarthCoordinates(
j.departurePoint.latitude,
j.departurePoint.longitude, j.destination.latitude,
j.destination.longitude)
else:
pass
d.append(d1)
num = num + 1
self.data = d
self.maxTime, self.minTime = self.maxTimeCalc(data)
self.maxPauseTime, self.minPauseTime = self.maxPauseTimeCalc(data)
self.trainData = []
self.generateLocationsArray = []
self.generateParametersArray = []
num = 0
for arr in self.data:
if (len(arr) == 0):
continue
self.generateLocationsArray.append([])
self.generateParametersArray.append([])
x = []
a = [0] * self.locationsCount
a[arr[0].departure.Number] = 1
x.append(a)
if not len(self.generateLocationsArray[num]) == self.window:
self.generateLocationsArray[num].append(a)
t = []
for i in range(len(arr)):
a = [0] * self.locationsCount
a[arr[i].destination.Number] = 1
x.append(a)
if not len(self.generateLocationsArray[num]) == self.window:
self.generateLocationsArray[num].append(a)
a = [
Levy.levyCalculate(arr[i].time, self.minTime,
self.maxTime / self.levyCoeff),
Levy.levyCalculate(arr[i].pauseTime, self.minPauseTime,
self.maxPauseTime / self.levyCoeff)
]
t.append(a)
if not len(
self.generateParametersArray[num]) == self.window - 1:
self.generateParametersArray[num].append(a)
self.trainData.append([x, t])
num = num + 1
Plotter.setTrainIntervals(self.trainData, self.distancesInLocation,
self.trainPoints, self.locationsCount,
self.minTime, self.maxTime,
self.minPauseTime, self.maxPauseTime,
self.levyCoeff)
self.histCount = len(
Plotter.trainIntervals.tracesLength) + 1 - 1 + len(
Plotter.trainIntervals.pauseTimes) - 1
def plotCharacteristics(self, data):
Plotter.plot(data, self.locationsCount, self.minTime, self.maxTime,
self.minPauseTime, self.maxPauseTime, self.levyCoeff)
def plotTrain(self):
Plotter.plotTrained(self.trainData, self.distancesInLocation,
self.trainPoints, self.locationsCount,
self.minTime, self.maxTime, self.minPauseTime,
self.maxPauseTime, self.levyCoeff)
def train(self, data, trainPercent, minTime, maxTime, minPauseTime,
maxPauseTime, stopTrainData, distancesInTrainLocations, rawData,
trainPoints, levyCoeff):
totalTrainX = []
totalTrainY = []
totalTestX = []
totalTestY = []
totalTrainXParameters = []
totalTrainYParameters = []
totalTestXParameters = []
totalTestYParameters = []
trainXWithoutGlue = []
trainParamWithoutGlue = []
rawXParameters = None
betweenLocationsX = []
betweenLocationsY = []
#self.normalizer.Restore(None)
for arr in data:
if len(arr) < self.window:
trainXWithoutGlue.append([])
trainParamWithoutGlue.append([])
continue
train_size = int(len(arr) * (trainPercent / 100))
test_size = len(arr) - train_size
x = []
a = [0] * self.locationsCount
wg = []
wgt = []
a[arr[0].departure.Number] = 1
x.append(a)
t = []
wg.append(a.copy())
for i in range(len(arr)):
a1 = [0] * self.locationsCount
a1[arr[i].destination.Number] = 1
x.append(a1)
wg.append(a1.copy())
a = [
Levy.levyCalculate(arr[i].time, minTime,
maxTime / levyCoeff),
Levy.levyCalculate(arr[i].pauseTime, minPauseTime,
maxPauseTime / levyCoeff)
]
wgt.append(a.copy())
t.append(a)
#train, test = x[0:train_size], x[train_size:len(x)]
trainX, trainY = self.create_dataset(x, self.window)
#testX, testY = self.create_dataset(test, self.window)
trainXWithoutGlue.append(wg)
trainParamWithoutGlue.append(wgt)
#seq = json.dumps(x)
#seq = json.dumps(t)
for i in range(0, len(trainX)):
trainX[i] = self.glueArrays(trainX[i])
#for i in range(0, len(testX)):
# testX[i] = self.glueArrays(testX[i])
totalTrainX = totalTrainX + trainX
#totalTestX = totalTestX + testX
totalTrainY = totalTrainY + trainY
#totalTestY = totalTestY + testY
trainXParameters, trainYParameters = self.create_dataset(
t, self.window - 1)
#testXParameters, testYParameters = self.create_dataset(t[train_size:len(t)], self.window)
for i in range(0, len(trainX)):
trainXParameters[i] = self.glueArrays(
trainXParameters[i]) + trainX[i].copy()
#for i in range(0, len(testXParameters)):
# testXParameters[i] = self.glueArrays(testXParameters[i])
totalTrainXParameters = totalTrainXParameters + trainXParameters
#totalTestXParameters = totalTestXParameters + testXParameters
totalTrainYParameters = totalTrainYParameters + trainYParameters
#totalTestYParameters = totalTestYParameters + testYParameters
self.fit(self.model,
np.array(totalTrainX),
np.array(totalTrainY),
validation_data=(np.array(totalTestX), np.array(totalTestY)),
batchSize=2,
epochs=60)
self.fit(self.timeModel,
np.array(totalTrainXParameters),
np.array(totalTrainYParameters),
batchSize=2)
stopData = []
stopDataY = []
for i in range(len(data)):
if len(data[i]) < self.window:
continue
res = self.glueArrays(trainXWithoutGlue[i][-self.stopWindow:])
d, _ = Plotter.tracesTime([[[], [x]]
for x in trainParamWithoutGlue[i]],
minTime, maxTime, minPauseTime,
maxPauseTime, levyCoeff, False)
res.append(sum(d) / Plotter.maxTraceTime)
d, _ = Plotter.calculateTracesLength(
[distancesInTrainLocations[i]], False)
count = Plotter.countInIntervals(
d, Plotter.trainIntervals.tracesLength)
s = sum(count)
if not s == 0:
count = [c1 / s for c1 in count]
res = res + list(count)
d, _ = Plotter.calculatePauseTimes(
[[[], [x]] for x in trainParamWithoutGlue[i]], minPauseTime,
maxPauseTime, levyCoeff, False)
count = Plotter.countInIntervals(d,
Plotter.trainIntervals.pauseTimes)
s = sum(count)
if not s == 0:
count = [c1 / s for c1 in count]
res = res + count
stopData.append(res)
stopDataY.append(1)
for c in range(self.stopWindow, len(trainXWithoutGlue[i]), 1):
sx = trainXWithoutGlue[i][c - self.stopWindow:c]
sx = self.glueArrays(sx)
sy = 0
d, _ = Plotter.tracesTime(
[[[], [x]] for x in trainParamWithoutGlue[i][:c]], minTime,
maxTime, minPauseTime, maxPauseTime, levyCoeff, False)
sx.append(sum(d) / Plotter.maxTraceTime)
length = [0] * self.locationsCount
currLocs = trainXWithoutGlue[i][:c]
try:
for locIndex in range(len(currLocs) - 1):
if (currLocs[locIndex].index(1) == currLocs[
locIndex + 1].index(1)):
length[currLocs[locIndex].index(
1
)] += PathPoint.distanceInMetersBetweenEarthCoordinates(
trainPoints[i]
[locIndex].departurePoint.latitude,
trainPoints[i]
[locIndex].departurePoint.longitude,
trainPoints[i][locIndex].destination.latitude,
trainPoints[i][locIndex].destination.longitude)
except:
exc = 0
d, _ = Plotter.calculateTracesLength([length], False)
count = Plotter.countInIntervals(
d, Plotter.trainIntervals.tracesLength)
s = sum(count)
if not s == 0:
count = [c1 / s for c1 in count]
sx = sx + count
d, _ = Plotter.calculatePauseTimes(
[[[], [x]] for x in trainParamWithoutGlue[i][:c]],
minPauseTime, maxPauseTime, levyCoeff, False)
count = Plotter.countInIntervals(
d, Plotter.trainIntervals.pauseTimes)
s = sum(count)
if not s == 0:
count = [c1 / s for c1 in count]
sx = sx + count
stopData.append(sx)
stopDataY.append(0)
self.fit(self.stopModel,
np.array(stopData),
np.array(stopDataY),
batchSize=3)
totalInside = 0
totalOutside = 0
for i in range(len(data)):
if len(data[i]) < self.window:
continue
for j in range(len(data[i]) - self.window):
res = self.glueArrays(trainXWithoutGlue[i][j:j + self.window])
currLoc = trainXWithoutGlue[i][j + self.window - 1].index(1)
nextLoc = trainXWithoutGlue[i][j + self.window].index(1)
betweenLocationsX.append(
res + trainParamWithoutGlue[i][j + self.window - 1])
if not currLoc == nextLoc:
totalOutside += 1
betweenLocationsY.append([0, 1])
else:
totalInside += 1
betweenLocationsY.append([1, 0])
print('totalInside: ' + str(totalInside) + ' totalOutside: ' +
str(totalOutside))
self.fit(self.betweenLocationsModel,
np.array(betweenLocationsX),
np.array(betweenLocationsY),
batchSize=2)
def generate(self, locations, parameters, countArr, firstPointsArray,
minPauseTime, maxPauseTime, levyCoeff):
totalInside = 0
totalOutside = 0
globalResult = []
betweenLocsRes = []
#model = Model(inputs=self.model.input, outputs=self.model.layers[-1].input)
for i in range(len(locations)):
result = locations[i].copy()
#result.append(locations[i][len(locations[i])-1])
timeResult = parameters[i].copy()
pointsResult = []
i1 = 0
startPoints = locations[i].copy()
timePoints = parameters[i].copy()
firstPoints = firstPointsArray[i].copy()
isStop = False
if (len(timePoints) < self.window - 1):
print('not enough points')
continue
blr = []
blt = []
blp = []
for j in range(len(locations[i]) - 1):
if (not locations[i][j].index(1)
== locations[i][j + 1].index(1)):
blr.append(locations[i][j].copy())
blr.append(locations[i][j + 1].copy())
blt.append(parameters[i][j].copy())
blp.append(firstPointsArray[i][j])
blp.append(firstPointsArray[i][j + 1])
while (i1 < 10 and not isStop == True): #countArr[i]):
print(str(i) + ':' + str(i1 + 1) + ' from ' + str(countArr[i]))
glue = self.glueArrays(startPoints)
betweenLocations = self.betweenLocationsModel.predict(
[glue + timePoints[len(timePoints) - 1]])
betweenLocations = list(betweenLocations[0])
print('Inside: ' + str(betweenLocations[0]) + ' Outside: ' +
str(betweenLocations[1]))
betweenLocations = betweenLocations.index(
max(betweenLocations))
a = [0] * self.locationsCount
outside = False
if betweenLocations == 1:
res = self.model.predict([glue])
res = list(res[0])
rand = random()
res.pop(startPoints[len(startPoints) - 1].index(1))
dictionary = dict(enumerate(res))
dictionary = sorted(dictionary.items(), key=lambda x: x[1])
selected = -1
for k, v in dictionary:
if v > rand:
if not res.index(max(res)) == k:
print('next location is not max')
selected = k
break
if selected == -1:
selected = res.index(max(res))
totalOutside += 1
blr.append(startPoints[len(startPoints) - 1].copy())
outside = True
else:
selected = startPoints[len(startPoints) - 1].index(1)
totalInside += 1
a[selected] = 1
point = LocationsCalculator.getNextPointBetweenPointAndLocation(
firstPoints[len(firstPoints) - 1], selected)
if (outside == True):
blr.append(a.copy())
blp.append(firstPoints[len(firstPoints) - 1])
blp.append(point)
firstPoints.append(point)
res = a
startPoints.append(res)
result.append(res)
startPoints.pop(0)
glue = self.glueArrays(timePoints) + glue
res = self.timeModel.predict(np.array([glue]))
res = list(res[0])
if res[1] < 0:
res[1] = 0
print('something strange')
print(str(timePoints))
if res[0] < 0:
res[0] = 0
print('something very strange')
print(str(timePoints))
if (outside == True):
blt.append(res.copy())
timePoints.append(res)
timeResult.append(res)
timePoints.pop(0)
stopEnter = result[-self.stopWindow:]
stopEnter = self.glueArrays(stopEnter)
traceTime = 0
for tr in timeResult:
traceTime = traceTime + tr[0] + tr[1]
stopEnter.append(traceTime / Plotter.maxTraceTime)
length = [0] * self.locationsCount
for locIndex in range(len(result) - 1):
if (result[locIndex].index(1) == result[locIndex +
1].index(1)):
length[result[locIndex].index(
1
)] += PathPoint.distanceInMetersBetweenEarthCoordinates(
firstPoints[locIndex].latitude,
firstPoints[locIndex].longitude,
firstPoints[locIndex + 1].latitude,
firstPoints[locIndex + 1].longitude)
d, _ = Plotter.calculateTracesLength([length], False)
count = Plotter.countInIntervals(
d, Plotter.trainIntervals.tracesLength)
s = sum(count)
if not s == 0:
count = [c / s for c in count]
stopEnter = stopEnter + count
d, _ = Plotter.calculatePauseTimes([[[], [x]]
for x in timeResult],
minPauseTime, maxPauseTime,
levyCoeff, False)
count = Plotter.countInIntervals(
d, Plotter.trainIntervals.pauseTimes)
s = sum(count)
if not s == 0:
count = [c / s for c in count]
stopEnter = stopEnter + count
stop = list(self.stopModel.predict(np.array([stopEnter
]))[0])[0]
print('stop: ' + str(stop))
#if stop > 0.5:
# rand = random()
# if stop > rand:
# isStop = True
rand = random()
if stop > rand:
isStop = True
#if stop > 0.5:
# rand = random()
# if rand <= stop:
# break
i1 = i1 + 1
globalResult.append([result, timeResult, firstPoints])
betweenLocsRes.append([blr, blt, blp])
with open('lastGenerated' + str(i) + '.txt', 'w+') as f:
json.dump(
{
"Locations": result,
"Time": np.array(timeResult).tolist(),
"Points": firstPoints
},
f,
default=CentralPoint.serialize)
#with open('lastGenerated' + str(i) + '.txt', 'r') as f:
# obj = json.load(f, object_hook=CentralPoint.deserialize)
# m = obj[0]
# m1 = obj[1]
# m2 = obj[2]
#countOfErrors = self.checkGeneration(result)
print('totalInside: ' + str(totalInside) + ' totalOutside: ' +
str(totalOutside))
return globalResult #betweenLocsRes#