class Ui_MainWindow(QtWidgets.QMainWindow):
def setupUi(self):
self.setObjectName("MainWindow")
self.resize(850, 550)
self.setWindowTitle("Rastrigin")
self.centralwidget = QtWidgets.QWidget(self)
self.centralwidget.setObjectName("centralwidget")
self.frameChart = QChartView(self.centralwidget)
self.frameChart.setGeometry(QtCore.QRect(10, 10, 620, 500))
self.frameChart.setFrameShape(QtWidgets.QFrame.Box)
self.frameChart.setFrameShadow(QtWidgets.QFrame.Sunken)
self.frameChart.setRenderHint(QPainter.Antialiasing)
self.frameChart.setObjectName("frameChart")
self.genParams = QtWidgets.QGroupBox(self.centralwidget)
self.genParams.setGeometry(QtCore.QRect(650, 10, 161, 110))
self.genParams.setObjectName("genParams")
self.genParams.setTitle("General parameters")
self.label1 = QtWidgets.QLabel(self.genParams)
self.label1.setGeometry(QtCore.QRect(10, 20, 61, 16))
self.label1.setObjectName("label1")
self.label1.setText("Population:")
self.label2 = QtWidgets.QLabel(self.genParams)
self.label2.setGeometry(QtCore.QRect(10, 50, 91, 16))
self.label2.setObjectName("label2")
self.label2.setText("No. generations:")
self.label3 = QtWidgets.QLabel(self.genParams)
self.label3.setGeometry(QtCore.QRect(10, 80, 81, 16))
self.label3.setObjectName("label3")
self.label3.setText("No. dimensions:")
self.tbxPopulation = QtWidgets.QLineEdit(self.genParams)
self.tbxPopulation.setGeometry(QtCore.QRect(100, 20, 51, 20))
self.tbxPopulation.setObjectName("tbxPopulation")
self.tbxGenerations = QtWidgets.QLineEdit(self.genParams)
self.tbxGenerations.setGeometry(QtCore.QRect(100, 50, 51, 20))
self.tbxGenerations.setObjectName("tbxGenerations")
self.tbxDimensions = QtWidgets.QLineEdit(self.genParams)
self.tbxDimensions.setGeometry(QtCore.QRect(100, 80, 51, 20))
self.tbxDimensions.setObjectName("tbxDimensions")
self.gaParams = QtWidgets.QGroupBox(self.centralwidget)
self.gaParams.setGeometry(QtCore.QRect(650, 130, 191, 105))
self.gaParams.setObjectName("gaParams")
self.gaParams.setTitle("GA parameters")
self.label4 = QtWidgets.QLabel(self.gaParams)
self.label4.setGeometry(QtCore.QRect(10, 20, 61, 16))
self.label4.setObjectName("label4")
self.label4.setText("Mutation:")
self.label5 = QtWidgets.QLabel(self.gaParams)
self.label5.setGeometry(QtCore.QRect(10, 50, 91, 16))
self.label5.setObjectName("label5")
self.label5.setText("Elite members:")
self.label9 = QtWidgets.QLabel(self.gaParams)
self.label9.setGeometry(QtCore.QRect(10, 80, 61, 16))
self.label9.setObjectName("label9")
self.label9.setText("Max abs.:")
self.tbxMutation = QtWidgets.QLineEdit(self.gaParams)
self.tbxMutation.setGeometry(QtCore.QRect(100, 20, 51, 20))
self.tbxMutation.setObjectName("tbxMutation")
self.tbxElite = QtWidgets.QLineEdit(self.gaParams)
self.tbxElite.setGeometry(QtCore.QRect(100, 50, 51, 20))
self.tbxElite.setObjectName("tbxElite")
self.tbxMaxAbs = QtWidgets.QLineEdit(self.gaParams)
self.tbxMaxAbs.setGeometry(QtCore.QRect(100, 80, 51, 20))
self.tbxMaxAbs.setObjectName("tbxMAxAbs")
self.psoParams = QtWidgets.QGroupBox(self.centralwidget)
self.psoParams.setGeometry(QtCore.QRect(650, 240, 161, 110))
self.psoParams.setObjectName("psoParams")
self.psoParams.setTitle("PSO parameters")
self.label6 = QtWidgets.QLabel(self.psoParams)
self.label6.setGeometry(QtCore.QRect(10, 20, 61, 16))
self.label6.setObjectName("label6")
self.label6.setText("Inertia factor:")
self.label7 = QtWidgets.QLabel(self.psoParams)
self.label7.setGeometry(QtCore.QRect(10, 50, 91, 16))
self.label7.setObjectName("label7")
self.label7.setText("Personal factor:")
self.label8 = QtWidgets.QLabel(self.psoParams)
self.label8.setGeometry(QtCore.QRect(10, 80, 81, 16))
self.label8.setObjectName("label8")
self.label8.setText("Social factor:")
self.tbxInertia = QtWidgets.QLineEdit(self.psoParams)
self.tbxInertia.setGeometry(QtCore.QRect(100, 20, 51, 20))
self.tbxInertia.setObjectName("tbxInertia")
self.tbxPersonal = QtWidgets.QLineEdit(self.psoParams)
self.tbxPersonal.setGeometry(QtCore.QRect(100, 50, 51, 20))
self.tbxPersonal.setObjectName("tbxPersonal")
self.tbxSocial = QtWidgets.QLineEdit(self.psoParams)
self.tbxSocial.setGeometry(QtCore.QRect(100, 80, 51, 20))
self.tbxSocial.setObjectName("tbxSocial")
self.cbxNoVis = QtWidgets.QCheckBox(self.centralwidget)
self.cbxNoVis.setGeometry(QtCore.QRect(650, 350, 170, 17))
self.cbxNoVis.setObjectName("cbxNoVis")
self.cbxNoVis.setText("No visualization per generation")
self.btnStartGA = QtWidgets.QPushButton(self.centralwidget)
self.btnStartGA.setGeometry(QtCore.QRect(650, 370, 75, 23))
self.btnStartGA.setObjectName("btnStartGA")
self.btnStartGA.setText("Start GA")
self.btnStartPSO = QtWidgets.QPushButton(self.centralwidget)
self.btnStartPSO.setGeometry(QtCore.QRect(650, 400, 75, 23))
self.btnStartPSO.setObjectName("btnStartPSO")
self.btnStartPSO.setText("Start PSO")
self.btnStop = QtWidgets.QPushButton(self.centralwidget)
self.btnStop.setEnabled(False)
self.btnStop.setGeometry(QtCore.QRect(740, 370, 75, 53))
self.btnStop.setObjectName("btnStop")
self.btnStop.setText("Stop")
self.btnSaveChart = QtWidgets.QPushButton(self.centralwidget)
self.btnSaveChart.setGeometry(QtCore.QRect(650, 450, 121, 41))
self.btnSaveChart.setObjectName("btnSaveChart")
self.btnSaveChart.setText("Save chart as image")
self.btnSaveChartSeries = QtWidgets.QPushButton(self.centralwidget)
self.btnSaveChartSeries.setGeometry(QtCore.QRect(650, 500, 121, 41))
self.btnSaveChartSeries.setObjectName("btnSaveChartSeries")
self.btnSaveChartSeries.setText("Save chart as series")
self.setCentralWidget(self.centralwidget)
QtCore.QMetaObject.connectSlotsByName(self)
#Connect events
self.btnStartGA.clicked.connect(self.btnStartGA_Click)
self.btnStartPSO.clicked.connect(self.btnStartPSO_Click)
self.btnStop.clicked.connect(self.btnStop_Click)
self.btnSaveChart.clicked.connect(self.btnSaveChart_CLick)
self.btnSaveChartSeries.clicked.connect(self.btnSaveChartSeries_Click)
#Set default variables
self.tbxGenerations.insert(str(NGEN))
self.tbxPopulation.insert(str(POP_SIZE))
self.tbxDimensions.insert(str(NO_DIMS))
self.tbxMutation.insert(str(GA_MUTPB))
self.tbxElite.insert(str(GA_NELT))
self.tbxMaxAbs.insert(str(GA_MAX_ABS))
self.tbxInertia.insert(str(PSO_INERTIA))
self.tbxPersonal.insert(str(PSO_PERSONAL))
self.tbxSocial.insert(str(PSO_SOCIAL))
def btnStartGA_Click(self):
global combination_series # List of lists containing min_series of 5 results -- Added by Denis Lazor
global parameter_name # Name of parameter used for writing its data to .csv file -- Added by Denis Lazor
global best_fit_values # List containing best fitness values for every of 5 experiments per combination -- Added by Denis Lazor
global DIM_SIZES
global ELITE_SIZES
global MAX_ABS_SIZES
global MUTATION_SIZES
# Checking if files are empty or not -- Added by Denis Lazor
csv_contains_ga = os.stat("graphs_csv/original_ga.csv").st_size != 0
# Clearing non empty files if we are trying to write to them -- Added by Denis Lazor
if csv_contains_ga:
clear_all_csv("ga")
parameter_name = "original"
n = 5000
print("GA:\n")
# Automation for all necessary combinations -- Added by Denis Lazor
for d in DIM_SIZES:
MUTATION_SIZES = [0.05, 0.1, 0.2]
ELITE_SIZES = [4, 8, 16]
MAX_ABS_SIZES = [0.4]
for m in MUTATION_SIZES:
for e in ELITE_SIZES:
for ma in MAX_ABS_SIZES:
for i in range(5):
# Set global variables
global stop_evolution
global q_min_series
global q_max_series
global q_avg_series
stop_evolution = False
q_min_series.clear()
q_max_series.clear()
q_avg_series.clear()
# Set global variables from information on UI
global NGEN
global POP_SIZE
global GA_MUTPB
global GA_NELT
global GA_MAX_ABS
NGEN = int(self.tbxGenerations.text())
POP_SIZE = int(self.tbxPopulation.text())
GA_MUTPB = m
GA_NELT = e
GA_MAX_ABS = ma
####Initialize deap GA objects####
# Make creator that minimize. If it would be 1.0 instead od -1.0 than it would be maxmize
self.creator = creator
self.creator.create("FitnessMin",
base.Fitness,
weights=(-1.0, ))
# Create an individual (a blueprint for cromosomes) as a list with a specified fitness type
self.creator.create(
"Individual",
list,
fitness=self.creator.FitnessMin)
# Create base toolbox for finishing creation of a individual (cromosome)
self.toolbox = base.Toolbox()
# Define what type of data (number, gene) will it be in the cromosome
self.toolbox.register("attr_float", random.uniform,
F_MIN, F_MAX)
# Initialization procedure (initRepeat) for the cromosome. For the individual to be completed we need to run initRepeat for the amaout of genes the cromosome includes
self.toolbox.register("individual",
tools.initRepeat,
self.creator.Individual,
self.toolbox.attr_float,
n=NO_DIMS)
# Create a population of individuals (cromosomes). The population is then created by toolbox.population(n=300) where 'n' is the number of cromosomes in population
self.toolbox.register("population",
tools.initRepeat, list,
self.toolbox.individual)
# Register evaluation function
self.toolbox.register("evaluate", evaluateInd)
# Register what genetic operators to use
# Standard coding
self.toolbox.register(
"mate", tools.cxTwoPoint
) # Use two point recombination
self.toolbox.register("mutate",
tools.mutGaussian,
mu=0,
sigma=GA_MAX_ABS,
indpb=0.5)
self.toolbox.register(
"select", tools.selTournament,
tournsize=3) # Use tournament selection
##################################
# Generate initial poplation. Will be a member variable so we can easely pass everything to new thread
self.pop = self.toolbox.population(n=POP_SIZE)
# Evaluate initial population, we map() the evaluation function to every individual and then assign their respective fitness, map runs evaluate function for each individual in pop
fitnesses = list(
map(self.toolbox.evaluate, self.pop))
for ind, fit in zip(self.pop, fitnesses):
ind.fitness.values = fit # Assign calcualted fitness value to individuals
# Extracting all the fitnesses of all individuals in a population so we can monitor and evovlve the algorithm until it reaches 0 or max number of generation is reached
self.fits = [
ind.fitness.values[0] for ind in self.pop
]
# Disable start and enable stop
self.btnStartGA.setEnabled(False)
self.btnStartPSO.setEnabled(False)
self.btnStop.setEnabled(False)
self.genParams.setEnabled(False)
self.gaParams.setEnabled(False)
self.psoParams.setEnabled(False)
self.cbxNoVis.setEnabled(False)
# Start evolution
self.evolveGA()
# Best fitness value -- Added by Denis Lazor
best_fit = np.array(min(best_fit_values))[0]
mean_fit = np.array(
min(best_fit_values,
key=lambda x: abs(x - statistics.mean(
np.asarray(best_fit_values).flatten())))
)[0]
# Index of best fitness value -- Added by Denis Lazor
mean_fit_idx = best_fit_values.index(mean_fit)
write_to_file(combination_series[mean_fit_idx],
parameter_name, "ga")
# First name will be "original", second one "max_abs" -- Added by Denis Lazor
parameter_name = "max_abs"
print_results_GA(POP_SIZE, m, e, ma, mean_fit,
best_fit, NGEN, d)
# Clearing past lists -- Added by Denis Lazor
combination_series = []
best_fit_values = []
# Reducing number of combinations and changing .csv file for writing -- Added by Denis Lazor
MAX_ABS_SIZES = MAX_ABS_SIZES[0:1]
parameter_name = "elites"
ELITE_SIZES = ELITE_SIZES[0:1]
parameter_name = "mutation"
MUTATION_SIZES = MUTATION_SIZES[0:1]
parameter_name = "original"
def btnStartPSO_Click(self):
global combination_series # List of lists containing min_series of 5 results -- Added by Denis Lazor
global parameter_name # Name of parameter used for writing its data to .csv file -- Added by Denis Lazor
global best_fit_values # List containing best fitness values for every of 5 experiments per combination -- Added by Denis Lazor
global DIM_SIZES
global INERTIA_SIZES
global PERSONAL_F_SIZES
global SOCIAL_F_SIZES
# Checking if files are empty or not -- Added by Denis Lazor
csv_contains_pso = os.stat("graphs_csv/original_pso.csv").st_size != 0
# Clearing non empty files if we are trying to write to them -- Added by Denis Lazor
if csv_contains_pso:
clear_all_csv("pso")
n = 5000
parameter_name = "original"
print("PSO:\n")
# Automation for all necessary combinations -- Added by Denis Lazor
for d in DIM_SIZES:
INERTIA_SIZES = [0.0, 0.37, 0.74]
PERSONAL_F_SIZES = [0.5, 1.0, 1.5]
SOCIAL_F_SIZES = [0.5, 1.0, 1.5]
for in_f in INERTIA_SIZES:
for pers_f in PERSONAL_F_SIZES:
for soc_f in SOCIAL_F_SIZES:
for i in range(5):
# Set global variables
global stop_evolution
global q_min_series
global q_max_series
global q_avg_series
stop_evolution = False
q_min_series.clear()
q_max_series.clear()
q_avg_series.clear()
# Set global variables from information on UI
global NGEN
global POP_SIZE
global PSO_INERTIA
global PSO_PERSONAL
global PSO_SOCIAL
NGEN = int(self.tbxGenerations.text())
POP_SIZE = int(self.tbxPopulation.text())
PSO_INERTIA = in_f
PSO_PERSONAL = pers_f
PSO_SOCIAL = soc_f
####Initialize deap PSO objects####
# Make creator that minimize. If it would be 1.0 instead od -1.0 than it would be maxmize
self.creator = creator
self.creator.create("FitnessMin",
base.Fitness,
weights=(-1.0, ))
# Create an individual (a blueprint for cromosomes) as a list with a specified fitness type
self.creator.create(
"Particle",
list,
fitness=self.creator.FitnessMin,
speed=list,
best=None)
# Create base toolbox for finishing creation of a individual (particle) and population
self.toolbox = base.Toolbox()
# Particle initialization
self.toolbox.register("particle",
generateParticle,
cr=self.creator,
size=NO_DIMS,
min_val=F_MIN,
max_val=F_MAX)
# Create a population of individuals (particles). The population is then created by e.g. toolbox.population(n=300) where 'n' is the number of particles in population
self.toolbox.register("population",
tools.initRepeat, list,
self.toolbox.particle)
# Update function for each particle
self.toolbox.register("update", updateParticle)
# Evaluation function for each particle
self.toolbox.register("evaluate", evaluateInd)
##################################
# Create population
self.pop = self.toolbox.population(n=POP_SIZE)
# Evaluate initial population, we map() the evaluation function to every individual and then assign their respective fitness, map runs emaluet function for each individual in pop
fitnesses = list(
map(self.toolbox.evaluate, self.pop))
for ind, fit in zip(self.pop, fitnesses):
ind.fitness.values = fit
# Extracting all the fitnesses of all individuals in a population so we can monitor and evovlve the algorithm until it reaches 0 or max number of generation is reached
self.fits = [
ind.fitness.values[0] for ind in self.pop
]
# Extraction current best position
self.global_best_position = tools.selBest(
self.pop, 1)[0][:]
# Disable start and enable stop
self.btnStartGA.setEnabled(False)
self.btnStartPSO.setEnabled(False)
self.btnStop.setEnabled(False)
self.genParams.setEnabled(False)
self.gaParams.setEnabled(False)
self.psoParams.setEnabled(False)
self.cbxNoVis.setEnabled(False)
# Start evolution
self.evolvePSO()
# Best fitness value -- Added by Denis Lazor
best_fit = np.array(min(best_fit_values))[0]
mean_fit = np.array(
min(best_fit_values,
key=lambda x: abs(x - statistics.mean(
np.asarray(best_fit_values).flatten())))
)[0]
# Index of best fitness value -- Added by Denis Lazor
mean_fit_idx = best_fit_values.index(mean_fit)
write_to_file(combination_series[mean_fit_idx],
parameter_name, "pso")
# First name will be "original", second one "social_factor" -- Added by Denis Lazor
parameter_name = "social_factor"
print_results_PSO(POP_SIZE, in_f, pers_f, soc_f,
mean_fit, best_fit, NGEN, d)
# Clearing past lists -- Added by Denis Lazor
print(best_fit_values)
combination_series = []
best_fit_values = []
# Reducing number of combinations and changing .csv file for writing -- Added by Denis Lazor
SOCIAL_F_SIZES = SOCIAL_F_SIZES[0:1]
parameter_name = "personal_factor"
PERSONAL_F_SIZES = PERSONAL_F_SIZES[0:1]
parameter_name = "inertia"
INERTIA_SIZES = INERTIA_SIZES[0:1]
parameter_name = "original"
def btnStop_Click(self):
global stop_evolution
stop_evolution = True
#Disable stop and enable start
self.btnStartGA.setEnabled(True)
self.btnStartPSO.setEnabled(True)
self.btnStop.setEnabled(False)
self.genParams.setEnabled(True)
self.gaParams.setEnabled(True)
self.psoParams.setEnabled(True)
self.cbxNoVis.setEnabled(True)
#Function for GA evolution
def evolveGA(self):
global q_min_series
global q_max_series
global q_avg_series
global combination_series
global best_fit_values
combination_current_series = [
] # Clearing fitness values series -- Added by Denis Lazor
# Variable for keeping track of the number of generations
curr_g = 0
# Begin the evolution till goal is reached or max number of generation is reached
while min(self.fits) != 0 and curr_g < NGEN:
#Check if evolution and thread need to stop
if stop_evolution:
break #Break the evolution loop
# A new generation
curr_g = curr_g + 1
# print("-- Generation %i --" % curr_g)
# Select the next generation individuals
#Select POP_SIZE - NELT number of individuals. Since recombination is between neigbours, not two naighbours should be the clone of the same individual
offspring = []
offspring.append(self.toolbox.select(
self.pop, 1)[0]) #add first selected individual
for i in range(
POP_SIZE - GA_NELT - 1
): # -1 because the first seleceted individual is already added
while True:
new_o = self.toolbox.select(self.pop, 1)[0]
if new_o != offspring[len(
offspring
) - 1]: #if it is different than the last inserted then add to offspring and break
offspring.append(new_o)
break
# Clone the selected individuals because all of the changes are inplace
offspring = list(map(self.toolbox.clone, offspring))
# Apply crossover on the selected offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
self.toolbox.mate(child1, child2) #inplace recombination
#Invalidate new children fitness values
del child1.fitness.values
del child2.fitness.values
#Apply mutation on the offspring
for mutant in offspring:
if random.random() < GA_MUTPB:
self.toolbox.mutate(mutant)
del mutant.fitness.values
#Add elite individuals #Is clonning needed?
offspring.extend(
list(map(self.toolbox.clone, tools.selBest(self.pop,
GA_NELT))))
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = map(self.toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
# print(" Evaluated %i individuals" % len(invalid_ind))
#Replace population with offspring
self.pop[:] = offspring
# Gather all the fitnesses in one list and print the stats
self.fits = [ind.fitness.values[0] for ind in self.pop]
length = len(self.pop)
mean = sum(self.fits) / length
sum2 = sum(x * x for x in self.fits)
std = abs(sum2 / length - mean**2)**0.5
q_min_series.append(curr_g, min(self.fits))
q_max_series.append(curr_g, max(self.fits))
q_avg_series.append(curr_g, mean)
combination_current_series.append(
min(self.fits)
) # Saving min_series fitness values of an experiment -- Added by Denis Lazor
# print(" Min %s" % q_min_series.at(q_min_series.count()-1).y())
# print(" Max %s" % q_max_series.at(q_max_series.count()-1).y())
# print(" Avg %s" % mean)
# print(" Std %s" % std)
#
if self.cbxNoVis.isChecked():
app.processEvents()
else:
self.chart = QChart()
self.chart.addSeries(q_min_series)
self.chart.addSeries(q_max_series)
self.chart.addSeries(q_avg_series)
self.chart.setTitle("Fitness value over time")
self.chart.setAnimationOptions(QChart.NoAnimation)
self.chart.createDefaultAxes()
self.frameChart.setChart(self.chart)
self.frameChart.repaint()
app.processEvents()
#Printing best individual
best_ind = tools.selBest(self.pop, 1)[0]
# print("Best individual is %s, %s" % (best_ind, best_ind.fitness.values))
#Visulaize final solution
if self.cbxNoVis.isChecked():
self.chart = QChart()
self.chart.addSeries(q_min_series)
self.chart.addSeries(q_max_series)
self.chart.addSeries(q_avg_series)
self.chart.setTitle("Fitness value over time")
self.chart.setAnimationOptions(QChart.NoAnimation)
self.chart.createDefaultAxes()
self.frameChart.setChart(self.chart)
self.frameChart.repaint()
#Disable stop and enable start
self.btnStartGA.setEnabled(True)
self.btnStartPSO.setEnabled(True)
self.btnStop.setEnabled(False)
self.genParams.setEnabled(True)
self.gaParams.setEnabled(True)
self.psoParams.setEnabled(True)
self.cbxNoVis.setEnabled(True)
app.processEvents()
combination_series.append(combination_current_series
) # Saving 5 results -- Added by Denis Lazor
best_fit_values.append(
best_ind.fitness.values
) # Adding best fitness value of experiment -- Added by Denis Lazor
#Function for GA evolution
def evolvePSO(self):
global q_min_series
global q_max_series
global q_avg_series
global combination_series
global best_fit_values
combination_current_series = [
] # Clearing fitness values series -- Added by Denis Lazor
# Variable for keeping track of the number of generations
curr_g = 0
while min(self.fits) != 0.0 and curr_g < NGEN:
#Check if evolution and thread need to stop
if stop_evolution:
break #Break the evolution loop
# A new generation
curr_g = curr_g + 1
# print("-- Generation %i --" % curr_g)
#Update particle position and evaluate particle
for particle in self.pop:
#Update
self.toolbox.update(particle, self.global_best_position,
PSO_INERTIA, PSO_PERSONAL, PSO_SOCIAL)
#Evaluate
fit = self.toolbox.evaluate(particle)
#Update best position
if fit[0] < particle.fitness.values[0]:
particle.best = particle[:]
#Update fitness
particle.fitness.values = fit
#Extracting all the fitnesses of all individuals in a population so we can monitor and evovlve the algorithm until it reaches 0 or max number of generation is reached
self.fits = [ind.fitness.values[0] for ind in self.pop]
#Extraction current best position
self.global_best_position = tools.selBest(self.pop, 1)[0][:]
#Stats
length = len(self.pop)
mean = sum(self.fits) / length
sum2 = sum(x * x for x in self.fits)
std = abs(sum2 / length - mean**2)**0.5
q_min_series.append(curr_g, min(self.fits))
q_max_series.append(curr_g, max(self.fits))
q_avg_series.append(curr_g, mean)
combination_current_series.append(
min(self.fits)
) # Saving min_series fitness values of an experiment -- Added by Denis Lazor
# print(" Min %s" % q_min_series.at(q_min_series.count()-1).y())
# print(" Max %s" % q_max_series.at(q_max_series.count()-1).y())
# print(" Avg %s" % mean)
# print(" Std %s" % std)
#
if self.cbxNoVis.isChecked():
app.processEvents()
else:
self.chart = QChart()
self.chart.addSeries(q_min_series)
self.chart.addSeries(q_max_series)
self.chart.addSeries(q_avg_series)
self.chart.setTitle("Fitness value over time")
self.chart.setAnimationOptions(QChart.NoAnimation)
self.chart.createDefaultAxes()
self.frameChart.setChart(self.chart)
self.frameChart.repaint()
app.processEvents()
#Printing best individual
best_ind = tools.selBest(self.pop, 1)[0]
# print("Best individual is %s, %s" % (best_ind, best_ind.fitness.values))
#Visulaize final solution
if self.cbxNoVis.isChecked():
self.chart = QChart()
self.chart.addSeries(q_min_series)
self.chart.addSeries(q_max_series)
self.chart.addSeries(q_avg_series)
self.chart.setTitle("Fitness value over time")
self.chart.setAnimationOptions(QChart.NoAnimation)
self.chart.createDefaultAxes()
self.frameChart.setChart(self.chart)
self.frameChart.repaint()
#Disable stop and enable start
self.btnStartGA.setEnabled(True)
self.btnStartPSO.setEnabled(True)
self.btnStop.setEnabled(False)
self.genParams.setEnabled(True)
self.gaParams.setEnabled(True)
self.psoParams.setEnabled(True)
self.cbxNoVis.setEnabled(True)
app.processEvents()
combination_series.append(combination_current_series
) # Saving 5 results -- Added by Denis Lazor
best_fit_values.append(
best_ind.fitness.values
) # Adding best fitness value of experiment -- Added by Denis Lazor
def btnSaveChart_CLick(self):
p = self.frameChart.grab()
filename, _ = QFileDialog.getSaveFileName(
None, "Save series chart as a image", "", "Image Files (*.png)")
p.save(filename, "PNG")
print("Chart series image saved to: ", filename)
def btnSaveChartSeries_Click(self):
global q_min_series
global q_max_series
global q_avg_series
filename, _ = QFileDialog.getSaveFileName(None,
"Save series to text file",
"",
"Text Files (*.txt, *.csv)")
with open(filename, 'w') as dat:
for i in range(q_min_series.count()):
dat.write('%f,%f,%f\n' %
(q_min_series.at(i).y(), q_avg_series.at(i).y(),
q_max_series.at(i).y()))
print("Chart series saved to: ", filename)
class Ui_MainWindow(QtWidgets.QMainWindow):
def setupUi(self):
self.setObjectName("MainWindow")
self.resize(600, 830)
self.setWindowTitle("GA - Queens")
self.centralwidget = QtWidgets.QWidget(self)
self.centralwidget.setObjectName("centralwidget")
self.frameWorld = MyQFrame(self.centralwidget)
self.frameWorld.img = QPixmap(1000, 1000)
self.frameWorld.setGeometry(QtCore.QRect(10, 10, 400, 400))
self.frameWorld.setFrameShape(QtWidgets.QFrame.Box)
self.frameWorld.setFrameShadow(QtWidgets.QFrame.Sunken)
self.frameWorld.setObjectName("frameWorld")
self.frameChart = QChartView(self.centralwidget)
self.frameChart.setGeometry(QtCore.QRect(10, 420, 400, 400))
self.frameChart.setFrameShape(QtWidgets.QFrame.Box)
self.frameChart.setFrameShadow(QtWidgets.QFrame.Sunken)
self.frameChart.setRenderHint(QPainter.Antialiasing)
self.frameChart.setObjectName("frameChart")
self.gaParams = QtWidgets.QGroupBox(self.centralwidget)
self.gaParams.setGeometry(QtCore.QRect(430, 10, 161, 171))
self.gaParams.setObjectName("gaParams")
self.gaParams.setTitle("GA parameters")
self.label1 = QtWidgets.QLabel(self.gaParams)
self.label1.setGeometry(QtCore.QRect(10, 20, 61, 16))
self.label1.setObjectName("label1")
self.label1.setText("Population:")
self.label2 = QtWidgets.QLabel(self.gaParams)
self.label2.setGeometry(QtCore.QRect(10, 50, 47, 16))
self.label2.setObjectName("label2")
self.label2.setText("Mutation:")
self.label3 = QtWidgets.QLabel(self.gaParams)
self.label3.setGeometry(QtCore.QRect(10, 80, 81, 16))
self.label3.setObjectName("label3")
self.label3.setText("Elite members:")
self.label4 = QtWidgets.QLabel(self.gaParams)
self.label4.setGeometry(QtCore.QRect(10, 110, 91, 16))
self.label4.setObjectName("label4")
self.label4.setText("No. generations:")
self.cbxPermutation = QtWidgets.QCheckBox(self.gaParams)
self.cbxPermutation.setGeometry(QtCore.QRect(35, 140, 91, 17))
self.cbxPermutation.setObjectName("cbxPermutation")
self.cbxPermutation.setText("Permutation")
self.tbxPopulation = QtWidgets.QLineEdit(self.gaParams)
self.tbxPopulation.setGeometry(QtCore.QRect(100, 20, 51, 20))
self.tbxPopulation.setObjectName("tbxPopulation")
self.tbxMutation = QtWidgets.QLineEdit(self.gaParams)
self.tbxMutation.setGeometry(QtCore.QRect(100, 50, 51, 20))
self.tbxMutation.setObjectName("tbxMutation")
self.tbxElite = QtWidgets.QLineEdit(self.gaParams)
self.tbxElite.setGeometry(QtCore.QRect(100, 80, 51, 20))
self.tbxElite.setObjectName("tbxElite")
self.tbxGenerations = QtWidgets.QLineEdit(self.gaParams)
self.tbxGenerations.setGeometry(QtCore.QRect(100, 110, 51, 20))
self.tbxGenerations.setObjectName("tbxGenerations")
self.label5 = QtWidgets.QLabel(self.centralwidget)
self.label5.setGeometry(QtCore.QRect(440, 190, 61, 16))
self.label5.setObjectName("label5")
self.label5.setText("No. queens:")
self.tbxNoQueens = QtWidgets.QLineEdit(self.centralwidget)
self.tbxNoQueens.setGeometry(QtCore.QRect(510, 190, 51, 20))
self.tbxNoQueens.setObjectName("tbxNoQueens")
self.cbxNoVis = QtWidgets.QCheckBox(self.centralwidget)
self.cbxNoVis.setGeometry(QtCore.QRect(420, 215, 170, 17))
self.cbxNoVis.setObjectName("cbxNoVis")
self.cbxNoVis.setText("No visualization per generation")
self.btnStart = QtWidgets.QPushButton(self.centralwidget)
self.btnStart.setGeometry(QtCore.QRect(430, 250, 75, 23))
self.btnStart.setObjectName("btnStart")
self.btnStart.setText("Start")
self.btnStop = QtWidgets.QPushButton(self.centralwidget)
self.btnStop.setEnabled(False)
self.btnStop.setGeometry(QtCore.QRect(510, 250, 75, 23))
self.btnStop.setObjectName("btnStop")
self.btnStop.setText("Stop")
self.btnSaveWorld = QtWidgets.QPushButton(self.centralwidget)
self.btnSaveWorld.setGeometry(QtCore.QRect(430, 370, 121, 41))
self.btnSaveWorld.setObjectName("btnSaveWorld")
self.btnSaveWorld.setText("Save world as image")
self.btnSaveChart = QtWidgets.QPushButton(self.centralwidget)
self.btnSaveChart.setGeometry(QtCore.QRect(430, 730, 121, 41))
self.btnSaveChart.setObjectName("btnSaveChart")
self.btnSaveChart.setText("Save chart as image")
self.btnSaveChartSeries = QtWidgets.QPushButton(self.centralwidget)
self.btnSaveChartSeries.setGeometry(QtCore.QRect(430, 780, 121, 41))
self.btnSaveChartSeries.setObjectName("btnSaveChartSeries")
self.btnSaveChartSeries.setText("Save chart as series")
self.setCentralWidget(self.centralwidget)
QtCore.QMetaObject.connectSlotsByName(self)
#Connect events
self.btnStart.clicked.connect(self.btnStart_Click)
self.btnStop.clicked.connect(self.btnStop_Click)
self.btnSaveWorld.clicked.connect(self.btnSaveWorld_Click)
self.btnSaveChart.clicked.connect(self.btnSaveChart_CLick)
self.btnSaveChartSeries.clicked.connect(self.btnSaveChartSeries_Click)
#Set default GA variables
self.tbxNoQueens.insert(str(NO_QUEENS))
self.tbxGenerations.insert(str(NGEN))
self.tbxPopulation.insert(str(POP_SIZE))
self.tbxMutation.insert(str(MUTPB))
self.tbxElite.insert(str(NELT))
self.new_image = QPixmap(1000, 1000)
def btnStart_Click(self):
global success # Number of times when fitness function reached 0 -- Added by Denis Lazor
global generations # Number of times when fitness function reached 0 -- Added by Denis Lazor
global combination_series # List of lists containing min_series of 5 correct results -- Added by Denis Lazor
global parameter_name # Name of parameter used for writing its data to .csv file -- Added by Denis Lazor
global ELITE_SIZES
global BOARD_SIZES
global POPULATION_SIZES
global MUTATION_SIZES
# Checking if files are empty or not -- Added by Denis Lazor
csv_contains = os.stat("graphs_csv/original_.csv").st_size != 0
csv_contains_permutation = os.stat(
"graphs_csv/original_permutation.csv").st_size != 0
permutation_checked = self.cbxPermutation.isChecked()
# Clearing non empty files if we are trying to write to them -- Added by Denis Lazor
if csv_contains_permutation and permutation_checked:
clear_all_csv("p")
if csv_contains and not permutation_checked:
clear_all_csv("np")
BOARD_SIZES = [12, 24]
n = 5000
# Automation for all necessary combinations -- Added by Denis Lazor
for b in BOARD_SIZES:
# Because we use slicing we need to restore parameters lists after changing board size
POPULATION_SIZES = [50, 100, 200]
MUTATION_SIZES = [0.04, 0.08, 0.16]
ELITE_SIZES = [4, 8, 16]
for p in POPULATION_SIZES:
for m in MUTATION_SIZES:
for e in ELITE_SIZES:
success = 0
trials = 0 # Number of tries needed to find 5 'correct' results -- Added by Denis Lazor
while success < 5: # Doing same combination till we get 5 'correct' results -- Added by Denis Lazor
trials = trials + 1
# Set global variables
global stop_evolution
global q_min_series
global q_max_series
global q_avg_series
stop_evolution = False
q_min_series.clear()
q_max_series.clear()
q_avg_series.clear()
# Set global variables from information on UI
global NO_QUEENS
global NGEN
global POP_SIZE
global MUTPB
global NELT
NO_QUEENS = b
NGEN = n
POP_SIZE = p
MUTPB = m
NELT = e
# Painting chess table
self.img = QPixmap(1000, 1000)
self.img.fill()
painter = QPainter(self.img)
painter.setPen(QPen(Qt.black, 10, Qt.SolidLine))
width = 1000 / NO_QUEENS
cur_width = 0
for i in range(
NO_QUEENS + 1
): # +1 in order to draw the last line as well
painter.drawLine(cur_width, 0, cur_width, 1000)
painter.drawLine(0, cur_width, 1000, cur_width)
cur_width += width
painter.end()
self.frameWorld.img = self.img
# Redrawing frames
self.frameWorld.repaint()
app.processEvents()
####Initialize deap GA objects####
# Make creator that minimize. If it would be 1.0 instead od -1.0 than it would be maxmize
creator.create("FitnessMin",
base.Fitness,
weights=(-1.0, ))
# Create an individual (a blueprint for cromosomes) as a list with a specified fitness type
creator.create("Individual",
list,
fitness=creator.FitnessMin)
# Create base toolbox for finishing creation of a individual (cromosome)
self.toolbox = base.Toolbox()
# Define what type of data (number, gene) will it be in the cromosome
if self.cbxPermutation.isChecked():
# Permutation coding
self.toolbox.register("indices", random.sample,
range(NO_QUEENS),
NO_QUEENS)
# initIterate requires that the generator of genes (such as random.sample) generates an iterable (a list) variable
self.toolbox.register("individual",
tools.initIterate,
creator.Individual,
self.toolbox.indices)
else:
# Standard coding
self.toolbox.register(
"attr_int", random.randint, 0,
NO_QUEENS - 1
) # number in cromosome is from 0 till IND_SIZE - 1
# Initialization procedure (initRepeat) for the cromosome. For the individual to be completed we need to run initRepeat for the amaout of genes the cromosome includes
self.toolbox.register("individual",
tools.initRepeat,
creator.Individual,
self.toolbox.attr_int,
n=NO_QUEENS)
# Create a population of individuals (cromosomes). The population is then created by toolbox.population(n=300) where 'n' is the number of cromosomes in population
self.toolbox.register("population",
tools.initRepeat, list,
self.toolbox.individual)
# Register evaluation function
self.toolbox.register("evaluate", evaluateInd)
# Register what genetic operators to use
if self.cbxPermutation.isChecked():
# Permutation coding
self.toolbox.register(
"mate",
tools.cxUniformPartialyMatched,
indpb=0.2
) # Use uniform recombination for permutation coding
self.toolbox.register("mutate",
tools.mutShuffleIndexes,
indpb=0.2)
else:
# Standard coding
self.toolbox.register(
"mate", tools.cxTwoPoint
) # Use two point recombination
self.toolbox.register(
"mutate",
tools.mutUniformInt,
low=0,
up=NO_QUEENS - 1,
indpb=0.2) # 20% that the gene will change
self.toolbox.register(
"select", tools.selTournament,
tournsize=3) # Use tournament selection
##################################
# Generate initial poplation. Will be a member variable so we can easely pass everything to new thread
self.pop = self.toolbox.population(n=POP_SIZE)
# Evaluate initial population, we map() the evaluation function to every individual and then assign their respective fitness, map runs evaluate function for each individual in pop
fitnesses = list(
map(self.toolbox.evaluate, self.pop))
for ind, fit in zip(self.pop, fitnesses):
ind.fitness.values = fit # Assign calcualted fitness value to individuals
# Extracting all the fitnesses of all individuals in a population so we can monitor and evovlve the algorithm until it reaches 0 or max number of generation is reached
self.fits = [
ind.fitness.values[0] for ind in self.pop
]
self.fits
# Disable start and enable stop
self.btnStart.setEnabled(False)
self.btnStop.setEnabled(True)
self.gaParams.setEnabled(False)
self.tbxNoQueens.setEnabled(False)
self.cbxNoVis.setEnabled(False)
# Start evolution
self.evolve()
# Mean number of generations nedeed for finding 5 correct solutions -- Added by Denis Lazor
mean_gen = min(generations,
key=lambda x: abs(x - statistics.mean(
generations)))
# Index of mean_gen value -- Added by Denis Lazor
mean_idx = generations.index(mean_gen)
write_to_file(
combination_series[mean_idx], parameter_name,
permutation_checked
) # First name will be "original", second one "elites" -- Added by Denis Lazor
parameter_name = "elites"
print_results(b, p, m, e, trials, generations,
mean_gen, NGEN)
# Clearing past lists -- Added by Denis Lazor
generations = []
combination_series = []
# Reducing number of combinations and changing .csv file for writing -- Added by Denis Lazor
ELITE_SIZES = ELITE_SIZES[0:1]
parameter_name = "mutation"
MUTATION_SIZES = MUTATION_SIZES[0:1]
parameter_name = "population"
POPULATION_SIZES = POPULATION_SIZES[0:1]
parameter_name = "original"
n = 30000 # Increasing generation size for 24x24 board
def btnStop_Click(self):
global stop_evolution
stop_evolution = True
#Disable stop and enable start
self.btnStop.setEnabled(False)
self.btnStart.setEnabled(True)
self.gaParams.setEnabled(True)
self.tbxNoQueens.setEnabled(True)
self.cbxNoVis.setEnabled(True)
#Function for GA evolution
def evolve(self):
global q_min_series
global q_max_series
global q_avg_series
global success
global generations
global combination_series
global NO_QUEENS
global NGEN
global POP_SIZE
global MUTPB
global NELT
combination_current_series = []
# Variable for keeping track of the number of generations
curr_g = 0
# Begin the evolution till goal is reached or max number of generation is reached
while min(self.fits) != 0 and curr_g < NGEN:
#Check if evolution and thread need to stop
if stop_evolution:
break #Break the evolution loop
# A new generation
curr_g = curr_g + 1
#print("-- Generation %i --" % curr_g)
# Select the next generation individuals
#Select POP_SIZE - NELT number of individuals. Since recombination is between neigbours, not two naighbours should be the clone of the same individual
offspring = []
offspring.append(self.toolbox.select(
self.pop, 1)[0]) #add first selected individual
for i in range(
POP_SIZE - NELT - 1
): # -1 because the first seleceted individual is already added
while True:
new_o = self.toolbox.select(self.pop, 1)[0]
if new_o != offspring[len(
offspring
) - 1]: #if it is different than the last inserted then add to offspring and break
offspring.append(new_o)
break
# Clone the selected individuals because all of the changes are inplace
offspring = list(map(self.toolbox.clone, offspring))
# Apply crossover on the selected offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
self.toolbox.mate(child1, child2) #inplace recombination
#Invalidate new children fitness values
del child1.fitness.values
del child2.fitness.values
#Apply mutation on the offspring
for mutant in offspring:
if random.random() < MUTPB:
self.toolbox.mutate(mutant)
del mutant.fitness.values
#Add elite individuals #Is clonning needed?
offspring.extend(
list(map(self.toolbox.clone, tools.selBest(self.pop, NELT))))
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = map(self.toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
#print(" Evaluated %i individuals" % len(invalid_ind))
#Replace population with offspring
self.pop[:] = offspring
# Gather all the fitnesses in one list and print the stats
self.fits = [ind.fitness.values[0] for ind in self.pop]
length = len(self.pop)
mean = sum(self.fits) / length
sum2 = sum(x * x for x in self.fits)
std = abs(sum2 / length - mean**2)**0.5
q_min_series.append(curr_g, min(self.fits))
q_max_series.append(curr_g, max(self.fits))
q_avg_series.append(curr_g, mean)
combination_current_series.append(
min(self.fits)
) # Saving min_series fitness values of an experiment -- Added by Denis Lazor
# Checking if fitness value of 0 is reached -- Added by Denis Lazor
for f in self.fits:
if f == 0:
success = success + 1
generations.append(curr_g)
combination_series.append(combination_current_series)
break
#print(" Min %s" % q_min_series.at(q_min_series.count()-1).y())
#print(" Max %s" % q_max_series.at(q_max_series.count()-1).y())
#print(" Avg %s" % mean)
#print(" Std %s" % std)
if self.cbxNoVis.isChecked():
app.processEvents()
else:
#Draw queen positions of best individual on a image
best_ind = tools.selBest(self.pop, 1)[0]
self.updateWorldFrame(generateQueenImage(best_ind))
self.chart = QChart()
#self.chart.addSeries(q_min_series)
#self.chart.addSeries(q_max_series)
#q_avg_series.setName("Board: " + str(b) + " Population: " + str(p) + " Elite: " + str(e) + " Mutation:" + str(m*100) + "% " + "Generations:" + str(NGSEN))
self.chart.addSeries(q_avg_series)
self.chart.setTitle("QN: " + str(NO_QUEENS) + " POP: " +
str(POP_SIZE) + " EL: " + str(NELT) +
" MT: " + str(MUTPB * 100) + "% ")
self.chart.setAnimationOptions(QChart.NoAnimation)
self.chart.createDefaultAxes()
self.frameChart.setChart(self.chart)
#Printing best individual
best_ind = tools.selBest(self.pop, 1)[0]
#print("Best individual is %s, %s \n" % (best_ind, best_ind.fitness.values))
#Visulaize final solution
if self.cbxNoVis.isChecked():
#Draw queen positions of best individual on a image
best_ind = tools.selBest(self.pop, 1)[0]
self.updateWorldFrame(generateQueenImage(best_ind))
self.chart = QChart()
#self.chart.addSeries(q_min_series)
#self.chart.addSeries(q_max_series)
self.chart.addSeries(q_avg_series)
self.chart.setTitle("QN: " + str(NO_QUEENS) + " POP: " +
str(POP_SIZE) + " EL: " + str(NELT) + " MT: " +
str(MUTPB * 100) + "% ")
self.chart.setAnimationOptions(QChart.NoAnimation)
self.chart.createDefaultAxes()
self.frameChart.setChart(self.chart)
#Disable stop and enable start
self.btnStop.setEnabled(False)
self.btnStart.setEnabled(True)
self.gaParams.setEnabled(True)
self.tbxNoQueens.setEnabled(True)
self.cbxNoVis.setEnabled(True)
def updateWorldFrame(self, queens_img):
#new_image = QPixmap(1000,1000)
self.new_image.fill() #White color is default
painter = QPainter(self.new_image)
#First draw the table
painter.drawPixmap(self.new_image.rect(), self.img)
#Then draw the queens
painter.drawImage(self.new_image.rect(), queens_img)
painter.end()
#Set new image to the frame
self.frameWorld.img = self.new_image
#Redrawing frames
self.frameWorld.repaint()
self.frameChart.repaint()
app.processEvents()
def btnSaveWorld_Click(self):
filename, _ = QFileDialog.getSaveFileName(None,
"Save world as a image", "",
"Image Files (*.png)")
self.frameWorld.img.save(filename, "PNG")
print("World image saved to: ", filename)
def btnSaveChart_CLick(self):
p = self.frameChart.grab()
filename, _ = QFileDialog.getSaveFileName(
None, "Save series chart as a image", "", "Image Files (*.png)")
p.save(filename, "PNG")
print("Chart series image saved to: ", filename)
def btnSaveChartSeries_Click(self):
global q_min_series
global q_max_series
global q_avg_series
filename, _ = QFileDialog.getSaveFileName(None,
"Save series to text file",
"",
"Text Files (*.txt, *.csv)")
with open(filename, 'w') as dat:
for i in range(q_min_series.count()):
dat.write('%f,%f,%f\n' %
(q_min_series.at(i).y(), q_avg_series.at(i).y(),
q_max_series.at(i).y()))
print("Chart series saved to: ", filename)
class Ui(mainwindow.Ui_MainWindow):
def __init__(self, MainWindow):
#super(Ui, self).__init__(M)
super(Ui, self).setupUi(MainWindow)
#uic.loadUi('UI/mainwindow.ui', self)
self.MainWindow = MainWindow
self.initialize()
def close(self):
self.MainWindow.close()
def style(self):
return self.MainWindow.style()
def initialize(self):
self.tabWidget.setCurrentIndex(0)
self.actionExit.triggered.connect(self.close)
self.action_Plot.setEnabled(False)
self.actionNext.setIcon(
self.style().standardIcon(
QtWidgets.QStyle.SP_ArrowForward))
self.actionPrevious.setIcon(
self.style().standardIcon(
QtWidgets.QStyle.SP_ArrowBack))
self.action_Open.setIcon(
self.style().standardIcon(
QtWidgets.QStyle.SP_DialogOpenButton))
self.actionSave.setIcon(
self.style().standardIcon(
QtWidgets.QStyle.SP_DriveFDIcon))
self.actionSave.triggered.connect(self.save)
self.action_Open.triggered.connect(self.getOpenFilename)
self.actionNext.triggered.connect(self.nextPacket)
self.actionPrevious.triggered.connect(self.previousPacket)
self.actionAbout.triggered.connect(self.about)
self.actionPrevious.setEnabled(False)
self.actionNext.setEnabled(False)
self.actionSave.setEnabled(False)
self.action_Plot.setEnabled(False)
self.actionPaste.triggered.connect(self.onPasteTriggered)
# self.actionLog.triggered.connect(self.dockWidget_2.show)
self.actionSet_IDB.triggered.connect(self.onSetIDBClicked)
self.plotButton.clicked.connect(self.onPlotButtonClicked)
self.exportButton.clicked.connect(self.onExportButtonClicked)
self.action_Plot.triggered.connect(self.onPlotActionClicked)
self.actionLoad_mongodb.triggered.connect(self.onLoadMongoDBTriggered)
self.mdb=None
self.current_row = 0
self.data=[]
self.x=[]
self.y=[]
self.xlabel='x'
self.ylabel='y'
self.chart = QChart()
self.chart.layout().setContentsMargins(0,0,0,0)
self.chart.setBackgroundRoundness(0)
self.savePlotButton.clicked.connect(self.savePlot)
self.chartView = QChartView(self.chart)
self.gridLayout.addWidget(self.chartView, 1, 0, 1, 15)
# IDB location
self.settings = QtCore.QSettings('FHNW', 'stix_parser')
def onExportButtonClicked(self):
if self.y:
filename = str(QtWidgets.QFileDialog.getSaveFileName(
None, "Save file", "", "*.csv")[0])
if filename:
with open(filename,'w') as f:
f.write('{},{}\n'.format(self.xlabel,self.ylabel))
for xx,yy in zip(self.x,self.y):
f.write('{},{}\n'.format(xx,yy))
self.showMessage('The data has been written to {}'.format(filename))
else:
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText('Plot first!')
msgBox.setWindowTitle("Warning")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def savePlot(self):
#if self.figure.get_axes():
if self.chart:
filename = str(QtWidgets.QFileDialog.getSaveFileName(
None, "Save file", "", "*.png *.jpg")[0])
if filename:
if not filename.endswith(('.png','.jpg')):
filename+='.png'
#self.figure.savefig(filename)
p=self.chartView.grab()
p.save(filename)
self.showMessage(('Saved to %s.' % filename))
else:
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText('The canvas is empty!')
msgBox.setWindowTitle("STIX DATA VIEWER")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def onPasteTriggered(self):
pass
def showMessageBox(self, message, content):
msg = QtWidgets.QMessageBox()
msg.setIcon(QtWidgets.QMessageBox.Critical)
msg.setText("Error")
msg.setInformativeText(message)
msg.setWindowTitle("Error")
msg.setDetailedText(content)
msg.setStandardButtons(QtWidgets.QMessageBox.Ok |
QtWidgets.QMessageBox.Cancel)
retval = msg.exec_()
def showMessage(self, msg):
# if destination != 1:
self.statusbar.showMessage(msg)
# if destination !=0 :
# self.listWidget_2.addItem(msg)
def onSetIDBClicked(self):
pass
def save(self):
pass
def setListViewSelected(self, row):
#index = self.model.createIndex(row, 0);
# if index.isValid():
# self.model.selectionModel().select( index, QtGui.QItemSelectionModel.Select)
pass
def about(self):
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText("STIX raw data parser and viewer, [email protected]")
msgBox.setWindowTitle("Stix data viewer")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def nextPacket(self):
self.current_row += 1
length=len(self.data)
if self.current_row>=length:
self.current_row=length-1
self.showMessage('No more packet!')
self.showPacket(self.current_row)
self.setListViewSelected(self.current_row)
def previousPacket(self):
self.current_row -= 1
if self.current_row <0:
self.current_row=0
self.showMessage('Reach the first packet!')
self.showPacket(self.current_row)
self.setListViewSelected(self.current_row)
def getOpenFilename(self):
pass
def openFile(self, filename):
pass
def onDataLoaded(self, data,clear=True):
if not clear:
self.data.append(data)
else:
self.data = data
self.displayPackets(clear)
if self.data:
self.actionPrevious.setEnabled(True)
self.actionNext.setEnabled(True)
self.actionSave.setEnabled(True)
self.action_Plot.setEnabled(True)
def displayPackets(self,clear=True):
if clear:
self.packetTreeWidget.clear()
t0=0
for p in self.data:
if type(p) is not dict:
continue
header = p['header']
root = QtWidgets.QTreeWidgetItem(self.packetTreeWidget)
if t0==0:
t0=header['time']
root.setText(0, '{:.2f}'.format(header['time']-t0))
root.setText(1,
('TM({},{}) - {}').format(header['service_type'],
header['service_subtype'],
header['DESCR']))
self.total_packets = len(self.data)
self.showMessage((('%d packets loaded') % (self.total_packets)))
self.packetTreeWidget.currentItemChanged.connect(self.onPacketSelected)
self.showPacket(0)
def onLoadMongoDBTriggered(self):
diag=QtWidgets.QDialog()
diag_ui=mongo_dialog.Ui_Dialog()
diag_ui.setupUi(diag)
diag_ui.pushButton.setFocus(True)
#self.settings = QtCore.QSettings('FHNW', 'stix_parser')
self.mongo_server= self.settings.value('mongo_server', [], str)
self.mongo_port= self.settings.value('mongo_port', [], str)
self.mongo_user= self.settings.value('mongo_user', [], str)
self.mongo_pwd= self.settings.value('mongo_pwd', [], str)
if self.mongo_server:
diag_ui.serverLineEdit.setText(self.mongo_server)
if self.mongo_port:
diag_ui.portLineEdit.setText(self.mongo_port)
if self.mongo_user:
diag_ui.userLineEdit.setText(self.mongo_user)
if self.mongo_pwd:
diag_ui.pwdLineEdit.setText(self.mongo_pwd)
diag_ui.pushButton.clicked.connect(partial(self.loadRunsFromMongoDB,diag_ui))
diag_ui.buttonBox.accepted.connect(partial(self.loadDataFromMongoDB,diag_ui,diag))
diag.exec_()
def loadRunsFromMongoDB(self,dui):
server=dui.serverLineEdit.text()
port=dui.portLineEdit.text()
user=dui.userLineEdit.text()
pwd=dui.pwdLineEdit.text()
self.showMessage('saving setting...')
if self.mongo_server!=server:
self.settings.setValue('mongo_server', server)
if self.mongo_port!=port:
self.settings.setValue('mongo_port', port)
if self.mongo_user!=user:
self.settings.setValue('mongo_user', user)
if self.mongo_pwd!=pwd:
self.settings.setValue('mongo_pwd', pwd)
self.showMessage('connecting Mongo database ...')
self.mdb=mgdb.MongoDB(server,int(port),user,pwd)
dui.treeWidget.clear()
self.showMessage('Fetching data...')
for run in self.mdb.get_runs():
root = QtWidgets.QTreeWidgetItem(dui.treeWidget)
root.setText(0, str(run['_id']))
root.setText(1, run['file'])
root.setText(2, run['date'])
root.setText(3, str(run['start']))
root.setText(4, str(run['end']))
self.showMessage('Runs loaded!')
def loadDataFromMongoDB(self,dui,diag):
selected_runs=[]
for item in dui.treeWidget.selectedItems():
selected_runs.append(item.text(0))
if not selected_runs:
self.showMessage('Run not selected!')
if selected_runs:
diag.done(0)
self.showMessage('Loading data ...!')
data=self.mdb.get_packets(selected_runs[0])
if data:
self.onDataLoaded(data,clear=True)
else:
self.showMessage('No packets found!')
#close
def onPacketSelected(self, cur, pre):
self.current_row = self.packetTreeWidget.currentIndex().row()
self.showMessage((('Packet #%d selected') % self.current_row))
self.showPacket(self.current_row)
def showPacket(self, row):
if not self.data:
return
header = self.data[row]['header']
self.showMessage(
(('Packet %d / %d %s ') %
(row, self.total_packets, header['DESCR'])))
self.paramTreeWidget.clear()
header_root = QtWidgets.QTreeWidgetItem(self.paramTreeWidget)
header_root.setText(0, "Header")
rows = len(header)
for key, val in header.items():
root = QtWidgets.QTreeWidgetItem(header_root)
root.setText(0, key)
root.setText(1, str(val))
params = self.data[row]['parameters']
param_root = QtWidgets.QTreeWidgetItem(self.paramTreeWidget)
param_root.setText(0, "Parameters")
self.showParameterTree(params, param_root)
self.paramTreeWidget.expandItem(param_root)
self.paramTreeWidget.expandItem(header_root)
def showParameterTree(self, params, parent):
for p in params:
root = QtWidgets.QTreeWidgetItem(parent)
if not p:
continue
try:
param_name=p['name']
desc=''
scos_desc=''
try:
desc=param_desc.PCF[param_name]
scos_desc=param_desc.SW[param_name]
except KeyError:
pass
root.setToolTip(1,scos_desc)
root.setText(0, param_name)
root.setText(1, desc)
root.setText(2, str(p['raw']))
root.setText(3, str(p['value']))
if 'child' in p:
if p['child']:
self.showParameterTree(p['child'], root)
except KeyError:
self.showMessage(
('[Error ]: keyError occurred when adding parameter'))
self.paramTreeWidget.itemDoubleClicked.connect(self.onTreeItemClicked)
def walk(self, name, params, header, ret_x, ret_y, xaxis=0, data_type=0):
if not params:
return
timestamp = header['time']
#parameters=[p for p in params if p['name'] == name]
for p in params:
if type(p) is not dict:
continue
#for p in parameters:
if name == p['name']:
values = None
#print('data type:{}'.format(data_type))
if data_type == 0:
values = p['raw']
else:
values = p['value']
try:
yvalue = None
if (type(values) is tuple) or (type(values) is list):
yvalue = float(values[0])
else:
yvalue = float(values)
ret_y.append(yvalue)
if xaxis == 1:
ret_x.append(timestamp)
else:
self.showMessage((('Can not plot %s ') % str(yvalue)))
except Exception as e:
self.showMessage((('%s ') % str(e)))
if 'child' in p:
if p['child']:
self.walk(
name,
p['child'],
header,
ret_x,
ret_y,
xaxis,
data_type)
def onPlotButtonClicked(self):
if self.chart:
self.chart.removeAllSeries()
if not self.data:
return
self.showMessage('Preparing plot ...')
name = self.paramNameEdit.text()
packet_selection = self.comboBox.currentIndex()
xaxis_type = self.xaxisComboBox.currentIndex()
data_type = self.dataTypeComboBox.currentIndex()
timestamp = []
self.y = []
packet_id = self.current_row
params = self.data[packet_id]['parameters']
header = self.data[packet_id]['header']
current_spid=header['SPID']
if packet_selection == 0:
self.walk(
name,
params,
header,
timestamp,
self.y,
xaxis_type,
data_type)
elif packet_selection == 1:
for packet in self.data:
header = packet['header']
if packet['header']['SPID'] != current_spid:
continue
#only look for parameters in the packets of the same type
params = packet['parameters']
self.walk(
name,
params,
header,
timestamp,
self.y,
xaxis_type,
data_type)
self.x = []
if not self.y:
self.showMessage('No data points')
elif self.y:
style = self.styleEdit.text()
if not style:
style = '-'
title = '%s' % str(name)
desc = self.descLabel.text()
if desc:
title += '- %s' % desc
self.chart.setTitle(title)
ylabel = 'Raw value'
xlabel = name
if data_type == 1:
ylabel = 'Engineering value'
if xaxis_type == 0:
xlabel = "Packet #"
self.x = range(0, len(self.y))
if xaxis_type == 1:
self.x = [t - timestamp[0] for t in timestamp]
xlabel = 'Time -T0 (s)'
#if xaxis_type != 2:
if True:
series = QLineSeries()
series2 = None
# print(y)
# print(x)
for xx, yy in zip(self.x, self.y):
series.append(xx, yy)
if 'o' in style:
series2 = QScatterSeries()
for xx, yy in zip(self.x, self.y):
series2.append(xx, yy)
self.chart.addSeries(series2)
self.chart.addSeries(series)
self.showMessage('plotted!')
#self.chart.createDefaultAxes()
axisX = QValueAxis()
axisX.setTitleText(xlabel)
axisY = QValueAxis()
axisY.setTitleText(ylabel)
self.chart.setAxisX(axisX)
self.chart.setAxisY(axisY)
series.attachAxis(axisX)
series.attachAxis(axisY)
# histogram
#else:
# nbins = len(set(self.y))
# ycounts, xedges = np.histogram(self.y, bins=nbins)
# series = QLineSeries()
# for i in range(0, nbins):
# meanx = (xedges[i] + xedges[i + 1]) / 2.
# series.append(meanx, ycounts[i])
# # series.append(dataset)
# self.chart.addSeries(series)
# #self.chart.createDefaultAxes()
# self.showMessage('Histogram plotted!')
# axisX = QValueAxis()
# axisX.setTitleText(name)
# axisY = QValueAxis()
# axisY.setTitleText("Counts")
# self.chart.setAxisY(axisY)
# self.chart.setAxisX(axisX)
## series.attachAxis(axisX)
# series.attachAxis(axisY)
# self.widget.setChart(self.chart)
self.xlabel=xlabel
self.ylabel=ylabel
self.chartView.setRubberBand(QChartView.RectangleRubberBand)
self.chartView.setRenderHint(QtGui.QPainter.Antialiasing)
def plotParameter(self, name=None, desc=None):
self.tabWidget.setCurrentIndex(1)
if name:
self.paramNameEdit.setText(name)
if desc:
self.descLabel.setText(desc)
def onPlotActionClicked(self):
self.tabWidget.setCurrentIndex(1)
self.plotParameter()
def onTreeItemClicked(self, it, col):
#print(it, col, it.text(0))
self.plotParameter(it.text(0), it.text(1))
def error(self, msg, description=''):
self.showMessage((('Error: %s - %s') % (msg, description)))
def warning(self, msg, description=''):
self.showMessage((('Warning: %s - %s') % (msg, description)))
def info(self, msg, description=''):
self.showMessage((('Info: %s - %s') % (msg, description)))
class Ui(mainwindow.Ui_MainWindow):
def __init__(self, MainWindow):
super(Ui, self).setupUi(MainWindow)
self.MainWindow = MainWindow
self.socketPacketReceiver = None
self.timmer_is_on = False
self.hexParser = StixHexStringParser()
self.socketPacketReceiver = StixSocketPacketReceiver()
self.socketPacketServer = StixSocketPacketServer()
self.dataReader = StixFileReader()
slots = {
'info': self.onDataReaderInfo,
'warning': self.onDataReaderWarning,
'error': self.onDataReaderError,
'critical': self.onDataReaderCritical,
'dataLoaded': self.onDataReady,
'packetArrival': self.onPacketArrival,
'progress': self.onProgressUpdated
}
self.socketPacketReceiver.connectSignalSlots(slots)
self.dataReader.connectSignalSlots(slots)
self.hexParser.connectSignalSlots(slots)
self.socketPacketServer.connectSignalSlots(slots)
self.tabWidget.setCurrentIndex(0)
self.actionExit.triggered.connect(self.close)
self.actionPlot.setEnabled(False)
self.actionNext.setIcon(self.style().standardIcon(
QtWidgets.QStyle.SP_ArrowForward))
self.actionPrevious.setIcon(self.style().standardIcon(
QtWidgets.QStyle.SP_ArrowBack))
self.actionOpen.setIcon(self.style().standardIcon(
QtWidgets.QStyle.SP_DialogOpenButton))
self.actionSave.setIcon(self.style().standardIcon(
QtWidgets.QStyle.SP_DriveFDIcon))
self.actionSave.triggered.connect(self.save)
self.actionOpen.triggered.connect(self.getOpenFilename)
self.filterPattern = None
self.actionNext.triggered.connect(self.nextPacket)
self.actionPrevious.triggered.connect(self.previousPacket)
self.actionAbout.triggered.connect(self.about)
self.actionPrevious.setEnabled(False)
self.actionNext.setEnabled(False)
self.actionSave.setEnabled(False)
self.actionPlot.setEnabled(False)
self.actionCopy.triggered.connect(self.onCopyTriggered)
self.packetTreeWidget.currentItemChanged.connect(self.onPacketSelected)
self.actionCopy.setEnabled(False)
self.actionPaste.triggered.connect(self.onPasteTriggered)
self.actionLog.triggered.connect(self.dockWidget.show)
self.actionSetIDB.triggered.connect(self.onSetIDBClicked)
self.plotButton.clicked.connect(
partial(self.onPlotButtonClicked, None))
#self.progressBar = QtWidgets.QProgressBar()
#self.statusbar.addPermanentWidget(self.progressBar)
self.progressDiag = None
#self.progressBar.hide()
self.actionPacketServer.triggered.connect(self.startPacketServer)
self.exportButton.clicked.connect(self.onExportButtonClicked)
self.actionPlot.triggered.connect(self.onPlotActionClicked)
self.actionLoadMongodb.triggered.connect(self.onLoadMongoDBTriggered)
self.actionConnectTSC.triggered.connect(self.onConnectTSCTriggered)
self.actionPacketFilter.triggered.connect(self.filter)
self.actionPlugins.triggered.connect(self.onPluginTriggered)
self.actionOnlineHelp.triggered.connect(self.onOnlineHelpTriggered)
self.actionViewBinary.triggered.connect(self.onViewBinaryTriggered)
self.actionTimestampConvertor.triggered.connect(self.onTimestampConvertorTriggered)
#self.actionPythonConsole.triggered.connect(self.startPythonConsole)
self.autoUpdateButton.clicked.connect(
self.onPlotAutoUpdateButtonClicked)
self.packetTreeWidget.customContextMenuRequested.connect(
self.packetTreeContextMenuEvent)
#self.statusListWidget.setContextMenuPolicy(QtCore.Qt.CustomContextMenu)
#self.statusListWidget.customContextMenuRequested.connect(self.statusListContextMenuEvent)
self.mdb = None
self.current_row = 0
self.data = []
self.x = []
self.y = []
self.xlabel = 'x'
self.ylabel = 'y'
self.buttons_enabled = False
self.chart = QChart()
self.chart.layout().setContentsMargins(0, 0, 0, 0)
self.chart.setBackgroundRoundness(0)
self.savePlotButton.clicked.connect(self.savePlot)
self.chartView = QChartView(self.chart)
self.gridLayout.addWidget(self.chartView, 1, 0, 1, 15)
self.selected_services = SELECTED_SERVICES
self.selected_SPID = []
self.selected_tmtc = 3
# IDB location
self.settings = QtCore.QSettings('FHNW', 'stix_parser')
self.idb_filename = self.settings.value('idb_filename', [], str)
if self.idb_filename:
STIX_IDB.reload(self.idb_filename)
if not STIX_IDB.is_connected():
self.showMessage('IDB has not been set!')
else:
idb_filename = STIX_IDB.get_idb_filename()
self.showMessage('IDB loaded from : {} '.format(idb_filename), 1)
if idb_filename != self.idb_filename:
self.settings.setValue('idb_filename', idb_filename)
self.idb_filename = idb_filename
#def startPythonConsole(self):
# console.start({'packets': self.data})
def close(self):
self.MainWindow.close()
def style(self):
return self.MainWindow.style()
def startPacketServer(self):
host = 'localhost'
port = 9096
self.socketPacketServer.connect(host, port)
self.socketPacketServer.setData(self.current_row, self.data)
self.socketPacketServer.start()
abspath = os.path.dirname(os.path.abspath(__file__))
template = (
"import sys\nsys.path.append('{}')\nimport client_packet_request as req\n"
"packets=req.request(query_str='len', host='{}',port={}, verbose_level=1)\n"
"#a query_string can be \n"
"# - a python slice notation, for example, ':' '0:-1', 3:-1\n"
"# - 'len', to get the total number of packets,\n"
"# - index , to get a packet of the given index"
"#set verbose_level to 0, to suppress print output ").format(
abspath, host, port)
cb = QtWidgets.QApplication.clipboard()
cb.clear(mode=cb.Clipboard)
cb.setText(template, mode=cb.Clipboard)
msg = QtWidgets.QMessageBox()
msg.setIcon(QtWidgets.QMessageBox.Information)
msg.setText(
"Packet server started and a template to request packet has been copied to your clipboard!"
)
retval = msg.exec_()
def onPlotAutoUpdateButtonClicked(self):
if not self.timmer_is_on:
if not self.data:
return
num_packets = len(self.data)
if num_packets > 200:
packets = self.data[-200:-1]
else:
packets = self.data
self.timer = QTimer()
self.timer.timeout.connect(
partial(self.onPlotButtonClicked, packets))
self.timer.start(2000)
self.timmer_is_on = True
self.autoUpdateButton.setText('Stop Auto Update')
else:
if self.timer:
self.timer.stop()
self.timmer_is_on = False
self.autoUpdateButton.setText('Start Auto Update')
#def statusListContextMenuEvent(self,pos):
# menu = QtWidgets.QMenu()
# clearLogAction= menu.addAction('Empty log')
# clearLogAction.triggered.connect(self.clearLog)
#def clearLog(self):
# self.statusListWidget.clear()
def packetTreeContextMenuEvent(self, pos):
menu = QtWidgets.QMenu()
filterAction = menu.addAction('Filter')
menu.addSeparator()
rawDataAction = menu.addAction('Raw binary data')
menu.addSeparator()
copyPacketAction = menu.addAction('Copy packet')
menu.addSeparator()
deleteAllAction = menu.addAction('Delete all packets')
self.current_row = self.packetTreeWidget.currentIndex().row()
rawDataAction.triggered.connect(self.onViewBinaryTriggered)
filterAction.triggered.connect(self.filter)
copyPacketAction.triggered.connect(self.onCopyTriggered)
deleteAllAction.triggered.connect(self.onDeleteAllTriggered)
action = menu.exec_(self.packetTreeWidget.viewport().mapToGlobal(pos))
def filter(self):
text, okPressed = QtWidgets.QInputDialog.getText(
None, "Packet filtering",
"Filtering by SPID or description (! to exclude):",
QtWidgets.QLineEdit.Normal, "")
if okPressed:
self.filterPattern = text
self.addPacketsToView(self.data, True, show_stat=False)
def onDeleteAllTriggered(self):
self.data.clear()
self.current_row = 0
self.packetTreeWidget.clear()
self.paramTreeWidget.clear()
#def onPacketTreeItemDoubleClicked(self):
# self.onViewBinaryTriggered()
def onViewBinaryTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = raw_viewer.Ui_Dialog()
diag_ui.setupUi(diag)
if self.data:
try:
raw = self.data[self.current_row]['bin']
header = self.data[self.current_row]['header']
diag_ui.setPacketInfo('{}({},{}) {}'.format(
header['TMTC'], header['service_type'],
header['service_subtype'], header['descr']))
diag_ui.displayRaw(raw)
except (IndexError, KeyError):
diag_ui.setText('Raw data not available.')
diag.exec_()
def onOnlineHelpTriggered(self):
webbrowser.open(
'https://github.com/i4Ds/STIX-python-data-parser', new=2)
def onTimestampConvertorTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = timestamp_convertor.Ui_Dialog()
diag_ui.setupUi(diag)
diag.exec_()
def onPluginTriggered(self):
self.plugin_location = self.settings.value('plugin_location', [], str)
diag = QtWidgets.QDialog()
diag_ui = plugin.Ui_Dialog()
diag_ui.setupUi(diag)
if self.plugin_location:
diag_ui.setPluginLocation(self.plugin_location)
diag_ui.setData(self.data, self.current_row)
diag.exec_()
location = diag_ui.getPluginLocation()
if location != self.plugin_location:
self.settings.setValue('plugin_location', location)
def onPacketFilterTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = packet_filter.Ui_Dialog()
diag_ui.setupUi(diag)
self.filterPattern = '' #empty search string
diag_ui.setSelectedServices(self.selected_services)
diag_ui.buttonBox.accepted.connect(
partial(self.applyServiceFilter, diag_ui))
diag.exec_()
def applyServiceFilter(self, diag_ui):
self.selected_SPID = diag_ui.getSelectedSPID()
self.selected_services = diag_ui.getSelectedServices()
self.selected_tmtc = diag_ui.getTMTC()
self.showMessage('Applying filter...')
self.addPacketsToView(self.data, True, show_stat=False)
def onExportButtonClicked(self):
if self.y:
filename = str(
QtWidgets.QFileDialog.getSaveFileName(
None, "Save data to file", "", "CSV(*.csv)")[0])
if filename:
with open(filename, 'w') as f:
f.write('{},{}\n'.format(self.xlabel, self.ylabel))
for xx, yy in zip(self.x, self.y):
f.write('{},{}\n'.format(xx, yy))
self.showMessage(
'The data has been written to {}'.format(filename))
else:
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText('Plot first!')
msgBox.setWindowTitle("Warning")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def savePlot(self):
# if self.figure.get_axes():
if self.chart:
filetypes = "PNG (*.png);;JPEG (*.jpg)"
filename = str(
QtWidgets.QFileDialog.getSaveFileName(
None, "Save plot to file", "", filetypes)[0])
if filename:
if not filename.endswith(('.png', '.jpg')):
filename += '.png'
# self.figure.savefig(filename)
p = self.chartView.grab()
p.save(filename)
self.showMessage(('Has been saved to %s.' % filename))
else:
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText('No figure to save')
msgBox.setWindowTitle("STIX raw data viewer")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def onCopyTriggered(self):
packet_id = self.current_row
try:
packet = self.data[packet_id]
ss = pprint.pformat(packet)
cb = QtWidgets.QApplication.clipboard()
cb.clear(mode=cb.Clipboard)
cb.setText(ss, mode=cb.Clipboard)
msg = QtWidgets.QMessageBox()
msg.setIcon(QtWidgets.QMessageBox.Information)
msg.setText(
"The data of the selected packet has been copied to the clipboard."
)
msg.setWindowTitle("Information")
msg.setStandardButtons(QtWidgets.QMessageBox.Ok)
retval = msg.exec_()
except Exception as e:
self.showMessage(str(e), 0)
def onPasteTriggered(self):
raw_hex = QtWidgets.QApplication.clipboard().text()
if len(raw_hex) < 16:
self.showMessage('No data in the clipboard.')
return
self.hexParser.setHex(raw_hex)
self.hexParser.start()
def showMessage(self, msg, where=0):
if where != 1:
self.statusbar.showMessage(msg)
if where != 0:
self.statusListWidget.addItem(msg)
def onSetIDBClicked(self):
self.idb_filename = QtWidgets.QFileDialog.getOpenFileName(
None, 'Select file', '.', 'IDB file(*.db *.sqlite *.sqlite3)')[0]
if not self.idb_filename:
return
STIX_IDB.reload(self.idb_filename)
if STIX_IDB.is_connected():
#settings = QtCore.QSettings('FHNW', 'stix_parser')
self.settings.setValue('idb_filename', self.idb_filename)
self.showMessage(
'IDB location: {} '.format(STIX_IDB.get_idb_filename()), 1)
def save(self):
filetypes = 'python compressed pickle (*.pklz);; python pickle file (*.pkl);; binary data (*.dat)'
self.output_filename = str(
QtWidgets.QFileDialog.getSaveFileName(None, "Save packets to", "",
filetypes)[0])
if not self.output_filename.endswith(('.pklz', '.pkl', '.dat')):
msg = 'unsupported file format !'
self.showMessage(msg)
return
msg = 'Writing data to file %s' % self.output_filename
self.showMessage(msg)
if self.output_filename.endswith(('.pklz', '.pkl')):
stw = stix_writer.StixPickleWriter(self.output_filename)
stw.register_run(str(self.input_filename))
stw.write_all(self.data)
elif self.output_filename.endswith('.dat'):
stw = stix_writer.StixBinaryWriter(self.output_filename)
stw.write_all(self.data)
num_ok = stw.get_num_sucess()
msg = (
'The binary data of {} packets written to file {}, total packets {}'
.format(num_ok, self.output_filename, len(self.data)))
self.showMessage(msg)
msg = 'Packets have been written to %s' % self.output_filename
self.showMessage(msg)
def setListViewSelected(self, row):
#index = self.model.createIndex(row, 0);
# if index.isValid():
# self.model.selectionModel().select( index, QtGui.QItemSelectionModel.Select)
pass
def about(self):
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText("STIX raw data parser and viewer, [email protected]")
msgBox.setWindowTitle("Stix data viewer")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def nextPacket(self):
self.current_row += 1
length = len(self.data)
if self.current_row >= length:
self.current_row = length - 1
self.showMessage('No more packet!')
self.showPacket(self.current_row)
self.setListViewSelected(self.current_row)
def previousPacket(self):
self.current_row -= 1
if self.current_row < 0:
self.current_row = 0
self.showMessage('Reach the first packet!')
self.showPacket(self.current_row)
self.setListViewSelected(self.current_row)
def getOpenFilename(self):
location = self.settings.value('location', [], str)
if not location:
location = '.'
filetypes = (
'Supported file (*.dat *.bin *.binary *.pkl *.pklz *.xml *ascii *BDF *txt) ;; All(*)'
)
self.input_filename = QtWidgets.QFileDialog.getOpenFileName(
None, 'Select file', location, filetypes)[0]
if not self.input_filename:
return
self.settings.setValue('location', os.path.abspath(self.input_filename))
diag = QtWidgets.QDialog()
diag_ui = packet_filter.Ui_Dialog()
diag_ui.setupUi(diag)
diag_ui.setSelectedServices(SELECTED_SERVICES)
diag_ui.buttonBox.accepted.connect(
partial(self.onOpenFile, self.input_filename, diag_ui))
diag.exec_()
def onOpenFile(self, input_filename, diag):
self.selected_SPID = diag.getSelectedSPID()
self.selected_services = diag.getSelectedServices()
self.openFile(input_filename, self.selected_services,
self.selected_SPID)
def openFile(self, filename, selected_services=None, selected_SPID=None):
msg = 'Loading file %s ...' % filename
self.progressDiag = QtWidgets.QProgressDialog()
#self.showMessage(msg)
self.progressDiag.setLabelText(msg)
self.progressDiag.setWindowTitle('Loading data')
self.progressDiag.setCancelButtonText('Cancel')
self.progressDiag.setRange(0, 100)
self.progressDiag.setMinimumWidth(300)
self.progressDiag.canceled.connect(self.stopParsing)
self.filterPattern = ''
self.dataReader.setPacketFilter(selected_services, selected_SPID)
self.dataReader.setFilename(filename)
self.dataReader.start()
self.progressDiag.show()
def stopParsing(self):
if self.dataReader:
self.dataReader.stopParsing()
self.progressDiag.hide()
def onProgressUpdated(self, progress):
if not self.progressDiag:
return
self.progressDiag.setValue(progress)
if progress >=99:
self.progressDiag.hide()
def onDataReaderCritical(self, msg):
self.showMessage(msg, 1)
def onDataReaderInfo(self, msg):
self.showMessage(msg, 0)
def onDataReaderWarning(self, msg):
self.showMessage(msg, 1)
def onDataReaderError(self, msg):
self.showMessage(msg, 1)
def onDataReady(self, data, clear=True, show_stat=True):
#self.progressBar.hide()
if not clear:
self.data.extend(data)
else:
self.data = data
if data:
self.addPacketsToView(data, clear=clear, show_stat=show_stat)
self.enableButtons()
else:
self.showMessage('No packet loaded')
def enableButtons(self):
if not self.buttons_enabled:
self.actionPrevious.setEnabled(True)
self.actionNext.setEnabled(True)
self.actionSave.setEnabled(True)
self.actionCopy.setEnabled(True)
self.actionPlot.setEnabled(True)
self.actionViewBinary.setEnabled(True)
self.buttons_enabled = True
def addPacketsToView(self, data, clear=True, show_stat=True):
if clear:
self.packetTreeWidget.clear()
for p in data:
if not isinstance(p, dict):
continue
header = p['header']
root = QtWidgets.QTreeWidgetItem(self.packetTreeWidget)
colors = {2: '#FFA500', 1: '#000080', 3: '#FF0000', 4: '#800000'}
tc_color = '#78281F'
if header['TMTC'] == 'TC':
root.setForeground(0, QtGui.QBrush(QtGui.QColor(tc_color)))
root.setForeground(1, QtGui.QBrush(QtGui.QColor(tc_color)))
else:
if header['service_type'] == 5:
if header['service_subtype'] in colors.keys():
root.setForeground(
0,
QtGui.QBrush(
QtGui.QColor(
colors[header['service_subtype']])))
root.setForeground(
1,
QtGui.QBrush(
QtGui.QColor(
colors[header['service_subtype']])))
timestamp_str = stix_datetime.format_datetime(header['unix_time'])
root.setText(0, timestamp_str)
description = '{}({},{}) - {}'.format(
header['TMTC'], header['service_type'],
header['service_subtype'], header['descr'])
root.setText(1, description)
hidden = False
if self.selected_SPID:
if header['TMTC'] == 'TC':
hidden = True
elif -int(header['SPID']) in self.selected_SPID or int(
header['SPID']) not in self.selected_SPID:
hidden = True
else:
if int(header['service_type']) not in self.selected_services:
hidden = True
TMTC = header['TMTC']
if TMTC == 'TM' and self.selected_tmtc in [2, 0]:
hidden = True
if TMTC == 'TC' and self.selected_tmtc in [1, 0]:
hidden = True
if self.filterPattern:
to_exclude = False
pattern = self.filterPattern.strip()
if pattern.startswith('!'):
to_exclude = True
pattern = pattern[1:]
try:
spid = int(pattern)
hidden = to_exclude == (header['SPID'] == spid)
#XNOR operation
except (TypeError, ValueError):
hidden = to_exclude == (pattern in description)
root.setHidden(hidden)
if show_stat:
total_packets = len(self.data)
self.showMessage(('Total packet(s): %d' % total_packets))
def onConnectTSCTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = tsc_connection.Ui_Dialog()
diag_ui.setupUi(diag)
self.tsc_host = self.settings.value('tsc_host', [], str)
self.tsc_port = self.settings.value('tsc_port', [], str)
if self.tsc_host:
diag_ui.serverLineEdit.setText(self.tsc_host)
if self.tsc_port:
diag_ui.portLineEdit.setText(self.tsc_port)
diag_ui.buttonBox.accepted.connect(partial(self.connectToTSC, diag_ui))
diag.exec_()
def connectToTSC(self, dui):
host = dui.serverLineEdit.text()
port = dui.portLineEdit.text()
self.showMessage('Connecting to TSC...')
self.socketPacketReceiver.connect(host, int(port))
self.socketPacketReceiver.start()
def onPacketArrival(self, packets):
clear = False
if packets:
if len(self.data) > MAX_NUM_PACKET_IN_BUFFER:
clear = True
self.onDataReady(packets, clear=clear, show_stat=True)
def onLoadMongoDBTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = mongo_dialog.Ui_Dialog()
diag_ui.setupUi(diag)
#self.settings = QtCore.QSettings('FHNW', 'stix_parser')
self.mongo_server = self.settings.value('mongo_server', [], str)
self.mongo_port = self.settings.value('mongo_port', [], str)
self.mongo_user = self.settings.value('mongo_user', [], str)
self.mongo_pwd = self.settings.value('mongo_pwd', [], str)
if self.mongo_server:
diag_ui.serverLineEdit.setText(self.mongo_server)
if self.mongo_port:
diag_ui.portLineEdit.setText(self.mongo_port)
if self.mongo_user:
diag_ui.userLineEdit.setText(self.mongo_user)
if self.mongo_pwd:
diag_ui.pwdLineEdit.setText(self.mongo_pwd)
diag_ui.pushButton.clicked.connect(
partial(self.loadRunsFromMongoDB, diag_ui))
diag_ui.buttonBox.accepted.connect(
partial(self.loadDataFromMongoDB, diag_ui, diag))
diag.exec_()
def loadRunsFromMongoDB(self, dui):
server = dui.serverLineEdit.text()
port = dui.portLineEdit.text()
user = dui.userLineEdit.text()
pwd = dui.pwdLineEdit.text()
self.showMessage('saving setting...')
if self.mongo_server != server:
self.settings.setValue('mongo_server', server)
if self.mongo_port != port:
self.settings.setValue('mongo_port', port)
if self.mongo_user != user:
self.settings.setValue('mongo_user', user)
if self.mongo_pwd != pwd:
self.settings.setValue('mongo_pwd', pwd)
self.showMessage('connecting Mongo database ...')
self.mdb = mgdb.MongoDB(server, int(port), user, pwd)
if not self.mdb.is_connected():
self.showMessage('Failed to connect to MongoDB')
return
dui.treeWidget.clear()
self.showMessage('Fetching data...')
for run in self.mdb.select_all_runs():
root = QtWidgets.QTreeWidgetItem(dui.treeWidget)
root.setText(0, str(run['_id']))
root.setText(1, run['filename'])
root.setText(2, stix_datetime.format_datetime(run['date']))
root.setText(3, stix_datetime.format_datetime(run['data_start_unix_time']))
root.setText(4, stix_datetime.format_datetime(run['data_stop_unix_time']))
def loadDataFromMongoDB(self, dui, diag):
self.showMessage('Loading packets ...')
selected_runs = []
for item in dui.treeWidget.selectedItems():
selected_runs.append(item.text(0))
if not selected_runs:
self.showMessage('Run not selected!')
if selected_runs:
diag.done(0)
self.showMessage('Loading data ...!')
data = list(self.mdb.select_packets_by_run(selected_runs[0]))
if data:
self.onDataReady(data, clear=True)
else:
self.showMessage('No packets found!')
# close
def onPacketSelected(self, cur, pre):
self.current_row = self.packetTreeWidget.currentIndex().row()
self.showMessage(('Packet #%d selected' % self.current_row))
self.showPacket(self.current_row)
def showPacket(self, row):
if not self.data:
return
header = self.data[row]['header']
total_packets = len(self.data)
self.showMessage(
('Packet %d / %d %s ' % (row, total_packets, header['descr'])))
self.paramTreeWidget.clear()
header_root = QtWidgets.QTreeWidgetItem(self.paramTreeWidget)
header_root.setText(0, "Header")
rows = len(header)
for key, val in header.items():
root = QtWidgets.QTreeWidgetItem(header_root)
root.setText(0, key)
root.setText(1, str(val))
params = self.data[row]['parameters']
param_root = QtWidgets.QTreeWidgetItem(self.paramTreeWidget)
param_root.setText(0, "Parameters")
self.showParameterTree(params, param_root)
self.paramTreeWidget.expandItem(param_root)
self.paramTreeWidget.expandItem(header_root)
self.current_row = row
def showParameterTree(self, params, parent, parent_id=[]):
if not params:
return
for i, p in enumerate(params):
root = QtWidgets.QTreeWidgetItem(parent)
if not p:
continue
param = Parameter(p)
param_name = param['name']
desc = param['desc']
current_ids = parent_id[:]
current_ids.append(i)
root.setText(0, param_name)
root.setText(1, desc)
root.setText(2, str(param['raw']))
tip='parameter'+''.join(['[{}]'.format(x) for x in current_ids])
root.setToolTip(0, tip)
long_desc = STIX_IDB.get_scos_description(param_name)
if long_desc:
root.setToolTip(1, long_desc)
try:
root.setToolTip(2, hex(param['raw_int']))
except:
pass
unit=STIX_IDB.get_parameter_unit(param_name)
eng=str(param['eng'])
root.setText(3, eng)
root.setText(4, unit)
if 'NIXG' in param_name:
root.setHidden(True)
#groups should not be shown
if param.children:
self.showParameterTree(param['children'], root, current_ids)
self.paramTreeWidget.itemDoubleClicked.connect(self.onTreeItemClicked)
def walk(self, name, params, header, ret_x, ret_y, xaxis=0, data_type=0):
if not params:
return
timestamp = header['unix_time']
for p in params:
if not p:
continue
param = Parameter(p)
if name == param.name:
values = None
if data_type == 0:
values = param.raw
else:
values = param.eng
try:
yvalue = float(values)
ret_y.append(yvalue)
if xaxis == 1:
ret_x.append(timestamp)
else:
self.showMessage(('Can not plot %s ' % str(yvalue)))
except Exception as e:
self.showMessage(('%s ' % str(e)))
if param.children:
self.walk(name, param.children, header, ret_x, ret_y, xaxis,
data_type)
def onPlotButtonClicked(self, packets=None):
if self.chart:
self.chart.removeAllSeries()
if packets is None:
packets = self.data
if not packets:
return
self.showMessage('Preparing plot ...')
name = self.paramNameEdit.text()
packet_selection = self.comboBox.currentIndex()
xaxis_type = self.xaxisComboBox.currentIndex()
data_type = self.dataTypeComboBox.currentIndex()
timestamp = []
self.y = []
params = self.paramNameEdit.text()
header = packets[0]['header']
current_spid = 0
spid_text = self.spidLineEdit.text()
if spid_text:
current_spid = int(spid_text)
selected_packets=[]
if packet_selection == 0:
selected_packets=[packets[self.current_row]]
elif packet_selection == 1:
selected_packets=packets
for packet in selected_packets:
header = packet['header']
if packet['header']['SPID'] != current_spid:
continue
params = packet['parameters']
self.walk(name, params, header, timestamp, self.y, xaxis_type,
data_type)
self.x = []
if not self.y:
self.showMessage('No data points')
elif self.y:
style = self.styleEdit.text()
if not style:
style = '-'
title = '%s' % str(name)
desc = self.descLabel.text()
if desc:
title += '- %s' % desc
self.chart.setTitle(title)
ylabel = 'Raw value'
xlabel = name
if data_type == 1:
ylabel = 'Engineering / Decompressed value'
if xaxis_type == 0:
if packet_selection == 1:
xlabel = "Packet #"
else:
xlabel = "Repeat #"
self.x = range(0, len(self.y))
if xaxis_type == 1:
self.x = [t - timestamp[0] for t in timestamp]
xlabel = 'Time -T0 (s)'
if xaxis_type != 2:
series = QLineSeries()
series2 = None
for xx, yy in zip(self.x, self.y):
series.append(xx, yy)
if 'o' in style:
series2 = QScatterSeries()
for xx, yy in zip(self.x, self.y):
series2.append(xx, yy)
self.chart.addSeries(series2)
self.chart.addSeries(series)
axisX = QValueAxis()
axisX.setTitleText(xlabel)
axisY = QValueAxis()
axisY.setTitleText(ylabel)
self.chart.setAxisX(axisX)
self.chart.setAxisY(axisY)
series.attachAxis(axisX)
series.attachAxis(axisY)
else:
nbins = len(set(self.y))
ycounts, xedges = np.histogram(self.y, bins=nbins)
series = QLineSeries()
for i in range(0, nbins):
meanx = (xedges[i] + xedges[i + 1]) / 2.
series.append(meanx, ycounts[i])
self.chart.addSeries(series)
axisX = QValueAxis()
axisX.setTitleText(name)
axisY = QValueAxis()
axisY.setTitleText("Counts")
self.chart.setAxisY(axisY)
self.chart.setAxisX(axisX)
series.attachAxis(axisX)
series.attachAxis(axisY)
self.xlabel = xlabel
self.ylabel = ylabel
self.chartView.setRubberBand(QChartView.RectangleRubberBand)
self.chartView.setRenderHint(QtGui.QPainter.Antialiasing)
msg = 'Number of data points: {}, Ymin: {}, Ymax: {}'.format(
len(self.y), min(self.y), max(self.y))
self.showMessage(msg, 1)
self.showMessage('The canvas updated!')
def plotParameter(self, SPID=None, pname=None, desc=None):
self.tabWidget.setCurrentIndex(1)
if pname:
self.paramNameEdit.setText(pname)
if desc:
self.descLabel.setText(desc)
if SPID:
self.spidLineEdit.setText(str(SPID))
def onPlotActionClicked(self):
self.tabWidget.setCurrentIndex(1)
self.plotParameter()
def onTreeItemClicked(self, it, col):
SPID = None
try:
SPID = self.data[self.current_row]['header']['SPID']
except IndexError:
pass
self.plotParameter(SPID, it.text(0), it.text(1))
class widgetFDChart(QtWidgets.QWidget, qdesignFDChart.Ui_Form):
def __init__(self, parent=None):
super(widgetFDChart, self).__init__(parent)
self.setupUi(self)
tribchartmenustyle(self)
# SetUp for Chart to Display Distribution curves and inputs.
self.chart = QChart()
self.chart.legend().setVisible(False)
self.chart.axisX = QValueAxis()
self.chart.axisY = QValueAxis()
self.chart.axisX.setTickCount(10)
self.chart.axisY.setTickCount(10)
self.chart.axisX.setTitleText("Value")
self.chart.axisY.setTitleText("Probability")
self.chartview = QChartView(self.chart)
self.verticalLayout.addWidget(self.chartview)
# self.setAxesMinMax(-3,3,0.01,1.5)
self.chartview.setRenderHint(QPainter.Antialiasing)
# self.legend().setVisible(True)
# self.setAnimationOptions(QChart.SeriesAnimations)
# self.legend().setAlignment(Qt.AlignBottom)
# Connect Buttons
self.pushButtonExportPNG.pressed.connect(self._onActionSavePNG)
def axesMinMax(self):
# returns a length 4 list of the axes min and max values [x1,x2,y1,y2]
return [
self.chart.axisX.min(),
self.chart.axisX.max(),
self.chart.axisY.min(),
self.chart.axisY.max()
]
def setAxes(self, series):
# assigns a series to the chart default axes
self.chart.setAxisX(self.chart.axisX, series)
self.chart.setAxisY(self.chart.axisY, series)
def setAxesMinMax(self, x1, x2, y1, y2):
# sets the min max values in X and Y
self.chart.axisX.setMin(x1)
self.chart.axisX.setMax(x2)
self.chart.axisY.setMin(y1)
self.chart.axisY.setMax(y2)
def setTitle(self, title):
self.chart.setTitle(title)
def addDistrLine(self, type, n, kstats):
data = distr.distrpdf(type, n, **kstats)
xmin = min(data['X'])
xmax = max(data['X'])
ymin = min(data['Y'])
ymax = max(data['Y'])
self.lineDist = XLineSeries(data, xkey='X')[0]
self.chart.addSeries(self.lineDist)
self.setAxes(self.lineDist)
xscal = 0.1 * (xmax - xmin)
yscal = 0.1 * (ymax - ymin)
self.setAxesMinMax(xmin - xscal, xmax + xscal, ymin, ymax + yscal)
def updateChart(self, tabledata):
self.activeDist = tabledata[0]
self.kstats = tabledata[1]
self.chart.removeAllSeries()
self.addDistrLine(self.activeDist, 100, self.kstats)
@pyqtSlot(str)
def paintChart(self, filename):
#pixmap = QPixmap()
pixmap = self.chartview.grab()
pixmap.save(filename)
def _onActionSavePNG(self):
pngfile = QtWidgets.QFileDialog.getSaveFileName(
self, caption='Export PNG As', directory='~', filter='*.png')
#try:
self.paintChart(pngfile[0])
class graficoBarras(QWidget):
def __init__(self, parent=None):
super(graficoBarras, self).__init__(parent)
self.initUI()
def initUI(self):
comboBoxColor = QComboBox()
comboBoxColor.addItems([
"Color del fondo del CharView", "Color del fondo del Chart",
"Color del título del Chart", "Color de las etiquetas del eje X",
"Color de las etiquetas del eje Y",
"Color de las etiquetas de la leyenda"
])
checkBoxVisibilidadFondoChart = QCheckBox("Visibilidad fondo (chart)")
checkBoxMargenesChart = QCheckBox("Margenes del chart")
checkBoxEsquinasChart = QCheckBox("Esquinas del chart")
buttonGuardar = QPushButton("Guardar gráfico")
# Crear gráficos.
self.vistaGrafico = QChartView(self.crearGraficoBarras())
self.vistaGrafico.setSizePolicy(QSizePolicy.Ignored,
QSizePolicy.Ignored)
self.vistaGrafico.setRenderHint(QPainter.Antialiasing, True)
# ========================== DISEÑO ==========================
disenioConfiguracion = QVBoxLayout()
disenioConfiguracion.addWidget(QLabel("Cambiar colores:"))
disenioConfiguracion.addWidget(comboBoxColor)
disenioConfiguracion.addWidget(checkBoxVisibilidadFondoChart)
disenioConfiguracion.addWidget(checkBoxMargenesChart)
disenioConfiguracion.addWidget(checkBoxEsquinasChart)
disenioConfiguracion.setSpacing(4)
disenioConfiguracion.addStretch()
disenioConfiguracion.addWidget(buttonGuardar)
baseDisenio = QGridLayout()
baseDisenio.addLayout(disenioConfiguracion, 0, 0, 0, 1)
baseDisenio.addWidget(self.vistaGrafico, 0, 1, 0, 4)
baseDisenio.setSpacing(10)
baseDisenio.setContentsMargins(10, 10, 10, 10)
self.setLayout(baseDisenio)
# ===================== CONECTAR SEÑALES =====================
comboBoxColor.activated.connect(self.Color)
checkBoxVisibilidadFondoChart.toggled.connect(
self.visibilidadFondoChart)
checkBoxMargenesChart.toggled.connect(self.margenesChart)
checkBoxEsquinasChart.toggled.connect(self.esquinasChart)
buttonGuardar.clicked.connect(self.Guardar)
# Establecer los valores predeterminados
comboBoxColor.setCurrentIndex(-1)
checkBoxVisibilidadFondoChart.setChecked(True)
checkBoxMargenesChart.setChecked(True)
checkBoxEsquinasChart.setChecked(True)
# ========================== FUNCIONES ===========================
def crearGraficoBarras(self):
paises = [
"EEUU", "China", "Japon", "Alemania", "Reino Unido",
"Resto del mundo"
]
valores = [24.32, 14.85, 8.91, 12.54, 7.85, 31.53]
colores = [
Qt.blue, Qt.red, Qt.darkYellow, Qt.gray, Qt.black, Qt.darkCyan
]
grafico = QChart()
grafico.setMargins(QMargins(30, 30, 30, 30))
grafico.setTheme(QChart.ChartThemeLight)
grafico.setTitle("% Distribución del PIB global")
grafico.setAnimationOptions(QChart.SeriesAnimations)
for i in range(len(paises)):
series = QBarSeries()
barSet = QBarSet(paises[i])
barSet.setColor(colores[i])
barSet.setLabelColor(Qt.yellow)
barSet.append(valores[i])
series.append(barSet)
series.setLabelsVisible(True)
series.setLabelsAngle(-90)
# series.setLabelsPrecision(2)
series.setLabelsFormat("@value %")
series.setLabelsPosition(QAbstractBarSeries.LabelsCenter)
grafico.addSeries(series)
axisX = QBarCategoryAxis()
axisX.append(paises)
axisY = QValueAxis()
axisY.setRange(0, 31.53)
axisY.setTickCount(10)
axisY.setLabelFormat("%.2f %")
grafico.createDefaultAxes()
grafico.setAxisX(axisX, None)
grafico.setAxisY(axisY, None)
grafico.legend().setVisible(True)
grafico.legend().setAlignment(Qt.AlignBottom)
return grafico
def Color(self, item):
if item == 0:
color = (self.vistaGrafico.backgroundBrush().color()
if self.vistaGrafico.backgroundBrush().color().isValid()
else QColor(Qt.white))
elif item == 1:
color = (
self.vistaGrafico.chart().backgroundBrush().color() if
self.vistaGrafico.chart().backgroundBrush().color().isValid()
else QColor(Qt.white))
elif item == 2:
color = (self.vistaGrafico.chart().titleBrush().color() if
self.vistaGrafico.chart().titleBrush().color().isValid()
else QColor(Qt.black))
elif item == 3:
color = (self.vistaGrafico.chart().axisX().labelsBrush().color()
if self.vistaGrafico.chart().axisX().labelsBrush().color(
).isValid() else QColor(Qt.black))
elif item == 4:
color = (self.vistaGrafico.chart().axisY().labelsBrush().color()
if self.vistaGrafico.chart().axisY().labelsBrush().color(
).isValid() else QColor(Qt.black))
elif item == 5:
color = (self.vistaGrafico.chart().legend().labelColor() if
self.vistaGrafico.chart().legend().labelColor().isValid()
else QColor(Qt.black))
color = QColorDialog.getColor(color, self)
if color.isValid():
if item == 0:
self.vistaGrafico.setBackgroundBrush(color)
elif item == 1:
self.vistaGrafico.chart().setBackgroundBrush(color)
elif item == 2:
self.vistaGrafico.chart().setTitleBrush(color)
elif item == 3:
self.vistaGrafico.chart().axisX().setLabelsBrush(color)
elif item == 4:
self.vistaGrafico.chart().axisY().setLabelsBrush(color)
elif item == 5:
self.vistaGrafico.chart().legend().setLabelColor(color)
def visibilidadFondoChart(self, bool):
self.vistaGrafico.chart().setBackgroundVisible(bool)
def margenesChart(self, bool):
if bool:
self.vistaGrafico.chart().layout().setContentsMargins(9, 9, 9, 9)
else:
self.vistaGrafico.chart().layout().setContentsMargins(0, 0, 0, 0)
def esquinasChart(self, bool):
if bool:
self.vistaGrafico.chart().setBackgroundRoundness(5)
else:
self.vistaGrafico.chart().setBackgroundRoundness(0)
def Guardar(self):
nombre, extension = QFileDialog.getSaveFileName(
self,
"Guardar como",
"Gráfico de barras",
"JPG (*.jpg);;PNG (*.png)",
options=QFileDialog.Options())
if nombre:
guardar = QPixmap(self.vistaGrafico.grab())
guardar.save(nombre, quality=100)
if guardar:
QMessageBox.information(self, "Guardar gráfico",
"Gráfico guardado con éxito.",
QMessageBox.Ok)
else:
QMessageBox.critical(self, "Guardar gráfico",
"Error al guardar el gráfico.",
QMessageBox.Ok)
class Ui(mainwindow.Ui_MainWindow):
def __init__(self, MainWindow):
super(Ui, self).setupUi(MainWindow)
self.MainWindow = MainWindow
self.stix_tctm_parser = stix_parser.StixTCTMParser()
self.initialize()
def close(self):
self.MainWindow.close()
def style(self):
return self.MainWindow.style()
def initialize(self):
self.tabWidget.setCurrentIndex(0)
self.actionExit.triggered.connect(self.close)
self.actionPlot.setEnabled(False)
self.actionNext.setIcon(self.style().standardIcon(
QtWidgets.QStyle.SP_ArrowForward))
self.actionPrevious.setIcon(self.style().standardIcon(
QtWidgets.QStyle.SP_ArrowBack))
self.actionOpen.setIcon(self.style().standardIcon(
QtWidgets.QStyle.SP_DialogOpenButton))
self.actionSave.setIcon(self.style().standardIcon(
QtWidgets.QStyle.SP_DriveFDIcon))
self.actionSave.triggered.connect(self.save)
self.actionOpen.triggered.connect(self.getOpenFilename)
self.actionNext.triggered.connect(self.nextPacket)
self.actionPrevious.triggered.connect(self.previousPacket)
self.actionAbout.triggered.connect(self.about)
self.actionPrevious.setEnabled(False)
self.actionNext.setEnabled(False)
self.actionSave.setEnabled(False)
self.actionPlot.setEnabled(False)
self.actionCopy.triggered.connect(self.onCopyTriggered)
self.packetTreeWidget.currentItemChanged.connect(self.onPacketSelected)
self.actionCopy.setEnabled(False)
self.actionPaste.triggered.connect(self.onPasteTriggered)
self.actionLog.triggered.connect(self.dockWidget.show)
self.actionSetIDB.triggered.connect(self.onSetIDBClicked)
self.plotButton.clicked.connect(self.onPlotButtonClicked)
self.exportButton.clicked.connect(self.onExportButtonClicked)
self.actionPlot.triggered.connect(self.onPlotActionClicked)
self.actionLoadMongodb.triggered.connect(self.onLoadMongoDBTriggered)
self.actionConnectTSC.triggered.connect(self.onConnectTSCTriggered)
self.actionPacketFilter.triggered.connect(self.onPacketFilterTriggered)
self.actionPlugins.triggered.connect(self.onPluginTriggered)
self.actionOnlineHelp.triggered.connect(self.onOnlineHelpTriggered)
self.actionViewBinary.triggered.connect(self.onViewBinaryTriggered)
self.packetTreeWidget.customContextMenuRequested.connect(
self.packetTreeContextMenuEvent)
self.mdb = None
self.current_row = 0
self.data = []
self.x = []
self.y = []
self.xlabel = 'x'
self.ylabel = 'y'
self.buttons_enabled = False
self.chart = QChart()
self.chart.layout().setContentsMargins(0, 0, 0, 0)
self.chart.setBackgroundRoundness(0)
self.savePlotButton.clicked.connect(self.savePlot)
self.chartView = QChartView(self.chart)
self.gridLayout.addWidget(self.chartView, 1, 0, 1, 15)
#self.packetTreeWidget.itemDoubleClicked.connect(self.onPacketTreeItemDoubleClicked)
self.selected_services = SELECTED_SERVICES
self.selected_SPID = []
# IDB location
self.settings = QtCore.QSettings('FHNW', 'stix_parser')
self.idb_filename = self.settings.value('idb_filename', [], str)
if self.idb_filename:
idb._stix_idb.reload(self.idb_filename)
if not idb._stix_idb.is_connected():
self.showMessage('IDB has not been set!')
else:
self.showMessage(
'IDB location: {} '.format(idb._stix_idb.get_idb_filename()),
1)
def packetTreeContextMenuEvent(self, pos):
menu = QtWidgets.QMenu()
rawDataAction = menu.addAction('Show raw data')
menu.addSeparator()
filterAction = menu.addAction('Filter packets')
copyPacketAction = menu.addAction('Copy packet')
menu.addSeparator()
deleteAllAction = menu.addAction('Delete all packets')
self.current_row = self.packetTreeWidget.currentIndex().row()
rawDataAction.triggered.connect(self.onViewBinaryTriggered)
filterAction.triggered.connect(self.onPacketFilterTriggered)
copyPacketAction.triggered.connect(self.onCopyTriggered)
deleteAllAction.triggered.connect(self.onDeleteAllTriggered)
action = menu.exec_(self.packetTreeWidget.viewport().mapToGlobal(pos))
def onDeleteAllTriggered(self):
self.data.clear()
self.packetTreeWidget.clear()
self.paramTreeWidget.clear()
#def onPacketTreeItemDoubleClicked(self):
# self.onViewBinaryTriggered()
def onViewBinaryTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = raw_viewer.Ui_Dialog()
diag_ui.setupUi(diag)
if self.data:
try:
raw = self.data[self.current_row]['bin']
header = self.data[self.current_row]['header']
diag_ui.setPacketInfo('{}({},{}) {}'.format(
header['TMTC'], header['service_type'],
header['service_subtype'], header['DESCR']))
diag_ui.displayRaw(raw)
except (IndexError, KeyError):
diag_ui.setText('Raw data not available.')
diag.exec_()
def onOnlineHelpTriggered(self):
webbrowser.open('https://github.com/i4Ds/STIX-dataviewer', new=2)
def onPluginTriggered(self):
self.plugin_location = self.settings.value('plugin_location', [], str)
diag = QtWidgets.QDialog()
diag_ui = plugin.Ui_Dialog()
diag_ui.setupUi(diag)
if self.plugin_location:
diag_ui.setPluginLocation(self.plugin_location)
diag_ui.setData(self.data, self.current_row)
diag.exec_()
location = diag_ui.getPluginLocation()
if location != self.plugin_location:
self.settings.setValue('plugin_location', location)
def onPacketFilterTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = packet_filter.Ui_Dialog()
diag_ui.setupUi(diag)
diag_ui.setSelectedServices(self.selected_services)
diag_ui.buttonBox.accepted.connect(
partial(self.applyServiceFilter, diag_ui))
diag.exec_()
def applyServiceFilter(self, diag_ui):
self.selected_SPID = diag_ui.getSelectedSPID()
self.selected_services = diag_ui.getSelectedServices()
self.showMessage('Applying filter...')
self.addPacketsToView(self.data, True, show_stat=False)
def onExportButtonClicked(self):
if self.y:
filename = str(
QtWidgets.QFileDialog.getSaveFileName(None,
"Save data to file", "",
"CSV(*.csv)")[0])
if filename:
with open(filename, 'w') as f:
f.write('{},{}\n'.format(self.xlabel, self.ylabel))
for xx, yy in zip(self.x, self.y):
f.write('{},{}\n'.format(xx, yy))
self.showMessage(
'The data has been written to {}'.format(filename))
else:
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText('Plot first!')
msgBox.setWindowTitle("Warning")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def savePlot(self):
# if self.figure.get_axes():
if self.chart:
filetypes = "PNG (*.png);;JPEG (*.jpg)"
filename = str(
QtWidgets.QFileDialog.getSaveFileName(None,
"Save plot to file", "",
filetypes)[0])
if filename:
if not filename.endswith(('.png', '.jpg')):
filename += '.png'
# self.figure.savefig(filename)
p = self.chartView.grab()
p.save(filename)
self.showMessage(('Saved to %s.' % filename))
else:
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText('No figure to save')
msgBox.setWindowTitle("STIX raw data viewer")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def onCopyTriggered(self):
packet_id = self.current_row
try:
packet = self.data[packet_id]
ss = pprint.pformat(packet)
cb = QtWidgets.QApplication.clipboard()
cb.clear(mode=cb.Clipboard)
cb.setText(ss, mode=cb.Clipboard)
msg = QtWidgets.QMessageBox()
msg.setIcon(QtWidgets.QMessageBox.Information)
msg.setText(
"The data of the selected packet has been copied to the clipboard."
)
msg.setWindowTitle("Information")
msg.setStandardButtons(QtWidgets.QMessageBox.Ok)
retval = msg.exec_()
except Exception as e:
self.showMessage(str(e), 0)
def onPasteTriggered(self):
raw_hex = QtWidgets.QApplication.clipboard().text()
if len(raw_hex) < 16:
self.showMessage('No data in the clipboard.')
return
data_hex = re.sub(r"\s+", "", raw_hex)
try:
data_binary = binascii.unhexlify(data_hex)
packets = self.stix_tctm_parser.parse_binary(data_binary,
0,
store_binary=True)
if not packets:
return
self.showMessage('%d packets read from the clipboard' %
len(packets))
self.onDataReady(packets, clear=False, show_stat=False)
except Exception as e:
self.showMessageBox(str(e), data_hex)
def showMessageBox(self, message, content):
msg = QtWidgets.QMessageBox()
msg.setIcon(QtWidgets.QMessageBox.Critical)
msg.setText("Error")
msg.setInformativeText(message)
msg.setWindowTitle("Error")
msg.setDetailedText(content)
msg.setStandardButtons(QtWidgets.QMessageBox.Ok
| QtWidgets.QMessageBox.Cancel)
retval = msg.exec_()
def showMessage(self, msg, where=0):
if where != 1:
self.statusbar.showMessage(msg)
if where != 0:
self.statusListWidget.addItem(msg)
# if destination !=0 :
# self.listWidget_2.addItem(msg)
def onSetIDBClicked(self):
self.idb_filename = QtWidgets.QFileDialog.getOpenFileName(
None, 'Select file', '.', 'IDB file(*.db *.sqlite *.sqlite3)')[0]
if not self.idb_filename:
return
idb._stix_idb.reload(self.idb_filename)
if idb._stix_idb.is_connected():
#settings = QtCore.QSettings('FHNW', 'stix_parser')
self.settings.setValue('idb_filename', self.idb_filename)
self.showMessage(
'IDB location: {} '.format(idb._stix_idb.get_idb_filename()), 1)
def save(self):
filetypes = 'python compressed pickle (*.pklz);; python pickle file (*.pkl);; binary data (*.dat)'
self.output_filename = str(
QtWidgets.QFileDialog.getSaveFileName(None, "Save packets to", "",
filetypes)[0])
if not self.output_filename.endswith(('.pklz', '.pkl', '.dat')):
msg = 'unsupported file format !'
self.showMessage(msg)
return
msg = 'Writing data to file %s' % self.output_filename
self.showMessage(msg)
if self.output_filename.endswith(('.pklz', '.pkl')):
stw = stix_writer.StixPickleWriter(self.output_filename)
stw.register_run(str(self.input_filename))
stw.write_all(self.data)
#stw.done()
elif self.output_filename.endswith('.dat'):
stw = stix_writer.StixBinaryWriter(self.output_filename)
stw.write_all(self.data)
num_ok = stw.get_num_sucess()
msg = (
'The binary data of {} packets written to file {}, total packets {}'
.format(num_ok, self.output_filename, len(self.data)))
self.showMessage(msg)
def setListViewSelected(self, row):
#index = self.model.createIndex(row, 0);
# if index.isValid():
# self.model.selectionModel().select( index, QtGui.QItemSelectionModel.Select)
pass
def about(self):
msgBox = QtWidgets.QMessageBox()
msgBox.setIcon(QtWidgets.QMessageBox.Information)
msgBox.setText("STIX raw data parser and viewer, [email protected]")
msgBox.setWindowTitle("Stix data viewer")
msgBox.setStandardButtons(QtWidgets.QMessageBox.Ok)
msgBox.exec_()
def nextPacket(self):
self.current_row += 1
length = len(self.data)
if self.current_row >= length:
self.current_row = length - 1
self.showMessage('No more packet!')
self.showPacket(self.current_row)
self.setListViewSelected(self.current_row)
def previousPacket(self):
self.current_row -= 1
if self.current_row < 0:
self.current_row = 0
self.showMessage('Reach the first packet!')
self.showPacket(self.current_row)
self.setListViewSelected(self.current_row)
def getOpenFilename(self):
filetypes = (
'STIX raw data(*.dat *.bin *.binary);; python pickle files (*.pkl *pklz);;'
'ESA xml files (*xml);;'
'ESA ascii files(*.ascii);; CMDVS archive files (*.BDF);; All(*)')
self.input_filename = QtWidgets.QFileDialog.getOpenFileName(
None, 'Select file', '.', filetypes)[0]
if not self.input_filename:
return
self.openFile(self.input_filename)
def openFile(self, filename):
self.packetTreeWidget.clear()
msg = 'Loading file %s ...' % filename
self.showMessage(msg)
self.dataReader = StixFileReader(filename)
#self.dataReader.dataLoaded.connect(self.onDataReady)
self.dataReader.packetArrival.connect(self.onPacketArrival)
self.dataReader.error.connect(self.onDataReaderError)
self.dataReader.info.connect(self.onDataReaderInfo)
self.dataReader.warn.connect(self.onDataReaderWarn)
self.dataReader.start()
def onDataReaderInfo(self, msg):
self.showMessage(msg, 0)
def onDataReaderWarn(self, msg):
self.showMessage(msg, 1)
def onDataReaderError(self, msg):
self.showMessage(msg, 1)
def onDataReady(self, data, clear=True, show_stat=True):
if not clear:
self.data.extend(data)
else:
self.data = data
if data:
self.addPacketsToView(data, clear=clear, show_stat=show_stat)
self.enableButtons()
def enableButtons(self):
if not self.buttons_enabled:
self.actionPrevious.setEnabled(True)
self.actionNext.setEnabled(True)
self.actionSave.setEnabled(True)
self.actionCopy.setEnabled(True)
self.actionPlot.setEnabled(True)
self.actionViewBinary.setEnabled(True)
self.buttons_enabled = True
def addPacketsToView(self, data, clear=True, show_stat=True):
if clear:
self.packetTreeWidget.clear()
#t0 = 0
for p in data:
if type(p) is not dict:
continue
header = p['header']
root = QtWidgets.QTreeWidgetItem(self.packetTreeWidget)
# if t0 == 0:
# t0 = header['time']
timestamp_str = ''
try:
timestamp_str = header['utc']
except KeyError:
timestamp_str = '{:.2f}'.format(header['time'])
#- t0)
root.setText(0, timestamp_str)
root.setText(
1,
'{}({},{}) - {}'.format(header['TMTC'], header['service_type'],
header['service_subtype'],
header['DESCR']))
if not self.selected_SPID:
#not set then apply service
if header['service_type'] not in self.selected_services:
root.setHidden(True)
else:
if header['SPID'] not in self.selected_SPID:
root.setHidden(True)
if show_stat:
total_packets = len(self.data)
self.showMessage(('Total packet(s): %d' % total_packets))
def onConnectTSCTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = tsc_connection.Ui_Dialog()
diag_ui.setupUi(diag)
self.tsc_host = self.settings.value('tsc_host', [], str)
self.tsc_port = self.settings.value('tsc_port', [], str)
if self.tsc_host:
diag_ui.serverLineEdit.setText(self.tsc_host)
if self.tsc_port:
diag_ui.portLineEdit.setText(self.tsc_port)
diag_ui.buttonBox.accepted.connect(partial(self.connectToTSC, diag_ui))
diag.exec_()
def connectToTSC(self, dui):
host = dui.serverLineEdit.text()
port = dui.portLineEdit.text()
self.showMessage('Connecting to TSC...')
self.socketPacketReceiver = StixSocketPacketReceiver(host, int(port))
self.socketPacketReceiver.packetArrival.connect(self.onPacketArrival)
self.socketPacketReceiver.error.connect(self.onDataReaderError)
self.socketPacketReceiver.info.connect(self.onDataReaderInfo)
self.socketPacketReceiver.warn.connect(self.onDataReaderWarn)
self.socketPacketReceiver.start()
def onPacketArrival(self, packets):
if packets:
self.onDataReady(packets, clear=False, show_stat=True)
def onLoadMongoDBTriggered(self):
diag = QtWidgets.QDialog()
diag_ui = mongo_dialog.Ui_Dialog()
diag_ui.setupUi(diag)
#self.settings = QtCore.QSettings('FHNW', 'stix_parser')
self.mongo_server = self.settings.value('mongo_server', [], str)
self.mongo_port = self.settings.value('mongo_port', [], str)
self.mongo_user = self.settings.value('mongo_user', [], str)
self.mongo_pwd = self.settings.value('mongo_pwd', [], str)
if self.mongo_server:
diag_ui.serverLineEdit.setText(self.mongo_server)
if self.mongo_port:
diag_ui.portLineEdit.setText(self.mongo_port)
if self.mongo_user:
diag_ui.userLineEdit.setText(self.mongo_user)
if self.mongo_pwd:
diag_ui.pwdLineEdit.setText(self.mongo_pwd)
diag_ui.pushButton.clicked.connect(
partial(self.loadRunsFromMongoDB, diag_ui))
diag_ui.buttonBox.accepted.connect(
partial(self.loadDataFromMongoDB, diag_ui, diag))
diag.exec_()
def loadRunsFromMongoDB(self, dui):
server = dui.serverLineEdit.text()
port = dui.portLineEdit.text()
user = dui.userLineEdit.text()
pwd = dui.pwdLineEdit.text()
self.showMessage('saving setting...')
if self.mongo_server != server:
self.settings.setValue('mongo_server', server)
if self.mongo_port != port:
self.settings.setValue('mongo_port', port)
if self.mongo_user != user:
self.settings.setValue('mongo_user', user)
if self.mongo_pwd != pwd:
self.settings.setValue('mongo_pwd', pwd)
self.showMessage('connecting Mongo database ...')
self.mdb = mgdb.MongoDB(server, int(port), user, pwd)
if not self.mdb.is_connected():
return
dui.treeWidget.clear()
self.showMessage('Fetching data...')
for run in self.mdb.get_runs():
root = QtWidgets.QTreeWidgetItem(dui.treeWidget)
root.setText(0, str(run['_id']))
root.setText(1, run['filename'])
root.setText(2, run['date'])
root.setText(3, str(run['start']))
root.setText(4, str(run['end']))
def loadDataFromMongoDB(self, dui, diag):
self.showMessage('Loading packets ...')
selected_runs = []
for item in dui.treeWidget.selectedItems():
selected_runs.append(item.text(0))
if not selected_runs:
self.showMessage('Run not selected!')
if selected_runs:
diag.done(0)
self.showMessage('Loading data ...!')
data = self.mdb.get_packets(selected_runs[0])
if data:
self.onDataReady(data, clear=True)
else:
self.showMessage('No packets found!')
# close
def onPacketSelected(self, cur, pre):
self.current_row = self.packetTreeWidget.currentIndex().row()
self.showMessage(('Packet #%d selected' % self.current_row))
self.showPacket(self.current_row)
def showPacket(self, row):
if not self.data:
return
header = self.data[row]['header']
total_packets = len(self.data)
self.showMessage(
('Packet %d / %d %s ' % (row, total_packets, header['DESCR'])))
self.paramTreeWidget.clear()
header_root = QtWidgets.QTreeWidgetItem(self.paramTreeWidget)
header_root.setText(0, "Header")
rows = len(header)
for key, val in header.items():
root = QtWidgets.QTreeWidgetItem(header_root)
root.setText(0, key)
root.setText(1, str(val))
params = self.data[row]['parameters']
param_root = QtWidgets.QTreeWidgetItem(self.paramTreeWidget)
param_root.setText(0, "Parameters")
self.showParameterTree(params, param_root)
self.paramTreeWidget.expandItem(param_root)
self.paramTreeWidget.expandItem(header_root)
self.current_row = row
def showParameterTree(self, params, parent):
if not params:
return
for p in params:
root = QtWidgets.QTreeWidgetItem(parent)
if not p:
continue
try:
param_name = p['name']
desc = '' #description of parameter
if 'desc' in p:
desc = p['desc']
if not desc:
desc = idb._stix_idb.get_PCF_description(param_name)
scos_desc = idb._stix_idb.get_scos_description(param_name)
if scos_desc:
root.setToolTip(1, scos_desc)
root.setText(0, param_name)
root.setText(1, desc)
root.setText(2, str(p['raw']))
try:
root.setToolTip(2, hex(p['raw'][0]))
except:
pass
root.setText(3, str(p['eng']))
if 'children' in p:
if p['children']:
self.showParameterTree(p['children'], root)
except KeyError:
self.showMessage(
'[Error ]: keyError occurred when adding parameter')
self.paramTreeWidget.itemDoubleClicked.connect(self.onTreeItemClicked)
def walk(self, name, params, header, ret_x, ret_y, xaxis=0, data_type=0):
if not params:
return
timestamp = header['time']
for p in params:
if type(p) is not dict:
continue
if name == p['name']:
values = None
#print('data type:{}'.format(data_type))
if data_type == 0:
values = p['raw']
else:
values = p['eng']
try:
yvalue = None
if (type(values) is tuple) or (type(values) is list):
yvalue = float(values[0])
else:
yvalue = float(values)
ret_y.append(yvalue)
if xaxis == 1:
ret_x.append(timestamp)
else:
self.showMessage(('Can not plot %s ' % str(yvalue)))
except Exception as e:
self.showMessage(('%s ' % str(e)))
if 'children' in p:
if p['children']:
self.walk(name, p['children'], header, ret_x, ret_y, xaxis,
data_type)
def onPlotButtonClicked(self):
if self.chart:
self.chart.removeAllSeries()
if not self.data:
return
self.showMessage('Preparing plot ...')
name = self.paramNameEdit.text()
packet_selection = self.comboBox.currentIndex()
xaxis_type = self.xaxisComboBox.currentIndex()
data_type = self.dataTypeComboBox.currentIndex()
timestamp = []
self.y = []
packet_id = self.current_row
params = self.data[packet_id]['parameters']
header = self.data[packet_id]['header']
current_spid = header['SPID']
if packet_selection == 0:
self.walk(name, params, header, timestamp, self.y, xaxis_type,
data_type)
elif packet_selection == 1:
for packet in self.data:
header = packet['header']
if packet['header']['SPID'] != current_spid:
continue
params = packet['parameters']
self.walk(name, params, header, timestamp, self.y, xaxis_type,
data_type)
self.x = []
if not self.y:
self.showMessage('No data points')
elif self.y:
style = self.styleEdit.text()
if not style:
style = '-'
title = '%s' % str(name)
desc = self.descLabel.text()
if desc:
title += '- %s' % desc
self.chart.setTitle(title)
ylabel = 'Raw value'
xlabel = name
if data_type == 1:
ylabel = 'Engineering value'
if xaxis_type == 0:
if packet_selection == 1:
xlabel = "Packet #"
else:
xlabel = "Repeat #"
self.x = range(0, len(self.y))
if xaxis_type == 1:
self.x = [t - timestamp[0] for t in timestamp]
xlabel = 'Time -T0 (s)'
if xaxis_type != 2:
series = QLineSeries()
series2 = None
for xx, yy in zip(self.x, self.y):
series.append(xx, yy)
if 'o' in style:
series2 = QScatterSeries()
for xx, yy in zip(self.x, self.y):
series2.append(xx, yy)
self.chart.addSeries(series2)
self.chart.addSeries(series)
axisX = QValueAxis()
axisX.setTitleText(xlabel)
axisY = QValueAxis()
axisY.setTitleText(ylabel)
self.chart.setAxisX(axisX)
self.chart.setAxisY(axisY)
series.attachAxis(axisX)
series.attachAxis(axisY)
else:
nbins = len(set(self.y))
ycounts, xedges = np.histogram(self.y, bins=nbins)
series = QLineSeries()
for i in range(0, nbins):
meanx = (xedges[i] + xedges[i + 1]) / 2.
series.append(meanx, ycounts[i])
self.chart.addSeries(series)
axisX = QValueAxis()
axisX.setTitleText(name)
axisY = QValueAxis()
axisY.setTitleText("Counts")
self.chart.setAxisY(axisY)
self.chart.setAxisX(axisX)
series.attachAxis(axisX)
series.attachAxis(axisY)
self.xlabel = xlabel
self.ylabel = ylabel
self.chartView.setRubberBand(QChartView.RectangleRubberBand)
self.chartView.setRenderHint(QtGui.QPainter.Antialiasing)
msg = 'Y length: {}, min-Y: {}, max-Y: {}'.format(
len(self.y), min(self.y), max(self.y))
self.showMessage(msg, 1)
self.showMessage('The canvas updated!')
def plotParameter(self, name=None, desc=None):
self.tabWidget.setCurrentIndex(1)
if name:
self.paramNameEdit.setText(name)
if desc:
self.descLabel.setText(desc)
def onPlotActionClicked(self):
self.tabWidget.setCurrentIndex(1)
self.plotParameter()
def onTreeItemClicked(self, it, col):
self.plotParameter(it.text(0), it.text(1))
class Ui_MainWindow(QtWidgets.QMainWindow):
def setupUi(self):
self.setObjectName("MainWindow")
self.resize(850, 1080)
self.setWindowTitle("GA - Queens")
self.centralwidget = QtWidgets.QWidget(self)
self.centralwidget.setObjectName("centralwidget")
self.frameWorld = MyQFrame(self.centralwidget)
self.frameWorld.img = QPixmap(1000, 1000)
self.frameWorld.setGeometry(QtCore.QRect(10, 10, 620, 600))
self.frameWorld.setFrameShape(QtWidgets.QFrame.Box)
self.frameWorld.setFrameShadow(QtWidgets.QFrame.Sunken)
self.frameWorld.setObjectName("frameWorld")
self.frameChart = QChartView(self.centralwidget)
self.frameChart.setGeometry(QtCore.QRect(10, 620, 620, 400))
self.frameChart.setFrameShape(QtWidgets.QFrame.Box)
self.frameChart.setFrameShadow(QtWidgets.QFrame.Sunken)
self.frameChart.setRenderHint(QPainter.Antialiasing)
self.frameChart.setObjectName("frameChart")
self.gaParams = QtWidgets.QGroupBox(self.centralwidget)
self.gaParams.setGeometry(QtCore.QRect(650, 10, 161, 145))
self.gaParams.setObjectName("gaParams")
self.gaParams.setTitle("GA parameters")
self.label1 = QtWidgets.QLabel(self.gaParams)
self.label1.setGeometry(QtCore.QRect(10, 20, 61, 16))
self.label1.setObjectName("label1")
self.label1.setText("Population:")
self.label2 = QtWidgets.QLabel(self.gaParams)
self.label2.setGeometry(QtCore.QRect(10, 50, 47, 16))
self.label2.setObjectName("label2")
self.label2.setText("Mutation:")
self.label3 = QtWidgets.QLabel(self.gaParams)
self.label3.setGeometry(QtCore.QRect(10, 80, 81, 16))
self.label3.setObjectName("label3")
self.label3.setText("Elite members:")
self.label4 = QtWidgets.QLabel(self.gaParams)
self.label4.setGeometry(QtCore.QRect(10, 110, 91, 16))
self.label4.setObjectName("label4")
self.label4.setText("No. generations:")
self.tbxPopulation = QtWidgets.QLineEdit(self.gaParams)
self.tbxPopulation.setGeometry(QtCore.QRect(100, 20, 51, 20))
self.tbxPopulation.setObjectName("tbxPopulation")
self.tbxMutation = QtWidgets.QLineEdit(self.gaParams)
self.tbxMutation.setGeometry(QtCore.QRect(100, 50, 51, 20))
self.tbxMutation.setObjectName("tbxMutation")
self.tbxElite = QtWidgets.QLineEdit(self.gaParams)
self.tbxElite.setGeometry(QtCore.QRect(100, 80, 51, 20))
self.tbxElite.setObjectName("tbxElite")
self.tbxGenerations = QtWidgets.QLineEdit(self.gaParams)
self.tbxGenerations.setGeometry(QtCore.QRect(100, 110, 51, 20))
self.tbxGenerations.setObjectName("tbxGenerations")
self.cbxNoVis = QtWidgets.QCheckBox(self.centralwidget)
self.cbxNoVis.setGeometry(QtCore.QRect(650, 170, 170, 17))
self.cbxNoVis.setObjectName("cbxNoVis")
self.cbxNoVis.setText("No visualization per generation")
self.cbxBorder = QtWidgets.QCheckBox(self.centralwidget)
self.cbxBorder.setGeometry(QtCore.QRect(650, 200, 100, 17))
self.cbxBorder.setObjectName("cbxBorder")
self.cbxBorder.setText("Border patrol")
self.btnStart = QtWidgets.QPushButton(self.centralwidget)
self.btnStart.setGeometry(QtCore.QRect(650, 230, 75, 23))
self.btnStart.setObjectName("btnStart")
self.btnStart.setText("Start")
self.btnStop = QtWidgets.QPushButton(self.centralwidget)
self.btnStop.setEnabled(False)
self.btnStop.setGeometry(QtCore.QRect(730, 230, 75, 23))
self.btnStop.setObjectName("btnStop")
self.btnStop.setText("Stop")
self.btnSaveWorld = QtWidgets.QPushButton(self.centralwidget)
self.btnSaveWorld.setGeometry(QtCore.QRect(650, 570, 121, 41))
self.btnSaveWorld.setObjectName("btnSaveWorld")
self.btnSaveWorld.setText("Save world as image")
self.btnSaveChart = QtWidgets.QPushButton(self.centralwidget)
self.btnSaveChart.setGeometry(QtCore.QRect(650, 930, 121, 41))
self.btnSaveChart.setObjectName("btnSaveChart")
self.btnSaveChart.setText("Save chart as image")
self.btnSaveChartSeries = QtWidgets.QPushButton(self.centralwidget)
self.btnSaveChartSeries.setGeometry(QtCore.QRect(650, 980, 121, 41))
self.btnSaveChartSeries.setObjectName("btnSaveChartSeries")
self.btnSaveChartSeries.setText("Save chart as series")
self.setCentralWidget(self.centralwidget)
QtCore.QMetaObject.connectSlotsByName(self)
#Connect events
self.btnStart.clicked.connect(self.btnStart_Click)
self.btnStop.clicked.connect(self.btnStop_Click)
self.btnSaveWorld.clicked.connect(self.btnSaveWorld_Click)
self.btnSaveChart.clicked.connect(self.btnSaveChart_CLick)
self.btnSaveChartSeries.clicked.connect(self.btnSaveChartSeries_Click)
#Set default GA variables
self.tbxGenerations.insert(str(NGEN))
self.tbxPopulation.insert(str(POP_SIZE))
self.tbxMutation.insert(str(MUTPB))
self.tbxElite.insert(str(NELT))
self.new_image = QPixmap(1000, 1000)
def btnStart_Click(self):
global combination_series # List of lists containing min_series of 5 correct results -- Added by Denis Lazor
global parameter_name # Name of parameter used for writing its data to .csv file -- Added by Denis Lazor
global best_fit_values # List containing best fitness values for every of 5 experiments per combination -- Added by Denis Lazor
global best_individual # Saving best individuals for drawing the best solution -- Added by Denis Lazor
global ELITE_SIZES
global POPULATION_SIZES
global MUTATION_SIZES
# Checking if files are empty or not -- Added by Denis Lazor
csv_contains = os.stat("graphs_csv/original.csv").st_size != 0
if csv_contains:
clear_all_csv()
n = 5000
# Automation for all necessary combinations -- Added by Denis Lazor
for p in POPULATION_SIZES:
for m in MUTATION_SIZES:
for e in ELITE_SIZES:
for i in range(5):
# Set global variables
global stop_evolution
global q_min_series
global q_max_series
global q_avg_series
stop_evolution = False
q_min_series.clear()
q_max_series.clear()
q_avg_series.clear()
# Set global variables from information on UI
global NGEN
global POP_SIZE
global MUTPB
global NELT
NGEN = n
POP_SIZE = p
MUTPB = m
NELT = e
global border_check
border_check = self.cbxBorder.isChecked()
# Loading Croatia map
self.img = QPixmap(620, 600)
self.img.load('Croatia620.png')
self.frameWorld.img = self.img
# Drawing towns
painter = QPainter(self.img)
painter.setPen(QPen(Qt.black, 10, Qt.SolidLine))
painter.setFont(QFont('Arial', 12))
for i in range(len(gradovi)):
x, y = GlobToImgCoords(sirine[i], duzine[i])
painter.drawPoint(x, y)
painter.drawText(x + 5, y + 5, gradovi[i])
painter.end()
# Redrawing frames
self.frameWorld.repaint()
app.processEvents()
####Initialize deap GA objects####
# Make creator that minimize. If it would be 1.0 instead od -1.0 than it would be maxmize
creator.create("FitnessMin",
base.Fitness,
weights=(-1.0, ))
# Create an individual (a blueprint for cromosomes) as a list with a specified fitness type
creator.create("Individual",
list,
fitness=creator.FitnessMin)
# Create base toolbox for finishing creation of a individual (cromosome)
self.toolbox = base.Toolbox()
# This is if we want a permutation coding of genes in the cromosome
self.toolbox.register("indices", random.sample,
range(IND_SIZE), IND_SIZE)
# initIterate requires that the generator of genes (such as random.sample) generates an iterable (a list) variable
self.toolbox.register("individual", tools.initIterate,
creator.Individual,
self.toolbox.indices)
# Create a population of individuals (cromosomes). The population is then created by toolbox.population(n=300) where 'n' is the number of cromosomes in population
self.toolbox.register("population", tools.initRepeat,
list, self.toolbox.individual)
# Register evaluation function
self.toolbox.register("evaluate", evaluateInd)
# Register what genetic operators to use
self.toolbox.register(
"mate", tools.cxUniformPartialyMatched, indpb=0.2
) # Use uniform recombination for permutation coding
# Permutation coding
self.toolbox.register("mutate",
tools.mutShuffleIndexes,
indpb=0.2)
self.toolbox.register(
"select", tools.selTournament,
tournsize=3) # Use tournament selection
##################################
# Generate initial poplation. Will be a member variable so we can easely pass everything to new thread
self.pop = self.toolbox.population(n=POP_SIZE)
# Evaluate initial population, we map() the evaluation function to every individual and then assign their respective fitness, map runs evaluate function for each individual in pop
fitnesses = list(map(self.toolbox.evaluate, self.pop))
for ind, fit in zip(self.pop, fitnesses):
ind.fitness.values = fit # Assign calcualted fitness value to individuals
# Extracting all the fitnesses of all individuals in a population so we can monitor and evovlve the algorithm until it reaches 0 or max number of generation is reached
self.fits = [ind.fitness.values[0] for ind in self.pop]
# Disable start and enable stop
self.btnStart.setEnabled(False)
self.btnStop.setEnabled(True)
self.gaParams.setEnabled(False)
self.cbxBorder.setEnabled(False)
self.cbxNoVis.setEnabled(False)
# Start evolution
self.evolve()
# Best fitness value -- Added by Denis Lazor
best_fit = np.array(min(best_fit_values))[0]
mean_fit = min(
best_fit_values,
key=lambda x: abs(x - statistics.mean(
(np.asarray(best_fit_values)).flatten())))[0]
# Index of best fitness value -- Added by Denis Lazor
best_fit_idx = best_fit_values.index(best_fit)
write_to_file(combination_series[best_fit_idx],
parameter_name)
# First name will be "original", second one "elites" -- Added by Denis Lazor
parameter_name = "elites"
print_results(p, m, e, best_fit, mean_fit, NGEN)
# Clearing past lists -- Added by Denis Lazor
combination_series = []
best_fit_values = []
# Reducing number of combinations and changing .csv file for writing -- Added by Denis Lazor
ELITE_SIZES = ELITE_SIZES[0:1]
parameter_name = "mutation"
MUTATION_SIZES = MUTATION_SIZES[0:1]
parameter_name = "population"
print("Best individual: " + str(best_individual))
self.updateWorldFrame(generateWorldImage(
best_individual)) # Drawing best solution -- Added by Denis Lazor
def btnStop_Click(self):
global stop_evolution
stop_evolution = True
#Disable stop and enable start
self.btnStop.setEnabled(False)
self.btnStart.setEnabled(True)
self.gaParams.setEnabled(True)
self.cbxBorder.setEnabled(True)
self.cbxNoVis.setEnabled(True)
#Function for GA evolution
def evolve(self):
global q_min_series
global q_max_series
global q_avg_series
global best_fit_values
global combination_series
global best_individual
combination_current_series = [
] # Clearing fitness values series -- Added by Denis Lazor
# Variable for keeping track of the number of generations
curr_g = 0
# Begin the evolution till goal is reached or max number of generation is reached
while min(self.fits) != 0 and curr_g < NGEN:
#Check if evolution and thread need to stop
if stop_evolution:
break #Break the evolution loop
# A new generation
curr_g = curr_g + 1
#print("-- Generation %i --" % curr_g)
# Select the next generation individuals
#Select POP_SIZE - NELT number of individuals. Since recombination is between neigbours, not two naighbours should be the clone of the same individual
offspring = []
offspring.append(self.toolbox.select(
self.pop, 1)[0]) #add first selected individual
for i in range(
POP_SIZE - NELT - 1
): # -1 because the first seleceted individual is already added
while True:
new_o = self.toolbox.select(self.pop, 1)[0]
if new_o != offspring[len(
offspring
) - 1]: #if it is different than the last inserted then add to offspring and break
offspring.append(new_o)
break
# Clone the selected individuals because all of the changes are inplace
offspring = list(map(self.toolbox.clone, offspring))
# Apply crossover on the selected offspring
for child1, child2 in zip(offspring[::2], offspring[1::2]):
self.toolbox.mate(child1, child2) #inplace recombination
#Invalidate new children fitness values
del child1.fitness.values
del child2.fitness.values
#Apply mutation on the offspring
for mutant in offspring:
if random.random() < MUTPB:
self.toolbox.mutate(mutant)
del mutant.fitness.values
#Add elite individuals #Is clonning needed?
offspring.extend(
list(map(self.toolbox.clone, tools.selBest(self.pop, NELT))))
# Evaluate the individuals with an invalid fitness
invalid_ind = [ind for ind in offspring if not ind.fitness.valid]
fitnesses = map(self.toolbox.evaluate, invalid_ind)
for ind, fit in zip(invalid_ind, fitnesses):
ind.fitness.values = fit
#print(" Evaluated %i individuals" % len(invalid_ind))
#Replace population with offspring
self.pop[:] = offspring
# Gather all the fitnesses in one list and print the stats
self.fits = [ind.fitness.values[0] for ind in self.pop]
length = len(self.pop)
mean = sum(self.fits) / length
sum2 = sum(x * x for x in self.fits)
std = abs(sum2 / length - mean**2)**0.5
q_min_series.append(curr_g, min(self.fits))
q_max_series.append(curr_g, max(self.fits))
q_avg_series.append(curr_g, mean)
combination_current_series.append(
min(self.fits)
) # Saving min_series fitness values of an experiment -- Added by Denis Lazor
#print(" Min %s" % q_min_series.at(q_min_series.count()-1).y())
#print(" Max %s" % q_max_series.at(q_max_series.count()-1).y())
#print(" Avg %s" % mean)
#print(" Std %s" % std)
if self.cbxNoVis.isChecked():
app.processEvents()
else:
self.chart = QChart()
self.chart.addSeries(q_min_series)
self.chart.addSeries(q_max_series)
self.chart.addSeries(q_avg_series)
self.chart.setTitle("Fitness value over time")
self.chart.setAnimationOptions(QChart.NoAnimation)
self.chart.createDefaultAxes()
self.frameChart.setChart(self.chart)
#Draw queen positions of best individual on a image
best_ind = tools.selBest(self.pop, 1)[0]
self.updateWorldFrame(generateWorldImage(best_ind))
#Printing best individual
best_ind = tools.selBest(self.pop, 1)[0]
#print("Best individual is %s, %s" % (best_ind, best_ind.fitness.values))
combination_series.append(
combination_current_series
) # Saving best 5 results -- Added by Denis Lazor
if not best_individual:
best_individual = best_ind
elif best_ind.fitness.values < best_individual.fitness.values: # Saving best individual in all combinations -- Added by Denis Lazor
best_individual = best_ind
best_fit_values.append(
best_ind.fitness.values
) # Adding best fitness value of experiment -- Added by Denis Lazor
# #Visulaize final solution
# if self.cbxNoVis.isChecked():
# self.chart = QChart()
# self.chart.addSeries(q_min_series)
# self.chart.addSeries(q_max_series)
# self.chart.addSeries(q_avg_series)
# self.chart.setTitle("Fitness value over time")
# self.chart.setAnimationOptions(QChart.NoAnimation)
# self.chart.createDefaultAxes()
# self.frameChart.setChart(self.chart)
#
# #Draw queen positions of best individual on a image
# best_ind = tools.selBest(self.pop, 1)[0]
# self.updateWorldFrame(generateWorldImage(best_ind))
#Disable stop and enable start
self.btnStop.setEnabled(False)
self.btnStart.setEnabled(True)
self.gaParams.setEnabled(True)
self.cbxBorder.setEnabled(True)
self.cbxNoVis.setEnabled(True)
def updateWorldFrame(self, best_individual_img):
#new_image = QPixmap(1000,1000)
self.new_image.fill() #White color is default
painter = QPainter(self.new_image)
#First draw the map with towns
painter.drawPixmap(self.new_image.rect(), self.img)
#Then draw the best individual
painter.drawImage(self.new_image.rect(), best_individual_img)
painter.end()
#Set new image to the frame
self.frameWorld.img = self.new_image
#Redrawing frames
self.frameWorld.repaint()
self.frameChart.repaint()
app.processEvents()
def btnSaveWorld_Click(self):
filename, _ = QFileDialog.getSaveFileName(None,
"Save world as a image", "",
"Image Files (*.png)")
self.frameWorld.img.save(filename, "PNG")
print("World image saved to: ", filename)
def btnSaveChart_CLick(self):
p = self.frameChart.grab()
filename, _ = QFileDialog.getSaveFileName(
None, "Save series chart as a image", "", "Image Files (*.png)")
p.save(filename, "PNG")
print("Chart series image saved to: ", filename)
def btnSaveChartSeries_Click(self):
global q_min_series
global q_max_series
global q_avg_series
filename, _ = QFileDialog.getSaveFileName(None,
"Save series to text file",
"",
"Text Files (*.txt, *.csv)")
with open(filename, 'w') as dat:
for i in range(q_min_series.count()):
dat.write('%f,%f,%f\n' %
(q_min_series.at(i).y(), q_avg_series.at(i).y(),
q_max_series.at(i).y()))
print("Chart series saved to: ", filename)
