def test_learning_rates(self): mean_accuracies = [] standard_errors = [] best_mean_accuracy = 0.0 best_learning_rate = 0.0 for learning_rate in self.learning_rates: # initialise list to store accuracy results accuracy_cache = [] print "Testing network with learning rate: " + str(learning_rate) # The Neural network is trained and tested for the specified number of iterations for each # hidden layer size value for i in range(0, 10, 1): # Reinitialise dataset importer object csv_delegate = CSVFileDelegate("Datasets/owls15.csv") # Initialise Neural Network Classifier with data and target from data importer neural_network_classifier = NeuralNetworkClassifier( csv_delegate.training_data, csv_delegate.training_target) # Build and train network with <hidden_layer_size> nodes in the hidden layer neural_network_classifier.build_network(3, 2000, learning_rate) # Use classifier to classify testing data results = neural_network_classifier.classify_set( csv_delegate.testing_data) # Compare results to testing target accuracy = self.compare_result_to_target( results, csv_delegate.testing_target) accuracy_cache.append(accuracy) print accuracy # Store the mean and standard error values for each set of results mean_accuracy = (float(sum(accuracy_cache)) / 10) if mean_accuracy > best_mean_accuracy: best_mean_accuracy = mean_accuracy best_learning_rate = learning_rate mean_accuracies.append(mean_accuracy) standard_errors.append(scipy.stats.sem(accuracy_cache)) print "Best learning rate = " + str(best_learning_rate) plotter = ResultPlotter(self.learning_rates, mean_accuracies, standard_errors, 0, 0, "Datasets/owls15.csv") plotter.generate_learning_rate_plot_with_errors( self.learning_rates, mean_accuracies, standard_errors)
def test_network_and_plot_results(self): accuracy_results = [] for training_iterations in self.training_iterations_list: print "Testing network for " + str( training_iterations) + " iterations" # Initialise lists to store mean accuracy and standard error values mean_accuracies = [] standard_errors = [] # Iterates through the hidden layer sizes specified for hidden_layer_size in self.hidden_layer_sizes: # initialise list to store accuracy results accuracy_cache = [] print "Testing network with hidden layer size: " + str( hidden_layer_size) # The Neural network is trained and tested for the specified number of iterations for each # hidden layer size value for i in range(0, 10, 1): # Reinitialise dataset importer object csv_delegate = CSVFileDelegate(self.filepath) # Initialise Neural Network Classifier with data and target from data importer neural_network_classifier = NeuralNetworkClassifier( csv_delegate.training_data, csv_delegate.training_target) # Build and train network with <hidden_layer_size> nodes in the hidden layer neural_network_classifier.build_network( hidden_layer_size, training_iterations) # Use classifier to classify testing data results = neural_network_classifier.classify_set( csv_delegate.testing_data) # Compare results to testing target accuracy = self.compare_result_to_target( results, csv_delegate.testing_target) accuracy_cache.append(accuracy) print accuracy # Store the mean and standard error values for each set of results mean_accuracies.append((float(sum(accuracy_cache)) / 10)) standard_errors.append(scipy.stats.sem(accuracy_cache)) # Plot accuracy vs number of hidden nodes with the standard error plotter = ResultPlotter(self.hidden_layer_sizes, mean_accuracies, standard_errors, training_iterations, self.baseline_accuracy, ntpath.basename(self.filepath)) plotter.generate_plot_with_errors() accuracy_results.append(mean_accuracies) if plotter: plotter.generate_combined_plot(self.hidden_layer_sizes, accuracy_results, self.training_iterations_list)
# loop of different calibration methods for m in config['calibrationMethods']: calMethod = locals()[m] calB = [] calTestData = [] calStats = [] # loop over the hops, i.e. perform calibration hop-by-hop for hop in range(config['numHops']): # get the data for calibration X, Y = myDataCreator.get_train_data(hop) if hop == 0: # in the first hop we calibrate the sensor array to the true values calB.append(calMethod(X, Y)) else: # in the following hops (hop > 0) we calibrate the array to the previously calibrated one calB.append(calMethod(X, virtY)) Xt, GT = myDataCreator.get_test_data(hop) virtY = calB[hop].dot(Xt) calTestData.append(virtY) trueB = myDataCreator.get_true_B(hop) calStats.append(CalibrationStatistics(virtY, GT, calB[hop], trueB)) Results.update({m + '_run_' + str(run): calStats}) # plot the results if config['statsToPlot']: ResultPlotter(Results, config['statsToPlot'], config['calibrationMethods'], config['numRuns'], config['numHops'])
def __init__(self, plotterTerminal="png"): self.plotter = ResultPlotter(plotterTerminal) self.metrics = MetricsCalc() self.pareto = None self.setResultDirectories([])
class Analyzer: __PARETO__ = "pareto" __IMAGES_DIR__ = "images/" __COLORS__ = ["#dddddd", "#9400D3", "green", "blue", "red", "orange"] def __init__(self, plotterTerminal="png"): self.plotter = ResultPlotter(plotterTerminal) self.metrics = MetricsCalc() self.pareto = None self.setResultDirectories([]) def setPareto(self, pareto): self.pareto = pareto self.addResultDirectory(pareto) def addResultDirectory(self, result): if result not in self.resultDirectories: self.nResults += 1 self.resultDirectories.append(result) self.resultNames.append(self.getResultName(result)) self._scanDirectory(result) def removeResultDirectory(self, result): if result in self.resultDirectories: self.resultDirectories.remove(result) self.setResultDirectories(self.resultDirectories) def setResultDirectories(self, results): self.resultDirectories = [] self.resultNames = [] self.functions = {} self.nResults = 0 if self.pareto is not None: self.setPareto(self.pareto) for result in results: self.addResultDirectory(result) def getResultName(self, directory): directory = str(directory) if directory[-1] == "/": directory = directory[:-1] slash = max(directory.rfind("/"), directory.rfind("\\")) return directory[slash+1:] def getResultsForFunction(self, functionName): return self.functions[functionName.lower()] def getFunctionNames(self, includeNotSolved=False): return [function.functionName for function in self.functions.values() if includeNotSolved or \ self._hasNonPareto(function.functionImplementation) or \ self._hasNonPareto(function.variableImplementation)] def exportAllImages(self, directory, resultNames=None): if resultNames is None: resultNames = self.resultNames for functionName in self.getFunctionNames(): function = self.functions[functionName.lower()] filename = directory + "/" + function.functionName generation = [0] * len(resultNames) self.exportToImage(function, generation, True, resultNames, filename + "_fun.png") self.exportToImage(function, generation, False, resultNames, filename + "_var.png") def exportToImage(self, function, generation, functionSpace, resultNames, filename, latex=False, window=None): toPlot = self._getSolutionsToPlot(function, generation, functionSpace, resultNames) if functionSpace: if latex: axis = ["$f_1$", "$f_2$", "$f_3$"] else: axis = ["F1", "F2", "F3"] else: if latex: axis = ["$x_1$", "$x_2$", "$x_3$"] else: axis = ["x1", "x2", "x3"] functionName = Utils.getFunctionNameLatex(function.functionName) if latex else function.functionName if latex: for solution in toPlot: solution[0] = Utils.getResultNameLatex(solution[0]) self.plotter.plotSolution(toPlot, functionName, None if functionSpace else "Parameter space", \ axis[0], axis[1], axis[2], window, filename) def _getSolutionsToPlot(self, problem, generation, functionSpace, resultNames): solutions = [] if Analyzer.__PARETO__ in resultNames: resultNames.remove(Analyzer.__PARETO__) resultNames.insert(0, Analyzer.__PARETO__) k = 0 for name in resultNames: k += 1 if functionSpace: solution = problem.getFunctionSolution(name) else: solution = problem.getVariableSolution(name) if solution is not None: colorIdx = 0 if name != Analyzer.__PARETO__: colorIdx = self.resultNames.index(name) + 1 rgb = Analyzer.__COLORS__[colorIdx] # rgb = 3*[0] # for p in xrange(3): # if k & (1 << p) > 0: # rgb[p] = 255 points = solution.getSolutionPoints(generation[k-1]) solutions.append([name, points, rgb]) return solutions def _hasNonPareto(self, implementations): if len(implementations) > 1: return True if len(implementations) == 0: return False return Analyzer.__PARETO__ not in implementations.keys() def _scanDirectory(self, directory): if not os.path.exists(directory) or not os.path.isdir(directory): print >> sys.stderr, "Directory '%s' does not exist!" % directory return resultName = self.getResultName(directory) for filename in dircache.listdir(directory): filename = str(directory + "/" + filename) # fileType, _ = mimetypes.guess_type(filename) #if fileType is None or "text" not in fileType or not self.isSolutionFile(filename): if not Utils.isSolutionFile(filename): continue functionName = Utils.getFunctionName(filename) genPos = max(-1, functionName.rfind("."), functionName.rfind("-"), functionName.rfind("_")) generation = 1 << 30 if genPos >= 0: try: generation = int(functionName[genPos+1:]) functionName = functionName[:genPos] except: pass fnId = functionName.lower() if fnId in self.functions: function = self.functions[fnId] else: function = MOSolution(functionName) self.functions[fnId] = function if Utils.isFunctionFile(filename): function.addFunctionSolution(resultName, filename, generation) else: function.addVariableSolution(resultName, filename, generation) def getFunctionResults(self, functionName, resultNames): function = self.getResultsForFunction(functionName) solutions = [] for name in resultNames: if name.lower() != Analyzer.__PARETO__: result = function.getFunctionSolution(name) if result is not None: solutions.append([name, [result.getSolutionPoints(idx) for idx in xrange(result.count())]]) return solutions def getFunctionPareto(self, functionName): pareto = self.getResultsForFunction(functionName).getFunctionSolution(Analyzer.__PARETO__) if pareto is None: return None return pareto.getSolutionPoints(0) def getFormattedValue(self, data1, data2, best, decimalFormat="%.4f", bestFormat="\\textbf{%s}"): if data1 is None: return "---" if isinstance(data1, types.StringType): return data1 value = decimalFormat % data1 if data2 is not None: value += " / " + (decimalFormat % data2) if best: value = bestFormat % value return value def getCurrentBestResult(self): convIdx = len(self.metrics.labels) - 2 distIdx = convIdx + 1 convergence = self.metrics.metricMean[convIdx] distribution = self.metrics.metricMean[distIdx] best = 0 for i in xrange(1, self.nResults - 1): if convergence[i] > convergence[best] + Utils.__EPS__: best = i elif abs(convergence[i] - convergence[best]) < Utils.__EPS__ and distribution[i] > distribution[best]: best = i return best def generateMetricImage(self, functionName, metricName, metricIdx, filename, latex=False): print " Generating figure for metric %s in problem %s" % (metricName, functionName) results = [] for i in xrange(self.nResults - 1): name = Utils.getResultNameLatex(self.resultNames[i]) if latex else self.resultNames[i] result = [name, [self.metrics.metricMin[metricIdx][i], self.metrics.metricQ1[metricIdx][i], self.metrics.metricMean[metricIdx][i], \ self.metrics.metricQ3[metricIdx][i], self.metrics.metricMax[metricIdx][i]], Analyzer.__COLORS__[i+1]] results.append(result) fname = Utils.getFunctionNameLatex(functionName) if latex else functionName mname = Utils.getMetricNameLatex(metricName) if latex else metricName self.plotter.plotIndicators(results, fname, "", "", mname, filename) def _getRun(self, functionName, resultName, rank=0.0): results = self.getFunctionResults(functionName, [resultName]) runs = [] pareto = self.getFunctionPareto(functionName) metrics = Metrics(pareto, [results[0][1]]) for run in xrange(len(results[0][1])): metrics.setSolutionsToCompare(0, run, None, None) value = metrics.deltaP() runs.append([value, run]) runs.sort(key = lambda x : x[0]) idx = max(0, int(round(len(runs) * rank)) - 1) return runs[idx][1] def generateImages(self, functionName, results, filenames, rank=0.0, latex=False): function = self.functions[functionName.lower()] window = [[1<<30, -(1<<30)] for _ in xrange(self.metrics.dim)] bestRun = [0] * len(results) for point in self.getFunctionPareto(functionName): for d in xrange(self.metrics.dim): window[d][0] = min(window[d][0], point[d]) window[d][1] = max(window[d][1], point[d]) for i in xrange(len(results)): resultName = results[i] bestRun[i] = self._getRun(functionName, resultName, rank) run = self.getFunctionResults(functionName, [resultName])[0][1][bestRun[i]] for point in run: for d in xrange(self.metrics.dim): #window[d] = max(window[d], math.ceil(point[d] * 10) / 10.0) window[d][0] = min(window[d][0], point[d]) window[d][1] = max(window[d][1], point[d]) for i in xrange(len(results)): resultName = results[i] print " Generating figure of %s for %s" % (resultName, functionName) resultNames = [Analyzer.__PARETO__, resultName] generation = [0, bestRun[i]] self.exportToImage(function, generation, True, resultNames, filenames[i], latex, window) def computeMetrics(self, functionName): pareto = self.getFunctionPareto(functionName) results = self.getFunctionResults(functionName, self.resultNames) self.metrics.computeMetrics(pareto, results, functionName)