def __init__(self, model):
QtGui.QMainWindow.__init__(self)
# model, holds data.
self.model = model
# used to validate user inputs.
self.validate = Validation()
# main view.
self.setGeometry(300, 300, 500, 380)
self.setWindowTitle('Air Quality Model')
# smaller search bar.
self.search = Search(self.model.schemas)
# connect search bar widget to the controler. The controller connects the search bar to the Table of data.
self.search.procDone.connect(self._widgetSearch)
# connection to show the table search.
self.search.procTable.connect(self._getTables)
# menu bar.
self.addMenuBar()
def __init__(self, model):
QtGui.QMainWindow.__init__(self)
# model, holds data.
self.model = model
# used to validate user inputs.
self.validate = Validation()
# main view.
self.setGeometry(300, 300, 500, 380)
self.setWindowTitle('Air Quality Model')
# smaller search bar.
self.search = Search(self.model.schemas)
# connect search bar widget to the controler. The controller connects the search bar to the Table of data.
self.search.procDone.connect(self._widgetSearch)
# connection to show the table search.
self.search.procTable.connect(self._getTables)
# menu bar.
self.addMenuBar()
def get_trained_best_network(self, num_augs=1):
"""selects the best network, and trains it fully"""
best_graphs = self.get_highest_accuracy(num=max(len(self.best_members) - 8, 1), check_set=self.best_members)
best = best_graphs[0]
print("Fully training", Config.run_name, "reported acc:", best.fitness_values[0])
augs = [x.data_augmentation_schemes[0] for x in best_graphs if len(x.data_augmentation_schemes) > 0]
aug_names = set()
unique_augs = []
for aug in augs:
name = repr(aug).split("Nodes:")[1].replace("'No_Operation'", "").replace("[]", "").replace('\\n',
"").replace(",",
"").replace(
'"', "").replace(" ", "")
if name not in aug_names:
aug_names.add(name)
unique_augs.append(aug)
unique_augs = unique_augs[:num_augs]
return Validation.get_fully_trained_network(best, unique_augs, num_epochs=Config.num_epochs_in_full_train)
class Controller(QtGui.QMainWindow):
'''
Create the starting window.
@param model: data from the db nicely ordered.
'''
def __init__(self, model):
QtGui.QMainWindow.__init__(self)
# model, holds data.
self.model = model
# used to validate user inputs.
self.validate = Validation()
# main view.
self.setGeometry(300, 300, 500, 380)
self.setWindowTitle('Air Quality Model')
# smaller search bar.
self.search = Search(self.model.schemas)
# connect search bar widget to the controler. The controller connects the search bar to the Table of data.
self.search.procDone.connect(self._widgetSearch)
# connection to show the table search.
self.search.procTable.connect(self._getTables)
# menu bar.
self.addMenuBar()
'''
Add menu bar to program.
'''
def addMenuBar(self):
# exit program action
exitAction = QtGui.QAction('Exit', self)
exitAction.setShortcut('Ctrl+Q')
exitAction.setStatusTip('Exit application')
exitAction.triggered.connect(self.close)
# create new model action.
modelAction = QtGui.QAction('New Model', self)
modelAction.setStatusTip('New model')
modelAction.triggered.connect(self._newModel)
# View a old model action.
viewAction = QtGui.QAction('View Model', self)
viewAction.setStatusTip('View model')
viewAction.triggered.connect(self._viewModel)
# add a menu bar.
menubar = self.menuBar()
fileMenu = menubar.addMenu('&File')
fileMenu.addAction(exitAction)
fileMenu.addAction(modelAction)
fileMenu.addAction(viewAction)
'''
Receives signals from Search object. Creates a new table from it.
@param schema: Message from Search. Name of the schema to create data model from.
@return: Table view that is made into the central widget.
@QtCore.pyqtSlot(str): Slot for receiving message.
'''
def _widgetSearch(self, data):
# convert to lower case b/c all of the db is saved as lower case.
schema = str(data[0]).lower()
table = str(data[1]).lower()
# create a data model from the db. summedemissions
self.model.newModel(schema, table)
# Create a new table view.
self.table = Table(self.model.model, self.model.header, self)
# add text to top of table.
self.table.leSchema.setText(schema)
self.table.leTable.setText(table)
# make table the central widget.
self.setCentralWidget(self.table)
# raise any errors.
self.raise_()
'''
Used in the Search view. Once a schema has been selected,
then allow the user to select a table to pick data from.
@param schema: Schema in the db corresponding to a run title.
'''
def _getTables(self, schema):
tables = self.model.getTables(schema)
self.search.addTable(tables)
'''
Set the central widget to the NewModel widget to create a new model
scenario.
'''
def _newModel(self):
# create new model.
self.newModel = NewModel()
# connect the new model to the controller.
self.newModel.procDone.connect(self._runNewModel)
# set new model screen to main widget.
self.setCentralWidget(self.newModel)
# close ferts.
self.newModel.closeAllFerts()
# add status bar.
self.statusBar().showMessage('Ready to Run New Model')
'''
Run a new model.
@param inputs: Inputs from the NewModel view.
A dictionary with various values such as title and run codes. dict(string, list)
'''
def _runNewModel(self, inputs):
self.statusBar().showMessage('Starting New Model')
inputs = Inputs(inputs)
# only used for validation purposes. Are boolean.
self.validate.title(inputs.title)
self.validate.runCodes(inputs.run_codes)
self.validate.fertDist(inputs.fertDist)
self.validate.ferts(inputs.ferts)
self.validate.pest(inputs.pestFeed)
print 'Fertilizer distribution.'
print inputs.fertDist
print 'Allocation from cg to ws and cs.'
print inputs.alloc
print 'run codes.'
print inputs.run_codes
print 'feed stocks using fertilizers.'
print inputs.ferts
print 'feed stocks using pesticides.'
print inputs.pestFeed
print 'Operations for feed stocks'
print inputs.operations
# make sure all of the variables to run the model have been created.
if not self.validate.errors:
self.statusBar().showMessage('Initiating Air Model.')
# get db and add schema
db = self.model.db
db.schema = inputs.title
# create the subprocess module that will run NONROAD in the background.
self.qprocess = QtCore.QProcess(self)
# send signal when subprocess has started.
self.qprocess.started.connect(self._processStart)
# when the subprocess is finished, it will send a signal to the controller to finish running the air model.
self.qprocess.finished.connect(self._processSave)
# read any new data.
self.qprocess.readyReadStandardOutput.connect(self._processStatus)
# send signal if a error has occured.
self.qprocess.error.connect(self._processError)
# create air model.
self.airModel = Driver(inputs.title, inputs.run_codes, db)
#self.airModel.setupNONROAD()
# create progress bar before running NONROAD model, to keep track of progress.
self.timer = 0
#create a global fertilizer to pass to the model.
self.fertDist = inputs.fertDist
# weather each feed stock should calculate emmisions from fertilizers.
self.ferts = inputs.ferts
# weather some feedstocks should calculate emmisions from pesticides.
self.pestFeed = inputs.pestFeed
# which operations to use for run.
self.operations = inputs.operations
self.alloc = inputs.alloc
# grab the total number of files that need to be ran.
batchFiles = self.airModel.batch.getBatchFiles()
self.bar = QtGui.QProgressBar()
self.bar.setRange(self.timer, batchFiles)
self.bar.setValue(self.timer)
self.bar.setWindowTitle("NONROAD Progress")
self.bar.show()
# run NONROAD.
#self.airModel.runNONROAD(self.qprocess)
self.airModel.saveData(self.ferts, self.fertDist, self.pestFeed, self.operations, self.alloc)
# if not able to validate inputs.
else:
self.statusBar().showMessage('ERROR: could not run model')
# output errors in pop up boxes.
for error in self.validate.getErrors():
QtGui.QMessageBox.about(self, "Validation", error)
'''
Check if qprocess has began running the NONROAD model.
@param qprocess: QtCore.QProcess, runs subprocess. Emits signal.
'''
def _processStart(self):
self.statusBar().showMessage('Running NONROAD')
'''
Uses a progress bar to update the user on how far the NONROAD model has ran.
Works by seeing how many files the NONROAD model has to process and making this te range for the bar.
The subprocess reads data from the NONROAD output, and detects when it has completed a file.
Every time this occurs a timer is updated, and the bar increase in percentage.
'''
def _processStatus(self):
# get the current data from the subprocess.
output = self.qprocess.readAllStandardOutput()
# check to see if a file has been finished processing.
if 'Successful completion' in output:
self.timer += 1
# Increase progress bar value.
self.bar.setValue(self.timer)
'''
Save the data from the air model, once the subprocess is finished.
Slot connecting Air model to the Controller.
@param qprocess: QtCore.QProcess, runs subprocess. Emits signal.
'''
def _processSave(self):
self.bar.close()
self.statusBar().showMessage('Finished Running NONROAD')
self.airModel.saveData(self.fertDist)
self._newModel()
'''
Tell the user if qprocess detects a error when running NONROAD.
@param qprocess: QtCore.QProcess, runs subprocess. Emits signal.
'''
def _processError(self):
self.statusBar().showMessage('ERROR: while running NONROAD')
'''
Show the search bar to look through the data.
'''
def _viewModel(self):
self.search.show()
def evaluate_blueprint(self, blueprint_individual, inputs, index):
blueprint_individual: BlueprintGenome
# Validation
if blueprint_individual.modules_used_index:
raise Exception('Modules used index is not empty',
blueprint_individual.modules_used_index)
if blueprint_individual.modules_used:
raise Exception('Modules used is not empty',
blueprint_individual.modules_used)
bpcp = copy.deepcopy(blueprint_individual)
# Testing old
s_constr = time.time()
blueprint_graph = blueprint_individual.to_blueprint()
module_graph = blueprint_graph.parse_to_module_graph(
self, allow_ignores=True if index >= Props.BP_POP_SIZE else False)
blueprint_individual: BlueprintGenome
if index == 1:
blueprint_individual.plot_tree_with_graphvis(view=True, file='bp')
for i, (spc, mod) in enumerate(
blueprint_individual.species_module_index_map.items()):
module = self.module_population.species[spc][mod]
module.plot_tree_with_graphvis(view=True, file='mod' + str(i))
module_graph.plot_tree_with_graphvis(view=True)
net = Validation.create_nn(module_graph, inputs)
old_construction_time = time.time() - s_constr
bpcp.species_module_index_map = blueprint_individual.species_module_index_map
Config.use_graph = True
bp = copy.deepcopy(bpcp)
s_constr = time.time()
new_net = Network(bp, self.module_population.species,
list(inputs.size())).to(Config.get_device())
new_construction_time = time.time() - s_constr
# Visualizing stuff
# blueprint_individual.plot_tree_with_graphvis(view=True, file='bp')
# blueprint_individual.modules_used[0].plot_tree_with_graphvis(view=True, file='mod')
# net.module_graph.plot_tree_with_graphvis(title='old', view=True, file='old')
# new_net.visualize(view=True)
# Number of parameters
# par = list(net.module_graph.module_graph_root_node.get_parameters({}))
# par.extend(net.final_layer.parameters())
# import numpy as np
# model_parameters = filter(lambda p: p.requires_grad, par)
# params = sum([np.prod(p.size()) for p in model_parameters])
# print('old', params)
# model_parameters = filter(lambda p: p.requires_grad, new_net.parameters())
# params = sum([np.prod(p.size()) for p in model_parameters])
# print('new', params)
if Config.evolve_data_augmentations:
if Config.allow_da_scheme_ignores and random.random(
) < Config.da_ignore_chance:
# ignore da scheme to try different one
blueprint_individual.da_scheme = None
da_indv = blueprint_individual.pick_da_scheme(
self.da_population)
else:
# use existing da scheme
da_indv = blueprint_individual.da_scheme
if da_indv.has_branches():
da_indv.plot_tree_with_graphvis()
raise Exception(
'Found data augmentation with branches, this should not happen'
)
da_scheme = da_indv.to_phenotype()
module_graph.data_augmentation_schemes.append(da_indv)
else:
da_scheme = None
# Testing old train time
s_train = time.time()
accuracy = Validation.get_accuracy_estimate_for_network(
net, da_scheme=da_scheme, batch_size=Config.batch_size)
old_train_time = time.time() - s_train
# -------------------Testing new phenotype------------------------------
# Creating the network with the same modules as used previously
Config.use_graph = False
n2 = Network(copy.deepcopy(bpcp), self.module_population.species,
list(inputs.size())).to(Config.get_device())
s_train = time.time()
new_acc = Validation.get_accuracy_estimate_for_network(
n2, da_scheme=None, batch_size=Config.batch_size)
new_train_time = time.time() - s_train
Config.use_graph = True
n3 = Network(bpcp, self.module_population.species,
list(inputs.size())).to(Config.get_device())
s_train = time.time()
new_acc_graph = Validation.get_accuracy_estimate_for_network(
n3, da_scheme=None, batch_size=Config.batch_size)
new_train_time_graph = time.time() - s_train
with open('traintime.txt', 'a+') as f:
f.write('\nnew:' + str(new_train_time))
f.write('\nnew_graph:' + str(new_train_time_graph))
f.write('\nold:' + str(old_train_time))
with open('constructiontime.txt', 'a+') as f:
f.write('\nnew:' + str(new_construction_time))
f.write('\nold:' + str(old_construction_time))
with open('acc.txt', 'a+') as f:
f.write('\nnew:' + str(new_acc))
f.write('\nnew_graph:' + str(new_acc_graph))
f.write('\nold:' + str(accuracy))
# End of logging
objective_names = [Config.second_objective, Config.third_objective]
results = [accuracy]
for objective_name in objective_names:
if objective_name == 'network_size':
results.append(net.module_graph.get_net_size())
elif objective_name == 'network_size_adjusted':
results.append(net.module_graph.get_net_size() /
(accuracy * accuracy))
elif objective_name == 'network_size_adjusted_2':
results.append(
math.sqrt(net.module_graph.get_net_size()) /
(accuracy * accuracy))
elif objective_name == '':
pass
else:
print("Error: did not recognise second objective",
Config.second_objective)
module_graph.delete_all_layers()
module_graph.fitness_values = results
return module_graph, blueprint_individual, results, new_acc, new_acc_graph, new_train_time, new_train_time_graph, old_train_time
class Controller(QtGui.QMainWindow):
'''
Create the starting window.
@param model: data from the db nicely ordered.
'''
def __init__(self, model):
QtGui.QMainWindow.__init__(self)
# model, holds data.
self.model = model
# used to validate user inputs.
self.validate = Validation()
# main view.
self.setGeometry(300, 300, 500, 380)
self.setWindowTitle('Air Quality Model')
# smaller search bar.
self.search = Search(self.model.schemas)
# connect search bar widget to the controler. The controller connects the search bar to the Table of data.
self.search.procDone.connect(self._widgetSearch)
# connection to show the table search.
self.search.procTable.connect(self._getTables)
# menu bar.
self.addMenuBar()
'''
Add menu bar to program.
'''
def addMenuBar(self):
# exit program action
exitAction = QtGui.QAction('Exit', self)
exitAction.setShortcut('Ctrl+Q')
exitAction.setStatusTip('Exit application')
exitAction.triggered.connect(self.close)
# create new model action.
modelAction = QtGui.QAction('New Model', self)
modelAction.setStatusTip('New model')
modelAction.triggered.connect(self._newModel)
# View a old model action.
viewAction = QtGui.QAction('View Model', self)
viewAction.setStatusTip('View model')
viewAction.triggered.connect(self._viewModel)
# add a menu bar.
menubar = self.menuBar()
fileMenu = menubar.addMenu('&File')
fileMenu.addAction(exitAction)
fileMenu.addAction(modelAction)
fileMenu.addAction(viewAction)
'''
Receives signals from Search object. Creates a new table from it.
@param schema: Message from Search. Name of the schema to create data model from.
@return: Table view that is made into the central widget.
@QtCore.pyqtSlot(str): Slot for receiving message.
'''
def _widgetSearch(self, data):
# convert to lower case b/c all of the db is saved as lower case.
schema = str(data[0]).lower()
table = str(data[1]).lower()
# create a data model from the db. summedemissions
self.model.newModel(schema, table)
# Create a new table view.
self.table = Table(self.model.model, self.model.header, self)
# add text to top of table.
self.table.leSchema.setText(schema)
self.table.leTable.setText(table)
# make table the central widget.
self.setCentralWidget(self.table)
# raise any errors.
self.raise_()
'''
Used in the Search view. Once a schema has been selected,
then allow the user to select a table to pick data from.
@param schema: Schema in the db corresponding to a run title.
'''
def _getTables(self, schema):
tables = self.model.getTables(schema)
self.search.addTable(tables)
'''
Set the central widget to the NewModel widget to create a new model
scenario.
'''
def _newModel(self):
# create new model.
self.newModel = NewModel()
# connect the new model to the controller.
self.newModel.procDone.connect(self._runNewModel)
# set new model screen to main widget.
self.setCentralWidget(self.newModel)
# close ferts.
self.newModel.closeAllFerts()
# add status bar.
self.statusBar().showMessage('Ready to Run New Model')
'''
Run a new model.
@param inputs: Inputs from the NewModel view.
A dictionary with various values such as title and run codes. dict(string, list)
'''
def _runNewModel(self, inputs):
self.statusBar().showMessage('Starting New Model')
inputs = Inputs(inputs)
# only used for validation purposes. Are boolean.
self.validate.title(inputs.title)
self.validate.runCodes(inputs.run_codes)
self.validate.fertDist(inputs.fertDist)
self.validate.ferts(inputs.ferts)
self.validate.pest(inputs.pestFeed)
print 'Fertilizer distribution.'
print inputs.fertDist
print 'Allocation from cg to ws and cs.'
print inputs.alloc
print 'run codes.'
print inputs.run_codes
print 'feed stocks using fertilizers.'
print inputs.ferts
print 'feed stocks using pesticides.'
print inputs.pestFeed
print 'Operations for feed stocks'
print inputs.operations
# make sure all of the variables to run the model have been created.
if not self.validate.errors:
self.statusBar().showMessage('Initiating Air Model.')
# get db and add schema
db = self.model.db
db.schema = inputs.title
# create the subprocess module that will run NONROAD in the background.
self.qprocess = QtCore.QProcess(self)
# send signal when subprocess has started.
self.qprocess.started.connect(self._processStart)
# when the subprocess is finished, it will send a signal to the controller to finish running the air model.
self.qprocess.finished.connect(self._processSave)
# read any new data.
self.qprocess.readyReadStandardOutput.connect(self._processStatus)
# send signal if a error has occured.
self.qprocess.error.connect(self._processError)
# create air model.
self.airModel = Driver(inputs.title, inputs.run_codes, db)
#self.airModel.setupNONROAD()
# create progress bar before running NONROAD model, to keep track of progress.
self.timer = 0
#create a global fertilizer to pass to the model.
self.fertDist = inputs.fertDist
# weather each feed stock should calculate emmisions from fertilizers.
self.ferts = inputs.ferts
# weather some feedstocks should calculate emmisions from pesticides.
self.pestFeed = inputs.pestFeed
# which operations to use for run.
self.operations = inputs.operations
self.alloc = inputs.alloc
# grab the total number of files that need to be ran.
batchFiles = self.airModel.batch.getBatchFiles()
self.bar = QtGui.QProgressBar()
self.bar.setRange(self.timer, batchFiles)
self.bar.setValue(self.timer)
self.bar.setWindowTitle("NONROAD Progress")
self.bar.show()
# run NONROAD.
#self.airModel.runNONROAD(self.qprocess)
self.airModel.saveData(self.ferts, self.fertDist, self.pestFeed,
self.operations, self.alloc)
# if not able to validate inputs.
else:
self.statusBar().showMessage('ERROR: could not run model')
# output errors in pop up boxes.
for error in self.validate.getErrors():
QtGui.QMessageBox.about(self, "Validation", error)
'''
Check if qprocess has began running the NONROAD model.
@param qprocess: QtCore.QProcess, runs subprocess. Emits signal.
'''
def _processStart(self):
self.statusBar().showMessage('Running NONROAD')
'''
Uses a progress bar to update the user on how far the NONROAD model has ran.
Works by seeing how many files the NONROAD model has to process and making this te range for the bar.
The subprocess reads data from the NONROAD output, and detects when it has completed a file.
Every time this occurs a timer is updated, and the bar increase in percentage.
'''
def _processStatus(self):
# get the current data from the subprocess.
output = self.qprocess.readAllStandardOutput()
# check to see if a file has been finished processing.
if 'Successful completion' in output:
self.timer += 1
# Increase progress bar value.
self.bar.setValue(self.timer)
'''
Save the data from the air model, once the subprocess is finished.
Slot connecting Air model to the Controller.
@param qprocess: QtCore.QProcess, runs subprocess. Emits signal.
'''
def _processSave(self):
self.bar.close()
self.statusBar().showMessage('Finished Running NONROAD')
self.airModel.saveData(self.fertDist)
self._newModel()
'''
Tell the user if qprocess detects a error when running NONROAD.
@param qprocess: QtCore.QProcess, runs subprocess. Emits signal.
'''
def _processError(self):
self.statusBar().showMessage('ERROR: while running NONROAD')
'''
Show the search bar to look through the data.
'''
def _viewModel(self):
self.search.show()