def fitness_func(chromosome: Chromosome) -> float:
fitness = 0.0
create_tf_session()
params = chromosome.values
print("Current Chromosome Params:")
print(params)
model = Model(train_x,
train_y,
validation_x,
validation_y,
preprocessor.get_seq_info_str(),
architecture=Architecture.GRU.value,
is_bidirectional=False,
batch_size=round(params["batch_size"]),
hidden_layers=round(params["hidden_layers"]),
neurons_per_layer=round(params["neurons_per_layer"]),
dropout=params["dropout"],
initial_learn_rate=params["initial_learn_rate"],
is_classification=IS_CLASSIFICATION)
model.train()
if IS_CLASSIFICATION:
fitness = -model.score["sparse_categorical_crossentropy"]
chromosome.other["sparse_categorical_crossentropy"] = model.score[
"sparse_categorical_crossentropy"]
chromosome.other["accuracy"] = model.score["accuracy"]
else:
fitness = model.score["RSquaredMetric"]
chromosome.other["mae"] = model.score["mae"]
## Cleanup
del model
K.clear_session()
return fitness
def db(params):
if len(params) == 0:
return
if params[0] == 'list':
dbs = os.listdir("./db/")
App_tpl.list_dbs(dbs)
elif params[0] == 'use' and len(params) == 2:
print("Now using: " + params[1])
Model.init(params[1])
def test_word_recognition(self):
model = Model(open('../model/charList.txt').read(), "../model/")
scan_number = 0
line_number = 7
word_number = 0
print(test_word(scan_number, line_number, word_number, model))
self.assertEqual(True, True)
def run_program(pubquiz_answer_sheets, save_image=False, db=None):
print("De officiele Ordina pub-quiz antwoord vinder")
model = Model(open('model/charList.txt').read())
answersheet_id = -1
for answer_sheets in pubquiz_answer_sheets:
# The pdf file. We can it and it returns 1 to multiple answer pages
pages = convert_pdf_to_image(answer_sheets)
for p in range(0, len(pages)):
# We take the name from the file. But we want it without any extension.
file_extension = os.path.splitext(answer_sheets)
sheet_name = answer_sheets
if file_extension[1] == ".pdf":
sheet_name = sheet_name[0:-4]
sheet_name = sheet_name + "_" + str(p)
answersheet_id = save_to_database.save_answersheet_database(
db, pages[p])
if answersheet_id == -1:
print("something went wrong, no answersheet id present")
exit()
else:
process_sheet(pages[p], model, save_image, sheet_name, db,
answersheet_id)
def __init__(self, app: QApplication):
super(Ui, self).__init__()
self.parent = app
self.model = Model(n=50)
self.ui = uic.loadUi('MainWindow.ui', self)
self.ui.openFile.clicked.connect(self.onOpenFile)
self.ui.exec.clicked.connect(self.Exec)
self.ui.center()
self.ui.setWindowIcon(QIcon('web.jpg'))
self.init_menu()
self.statusBar().showMessage('Готов к работе...')
self.show()
def test_test_word(self):
model = Model(open('../model/charList.txt').read(), "../model/")
path = "test_words/test_1.PNG"
word = cv2.imread(path)
show_image(word)
if word is not None:
read_results = read_word_from_image(word, model)
return read_results
return None
def test_all_words(self):
model = Model(open('../model/charList.txt').read(), "../model/")
scan_path = "test_files/word_files/"
scans = [scans for scans in os.listdir(scan_path)]
for scan in range(0, len(scans)):
lines = [lines for lines in os.listdir(scan_path + scans[scan])]
for line in range(0, len(lines)):
words = [words for words in os.listdir(scan_path + scans[scan] + "/" + lines[line])]
for word in range(0, len(words)):
print(test_word(scan, line, word, model))
# If it arrives here without errors it has successfully attempted to recognize all words without errors
self.assertTrue(True)
def run_pubquiz_program(answer_sheets):
print("De officiele Ordina pub-quiz antwoord vinder")
model = Model(open('model/charList.txt').read())
# Not the cleanest way to detect an error, but the fastest to implement
problem = False
for p in convert_pdf_to_image(answer_sheets):
if p is not None:
try:
print("start processing answersheet")
if process_sheet(p, model):
problem = True
except:
return "Bestand uploaden mislukt. Het bestand kan niet uitgelezen worden."
if problem:
return "Bestand uploaden mislukt. Het bestand kan niet uitgelezen worden."
def load_model(self, filename):
self.model = Model.Model()
self.model.initialize(filename)
self.open_file = filename
self.opened = True
self.view.setWindowTitle("MPect - " + self.open_file)
self.mode = "view"
self.labelText.setText("Opened " + self.open_file)
self.showPlot()
self.mode = "slice"
self.view.plotText.setText("<p>Slice Mode</p>")
self.view.resetAllRatios()
def train_model():
preprocessor = DataPreprocesser(
f"{os.environ['WORKSPACE']}/data/crypto/{SYMBOL_TO_PREDICT}.parquet",
col_names=["open", "high", "low", "close", "volume"],
forecast_col_name="close",
sequence_length=100,
forecast_period=10,
is_classification=IS_CLASSIFICATION)
if not preprocessor.has_loaded and SHOULD_USE_INDICATORS:
indicator_df = get_select_indicator_values(preprocessor.df_original)
preprocessor.change_data(indicator_df)
preprocessor.print_df()
preprocessor.print_df_no_std()
preprocessor.preprocess()
train_x, train_y = preprocessor.get_train()
validation_x, validation_y = preprocessor.get_validation()
# Best hyperparams from GA:
# batch_size=1534,
# hidden_layers=2,
# neurons_per_layer=60,
# dropout=0.4714171367290059,
# initial_learn_rate=0.003725545984696872,
model = Model(train_x,
train_y,
validation_x,
validation_y,
preprocessor.get_seq_info_str(),
architecture=Architecture.GRU.value,
is_bidirectional=False,
batch_size=1534,
hidden_layers=2,
neurons_per_layer=60,
dropout=0.4164892976819227,
initial_learn_rate=0.004587161725770879,
early_stop_patience=10,
max_epochs=100,
is_classification=IS_CLASSIFICATION)
preprocessor.print_dataset_totals()
del preprocessor # Save memory
model.train()
model.save_model()
class Ui(QMainWindow):
parent = False
def __init__(self, app: QApplication):
super(Ui, self).__init__()
self.parent = app
self.model = Model(n=50)
self.ui = uic.loadUi('MainWindow.ui', self)
self.ui.openFile.clicked.connect(self.onOpenFile)
self.ui.exec.clicked.connect(self.Exec)
self.ui.center()
self.ui.setWindowIcon(QIcon('web.jpg'))
self.init_menu()
self.statusBar().showMessage('Готов к работе...')
self.show()
menuActions = {}
menuSubActions = {}
def onOpenFile(self):
self.statusBar().showMessage('Выберите файл...')
settings = QSettings()
ref = settings.value('settings/ref', '', type=str)
data = settings.value('settings/data', '', type=str)
if Validator.fileExist(ref) and Validator.fileExist(data):
question = QMessageBox.question(
self, 'Кэш',
'Использовать прошлые файлы, чтобы не выбирать новые? \n{}\n{}'
.format(ref, data), QMessageBox.Yes | QMessageBox.No)
if question == QMessageBox.Yes:
self.model.set_two_files(data, ref)
return QMessageBox.information(
self, "Успешно", "Теперь можно настраивать ограничения")
QMessageBox.information(self, "Файл",
"Выберите сначала ref файл, а затем data")
fileRef = QFileDialog.getOpenFileName(self, "Выберите ref файл")[0]
fileData = QFileDialog.getOpenFileName(self, "Выберите data файл")[0]
if Validator.fileExist(fileRef) is False or Validator.fileExist(
fileData) is False:
return QMessageBox.warning(
self, 'Ошибка', 'Вы не выбрали файл или его не существует')
settings.setValue('settings/ref', fileRef)
settings.setValue('settings/data', fileData)
self.model.set_two_files(fileData, fileRef)
return QMessageBox.information(self, "Успешно",
"Теперь можно настраивать ограничения")
def Exec(self):
self.model.set_delete_num(self.ui.deleteLine.text())
self.model.set_speed_limit(self.ui.speedLine.text())
self.model.set_average_num(self.ui.averageLine.text())
ret = False
try:
ret = self.model.from_two_files()
except BaseException as e:
print(str(e))
if (ret):
QMessageBox.information(self, "Успешно",
"Операция успешно завершилась")
self.statusBar().showMessage('Операция успешно завершена')
else:
self.statusBar().showMessage('Произошла ошибка')
def init_menu(self):
self.ui.addMenuMain("Файл")
self.ui.AddMenuSub("Файл", "Выход", "exit.png", "close")
self.setWindowTitle("ADCP. Главное окно")
def AddMenuSep(self, name):
self.menuActions[name].addSeparator()
def addMenuMain(self, name: str):
self.menuActions[name] = self.menuBar().addMenu(name)
def AddMenuSub(self, mainPoint: str, name: str, icon: str, connect):
self.menuSubActions[name] = QAction(QIcon(icon), name, self)
if (type(connect) == type("str") and connect == 'close'):
self.menuSubActions[name].triggered.connect(self.close)
else:
self.menuSubActions[name].triggered.connect(connect)
self.menuActions[mainPoint].addAction(self.menuSubActions[name])
def center(self):
qr = self.frameGeometry()
cp = QDesktopWidget().availableGeometry().center()
qr.moveCenter(cp)
self.move(qr.topLeft())
#! /usr/bin/python3 from app.App import App from app.Model import Model # APP GLOBALS # INIT Model.init() App.init() # MAIN App.start()
def test_model():
# model = Model.instance('Test_OTS_database.xlsx', is_trace=True)
# model = Model.instance('OTS_database.xlsx', is_trace=True)
model = Model.instance('OTS_database_dup.xlsx', is_trace=True)
model.is_trace = True
def multi_test(SYMBOL_TO_PREDICT: str):
params = {
"architecture": Architecture.GRU.value,
"is_bidirectional": False,
"indicators": False,
"sequence_length": 200,
"forecast_period": 10,
}
tests = [
# Format:
# (num_repetitions, {"param to change": new_value})
(5, {"architecture": Architecture.LSTM.value, "is_bidirectional": False,}),
(5, {"architecture": Architecture.GRU.value, "is_bidirectional": False,}),
(5, {"architecture": Architecture.LSTM.value, "is_bidirectional": True,}),
(5, {"architecture": Architecture.GRU.value, "is_bidirectional": True,}),
(5, {"indicators": False}),
(5, {"indicators": True}),
(1, {"sequence_length": 50}),
(1, {"sequence_length": 100}),
(1, {"sequence_length": 150}),
(1, {"sequence_length": 200}),
(1, {"sequence_length": 250}),
(1, {"sequence_length": 300}),
(1, {"sequence_length": 350}),
(1, {"sequence_length": 400}),
(1, {"forecast_period": 1}),
(1, {"forecast_period": 5}),
(1, {"forecast_period": 10}),
(1, {"forecast_period": 20}),
(1, {"forecast_period": 30}),
(1, {"forecast_period": 50}),
]
preprocessor = DataPreprocesser(
f"{os.environ['WORKSPACE']}/data/crypto/{SYMBOL_TO_PREDICT}.parquet",
col_names=["open", "high", "low", "close", "volume"],
forecast_col_name="close",
forecast_period=params["forecast_period"],
sequence_length=params["sequence_length"],
should_ask_load=False,
is_classification=IS_CLASSIFICATION
)
preprocessor.preprocess()
folder = f"{os.environ['WORKSPACE']}/results/tests"
for test_num, test in enumerate(tests):
test_num = test_num
num_repeats = test[0]
additional_params = test[1]
new_params = {**params, **additional_params}
pre = None
try: pre = preprocessor
except: pass
if "sequence_length" in additional_params or "forecast_period" in additional_params or "indicators" in additional_params:
try:
del preprocessor
del pre
except: pass
pre = DataPreprocesser(
f"{os.environ['WORKSPACE']}/data/crypto/{SYMBOL_TO_PREDICT}.parquet",
col_names=["open", "high", "low", "close", "volume"],
forecast_col_name="close",
forecast_period=new_params["forecast_period"],
sequence_length=new_params["sequence_length"],
should_ask_load=False,
is_classification=IS_CLASSIFICATION
)
pre.preprocess()
if new_params["indicators"]:
indicator_df = get_select_indicator_values(pre.df_original)
pre.change_data(indicator_df)
pre.print_df()
pre.print_df_no_std()
for i in range(num_repeats):
create_tf_session()
print(f"Test {test_num} repeat {i}")
print("Params")
print(new_params)
train_x, train_y = pre.get_train()
validation_x, validation_y = pre.get_validation()
model = Model(
train_x, train_y, validation_x, validation_y,
pre.get_seq_info_str(),
architecture=new_params["architecture"],
is_bidirectional=new_params["is_bidirectional"],
batch_size=1024,
hidden_layers=2,
neurons_per_layer=100,
dropout=0.2,
initial_learn_rate=0.001,
is_classification=IS_CLASSIFICATION
)
model.train()
train_time = float(model.train_time) / 60.0 # In Minutes
file_path = f"{folder}/test{test_num}.csv"
if not os.path.exists(file_path):
with open(file_path, 'a') as file:
if IS_CLASSIFICATION:
file.write("Accuracy,Sparse Categorical Crossentropy,Train Time\n")
else:
file.write("R Square,MAE,Train Time\n")
with open(file_path, 'a') as file:
if IS_CLASSIFICATION:
file.write(f"{model.score['accuracy']},{model.score['sparse_categorical_crossentropy']},{train_time}\n")
else:
file.write(f"{model.score['RSquaredMetric']},{model.score['mae']},{train_time}\n")
## Cleanup
del model
K.clear_session()
