class MainWindow(QWidget):
"""
Main window class for capraz_sevkiyat project
"""
def __init__(self):
QWidget.__init__(self)
self.model = None
self.setWindowTitle("Cross Docking Project")
self.setGeometry(400, 400, 400, 400)
self.set_buttons()
self.set_layout()
self.truck_image_list = {}
self.truckDataWindow = None
self.data = DataStore()
self.model = None
self.current_iteration = 1
self.iteration_limit = 100
self.current_data_set = 0
self.algorithms = None
self.solution_choice = None
self.scn = QGraphicsScene()
self.simulation = GraphView(self.scn)
def set_buttons(self):
self.new_data_set_button = QPushButton('New Data Set')
self.load_data_set_button = QPushButton('Load Data Set')
self.save_data_set_button = QPushButton('Save Data Set')
self.truck_data_button = QPushButton('Truck Data')
self.system_data_button = QPushButton('System Data')
self.algorithm_data_button = QPushButton('Algorithm Data')
self.generate_data_set_button = QPushButton('Generate Data Set')
self.show_data_button = QPushButton('Show Data Set')
self.print_gams_button = QPushButton('Print gams output')
self.data_set_ready_button = QPushButton('Data Set Ready')
self.solve_step_button = QPushButton('Solve Next Step')
self.solve_iteration_button = QPushButton('Solve Next Iteration')
self.solve_next_data_set_button = QPushButton('Solve Next Data Set')
self.show_logger_button = QPushButton('Show Logger')
self.show_simulation_button = QPushButton('Show Simulation')
self.show_data_table = QPushButton('Show Run Time Data Table')
self.data_set_number = QSpinBox()
self.data_set_number.setMinimum(0)
self.new_data_set_button.clicked.connect(self.new_data_set)
self.load_data_set_button.clicked.connect(self.load_data)
self.save_data_set_button.clicked.connect(self.save_data)
self.truck_data_button.clicked.connect(self.show_truck_data)
self.system_data_button.clicked.connect(self.show_system_data)
self.algorithm_data_button.clicked.connect(self.show_algorithm_data)
self.generate_data_set_button.clicked.connect(self.generate_data_set)
self.show_data_button.clicked.connect(self.show_data)
self.print_gams_button.clicked.connect(self.print_gams)
self.data_set_ready_button.clicked.connect(self.data_set_ready)
self.show_logger_button.clicked.connect(self.show_logger)
self.show_data_table.clicked.connect(self.show_runtime_table)
self.solve_next_data_set_button.clicked.connect(self.data_set_button)
self.solve_iteration_button.clicked.connect(self.iteration_button)
self.solve_step_button.clicked.connect(self.step_button)
self.data_set_number.valueChanged.connect(self.set_data_set_number)
def set_layout(self):
self.data_set_layout = QGridLayout()
self.data_set_layout.addWidget(self.new_data_set_button, 1 ,1)
self.data_set_layout.addWidget(self.load_data_set_button, 1 ,2)
self.data_set_layout.addWidget(self.save_data_set_button, 1 ,3)
self.data_set_layout.addWidget(self.truck_data_button, 2 ,1)
self.data_set_layout.addWidget(self.system_data_button, 2 ,2)
self.data_set_layout.addWidget(self.algorithm_data_button, 2 ,3)
self.data_set_layout.addWidget(self.generate_data_set_button, 3, 1)
self.data_set_layout.addWidget(self.show_data_button, 3, 2)
self.data_set_layout.addWidget(self.print_gams_button, 3, 3)
self.data_set_layout.addWidget(self.data_set_ready_button, 4, 1)
self.solver_layout = QGridLayout()
self.solver_layout.addWidget(self.solve_step_button, 1, 1)
self.solver_layout.addWidget(self.solve_iteration_button, 1, 2)
self.solver_layout.addWidget(self.solve_next_data_set_button, 1, 3)
self.solver_layout.addWidget(self.data_set_number, 1, 4)
self.interaction_layout = QGridLayout()
self.interaction_layout.addWidget(self.show_logger_button, 1, 1)
self.interaction_layout.addWidget(self.show_simulation_button, 1, 3)
self.interaction_layout.addWidget(self.show_data_table, 1, 4)
self.layout = QVBoxLayout()
self.layout.addLayout(self.data_set_layout)
self.layout.addLayout(self.solver_layout)
self.layout.addLayout(self.interaction_layout)
self.setLayout(self.layout)
self.pause_bool = False
def new_data_set(self):
"""
:return:
"""
self.data = DataStore()
def load_data(self):
"""
loads prev saved data
:return:
"""
file_name, _ = QFileDialog.getOpenFileName(self, 'Open file', '/home')
self.data = pickle.load(open(file_name, 'rb'))
def save_data(self):
"""
saves current data
:return:
"""
file_name, _ = QFileDialog.getSaveFileName(self, 'Save file', '/home')
pickle.dump(self.data, open(file_name, 'wb'))
def generate_data_set(self):
# ask if sure
self.data.arrival_times = []
self.data.boundaries = []
self.model = Solver(self.data)
for i in range(len(self.data.data_set_list)):
self.model.current_data_set = i
self.model.set_data()
def show_data(self):
self.data_show = ShowData(self.data)
self.data_show.exec_()
def print_gams(self):
file_name, _ = QFileDialog.getSaveFileName(self, 'Open file', '/home')
for i in range(len(self.data.data_set_list)):
gams_writer(file_name + str(i), i, self.data )
def show_truck_data(self):
"""
shows data about the trucks
:return:
"""
self.truckDataWindow = TruckDataWindow(self.data)
self.truckDataWindow.exec_()
def show_system_data(self):
"""
shows data set
:return:
"""
self.dataWindow = DataSetWindow(self.data)
self.dataWindow.exec_()
def show_algorithm_data(self):
pass
def data_set_ready(self):
# setup for one data set
self.algorithms = Algorithms()
self.model = Solver(self.data)
self.model.current_data_set = self.current_data_set
self.model.load_data_set()
self.algorithms.set_algorithms(self.model)
def show_logger(self):
self.logger = LogData()
root = logging.getLogger()
root.setLevel(logging.INFO)
ch = logging.StreamHandler(self.logger)
ch.setLevel(logging.INFO)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
root.addHandler(ch)
self.logger.show()
logging.info('Logger Started')
def data_set_button(self):
self.solution_choice = 'data_set'
self.solve_dataset()
def iteration_button(self):
self.solution_choice = 'iteration'
self.solution_type_choice()
self.solve_dataset()
def step_button(self):
self.solution_choice = 'step'
self.solution_type_choice()
self.solve_dataset()
def set_data_set_number(self):
self.current_data_set = self.data_set_number.value()
def solve_dataset(self):
"""
solves one data set
:return:
"""
logging.info('Data Set Number: {0}'.format(self.current_data_set))
self.model.current_data_set = self.current_data_set
if self.data_set_bool:
#print('one_set')1
if self.current_iteration == 1 and self.model.current_time == 0:
self.model.load_data_set()
self.solve_iteration()
if self.current_data_set == len(self.data.data_set_list):
# print('finish')
self.current_iteration = 1
self.current_data_set = 0
self.trial_time = 0
else:
while self.current_data_set < len(self.data.data_set_list):
if self.pause_bool:
break
self.model.load_data_set()
self.solve_iteration()
# print(self.current_data_set)
self.current_data_set = 0
def solve_iteration(self):
"""
solves one iteration
:return:
"""
if self.iteration_bool:
if self.model.current_time == 0:
if self.current_iteration == 1:
self.algorithms.start()
self.model.set_sequence(self.algorithms.solution_sequence)
self.solve_whole_step()
self.model.reset()
self.algorithms.next()
self.model.set_sequence(self.algorithms.solution_sequence)
else:
self.algorithms.next()
self.model.set_sequence(self.algorithms.solution_sequence)
self.solve_step()
if self.current_iteration == self.iteration_limit:
self.log_results()
self.current_iteration = 1
else:
while self.current_iteration < self.iteration_limit:
if self.pause_bool:
break
if self.model.current_time == 0:
if self.current_iteration == 1:
self.algorithms.start()
else:
self.algorithms.next()
self.model.set_sequence(self.algorithms.solution_sequence)
# next sequence
self.solve_step()
self.current_iteration = 1
self.log_results()
def solve_step(self):
if self.step_bool:
self.solve_one_step()
else:
self.solve_whole_step()
def solve_whole_step(self):
"""
solves one iterations
:return:
"""
while not self.model.finish:
# if self.model.current_time > 800:
#
# break
if self.pause_bool:
break
self.model.next_step()
#finished
for truck in itertools.chain(self.model.outbound_trucks.values(), self.model.compound_trucks.values()):
truck.calculate_error()
if self.runtime_table:
self.runtime_table.update_tables()
self.runtime_table.activateWindow()
#add reset
self.model.finish = False
self.algorithms.solution_sequence['error'] = self.add_errors()
self.model.reset()
if self.current_iteration > 1:
self.algorithms.calculate()
self.current_iteration += 1
def solve_one_step(self):
"""
goes one time step forward
:return:
"""
self.model.next_step()
# self.simulation.update_image()
if self.runtime_table:
self.runtime_table.update_tables()
self.runtime_table.activateWindow()
if self.model.finish:
#finished
logging.info("Finished iteration {0}".format(self.current_iteration))
for truck in self.model.outbound_trucks.values():
truck.calculate_error()
self.add_errors()
self.model.reset()
# add reset
self.current_iterpation += 1
self.model.finish = False
# update algorithm
self.algorithms.solution_sequence['error'] = self.add_errors()
self.algorithms.calculate()
def add_errors(self):
"""
adds absolute values of the errors
:return:
"""
total_error = 0
for truck in itertools.chain(self.model.outbound_trucks.values(), self.model.compound_trucks.values()):
total_error += abs(truck.error)
logging.info("Error: {0}\n".format(total_error))
return total_error
def show_runtime_table(self):
"""
shows data table of the
:return:
"""
self.runtime_table = TruckDataTable(self.algorithms, self.model)
self.runtime_table.show()
def simulation_cycle(self):
i = 0
for inbound_trucks in self.model.inbound_trucks.values():
truck_name = inbound_trucks.truck_name
self.truck_image_list[truck_name] = self.scn.addPixmap(self.truckPixmap)
self.truck_image_list[truck_name].scale(0.2,0.2)
self.truck_image_list[truck_name].setPos(-600,i*100)
i = i +1
self.simulation.show()
def solution_type_choice(self):
"""
update bools for the choosen solution type
:return:
"""
if self.solution_choice == 'solve':
self.solve_bool = True
self.data_set_bool = False
self.iteration_bool = False
self.step_bool = False
elif self.solution_choice == 'data_set':
self.solve_bool = True
self.data_set_bool = True
self.iteration_bool = False
self.step_bool = False
self.data_set_ready()
elif self.solution_choice == 'iteration':
self.solve_bool = True
self.data_set_bool = True
self.iteration_bool = True
self.step_bool = False
elif self.solution_choice == 'step':
self.solve_bool = True
self.data_set_bool = True
self.iteration_bool = True
self.step_bool = True
def print_simulation_data(self):
logging.info("Iteration Number: {0}\n".format(self.current_iteration))
logging.info("Inbound Sequence: {0}\n".format(self.algorithms.solution_sequence['inbound']))
logging.info("Outbound Sequence: {0}\n".format(self.algorithms.solution_sequence['outbound']))
logging.info("Error value: {0}\n".format(self.algorithms.solution_sequence['error']))
def log_results(self):
logging.info("Best result:")
logging.info("Inbound Sequence: {0}\n".format(self.algorithms.best_sequence['inbound']))
logging.info("Outbound Sequence: {0}\n".format(self.algorithms.best_sequence['outbound']))
logging.info("Error value: {0}\n".format(self.algorithms.best_sequence['error']))
def show_data(self):
self.data_show = ShowData(self.data)
self.data_show.exec_()
def main():
svhn_train_data, svhn_train_label = create_svhn_dataset("SVHN/train_32x32.mat")
svhn_pre_train_data = svhn_train_data[:60000]
svhn_pre_train_label = svhn_train_label[:60000]
svhn_fine_tuning_data = svhn_train_data[svhn_train_data.shape[0]-10000:]
svhn_fine_tuning_label = svhn_train_label[svhn_train_data.shape[0]-10000:]
svhn_test_data, svhn_test_label = create_svhn_dataset("SVHN/test_32x32.mat")
svhn_pre_test_data = svhn_train_data[6000:svhn_train_data.shape[0]-10000]
svhn_pre_test_label = svhn_train_label[6000:svhn_train_data.shape[0]-10000]
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
mnist_train_data = np.concatenate([mnist.train.images, mnist.validation.images])
mnist_train_label = np.concatenate([mnist.train.labels, mnist.validation.labels])
mnist_train_data = color_resize(mnist_train_data)
mnist_test_data = color_resize(mnist.test.images)
cnn = ConvolutionalNeuralNetwork(32, 10)
cnn.set_model(0.001)
show_data = ShowData()
sess = tf.Session()
saver = tf.train.Saver()
init = tf.global_variables_initializer()
sess.run(init)
saver.restore(sess, "model_m.dump")
save_model = "model_m_s.dump"
save_accuracy = "accuracy_curve_finetuning_mnist.png"
save_loss = "loss_curve_finetuning_mnist.png"
train_data = svhn_fine_tuning_data
train_label = svhn_fine_tuning_label
test_data = svhn_test_data
test_label = svhn_test_label
epoch = 500
# Pre-training
accuracy_list = []
loss_list = []
for i in range(epoch):
choice_id = np.random.choice(train_data.shape[0], 100, replace=False)
batch_data = train_data[choice_id]
batch_label = train_label[choice_id]
if i % 100 == 0:
accuracy = 0
for j in range(0, test_data.shape[0], 100):
data = test_data[j:j+100]
label = test_label[j:j+100]
accuracy += int(cnn.test(sess, data, label)[0]*data.shape[0])
print("step {}, training accuracy {}".format(i, accuracy/test_data.shape[0]*100.0))
_, loss = cnn.train(sess, batch_data, batch_label)
loss_list.append(loss)
accuracy = 0
for j in range(0, test_data.shape[0], 100):
data = test_data[j:j+100]
label = test_label[j:j+100]
accuracy += int(cnn.test(sess, data, label)[0]*data.shape[0])
accuracy_list.append(accuracy/test_data.shape[0]*100.0)
print("test accuracy {}".format(accuracy/test_data.shape[0]*100.0))
#show_data.show_accuracy_curve(accuracy_list, save_accuracy)
#show_data.show_loss_curve(loss_list, save_loss)
saver.save(sess, save_model)
saver.restore(sess, "model_s.dump")
save_model = "model_s_s.dump"
save_accuracy = "accuracy_curve_finetuning.png"
save_loss = "loss_curve_finetuning_svhn.png"
accuracy_list_ = []
loss_list_ = []
for i in range(epoch):
choice_id = np.random.choice(train_data.shape[0], 100, replace=False)
batch_data = train_data[choice_id]
batch_label = train_label[choice_id]
if i % 100 == 0:
accuracy = 0
for j in range(0, test_data.shape[0], 100):
data = test_data[j:j+100]
label = test_label[j:j+100]
accuracy += int(cnn.test(sess, data, label)[0]*data.shape[0])
print("step {}, training accuracy {}".format(i, accuracy/test_data.shape[0]*100.0))
_, loss = cnn.train(sess, batch_data, batch_label)
loss_list_.append(loss)
accuracy = 0
for j in range(0, test_data.shape[0], 100):
data = test_data[j:j+100]
label = test_label[j:j+100]
accuracy += int(cnn.test(sess, data, label)[0]*data.shape[0])
accuracy_list_.append(accuracy/test_data.shape[0]*100.0)
print("test accuracy {}".format(accuracy/test_data.shape[0]*100.0))
show_data.show_accuracy_curves(accuracy_list, accuracy_list_, save_accuracy)
#show_data.show_loss_curve(loss_list, save_loss)
saver.save(sess, save_model)
def main():
database = Database()
managedb = ManageDatabase()
showdata = ShowData()
database.create_database()
database.insert_data()
showdata.show_list_of_employees()
print(showdata.show_monthly_spending())
print(showdata.show_yearly_spending())
managedb.add_employee()
showdata.show_list_of_employees()
managedb.delete_employee()
showdata.show_list_of_employees()
print(showdata.show_monthly_spending())
print(showdata.show_yearly_spending())
managedb.update_employee()
showdata.show_all_employees()
def main():
"""Main training function. Use flags to sort the dataset or show the dataset.
"""
parser = argparse.ArgumentParser(
description="Training and Dataset functions")
parser.add_argument('--manage_dataset',
action='store_true',
help='if set, create dataset from raw data')
parser.add_argument('--show_dataset',
action='store_true',
help='if set, call show dataset')
parser.add_argument('--train',
action='store_true',
help='if set, train the network')
parser.add_argument('--data_path',
help='path to sorted data folder',
default='data/sorted_data')
parser.add_argument('--raw_data_path',
help='path to raw data folder',
default='data/raw_data')
parser.add_argument('--checkpoint_path',
help='path to store the checkpoints',
default='checkpoints')
parser.add_argument('--batch_size',
help='batch Size',
type=int,
default=64)
parser.add_argument('--epochs', help='epochs', type=int, default=50)
parser.add_argument('--learning_rate',
help='Learning Rate',
type=float,
default=1e-3)
parser.add_argument('--train_from_checkpoint',
help='Path to checkpoint',
default=None)
parser.add_argument('--logdir',
help='Path to Tensorboard Logs',
default="runs")
args = parser.parse_args()
#mode
manage_dataset = args.manage_dataset
show_dataset = args.show_dataset
train = args.train
#training
batch_size = args.batch_size
learning_rate = args.learning_rate
train_from_checkpoint = args.train_from_checkpoint
epochs = args.epochs
project_path = Path.cwd().parent.parent.parent
#paths
raw_data_path = project_path / args.raw_data_path
sorted_data_path = project_path / args.data_path
checkpoint_path = project_path / args.checkpoint_path
logdir_path = project_path / args.logdir
if manage_dataset:
# sort dataset
print('manage dataset')
data_manager = ManageData(raw_data_path, sorted_data_path)
data_manager.sort_dataset()
elif show_dataset:
# show dataset
print('show dataset')
data_show = ShowData(raw_data_path)
data_show.show()
elif train:
# training
print('start training')
train = Train(sorted_data_path, checkpoint_path, logdir_path)
train.train_network(batch_size, learning_rate, epochs,
train_from_checkpoint)
else:
raise Exception(
"No flag is set. Set either 'manage_dataset', 'show_dataset' or 'train' flag."
)
