def main():
validation = Validation('standard_data.json')
invalid = collections.defaultdict(set)
dir = os.path.dirname(__file__)
filename_in = os.path.join(dir, 'budapest_hungary.osm')
for _, element in ET.iterparse(filename_in):
address = get_address(element)
if address is not None:
# validate each keys one by one
for key in address.keys():
value = address[key]
# do not have to validate if we know it is invalid
if value not in invalid[key]:
# if invalid
if validation.validate(key, value) is False:
# store it to make sure we do not repeat this report
invalid[key].add(value)
# report the invalid value
print 'Invalid {0}: "{1}"'\
.format(key, value.encode('utf-8'))
def build(save = False):
infos = AllFolds()
val = Validation(infos)
val.buildPredictors()
if save:
val.savePredictors()
return val
def add(self, name, space): name = name + '.app' validate = Validation() if validate.matchAgainstDirectory(name): self.storage.addApplication(name, space) fuzzy_result = validate.fuzzyAgainstDirectory(name) if fuzzy_result != None: self.storage.addApplication(fuzzy_result, space)
def __init__(self,
data=None,
model=None,
training_data_span_months=1):
self._data_df = data
self._model = model
self.training_data_span_months = training_data_span_months
self.validation = Validation(data)
def fit(self, training_data):
# self.training_data = training_data
self.validation = Validation(training_data)
self._training_data = training_data
# Want to go for minimum mean training error * std.dev
if self.should_optimize:
optimized_args = self.optimize_interpolator()
self.x_offset = optimized_args[0]
self.x_divisor = optimized_args[1]
def __init__(self):
#importing the client name and raw table name passed as parameters
clientName = sys.argv[1]
fileName = sys.argv[2]
#if needed to give filepath and validate a file not in the same folder
#fileNamewithPath = sys.argv[2]
#tempfileName = re.findall('[A-Za-z0-9_-\s]+.csv',fileNamewithPath)
#fileName = tempfileName[0]
fileInfo = FileInfo()
fileInfo.setFileInfo(fileName)
sortFileType = SortFileType()
fileType,rawTableName = sortFileType.getFileType(clientName,fileName)
#print fileType
#print rawTableName
fileReader = FileReader(fileName)
tableMetadata = TableMetadata()
validation = Validation()
error = None
fileColumns = fileReader.readHeader()
dbColumns = tableMetadata.readTableHeader(clientName,rawTableName)
if validation.validateHeaders(dbColumns,fileColumns):
dataType = tableMetadata.readMetadata(clientName,rawTableName)
fileRows = fileReader.readRows()
for eachRow in fileRows :
self.rowNum = self.rowNum + 1
if validation.validateRow(dataType,eachRow,self.rowNum) is False:
error = "Error in dataType"
break
else:
error = "Error in header"
if error:
print error
else:
print"File sucessfully validated"
def signup():
if request.method == "POST":
un = request.form['username']
email = request.form['email']
password = request.form['password']
v = Validation()
if v.check_validation(un, email, password):
#put the user in the db
u = models.User(username = un, email = email, password = password)
db.session.add(u)
db.session.commit()
# render the template where the user is logged in
return render_template("user_logged.html")
else:
#validation did not succeed, showing the user a flash message
flash("Sorry, but some of your input is not enough secured")
return render_template("signup.html")
#get request for signup
return render_template("signup.html")
def __init__(self):
# Instantiate a data access object
# Contains methods to access the database
self.staff_dao = StaffDAO()
# Instantiate a validation object
# Contains methods to validate input fields
self.validator = Validation()
# Form fields
# Instantiate stringvars - hold data entered in fields of form
self.staff_id = tk.StringVar()
self.staff_name = tk.StringVar()
self.staff_surname = tk.StringVar()
self.staff_password = tk.StringVar()
# List of staff ids - lb for listbox
self.lb_ids = None
# Messagebox title
self.mb_title_bar = "Staff CRUD"
pass
def creat_task():
task_info = request.get_json()
if "title" in task_info.keys():
title = task_info.get("title")
status = "to-do"
task_id = len(todo_list)+1
logged_user = get_jwt_identity()
validate = Validation.validate_task(title)
if validate:
return jsonify({"message": validate}), 400
new_task = Task(task_id=task_id, title=title, owner=logged_user, status=status)
if(new_task.create_task()):
return jsonify({"New task Created": new_task.__dict__}), 201
return jsonify({"message": "Task not added"}), 400
return jsonify({"message": "a 'key' is missing in your task body"}), 400
def make_move(self, tictacgame):
if self.option == Level.CUSTOM:
while True:
coordinates = input("Enter the coordinates > ")
if not Validation.check_move_validity(coordinates,
tictacgame.matrix):
continue
x, y = coordinates.split()
x = int(x)
y = int(y)
tictacgame.make_move(x - 1, y - 1)
tictacgame.output()
break
elif self.option == Level.EASY or self.option == Level.MEDIUM or self.option == Level.HARD:
tictacgame.make_computer_move(self.option, self)
tictacgame.output()
def __init__(self, data):
SkuBase.__init__(self, data)
if 'commentList' in self.data.keys():
comments = json.loads(self.data.pop('commentList'))
self.data['commentList'] = list()
for comment in comments:
comment['commentData'] = unhexlifyUtf8(
comment.pop('commentData'))
if Validation.isCommentBad(comment['commentData']):
continue
self.data['commentList'].append(comment)
def generateList(linked_list: Linked_list, lname='l'):
options = ('🗿 🗿 🗿 🗿 🗿 🗿 🗿 🗿\n' + '🗿 1 - Strategy 1 🗿\n' +
'🗿 2 - Strategy 2 🗿\n' + '🗿 3 - generate data 🗿\n' +
'🗿 4 - print list 🗿\n' + '🗿 5 - exit 🗿\n' +
'🗿 🗿 🗿 🗿 🗿 🗿 🗿 🗿\n')
context = Context(StrategyIterator)
while True:
print(options)
choice = v.intValidateInRange('Enter choice ', 1, 5)
if choice == 1: context.strategy = StrategyIterator()
elif choice == 2: context.strategy = StrategyReadFile()
elif choice == 3:
linked_list = generateMenu(linked_list, context, lname)
elif choice == 4:
print(linked_list)
elif choice == 5:
return linked_list
def register_user():
reg_info = request.get_json()
search_keys = ("username", "email", "password")
if all(key in reg_info.keys() for key in search_keys):
user_name = reg_info.get("username")
email = reg_info.get("email")
password = reg_info.get("password")
validate = Validation.auth_validation(user_name=user_name, email=email, password=password)
if validate:
return jsonify({"message": validate}), 400
new_user = User(username=user_name, email=email, password=password)
if (new_user.add_account()):
return jsonify({"New User Created": new_user.__dict__}), 201
else:
return jsonify({"message": "User not added or username already exists"}), 400
return jsonify({"message": "a 'key(s)' is missing in your registration body"}), 400
def buy_product(self):
self.display_product(False)
max_index = len(Product.list_products())
while 1:
i = Validation.get_int_input(
input(
"Enter Index of Product You Want To Buy or 0 to cancel : "
),
max_index,
True,
)
if i == 0:
break
current_product = Product.list_products()[i - 1]
if int(current_product.stock_quanitity) > 0:
current_product.purchase_product(Authentication.user)
break
else:
print("Out Of Stock")
self.logged_in_menu()
def login():
login_info = request.get_json()
search_keys = ("username", "password")
if all(key in login_info.keys() for key in search_keys):
user_name = login_info.get("username").strip()
password = login_info.get("password")
validate = Validation.login_validation(user_name, password)
if validate:
return jsonify({"message": validate}), 400
login = User.login(username=user_name, password=password)
user_token = {}
if login:
access_token = create_access_token(identity= user_name)
refresh_token = create_refresh_token(identity=user_name)
user_token["access_token"] = access_token
user_token["refresh_token"] = refresh_token
return jsonify(user_token), 200
return jsonify({"message": "user does not exist, register and login again"}), 404
return jsonify({"message": "a 'key(s)' is missing in login body"}), 400
def add_user():
request_form = [('alpha', 'first_name', request.form['first_name']),
('alpha', 'last_name', request.form['last_name']),
('email', 'email', request.form['email']),
('pass_check', 'password', request.form['password'],
request.form['confirm_password'], 8, 16)]
sanitize = Validation(request_form)
if len(sanitize.errors) == 0:
query = 'INSERT INTO users (first_name, last_name, email, password, created_at, updated_at) VALUES ( :first_name, :last_name, :email, :password, now(), now())'
data = sanitize.data
user_id = mysql.query_db(query, data)
flash("You have successfully registered! You user ID is " +
str(user_id) + ".")
if 'data' in session:
session.pop('data')
return redirect('/')
else:
session['data'] = sanitize.data
for error in sanitize.errors:
flash(error)
return redirect('/')
def add_user():
request_form = [('alpha', 'first_name', request.form['first_name']),
('alpha', 'last_name', request.form['last_name']),
('email', 'email', request.form['email']),
('pass_check', 'password', request.form['password'],
request.form['confirm_password'], 8, 16)]
sanitize = Validation(request_form)
email_query = 'SELECT id FROM users WHERE email = :email'
email_exists = mysql.query_db(email_query,
{'email': request.form['email']})
if len(email_exists) > 0:
sanitize.errors.append('User exists with email provided.')
if len(sanitize.errors) == 0:
data = sanitize.data
query = 'INSERT INTO users (first_name, last_name, email, password, salt, created_at, updated_at) VALUES ( :first_name, :last_name, :email, :password, :salt, now(), now())'
user_id = mysql.query_db(query, data)
flash('You have successfully registered! You user ID is ' +
str(user_id) + '.')
session['user'] = {
'status': 'active',
'user_id': user_id,
'first_name': sanitize.data['first_name'],
'last_name': sanitize.data['last_name'],
'email': sanitize.data['email'],
}
if 'data' in session:
session.pop('data')
return redirect('/')
else:
session['data'] = sanitize.data
for error in sanitize.errors:
flash(error)
return redirect('/register')
def train(self,
x_chunks_dir,
y_chunks_dir,
model_name,
logs_dir,
mode,
batch_size,
model_dir,
official_perl_rouge=True,
start_epoch_no=1,
training_on_collocations=False):
total_start_time = time.time()
avg_loss_per_batches_list = [
] # list of avg loss per a number of batches
avg_loss_per_epoch = []
rouge1_per_epoch_list = []
rouge2_per_epoch_list = []
rougeL_per_epoch_list = []
best_loss = {
'loss': 999888,
'epoch': 0,
'batch': 0
} # a huge initial value of best_loss
best_rouge1_f1 = {'rouge': 0.0, 'epoch': 0, 'batch': 0}
best_rouge2_f1 = {'rouge': 0.0, 'epoch': 0, 'batch': 0}
best_rougeL_f1 = {'rouge': 0.0, 'epoch': 0, 'batch': 0}
if not os.path.exists(logs_dir):
os.makedirs(logs_dir)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
model_checkpoint_filename = model_name + '_checkpoint.txt'
now = datetime.datetime.now()
logfile_name = str(
now.strftime(model_name + '_%Y%m%d_%H%M_train_logs.txt'))
logfile_path = logs_dir + logfile_name
logfile_writer_e = None
logfile_writer_w = open(logfile_path, 'w', encoding='utf8')
sess = None
try:
print_str = 'Logs\nLogs File: {}\nMode: {}\nModel: {}\nstarting: {}\nParameters:\n{}\n' \
'Loading dictionary and training dataset...\n' \
''.format(logfile_name, mode, model_name, now.strftime("%Y-%m-%d %H:%M"), self.parameters_str)
print(print_str, end='')
logfile_writer_w.write(print_str)
word2int_dict, int2word_dict, article_max_len, summary_max_len, vocabulary_size = self.load_training_data(
)
summary_max_len += 1 # because of adding </S> or <S>
x_and_y_file_path_pairs_list = self.file_path_pairs(
x_chunks_dir=x_chunks_dir, y_chunks_dir=y_chunks_dir)
num_of_chunks = len(x_and_y_file_path_pairs_list)
num_of_batches_per_epoch = self.num_of_batches_per_epoch(
x_and_y_file_path_pairs_list, batch_size)
print_str = 'Batches per epoch: {}\n' \
'vocabulary_size: {}\n' \
'Article and summary max_len: {}, {}\n'.format(num_of_batches_per_epoch, vocabulary_size,
article_max_len, summary_max_len)
print(print_str, end='')
logfile_writer_w.write(print_str)
sess = tf.Session() # as sess:
print("Loading word2vec...")
word2vec_embeddings = Model.get_init_embedding(
int2word_dict, param.embedding_dim, paths.word2vec_file_path)
print_str = 'Word embeddings have been loaded.\n'
print(print_str, end='')
logfile_writer_w.write(print_str)
model = Model(
article_max_len,
summary_max_len,
param.embedding_dim,
param.hidden_dim,
param.layers_num,
param.learning_rate,
param.beam_width,
param.keep_prob,
vocabulary_size,
batch_size,
word2vec_embeddings,
forward_only=False,
using_word2vec_embeddings=param.using_word2vec_embeddings)
model_validation = Model(article_max_len,
summary_max_len,
param.embedding_dim,
param.hidden_dim,
param.layers_num,
param.learning_rate,
param.beam_width,
param.keep_prob,
vocabulary_size,
batch_size,
word2vec_embeddings,
forward_only=True,
using_word2vec_embeddings=False)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(tf.global_variables(), max_to_keep=15)
if param.train_restored_saved_model and os.path.exists(
model_dir + model_checkpoint_filename):
print('Restoring the saved model.')
logfile_writer_w.write('Restoring the saved model.\n')
ckpt = tf.train.get_checkpoint_state(
checkpoint_dir=paths.model_dir,
latest_filename=model_checkpoint_filename)
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print('Training of new model.')
logfile_writer_w.write('Training of new model.\n')
logfile_writer_w.close()
epoch = start_epoch_no
epoch_no = start_epoch_no
end_epoch_no = epoch_no + param.epochs_num - 1
remaining_epochs = end_epoch_no - start_epoch_no
for _ in range(param.epochs_num):
end_epoch_no = epoch + remaining_epochs
remaining_epochs -= 1
logfile_writer_e = open(logfile_path, 'a', encoding='utf8')
epoch_loss_list = []
epoch_start_time = time.time()
print_str = '\n{} of {} epoch\n'.format(epoch, end_epoch_no)
print(print_str, end='')
logfile_writer_e.write(print_str)
x_and_y_file_path_pairs_list = shuffle(
x_and_y_file_path_pairs_list,
random_state=round(time.time() // 12345))
chunk_index = 0
for (x_file_path, y_file_path) in x_and_y_file_path_pairs_list:
#chunk_start_time = time.time()
chunk_index += 1
print_str = '\t{} of {} chunk (files: {} & {})\n'.format(
chunk_index, num_of_chunks, x_file_path[-60:],
y_file_path[-60:])
print(print_str, end='')
logfile_writer_e.write(print_str)
train_x, train_y, _ = self.load_and_shuffle_training_set(
x_file_path, y_file_path)
# if param.training_size is not None:
# remaining_training_size = remaining_training_size - current_training_size
print_str = '\t\tx & y chunk shape: {}, {}\n'.format(
np.array(train_x).shape,
np.array(train_y).shape)
logfile_writer_e.write(print_str)
print(print_str, end='')
batches = self.batch_iter_v2(train_x, train_y, batch_size)
num_batches_of_chunk = (len(train_x) - 1) // batch_size + 1
print("\t\tBatches of chunk: {}".format(
num_batches_of_chunk))
# print("Iteration starts.")
logfile_writer_e.write("\t\tBatches of chunk: {}\n".format(
num_batches_of_chunk))
# logfile_writer.write("\nIteration starts.\n")
batch_index = 0
batches_start_time = time.time()
batches_loss_list = []
for batch_x, batch_y in batches:
batch_index += 1
batch_x_len = list(
map(lambda x: len([y for y in x if y != 0]),
batch_x))
batch_decoder_input = list(
map(lambda x: [word2int_dict["<S>"]] + list(x),
batch_y)) # y starts with <s>
batch_decoder_len = list(
map(lambda x: len([y for y in x if y != 0]),
batch_decoder_input))
batch_decoder_output = list(
map(lambda x: list(x) + [word2int_dict["</S>"]],
batch_y)) # y ends with </s>
batch_decoder_input = list(
map(
lambda d: d + (summary_max_len - len(d)) *
[word2int_dict["<PAD>"]], batch_decoder_input))
batch_decoder_output = list(
map(
lambda d: d + (summary_max_len - len(d)
) * [word2int_dict["<PAD>"]],
batch_decoder_output))
train_feed_dict = {
model.batch_size: len(batch_x),
model.X: batch_x,
model.X_len: batch_x_len,
model.decoder_input: batch_decoder_input,
model.decoder_len: batch_decoder_len,
model.decoder_target: batch_decoder_output
}
fetches = [model.update, model.global_step, model.loss]
_, batch_no, loss = sess.run(fetches,
feed_dict=train_feed_dict)
epoch_loss_list.append(loss)
batches_loss_list.append(loss)
if batch_no % param.print_loss_per_steps == 0:
avg_batch_loss = np.average(
np.array(batches_loss_list))
avg_loss_per_batches_list.append(avg_batch_loss)
batches_loss_list = []
print_str = "\t\tEpoch: {} Batch: {}, Loss: {:.5f}, " \
"Time (batches, epoch & total): {}, {} & {}\n" \
"".format(epoch, batch_no, avg_batch_loss,
datetime.timedelta(
seconds=round(time.time() - batches_start_time, 0)),
datetime.timedelta(
seconds=round(time.time() - epoch_start_time, 0)),
datetime.timedelta(
seconds=round(time.time() - total_start_time, 0)))
print(print_str, end='')
logfile_writer_e.write(print_str)
batches_start_time = time.time()
# end of epoch --> saving the best model and print validation values
if batch_no % num_of_batches_per_epoch == 0: # end of epoch
epoch = batch_no // num_of_batches_per_epoch
temp = np.array(epoch_loss_list)
avg_epoch_loss = np.average(temp)
del temp
avg_loss_per_epoch.append(avg_epoch_loss)
file_prefix = '{}_{}epoch_{}batch'.format(
model_name, self.int_to_two_digits_str(epoch),
batch_no)
validation = Validation()
rouge_scores_dict = \
validation.rouge_scores_of_validation_set(sess,
model_validation,
int2word_dict,
batch_size,
file_prefix=file_prefix,
official_perl_rouge=official_perl_rouge,
#training_on_collocations=training_on_collocations
)
rouge1_per_epoch_list.append(
rouge_scores_dict['rouge_1_f_score'])
rouge2_per_epoch_list.append(
rouge_scores_dict['rouge_2_f_score'])
rougeL_per_epoch_list.append(
rouge_scores_dict['rouge_l_f_score'])
saving_flag = False
if rouge_scores_dict[
'rouge_1_f_score'] > best_rouge1_f1[
'rouge']:
best_rouge1_f1['rouge'] = rouge_scores_dict[
'rouge_1_f_score']
best_rouge1_f1['epoch'] = epoch
best_rouge1_f1['batch'] = batch_no
saving_flag = True
if rouge_scores_dict[
'rouge_2_f_score'] > best_rouge2_f1[
'rouge']:
best_rouge2_f1['rouge'] = rouge_scores_dict[
'rouge_2_f_score']
best_rouge2_f1['epoch'] = epoch
best_rouge2_f1['batch'] = batch_no
saving_flag = True
if rouge_scores_dict[
'rouge_l_f_score'] > best_rougeL_f1[
'rouge']:
best_rougeL_f1['rouge'] = rouge_scores_dict[
'rouge_l_f_score']
best_rougeL_f1['epoch'] = epoch
best_rougeL_f1['batch'] = batch_no
saving_flag = True
if avg_epoch_loss < best_loss['loss']:
best_loss['loss'] = avg_epoch_loss
best_loss['epoch'] = epoch
best_loss['batch'] = batch_no
saving_flag = True
# Save all variables of the TensorFlow graph to file.
print_str = 'Model: {}\n' \
' Avg_epoch_loss (current & best): {:.5f} & {:.5f} (epoch {}, batch {})\n' \
' Rouge_1_f-score (current & best): {:.5f} & {:.5f} (epoch {}, batch {})\n' \
' Rouge_2_f-score (current & best): {:.5f} & {:.5f} (epoch {}, batch {})\n' \
' Rouge_L_f-score (current & best): {:.5f} & {:.5f} (epoch {}, batch {})\n' \
' Time (epoch & total): {} & {}.\n'.format(model_name,
avg_epoch_loss, best_loss['loss'], best_loss['epoch'], best_loss['batch'],
rouge_scores_dict['rouge_1_f_score'],
best_rouge1_f1['rouge'], best_rouge1_f1['epoch'], best_rouge1_f1['batch'],
rouge_scores_dict['rouge_2_f_score'],
best_rouge2_f1['rouge'], best_rouge2_f1['epoch'], best_rouge2_f1['batch'],
rouge_scores_dict['rouge_l_f_score'],
best_rougeL_f1['rouge'], best_rougeL_f1['epoch'], best_rougeL_f1['batch'],
datetime.timedelta(seconds=round(time.time() - epoch_start_time, 0)),
datetime.timedelta(seconds=round(time.time() - total_start_time, 0)))
if saving_flag:
current_model_name = model_name + '_' + self.int_to_two_digits_str(
epoch) + 'epoch.ckpt'
# model_checkpoint_filename = model_name + 'checkpoint'
checkpoint = saver.save(
sess=sess,
save_path=model_dir + current_model_name,
latest_filename=model_checkpoint_filename,
global_step=batch_no)
add_str = '\nEpoch {}: The model is saved as the best one (...{}).\n' \
''.format(epoch, checkpoint[-25:])
print(add_str + print_str, end='')
logfile_writer_e.write(add_str + print_str)
else:
add_str = '\nEpoch {}: Τhe model was not saved as it is not the best one.\n' \
''.format(epoch)
print(add_str + print_str, end='')
logfile_writer_e.write(add_str + print_str)
epoch = epoch + 1
logfile_writer_e.close()
except KeyboardInterrupt:
if logfile_writer_w is not None:
logfile_writer_w.close()
if logfile_writer_e is not None:
logfile_writer_e.close()
#print_keyboard_interrupt_str = '\nException: KeyboardInterrupt\n'
logfile_writer = open(logfile_path, 'a', encoding='utf8')
print_str = '\n' + "-" * 60 + '\nException: KeyboardInterrupt\n'
print(print_str)
logfile_writer.write(print_str)
traceback.print_exc(file=sys.stdout)
traceback.print_exc(file=logfile_writer)
print_str = "-" * 60
print(print_str)
logfile_writer.write(print_str)
logfile_writer.close()
except Exception:
if logfile_writer_w is not None:
logfile_writer_w.close()
if logfile_writer_e is not None:
logfile_writer_e.close()
logfile_writer = open(logfile_path, 'a', encoding='utf8')
print_str = '\n' + "-" * 60 + '\n' + 'Exception:\n'
print(print_str)
logfile_writer.write(print_str)
traceback.print_exc(file=sys.stdout)
traceback.print_exc(file=logfile_writer)
print_str = "-" * 60
print(print_str)
logfile_writer.write(print_str)
logfile_writer.close()
finally:
self.finally_of_train_method(
logfile_path, avg_loss_per_batches_list, avg_loss_per_epoch,
rouge1_per_epoch_list, rouge2_per_epoch_list,
rougeL_per_epoch_list, total_start_time, now)
sess.close()
class OptimizedTimezoneProbModel():
def __init__(self):
self.training_data_span_months = 15
def fit(self, training_data):
self.validation = Validation(training_data)
self._training_data = training_data
self.smoother = self.optimize_smoother()
# get validation set
# optimize the smoother using validation set
def optimize_smoother(self, default=np.array([0.01, 0.01, 0.01, 0.01, 0.01, 0.01])):
cons = ({'fun': lambda x: x[0], 'type': 'ineq' },
{'fun': lambda x: x[1], 'type': 'ineq' },
{'fun': lambda x: x[2], 'type': 'ineq' },
{'fun': lambda x: x[3], 'type': 'ineq' },
{'fun': lambda x: x[4], 'type': 'ineq' },
{'fun': lambda x: x[5], 'type': 'ineq' },
{'fun': lambda x: 6 - sum(x), 'type': 'ineq' }) # only use low weights
val = minimize(self.evaluate_training_error,
default,
method='COBYLA',
constraints=cons,
options={'maxiter': 50, 'disp': True})
return val.x
def evaluate_training_error(self, smoother):
self.smoother = smoother
errors = []
for i in range(0,self.num_months()-self.training_data_span_months):
training_data = self.training_data(i, self.training_data_span_months)
test_data = self.test_data(i+self.training_data_span_months)
pred = self.predict(BusinessForecast(test_data).convert(),
training_data=training_data)
actual = ActualSchedule(test_data).bins()
errors.append(mean_squared_error(pred, actual))
sum_errors = np.array(errors).sum()
print "Smoother: {}, error: {}".format(self.smoother, sum_errors)
return sum_errors
def all_smoothing_values_positive(self):
truth = True
for i in self.smoother:
truth = truth & (i >= 0)
return truth
def predict(self, business_forecast, training_data=pd.DataFrame()):
sum_bins = np.zeros(6)
for i in business_forecast:
sched = ActualSchedule(training_data[\
training_data.user_tz == i['user_tz']])
stuff = (sched.bins() + self.smoother) / (sched.bins() + self.smoother).sum()
sum_bins += stuff * i['frequency'] * i['schedule_type']
return sum_bins
#private
def size(self):
return self.validation.size()
def num_months(self):
return self.validation.num_months()
def training_data(self, *args):
return self.validation.training_data(*args)
def year_month_index(self, *args):
return self.validation.year_month_index(*args)
def test_data(self, *args):
return self.validation.test_data(*args)
def plot():
validation = Validation()
validation.plot_ea()
validation.plot_hc()
# standard imports
from flask import Flask, request, render_template, session, redirect, url_for
from env import secret_key
from database import SqlOperations
from validation import Validation
from config import Config
# variable configurations
app = Flask(__name__)
app.secret_key = secret_key
config = Config()
sql = SqlOperations(config)
val = Validation(config)
@app.route("/")
def root():
for key in list(session.keys()):
session.pop(key)
return render_template("index.html")
@app.route("/routes", methods=["POST", "GET"])
def routes():
"""
/routes handles input to search for bus services between two bus stops, output the result and redirects
to /list_stops
"""
# list of error messages
err_msgs = [None] * 5
# list of headers to be displayed
# The repos.json file should be readable by the user running the Flask app,
# and the absolute path should be given by this environment variable.
BASE_APP_PATH = os.path.dirname(os.path.realpath(__file__))
repos_config = json.loads(
io.open(BASE_APP_PATH + '/config/repos.json', 'r').read())
global_config = json.loads(
io.open(BASE_APP_PATH + '/config/global.json', 'r').read())
logging_config = json.loads(
io.open(BASE_APP_PATH + '/config/logging.json', 'r').read())
if 'use_proxyfix' in global_config and global_config['use_proxyfix']:
from werkzeug.contrib.fixers import ProxyFix
dictConfig(logging_config)
app = Flask(__name__)
validation = Validation(global_config)
@app.route("/", methods=['GET', 'POST'])
def index():
if not validation.validate_ip():
abort(403)
if request.method == 'GET':
return 'OK'
elif request.method == 'POST':
event = request.headers.get('X-GitHub-Event')
payload = json.loads(
request.data.decode('utf8')) if request.data else None
handler = GithubEventHandler(app, event, payload)
handler.set_config(global_config, repos_config)
repo_config = handler.get_repo_config()
def train(param, model, forest_net, dataset):
# Launch the graph
with tf.Session() as sess:
sess.run(model.init)
step = 0
redisual = list()
# Keep training until reach max iterations
sum_time = []
while step < param.training_iters:
s_time = float(time.time())
#generate train data---mini batch
#using the forest net generate hidden data : n_steps * batch_size * n_input
batch_x, batch_y = generate_lstm_input(param, forest_net, dataset,
redisual)
#batch_x = forest_net.get_train_data(mnist.train.images, redisual, mnist.test.images)
m_time = float(time.time())
sum_time.append(m_time - s_time)
batch_x = np.transpose(batch_x, [1, 0, 2])
#batch_y = np.reshape(batch_y,(-1,param.n_classes))
fd = {model.xx: batch_x, model.y: batch_y}
# Run optimization op (backprop)
sess.run(model.optimizer, feed_dict=fd)
#get the gradient or residual
m1_time = float(time.time())
grad_val = np.array(
sess.run(tf.gradients(model.cost, model.xx), feed_dict=fd)[0])
m2_time = float(time.time())
redisual = param.step * np.transpose(grad_val, [1, 0, 2])
e_time = float(time.time())
#print (m_time-s_time, m2_time-m1_time, e_time-m2_time)
if step % param.display_step == 0:
# Calculate batch accuracy
acc = sess.run(model.accuracy, feed_dict=fd)
# Calculate batch loss
loss = sess.run(model.cost, feed_dict=fd)
print("Iter " + str(step) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
step += 1
batch_x = list()
batch_y = list()
print(sum(sum_time) / len(sum_time))
print("Optimization Finished!")
#test data
test_label = dataset.y_test #[:test_len]
#test_data_temp = forest_net.forest_test_predict(test_d)
test_data_temp = np.array(forest_net.get_test_data())
#print (test_data_temp.shape)
test_data = sess.run(tf.transpose(test_data_temp, [1, 0, 2]))
fd = {model.xx: test_data, model.y: test_label}
loss = sess.run(model.cost, feed_dict=fd)
print("{:.6f}".format(loss))
print("Testing Accuracy:", \
sess.run(model.accuracy, feed_dict=fd))
#get the predict classifier and probability value
classifier_value = sess.run([model.classifier_collection],
feed_dict={model.xx: test_data})
proba_value = sess.run([model.proba_collection],
feed_dict={model.xx: test_data})
#print (np.array(classifier_value).shape, np.array(proba_value).shape)
v = Validation()
result = [str(item + 1) for item in classifier_value[0]]
v.calculateF1(dataset.y_input, result)
def destroy(self):
request = Request(self)
url = self.base_url() + '/' + self.id
response = request.destroy(url)
return Validation(self).validate_response(response)
class Receptionist:
"""
Receptionist class used to represent an reception py functionality of user registration and login.
Objects:
__db :
Object created for UserDB class naming __db as protected object.
__secure :
Object created for the Security class naming __secure as protected object.
__valid :
Object created for the validation class naming __valid as protected object.
Methods
-------
register
Lets a new user to register an account with valid user name, password, Proper name, email.
login
Lets the existing user login to the system using valid user id and password.
"""
__db = UserDB()
__secure = Security()
__valid = Validation()
def register(self):
"""
This method takes in user information as username, password and asks to user to confirm the password
if the user name exists already in the system it asks to choose another username.
Then the password gets checked if its a valid one or not as well as the inserted password matching the
confirmation password. If the password does not match the confirmation password it shows messege as password does not match.
Further it prompts the user for first name, last name, email id and then user object gets created using User method of the UserDB class
and get those all user inserted data as parameters.
Using the methods of the UserDB class the database gets created, the overall data gets inserted as well in this method.
"""
while True:
username = input("Insert your UserName Please: ")
if self.__db.isExist(
username
) is not True and self.__valid.userName_regex.match(username):
username.lower()
break
print("Sorry the username is exist.")
print("Please choose another username")
while True:
password = getpass.getpass("Insert your Password Please: ")
confirmPassword = getpass.getpass("Confirm your Password : "******"password does not match")
while True:
firstname = str(input("Insert your First Name Please: "))
lastname = str(input("Insert your Last Name Please: "))
if self.__valid.name_regex.match(
firstname) and self.__valid.name_regex.match(lastname):
break
print("You should enter letters only")
while True:
email = input("Insert Your Email Please: ")
if self.__valid.email_regex.match(email):
break
print("you should enter a vaild email address")
self.__db.insert(username, password, firstname, lastname, email)
self.__db.displayDB()
def login(self):
"""
This method asks user for user name and password and checks if the user name and password exists in the database
if it finds the existing match it lets the user login the system.
"""
count = 0
while True:
self.__db = UserDB()
while True:
userName = input("Insert Your UserName: "******"Insert Your Password: "******"You are not registered you should register by choose 1 from the menu"
)
return False
break
count += 1
result = self.__db.get_Pass(userName)
if count == 4:
return False
break
else:
if result is not None:
result = result[0]
plain_pass = self.__secure.decrypt(result, userName)
if bytes.decode(plain_pass) == password:
data = self.__db.getUserInformation(userName)
return data
break
else:
print("Username or password is wrong")
else:
print("It is wrong you can try again")
class Inflow:
def __init__(self):
self.data_loader = DataLoader(cfg)
self.__build_model()
self.log_config('config', cfg, self.train_summary_writer, 0)
self.validation = Validation(cfg.validation, self.data_loader,
self.valid_summary_writer, self.network,
'Valid')
def log_config(self, name, config, summary_writer, step):
general_keys = list(config.keys())
rows = []
for key in general_keys:
try:
subkeys = list(config[key])
for subkey in subkeys:
rows.append(
['%s.%s' % (key, subkey),
str(config[key][subkey])])
except:
rows.append([key, str(config[key])])
hyperparameters = [tf.convert_to_tensor(row) for row in rows]
with summary_writer.as_default():
tf.summary.text(name, tf.stack(hyperparameters), step=step)
return
def __build_model(self):
self.network = TabNet(self.data_loader.get_columns(),
self.data_loader.num_features,
feature_dim=cfg.train.feature_dim,
output_dim=cfg.train.output_dim,
num_decision_steps=cfg.train.num_decision_steps,
relaxation_factor=cfg.train.relaxation_factor,
batch_momentum=cfg.train.batch_momentum,
virtual_batch_size=cfg.train.virtual_batch_size,
num_classes=cfg.train.num_classes,
encoder_type=cfg.train.encoder_type,
epsilon=0.00001)
lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(
cfg.train.learning_rate,
decay_steps=cfg.train.learning_rate_decay_steps,
decay_rate=cfg.train.learning_rate_decay_factor,
staircase=True)
self.optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)
self.log_loss = tf.losses.BinaryCrossentropy()
self.loss_metric = tf.keras.metrics.Mean(name='train_loss')
self.AUC_metric = tf.keras.metrics.AUC(name='train_AUC')
self.binary_metric = tf.keras.metrics.BinaryAccuracy(
name='train_BinaryAccuracy')
train_log_dir = os.path.join(cfg.train.logs_base_dir, 'train')
valid_log_dir = os.path.join(cfg.train.logs_base_dir, 'valid')
self.train_summary_writer = tf.summary.create_file_writer(
train_log_dir)
self.valid_summary_writer = tf.summary.create_file_writer(
valid_log_dir)
return
def __reset_all_metrics(self):
self.loss_metric.reset_states()
self.AUC_metric.reset_states()
self.binary_metric.reset_states()
return
def __load_variables(self):
if cfg.train.restore_model_path != '':
print(cfg.train.restore_model_path)
output = self.network(dict(
self.data_loader.valid_X.take([0, 1], axis=0)),
training=False)
file = h5py.File(cfg.train.restore_model_path, 'r')
weights = []
for i in range(len(file.keys())):
weights.append(file['weight' + str(i)].value)
self.network.set_weights(weights)
file.close()
# file = h5py.File(cfg.train.restore_model_path.replace('model-', 'optimizer-'), 'r')
# weights = []
# for i in range(len(file.keys())):
# weights.append(file['weight' + str(i)].value)
# self.optimizer.set_weights(weights)
# file.close()
step = int(
cfg.train.restore_model_path.split('.h5')[0].split('-')[-1])
self.optimizer.iterations.assign(step)
print('Model restored')
else:
step = 0
return step
def __save_model(self, step):
if not os.path.exists(cfg.train.models_base_dir):
os.makedirs(cfg.train.models_base_dir)
file_path = os.path.join(cfg.train.models_base_dir,
'model-%d.h5' % step)
file = h5py.File(file_path, 'w')
weights = self.network.get_weights()
for i in range(len(weights)):
file.create_dataset('weight' + str(i), data=weights[i])
file.close()
file_path = os.path.join(cfg.train.models_base_dir,
'optimizer-%d.h5' % step)
file = h5py.File(file_path, 'w')
weights = self.optimizer.get_weights()
for i in range(len(weights)):
file.create_dataset('weight' + str(i), data=weights[i])
file.close()
print('Model saved')
return
def run_train(self):
step = self.__load_variables()
self.validation.run_validation(step)
for epoch in range(cfg.train.max_nrof_epochs):
print('Start epoch %d' % epoch)
self.__reset_all_metrics()
batch_id = 0
for batch_features, labels in tqdm(self.data_loader.train_loader):
with tf.GradientTape() as tape:
output, output_aggregated, total_entropy,\
aggregated_mask_values_all, mask_values_all = self.network(batch_features, training=True)
loss = self.log_loss(labels, output)
# reg_loss = sum(self.network.losses)
reg_loss = cfg.train.weight_decay * tf.add_n([
tf.nn.l2_loss(w)
for w in self.network.trainable_variables
])
total_loss = loss + cfg.train.sparsity_loss_weight * total_entropy + reg_loss
grads = tape.gradient(total_loss,
self.network.trainable_variables)
capped_gvs = [
tf.clip_by_value(grad, -cfg.train.gradient_thresh,
cfg.train.gradient_thresh)
for grad in grads
]
self.optimizer.apply_gradients(
zip(capped_gvs, self.network.trainable_variables))
self.loss_metric(total_loss)
self.AUC_metric(labels, output)
self.binary_metric(labels, output)
with self.train_summary_writer.as_default():
# Visualization of the feature selection mask at decision step ni
for ni in range(len(mask_values_all)):
tf.summary.image("Mask for step" + str(ni),
tf.expand_dims(
tf.expand_dims(
mask_values_all[ni], 0), 3),
max_outputs=1,
step=step)
# Visualization of the aggregated feature importances
for ni in range(len(aggregated_mask_values_all)):
tf.summary.image(
"Aggregated mask",
tf.expand_dims(
tf.expand_dims(aggregated_mask_values_all[ni],
0), 3),
max_outputs=1,
step=step)
tf.summary.scalar('Total_loss',
self.loss_metric.result(),
step=step)
tf.summary.scalar('Total entropy',
total_entropy,
step=step)
tf.summary.scalar('Log_Loss', loss, step=step)
tf.summary.scalar('Reg_Loss', reg_loss, step=step)
tf.summary.scalar('AUC_score',
self.AUC_metric.result(),
step=step)
tf.summary.scalar('Accuracy',
self.binary_metric.result(),
step=step)
tf.summary.scalar('Gini_index',
2 * self.AUC_metric.result() - 1,
step=step)
tf.summary.scalar(
'Learning_rate',
self.optimizer.learning_rate.__call__(step).numpy(),
step=step)
template = 'Epoch [{}][{}/{}], loss: {}, binary accuracy: {}, AUC: {}'
print(
template.format(epoch, batch_id,
self.data_loader.nrof_batches, loss,
self.binary_metric.result() * 100,
self.AUC_metric.result() * 100))
batch_id += 1
step += 1
self.__save_model(step)
self.validation.run_validation(step)
self.data_loader.shuffle_data()
return
def output_variables(unsafe_building_type):
validate = Validation()
validate.unsafe_data['building_type'] = unsafe_building_type
building_type = validate._clean_building_type()
return _output_variables(building_type)
def run(run = False):
if run:
val = Validation({})
val.loadPredictors()
else:
val = build()
val.testPredictors()
val.evalPredictors()
val.mergeAllConfusion()
val.printPredictorsPerformance()
class InterpolatedProbModel(Model):
def __init__(self):
self.training_data_span_months = 15
self.x_offset = 139.016
self.x_divisor = 20
self.max_iter = 50
self.should_optimize = True
def fit(self, training_data):
# self.training_data = training_data
self.validation = Validation(training_data)
self._training_data = training_data
# Want to go for minimum mean training error * std.dev
if self.should_optimize:
optimized_args = self.optimize_interpolator()
self.x_offset = optimized_args[0]
self.x_divisor = optimized_args[1]
# get validation set
# first is x_offset, then x_divisor
def optimize_interpolator(self):
cons = ({'fun': lambda x: x[0], 'type': 'ineq' },
{'fun': lambda x: x[1] - 1, 'type': 'ineq' }) # only use low weights
val = minimize(self.evaluate_training_error,
[self.x_offset, self.x_divisor],
method='COBYLA',
constraints=cons,
options={'maxiter': self.max_iter, 'disp': True})
return val.x
def evaluate_training_error(self, args):
self.x_offset = args[0]
self.x_divisor = args[1]
minimizables = []
for i in range(0,self.num_months()-self.training_data_span_months):
training_data = self.training_data(i, self.training_data_span_months)
test_data = self.test_data(i+self.training_data_span_months)
pred = self.predict(BusinessForecast(test_data).convert(),
training_data=training_data)
actual = ActualSchedule(test_data).bins()
to_minimize = mean_absolute_error(pred,actual) * (pred - actual).std()
minimizables.append(to_minimize)
np_to_minimize = np.array(minimizables)
avg = np_to_minimize.mean()
print "x_offset: {}, x_divisor: {}, minimize: {}".format(self.x_offset,
self.x_divisor,
avg)
return avg
def predict(self, business_forecast, training_data=pd.DataFrame()):
sum_bins = np.zeros(6)
for i in business_forecast:
gpm = GeneralProbModel()
gpm.fit(training_data)
gpm_prediction = gpm.predict([i])
tpm = TimezoneProbModel()
tpm.fit(training_data)
tpm_prediction = tpm.predict([i], training_data=training_data)
people_at_user_tz = training_data[training_data.user_tz == i['user_tz']]
tmp = self.timezone_model_probability(people_at_user_tz.schedule_type.sum())
sum_bins += gpm_prediction * (1 - tmp) + tpm_prediction * tmp
return sum_bins
#private
def timezone_model_probability(self, sample_size):
stuff = (sample_size - self.x_offset) / self.x_divisor
print "sample_size: {}, x_offset: {}, x_divisor: {}, stuff: {}".format(sample_size,
self.x_offset, self.x_divisor, stuff)
# prevent overflow error
if stuff < -549:
exponent = -549
else:
exponent = stuff
return 1.0 / (1 + math.exp(-exponent))
def size(self):
return self.validation.size()
def num_months(self):
return self.validation.num_months()
def training_data(self, *args):
return self.validation.training_data(*args)
def year_month_index(self, *args):
return self.validation.year_month_index(*args)
def test_data(self, *args):
return self.validation.test_data(*args)
import csv
import readValidate
from validation import Validation
try:
f = open('../preprocessed_data/probabilities.txt', 'rb')
h = open('../preprocessed_data/hops.txt', 'rb')
except IOError:
print "No probabilities to validate "
f.close()
return
else:
prob = pickle.load(f)
hops = pickle.load(h)
f.close()
h.close()
shouldBeResult = readValidate.read("../data/validate.csv")
results = Validation()
results.validation(shouldBeResult,prob,hops)
step += 1
start = start + batch_size
end = end + batch_size
print("Optimization Finished!")
# Calculate accuracy for 128 mnist test images
#test_len = 128
#test_data = mnist.test.images[:test_len].reshape((-1, n_steps, n_input))
#test_label = mnist.test.labels[:test_len]
test_data = x_test.reshape((-1, n_steps, n_input))
test_label = np.reshape(y_test, (-1, 2))
loss = sess.run(cost, feed_dict={x: test_data, y: test_label})
print("{:.6f}".format(loss))
print("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
classifier_value = sess.run([classifier_collection],
feed_dict={x: test_data})
print(len(classifier_value))
for value in classifier_value:
print(value)
proba_value = sess.run([proba_collection], feed_dict={x: test_data})
for value in proba_value:
print(value)
v = Validation()
v.calculateF1(input_value, classifier_value[0])
v.allValidation(input_value, proba_value[0][:, 1])
from validation import Validation validation = Validation() validation.plot_aco()
def __init__(self, required: bool = False, nullable: bool = True):
super().__init__(required, nullable,
validation=Validation(lambda name, value: Validation.validate_gender(name, value)))
"No such file exists in the specified path...Please see the 'dir' option under savingAndLoading in the config and ensure that your file is present there"
)
lastEpoch = getLastEpochFromFileName(fileName)
startEpoch = lastEpoch + 1
stage = getStageFromFileName(fileName)
if stageChangeRequired(stage, lastEpoch):
changeStage = True
startEpoch = 1
stage += 1
lipreader = Lipreader(stage)
lipreader = loadModel(lipreader, fileName, changeStage)
else:
lipreader = Lipreader()
if config.gpuAvailable:
lipreader = lipreader.cuda()
if config.deviceCount > 1:
lipreader = nn.DataParallel(lipreader)
trainer = Trainer(lipreader)
validator = Validation(lipreader)
totalEpochs = config.training["Stage " + str(stage)]["epochs"]
print("Started training at", datetime.now())
with tqdm(total=totalEpochs - startEpoch + 1, desc="Epochs",
position=0) as t:
for epoch in range(startEpoch - 1, totalEpochs):
trainer.train(epoch)
validator.validate(epoch)
t.update()
saveModel(lipreader, epoch + 1)
print(f"Successfully completed training of Stage {stage}")
class BackTest():
def __init__(self,
data=None,
model=None,
training_data_span_months=1):
self._data_df = data
self._model = model
self.training_data_span_months = training_data_span_months
self.validation = Validation(data)
def errors(self):
indices = [] # index of a row, formatted YYYY-M (e.g. 2016-8, 2016-9)
predictions = []
errors = [] # errors. Lower is better
test_data_size = []
actuals = []
for i in range(0,self.num_months()-self.training_data_span_months):
m = self._model()
training_data = self.training_data(i, self.training_data_span_months)
m.fit(training_data)
td = self.test_data(i+self.training_data_span_months)
print "Generating prediction..."
prediction = m.predict(\
BusinessForecast(\
td).convert(),
training_data=training_data)
actual = ActualSchedule(\
td).bins()
print "prediction "
print prediction
print "\n"
print "actual"
print actual
print "\n"
error = mean_absolute_error(actual, prediction)
errors.append(error)
index = self.year_month_index(i+self.training_data_span_months)
indices.append(index)
print "time: {}, error: {}".format(index, error)
predictions.append(prediction)
actuals.append(actual)
test_data_size.append(actual.sum())
return pd.DataFrame({
'errors': errors,
'test_data_size': test_data_size,
'predictions': predictions,
'actuals': actuals
}).set_index([indices])
#private
def size(self):
return self.validation.size()
def num_months(self):
return self.validation.num_months()
def training_data(self, *args):
return self.validation.training_data(*args)
def year_month_index(self, *args):
return self.validation.year_month_index(*args)
def test_data(self, *args):
return self.validation.test_data(*args)
if step % display_step == 0:
# Calculate batch accuracy
acc = sess.run(accuracy, feed_dict={x: batch_x, y: batch_y})
# Calculate batch loss
loss = sess.run(cost, feed_dict={x: batch_x, y: batch_y})
print("Iter " + str(step*batch_size) + ", Minibatch Loss= " + \
"{:.6f}".format(loss) + ", Training Accuracy= " + \
"{:.5f}".format(acc))
#print(sess.run(tf.gradients(cost, x), feed_dict={x:batch_x, y:batch_y})[0])
step += 1
print("Optimization Finished!")
# Calculate accuracy for 128 mnist test images
#test_data = x_test.reshape((-1, n_steps, n_input))
#test_data = np.transpose(x_test, [1,0,2])
test_data = x_test
test_label = y_test
print("Testing Accuracy:", \
sess.run(accuracy, feed_dict={x: test_data, y: test_label}))
classifier_value = sess.run([classifier_collection],
feed_dict={x: test_data})[0]
proba_value = sess.run([proba_collection], feed_dict={x: test_data})
v = Validation()
v.calculateF1(y_input, classifier_value + 1)
v.allValidation(y_input - 1, proba_value[0][:, 1])
#v.top_accuacy(y_input, classifier_value[0], proba_value[0][:,0],[1000,10000,50000])
def __enter_file_name():
file_name = v.enterStr('Enter file name: ')
return file_name
class SalespersonGUI():
"""
GUI class to perform CRUD operations on the Salesperson table in the database.
"""
def __init__(self):
"""
Initialise Salesperson class.
"""
# Data access object
self.salesperson_dao = SalespersonDAO()
# Validation object
self.validator = Validation()
# Form fields
self.salesperson_id = tk.StringVar()
self.title = tk.StringVar()
self.firstname = tk.StringVar()
self.surname = tk.StringVar()
self.position = tk.StringVar()
self.work_phone = tk.StringVar()
self.email = tk.StringVar()
# List of salesperson ids
self.lb_ids = None
# Messagebox title
self.mb_title_bar = "Salesperson CRUD"
pass
def create_gui(self, root):
"""
Create the GUI.
This is the interface for the user to input data or query the database.
It is composed of frames containing widgets (e.g. labels, input fields).
Parameters (apart from self):
root: main window of application
Return:
salesperson_frame: the frame containing all the widgets for the
salesperson CRUD
"""
print("Creating Salesperson GUI ...")
salesperson_frame = tk.Frame(root)
salesperson_frame.pack()
# FORM FRAME
form_frame = tk.Frame(salesperson_frame)
form_frame.pack()
# FORM FRAME: ROW 0
# Heading for Salesperson frame
tk.Label(form_frame, text="Salesperson",
font=('arial', 10)).grid(row=0, column=0, columnspan=3)
# FORM FRAME: ROW 1
# Label and entry field (disabled) for "Salesperson ID"
tk.Label(form_frame,
text="Salesperson ID",
font=('arial', 10),
width=20,
anchor='e',
bd=1,
pady=10,
padx=10).grid(row=1, column=0)
tk.Entry(form_frame,
textvariable=self.salesperson_id,
width=30,
bd=1,
state=tk.DISABLED).grid(row=1, column=1)
# Heading for Listbox to display and select Salesperson IDs
tk.Label(form_frame, text="Salesperson IDs",
font=('arial', 10)).grid(row=1, column=2)
# FORM FRAME: ROW 2
# Label and combobox field for "Title"
tk.Label(form_frame,
text="Title",
font=('arial', 10),
width=20,
anchor='e',
bd=1,
pady=10,
padx=10).grid(row=2, column=0)
TITLE_VALUES = ("Mr", "Mrs", "Ms", "Miss", "Dr")
ttk.Combobox(form_frame,
state='readonly',
textvariable=self.title,
values=TITLE_VALUES,
width=5).grid(row=2, column=1, sticky='w')
# Listbox to display and select Salesperson IDs
self.lb_ids = tk.Listbox(form_frame)
self.lb_ids.grid(row=2, column=2, rowspan=6)
self.lb_ids.bind('<<ListboxSelect>>', self.on_list_select)
# FORM FRAME: ROW 3
# Label and entry field for "First Name"
tk.Label(form_frame,
text="First Name",
font=('arial', 10),
width=20,
anchor='e',
bd=1,
pady=10,
padx=10).grid(row=3, column=0)
tk.Entry(form_frame, textvariable=self.firstname, width=30,
bd=1).grid(row=3, column=1)
# FORM FRAME: ROW 4
# Label and entry field for "Surname"
tk.Label(form_frame,
text="Surname",
font=('arial', 10),
width=20,
anchor='e',
bd=1,
pady=10,
padx=10).grid(row=4, column=0)
tk.Entry(form_frame, textvariable=self.surname, width=30,
bd=1).grid(row=4, column=1)
# FORM FRAME: ROW 5
# Label and entry field for "Position"
tk.Label(form_frame,
text="Position",
font=('arial', 10),
width=20,
anchor='e',
bd=1,
pady=10,
padx=10).grid(row=5, column=0)
POSITION_VALUES = ("Junior", "Associate", "Senior")
ttk.Combobox(form_frame,
state='readonly',
textvariable=self.position,
values=POSITION_VALUES,
width=10).grid(row=5, column=1, sticky='w')
# FORM FRAME: ROW 6
# Label and entry field for "Work Phone"
tk.Label(form_frame,
text="Work Phone",
font=('arial', 10),
width=20,
anchor='e',
bd=1,
pady=10,
padx=10).grid(row=6, column=0)
tk.Entry(form_frame, textvariable=self.work_phone, width=30,
bd=1).grid(row=6, column=1)
# FORM FRAME: ROW 7
# Label and entry field for "Email"
tk.Label(form_frame,
text="E-mail",
font=('arial', 10),
width=20,
anchor='e',
bd=1,
pady=10,
padx=10).grid(row=7, column=0)
tk.Entry(form_frame, textvariable=self.email, width=30,
bd=1).grid(row=7, column=1)
# BUTTON FRAME
button_frame = tk.Frame(salesperson_frame, pady=10)
button_frame.pack()
tk.Button(button_frame,
width=10,
text="Clear",
command=self.clear_fields).pack(side=tk.LEFT)
tk.Button(button_frame, width=10, text="Save",
command=self.save).pack(side=tk.LEFT)
tk.Button(button_frame, width=10, text="Delete",
command=self.delete).pack(side=tk.LEFT)
tk.Button(button_frame, width=10, text="Load",
command=self.load).pack(side=tk.LEFT)
return salesperson_frame
def clear_fields(self):
"""
Clear the fields of the form
Parameters (apart from self): None
Return: None
"""
self.salesperson_id.set("")
self.title.set("")
self.firstname.set("")
self.surname.set("")
self.position.set("")
self.work_phone.set("")
self.email.set("")
pass
def save(self):
"""
Save the data displayed on the form to the database.
Get salesperson data to be saved from the global instance attributes,
then validate data by calling validate_fields()
If the data is invalid, a message box is presented to the user.
If the data is valid, the data is either saved or updated
If salesperson_id is present, the data is updated
If not, a new salesperson record is created in the database
Parameters (apart from self): None
Return: None
"""
print("Saving a salesperson ...")
data = self.get_fields()
valid_data, message = self.validate_fields(data)
if valid_data:
if (len(data['salesperson_id']) == 0):
print("Calling create() as salesperson_id is absent")
self.create(data)
else:
print("Calling update() as salesperson_id is present")
self.update(data)
pass
else:
message_text = "Invalid fields.\n" + message
messagebox.showwarning(self.mb_title_bar,
message_text,
icon='warning')
pass
def get_fields(self):
sp = {}
sp['salesperson_id'] = self.salesperson_id.get()
sp['title'] = self.title.get()
sp['firstname'] = self.firstname.get()
sp['surname'] = self.surname.get()
sp['position'] = self.position.get()
sp['work_phone'] = self.work_phone.get()
sp['email'] = self.email.get()
return sp
def validate_fields(self, data):
"""
Validate the data entered in the fields of the form
Parameters (apart from self):
data: dictionary object containing all the information entered on the form
Return:
valid_data: a boolean indication whether the data is valid (True) or
not valid (False)
message: a string containing details about the fields that are not valid
Returned as a tuple (valid_data, message)
"""
valid_data = True
message_list = []
# Check field is not empty
if len(data['title']) == 0:
valid_data = False
message_list.append("title is empty")
if len(data['firstname']) == 0:
valid_data = False
message_list.append("firstname is empty")
if len(data['surname']) == 0:
valid_data = False
message_list.append("surname is empty")
if len(data['position']) == 0:
valid_data = False
message_list.append("position year is empty")
if len(data['work_phone']) == 0:
valid_data = False
message_list.append("work phone is empty")
if len(data['email']) == 0:
valid_data = False
message_list.append("e-mail is empty")
# Position and title are selected from dropdown menu
# so no further validation needed
# Check that firstname and surname are alphabetic
if not self.validator.is_alphabetic(data['firstname']):
valid_data = False
message_list.append("invalid firstname")
if not self.validator.is_alphabetic(data['surname']):
valid_data = False
message_list.append("invalid surname")
# Other checks
if not self.validator.is_email(data['email']):
valid_data = False
message_list.append("invalid email format")
if not self.validator.is_phone_number(data['work_phone']):
valid_data = False
message_list.append("invalid phone number format")
message = ', '.join(message_list)
return valid_data, message
pass
def create(self, data):
"""
Create a new record in the database.
A messagebox is used display the outcome (success or failure)
of the create operation to the user.
Parameters (apart from self):
data: dictionary object containing salesperson data to be saved
Return: None
"""
print("Creating a salesperson ...")
print(data)
session = db.get_db_session()
result = self.salesperson_dao.create(session, data)
session.close()
messagebox.showinfo(self.mb_title_bar, result)
pass
def update(self, data):
"""
Update a record in the database.
A messagebox is used display the outcome (success or failure)
of the update operation to the user.
Parameters (apart from self):
data: dictionary object containing salesperson data to be saved
Return: None
"""
print("Updating a salesperson ...")
print(data)
session = db.get_db_session()
result = self.salesperson_dao.update(session, data['salesperson_id'],
data)
session.close()
messagebox.showinfo(self.mb_title_bar, result)
pass
def delete(self):
"""
Delete a record from the database.
The salesperson_id of the record to be deleted is obtained from a
global attribute.
A messagebox is used display the outcome (success or failure)
of the delete operation to the user.
Parameters (apart from self): None
Return: None
"""
print("Deleting a salesperson ...")
sp_id = self.salesperson_id.get()
print(id)
session = db.get_db_session()
result = self.salesperson_dao.delete(session, sp_id)
session.close()
messagebox.showinfo(self.mb_title_bar, result)
pass
def load(self):
"""
Retrieve a list of IDs from the database and load them into a listbox.
Parameters (apart from self):
Return: None
"""
session = db.get_db_session()
result = self.salesperson_dao.find_ids(session)
session.close()
print("result: ", result)
if "salesperson_ids" in result:
list_ids = result["salesperson_ids"]
self.lb_ids.delete(0, tk.END)
print("Setting salesperson_id in listbox")
for x in list_ids:
self.lb_ids.insert(tk.END, x)
pass
def on_list_select(self, evt):
"""
Actions to be triggered when a user clicks an item in the listbox.
Defined above in create_gui(), where on_list_select is bound to the
listbox selection.
Parameters (apart from self):
evt: object containing information about the mouse click
Return: None
"""
w = evt.widget
index = int(w.curselection()[0])
value = w.get(index)
print(index)
print(value)
session = db.get_db_session()
result = self.salesperson_dao.find_by_id(session, value)
session.close()
print("result: ", result)
sp = result['salesperson']
self.populate_fields(sp)
pass
def populate_fields(self, sp):
"""
Populate the fields of the form with data.
Parameters (apart from self):
sp: dictionary object containing all the information
about a salesperson
Return: None
"""
self.salesperson_id.set(sp['salesperson_id'])
self.title.set(sp['title'])
self.firstname.set(sp['firstname'])
self.surname.set(sp['surname'])
self.position.set(sp['position'])
self.work_phone.set(sp['work_phone'])
self.email.set(sp['email'])
pass
def get_result(self, predict_classifier, predict_probability):
v = Validation()
v.calculateF1(self.y_test, predict_classifier)
v.allValidation(self.y_test, predict_probability)
class StaffGUI():
"""
GUI class to perform CRUD operations on the employee table in the database.
"""
def __init__(self):
"""
The initialiser is used to "instantiate" attributes of the class.
The attributes are the "variables" that have been declared "outside"
of the methods of the class.
Some attributes may have not been declared as they may be created
any where in the class methods (python allows this).
Attributes are like global variables for the class, as they are
available to any method in the class.
And, to use them, they must be prefixed with "self."
This differentiates them from "local" variables which are
defined/created and used within a single method
If you need to pass the value of a local variable from one method
to another, then you must pass "parameters" to the method
We cannot create the GUI here as we need to return a reference to
the frame created.
Hence, we need to implement a 'normal' function to do this e.g. create_gui()
Parameters (apart from self): None
Return: None
"""
# Instantiate a data access object
# Contains methods to access the database
self.emp_dao = StaffDAO()
# Instantiate a validation object
# Contains methods to validate input fields
self.validator = Validation()
# Form fields
# Instantiate stringvars - hold data entered in fields of form
self.staff_id = tk.StringVar()
self.first_name = tk.StringVar()
self.last_name = tk.StringVar()
self.title = tk.StringVar()
self.email = tk.StringVar()
self.sex = tk.StringVar()
self.contact_no = tk.IntVar()
# List of employee ids - lb for listbox
self.lb_ids = None
# Messagebox title
self.mb_title_bar = "Staff CRUD"
pass
"""
Create a high level frame which contains the entire GUI
"""
def create_gui(self, root):
print("Creating Staff GUI ...")
emp_frame = tk.Frame(root)
emp_frame.pack()
form_frame = tk.Frame(emp_frame)
form_frame.pack()
tk.Label(form_frame, font=('arial', 10),
text="Staff").grid(row=0, column=0, columnspan=3)
# row 1: employee_id label, employee_id entry and list_of_ids label
tk.Label(form_frame,
text="Staff Id",
font=('arial', 10),
width=20,
anchor="e",
bd=1,
pady=10,
padx=10).grid(row=1, column=0)
tk.Entry(form_frame,
textvariable=self.staff_id,
width=30,
bd=1,
state=tk.DISABLED).grid(row=1, column=1)
tk.Label(form_frame, text="Staff IDs",
font=('arial', 10)).grid(row=1, column=2)
# row 2: title label and combobox (the listbox will go through)
tk.Label(form_frame,
text="Title",
font=('arial', 10),
width=20,
anchor="e",
bd=1,
pady=10,
padx=10).grid(row=2, column=0)
VALUES = ('Mr', 'Mrs', 'Ms', 'Miss', 'Dr')
ttk.Combobox(form_frame,
state="readonly",
textvariable=self.title,
values=VALUES,
width=10).grid(row=2, column=1, sticky="w")
self.title.set(VALUES[0])
# row 3: firstname label, firstname entry and listbox of ids
tk.Label(form_frame,
text="First name",
font=('arial', 10),
width=20,
anchor="e",
bd=1,
pady=10,
padx=10).grid(row=3, column=0)
tk.Entry(form_frame, textvariable=self.first_name, width=30,
bd=1).grid(row=3, column=1)
self.lb_ids = tk.Listbox(form_frame)
self.lb_ids.grid(row=3, column=2, rowspan=5)
self.lb_ids.bind('<<ListboxSelect>>', self.on_list_select)
# row 4: lastname label and entry (the listbox will go through)
tk.Label(form_frame,
text="Last name",
font=('arial', 10),
width=20,
anchor="e",
bd=1,
pady=10,
padx=10).grid(row=4, column=0)
tk.Entry(form_frame, textvariable=self.last_name, width=30,
bd=1).grid(row=4, column=1)
# row 5: title label and combobox (the listbox will go through)
tk.Label(form_frame,
text="Sex",
font=('arial', 10),
width=20,
anchor="e",
bd=1,
pady=10,
padx=10).grid(row=5, column=0)
VALUES = ('Male', 'Female', 'Undefined')
ttk.Combobox(form_frame,
state="readonly",
textvariable=self.sex,
values=VALUES,
width=10).grid(row=5, column=1, sticky="w")
# row 6: email label and combobox (the listbox will go through)
tk.Label(form_frame,
text="Email",
font=('arial', 10),
width=20,
anchor="e",
bd=1,
pady=10,
padx=10).grid(row=6, column=0)
tk.Entry(form_frame, textvariable=self.email, width=30,
bd=1).grid(row=6, column=1)
# row 7: work_phone label and combobox (the listbox will go through)
tk.Label(form_frame,
text="Contact No",
font=('arial', 10),
width=20,
anchor="e",
bd=1,
pady=10,
padx=10).grid(row=7, column=0)
tk.Entry(form_frame, textvariable=self.contact_no, width=30,
bd=1).grid(row=7, column=1)
# Buttons
button_frame = tk.Frame(emp_frame, pady=10)
button_frame.pack()
tk.Button(button_frame,
width=10,
text="Clear",
command=self.clear_fields).pack(side=tk.LEFT)
tk.Button(button_frame, width=10, text="Save",
command=self.save).pack(side=tk.LEFT)
tk.Button(button_frame, width=10, text="Delete",
command=self.delete).pack(side=tk.LEFT)
tk.Button(button_frame, width=10, text="Load",
command=self.load).pack(side=tk.LEFT)
# Return a reference to the high level frame created
return emp_frame
"""
Clears all the fields of the form
"""
def clear_fields(self):
self.staff_id.set("")
self.first_name.set("")
self.last_name.set("")
#self.sex.set("") # Do not clear if using dropdown
#self.title.set("") # Do not clear if using dropdown
self.email.set("")
self.contact_no.set("")
pass
"""
Save the data displayed on the form to the database
"""
def save(self):
print("Saving staff....")
# Get the data
data = self.get_fields()
# Validate the data
valid_data, message = self.validate_fields(data)
if valid_data:
if (len(data['staff_id']) == 0):
# If nothing has been entered in employee_id
# i.e. its length is zero characters
print("Calling create() as staff_id is absent")
self.create(data)
else:
print("Calling update() as staff_id is present")
self.update(data)
pass
else:
message_text = "Invalid fields.\n" + message
messagebox.showwarning(self.mb_title_bar,
message_text,
icon="warning")
pass
"""
Get the data entered in the fields of the form
"""
def get_fields(self):
emp = {}
# employee_id is ignored when creating a record
emp['staff_id'] = self.staff_id.get()
emp['first_name'] = self.first_name.get()
emp['last_name'] = self.last_name.get()
emp['title'] = self.title.get()
emp['sex'] = self.sex.get()
emp['email'] = self.email.get()
emp['contact_no'] = self.contact_no.get()
return emp
"""
Validate the data entered in the fields of the form
"""
def validate_fields(self, data):
# By default set to true, anything wrong will turn it to false
valid_data = True
# Instantiate an empty list to contain the messages
message_list = []
# Check for blank fields
# Do not check employee_id as this is generated by the database
#if len(data['employee_id']==0:
# valid_data = False
# message_list.append("employee_id is empty")
if len(str(data['first_name'])) == 0:
valid_data = False
message_list.append("first_name is empty")
if len(str(data['last_name'])) == 0:
valid_data = False
message_list.append("last_name is empty")
if len(data['title']) == 0:
valid_data = False
message_list.append("title is empty")
if len(data['sex']) == 0:
valid_data = False
message_list.append("sex is empty")
if len(str(data['email'])) == 0:
valid_data = False
message_list.append("work_phone is empty")
if len(str(data['contact_no'])) == 0:
valid_data = False
message_list.append("contact_no is empty")
if not self.validator.is_alphabetic(data['first_name']):
valid_data = False
message_list.append("invalid first_name")
if not self.validator.is_alphabetic(data['last_name']):
valid_data = False
message_list.append("invalid last_name")
# Check if title is in a list [Mr, Ms, Mrs, Dr, etc]
if self.title.get() not in ["Mr", "Ms", "Mrs", "Miss", "Dr"]:
valid_data = False
message_list.append("title should be Mr, Ms, Mrs, Miss or Dr")
# Check if email follows a certain pattern
# i.e contains an @ followed by a dot
if not self.validator.is_email(data['email']):
valid_data = False
message_list.append("invalid email format")
# Join the items in the list as a string separated with a comma and a space
message = ', '.join(message_list)
return valid_data, message # return 2 values
"""
Create a new record in the database.
"""
def create(self, data):
print("Creating an staff ...")
print(data)
session = db.get_db_session() # Get a session (database.py)
result = self.emp_dao.create(session, data)
# result is a tuple e.g. ("Employee added successfully", 1004)
#result, employee_id = self.emp.create(data)
# if you wish to get the message and employee_id separately
session.close() # Close the session
messagebox.showinfo(self.mb_title_bar, result)
pass
"""
Create a new record in the database.
"""
def update(self, data):
print("Updating staff ...")
print(data)
session = db.get_db_session() # Get a session (database.py)
result = self.emp_dao.update(session, data['staff_id'], data)
session.close() # close the session
# Display the returned message to the user - use a messagebox
# Display everything that is returned in the result
messagebox.showinfo(self.mb_title_bar, result)
pass
"""
Delete a record in the database
"""
def delete(self):
# Grab the employee_id from the stringvar
id = self.staff_id.get()
print(id)
# Call the data access object to do the job
# Pass the id as parameter to the delete() method
session = db.get_db_session() # Get a session (database.py)
result = self.emp_dao.delete(session, id)
session.close() # Close the session
# Display the returned message to the user - use a messagebox
# Display everything that is returned in the result
messagebox.showinfo(self.mb_title_bar, result)
pass
"""
load records in the database
"""
def load(self):
session = db.get_db_session() # Get a session (database.py)
result = self.emp_dao.find_ids(session) # {"employee_ids": [1, 2, 3]}
session.close() # Close the session
print("result", result)
# Check if there is an entry in the result dictionary
if "staff_ids" in result:
list_ids = result['staff_ids']
self.lb_ids.delete(0, tk.END)
print("Setting staff_id in listbox ...")
for x in list_ids:
self.lb_ids.insert(tk.END, x)
#print(x)
pass
"""
on_list_select() is triggered when a user clicks an item in the listbox.
"""
def on_list_select(self, evt):
w = evt.widget
index = int(w.curselection()[0])
# index = position of the item clicked in the list, first item is item 0 not 1
value = w.get(index)
# value of the item clicked, in our case it's the employee_id
print(index)
print(value)
# Call find_by_id and populate the stringvars of the form
session = db.get_db_session() # Get a session (database.py)
result = self.emp_dao.find_by_id(session, value)
session.close() # close the session
print("result", result)
# { "employee" : {"employee_id": "", "firstname": "", etc}}
emp = result['staff']
self.populate_fields(emp)
pass
"""
Populate the fields of the form with data
"""
def populate_fields(self, emp):
# Set the values from the dict to the stringvars
self.staff_id.set(emp['staff_id'])
self.first_name.set(emp['first_name'])
self.last_name.set(emp['last_name'])
self.title.set(emp['title'])
self.sex.set(emp['sex'])
self.email.set(emp['email'])
self.contact_no.set(emp['contact_no'])
pass
from discord import Game
from discord.ext import commands
import validation
from gw2ApiKey import GW2Api
from mongoDB import MongoDB
from utils import Utils, UtilsCommand, UtilsDiscordRoles, ErrorMessages
from validation import Validation
utils = Utils()
description = utils.BOT_DESCRIPTION
askForAPIKey = utils.API_QUESTION
command_description = UtilsCommand()
server_roles = UtilsDiscordRoles()
error_messages = ErrorMessages()
validation = Validation()
mongoDb = MongoDB()
class DiscordBot(commands.Bot):
# old but u run the bot only on one server, its fine
# current_server_joined = None
def __init__(self):
super().__init__(command_prefix="!",
description=description,
pm_help=True,
has_permission=8,
status=discord.Status.idle)
self.add_command(self.be_rude)
self.add_command(self.reg)
def options(arr):
for i in arr:
print(i)
choice = v.intValidateInRange('Your choice - ', 0, len(arr))
return choice
def output_variables(unsafe_building_type):
validate = Validation()
validate.unsafe_data['building_type'] = unsafe_building_type
building_type = validate._clean_building_type()
return _output_variables(building_type)
def fit(self, training_data):
self.validation = Validation(training_data)
self._training_data = training_data
self.smoother = self.optimize_smoother()