def run_exponentiated_gradient_learning():
"""
Tests the performance of exponentiated gradient with different learning
rates.
"""
widgets.learning_alg_radio.value = const.ALG_EG
widgets.k_slider.value = 10
widgets.lambdas_range_slider.value = [-10, -10]
widgets.epoch_slider.value = 25
# Test exponentiated gradient with different learning rates
train_examples, test_examples = input_parser.parse()
learning_rate_vals = [
0.01, 0.05, 0.10, 0.2, 0.5, 1, 2, 4, 8, 16, 32, 64, 128
]
err_export = np.zeros([4, len(learning_rate_vals)])
for idx, learning_rate in enumerate(learning_rate_vals):
widgets.learning_rate_slider.value = learning_rate
train_err_run, validation_err_run, test_err_run = run_hw03(
train_examples, test_examples)
err_export[0, idx] = learning_rate
err_export[1, idx] = train_err_run[0, 0]
err_export[2, idx] = validation_err_run[0, 0]
err_export[3, idx] = test_err_run[0, 0]
np.savetxt("exponentiated_gradient_learning_rate.csv",
err_export,
delimiter=",")
import plotter
plotter.plot_eg_learning_rate(err_export)
def solve_c():
(days, books_num, libraries_num, books_scores, num_of_books_in_library, signup_time_for_library,
books_per_day_from_lib, book_ids_for_library) = parse_file("inputs/c_incunabula.txt")
res = parse("inputs/c_incunabula.txt")
# Sort by (total_book_score/signup_time), then do greedy
# print (initial_library_scores(res))
scores = initial_library_scores(res)
sorted_libs = np.flip(np.argsort([score[0] for score in scores]))
sorted_books = book_ids_for_library[sorted_libs]
return (sorted_libs, sorted_books , num_of_books_in_library, books_per_day_from_lib,
days, signup_time_for_library)
def parse_file(filename):
res = input_parser.parse(filename)
days = int(res['D'])
books_num = int(res['B'])
libraries_num = int(res['L'])
books_scores = np.asarray(res['scores'])
num_of_books_in_library = np.asarray(res['books_in_library'])
signup_time_for_library = np.asarray(res['signup_time_for_library'])
books_per_day_from_lib = np.asarray(res['books_per_day_from_lib'])
book_ids_for_library = np.asarray(res['book_ids_for_library'])
return (days, books_num, libraries_num, books_scores,
num_of_books_in_library, signup_time_for_library,
books_per_day_from_lib, book_ids_for_library)
def extract_train_and_test():
"""
Test and Training Data Extractor
Helper function for extracting the test and training data. It provides
additional flexibility for if one-hot is desired or not.
:return: Tuple(np.ndarray)
"""
train, test, full_vocab = input_parser.parse()
t_col = const.COL_TARGET
if USE_BAG_OF_WORDS:
_flatten_one_hot(train)
_flatten_one_hot(test)
x_col = const.COL_BAG_WORDS
return np.matrix(train[x_col].tolist()), \
np.matrix(train[t_col].tolist()), \
np.matrix(test[x_col].tolist()), \
full_vocab
else:
x_col = const.COL_ONE_HOT
return train[x_col], train[t_col], test[x_col], full_vocab
for move, grid in final_path[::-1]:
f.write(f'{move} {collapse_list(grid)}\n')
def write_visit(f, node):
f.write(f'{node.f} {node.g} {node.h} {collapse_list(node.state.grid)}\n')
if __name__ == '__main__':
desired_folder_path = os.path.join(os.getcwd(),
"out_bfs_h1_iterative_test/")
if os.path.isdir(desired_folder_path):
shutil.rmtree(desired_folder_path, ignore_errors=True)
os.mkdir(desired_folder_path)
for version, case_args in enumerate(parse(testfile)):
# counter = Counter()
path_length = 0
with open(
os.path.join(desired_folder_path, f'{version}_bfs_search.txt'),
'w+') as search_file:
initial_node = Node('0', Board(case_args[3]), 0)
s = time.time()
result, final_node = iterative_bfs(*case_args[:3], initial_node,
search_file, version,
path_length)
print(
def main():
# open the input file for reading, and examine
error_correction = int(sys.argv[3]) #error correction flag
x = sys.argv[1]
size_of_history = sys.argv[2] #size of history file
history_file_demo = False
custom_key = ""
if len(sys.argv) > 4:
#run the program and only show encryption and decryption of history file
history_file_demo = True
custom_key = sys.argv[4]
user_input = input_parser.parse(x) #build a table of all the user input
m = len(user_input[1][0])
# store the user's input passwords and build the first history table
history = init_history.initialize_history(user_input, size_of_history)
# will compute the history of the user's input and creates a history file
# need to choose a random hardened password and r and q
pwdArray = myMath.choose_hpwd(
) #this function generates all the random numbers needed for the assignment
hpwd = pwdArray[0]
q = pwdArray[1]
r = pwdArray[2]
# will return a list of mus per feature
mu_list = myMath.compute_mu_list(history)
sigma_list = myMath.compute_sigma_list(history)
pwd = user_input[0][0] #takes password from teh user, known to be correct
master_pwd = pwd
#this is where we store the pwd and hpwd, since our Lagrange function does not function properly
master_hpwd = hpwd
# figures out if we are slow or fast and computes the instruction table.
instruction_table = myMath.compute_instruction_table(
mu_list, sigma_list, hpwd, m, r, q, pwd)
#the answers list saves the values for us, so we can complete our work around for the Lagrange function
answers_list = instruction_table[1]
instruction_table = instruction_table[0]
#now we encrypt
encrypted_history_file = compute_history.encrypt_history_data_structure(
history, hpwd)
# compute_history.write_to_disk(encrypted_history_file)
# will demonstrate that the history file is encrypted and decrypted correctly
if history_file_demo:
encrypted_history_file = compute_history.encrypt_history_data_structure(
history, custom_key)
compute_history.demostrate_history_file(encrypted_history_file,
custom_key)
for i in range(5):
print 1
second_time = False
# check the rest of the user's inputs and see if he or she logged in correctly
for i in range(
5, len(user_input[0])
): #this list iterates through all the remaining attemps and allows or denies them
#these next lines clear out any saved data in our system, so the security check is done safetly
mu_list = []
sigma_list = []
hpwd = 0
history = []
pwd = user_input[0][i]
speeds_of_user = []
#this inspects the users input to determine if they were either fast or slow on their enteries for each feature
for j in user_input[1][i]:
if j < 10:
speeds_of_user.append(0)
else:
speeds_of_user.append(1)
#now we use the values to attempt to reconstruct the hpwd
hpwd = myMath.reconstruct_polynomial(instruction_table, speeds_of_user,
q, r, pwd)
#this is again part of the work around we are doing
login_attempt = True
#this is where the work around is for the hpwd setup
for k in range(len(speeds_of_user)):
if answers_list[k] != -1:
if answers_list[k] != speeds_of_user[k]:
login_attempt = False
break
#checks the password directly for a match
if master_pwd != pwd:
login_attempt = False
#if our secondary check passes, we allow the login to take the correct hpwd
if login_attempt:
hpwd = master_hpwd
decrypted_history_file = compute_history.decrypt_history_data_structure(
encrypted_history_file, hpwd)
#here we check for the token, and then update the entire history list and rebuild the instruction table
if decrypted_history_file[1] == "Nice Work!":
decrypted_history_file = compute_history.update_history_file(
decrypted_history_file, user_input[1][i])
mu_list = myMath.compute_mu_list(decrypted_history_file)
sigma_list = myMath.compute_sigma_list(decrypted_history_file)
pwdArray = myMath.choose_hpwd()
#here we choose a new random value r
r = pwdArray[2]
instruction_table = myMath.compute_instruction_table(
mu_list, sigma_list, hpwd, m, r, q, pwd)
#this updates the tables we are saving in teh system
answers_list = instruction_table[1]
instruction_table = instruction_table[0]
encrypted_history_file = compute_history.encrypt_history_data_structure(
decrypted_history_file, hpwd)
#this simply clears the history file so an attacker cannot gain access
decrypted_history_file = ""
print 1
#this function implements our error correction for the test case files
elif (master_pwd == pwd
and error_correction): #this is where we try error reduction
login_sucess = False
login_attempt = True
speeds_of_user2 = []
#this copies a version of the user speeds for us to vary
for item in speeds_of_user:
speeds_of_user2.append(item)
#now we vary each feature speed one at a time to see if it will be successful
for l in range(len(speeds_of_user)):
login_attempt = True
speeds_of_user = []
for item in speeds_of_user2:
speeds_of_user.append(item)
#this next set alters the value of the feature speed gathered from the user
if speeds_of_user[l] == 0:
speeds_of_user[l] = 1
else:
speeds_of_user[l] = 0
#now we rerun our login attempt to see if the new value works
for k in range(len(speeds_of_user)):
if answers_list[k] != -1:
if answers_list[k] != speeds_of_user[k]:
login_attempt = False
break
if login_attempt:
hpwd = master_hpwd
decrypted_history_file = compute_history.decrypt_history_data_structure(
encrypted_history_file, hpwd)
#this again checks to see if the value was a success, and if it was, then updates the history file and builds a new instruction table
#this is the same as the code above, other than at the bottom
if decrypted_history_file[1] == "Nice Work!":
decrypted_history_file = compute_history.update_history_file(
decrypted_history_file, user_input[1][i])
mu_list = myMath.compute_mu_list(decrypted_history_file)
sigma_list = myMath.compute_sigma_list(
decrypted_history_file)
pwdArray = myMath.choose_hpwd()
r = pwdArray[2]
instruction_table = myMath.compute_instruction_table(
mu_list, sigma_list, hpwd, m, r, q, pwd)
answers_list = instruction_table[1]
instruction_table = instruction_table[0]
encrypted_history_file = compute_history.encrypt_history_data_structure(
decrypted_history_file, hpwd)
decrypted_history_file = ""
print 1
login_sucess = True
second_time = False
#here we break out of the loop of checking new combinations of answers, since we already found one that works
break
#this code will execute if we were unable to make a correction that allows entry, and prints a 0 to indicate this
if (not login_sucess) and second_time:
print 0
#second_time = False
if (not login_sucess) and (not second_time):
print 0
#second_time = True
#this else is reached if error correction is off, and simply prints a failure for the user
else:
print 0
import tensorflow as tf
import input_parser
import const
from hw05 import build_output_file
from feed_forward import init
import numpy as np
# main
if __name__ == '__main__':
# # Create the inputs
train, test, vocab, dummy_word = input_parser.parse()
# get the integer transformation of the train x
train_x = [list(x) for x in train[const.COL_WORD2VEC].values]
# get the integer transformation of the train x
test_x = [list(x) for x in test[const.COL_WORD2VEC].values]
# get the training targets
train_y = np.array([np.array(y) for y in train['target']])
# make sequence length vector s
train_sequence_lengths = [len(l) for l in train_x]
test_sequence_lengths = [len(l) for l in test_x]
# pad the examples with dummy data
max_seq_len = max(max(train_sequence_lengths), max(test_sequence_lengths))
for x_vals in [train_x, test_x]:
for x_val in x_vals:
while len(x_val) < max_seq_len:
x_val.append(dummy_word)
from utils import total_library_scores
import numpy as np
from input_parser import parse
def c_score(dataset):
'''return the score for dataset c'''
scores = np.array(total_library_scores(dataset))
scores = scores / dataset['signup_time_for_library']
return scores
def c_score_update(dataset, scores, library):
return
c = parse('inputs/c_incunabula.txt')
@profile
def test_mem_iterative(case_args):
initial_node = Node('0', Board(case_args[3]), 0)
with open('mem_test_cases/mem_test_search_file.txt', 'w+') as search_file:
result, final_node = iterative_a_star(*case_args[:3], initial_node,
search_file, 0)
search_file.close()
return
@profile
def test_mem_recursive(case_args):
initial_node = Node('0', Board(case_args[3]), 0)
with open('mem_test_cases/mem_test_search_file.txt', 'w+') as search_file:
result, final_node = recursive_a_star(*case_args[:3], initial_node,
100000000, search_file, 0, 0)
search_file.close()
return
if __name__ == '__main__':
case1 = 'mem_test_cases/case1.txt'
case2 = 'mem_test_cases/case2.txt'
case3 = 'mem_test_cases/case3.txt'
start = time.time()
test_mem_iterative(parse(case1)[0])
print(time.time() - start)
def input_parse(data): parsed_data = [] for edition in data: parsed_data.append(input_parser.parse(configuration.input.keys()[edition-2011])) return parsed_data
return min(l), max(l), np.average(l), math.sqrt(np.var(l))
def print_mmav(what_is_this, l):
print("{} between {} and {}, avg is {} var is {}".format(what_is_this, *minmax_avgvar(l)))
def analyze(dataset):
print(f"Books/Libraries/Days: {dataset['B']},{dataset['L']},{dataset['D']}")
print_mmav("book scores", dataset['scores'])
print_mmav("num of books", dataset['books_in_library'])
print_mmav("days until signup", dataset['signup_time_for_library'])
print_mmav("books per day", dataset['books_per_day_from_lib'])
print_mmav("scores in library", total_library_scores(dataset))
print(f"Books missing from all libraries: {missing_books(dataset)}")
print_mmav("libraries containing a specific book:",
[len(dataset['books_to_libraries_containing'][book]) for book in dataset['books_to_libraries_containing']])
print(f"Histogram of books-per-library: {np.histogram([len(ids) for ids in dataset['book_ids_for_library']])}")
print("B")
analyze(parse('inputs/b_read_on.txt'))
print("C")
analyze(parse('inputs/c_incunabula.txt'))
print()
analyze(parse('inputs/d_tough_choices.txt'))
print()
analyze(parse('inputs/e_so_many_books.txt'))
print()
analyze(parse('inputs/f_libraries_of_the_world.txt'))
import sqlite3
from input_parser import parse
from input_parser import target, tgds
from skolemizer import skolemize
from sql_generator import generate_sql, generate_target
if __name__ == '__main__':
# Reads the input file from stdin
input_file = ""
for line in stdin:
input_file += line
# Parse input, skolemize & generates sql
parse(input_file)
skolemized_tgds = skolemize(tgds)
sql = generate_sql(skolemized_tgds)
# Is it integrated with sqlite ?
if len(argv) > 1 and argv[1] == "-sqlite3":
database = argv[2]
target_sql = generate_target(target)
print("/* Executing the following code on", database, "... */")
print(target_sql)
print(sql)
conn = sqlite3.connect(database)
c = conn.cursor()
follow_sets = follow.get(variables, rules, first_sets)
# change terminals to include $ instead of !
terminals[-1] = '$'
parsing_table = table_builder.build(variables, terminals, first_sets, follow_sets, rules_dict)
print('Rules: ')
print('\n'.join(map(str, rules)))
print(' ---------------')
print('First Sets: ')
print(first_sets)
print(' ---------------')
print('Follow Sets: ')
print(follow_sets)
print(' ---------------')
print('Parsing Table: ')
print(parsing_table)
print(' ---------------')
for i in range(1, 5):
input_file_name = input_string_file_template.format(i)
output_file_name = output_parse_tree_file_template.format(i)
input_to_parse = file_parser.parse(input_file_name, rules=False, terminals=terminals)
sucess, parse_tree = input_parser.parse(input_to_parse[0], variables[0], parsing_table)
parse_trees.append((output_file_name, sucess, parse_tree))
print('-- Input: ')
print('--', input_to_parse[0])
print('---- Parse Tree: =>', sucess)
print('\n'.join(map(str, parse_tree)))
print(' =====================')
output_writer.write_table(parsing_table)
for output_file_name, sucess, parse_tree in parse_trees:
output_writer.write_parse_tree(output_file_name, sucess, parse_tree)
"""Test implemeted scripts."""
from input_parser import parse
# from parser import source
# from parser import target
from input_parser import tgds
from skolemizer import skolemize
from sql_generator import generate_sql
if __name__ == '__main__':
file = "examples/homework_example.txt"
with open(file, 'r+') as f:
data = f.read()
parse(data)
skolemized_tgds = skolemize(tgds)
for i, sk_tgd in enumerate(skolemized_tgds):
print("m%d:" % (i + 1))
print("Left hand side: ")
for instance in sk_tgd.lhs:
print(instance.relation.name)
print(instance.variables)
print("Right hand side:")
for instance in sk_tgd.rhs:
print(instance.relation.name)
for var in instance.variables:
if type(var) == str:
print(var)
else:
print("f_%s,%s(%s)" % (
var.mapping_name,
def test_ast_builds_correctly(self):
tree = parse(self.contents_for_test)
# Perform verification on the tree
self.assertTrue(len(tree.children) == 1)
ast = ASTBuilder(tree).ast
def purge(testdocs):
for test_doc in testdocs:
for test_set in test_doc:
test_set['doc'] = filter (lambda x: len(x) != 0, test_set['doc'])
######### IO #####################################
def write_result(testdoc, name):
f = cache.open_cache(name,'w')
f.write(json.dumps(testdoc, sort_keys=True, indent=4 * ' '))
f.close()
######### TOKEN ALTERING #########################
token_map = {
"'m" : "am",
"n't" : "not",
"'s" : "is",
"'re" : "are",
"'d" : "would",
"'ve" : "have",
'll' : "will",
"--lrb--":"(",
"--rrb--":")"
}
def token_altering(token):
return token_map[token] if token in token_map else token
if __name__ == "__main__":
data = [parse("CLEF_2011_GS"), parse("CLEF_2012_GS")]
preprocess(data)
