def create_list_of_feature_dic(train_sents,
word_matrix_size,
word_map,
tag_matrix_size,
tag_map,
pkl_path,
pickles=True,
augmented=False):
if pickles:
list_feat_dict_path = pkl_path
if not os.path.exists(list_feat_dict_path):
list_feat_dics = []
for sent in tqdm(train_sents):
feat_dict = {}
for i, word1 in enumerate(sent):
for j, word2 in enumerate(sent):
if (i == j):
continue
feat_dict[(word1,
word2)] = feature_func(word1,
word2,
word_matrix_size,
tag_matrix_size,
word_map,
tag_map,
augmented=augmented)
list_feat_dics.append(feat_dict)
save_pickle(list_feat_dics, list_feat_dict_path)
else:
list_feat_dics = load_pickle(list_feat_dict_path)
else:
list_feat_dics = []
for sent in tqdm(train_sents):
feat_dict = {}
for i, word1 in enumerate(sent):
for j, word2 in enumerate(sent):
if (i == j):
continue
feat_dict[(word1,
word2)] = feature_func(word1,
word2,
word_matrix_size,
tag_matrix_size,
word_map,
tag_map,
augmented=augmented)
list_feat_dics.append(feat_dict)
return list_feat_dics
def run():
hook = Webhook(HOOK)
if not os.path.exists('old_urls.pkl'):
write_pickle([], 'old_urls.pkl')
while True:
post_urls = Redditbot().get_posts('dankmemes', 10000)
for url in post_urls:
if url in load_pickle('old_urls.pkl'):
pass
else:
hook.send(url)
append_pickle(url, 'old_urls.pkl')
sleep(600)
def list_of_word_tup_per_tree(trees_train, pkl_path):
list_tup_per_tree_path = pkl_path
if not os.path.exists(list_tup_per_tree_path):
tree_list_of_tup_words = []
for tree in trees_train:
tree_list = []
for node in range(len(tree.nodes)):
tail = tree.nodes[node]['word']
head_idx = tree.nodes[node]['head']
head = tree.nodes[head_idx]['word']
if tail == None or head == None:
continue
tree_list.append((head, tail))
tree_list_of_tup_words.append(tree_list)
save_pickle(tree_list_of_tup_words, list_tup_per_tree_path)
else:
tree_list_of_tup_words = load_pickle(list_tup_per_tree_path)
return tree_list_of_tup_words
def main(manually=True, test=False):
# Als ze al eerder allemaal gedetecteerd zijn, moeten we niet nog eens checken (tenzij handmatig opgeroepen)
if not manually and check_online():
return
year = 0 if not test else 1 # 0 is current year, 1 previous year
driver = get_results_page(year)
num_courses, results = check_all_courses(
driver, semesters_to_check=semesters_to_check)
num_results = len(results)
announcement = f"Momenteel hebben we punten voor {num_results} van de {num_courses} vakken uit semester {semesters_to_check}:\n"
for course in results:
announcement += f"- {course.description}\n"
if test:
announcement += "Note: Dit is een test en gaat over de resultaten van vorig jaar. \n"
if manually:
announcement += f"Laatste succesvolle automatische test was om {get_logged_time()}.\n"
# Kijk of er een punt is bijgekomen, en if so, post it
last_results = load_pickle("num_results.pickle", 0)
# Niet posten bij manuele oproepen om dubbele posts te vermijden
if not manually and num_results > last_results:
# Er zijn punten bijgekomen
make_announcement(announcement)
store_pickle("num_results.pickle", num_results)
if not manually:
log_time()
print(announcement)
driver.close()
return announcement
def get_logged_time():
load_pickle("last.pickle", "unknown")
def check_online():
# Checks whether this script has already detected that the results are online
return load_pickle("online.pickle", False)
def load_from_file(self, size):
self = pickles.load_pickle('\\feature_sets\\fs_' + str(size))
def load_from_file():
data = pickles.load_pickle('clfs', path='')
return data