def run_penn():
file_list = os.listdir(fileparser.resource_path)
# training
print('training...')
sentences_file = open('sentences.txt', 'w')
for file in file_list:
if file.startswith(fileparser.training_prefix):
training_file = open(fileparser.resource_path + file, 'r')
sentence_list = fileparser.parse(training_file)
sentences_file.write('%s\n' % (sentence_list, ))
train(aa, bb, cc, vv, sentence_list)
sentences_file.close()
a_file = open('a.txt', 'w')
util.prettywrite_nested_map(aa, a_file)
a_file.close()
b_file = open('b.txt', 'w')
util.prettywrite_nested_map(bb, b_file)
b_file.close()
c_file = open('c.txt', 'w')
util.prettywrite_map(cc, c_file)
c_file.close()
print('training done.')
# computing likelihood
print('computing likelihood...')
forward_file = open('forward.txt', 'w')
for file in file_list:
if file.startswith(fileparser.test_prefix):
training_file = open(fileparser.resource_path + file, 'r')
sentence_list = fileparser.parse(training_file)
for sentence in sentence_list:
(forward_p, forward_table) = forward_algorithm(aa, bb, cc, vv, sentence)
(backward_p, backward_table) = backward_algorithm(aa, bb, cc, vv, sentence)
forward_file.write('%s\n %s\n %s\n\n' % (sentence, forward_p, backward_p))
print('likelihood computed.')
forward_file.close()
# computing most likely tag sequencesanc accuracy
print('computing most likely tag sequence and tagger accuracy...')
viterbi_file = open('viterbi.txt', 'w')
match_count = 0.0
total_count = 0.0
for file in file_list:
if file.startswith(fileparser.test_prefix):
training_file = open(fileparser.resource_path + file, 'r')
sentence_list = fileparser.parse(training_file)
for sentence in sentence_list:
(p, viterbi_table, viterbi_backpointer, tagger_sequence) = viterbi_algorithm(aa, bb, cc, vv, sentence)
human_sequence = map(fileparser.map_extract_tag, sentence)
# update tagger accuracy information
for i in range(min(len(human_sequence), len(tagger_sequence))): # because of underflow it is possible that the tag sequences are not equal in length...s
if tagger_sequence[i] == human_sequence[i]:
match_count = match_count + 1.0
total_count = total_count + max(len(human_sequence), len(tagger_sequence))
viterbi_file.write('%s\n%s\nProbability: %f\n\n' % (human_sequence, tagger_sequence, p))
print('most likely tag sequence computed.')
print('accuracy of tagger is: %f' % (match_count / total_count, ))
viterbi_file.close()
return (aa, bb, cc)
def change_role(self):
lines = parse(self.path)
for line in lines:
course_name = line[0]
user = line[1]
role = line[2]
self.send_data_output(
"Cambiando rol de {} a {} en el curso {}".format(
user, role, course_name))
self.driver.get(self.url_dict["courses"].format(self.env))
self.change_combo_box_value("courseInfoSearchKeyString",
"CourseId")
self.search_bar("courseInfoSearchText", course_name)
self.click_text_link(course_name)
course_id = self.get_course_id(self.driver.current_url)
self.driver.get(
"https://{}.blackboard.com/webapps/blackboard/execute/userManager?"
.format(self.env) + course_id)
self.change_combo_box_value("userInfoSearchOperatorString",
"Equals")
self.search_bar("search_text", user)
try:
usermenu = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div[4]"
"/form/div[2]/div[3]/div/table/tbody/tr/th/span[2]/a")
except NoSuchElementException:
usermenu = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div[3]"
"/form/div[2]/div[3]/div/table/tbody/tr/th/span[2]/a")
usermenu.send_keys(Keys.ENTER)
modify_role = self.driver.find_element_by_id(
"cp_course_role_modify")
modify_role.click()
self.check_box_value(role)
self.submit_changes("bottom_Submit")
self.send_data_output(
"Rol de {} cambiado a {} en el curso {} exitosamente".format(
user, role, course_name))
def getProfiles(fileDirec):
# Takes in the directory path, read the profiles in them and
# return all the profiles
profiles = {}
files = listdir(fileDirec)
for filename in files:
with open(join(fileDirec, filename), "r") as f:
name, profile = fileparser.parse(f)
profiles[name] = profile
f.close()
return profiles
def change_date(self):
lines = parse(self.path)
for elem in lines:
course_name = elem[0]
if len(course_name) == 1:
return
value = elem[1]
self.driver.get(self.url_dict["courses"].format(self.env))
self.change_combo_box_value("courseInfoSearchKeyString",
"CourseId")
self.search_bar("courseInfoSearchText", course_name)
self.click_text_link(course_name)
course_id = self.get_course_id(self.driver.current_url)
self.driver.get(self.url_dict["properties"].format(
self.env, course_id))
end_date_chk = self.driver.find_element_by_id("end_duration")
end_date_chk.send_keys(Keys.NULL) # Focus
if value == "Y":
if not end_date_chk.is_selected():
end_date_chk.send_keys(Keys.SPACE)
elif value == "N":
if end_date_chk.is_selected():
end_date_chk.send_keys(Keys.SPACE)
self.submit_changes("bottom_Submit")
self.send_data_output(
"Completado, curso {} fue marcado con {} en el cierre".format(
course_name, value))
def post_tutor_info(self):
lines = parse(self.path)
for course_id in lines:
self.send_data_output(
"Publicando ficha de tutor {} en curso {}".format(
self.username, course_id))
self.driver.get(self.url_dict["courses"].format(self.env))
self.change_combo_box_value("courseInfoSearchKeyString",
"CourseId")
self.search_bar("courseInfoSearchText", course_id)
self.click_text_link(course_id)
edit_mode = self.driver.find_element_by_id('statusText')
if edit_mode.get_attribute('innerHTML') == 'DESACTIVADO':
edit_mode.click()
sleep(0.5)
sleep(1)
menu_puller = self.driver.find_element_by_id('menuPuller')
if menu_puller.get_attribute('title') == 'Mostrar menú Curso':
menu_puller.click()
sleep(0.5)
plus = self.driver.find_element_by_id('addCmItem')
plus.click()
sleep(1)
content_item = self.driver.find_element_by_id(
'addContentAreaButton')
content_item.send_keys(Keys.ENTER)
content_name = self.driver.find_element_by_id('addContentAreaName')
content_name.send_keys("Contacto Tutoría")
content_availabilty = self.driver.find_element_by_id(
'content_area_availability_ckbox')
content_availabilty.send_keys(Keys.SPACE)
try:
submit = self.driver.find_element_by_id(
'addContentAreaFormSubmit')
submit.send_keys(Keys.ENTER)
self.driver.find_element_by_link_text(
'Contacto Tutoría').click()
except ElementClickInterceptedException:
self.driver.find_element_by_id(
'addContentAreaFormCancel').click()
self.driver.find_element_by_link_text(
'Contacto Tutoría').click()
self.driver.find_element_by_xpath(
'/html/body/div[5]/div[2]/div/div/div/div/div[1]/div/span[1]/a'
).click()
text_only = self.driver.find_element_by_xpath(
'/html/body/div[7]/ul/li[5]/a')
if text_only.get_attribute('title') == 'Mostrar solo texto':
self.driver.get(text_only.get_attribute('href'))
add_element = self.driver.find_element_by_id(
'content-handler-resource/x-bb-document')
self.driver.get(add_element.get_attribute('href'))
elem_title = self.driver.find_element_by_id("user_title")
elem_title.send_keys('Contacto Tutoría')
self.driver.execute_script("window.scrollTo(0, 300)")
sleep(3)
try:
insert_image = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div["
"2]/form/div/div[2]/div/ol/li[3]/div["
"2]/table/tbody/tr/td/span[2]/table/tbody/tr["
"1]/td/div/span/div[2]/table[3]/tbody/tr/td[4]")
insert_image.click()
except ElementNotInteractableException:
self.driver.find_element_by_id(
"messagetext_bb_slidercontrol").click()
sleep(0.2)
insert_image = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div["
"2]/form/div/div[2]/div/ol/li[3]/div["
"2]/table/tbody/tr/td/span[2]/table/tbody/tr["
"1]/td/div/span/div[2]/table[3]/tbody/tr/td[4]")
insert_image.click()
finally:
self.send_data_output(
"Adjuntando URL de IMG en ventana emergente")
old_window = self.driver.window_handles[0]
new_window = self.driver.window_handles[1]
self.driver.switch_to.window(new_window)
img_text = self.driver.find_element_by_id(
"imagepackage_selectedCSFilePath")
img_text.send_keys(self.url_dict['tutor_info'])
alt = self.driver.find_element_by_id("alt")
alt.send_keys(".")
title = self.driver.find_element_by_id("title")
title.send_keys(".")
insert_btn = self.driver.find_element_by_id("insert")
insert_btn.click()
self.driver.switch_to.window(old_window)
self.submit_changes('bottom_Submit')
self.send_data_output('Publicada!')
sleep(1)
def post_announcements(self):
lines = parse(self.path)
for elem in lines:
course_name = elem[0]
post_type = elem[1]
subject = elem[2]
url_txt = elem[3]
send_immediately = elem[4]
self.send_data_output("Procesando anuncio tipo {} en {}".format(
post_type, course_name))
self.driver.get(self.url_dict["courses"].format(self.env))
self.change_combo_box_value("courseInfoSearchKeyString",
"CourseId")
self.search_bar("courseInfoSearchText", course_name)
self.click_text_link(course_name)
course_id = self.get_course_id(self.driver.current_url)
self.driver.get(self.url_dict["announcements"].format(course_id))
pk1 = course_id.split("=")
pk1 = pk1[1]
self.driver.execute_script(
"javascript:designer_participant.toggleEditMode('javascript:"
"Announcement.addAnnouncement()','{}','designer')".format(pk1))
subj = self.driver.find_element_by_id("subject")
subj.send_keys(subject)
self.driver.find_element_by_id("isPermanent_true").send_keys(
Keys.NULL)
sleep(3)
if post_type == "IMG":
try:
insert_image = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div["
"2]/form/div/div[2]/div/ol/li[2]/div["
"2]/table/tbody/tr/td/span[2]/table/tbody/tr["
"1]/td/div/span/div[2]/table[3]/tbody/tr/td[4]")
insert_image.click()
except ElementNotInteractableException:
self.driver.find_element_by_id(
"messagetext_bb_slidercontrol").click()
sleep(0.2)
insert_image = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div["
"2]/form/div/div[2]/div/ol/li[2]/div["
"2]/table/tbody/tr/td/span[2]/table/tbody/tr["
"1]/td/div/span/div[2]/table[3]/tbody/tr/td[4]")
insert_image.click()
finally:
self.send_data_output(
"Adjuntando URL de IMG en ventana emergente")
old_window = self.driver.window_handles[0]
new_window = self.driver.window_handles[1]
self.driver.switch_to.window(new_window)
img_text = self.driver.find_element_by_id(
"imagepackage_selectedCSFilePath")
img_text.send_keys(url_txt)
alt = self.driver.find_element_by_id("alt")
alt.send_keys(".")
title = self.driver.find_element_by_id("title")
title.send_keys(".")
insert_btn = self.driver.find_element_by_id("insert")
insert_btn.click()
self.driver.switch_to.window(old_window)
elif post_type == "HTML":
try:
insert_html = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div["
"2]/form/div/div[2]/div/ol/li[2]/div["
"2]/table/tbody/tr/td/span[2]/table/tbody/tr["
"1]/td/div/span/div[2]/table[3]/tbody/tr/td[28]")
insert_html.click()
except ElementNotInteractableException:
self.driver.find_element_by_id(
"messagetext_bb_slidercontrol").click()
sleep(0.2)
insert_html = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div["
"2]/form/div/div[2]/div/ol/li[2]/div["
"2]/table/tbody/tr/td/span[2]/table/tbody/tr["
"1]/td/div/span/div[2]/table[3]/tbody/tr/td[28]")
insert_html.click()
self.send_data_output(
"Insertando código HTML en ventana emergente")
old_window = self.driver.window_handles[0]
new_window = self.driver.window_handles[1]
self.driver.switch_to.window(new_window)
html = self.driver.find_element_by_id("htmlSource")
html.click()
html.clear()
relative_path = self.path[:self.path.rindex("/") + 1]
hmtl_file = open(relative_path + url_txt,
encoding="utf-8",
mode="r")
complete_html_file = hmtl_file.read()
complete_html_file = complete_html_file.replace("\n", "")
html.send_keys(complete_html_file)
insert_btn = self.driver.find_element_by_id("insert")
insert_btn.click()
self.driver.switch_to.window(old_window)
elif post_type == "TEXT":
self.driver.switch_to.frame("messagetext_ifr")
body = self.driver.find_element_by_xpath("html/body")
sleep(1)
body.send_keys(url_txt)
self.driver.switch_to.default_content()
if send_immediately == "Y":
self.send_data_output("Marcando envío inmediato")
box = self.driver.find_element_by_id("pushNotify_true")
box.send_keys(Keys.SPACE)
self.submit_changes("bottom_Submit")
self.send_data_output("Publicado!")
def disable_students(self):
lines = parse(self.path)
for course_id in lines:
self.send_data_output("Procesando curso {}".format(course_id))
term, nrc, code, ed_me = self.disect_course_id(course_id)
user_list = self.extract_data_banner_ws(term, nrc)
self.driver.get(self.url_dict["courses"].format(self.env))
self.change_combo_box_value("courseInfoSearchKeyString",
"CourseId")
self.search_bar("courseInfoSearchText", course_id)
self.click_text_link(course_id)
pk1 = self.get_course_id(self.driver.current_url)
self.driver.get(self.url_dict["user_manager"].format(
self.env, pk1))
self.list_show_all()
user_listbb, user_disabled = self.compare_bn_bb()
list_difference = [
item for item in user_listbb if item not in user_list
]
self.send_data_output(
"Ajustando diferencia de {} estudiantes en Blackboard".format(
len(list_difference)))
if len(user_disabled) > 0:
self.send_data_output(
"{} usuarios ya se encuentran deshabilitados:\n".format(
len(user_disabled)) +
("{}\n" * len(user_disabled)).format(*user_disabled))
list_difference = [
item for item in list_difference
if item not in user_disabled
]
if len(list_difference) == 0:
self.send_data_output(
"Curso {} completado, los usuarios ya estaban deshabilitados"
.format(course_id))
continue
for user in list_difference:
self.driver.get(self.url_dict["user_manager"].format(
self.env, pk1))
self.change_combo_box_value("userInfoSearchOperatorString",
"Equals")
self.search_bar("search_text", user)
try:
usermenu = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div[4]"
"/form/div[2]/div[3]/div/table/tbody/tr/th/span[2]/a")
except NoSuchElementException:
usermenu = self.driver.find_element_by_xpath(
"/html/body/div[5]/div[2]/div/div/div/div/div[3]"
"/form/div[2]/div[3]/div/table/tbody/tr/th/span[2]/a")
usermenu.click()
modify_role = self.driver.find_element_by_id(
"cp_course_availability")
modify_role.click()
self.change_combo_box_value("availableIndex", "false")
self.submit_changes("bottom_Submit")
self.change_combo_box_value("userInfoSearchOperatorString",
"NotBlank")
self.search_bar("search_text")
self.send_data_output(
"Curso {} completado, {} usuarios deshabilitados:\n".format(
course_id, len(list_difference)) +
("{}\n" * len(list_difference)).format(*list_difference))
def main():
aa = {}
bb = {}
vocabulary = set([])
file_list = os.listdir(fileparser.resource_path)
# training
print('Training...')
for file in file_list:
if file.startswith(fileparser.training_prefix):
training_file = open(fileparser.resource_path + file, 'r')
sentence_list = fileparser.parse(training_file)
train(aa, bb, sentence_list, vocabulary)
print('DONE')
# transform into a and b
t_start = time.time()
a = {}
b = {}
user_states = list(aa.iterkeys())
states = list(aa.iterkeys()) + [hmm.START, hmm.END]
for state in aa.iterkeys():
sum_counts = sum([aa[state][next_state] for next_state in aa[state].iterkeys()])
for next_state in states:
if aa[state].has_key(next_state):
a[(state, next_state)] = LogProbability(aa[state][next_state]) / sum_counts
else:
a[(state, next_state)] = LogProbability(0.0)
# Extract vocabulary
vocab = {}
for state in bb.iterkeys():
for output in bb[state].iterkeys():
vocab[output] = vocab.get(output, 0) + 1
# Create matrix B and apply smoothing
for state in bb.iterkeys():
sum_emmited = (sum([bb[state][output] for output in bb[state].iterkeys()]) if bb.has_key(state) else 0)
b[state] = {}
for output in vocab.iterkeys():
b[state][output] = LogProbability(bb.get(state, {}).get(output, 0.0) + 1.0) / (sum_emmited + len(vocab))
# Calculate average of singleton words
unknown_b = {}
singletons = [word for word, count in vocab.iteritems() if count == 1]
for s in bb.iterkeys():
sm = LogProbability(0.0)
for singleton in singletons:
sm += b[s].get(singleton, LogProbability(0.0))
b[s][hmm.UNKNOWN] = sm / len(singletons)
print hmm.states(a, b)
def unknown_b_mapper(s, word):
print 'Could not find word %s in state %s' % (word, s)
print 'State has: %s' % (list(b[s].iterkeys()))
assert word not in vocab
print '**UNKNOWN** %s' % word
return unknown_b[s]
# computing likelihood
print('computing likelihood...')
forward_file = open('forward.txt', 'w')
for file in file_list:
if file.startswith(fileparser.test_prefix):
training_file = open(fileparser.resource_path + file, 'r')
sentence_list = fileparser.parse(training_file)
for sentence in sentence_list:
words = [word for word, tag in sentence]
words = [(word if word in vocab else hmm.UNKNOWN) for word in words]
forward_table = {}
backward_table = {}
forward_p = hmm.forward_algorithm(words, a, b, forward=forward_table)
backward_p = hmm.backward_algorithm(words, a, b, backward=backward_table)
forward_file.write('%s\n %s\n %s\n\n' % (words, forward_p.logv, backward_p.logv))
forward_file.close()
print('likelihood computed.')
print 'Took %ds' % (time.time() - t_start)
# computing most likely tag sequencesanc accuracy
print('computing most likely tag sequence and tagger accuracy...')
match_count = 0.0
total_count = 0.0
for file in file_list:
if file.startswith(fileparser.test_prefix):
training_file = open(fileparser.resource_path + file, 'r')
sentence_list = fileparser.parse(training_file)
for sentence in sentence_list:
words = [word for word, tag in sentence]
words = [(word if word in vocab else hmm.UNKNOWN) for word in words]
tagger_sequence = hmm.viterbi(words, a, b)
human_sequence = [tag for word, tag in sentence]
#print tagger_sequence
#print human_sequence
#print '----'
# update tagger accuracy information
for i in range(min(len(human_sequence), len(tagger_sequence))): # because of underflow it is possible that the tag sequences are not equal in length...s
if tagger_sequence[i] == human_sequence[i]:
match_count = match_count + 1.0
total_count = total_count + max(len(human_sequence), len(tagger_sequence))
#print('%s\n%s\nProbability: %f\n' % (human_sequence, tagger_sequence, p))
print('most likely tag sequence computed.')
print('accuracy of tagger is: %f' % (match_count / total_count, ))
class FromImportScope(NodeScope):
def __init__(self, from_import_node, parent, root):
super(FromImportScope, self).__init__(from_import_node, parent, root)
def __str__(self):
#print 'type is from import scope, from name is',self.Node.Name,'line start is',self.LineStart,'line end is',self.LineEnd
return self.Node.Name
def EqualName(self, name):
for child_scope in self.ChildScopes:
if child_scope.EqualName(name):
return True
return False
class MainFunctionScope(NodeScope):
def __init__(self, main_function_node, parent, root):
super(MainFunctionScope, self).__init__(main_function_node, parent,
root)
if __name__ == "__main__":
module = fileparser.parse(r"D:\env\Noval\noval\parser\nodeast.py")
module_scope = ModuleScope(module, 100)
module_scope.MakeModuleScopes()
module_scope.RouteChildScopes()
func_scope = module_scope.FindDefinitionScope("FuncDef.__init__")
print(func_scope)
