def addParse(self, parseText):
parseText = parseText.strip()
for parse in self.parses:
if parseText == parse.text:
parse.frequency += 1
return
parse = Parse(parseText)
self.parses.append(parse)
self.sortParses()
def newFile(self):
fi = open(resp_dir + "__init__.py", "w")
fi.write("# coding=utf-8\n")
fi.close()
tags = Parse().getTags()
for tag in tags:
self.fileOne(tag)
fi = open(resp_dir + "__init__.py", "a+")
fi.write("from " + resp_pack + tag + " import *\n")
fi.close()
def find_top_5_sentences(self):
# return a list of 5 top sentences, sorted by sum(tfidf) for each word in a sentence
# extract terms from the entire document
parser = Parse(globs.constants, globs.stop_words, self.is_stemming)
parser.parse_document(self.content)
terms = parser.terms
# split the document into sentences
sentences = sent_tokenize(self.content)
parsed_sentences = []
try:
for position, sentence in enumerate(sentences):
# parse & assign grade for each sentence
sentence_parser = Parse(globs.constants, globs.stop_words,
self.is_stemming)
sentence_parser.parse_document(sentence)
sentence_terms = sentence_parser.terms
sentence_length = sum([
int(term.count)
for key, term in sentence_terms.iteritems()
])
if sentence_length in (0, 1):
continue
sentence_score = sum([
(float(sentence_terms[token].count) / self.max_tf) * log(
float(globs.num_of_documents) /
float(globs.main_dictionary[token].df), 2)
for token in sentence_terms
]) / log(float(sentence_length), 2)
# append each original sentence with it's grade & position
parsed_sentences.append((sentence, sentence_score, position))
# find the top 5 sentences and return their rank, position and of course the sentence itself
top_5_sentences = sorted(parsed_sentences,
key=lambda tup: tup[1],
reverse=True)[:5]
ranked_5_sentences = []
for rank, sentence in enumerate(top_5_sentences):
ranked_5_sentences.append((sentence[0], rank + 1, sentence[2]))
return ranked_5_sentences
except Exception as err:
print(err)
traceback.print_exc(file=sys.stdout)
def __init__(self, corpus_path, posting_path, terms_dict, cities_dict, docs_dict, avg_doc_length, with_stemming,
with_semantics):
self.terms_dict = terms_dict
self.cities_dict = cities_dict
self.docs_dict = docs_dict
self.parser = Parse(corpus_path) ## corpus path for stop words
self.parser.to_stem = with_stemming
self.posting_path = posting_path
self.ranker = Ranker(avg_doc_length)
self.model = None
self.with_semantics = with_semantics
self.with_stemming = with_stemming
def callRobot(txt, request):
parse = Parse()
result = []
decision = parse.outputString(str(txt))
if decision[0][0] == 1:
fb = FB()
temp = fb.searchUser(str(decision[1]))
if temp != None and len(temp) > 0:
result.append("Result from Facebook, Total results are " +
str(len(temp)) + "\n")
for i, y in temp.iteritems():
result.append("UserID:" + i + "\tUser Full Name: " + y + "\n")
if decision[0][1] == 1:
twitter = twitterSearch()
temp = twitter.searchName(str(txt))
result.append("Result from twitter Total results are :" +
str(len(temp)) + "\n")
for i in temp:
result.append("UserID:" + i[0] + "\tUser Full Name:" + i[1] +
"\tUser location" + i[2] + "\n")
if decision[0][2] == 1:
fb = FB()
temp = fb.searchDetailInfo(str(txt))
if "name" in temp:
result.append("Name: " + temp["name"])
if "category" in temp:
result.append("\tCategory: " + temp["category"])
if "birthday" in temp:
result.append("\tBirthday: " + temp["birthday"])
if "about" in temp:
result.append("\tAbout: " + temp["about"])
if "pic" in temp:
result.append("\tPicture: " + temp["pic"] + "\n")
#result = fb.searchUser(str(txt))
message = ""
for i in result:
message = message + i
#for i in range(1,1000):
# print message.decode('unicode-escape')
# for i in range(1,300):
# print message[1791:1793]
#msg = message.decode('unicode-escape')
#print result
msg = message
user = GetRobot(request, msg)
return msg
def __init__(self):
# creating an object for parser class
parser_obj = Parse()
# cycle count
self.cycle = 0
self.cy_needed = 0
# loop case check
self.loop = False
# flag for i-cache
self.spec_i_flag = False
# collecting the parsed data
self.inst = parser_obj.inst
self.config = parser_obj.conf
self.registers = parser_obj.regs
self.data = parser_obj.data
# hit count
self.i_hit_count = 0
self.i_access_count = 0
self.d_miss = 0
self.d_hit_count = 0
self.d_access_count = 0
self.stall = 0
self.fflag = False
self.next = 0
self.v = 0
# register set
self.register_set = {}
# d-cache
self.d_block_0 = {d_value_0: [] for d_value_0 in range(2)}
self.d_block_1 = {d_value_1: [] for d_value_1 in range(2)}
self.least_recently_used = 0
self.least_recently_used2 = 0
# initialize instruction sets
self.mem = ['LW', 'SW', 'L.D', 'S.D']
self.add_sub = ['ADD.D', 'SUB.D']
self.int_inst = ['DADD', 'DADDI', 'DSUB', 'DSUBI', 'AND', 'ANDI', 'OR', 'ORI']
self.jump = ['HLT', 'J', 'BEQ', 'BNE']
# tracking if busy or not
self.fetch_busy = self.decode_busy = self.mem_busy = self.add_busy = self.mul_busy = self.div_busy \
= self.write_back_busy = self.iu_busy = self.jump_busy = [False, None]
def getTagPaths(self):
paths = list(Parse().getPaths())
paths.sort()
tags = list(Parse().getTags())
tags.sort()
p = Parse()
tags_paths = {}
for tag in tags:
tags_paths[tag] = []
index = []
for path in paths:
p.setPath(path)
path_tag = p.getTag()
if path_tag == tag:
tags_paths[tag].append(path)
index.append(paths.index(path))
index.sort()
index.reverse()
for i in index:
paths.pop(i)
return tags_paths
def respOne(self, path):
p = Parse()
p.setPath(path)
tag = p.getTag()
f = open(resp_dir + tag + ".py", 'a+')
casename = p.getCaseName()
resp = p.getResponse()
f.write(casename + "_resp = ")
if isinstance(resp, dict):
self.writeDict(f, resp)
elif isinstance(resp, list):
self.writeList(f, resp)
else:
f.write("\'" + resp + "\'")
f.write("\n\n")
def test_run_OLCIL1_Oa01_radiance(self):
from test_OLCIL1_parse_data import OLCIL1_test_path, OLCIL1_test_attributes
from Parse import Parse
s3parse = Parse(OLCIL1_test_path)
# Test attributes attribute
# Assert opened attributes are the same as the true attributes
for key in OLCIL1_test_attributes.keys():
if type(OLCIL1_test_attributes[key]) == list:
self.assertItemsEqual(s3parse.attributes[key],
OLCIL1_test_attributes[key],
"Problem with %s" % key)
else:
self.assertEqual(s3parse.attributes[key],
OLCIL1_test_attributes[key],
"Problem with %s" % key)
def respAssertOne(self, path):
p = Parse()
p.setPath(path)
tag = p.getTag()
f = open(respAssert_dir + tag + ".robot", 'a+')
casename = p.getCaseName()
resp = p.getResponse()
f.write(casename + "_resp\n")
f.writelines("\t${resp} evaluate ${" + casename + "_resp}\n")
if isinstance(resp, dict):
self.writeDict(f, resp, "resp")
elif isinstance(resp, list):
self.writeList(f, resp, "resp")
else:
f.write("\tShould Be Equal ${resp} ${sql}\n")
f.write("\n\n")
def paramOne(self, path):
p = Parse()
p.setPath(path)
tag = p.getTag()
f = open(paramsnew_dir + tag + ".py", 'a+')
casename = p.getCaseName()
params = p.getParams()
if len(params) > 0:
f.write(casename + "_param = ")
for i in range(len(params)):
name = params[i]["name"]
if i == 0:
param = name
else:
param = dict(param, **name)
self.writeDict(f, param)
f.write("\n\n")
def testIndexing(self):
p = Parse(
"leader<table><tr><td>one</td><td>two</td><td>three</td></tr><tr><td>four</td></tr></table>trailer"
)
self.assertEquals("one", p.at(0, 0, 0).body)
self.assertEquals("two", p.at(0, 0, 1).body)
self.assertEquals("three", p.at(0, 0, 2).body)
self.assertEquals("three", p.at(0, 0, 3).body)
self.assertEquals("three", p.at(0, 0, 4).body)
self.assertEquals("four", p.at(0, 1, 0).body)
self.assertEquals("four", p.at(0, 1, 1).body)
self.assertEquals("four", p.at(0, 2, 0).body)
self.assertEquals(1, p.size())
self.assertEquals(2, p.parts.size())
self.assertEquals(3, p.parts.parts.size())
self.assertEquals("one", p.leaf().body)
self.assertEquals("four", p.parts.last().leaf().body)
def __init__(self, dataStorage):
"""
初始化
:return:
"""
self.driver = webdriver.Chrome()
# self.driver.implicitly_wait(1)
# self.driver = webdriver.PhantomJS() #########不能使用，个人成就和推荐信信息爬不出来
self.accountName = ACCOUNT.get('name')
self.urlName = LINKEDINURL.get('name')
self.usersName = LINKEDINUSERS.get('name')
self.postName = LINKEDINPOSTSURL.get('name')
self.__dataStorage = dataStorage
self.parse = Parse()
self.webAction = WebAction()
# self.linkedin_limit = 0
self.__initLogger()
def __init__(self, dataStorage):
"""
初始化
:return:
"""
self.driver = webdriver.Chrome()
# self.driver = webdriver.PhantomJS()
# self.driver.maximize_window()
# 设置页面超时时间
self.driver.set_page_load_timeout(30)
self.accountName = ACCOUNT.get('name')
self.urlName = LINKEDINURL.get('name')
self.usersName = LINKEDINUSERS.get('name')
self.__dataStorage = dataStorage
self.parse = Parse()
# self.linkedin_limit = 0
self.__initLogger()
def __init__(self, arr):
parser_obj = Parse()
# depending on the number of registers
if len(arr) == 1:
self.inst = arr[0]
self.reg1 = ''
self.reg2 = ''
self.reg3 = ''
if len(arr) == 3:
self.inst = arr[0]
self.reg1 = arr[1]
self.reg2 = arr[2]
self.reg3 = ''
if len(arr) == 4:
self.inst = arr[0]
self.reg1 = arr[1]
self.reg2 = arr[2]
self.reg3 = arr[3]
self.address = 0
self.x = ''
self.mem_check = False
self.sub_cycle = 0
self.int_cycle = 1
self.iu_cycle = 1
self.mem_cycle = int(parser_obj.conf[0]['Main memory'])
self.add_sub_cycle = int(parser_obj.conf[0]['FP adder'])
self.mul_cycle = int(parser_obj.conf[0]['FP Multiplier'])
self.div_cycle = int(parser_obj.conf[0]['FP divider'])
# cycle count for each stage
self.fetch = self.decode = self.execute = self.write_back = 0
# possible hazards
self.raw = self.war = self.waw = self.struct_haz = 'N'
self.status = 'IF'
# instruction cache flag
self.i_flag = [False, 0]
self.cache_miss_flag = False
self.d_flag = False
def test_Parse(self):
from Parse import Parse
# Create parser
test_parser_directory = abspath(
pjoin(dirname(dirname(__file__)), "tools", "test"))
createTestParsingTool(test_parser_directory)
# Ensure test data reader generated
self.assertTrue(exists(test_parser_directory), "Test parser missing")
# Parse test product
testParse = Parse("")
# Assert attributes expected
self.assertEqual(testParse.attributes, {})
# Remove test data parser
shutil.rmtree(test_parser_directory)
def __init__(self):
self.parse = Parse()
# Authorise my request using OAuth
self.t = Twitter(
auth=OAuth("2999654973-YS9XG2UdzWDNJhapOjAUDHjrv5wgG3az3MZ5JfF",
"NlP0PWQPJv1Z0lPsH6xv3bb7jw3Aos5xD18ni3ODuDZME",
"p6yoZBcF9NnteTV67YbVRwcis",
"nY0Q4ugFlef0AtlRTaXptshXxJNGymxFgn4KzaeF77UXlMkRuq"))
self.tweets = []
# Define target account and fetch tweet
self.t.statuses.user_timeline(screen_name="premierleague",
count=1,
exclude_replies=True,
include_rts=False)
tweets = self.t.statuses.user_timeline(screen_name="premierleague",
count=1,
exclude_replies=True,
include_rts=False)
for i in xrange(len(tweets)):
self.tweets.append(self.parse.parseOne(tweets[i]))
def getWordsAndTheirParses(self, words):
for word in words:
lines = word.split("\n")
firstLine = lines[0]
splittedFirstLine = firstLine.split(":")
if len(splittedFirstLine) < 2:
continue
wordText = splittedFirstLine[0]
unique_serialized_word = self.upsertWord(wordText)
correctMorphParseNumber = splittedFirstLine[1] # get correct
correctMorphParseFull = lines[int(correctMorphParseNumber)]
unique_serialized_word.addParse(correctMorphParseFull)
# if "Punc" not in correctMorphParseFull:
w = Word(wordText) # add all words to all_words_in_corpus
w.correct_parse = Parse(correctMorphParseFull)
self.all_words_in_corpus.append(w)
def __string_to_terms(self, string):
# given a string, parse into terms. in this case it's for a query
parser = Parse(globs.constants, globs.stop_words, self.is_stemming)
parser.parse_document(string)
return parser.terms.keys()
def paramAll(self): paths = Parse().getPaths() for path in paths: self.paramOne(path)
def newFile(self): tags = Parse().getTags() for tag in tags: self.fileOne(tag)
print("HI!\n")
from Parse import Parse
j = input("Enter 'a' for Yahoo and 'b' for rediffmoney : ")
if (j == 'a'):
i = str(input("enter the link for the stock : "))
Parse(i, 0)
else:
i = str(input("enter the link for the stock : "))
Parse(i, 1)
def respAssertAll(self):
paths = Parse().getPaths()
for path in paths:
self.respAssertOne(path)
def td(self, body):
return Parse(tag="td", body=self.gray(body))
:param n: 从0开始算
:param text: list
:return:
'''
if n < 0:
pass
else:
fp = open(filename, 'r')
lines = fp.readlines()
fp.close()
text = list(text)
lines[n] = text[0]
n = int(n)
for i in range(1, len(text)):
line = n + i
lines.insert(line, text[i])
fp = open(filename, 'w')
fp.writelines(lines)
fp.close()
if __name__ == '__main__':
tags = Parse().getTags()
for tag in tags:
filename = cases_dir + tag + ".robot"
text = ["Library ../../Common/Function.py\n"]
line = EditFile().getLine(filename,
"Library ../../Util/Function.py")
EditFile().editText(filename, line, text)
# EditFile().insertFileText(filename, line+1, text)
def fromString(self, string):
self.data = Parse(string)
self.make()
return self
def td(self, body):
return Parse(tag="td", body=self.info(body))
class Capture:
rfp = Parse(r'C:\Users\arile\Documents\renciTextExamples\rfpAI.txt')
rfp.find_title()
rfp.find_due_dates()
rfp.find_program_description(4) # argument is the number of sentences of description
rfp.find_congnizant_program_officers()
def parseLine(self, slots):
numSlots = len(slots)
initialParse = Parse(self, numSlots)
parses = initialParse.extend(slots[0])
parses[0].comparisonNums.add(1)
boundedParses = []
for slotN in range(1, numSlots):
newParses = []
for parse in parses:
newParses.append(parse.extend(slots[slotN]))
for parseSetIndex in range(len(newParses)):
parseSet = newParses[parseSetIndex]
for parseIndex in range(len(parseSet)):
parse = parseSet[parseIndex]
parse.comparisonParses = []
if len(parseSet) > 1 and parseIndex == 0:
parse.comparisonNums.add(parseSetIndex)
for comparisonIndex in parse.comparisonNums:
# should be a label break, but not supported in Python
# find better solution; redundant checking
if parse.isBounded:
break
try:
for comparisonParse in newParses[comparisonIndex]:
if parse is comparisonParse:
continue
if not comparisonParse.isBounded:
if parse.canCompare(comparisonParse):
boundingRelation = parse.boundingRelation(
comparisonParse)
if boundingRelation == Bounding.bounds:
# print parse.__report__()
# print '--> bounds -->'
# print comparisonParse.__report__()
# print
comparisonParse.isBounded = True
comparisonParse.boundedBy = parse
elif boundingRelation == Bounding.bounded:
# print
# print comparisonParse.__report__()
# print '--> bounds -->'
# print parse.__report__()
# print
parse.isBounded = True
parse.boundedBy = comparisonParse
break
elif boundingRelation == Bounding.equal:
parse.comparisonParses.append(
comparisonParse)
else:
parse.comparisonParses.append(
comparisonParse)
except IndexError:
pass
parses = []
#boundedParses=[]
parseNum = 0
for parseSet in newParses:
for parse in parseSet:
if parse.isBounded:
boundedParses += [parse]
elif parse.score() >= 1000:
parse.unmetrical = True
boundedParses += [parse]
else:
parse.parseNum = parseNum
parseNum += 1
parses.append(parse)
for parse in parses:
parse.comparisonNums = set()
for compParse in parse.comparisonParses:
if not compParse.isBounded:
parse.comparisonNums.add(compParse.parseNum)
return parses, boundedParses
from Parse import Parse # my parse class
postfixStrings = [
'4 5 +', # 9
'9 3 /', # 3
'17 8 -', # 9
'6 2 / 5 +', # 8
'4 5 + 7 2 - *', # 45
'4 5 7 2 + - *', # -16
'3 4 + 2 * 7 /', # 2
'5 7 + 6 2 - *', # 48
'4 2 3 5 1 - + * +', # 18
'4 2 + 3 5 1 - * +' # 18
]
# input format : 'operand operand operator'
for string in postfixStrings:
print '{:18} = {}'.format(string, Parse(string).evaluate())
