def save_latest_feed(self):
# save the latest feed of the topic
if self.url is None:
raise ValueError("Did not found url for the topic")
else:
if self.soup is None:
self.parser()
title = self.get_title()
try:
os.mkdir(title.replace("/", "") + " latest feed(topic)")
os.chdir(title.replace("/", "") + " latest feed(topic)")
except OSError:
os.chdir(title.replace("/", "") + " latest feed(topic)")
feed_raw = self.soup.find("div", class_="zu-top-feed-list")
question_links = []
question_links_raw = feed_raw.find_all("a", class_="question_link")
for question_link_raw in question_links_raw:
question_link_raw = Zhihu + question_link_raw["href"]
question_links.append(question_link_raw)
answer_links = []
answer_links_raw = feed_raw.find_all("span", class_="answer-date-link-wrap")
for answer_link_raw in answer_links_raw:
answer_link = Zhihu + answer_link_raw.a["href"]
answer.Answer(answer_link).save_answer_to_file()
answer_links.append(answer_link)
os.chdir("..")
return
def getInterview(InterviewID):
conn = sqlite3.connect('interview_portal.db')
conn.row_factory = sqlite3.Row
SELECT = ("SELECT ii.InterviewName, "
"ir.QuestionID, "
"qi.QuestionText, "
"qr.AnswerID, "
"a.AnswerText ")
FROM = "FROM InterviewInfo ii "
JOINS = (
"INNER JOIN InterviewRelation ir ON ii.InterviewID = ir.InterviewID "
"INNER JOIN QuestionInfo qi ON ir.QuestionID = qi.QuestionID "
"INNER JOIN QuestionRelation qr ON qi.QuestionID = qr.QuestionID "
"INNER JOIN AnswerInfo a on qr.AnswerID = a.AnswerID ")
rows = conn.execute(SELECT + FROM + JOINS + "WHERE ii.InterviewID = ?",
(InterviewID, )).fetchall()
try:
InterviewName = rows[0]['InterviewName']
except IndexError:
res = active_interview.ActiveInterview(-1, "No interview available",
[])
return res
Questions = {}
for row in rows:
if row['QuestionID'] in Questions:
Questions[row['QuestionID']].putAnswer(
answer.Answer(row['AnswerID'], row['AnswerText']))
else:
Questions[row['QuestionID']] = question.Question(
row['QuestionID'], row['QuestionText'])
Questions[row['QuestionID']].putAnswer(
answer.Answer(row['AnswerID'], row['AnswerText']))
res = active_interview.ActiveInterview(InterviewID, InterviewName,
list(Questions.values()))
conn.close()
return res
def save_top_answers(self, num):
# save the top i answers of the question
# if i is greater than the total number of answers, will save all answers
if self.url is None:
raise ValueError("Did not found url for the question")
else:
if self.soup is None:
self.parser()
top_answers_url_raw = self.soup.find_all("span", class_="answer-date-link-wrap")
title = self.get_title()[:-1]
try:
os.mkdir(title.replace("/", "") + "top {0} answers(question)".format(num))
os.chdir(title.replace("/", "") + "top {0} answers(question)".format(num))
except OSError:
os.chdir(title.replace("/", "") + "top {0} answers(question)".format(num))
if len(top_answers_url_raw) <= num:
for answer_url in top_answers_url_raw:
answer.Answer(Zhihu + answer_url.a["href"]).save_answer_to_file()
else:
for j in xrange(num):
answer.Answer(Zhihu + top_answers_url_raw[j].a["href"]).save_answer_to_file()
os.chdir("..")
return
def evaluate(param, file_name):
"""
Evaluate 4 parameter from question_test.txt and write result to file csv
"""
print("======= Start Evaluate =======")
print(param[1])
answer_obj = answer.Answer(param[0], param[1], param[2])
with open(file_name, "w") as csvfile:
fieldnames = ['question', 'answer']
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for index, sentence in enumerate(test_set):
data = answer_obj.answer(sentence)
writer.writerow({'question': sentence, 'answer': data})
print str(index) + ": " + sentence + "-----" + data
print("======= End Of Evaluate =======")
def upload_file():
if request.method == 'POST':
file = request.files['file']#获得文件
if file and allowed_file(file.filename):
#filename = secure_filename(file.filename)
filename = hashlib.md5(('admin' + str(time.time())).encode("utf-8")).hexdigest()[:15] + ".jpg"
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
file_url = url_for('uploaded_file',filename=filename)#获取图片地址
local_urlbase = "G:\makePoem" + file_url
local_url = open(local_urlbase,'rb')
res = answer.Answer(local_url)
ress = res.split('_')
poem = str(ress[0])
n = len(poem)
poem_writer = res[n:]
return render_template('show.html', pic_url=file_url, poem=poem, poem_writer=poem_writer)
return render_template('index.html')
def save_all_answers(self):
# save all answers of the question
if self.url is None:
raise ValueError("Did not found url for the question")
else:
if self.soup is None:
self.parser()
top_answers_url_raw = self.soup.find_all("span", class_="answer-date-link-wrap")
title = self.get_title()[:-1]
try:
os.mkdir(title.replace("/", "") + "all answers(question)")
os.chdir(title.replace("/", "") + "all answers(question)")
except OSError:
os.chdir(title.replace("/", "") + "all answers(question)")
for answer_url in top_answers_url_raw:
answer.Answer(Zhihu + answer_url.a["href"]).save_answer_to_file()
os.chdir("..")
return
def __init__(self, vocab_processor, max_length, w2v_W, data): self.epoch = FLAGS.epoch self.batch_size = FLAGS.batch_size self.lr = FLAGS.lr self.RNN_dim = FLAGS.RNN_dim self.embedding_size = FLAGS.embedding_size self.vocab = vocab_processor._reverse_mapping self.vocab_size = len(self.vocab) self.max_length = max_length self.num_sampled = FLAGS.num_sampled self.layer = FLAGS.layer self.w2v_W = w2v_W self.data = data config = tf.ConfigProto(allow_soft_placement = True) config.gpu_options.allow_growth = True self.sess = tf.Session(config = config) self.test = data_utils.load_n_process_test(FLAGS.test_data, vocab_processor) self.gen_path() self.answer = answer.Answer()
def save_questions_and_answers(self):
# save all the answers of the collection
if self.url is None:
raise ValueError("Did not found url for the collection")
else:
new_session = login.log_in()
for i in xrange(100):
collection_url = self.url + "?page={0}".format(i + 1)
r = new_session.get(collection_url)
soup = BeautifulSoup(r.content, "lxml")
items = soup.find_all("div", class_="zm-item")
author = self.get_author()
if items is None:
break
title = self.get_title()
try:
os.mkdir(
user.User(author).get_id().replace("/", "") + "-" +
title + "(collection)")
os.chdir(
user.User(author).get_id().replace("/", "") + "-" +
title + "(collection)")
except OSError:
os.chdir(
user.User(author).get_id().replace("/", "") + "-" +
title + "(collection)")
for item in items:
try:
answer_url = Zhihu + item.find(
"a",
class_="answer-date-link last_updated meta-item"
)["href"]
except TypeError:
answer_url = Zhihu + item.find(
"a", class_="answer-date-link meta-item")["href"]
answer.Answer(answer_url).save_answer_to_file()
os.chdir("..")
return
def get_answers(self):
if self.user_url == None:
print "I'm anonymous user."
return
yield
else:
answers_num = self.get_answers_num()
if answers_num == 0:
return
yield
else:
for i in xrange((answers_num - 1) / 20 + 1):
answer_url = self.user_url + "/answers?page=" + str(i + 1)
r = requests.get(answer_url)
soup = BeautifulSoup(r.content, "lxml")
for answer in soup.find_all("a", class_="question_link"):
question_url = "http://www.zhihu.com" + answer["href"][
0:18]
question_title = answer.string.encode("utf-8")
question = question.Question(question_url,
question_title)
yield answer.Answer(
"http://www.zhihu.com" + answer["href"], question,
self)
def save_top_answers(self):
# save the top answers of the topic
if self.url is None:
raise ValueError("Did not found url for the topic")
else:
new_session = login.log_in()
title = self.get_title()
try:
os.mkdir(title.replace("/", "") + " top answers(topic)")
os.chdir(title.replace("/", "") + " top answers(topic)")
except OSError:
os.chdir(title.replace("/", "") + " top answers(topic)")
for i in xrange(50):
top_answer_url = self.url + "/top-answers?page={0}".format(i + 1)
r = new_session.get(top_answer_url)
if r.status_code != 200:
break
soup = BeautifulSoup(r.content, "lxml")
answer_links_raw = soup.find_all("span", class_="answer-date-link-wrap")
for answer_link_raw in answer_links_raw:
answer_link_raw = Zhihu + answer_link_raw.a["href"]
answer.Answer(answer_link_raw).save_answer_to_file()
os.chdir("..")
return
def __init__(self, QuestionID, QuestionText, Answers=None):
self.qid = QuestionID
self.question = QuestionText
self.answers = [] if Answers == None else Answers
self.answered = answer.Answer(-1, 'default')
import falcon
import info
import answer
import mytime
from wsgiref import simple_server
api = falcon.API()
api.add_route('/info', info.Info())
api.add_route('/answer', answer.Answer())
api.add_route('/time', mytime.Mytime())
# Useful for debugging problems in your API; works with pdb.set_trace()
if __name__ == '__main__':
httpd = simple_server.make_server('127.0.0.1', 8000, api)
httpd.serve_forever()
def add_answer_key(self, answer_text):
answer_obj = answer.Answer(answer_text)
self._answer_key.append(answer_obj)
self.answers.setdefault(answer_text, answer_obj)
import os.path, sys
sys.path.append(
os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
from flask import Flask, render_template, request, json
import answer
app = Flask(__name__)
answer_obj = answer.Answer()
@app.route('/')
def index():
return render_template('ui.html')
@app.route('/get_reply')
def get_reply():
message = request.args.get('message')
if message:
data = answer_obj.answer(message)
response = app.response_class(response=json.dumps(data),
status=200,
mimetype='application/json')
else:
data = 'Invalid message'
response = app.response_class(response=json.dumps(data),
status=404,
mimetype='application/json')
return response
def get_all_answers(self):
answers_num = self.get_answers_num()
if answers_num == 0:
print "No answer."
return
yield
else:
error_answer_count = 0
my_answer_count = 0
for i in xrange((answers_num - 1) / 20 + 1):
if i == 0:
for j in xrange(min(answers_num, 20)):
if self.soup == None:
self.parser()
soup = BeautifulSoup(self.soup.encode("utf-8"), "lxml")
is_my_answer = False
if soup.find_all("div", class_="zm-item-answer")[j].find("span", class_="count") == None:
my_answer_count += 1
is_my_answer = True
if soup.find_all("div", class_="zm-item-answer")[j].find("div", class_="zm-editable-content clearfix") == None:
error_answer_count += 1
continue
author = None
if soup.find_all("div", class_="zm-item-answer-author-info")[j].get_text(strip='\n') == u"匿名用户":
author_url = None
author = user.User(author_url)
else:
author_tag = soup.find_all("div", class_="zm-item-answer-author-info")[j].find_all("a")[1]
author_id = author_tag.string.encode("utf-8")
author_url = "http://www.zhihu.com" + author_tag["href"]
author = user.User(author_url, author_id)
if is_my_answer == True:
count = soup.find_all("div", class_="zm-item-answer")[j].find("a", class_="zm-item-vote-count").string
else:
count = soup.find_all("span", class_="count")[j - my_answer_count].string
if count[-1] == "K":
upvote = int(count[0:(len(count) - 1)]) * 1000
elif count[-1] == "W":
upvote = int(count[0:(len(count) - 1)]) * 10000
else:
upvote = int(count)
answer_url = "http://www.zhihu.com" + soup.find_all("a", class_="answer-date-link")[j]["href"]
answer = soup.find_all("div", class_="zm-editable-content clearfix")[j - error_answer_count]
soup.body.extract()
soup.head.insert_after(soup.new_tag("body", **{'class': 'zhi'}))
soup.body.append(answer)
img_list = soup.find_all("img", class_="content_image lazy")
for img in img_list:
img["src"] = img["data-actualsrc"]
img_list = soup.find_all("img", class_="origin_image zh-lightbox-thumb lazy")
for img in img_list:
img["src"] = img["data-actualsrc"]
noscript_list = soup.find_all("noscript")
for noscript in noscript_list:
noscript.extract()
content = soup
answer = answer.Answer(answer_url, self, author, upvote, content)
yield answer
else:
post_url = "http://www.zhihu.com/node/QuestionAnswerListV2"
_xsrf = self.soup.find("input", attrs={'name': '_xsrf'})["value"]
offset = i * 20
params = json.dumps(
{"url_token": int(self.url[-8:-1] + self.url[-1]), "pagesize": 20, "offset": offset})
data = {
'_xsrf': _xsrf,
'method': "next",
'params': params
}
header = {
'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0",
'Host': "www.zhihu.com",
'Referer': self.url
}
r = requests.post(post_url, data=data, headers=header)
answer_list = r.json()["msg"]
for j in xrange(min(answers_num - i * 20, 20)):
soup = BeautifulSoup(self.soup.encode("utf-8"), "lxml")
answer_soup = BeautifulSoup(answer_list[j], "lxml")
if answer_soup.find("div", class_="zm-editable-content clearfix") == None:
continue
author = None
if answer_soup.find("div", class_="zm-item-answer-author-info").get_text(strip='\n') == u"匿名用户":
author_url = None
author = user.User(author_url)
else:
author_tag = answer_soup.find("div", class_="zm-item-answer-author-info").find_all("a")[1]
author_id = author_tag.string.encode("utf-8")
author_url = "http://www.zhihu.com" + author_tag["href"]
author = user.User(author_url, author_id)
if answer_soup.find("span", class_="count") == None:
count = answer_soup.find("a", class_="zm-item-vote-count").string
else:
count = answer_soup.find("span", class_="count").string
if count[-1] == "K":
upvote = int(count[0:(len(count) - 1)]) * 1000
elif count[-1] == "W":
upvote = int(count[0:(len(count) - 1)]) * 10000
else:
upvote = int(count)
answer_url = "http://www.zhihu.com" + answer_soup.find("a", class_="answer-date-link")["href"]
answer = answer_soup.find("div", class_="zm-editable-content clearfix")
soup.body.extract()
soup.head.insert_after(soup.new_tag("body", **{'class': 'zhi'}))
soup.body.append(answer)
img_list = soup.find_all("img", class_="content_image lazy")
for img in img_list:
img["src"] = img["data-actualsrc"]
img_list = soup.find_all("img", class_="origin_image zh-lightbox-thumb lazy")
for img in img_list:
img["src"] = img["data-actualsrc"]
noscript_list = soup.find_all("noscript")
for noscript in noscript_list:
noscript.extract()
content = soup
answer = answer.Answer(answer_url, self, author, upvote, content)
yield answer
import answer
import trio
async def foo(request):
return answer.Response(b'YAAY!\n')
app = answer.Answer()
app.router.add_route('/foo', foo)
trio.run(app.run)
rule.show_rules()
def open_cubes(self):
cube.show()
def open_history(self):
his.show()
his.show_history()
if __name__ == "__main__":
import sys
# create app
app = QtWidgets.QApplication(sys.argv)
# main window
showMain = Main()
showMain.show()
# answer
ans = answer.Answer()
# letters
letter = letters.Letters()
# rules
rule = rules.Rules()
# cubes
cube = cubes.Cubes()
# history
his = history.History()
# run main loop
sys.exit(app.exec_())
def add_answer(self, answer_text):
answer_obj = answer.Answer(answer_text)
self._answers.setdefault(answer_text, answer_obj)
return answer_obj
