def add_new_answer_crawler(question_zhihuid, answer_zhihuid, answer_id):
"""
:param question_zhihuid:
:param answer_zhihuid:
:param answer_id:
:return:
"""
_, answer_data = database_requester.get_model_by_id(Answer, answer_id)
vote_num, comment_num, rank, title = zh_spider.get_answer_data(
answer_zhihuid, question_zhihuid)
answer_data.title = title
answer_data.current_vote_nums = vote_num
answer_data.current_rank = rank
answer_data.current_comment_nums = comment_num
database_requester.commit()
new_vote_num = VoteNum(value=vote_num,
answer_id=answer_data.id,
record_time=get_time_stamp())
new_comment_num = CommentNum(value=comment_num,
answer_id=answer_data.id,
record_time=get_time_stamp())
new_rank = Rank(value=rank,
answer_id=answer_data.id,
record_time=get_time_stamp())
database_requester.add(new_vote_num)
database_requester.add(new_comment_num)
database_requester.add(new_rank)
#
database_requester.commit()
def update_data():
questions = database_requester.get_model_all(Question)
print("更新问题")
for question in questions:
question_zhihuid = question.question_zhihuid
follower_num, view_num, title = zh_spider.get_follower_view_title(
question_zhihuid=question_zhihuid)
# question.view_increment = view_num - question.current_follower_nums
question.increase_percentage = (
view_num - question.current_view_nums) / question.current_view_nums
question.view_increment = view_num - question.current_view_nums
question.current_follower_nums = follower_num
question.current_view_nums = view_num
new_view_num = ViewNum(value=view_num,
question_id=question.id,
record_time=get_time_stamp())
new_follower_num = FollowerNum(value=follower_num,
question_id=question.id,
record_time=get_time_stamp())
database_requester.add(new_view_num)
database_requester.add(new_follower_num)
database_requester.commit()
print('问题更新完成')
answers = database_requester.get_model_all(Answer)
print("更新回答")
for answer in answers:
answer_zhihuid = answer.answer_zhihuid
question_zhihuid = answer.question_zhihuid
vote_num, comment_num, rank, title = zh_spider.get_answer_data(
answer_zhihuid, question_zhihuid)
new_vote_num = VoteNum(value=vote_num,
answer_id=answer.id,
record_time=get_time_stamp())
new_comment_num = CommentNum(value=comment_num,
answer_id=answer.id,
record_time=get_time_stamp())
new_rank = Rank(value=rank,
answer_id=answer.id,
record_time=get_time_stamp())
database_requester.add(new_vote_num)
database_requester.add(new_comment_num)
database_requester.add(new_rank)
database_requester.commit()
answer.current_vote_nums = vote_num
answer.current_rank = rank
answer.current_comment_nums = comment_num
answer.title = title
database_requester.commit()
print("回答更新完成")
class CollectNums(db.Model):
__tablename__ = "collectnums"
id = db.Column(db.Integer, primary_key=True)
record_time = db.Column(db.BigInteger, default=get_time_stamp())
value = db.Column(db.BigInteger)
answer_id = db.Column(db.Integer, db.ForeignKey("answer.id"))
answer = db.relationship("Answer", back_populates="collect_nums")
def __init__(self):
super(Answer, self).__init__()
self.parser.add_argument('answerZhiHuId', type=int)
self.parser.add_argument('questionZhiHuId', type=int)
self.parser.add_argument('answerUrl', type=str)
self.parser.add_argument('startTime', type=int, default=0) # 开始时间
self.parser.add_argument('endTime', type=int, default=get_time_stamp())
self.fields = base_settings.answer_fields
def __init__(self):
super(Question, self).__init__()
# 请求添加问题的id
self.parser.add_argument('questionZhiHuId', type=int)
self.parser.add_argument("questionUrl", type=str)
self.parser.add_argument('startTime', type=int, default=0) # 开始时间
self.parser.add_argument('endTime', type=int,
default=get_time_stamp()) # 截至时间
self.fields = base_settings.question_fields
class Rank(db.Model):
id = db.Column(db.Integer, primary_key=True)
record_time = db.Column(db.BigInteger, default=get_time_stamp())
value = db.Column(db.BigInteger)
answer_id = db.Column(db.Integer, db.ForeignKey("answer.id"))
answer = db.relationship("Answer", back_populates="ranks")
@property
def time_stamp(self):
return int(self.record_time.timestamp())
class VoteNum(db.Model):
__tablename__ = "votenum"
id = db.Column(db.Integer, primary_key=True)
record_time = db.Column(db.BigInteger, default=get_time_stamp())
value = db.Column(db.BigInteger)
answer_id = db.Column(db.Integer, db.ForeignKey('answer.id'))
answer = db.relationship("Answer", back_populates="vote_nums")
@property
def time_stamp(self):
return int(self.record_time.timestamp())
class FollowerNum(db.Model):
# 多
__tablename__ = "followernum"
id = db.Column(db.Integer, primary_key=True)
record_time = db.Column(db.BigInteger, default=get_time_stamp())
value = db.Column(db.BigInteger)
question_id = db.Column(db.Integer, db.ForeignKey('question.id'))
question = db.relationship("Question", back_populates="follower_nums")
@property
def time_stamp(self):
return int(self.record_time.timestamp())
def put(self):
response_data = deepcopy(self.base_response_data)
question_zhihuid = re.findall(
r"question/(\d*)",
self.parser.parse_args().get('questionUrl'))[0]
question_zhihuid = int(question_zhihuid)
# question_zhihuid = self.parser.parse_args().get("questionZhiHuId")
# 获取当前时间该的问题关注数和浏览数和标题
follower_nums, view_nums, title = self.crawler.get_follower_view_title(
question_zhihuid)
# 根据问题id提交到数据库
new_question = self.build_new_question(question_zhihuid, title,
follower_nums, view_nums)
# 提交到数据库
try:
self.requester.add(new_question)
self.requester.commit()
new_follower_num = FollowerNum(question_id=new_question.id,
value=follower_nums,
record_time=get_time_stamp())
new_view_num = ViewNum(question_id=new_question.id,
value=view_nums,
record_time=get_time_stamp())
self.requester.add(new_follower_num)
self.requester.add(new_view_num)
# 提交
self.requester.commit()
return response_data
except:
response_data['res'] = 0
response_data['message'] = "error"
return response_data