def setUp(self):
"""Define test variables and initialize app."""
self.app = create_app()
self.client = self.app.test_client
self.database_path = database_path
setup_db(self.app, self.database_path)
# binds the app to the current context
with self.app.app_context():
self.db = SQLAlchemy()
self.db.init_app(self.app)
self.app.register_blueprint(hackathon_api)
# create all tables
engine = create_engine(self.app.config['SQLALCHEMY_DATABASE_URI'])
if not database_exists(engine.url):
create_database(engine.url)
# test status
status_pending = Status(name="Pending",
description="Submitted application")
status_approved = Status(name="Approved",
description="Hackathon approved")
status_pending.insert()
status_approved.insert()
self.status_pending_id = status_pending.id
self.status_approved_id = status_approved.id
self.lead_headers = {"Authorization": f"Bearer {get_lead_token()}"}
self.member_headers = {
"Authorization": f"Bearer {get_member_token()}"
}
def load_status_page(page, uid=crawler.uid):
r = crawler.get_json(config.STATUS_URL, {'userId': uid, 'curpage': page})
likes = r['likeInfoMap']
for s in r['doingArray']:
id = int(s['id'])
# TODO: location，locationUrl
status = {
'id': id,
'uid': uid,
't': datetime.fromtimestamp(int(s['createTime']) / 1000),
'content': s['content'], # 内容
'like': likes.get('status_{id}'.format(id=id), 0), # 点赞
'repeat': s['repeatCountTotal'], # 转发
'comment': s['comment_count'], # 评论
'rootContent': s.get('rootContent', ''), # 如果是转发，转发的原文
'rootUid': s.get('rootDoingUserId', 0), # 转发原 uid
'rootUname': s.get('rootDoingUserName', ''), # 转发原 username
}
Status.insert(**status).on_conflict('replace').execute()
if status['comment']:
get_comments(id, 'status', owner=uid)
if status['like']:
get_likes(id, 'status', owner=uid)
print(' on page {page}, {parsed} parsed'.format(page=page,
parsed=len(
r['doingArray'])))
return r['count']
def load_status_page(uid=crawler.uid, after=None):
r = crawler.get_json(config.STATUS_URL,
json_=get_common_payload(uid, after),
method="POST")
if "count" not in r:
return 0, None
for s in r["data"]:
sid = int(s["id"])
body = s["body"]
if "content" not in body:
continue
head_image = body.get("head_image", "")
if is_rr_pic(head_image):
head_image = get_image(head_image)
sfrom = s.get("from", {})
sbody = sfrom.get("body", {})
rootContent = sbody.get("content", "")
rootPic = sbody.get("head_image", "")
if is_rr_pic(rootPic):
rootPic = get_image(rootPic)
rootPublisher = sfrom.get("publisher", {})
rootUid = rootPublisher.get("id", 0)
rootUname = rootPublisher.get("nickname", "")
status = {
"id": sid,
"uid": uid,
"t": datetime.fromtimestamp(int(s["publish_time"]) / 1000),
"content": s["body"]["content"], # 内容
"headPic": head_image, # 附件图片
"like": s["like_count"], # 点赞
"repeat": 0, # s['repeatCountTotal'], # 转发
"comment": s["comment_count"], # 评论
"rootContent": rootContent, # 如果是转发，转发的原文
"rootPic": rootPic, # 如果是转发，转发的原文的附件图片
"rootUid": rootUid, # 转发原 uid
"rootUname": rootUname, # 转发原 username
"location": s.get("lbs", {}).get("position", ""), # 带地理位置的地名
"locationUrl":
"", # s.get('locationUrl', ''), # 地理位置的人人地点
}
Status.insert(**status).on_conflict("replace").execute()
logger.info("{parsed} parsed".format(parsed=len(r["data"])))
return r["count"], r["tail_id"]
