def save_source_status(status):
sql_user = get_sql_user(status)
if sql_user == None:
return None
sql_status = session.query(SingleSourceStatus).filter(
SingleSourceStatus.origin_id == str(status['id'])).first()
if not sql_status:
sql_status = SingleSourceStatus()
sql_status.origin_id = str(status['id'])
sql_status.url = "http://weibo.com/" + sql_user.origin_id + "/" + id2mid(
status['idstr'])
sql_status.created_at = weibo_date_str_to_datetime(status['created_at'])
sql_status.text = status['text']
sql_status.reposts_count = status['reposts_count']
sql_status.comments_count = status['comments_count']
sql_status.attitudes_count = status['attitudes_count']
sql_status.source_app = status['source']
if status.has_key('original_pic'):
sql_status.pic = status['original_pic']
sql_status.single_weibo_user = sql_user
return sql_status
def save_repost(status, sql_source_status, repost_depth, direct_reposts_count,
parent_origin_id):
sql_user = get_sql_user(status)
if sql_user == None:
return
sql_status = session.query(SingleRepostStatus).filter(
SingleRepostStatus.origin_id == str(status['id'])).first()
if not sql_status:
sql_status = SingleRepostStatus()
sql_status.origin_id = str(status['id'])
sql_status.url = "http://weibo.com/" + sql_user.origin_id + "/" + id2mid(
status['idstr'])
sql_status.created_at = weibo_date_str_to_datetime(status['created_at'])
sql_status.text = status['text']
sql_status.reposts_count = status['reposts_count']
sql_status.comments_count = status['comments_count']
sql_status.attitudes_count = status['attitudes_count']
sql_status.source_app = status['source']
if status.has_key('original_pic'):
sql_status.pic = status['original_pic']
sql_status.repost_depth = repost_depth
sql_status.direct_reposts_count = direct_reposts_count
sql_status.parent_origin_id = parent_origin_id
sql_status.single_weibo_user = sql_user
sql_status.single_source_status = sql_source_status
session.merge(sql_status)
def save_comment(comment, sql_source_status):
sql_user = get_sql_user(comment)
if sql_user == None:
return
sql_comment = session.query(SingleComment).filter(
SingleComment.origin_id == str(comment['id'])).first()
if not sql_comment:
sql_comment = SingleComment()
sql_comment.origin_id = str(comment['id'])
sql_comment.created_at = weibo_date_str_to_datetime(comment['created_at'])
sql_comment.text = comment['text']
sql_comment.source_app = comment['source']
sql_comment.single_weibo_user = sql_user
sql_comment.single_source_status = sql_source_status
session.merge(sql_comment)
def get_sql_user(status):
user = status['user']
if user is None or status['text'] is None: #Exception
return None
sql_user = session.query(SingleWeiboUser).filter(
SingleWeiboUser.origin_id == str(user['id'])).first()
if not sql_user:
sql_user = SingleWeiboUser()
sql_user.origin_id = str(user['id'])
sql_user.screen_name = user['screen_name']
sql_user.bi_followers_count = user['bi_followers_count']
sql_user.followers_count = user['followers_count']
sql_user.friends_count = user['friends_count']
sql_user.statuses_count = user['statuses_count']
sql_user.description = user['description']
sql_user.profile_image_url = user['profile_image_url']
sql_user.profile_url = user['profile_url']
sql_user.verified_type = get_verified_type(user['verified_type'])
sql_user.created_at = weibo_date_str_to_datetime(user['created_at'])
sql_user.gender = user['gender']
location = locationId2Str(user['province'], user['city'])
sql_user.province = location['province']
sql_user.city = location['city']
# 先提交 防重复
session.merge(sql_user)
session.flush()
session.commit()
sql_user = session.query(SingleWeiboUser).filter(
SingleWeiboUser.origin_id == str(user['id'])).first()
if not sql_user:
return None
return sql_user
def add_status_and_user_to_session(status, keyword_id):
user = status['user']
if user is None or status['text'] is None: #Exception
return
sql_user = session.query(User).filter(
User.user_origin_id == str(user['id']),
User.info_source_id == NETEASE_WEIBO_INFO_SOURCE_ID).first()
if not sql_user:
sql_user = User()
sql_user.user_origin_id = str(user['id'])
sql_user.info_source_id = NETEASE_WEIBO_INFO_SOURCE_ID
sql_user.screen_name = user['screen_name']
sql_user.profile_image_url = user['profile_image_url']
sql_user.status_count = user['statuses_count']
sql_user.follower_count = user['followers_count']
sql_user.following_count = user['friends_count']
sql_user.verified = user['verified']
if user['gender'] == '1':
sql_user.gender = 'm'
elif user['gender'] == '2':
sql_user.gender = 'f'
else:
sql_user.gender = 'n'
location = location_split(user['location'])
sql_user.geo_info_province = location['province']
sql_user.geo_info_city = location['city']
sql_status = session.query(Status).filter(
Status.weibo_origin_id == status['id'],
Status.info_source_id == NETEASE_WEIBO_INFO_SOURCE_ID).first()
if not sql_status:
sql_status = Status()
sql_status.weibo_origin_id = int(status['id'])
sql_status.url = "http://t.163.com/" + user['id'] + "/status/" + status[
'id']
sql_status.weibo_user_screen_name = user['screen_name']
sql_status.keyword_id = keyword_id
sql_status.info_source_id = NETEASE_WEIBO_INFO_SOURCE_ID
sql_status.text = status['text']
sql_status.created_at = weibo_date_str_to_datetime(status['created_at'])
sql_status.repost_count = status['retweet_count']
sql_status.comment_count = status['comments_count']
sql_status.attitude_count = 0
if status['in_reply_to_status_id'] != None:
sql_status.retweeted = True
else:
sql_status.retweeted = False
sql_status.with_pic = False
sql_status.geo_info_province = location['province']
sql_status.geo_info_city = location['city']
sql_status.user = sql_user #foreign key
session.merge(sql_status) #merge
session.flush()
session.commit()
sql_status = session.query(Status).filter(
Status.weibo_origin_id == status['id'],
Status.info_source_id == NETEASE_WEIBO_INFO_SOURCE_ID).first()
if sql_status:
store_category('weibo', str(sql_status.id))
def add_status_and_user_to_session(status, count, keyword_id):
user = status['user']
if user is None or status['text'] is None or status[
'text'] == "@#@_@#@": #Exception
return
sql_user = session.query(User).filter(
User.user_origin_id == str(user['id']),
User.info_source_id == SOHU_WEIBO_INFO_SOURCE_ID).first()
if not sql_user:
sql_user = User()
sql_user.user_origin_id = user['id']
sql_user.info_source_id = SOHU_WEIBO_INFO_SOURCE_ID
sql_user.screen_name = user['screen_name']
sql_user.profile_image_url = user['profile_image_url']
sql_user.status_count = user['statuses_count']
sql_user.follower_count = user['followers_count']
sql_user.following_count = user['friends_count']
sql_user.verified = user['verified']
sql_user.gender = 'n'
sql_user.geo_info_province = u'其它'
sql_user.geo_info_city = u''
sql_status = session.query(Status).filter(
Status.weibo_origin_id == status['id'],
Status.info_source_id == SOHU_WEIBO_INFO_SOURCE_ID).first()
if not sql_status:
sql_status = Status()
sql_status.weibo_origin_id = int(status['id'])
sql_status.url = "http://t.sohu.com/m/" + status['id']
sql_status.weibo_user_screen_name = user['screen_name']
sql_status.keyword_id = keyword_id
sql_status.info_source_id = SOHU_WEIBO_INFO_SOURCE_ID
sql_status.text = status['text']
sql_status.created_at = weibo_date_str_to_datetime(status['created_at'])
sql_status.repost_count = count['transmit_count']
sql_status.comment_count = count['comments_count']
sql_status.attitude_count = 0
if status.has_key('retweeted_status'):
sql_status.retweeted = True
else:
sql_status.retweeted = False
if status['middle_pic'] != '':
sql_status.with_pic = True
sql_status.pic_address = status['middle_pic']
else:
sql_status.with_pic = False
sql_status.geo_info_province = u'其它'
sql_status.geo_info_city = u''
sql_status.user = sql_user #foreign key
session.merge(sql_status) #merge
session.flush()
session.commit()
sql_status = session.query(Status).filter(
Status.weibo_origin_id == status['id'],
Status.info_source_id == SOHU_WEIBO_INFO_SOURCE_ID).first()
if sql_status:
store_category('weibo', str(sql_status.id))
def add_status_and_user_to_session(status, keyword_id):
user = status['user']
if user is None or status['text'] is None: #Exception
return
sql_user = session.query(User).filter(User.user_origin_id==str(user['id']), User.info_source_id==SEARCH_INFO_SOURCE_ID).first()
if not sql_user:
sql_user = User()
sql_user.user_origin_id = str(user['id'])
sql_user.info_source_id = SEARCH_INFO_SOURCE_ID
sql_user.screen_name = user['screen_name']
sql_user.profile_image_url = user['profile_image_url']
sql_user.status_count = user['statuses_count']
sql_user.follower_count = user['followers_count']
sql_user.following_count = user['friends_count']
sql_user.verified = user['verified']
sql_user.gender = user['gender']
location = locationId2Str(user['province'], user['city'])
sql_user.geo_info_province = location['province']
sql_user.geo_info_city = location['city']
sql_status = session.query(Status).filter(Status.weibo_origin_id==status['id'], Status.info_source_id==SEARCH_INFO_SOURCE_ID).first()
if not sql_status:
sql_status = Status()
sql_status.weibo_origin_id = status['id']
sql_status.url = "http://weibo.com/" + str(user['id']) + "/" + id2mid(status['idstr'])
sql_status.weibo_user_screen_name = user['screen_name']
sql_status.keyword_id = keyword_id
sql_status.info_source_id = SEARCH_INFO_SOURCE_ID
sql_status.text = status['text']
sql_status.created_at = weibo_date_str_to_datetime(status['created_at'])
sql_status.repost_count = status['reposts_count']
sql_status.comment_count = status['comments_count']
sql_status.attitude_count = status['attitudes_count']
if status.has_key('retweeted_status'):
sql_status.retweeted = True
else:
sql_status.retweeted = False
if status.has_key('thumbnail_pic'):
sql_status.with_pic = True
sql_status.pic_address = status['thumbnail_pic']
else:
sql_status.with_pic = False
sql_status.geo_info_province = location['province']
sql_status.geo_info_city = location['city']
sql_status.user = sql_user #foreign key
session.merge(sql_status) #merge
session.flush()
session.commit()
sql_status = session.query(Status).filter(Status.weibo_origin_id==status['id'],
Status.info_source_id==SEARCH_INFO_SOURCE_ID).first()
if sql_status:
store_category('weibo', str(sql_status.id))