def get_weibo_info_detail(each, html):
wb_data = WeiboData()
user_cont = each.find(attrs={'class': 'face'})
user_info = str(user_cont.find('a'))
user_pattern = 'id=(\\d+)&'
m = re.search(user_pattern, user_info)
if m:
wb_data.uid = m.group(1)
else:
parser.warning('未提取到用户id,页面源码是{}'.format(html))
return None
weibo_pattern = 'mid=(\\d+)'
m = re.search(weibo_pattern, str(each))
if m:
wb_data.weibo_id = m.group(1)
else:
parser.warning('未提取到页面的微博id,页面源码是{}'.format(html))
return None
time_url = each.find(attrs={'node-type': 'feed_list_item_date'})
wb_data.create_time = time_url.get('title', '')
wb_data.weibo_url = time_url.get('href', '')
if 'weibo.com' not in wb_data.weibo_url:
wb_data.weibo_url = 'http://weibo.com{}'.format(wb_data.weibo_url)
wb_data.weibo_cont = each.find(attrs={'node-type': 'feed_content'}).find\
(attrs={'node-type': 'feed_list_content'}).text.strip()
if '展开全文' in str(each):
is_all_cont = 0
else:
is_all_cont = 1
try:
wb_data.device = each.find(attrs={'class': 'WB_from'}).find(attrs={'action-type': 'app_source'}).text
except Exception as e:
parser.error('本次解析设备出错，具体是{}'.format(e))
wb_data.device = ''
try:
wb_data.repost_num = int(each.find(attrs={'action-type': 'fl_forward'}).find_all('em')[1].text)
except Exception:
wb_data.repost_num = 0
try:
wb_data.comment_num = int(each.find(attrs={'action-type': 'fl_comment'}).find_all('em')[1].text)
except Exception:
wb_data.comment_num = 0
try:
wb_data.praise_num = int(each.find(attrs={'action-type': 'fl_like'}).find_all('em')[1].text)
except Exception:
wb_data.praise_num = 0
return wb_data, is_all_cont
def get_weibo_info_detail(each, html):
wb_data = WeiboData()
user_cont = each.find(attrs={'class': 'face'})
user_info = str(user_cont.find('a'))
user_pattern = 'id=(\\d+)&'
m = re.search(user_pattern, user_info)
if m:
wb_data.uid = m.group(1)
else:
parser.warning('未提取到用户id,页面源码是{}'.format(html))
return None
weibo_pattern = 'mid=(\\d+)'
m = re.search(weibo_pattern, str(each))
if m:
wb_data.weibo_id = m.group(1)
else:
parser.warning('未提取到页面的微博id,页面源码是{}'.format(html))
return None
time_url = each.find(attrs={'node-type': 'feed_list_item_date'})
wb_data.create_time = time_url.get('title', '')
wb_data.weibo_url = time_url.get('href', '')
if 'weibo.com' not in wb_data.weibo_url:
wb_data.weibo_url = 'http://weibo.com{}'.format(wb_data.weibo_url)
wb_data.weibo_cont = each.find(attrs={'node-type': 'feed_content'}).find\
(attrs={'node-type': 'feed_list_content'}).text.strip()
# test for weibo_pic capture
# 先判断这条微博是否有带图片，再进行后续的处理
try:
weibo_pic = []
have_pic = 1
pic_list = each.find_all(attrs={'action-type': 'fl_pics'})
except Exception as e:
have_pic = 0
if have_pic == 1:
for pic in pic_list:
wb_pic = WeiboPic()
wb_pic.uid = wb_data.uid
wb_pic.weibo_id = wb_data.weibo_id
wb_pic.pic_url = pic.find('img').get('src')
# wb_pic.url_hash = md5Encode(wb_pic.pic_url)
wb_pic.url_hash = re.match('.*/thumb150/(.*).jpg',
wb_pic.pic_url).group(1)
wb_pic.dl_flag = 0
wb_pic.judge_flag = 0
weibo_pic.append(wb_pic)
# end
if '展开全文' in str(each):
is_all_cont = 0
else:
is_all_cont = 1
try:
wb_data.device = each.find(attrs={
'class': 'WB_from'
}).find(attrs={
'action-type': 'app_source'
}).text
except Exception as e:
parser.error('本次解析设备出错，具体是{}'.format(e))
wb_data.device = ''
try:
wb_data.repost_num = int(
each.find(attrs={
'action-type': 'fl_forward'
}).find_all('em')[1].text)
except Exception:
wb_data.repost_num = 0
try:
wb_data.comment_num = int(
each.find(attrs={
'action-type': 'fl_comment'
}).find_all('em')[1].text)
except Exception:
wb_data.comment_num = 0
try:
wb_data.praise_num = int(
each.find(attrs={
'action-type': 'fl_like'
}).find_all('em')[1].text)
except Exception:
wb_data.praise_num = 0
return wb_data, is_all_cont, weibo_pic
def get_weibo_info(each, html):
wb_data = WeiboData()
try:
user_cont = each.find(attrs={'class': 'face'})
user_info = user_cont.find('a')
m = re.match(user_pattern, user_info.img.get('usercard'))
if m:
wb_data.uid = m.group(1)
else:
parser.warning('未提取到用户id,页面源码是{}'.format(html))
return None
except Exception as why:
parser.error('解析用户信息出错，出错原因:{},页面源码是{}'.format(why, html))
return None
wb_data.weibo_id = each.find(attrs={
'class': 'WB_screen'
}).find('a').get('action-data')[4:]
try:
wb_data.weibo_url = each.find(
attrs={'node-type': 'feed_list_item_date'})['href']
except Exception as e:
parser.error('解析微博url出错，出错原因是{},页面源码是{}'.format(e, html))
return None
try:
wb_data.device = each.find(attrs={
'class': 'feed_from'
}).find(attrs={
'rel': 'nofollow'
}).text
except AttributeError:
wb_data.device = ''
try:
create_time = each.find(
attrs={'node-type': 'feed_list_item_date'})['date']
except (AttributeError, KeyError):
wb_data.create_time = ''
else:
create_time = int(create_time) / 1000 # 时间戳单位不同
create_time = datetime.fromtimestamp(create_time)
wb_data.create_time = create_time.strftime("%Y-%m-%d %H:%M")
try:
feed_action = each.find(attrs={'class': 'feed_action'})
except Exception as why:
parser.error('解析feed_action出错,出错原因:{},页面源码是{}'.format(why, each))
else:
feed_infos = feed_action.find_all('li')
try:
wb_data.repost_num = get_feed_info(feed_infos, '转发')
except (AttributeError, ValueError):
wb_data.repost_num = 0
try:
wb_data.comment_num = get_feed_info(feed_infos, '评论')
except (AttributeError, ValueError):
wb_data.comment_num = 0
try:
wb_data.praise_num = int(
feed_action.find(attrs={
'action-type': 'feed_list_like'
}).find('em').text)
except (AttributeError, ValueError):
wb_data.praise_num = 0
try:
wb_data.weibo_cont = each.find(attrs={
'class': 'comment_txt'
}).text.strip()
except Exception as why:
parser.error('解析微博内容出错:{}, 页面源码是{}'.format(why, html))
return None
if '展开全文' in str(each):
is_all_cont = 0
else:
is_all_cont = 1
return wb_data, is_all_cont
def get_weibo_info_detail(each, html):
wb_data = WeiboData()
user_cont = each.find(attrs={'class': 'face'})
user_info = str(user_cont.find('a'))
user_pattern = 'id=(\\d+)&'
m = re.search(user_pattern, user_info)
if m:
wb_data.uid = m.group(1)
else:
parser.warning("fail to get user'sid, the page source is{}".format(html))
return None
weibo_pattern = 'mid=(\\d+)'
m = re.search(weibo_pattern, str(each))
if m:
wb_data.weibo_id = m.group(1)
else:
parser.warning("fail to get weibo's id,the page source {}".format(html))
return None
time_url = each.find(attrs={'node-type': 'feed_list_item_date'})
wb_data.create_time = time_url.get('title', '')
wb_data.weibo_url = time_url.get('href', '')
if ROOT_URL not in wb_data.weibo_url:
wb_data.weibo_url = '{}://{}{}'.format(PROTOCOL, ROOT_URL, wb_data.weibo_url)
def url_filter(url):
return ':'.join([PROTOCOL, url]) if PROTOCOL not in url and ORIGIN not in url else url
try:
imgs = str(each.find(attrs={'node-type': 'feed_content'}).find(attrs={'node-type': 'feed_list_media_prev'}).
find_all('img'))
imgs_url = map(url_filter, re.findall(r"src=\"(.+?)\"", imgs))
wb_data.weibo_img = ';'.join(imgs_url)
except Exception:
wb_data.weibo_img = ''
try:
li = str(each.find(attrs={'node-type': 'feed_content'}).find(attrs={'node-type': 'feed_list_media_prev'}).
find_all('li'))
extracted_url = urllib.parse.unquote(re.findall(r"video_src=(.+?)&", li)[0])
wb_data.weibo_video = url_filter(extracted_url)
except Exception:
wb_data.weibo_video = ''
try:
wb_data.weibo_cont = each.find(attrs={'node-type': 'feed_content'}).find(
attrs={'node-type': 'feed_list_content'}).text.strip()
except Exception:
wb_data.weibo_cont = ''
if '展开全文' in str(each):
is_all_cont = 0
else:
is_all_cont = 1
try:
wb_data.device = each.find(attrs={'class': 'WB_from S_txt2'}).find(attrs={'action-type': 'app_source'}).text
except Exception:
wb_data.device = ''
try:
wb_data.repost_num = int(each.find(attrs={'action-type': 'fl_forward'}).find_all('em')[1].text)
except Exception:
wb_data.repost_num = 0
try:
wb_data.comment_num = int(each.find(attrs={'action-type': 'fl_comment'}).find_all('em')[1].text)
except Exception:
wb_data.comment_num = 0
try:
wb_data.praise_num = int(each.find(attrs={'action-type': 'fl_like'}).find_all('em')[1].text)
except Exception:
wb_data.praise_num = 0
return wb_data, is_all_cont
def get_weibo_info(each, html):
wb_data = WeiboData()
user_cont = each.find(attrs={'class': 'face'})
user_info = user_cont.find('a')
m = re.match(USER_PATTERN, user_info.img.get('usercard'))
if m:
wb_data.uid = m.group(1)
else:
parser.warning("fail to get user'sid, the page source is{}".format(html))
return None
try:
wb_data.weibo_id = each.find(attrs={'class': 'WB_screen'}).find('a').get('action-data')[4:]
except (AttributeError, IndexError, TypeError):
return None
try:
wb_data.weibo_url = each.find(attrs={'node-type': 'feed_list_item_date'})['href']
except Exception as e:
parser.error('fail to get weibo url, the error is {}, the source page is {}'.format(e, html))
return None
def url_filter(url):
return ':'.join([PROTOCOL, url]) if PROTOCOL not in url and ORIGIN not in url else url
try:
imgs = str(each.find(attrs={'node-type': 'feed_list_media_prev'}).find_all('li'))
imgs_url = map(url_filter, re.findall(r"src=\"(.+?)\"", imgs))
wb_data.weibo_img = ';'.join(imgs_url)
except Exception:
wb_data.weibo_img = ''
try:
a_tag = str(each.find(attrs={'node-type': 'feed_list_media_prev'}).find_all('a'))
extracted_url = urllib.parse.unquote(re.findall(r"full_url=(.+?)&", a_tag)[0])
wb_data.weibo_video = url_filter(extracted_url)
except Exception:
wb_data.weibo_video = ''
try:
wb_data.device = each.find(attrs={'class': 'feed_from'}).find(attrs={'rel': 'nofollow'}).text
except AttributeError:
wb_data.device = ''
try:
create_time = each.find(attrs={'node-type': 'feed_list_item_date'})['date']
except (AttributeError, KeyError):
wb_data.create_time = ''
else:
create_time = int(create_time) / 1000 # 时间戳单位不同
create_time = datetime.fromtimestamp(create_time)
wb_data.create_time = create_time.strftime("%Y-%m-%d %H:%M")
try:
feed_action = each.find(attrs={'class': 'feed_action'})
except Exception as why:
parser.error('failt to get feed_action, the error is {},the page source is {}'.format(why, each))
else:
feed_infos = feed_action.find_all('li')
try:
wb_data.repost_num = get_feed_info(feed_infos, '转发')
except (AttributeError, ValueError):
wb_data.repost_num = 0
try:
wb_data.comment_num = get_feed_info(feed_infos, '评论')
except (AttributeError, ValueError):
wb_data.comment_num = 0
try:
wb_data.praise_num = int(feed_action.find(attrs={'action-type': 'feed_list_like'}).find('em').text)
except (AttributeError, ValueError):
wb_data.praise_num = 0
try:
wb_data.weibo_cont = each.find(attrs={'class': 'comment_txt'}).text.strip()
except Exception as why:
parser.error('fail to get weibo cont, the error is {}, the page source is {}'.format(why, html))
return None
if '展开全文' in str(each):
is_all_cont = 0
else:
is_all_cont = 1
return wb_data, is_all_cont
def get_weibo_info(each, html):
wb_data = WeiboData()
try:
try:
user_cont = each.find(attrs={'class': 'face'})
user_info = user_cont.find('a')
m = re.match(user_pattern, user_info.img.get('usercard'))
if m:
wb_data.uid = m.group(1)
else:
parser.warning('未提取到用户id,页面源码是{}'.format(html))
return None
except Exception as why:
parser.error('解析用户信息出错，出错原因:{},页面源码是{}'.format(why, html))
return None
wb_data.weibo_id = each.find(attrs={
'class': 'WB_screen'
}).find('a').get('action-data')[4:]
try:
wb_data.weibo_url = each.find(
attrs={'node-type': 'feed_list_item_date'})['href']
except Exception as e:
parser.error('解析微博url出错，出错原因是{},页面源码是{}'.format(e, html))
return None
try:
feed_action = each.find(attrs={'class': 'feed_action'})
wb_data.create_time = each.find(
attrs={'node-type': 'feed_list_item_date'})['title']
except Exception as why:
parser.error('解析feed_action出错,出错原因:{},页面源码是{}'.format(why, html))
wb_data.device = ''
else:
try:
wb_data.repost_num = int(
feed_action.find(attrs={
'action-type': 'feed_list_forward'
}).find('em').text)
except (AttributeError, ValueError):
wb_data.repost_num = 0
try:
wb_data.comment_num = int(
feed_action.find(attrs={
'action-type': 'feed_list_comment'
}).find('em').text)
except (AttributeError, ValueError):
wb_data.comment_num = 0
try:
wb_data.praise_num = int(
feed_action.find(attrs={
'action-type': 'feed_list_like'
}).find('em').text)
except (AttributeError, ValueError):
wb_data.praise_num = 0
try:
wb_data.weibo_cont = each.find(attrs={
'class': 'comment_txt'
}).text.strip()
except Exception as why:
parser.error('解析微博内容出错:{}, 页面源码是{}'.format(why, html))
return None
except Exception as why:
parser.error('整条解析出错,原因为:{}, 页面源码是{}'.format(why, html))
return None
else:
return wb_data
def get_weibo_info(each, html):
wb_data = WeiboData()
user_cont = each.find(attrs={'class': 'face'})
user_info = user_cont.find('a')
m = re.match(USER_PATTERN, user_info.img.get('usercard'))
if m:
wb_data.uid = m.group(1)
else:
parser.warning("fail to get user'sid, the page source is{}".format(html))
return None
try:
wb_data.weibo_id = each.find(attrs={'class': 'WB_screen'}).find('a').get('action-data')[4:]
except (AttributeError, IndexError, TypeError):
return None
try:
wb_data.weibo_url = each.find(attrs={'node-type': 'feed_list_item_date'})['href']
except Exception as e:
parser.error('fail to get weibo url, the error is {}, the source page is {}'.format(e, html))
return None
def url_filter(url):
return ':'.join([PROTOCOL, url]) if PROTOCOL not in url and ORIGIN not in url else url
try:
imgs = str(each.find(attrs={'node-type': 'feed_list_media_prev'}).find_all('li'))
imgs_url = map(url_filter, re.findall(r"src=\"(.+?)\"", imgs))
wb_data.weibo_img = ';'.join(imgs_url)
except Exception:
wb_data.weibo_img = ''
try:
a_tag = str(each.find(attrs={'node-type': 'feed_list_media_prev'}).find_all('a'))
extracted_url = urllib.parse.unquote(re.findall(r"full_url=(.+?)&", a_tag)[0])
wb_data.weibo_video = url_filter(extracted_url)
except Exception:
wb_data.weibo_video = ''
try:
wb_data.device = each.find(attrs={'class': 'feed_from'}).find(attrs={'rel': 'nofollow'}).text
except AttributeError:
wb_data.device = ''
try:
create_time = each.find(attrs={'node-type': 'feed_list_item_date'})['date']
except (AttributeError, KeyError):
wb_data.create_time = ''
else:
create_time = int(create_time) / 1000 # 时间戳单位不同
create_time = datetime.fromtimestamp(create_time)
wb_data.create_time = create_time.strftime("%Y-%m-%d %H:%M")
try:
feed_action = each.find(attrs={'class': 'feed_action'})
except Exception as why:
parser.error('failt to get feed_action, the error is {},the page source is {}'.format(why, each))
else:
feed_infos = feed_action.find_all('li')
try:
wb_data.repost_num = get_feed_info(feed_infos, '转发')
except (AttributeError, ValueError):
wb_data.repost_num = 0
try:
wb_data.comment_num = get_feed_info(feed_infos, '评论')
except (AttributeError, ValueError):
wb_data.comment_num = 0
try:
wb_data.praise_num = int(feed_action.find(attrs={'action-type': 'feed_list_like'}).find('em').text)
except (AttributeError, ValueError):
wb_data.praise_num = 0
try:
wb_data.weibo_cont = each.find(attrs={'class': 'comment_txt'}).text.strip()
except Exception as why:
parser.error('fail to get weibo cont, the error is {}, the page source is {}'.format(why, html))
return None
if '展开全文' in str(each):
is_all_cont = 0
else:
is_all_cont = 1
return wb_data, is_all_cont