def _get_comment_list(self, all_comment_list, db_top_n_buyer_name_and_comment_date_list, db_sku_info):
"""
转换成结果集
:param all_comment_list:
:return:
"""
_comment_list = []
for item in all_comment_list:
buyer_name = item.get('member', '')
comment = []
# TODO item.get('rateItem', [])只取第一条comment
try:
first_rate_item = item.get('rateItem', [])[0]
except IndexError:
continue
_comment_content = wash_goods_comment(comment_content=first_rate_item.get('remarkContent', ''))
if not filter_invalid_comment_content(_comment_content):
continue
comment_date = self._get_comment_date2(item=first_rate_item)
comment.append({
'comment': _comment_content,
'comment_date': comment_date,
'sku_info': choice(db_sku_info), # 购买的商品规格(pc端1688商品没有规格)
'star_level': first_rate_item.get('starLevel', 5),
'img_url_list': [],
'video': '',
})
quantify = item.get('quantity', 1) # 购买数量
if comment == []: # 为空不录入
continue
if not filter_crawled_comment_content(
new_buyer_name=buyer_name,
new_comment_date=comment_date,
db_buyer_name_and_comment_date_info=db_top_n_buyer_name_and_comment_date_list,
logger=self.lg):
# 过滤已采集的comment
continue
_ = {
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 购买数量
'head_img': '', # 用户头像
'append_comment': {}, # 追评
}
_comment_list.append(_)
return _comment_list
def _get_comment_list(self, _tmp_comment_list):
'''
转换成需求的结果集
:param _tmp_comment_list:
:return:
'''
_comment_list = []
for item in _tmp_comment_list:
_comment_date = item.get('commentDate', '')
assert _comment_date != '', '得到的_comment_date为空str!请检查!'
# sku_info(有些商品评论是没有规格的所以默认为空即可，不加assert检查!)
ware_attributes = item.get('wareAttributes', [])
# self.lg.info(str(ware_attributes))
sku_info = ' '.join([i.get('key', '')+':'+i.get('value', '') for i in ware_attributes])
# assert sku_info != '', '得到的sku_info为空str!请检查!'
_comment_content = item.get('commentData', '')
assert _comment_content != '', '得到的评论内容为空str!请检查!'
_comment_content = self._wash_comment(comment=_comment_content)
buyer_name = item.get('userNickName', '')
assert buyer_name != '', '得到的用户昵称为空值!请检查!'
# jd设置默认 购买量为1
quantify = 1
head_img = item.get('userImgURL', '')
assert head_img != '', '得到的用户头像为空值!请检查!'
head_img = 'https://' + head_img
# 第一次评论图片
_comment_img_list = item.get('pictureInfoList', [])
if _comment_img_list != []:
_comment_img_list = [{'img_url': img.get('largePicURL', '')} for img in _comment_img_list]
'''追评'''
append_comment = {}
# star_level
star_level = int(item.get('commentScore', '5'))
if not filter_invalid_comment_content(_comment_content):
continue
comment = [{
'comment': _comment_content,
'comment_date': _comment_date,
'sku_info': sku_info,
'img_url_list': _comment_img_list,
'star_level': star_level,
'video': '',
}]
_comment_list.append({
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 评论数量
'head_img': head_img, # 头像
'append_comment': append_comment, # 追评
})
return _comment_list
def _get_comment_list(self, _tmp_comment_list):
'''
获取规范化的comment结果集
:param _tmp_comment_list:
:return:
'''
_comment_list = []
for item in _tmp_comment_list:
comment_date = item.get('createTime', '')
assert comment_date != '', '得到的comment_date为空str!请检查!'
comment_date = self._get_comment_date(comment_date)
buyer_name = item.get('nickname', '')
assert buyer_name != '', '得到的用户昵称为空值!请检查!'
_comment_content = item.get('content', '')
assert _comment_content != '', '得到的评论内容为空str!请检查!'
_comment_content = self._wash_comment(comment=_comment_content)
sku_info = item.get('skuDesc', '')
# self.lg.info(sku_info)
# 存在规格为空的
# assert sku_info != '', '得到的sku_info为空str!请检查!'
sku_info = self._wash_sku_info(sku_info)
# 第一次评论照片
img_url_list = item.get('firstEvidences', [])
# self.lg.info(img_url_list)
if img_url_list is None:
img_url_list = []
else:
img_url_list = [{
'img_url': _i.get('big', '')
} for _i in img_url_list]
'''追评'''
append_comment = {}
if item.get('appendTime', '') == '': # 追评时间为空即表示无追评
pass
else:
_tmp_append_comment_content = item.get('append', '')
# 追评的图片
_append_comment_img_list = [
{
'img_url': img.get('big', '')
} for img in item.get('appendEvidences')
] if item.get('appendEvidences') is not None else []
# self.lg.info(_append_comment_img_list)
append_comment = {
'comment_date': item.get('appendTime', ''),
'comment': self._wash_comment(_tmp_append_comment_content),
'img_url_list': _append_comment_img_list,
}
# 购买数量, 随机
quantify = randint(1, 2)
# 用户头像, 默认留空
head_img = ''
# 评论星级
star_level = int(item.get('levelStar', 5))
if not filter_invalid_comment_content(_comment_content):
continue
comment = [{
'comment': _comment_content,
'comment_date': comment_date,
'sku_info': sku_info,
'img_url_list': img_url_list,
'star_level': star_level,
'video': '',
}]
_ = {
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 购买数量
'head_img': head_img, # 头像
'append_comment': append_comment, # 追评
}
_comment_list.append(_)
return _comment_list
def _get_comment_data(self, goods_id):
if goods_id == '':
self.result_data = {}
return {}
self.my_lg.info('------>>>| 待处理的goods_id为: %s' % str(goods_id))
# # 原先采用phantomjs, 改用手机端抓html(speed slow, give up)
# tmp_url = 'https://m.1688.com/page/offerRemark.htm?offerId=' + str(goods_id)
# body = self.my_phantomjs.use_phantomjs_to_get_url_body(url=tmp_url, exec_code=self._exec_code)
# # self.my_lg.info(str(body))
#
# if body == '':
# self.result_data = {}
# self.my_lg.error('该地址的body为空值, 出错地址: ' + tmp_url)
# return {}
#
# _html_comment_list = list(Selector(text=body).css('div.remark-item').extract())
# if _html_comment_list != []:
# _comment_list = []
# for index, item in enumerate(_html_comment_list):
# if index > 25: # 就取前25条评论信息
# break
#
# buyer_name = str(Selector(text=item).css('span.member::text').extract_first())
# quantify = str(Selector(text=item).css('span.amount::text').extract_first())
# try:
# quantify = int(re.compile(r'\d+').findall(quantify)[0])
# except IndexError:
# self.my_lg.error('获取quantify时索引异常! 出错地址: ' + tmp_url)
# self.result_data = {}
# return {}
#
# comment_date = str(Selector(text=item).css('div.date span::text').extract_first())
# comment_date = self._get_comment_date(comment_date) # str '2017-01-25 17:06:00'
# tmp_sku_info = str(Selector(text=item).css('div.date::text').extract_first())
#
# _comment_content = self._wash_comment(str(Selector(text=item).css('div.bd::text').extract_first()))
# if not filter_invalid_comment_content(_comment_content):
# continue
#
# comment = [{
# 'comment': _comment_content,
# 'comment_date': comment_date, # 评论创建日期
# 'sku_info': re.compile(r'<span.*?</span>').sub('', tmp_sku_info), # 购买的商品规格
# 'img_url_list': [],
# 'star_level': randint(3, 5), # 几星好评
# 'video': '',
# }]
#
# _ = {
# 'buyer_name': buyer_name, # 买家昵称
# 'comment': comment, # 评论内容
# 'quantify': quantify, # 购买数量
# 'head_img': '', # 用户头像
# 'append_comment': {}, # 追评
# }
# _comment_list.append(_)
#
# _t = datetime.datetime.now()
#
# _r = CommentItem()
# _r['goods_id'] = str(goods_id)
# _r['create_time'] = _t
# _r['modify_time'] = _t
# _r['_comment_list'] = _comment_list
# self.result_data = _r
# # pprint(self.result_data)
# return self.result_data
# else:
# self.my_lg.error('该商品的comment为空list! 出错地址: ' + tmp_url)
# self.result_data = {}
# return {}
'''下面是模拟pc端好评接口'''
member_id = self._get_this_goods_member_id(goods_id=goods_id)
self.my_lg.info('------>>>| 获取到的member_id: {0}'.format(member_id))
if member_id == '':
self.my_lg.error('获取到的member_id为空值!请检查!')
self.result_data = {}
return {}
# 这里从db获取该商品原先的规格值
sku_info = self._get_sku_info_from_db(goods_id)
# self.my_lg.info('sku_info: {0}'.format(sku_info))
if sku_info == []:
self.result_data = {}
return {}
_comment_list = []
for page_num in range(1, 4):
self.my_lg.info('------>>>| 正在抓取第{0}页...'.format(page_num))
params = self._set_params(goods_id=goods_id, member_id=member_id, page_num=page_num)
url = 'https://rate.1688.com/remark/offerDetail/rates.json'
tmp_headers = self.headers
tmp_headers.update({
'referer': 'https://detail.1688.com/offer/{0}.html'.format(str(goods_id))
})
# 原先用MyRequests老是404，改用phantomjsy也还是老是404
body = MyRequests.get_url_body(url=url, headers=tmp_headers, params=params)
# self.my_lg.info(str(body))
# 用phantomjs
# url = self._set_url(url=url, params=params)
# self.my_lg.info(url)
# body = self.my_phantomjs.use_phantomjs_to_get_url_body(url)
# try:
# body = re.compile('<pre.*?>(.*)</pre>').findall(body)[0]
# except IndexError:
# self.my_lg.error('获取body时索引异常!')
# self.result_data = {}
# return {}
if body == '':
self.result_data = {}
self.my_lg.error('该地址的body为空值, 出错goods_id: {0}'.format(goods_id))
return {}
data = self.json_str_2_dict(json_str=body)
if data.get('url') is not None:
self.my_lg.info('------>>>| 被重定向到404页面, 休眠{0}s中...'.format(self._page_sleep_time))
sleep(self._page_sleep_time)
break
# self.my_lg.info(str(body))
data = data.get('data', {}).get('rates', [])
# pprint(data)
if data == []:
# sleep(self._page_sleep_time)
break
try:
for item in data:
buyer_name = item.get('member', '')
comment = []
for i in item.get('rateItem', []):
_comment_content = self._wash_comment(i.get('remarkContent', ''))
if not filter_invalid_comment_content(_comment_content):
continue
comment.append({
'comment': _comment_content,
'comment_date': str(i.get('remarkTime', '')), # 评论日期
'sku_info': choice(sku_info), # 购买的商品规格(pc端1688商品没有规格)
'star_level': i.get('starLevel', 5),
'img_url_list': [],
'video': '',
})
quantify = item.get('quantity', 1) # 购买数量
if comment == []: # 为空不录入
continue
_ = {
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 购买数量
'head_img': '', # 用户头像
'append_comment': {}, # 追评
}
_comment_list.append(_)
except Exception:
self.result_data = {}
self.my_lg.error('出错商品goods_id: {0}'.format(goods_id), exc_info=True)
return {}
sleep(self._page_sleep_time)
if _comment_list != []:
# pprint(_comment_list)
_t = datetime.datetime.now()
_r = CommentItem()
_r['goods_id'] = str(goods_id)
_r['create_time'] = _t
_r['modify_time'] = _t
_r['_comment_list'] = _comment_list
self.result_data = _r
return self.result_data
else:
self.my_lg.error('出错goods_id: {0}'.format(goods_id))
self.result_data = {}
return {}
def _get_comment_list(self, _tmp_comment_list):
'''
转化成需要的结果集
:param _tmp_comment_list:
:return:
'''
_comment_list = []
_sku_info_list = [] # 用于存已有的规格
for item in _tmp_comment_list:
comment_date = item.get('date', '')
assert comment_date != '', '得到的comment_date为空str!请检查!'
comment_date = self._get_comment_date(comment_date)
sku_info = item.get('auction', {}).get('sku', '')
# self.my_lg.info(sku_info)
if sku_info == '' and _sku_info_list == []: # 规格为空就跳过, 即只抓取有效评论
continue
if sku_info != '': # 不为空存入
_sku_info_list.append(sku_info)
_sku_info_list = list(set(_sku_info_list))
if sku_info == '': # 为空的，随机设置一个
sku_info = _sku_info_list[randint(0, len(_sku_info_list) - 1)]
# print(sku_info)
sku_info = self._wash_sku_info(sku_info)
# 评论照片
img_url_list = item.get('photos', [])
img_url_list = [{
'img_url': 'https:' + _i.get('url', '')
} for _i in img_url_list if _i.get('url', '') != '']
_comment_content = item.get('content', '')
assert _comment_content != '', '得到的评论内容为空str!请检查!'
_comment_content = self._wash_comment(comment=_comment_content)
buyer_name = item.get('user', {}).get('nick', '')
assert buyer_name != '', '得到的用户昵称为空值!请检查!'
quantify = int(item.get('buyAmount', 0)) if item.get('buyAmount', 0) != 0 else 1
tmp_head_img = item.get('user', {}).get('avatar', '')
if re.compile(r'/default/avatar-40.png').findall(tmp_head_img) != []:
head_img = ''
# 无法识别是否为同一张图 只能先拿到这种规律的然后请求图片看齐地址
# elif re.compile(r'vGNuOHcWv88YXF').findall(tmp_head_img) != []:
# # self.my_lg.info('https:' + tmp_head_img)
# if self._judge_is_taobao_head_img(url='https:' + tmp_head_img):
# self.my_lg.info('https:' + tmp_head_img)
# head_img = ''
# else:
# head_img = 'https:' + tmp_head_img
elif tmp_head_img != '//wwc.alicdn.com/avatar/getAvatar.do?userIdStr=vGNuOHcWv88YXF-HPmvbM07HvG8SvFI0Xm7Hvm80MkZhvkk0XmcSPFPhPHQWOmvG&width=40&height=40&type=sns' \
or tmp_head_img != '//gw.alicdn.com/tps/i3/TB1yeWeIFXXXXX5XFXXuAZJYXXX-210-210.png_40x40.jpg':
head_img = 'https:' + tmp_head_img
else:
head_img = ''
if not filter_invalid_comment_content(_comment_content):
continue
comment = [{
'comment': _comment_content,
'comment_date': comment_date,
'sku_info': sku_info,
'img_url_list': img_url_list,
'star_level': randint(4, 5),
'video': item.get('video', ''),
}]
_ = {
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 购买数量
'head_img': head_img, # 头像
'append_comment': {}, # 追评
}
_comment_list.append(_)
return _comment_list
def _get_comment_list(self, all_comment_list,
db_top_n_buyer_name_and_comment_date_list,
db_sku_info_list):
'''
转化成需要的结果集
:param all_comment_list:
:return:
'''
_comment_list = []
_sku_info_list = [] # 用于存已有的规格
for item in all_comment_list:
comment_date = self._get_comment_date(item=item)
sku_info = self._get_sku_info(item=item,
db_sku_info_list=db_sku_info_list)
# 评论照片
img_url_list = item.get('photos', [])
img_url_list = [{
'img_url': 'https:' + _i.get('url', '')
} for _i in img_url_list if _i.get('url', '') != '']
_comment_content = item.get('content', '')
assert _comment_content != '', '得到的评论内容为空str!请检查!'
_comment_content = wash_goods_comment(
comment_content=_comment_content)
buyer_name = item.get('user', {}).get('nick', '')
assert buyer_name != '', '得到的用户昵称为空值!请检查!'
quantify = int(item.get('buyAmount',
0)) if item.get('buyAmount', 0) != 0 else 1
head_img = self._get_head_img_url(item=item)
ori_video_info = item.get(
'video', {}) if item.get('video') is not None else {}
video_url = ori_video_info.get('cloudVideoUrl', '')
video_url = 'https:' + video_url if video_url != '' else ''
if not filter_invalid_comment_content(_comment_content):
continue
if not filter_crawled_comment_content(
new_buyer_name=buyer_name,
new_comment_date=comment_date,
db_buyer_name_and_comment_date_info=
db_top_n_buyer_name_and_comment_date_list,
logger=self.lg):
# 过滤已采集的comment
continue
comment = [{
'comment': _comment_content,
'comment_date': comment_date,
'sku_info': sku_info,
'img_url_list': img_url_list,
'star_level': randint(4, 5),
'video': video_url,
}]
_ = {
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 购买数量
'head_img': head_img, # 头像
'append_comment': {}, # 追评
}
_comment_list.append(_)
return _comment_list
def _get_comment_list(self, all_comment_list,
db_top_n_buyer_name_and_comment_date_list,
db_sku_info_list):
'''
转换成需求的结果集
:param all_comment_list:
:return:
'''
_comment_list = []
for item in all_comment_list:
# pprint(item)
_comment_date = self._get_comment_date(item=item)
sku_info = item.get('auctionSku', '')
# self.lg.info(sku_info)
if sku_info == '':
# 从所有规格里面随机一个
sku_info = str(choice(db_sku_info_list))
_comment_content = item.get('rateContent', '')
assert _comment_content != '', '得到的评论内容为空str!请检查!'
_comment_content = wash_goods_comment(
comment_content=_comment_content)
buyer_name = item.get('displayUserNick', '')
assert buyer_name != '', '得到的用户昵称为空值!请检查!'
# 天猫设置默认 购买量为1
quantify = 1
# 天猫没有head_img回传，就设置一个默认地址
head_img = ''
# 第一次评论图片
_comment_img_list = item.get(
'pics', []) if item.get('pics', '') != '' else []
if _comment_img_list != []:
_comment_img_list = [{
'img_url': 'https:' + img
} for img in _comment_img_list]
'''追评'''
_tmp_append_comment = item.get(
'appendComment',
{}) if item.get('appendComment') is not None else {}
# 追评的图片
# pprint(_tmp_append_comment)
_append_comment_img_list = _tmp_append_comment.get(
'pics',
[]) if _tmp_append_comment.get('pics', '') != '' else []
if _append_comment_img_list != []:
_append_comment_img_list = [{
'img_url': 'https:' + img
} for img in _append_comment_img_list]
if _tmp_append_comment != {}:
append_comment = {
'comment_date':
_tmp_append_comment.get('commentTime', ''),
'comment':
wash_goods_comment(comment_content=_tmp_append_comment.get(
'content', '')),
'img_url_list':
_append_comment_img_list,
}
else:
append_comment = {}
if not filter_invalid_comment_content(_comment_content):
continue
if not filter_crawled_comment_content(
new_buyer_name=buyer_name,
new_comment_date=_comment_date,
db_buyer_name_and_comment_date_info=
db_top_n_buyer_name_and_comment_date_list,
logger=self.lg):
# 过滤已采集的comment
continue
comment = [{
'comment': _comment_content,
'comment_date': _comment_date,
'sku_info': sku_info,
'img_url_list': _comment_img_list,
'star_level': randint(4, 5),
'video': '',
}]
_ = {
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 评论数量
'head_img': head_img, # 头像
'append_comment': append_comment, # 追评
}
_comment_list.append(_)
return _comment_list
def _get_comment_list(self, _tmp_comment_list):
'''
转换成需求的结果集
:param _tmp_comment_list:
:return:
'''
_comment_list = []
for item in _tmp_comment_list:
# pprint(item)
_comment_date = item.get('rateDate', '')
assert _comment_date != '', '得到的_comment_date为空str!请检查!'
# 天猫接口拿到的sku_info默认为空
# sku_info = ''
# 从所有规格里面随机一个
if self.random_sku_info_list == []:
self.random_sku_info_list = ['']
sku_info = str(choice(self.random_sku_info_list))
_comment_content = item.get('rateContent', '')
assert _comment_content != '', '得到的评论内容为空str!请检查!'
_comment_content = self._wash_comment(comment=_comment_content)
buyer_name = item.get('displayUserNick', '')
assert buyer_name != '', '得到的用户昵称为空值!请检查!'
# 天猫设置默认 购买量为1
quantify = 1
# 天猫没有head_img回传，就设置一个默认地址
head_img = ''
# 第一次评论图片
_comment_img_list = item.get(
'pics', []) if item.get('pics', '') != '' else []
if _comment_img_list != []:
_comment_img_list = [{
'img_url': 'https:' + img
} for img in _comment_img_list]
'''追评'''
_tmp_append_comment = item.get(
'appendComment',
{}) if item.get('appendComment') is not None else {}
# 追评的图片
# pprint(_tmp_append_comment)
_append_comment_img_list = _tmp_append_comment.get(
'pics',
[]) if _tmp_append_comment.get('pics', '') != '' else []
if _append_comment_img_list != []:
_append_comment_img_list = [{
'img_url': 'https:' + img
} for img in _append_comment_img_list]
if _tmp_append_comment != {}:
append_comment = {
'comment_date':
_tmp_append_comment.get('commentTime', ''),
'comment':
self._wash_comment(_tmp_append_comment.get('content', '')),
'img_url_list':
_append_comment_img_list,
}
else:
append_comment = {}
if not filter_invalid_comment_content(_comment_content):
continue
comment = [{
'comment': _comment_content,
'comment_date': _comment_date,
'sku_info': sku_info,
'img_url_list': _comment_img_list,
'star_level': randint(4, 5),
'video': '',
}]
_ = {
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 评论数量
'head_img': head_img, # 头像
'append_comment': append_comment, # 追评
}
_comment_list.append(_)
return _comment_list
def _get_comment_list(self, _tmp_comment_list,
db_top_n_buyer_name_and_comment_date_list):
'''
转换成需求的结果集
:param _tmp_comment_list:
:return:
'''
_comment_list = []
for item in _tmp_comment_list:
_comment_date = self._get_comment_date(item=item)
# sku_info(有些商品评论是没有规格的所以默认为空即可，不加assert检查!)
# eg: '颜域品牌女装2017冬季新品娃娃领加厚格纹绣花毛呢外套中长款大衣04W7135 黑色 M/38'
ware_attributes = item.get('referenceName', '')
# self.lg.info(str(ware_attributes))
sku_info = ' '.join(ware_attributes.split(' ')[1:])
# assert sku_info != '', '得到的sku_info为空str!请检查!'
_comment_content = item.get('content', '')
assert _comment_content != '', '得到的评论内容为空str!请检查!'
_comment_content = wash_goods_comment(
comment_content=_comment_content)
buyer_name = item.get('nickname', '')
assert buyer_name != '', '得到的用户昵称为空值!请检查!'
# jd设置默认 购买量为1
quantify = 1
head_img = self._get_head_img_url(item=item)
# 第一次评论图片
_comment_img_list = item.get('images', [])
if _comment_img_list != []:
_comment_img_list = [
{
'img_url':
img.get('imgUrl', '').replace('s128x96_jfs',
'jfs'), # 小图换成大图!
} for img in _comment_img_list
]
'''追评'''
append_comment = {}
# star_level
star_level = int(item.get('score', '5'))
if not filter_invalid_comment_content(_comment_content):
continue
comment = [{
'comment': _comment_content,
'comment_date': _comment_date,
'sku_info': sku_info,
'img_url_list': _comment_img_list,
'star_level': star_level,
'video': '',
}]
if not filter_crawled_comment_content(
new_buyer_name=buyer_name,
new_comment_date=_comment_date,
db_buyer_name_and_comment_date_info=
db_top_n_buyer_name_and_comment_date_list,
logger=self.lg):
# 过滤已采集的comment
continue
_comment_list.append({
'buyer_name': buyer_name, # 买家昵称
'comment': comment, # 评论内容
'quantify': quantify, # 评论数量
'head_img': head_img, # 头像
'append_comment': append_comment, # 追评
})
return _comment_list