def __init__(self):
self._set_logger()
self.msg = ''
self.debugging_api = self._init_debugging_api()
self._set_func_name_dict()
self.sql_str = r'insert into dbo.all_goods_comment(goods_id, create_time, modify_time, comment_info) values(%s, %s, %s, %s)'
if self._init_debugging_api().get(2):
self.my_lg.info('初始化 1688 phantomjs中...')
self.ali_1688 = ALi1688CommentParse(logger=self.my_lg)
if self._init_debugging_api().get(3) is True \
or self._init_debugging_api().get(4) is True\
or self._init_debugging_api().get(6) is True:
self.my_lg.info('初始化 天猫 phantomjs中...')
self.tmall = TmallCommentParse(logger=self.my_lg)
if self._init_debugging_api().get(7) is True \
or self._init_debugging_api().get(8) is True\
or self._init_debugging_api().get(9) is True\
or self._init_debugging_api().get(10) is True:
self.my_lg.info('初始化 京东 phantomjs中...')
self.jd = JdCommentParse(logger=self.my_lg)
self.my_lg.info('初始化完毕!!!')
def _ali_1688_comment(self, index, goods_id, site_id):
'''
处理阿里1688的商品comment
:param index: 索引
:param goods_id:
:param site_id:
:return:
'''
if self.debugging_api.get(site_id):
self.my_lg.info('------>>>| 阿里1688\t\t索引值(%s)' % str(index))
if index % 5 == 0:
try:
del self.ali_1688
except:
self.my_lg.info('del ali_1688失败!')
gc.collect()
self.ali_1688 = ALi1688CommentParse(logger=self.my_lg)
_r = self.ali_1688._get_comment_data(goods_id=goods_id)
if _r.get('_comment_list', []) != []:
if self._comment_pipeline.is_connect_success:
self._comment_pipeline._insert_into_table(
sql_str=self.sql_str,
params=self._get_db_insert_params(item=_r))
else:
self.my_lg.info('该商品_comment_list为空list! 此处跳过!')
else:
pass
def _update_ali_1688_comment(self, index, goods_id, site_id):
'''
处理阿里1688的商品comment
:param index: 索引
:param goods_id:
:param site_id:
:return:
'''
if self.debugging_api.get(site_id):
self.my_lg.info('------>>>| 阿里1688\t\t索引值(%s)' % str(index))
ali_1688 = ALi1688CommentParse(logger=self.my_lg)
_r = ali_1688._get_comment_data(goods_id=goods_id)
if _r != {}:
if self._comment_pipeline.is_connect_success:
self._comment_pipeline._update_table(
sql_str=self.sql_str,
params=self._get_db_update_params(item=_r))
try:
del ali_1688
except:
self.my_lg.info('del ali_1688失败!')
gc.collect()
else:
pass
def ali_1688_comment(self, index, goods_id):
'''
处理阿里1688的商品comment
:params index: 索引
:param goods_id:
:return:
'''
self.my_lg.info('阿里1688\t\t\t索引值(%s)' % str(index))
ali_1688 = ALi1688CommentParse(logger=self.my_lg)
ali_1688._get_comment_data(goods_id=goods_id)
try:
del ali_1688
except:
self.my_lg.info('del ali_1688失败!')
gc.collect()
def __init__(self):
self._set_logger()
self.msg = ''
self.debugging_api = self._init_debugging_api()
self._set_func_name_dict()
self.sql_str = cm_update_str_1
if self._init_debugging_api().get(2):
self.my_lg.info('初始化 1688 phantomjs中...')
self.ali_1688 = ALi1688CommentParse(logger=self.my_lg)
if self._init_debugging_api().get(3) is True \
or self._init_debugging_api().get(4) is True\
or self._init_debugging_api().get(6) is True:
self.my_lg.info('初始化 天猫 phantomjs中...')
self.tmall = TmallCommentParse(logger=self.my_lg)
if self._init_debugging_api().get(7) is True \
or self._init_debugging_api().get(8) is True\
or self._init_debugging_api().get(9) is True\
or self._init_debugging_api().get(10) is True:
self.my_lg.info('初始化 京东 phantomjs中...')
self.jd = JdCommentParse(logger=self.my_lg)
def __init__(self):
self._set_logger()
self.msg = ''
self.debugging_api = self._init_debugging_api()
self._set_func_name_dict()
self.sql_str = r'update dbo.all_goods_comment set modify_time=%s, comment_info=%s where goods_id=%s'
if self._init_debugging_api().get(2):
self.my_lg.info('初始化 1688 phantomjs中...')
self.ali_1688 = ALi1688CommentParse(logger=self.my_lg)
if self._init_debugging_api().get(3) is True \
or self._init_debugging_api().get(4) is True\
or self._init_debugging_api().get(6) is True:
self.my_lg.info('初始化 天猫 phantomjs中...')
self.tmall = TmallCommentParse(logger=self.my_lg)
if self._init_debugging_api().get(7) is True \
or self._init_debugging_api().get(8) is True\
or self._init_debugging_api().get(9) is True\
or self._init_debugging_api().get(10) is True:
self.my_lg.info('初始化 京东 phantomjs中...')
self.jd = JdCommentParse(logger=self.my_lg)