def __init__(self): self._set_logger() self.msg = '' self.debugging_api = self._init_debugging_api() self._set_func_name_dict() self.sql_str = r'insert into dbo.all_goods_comment(goods_id, create_time, modify_time, comment_info) values(%s, %s, %s, %s)' if self._init_debugging_api().get(2): self.my_lg.info('初始化 1688 phantomjs中...') self.ali_1688 = ALi1688CommentParse(logger=self.my_lg) if self._init_debugging_api().get(3) is True \ or self._init_debugging_api().get(4) is True\ or self._init_debugging_api().get(6) is True: self.my_lg.info('初始化 天猫 phantomjs中...') self.tmall = TmallCommentParse(logger=self.my_lg) if self._init_debugging_api().get(7) is True \ or self._init_debugging_api().get(8) is True\ or self._init_debugging_api().get(9) is True\ or self._init_debugging_api().get(10) is True: self.my_lg.info('初始化 京东 phantomjs中...') self.jd = JdCommentParse(logger=self.my_lg) self.my_lg.info('初始化完毕!!!')
def _ali_1688_comment(self, index, goods_id, site_id): ''' 处理阿里1688的商品comment :param index: 索引 :param goods_id: :param site_id: :return: ''' if self.debugging_api.get(site_id): self.my_lg.info('------>>>| 阿里1688\t\t索引值(%s)' % str(index)) if index % 5 == 0: try: del self.ali_1688 except: self.my_lg.info('del ali_1688失败!') gc.collect() self.ali_1688 = ALi1688CommentParse(logger=self.my_lg) _r = self.ali_1688._get_comment_data(goods_id=goods_id) if _r.get('_comment_list', []) != []: if self._comment_pipeline.is_connect_success: self._comment_pipeline._insert_into_table( sql_str=self.sql_str, params=self._get_db_insert_params(item=_r)) else: self.my_lg.info('该商品_comment_list为空list! 此处跳过!') else: pass
def _update_ali_1688_comment(self, index, goods_id, site_id): ''' 处理阿里1688的商品comment :param index: 索引 :param goods_id: :param site_id: :return: ''' if self.debugging_api.get(site_id): self.my_lg.info('------>>>| 阿里1688\t\t索引值(%s)' % str(index)) ali_1688 = ALi1688CommentParse(logger=self.my_lg) _r = ali_1688._get_comment_data(goods_id=goods_id) if _r != {}: if self._comment_pipeline.is_connect_success: self._comment_pipeline._update_table( sql_str=self.sql_str, params=self._get_db_update_params(item=_r)) try: del ali_1688 except: self.my_lg.info('del ali_1688失败!') gc.collect() else: pass
def ali_1688_comment(self, index, goods_id): ''' 处理阿里1688的商品comment :params index: 索引 :param goods_id: :return: ''' self.my_lg.info('阿里1688\t\t\t索引值(%s)' % str(index)) ali_1688 = ALi1688CommentParse(logger=self.my_lg) ali_1688._get_comment_data(goods_id=goods_id) try: del ali_1688 except: self.my_lg.info('del ali_1688失败!') gc.collect()
def __init__(self): self._set_logger() self.msg = '' self.debugging_api = self._init_debugging_api() self._set_func_name_dict() self.sql_str = cm_update_str_1 if self._init_debugging_api().get(2): self.my_lg.info('初始化 1688 phantomjs中...') self.ali_1688 = ALi1688CommentParse(logger=self.my_lg) if self._init_debugging_api().get(3) is True \ or self._init_debugging_api().get(4) is True\ or self._init_debugging_api().get(6) is True: self.my_lg.info('初始化 天猫 phantomjs中...') self.tmall = TmallCommentParse(logger=self.my_lg) if self._init_debugging_api().get(7) is True \ or self._init_debugging_api().get(8) is True\ or self._init_debugging_api().get(9) is True\ or self._init_debugging_api().get(10) is True: self.my_lg.info('初始化 京东 phantomjs中...') self.jd = JdCommentParse(logger=self.my_lg)
def __init__(self): self._set_logger() self.msg = '' self.debugging_api = self._init_debugging_api() self._set_func_name_dict() self.sql_str = r'update dbo.all_goods_comment set modify_time=%s, comment_info=%s where goods_id=%s' if self._init_debugging_api().get(2): self.my_lg.info('初始化 1688 phantomjs中...') self.ali_1688 = ALi1688CommentParse(logger=self.my_lg) if self._init_debugging_api().get(3) is True \ or self._init_debugging_api().get(4) is True\ or self._init_debugging_api().get(6) is True: self.my_lg.info('初始化 天猫 phantomjs中...') self.tmall = TmallCommentParse(logger=self.my_lg) if self._init_debugging_api().get(7) is True \ or self._init_debugging_api().get(8) is True\ or self._init_debugging_api().get(9) is True\ or self._init_debugging_api().get(10) is True: self.my_lg.info('初始化 京东 phantomjs中...') self.jd = JdCommentParse(logger=self.my_lg)