Exemplo n.º 1
0
 def log(self, cls_name, method_name, message, type):
     """
     Instantiate the logger obj and log to file depending on the msg type.
     """
     logger = logging.getLogger("%s - %s:" % (cls_name, method_name))
     
     # print the message out if the logger is set to be verbose
     if self.verbose:
         print u"%s: [%s - %s()] %s:" % (type.upper(), cls_name, method_name, message)
     
     # add extra params
     extras = {
         'class': cls_name,
         'method': method_name,
     }
     
     # determine the logging type and set as required - options are info, warning, error
     if type == 'info':
         if self.debug:
             logger.setLevel(logging.INFO)
             logger.info(jd_utils.encoding(message), extra=extras)
             
     elif type == 'warning':
         logger.setLevel(logging.WARNING)
         logger.warning(message, extra=extras)
         
     elif type == 'error':
         logger.setLevel(logging.ERROR)
         logger.error(message, extra=extras)
         raise Exception("%s: %s" % (type.upper(), message))
         
     elif type == 'critical':
         logger.setLevel(logging.CRITICAL)
         logger.critical(message, extra=extras)
         raise Exception("%s: %s" % (type.upper(), message))
Exemplo n.º 2
0
def get_product_ids(url, jdb, tid):
    flag = 0
    while True:
        try:
            request = urllib.request.Request(url, headers = jd_headers)
            g_response = urllib.request.urlopen(request)
            if g_response.info().get('Content-Encoding') == 'gzip':
                g_read = zlib.decompress(g_response.read(), 16+zlib.MAX_WBITS)
            else:    
                g_read = g_response.read()
                
            url_html = jd_utils.encoding(g_read)
            
            url_soup = BeautifulSoup(url_html)
            url_extend = url_soup.findAll('a', attrs = {"href": re.compile(r"^http://\w+.jd.com/.+\.(htm|html)$")})
            break
        except http.client.IncompleteRead:
            continue
        except Exception as e:
            if flag > 3 :
                print ("网络异常,放弃展开URL") 
                return
            if e.errno == errno.ECONNRESET:
                flag = flag + 1
                time.sleep(20)
                print ("重试中...[%d]" % flag)
                continue
            print ("展开产品链接异常:" + str(e) )
            return
    
    prds = []
    no_prds = []
    for url_item in url_extend:
        url_str = url_item.get("href")
        m = re.match(r'^http://item.jd.com/\d+.html$', url_str)
        if m:
            #jdb.db_insert_product(m.string)
            prds.append( m.string )           
        else:
            #("http://red.jd.com/", "http://tuan.jd.com/", "http://auction.jd.com/", "http://jr.jd.com/", "http://smart.jd.com/")
            if not re.match(r'^http://(help|red|tuan|auction|jr|smart|gongyi|app|en|media|m|myjd|chat|read|chongzhi|z|giftcard|fw|you|mobile|wiki|me).jd.com', url_str) and not re.match(r'^http://www.jd.com/compare/', url_str) and not re.match(r'^http://club.jd.com/consultation/', url_str) :
                no_prds.append( url_str )              
                #with gdb_lock:
                #    jdb.db_insert_no_product(url_str)
        
    # Really need to do with database
    if prds or no_prds :
        with gdb_lock:
            print ('线程[%d] 插入数据库...' % tid)
            for item in prds:
                jdb.db_insert_product(item)
            for item in no_prds:
                jdb.db_insert_no_product(item)
Exemplo n.º 3
0
    def get_product_consults(self, product_url):
        result_path = jd_config.JDSPR_RESULT
        self.agent = None
        page_id = 1    
        product_id = int(product_url.split('.')[2].split('/')[1])
        product_url = jd_item_url % product_id
        
        flag = 0
        while True:
            try:
                self.agent = random_jd_header(product_url)
                request = urllib.request.Request(product_url, headers = self.agent)
                g_response = urllib.request.urlopen(request)
                if g_response.info().get('Content-Encoding') == 'gzip':
                    g_read = zlib.decompress(g_response.read(), 16+zlib.MAX_WBITS)
                else:    
                    g_read = g_response.read()
                
                product_html = jd_utils.encoding(g_read)
                #操作正常
                break
            except UnicodeDecodeError:
                print ("GBK/Unicode编解码错误!")
                return
            except http.client.IncompleteRead:
                continue
            except Exception as e:
                if flag > 3 :
                    print ("咨询线程[%d]网络异常,放弃该产品" % self.tid) 
                    return
                if e.errno == errno.ECONNRESET:
                    flag = flag + 1
                    time.sleep(10)
                    print ("咨询线程[%d]重试中...[%d]"%( self.tid, flag) )
                    continue
                print ("1.其它异常:"+str(e))
                return
	
        product_name = None
        product_ts = None
        product_soup = BeautifulSoup(product_html)
        product_name = product_soup.find('h1')
        
        #产品类别
        product_type = product_soup.find('div', attrs={"class":"breadcrumb"})
        if product_type:
            product_ts = product_type.findAll('a')
        
        if not product_name or not product_ts:
            print("1.产品名称和类别提取错误,返回!Check[%s]" % product_url)
            print(self.agent['User-Agent'])
            return
        
        result_file = None
        try:
            i = 0    
            for pt_item in product_ts:
                if pt_item:
                    result_path = result_path + "/" + pt_item.string + "/"
                    i = i + 1
                    #目录类别的深度
                    if i > 3:
                        break
                else:
                    print("2.提取产品名称和目录错误!:%s, Check[%s]" % ( str(e), product_url) )
                    return
            
            if not os.path.exists(result_path):
                os.makedirs(result_path) 
            result_file = "%s/%d.txt"%(result_path,product_id)
            if os.path.exists(result_file):
                return
            
            #print ("产品保存地址:%s",result_path)    
            print ("咨询线程[%d]正在处理商品 %d" % ( self.tid, product_id ))
            f = codecs.open(result_file, 'wb',encoding = 'utf-8')   
            f.write("产品名称:" + product_name.text + "\n")
        except Exception as e:
            print("3.提取产品名称和目录错误!:%s, Check[%s]" % ( str(e), product_url) )
            if result_file and os.path.exists(result_file):
                try:
                    os.remove(result_file)
                except:
                    pass
            return
        count = 0 
        while  True:
            product_consult_url = jd_consult_url % ( product_id, page_id )
            flag = 0
            progress = "."
            while True:
                progress = progress + "."
                try:
                    self.agent = random_jd_header(product_url)
                    request = urllib.request.Request(product_consult_url, headers = self.agent)
                    g_response = urllib.request.urlopen(request)
                    if g_response.info().get('Content-Encoding') == 'gzip':
                        g_read = zlib.decompress(g_response.read(), 16+zlib.MAX_WBITS)
                    else:    
                        g_read = g_response.read()
                
                    consult_html = jd_utils.encoding(g_read)
                
                    #操作正常
                    break
                except UnicodeDecodeError:
                    print ("GBK/Unicode编解码错误!")
                    f.close()
                    return		
                except http.client.IncompleteRead:
                    continue	
                except Exception as e:
                    if flag > 3 :
                        print ("咨询线程[%d]网络异常,放弃该产品" % self.tid) 
                        f.close()
                        return
                    if e.errno == errno.ECONNRESET:
                        flag = flag + 1
                        time.sleep(10)
                        print ("咨询线程[%d]重试中...[%d]" % ( self.tid, flag) )
                        continue
                    print ("2.其它异常:"+str(e))
                    f.close()
                    return
            
            consult_soup = BeautifulSoup(consult_html)
            count = count + self.get_page_consult(consult_soup, f)
            if count == 0 and progress == "..":
                print("咨询线程[%d] - 商品咨询为空,删除商品文件:%s" %( self.tid , result_file))
                if result_file and os.path.exists(result_file):
                    try:
                        os.remove(result_file)
                    except:
                        pass
                return
                
            pagination = consult_soup.find('div', attrs = {"class":"Pagination"})
            if not pagination:
                break;
            if not pagination.findAll('a',attrs = {"class":"next"}) :
                break;
            else:
                page_id = page_id + 1;
                f.flush()
            
        print ("咨询线程[%d]处理完毕,咨询[%d] %d" % ( self.tid, count, product_id ))
        f.close()
Exemplo n.º 4
0
    def get_product_comments(self, product_url):
        result_path = ""
        self.agent = random_jd_header()
        page_id = 1    
        product_id = int(product_url.split('.')[2].split('/')[1])
        product_url = jd_item_url % product_id
        
        flag = 0
        while True:
            try:
                request = urllib.request.Request(product_url, headers = self.agent)
                g_response = urllib.request.urlopen(request)
                if g_response.info().get('Content-Encoding') == 'gzip':
                    g_read = zlib.decompress(g_response.read(), 16+zlib.MAX_WBITS)
                else:    
                    g_read = g_response.read()
                
                product_html = jd_utils.encoding(g_read)
                #操作正常
                break
            except UnicodeDecodeError:
                print ("GBK/Unicode编解码错误!")
                return
            except http.client.IncompleteRead:
                continue
            except Exception as e:
                if flag > 3 :
                    print ("评论线程[%d]网络异常,放弃该产品" % self.tid) 
                    return
                if e.errno == errno.ECONNRESET:
                    flag = flag + 1
                    time.sleep(10)
                    print ("评论线程[%d]重试中...[%d]"%( self.tid, flag) )
                    continue
                print ("1.其它异常:"+str(e))
                return
	
        product_name = None
        product_ts = None
        product_soup = BeautifulSoup(product_html)
        product_name = product_soup.find('h1')
        
        #产品类别
        product_type = product_soup.find('div', attrs={"class":"breadcrumb"})
        if product_type:
            product_ts = product_type.findAll('a')
        
        if not product_name or not product_ts:
            print("1.产品名称和类别提取错误,返回!Check[%s]" % product_url)
            print(self.agent['User-Agent'])
            return
        
        result_file = None
        try:
            i = 0    
            for pt_item in product_ts:
                if pt_item:
                    result_path = result_path + "/" + pt_item.string + "/"
                    i = i + 1
                    #目录类别的深度
                    if i > 3:
                        break
                else:
                    print("2.提取产品名称和目录错误!:%s, Check[%s]" % ( str(e), product_url) )
                    return
            
            #if not os.path.exists(result_path):
            #    os.makedirs(result_path) 
            result_file = "%s/%d_comm.txt"%(jd_config.JDSPR_RESULT_LOCAL, product_id)
            if os.path.exists(result_file):
                return
            
            #print ("产品保存地址:%s",result_path)    
            print ("评论线程[%d]正在处理商品 %d" % ( self.tid, product_id ))
            f = codecs.open(result_file, 'wb',encoding = 'utf-8')   
            f.write("产品名称:" + product_name.text + "\n")
        except Exception as e:
            print("3.提取产品名称和目录错误!:%s, Check[%s]" % ( str(e), product_url) )
            if result_file and os.path.exists(result_file):
                try:
                    os.remove(result_file)
                except:
                    pass
            return
            
        count = 0 
        retries = 0
        while  True:
            # random page url to avoid block
            product_comment_url = jd_comment_url % ( product_id, random.randint(3,10), page_id )
            #print ("=============> DOING... " + product_comment_url)
            flag = 0
            progress = "."
            while True:
                progress = progress + "."
                try:
                    self.agent = random_jd_header(product_comment_url)
                    request = urllib.request.Request(product_comment_url, headers = self.agent)
                    g_response = urllib.request.urlopen(request)
                    
                    if g_response.info().get('Content-Encoding') == 'gzip':
                        g_read = zlib.decompress(g_response.read(), 16+zlib.MAX_WBITS)
                    else:    
                        g_read = g_response.read()
                    
                    comment_html = jd_utils.encoding(g_read)
                
                    #操作正常
                    break
                except UnicodeDecodeError:
                    print ("GBK/Unicode编解码错误!")
                    f.close()
                    return
                except http.client.IncompleteRead:
                    continue
                except Exception as e:
                    if flag > 3 :
                        print ("评论线程[%d]网络异常,放弃该产品" % self.tid) 
                        f.close()
                        return
                    if e.errno == errno.ECONNRESET:
                        flag = flag + 1
                        time.sleep(2)
                        print ("评论线程[%d]重试中...[%d]" % ( self.tid, flag) )
                        continue
                    print ("2.其它异常:"+str(e))
                    f.close()
                    return
            
            comment_soup = BeautifulSoup(comment_html)
            count_t = self.get_page_comment(comment_soup, product_comment_url , f)
            
            # Retry about max 10 times here:
            # I hate JD
            lucky_flag = 1
            if count_t == 0:
                if retries < 10:
                    retries = retries + 1
                    if count != 0:
                        # Refresh user agent
                        self.agent = random_jd_header(product_comment_url)
                        time.sleep( random.randint(3, 9))
                    print("评论线程[%d] R[%d] %s" %( self.tid, retries, product_comment_url))               
                    continue
                else:
                    lucky_flag = 0
            
            retries = 0
            count = count + count_t
            
            if count == 0 and progress == "..":
                print("评论线程[%d] - 商品咨询为空,删除商品文件:%s" %( self.tid , result_file))
                if result_file and os.path.exists(result_file):
                    try:
                        os.remove(result_file)
                    except:
                        pass
                return                  
            
                           
            pagination = comment_soup.find('div', attrs = {"class":"pagin fr"})
            if not pagination:
                break
            if not pagination.findAll('a',attrs = {"class":"next"}) :
                break
            else:
                page_id = page_id + 1;
                f.flush()
            
        print ("评论线程[%d]处理完毕,产品[%d],评论[%d],LUCK[%s],PATH[%s]" % ( self.tid, product_id, count, lucky_flag, result_path ))
        f.close()
        return (product_url, count, lucky_flag, result_path, product_id)
Exemplo n.º 5
0
def get_product_ids(url, jdb, tid):
    flag = 0
    while True:
        try:
            request = urllib.request.Request(url, headers=jd_headers)
            g_response = urllib.request.urlopen(request)
            if g_response.info().get('Content-Encoding') == 'gzip':
                g_read = zlib.decompress(g_response.read(),
                                         16 + zlib.MAX_WBITS)
            else:
                g_read = g_response.read()

            url_html = jd_utils.encoding(g_read)

            url_soup = BeautifulSoup(url_html)
            url_extend = url_soup.findAll(
                'a',
                attrs={
                    "href": re.compile(r"^http://\w+.jd.com/.+\.(htm|html)$")
                })
            break
        except http.client.IncompleteRead:
            continue
        except Exception as e:
            if flag > 3:
                print("网络异常,放弃展开URL")
                return
            if e.errno == errno.ECONNRESET:
                flag = flag + 1
                time.sleep(20)
                print("重试中...[%d]" % flag)
                continue
            print("展开产品链接异常:" + str(e))
            return

    prds = []
    no_prds = []
    for url_item in url_extend:
        url_str = url_item.get("href")
        m = re.match(r'^http://item.jd.com/\d+.html$', url_str)
        if m:
            #jdb.db_insert_product(m.string)
            prds.append(m.string)
        else:
            #("http://red.jd.com/", "http://tuan.jd.com/", "http://auction.jd.com/", "http://jr.jd.com/", "http://smart.jd.com/")
            if not re.match(
                    r'^http://(help|red|tuan|auction|jr|smart|gongyi|app|en|media|m|myjd|chat|read|chongzhi|z|giftcard|fw|you|mobile|wiki|me).jd.com',
                    url_str) and not re.match(
                        r'^http://www.jd.com/compare/',
                        url_str) and not re.match(
                            r'^http://club.jd.com/consultation/', url_str):
                no_prds.append(url_str)
                #with gdb_lock:
                #    jdb.db_insert_no_product(url_str)

    # Really need to do with database
    if prds or no_prds:
        with gdb_lock:
            print('线程[%d] 插入数据库...' % tid)
            for item in prds:
                jdb.db_insert_product(item)
            for item in no_prds:
                jdb.db_insert_no_product(item)