示例#1
0
class ArmaniBag():
    '''A class of armani bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = 'http://www.armani.cn'
        self.price_url = ''
        self.refers = None

        # 品牌type
        self.brand_type = 'armani'

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self):
        tab_list = [
            ("giorgio armani",
             "http://www.armani.cn/cn/giorgioarmani/%E5%A5%B3%E5%A3%AB/onlinestore/%E5%8C%85%E8%A2%8B"
             ),
            ("emporio armani",
             "http://www.armani.cn/cn/emporioarmani/%E5%A5%B3%E5%A3%AB/onlinestore/%E5%8C%85%E8%A2%8B"
             ),
            ("armani jeans",
             "http://www.armani.cn/cn/armanijeans/%E5%A5%B3%E5%A3%AB/onlinestore/%E5%8C%85%E8%A2%8B"
             )
        ]
        for tab in tab_list:
            tab_name, tab_data_url = tab
            print '# tab:', tab_name, tab_data_url
            tab_page = self.crawler.getData(tab_data_url, self.home_url)

            p = re.compile(
                r'<div class="item hproduct".+?>.+?<a href="(.+?)".+?class="url">\s*<div class="hproductPhotoCont">\s*<img.+?(src|data-original)="(.+?)".*?/>\s*</div>\s*</a>\s*<div class="itemDesc">\s*<a.+?>\s*<h3.+?>(.+?)</h3>\s*</a>.+?<div class="itemPrice">.+?<span class="prezzoProdottoSaldo".*?>(.+?)</span>\s*</div>.+?</div>',
                flags=re.S)
            for item in p.finditer(tab_page):
                i_url, i_img, i_name, s_price = self.home_url + item.group(
                    1), item.group(2), item.group(4).strip(), item.group(5)
                print i_url, i_img, i_name, s_price
                i_unit = ""
                if s_price.find("¥") != -1:
                    i_unit = "CNY"
                i_price = s_price.replace('¥', '').strip()
                if i_url and i_url != '':
                    self.link_list.append((tab_name, tab_data_url, i_name,
                                           i_url, i_img, i_price, i_unit))
                else:
                    i = BagItem(self.brand_type)
                    i.initItem(tab_name, '', i_name, i_price, i_unit, '',
                               i_url, i_img)
                    self.items.append(i.outItem())

    def bagItems(self):
        #for link in self.link_list:
        #    self.itemPage(link)
        #    break
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type,
                                  len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        m = re.search(r'<h2 class="productName">(.+?)</h2>', page, flags=re.S)
        if m:
            i_name = m.group(1).strip()

        m = re.search(
            r'<div id="zoomImageWrapper">\s*<img.+?src="(.+?)".*?/>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_img = m.group(1)
        else:
            m = re.search(
                r'<div id="thumbsWrapper">.+?<div class="thumbElement".+?>\s*<img.+?src="(.+?)".*?/>\s*</div>',
                page,
                flags=re.S)
            if m:
                i_img = m.group(1)

        m = re.search(
            r'<span class="currency">(.+?)</span>.*?<span class="priceValue">(.+?)</span>',
            page,
            flags=re.S)
        if m:
            currency, i_price = m.group(1), re.sub(r'<.*>', '', m.group(2))
            if currency.find("¥") != -1:
                i_unit = "CNY"
            else:
                i_unit = currency

        m = re.search(r'<div class="attributes">(.+?)</div>', page, flags=re.S)
        if m:
            size_str = re.sub(r'<.*?>', '', m.group(1))
            #i_size = "".join(size_str.split())
            i_size = re.sub(r'\s*', '', size_str)
            print "".join(i_size.split())

        i_number = ''
        m = re.search(
            r' <div class="modelFabricColorWrapper">\s*<div class="inner".*?>\s*<span class="modelTitle">.+?</span>.+?<span.*?class="value">(.+?)</span>\s*</div>\s*</div>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_number = m.group(1)

        i = BagItem()
        i.initItem(serie_title, '', i_name, i_price, i_unit, i_size, i_url,
                   i_img, i_number)
        self.items.append(i.outItem)
        #print '# itemPage :', serie_title, i_name, i_price, i_unit, i_size, i_url, i_img

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#2
0
class GivenchyBag():
    '''A class of givenchy bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://www.givenchy.com'
        self.price_url  = ''
        self.refers     = None

        # 品牌type
        self.brand_type = 'givenchy'

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        
    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return
       
        tab_list = []
        m = re.search(r'<p>女装配饰</p>\s+<ul class="menu-level-4">(.+?)</ul>', page, flags=re.S)
        if m:
            tabs_list_info = m.group(1)

            p = re.compile(r'<li class="submenu-item">\s+<a.+?href="(.+?)">\s+<span lang="fr" class="lang-fr">(.+?)</span>(.+?)</a>\s+</li>', flags=re.S)
            for tab in p.finditer(tabs_list_info):
                tab_list.append((tab.group(2)+tab.group(3).strip(),tab.group(1)))

        for tab in tab_list:
            tab_name,tab_url = tab
            print '# tab:',tab_name,tab_url
            tab_page = self.crawler.getData(tab_url, url)

            m = re.search(r'<div id="layer-1".+?>\s+<a href="(.+?)" class="layer-links">.+?</a>', tab_page, flags=re.S)
            if m:
                ajax_url = self.home_url + m.group(1) + "?ajax=true&fragment=true"
                ajax_data = self.crawler.getData(ajax_url, tab_url)

                if ajax_data:
                    #data = json.loads(ajax_data)
                    #if data and data.has_key("html"):
                    #    print data["html"].decode("unicode-escape")
                    r_data = ajax_data.decode("unicode-escape")
                    if r_data:
                        m = re.search(r'"html":"(.+?)"}', r_data, flags=re.S)
                        if m:
                            data_html = m.group(1).replace("\/","/")
                            #print data_html
                            #break
                            p = re.compile(r'<li class="lookbook-item line" data-idlook="\d+">\s+<div class="disp-n">.+?<div class="look-info article">\s+<p>(.+?)</p>.+?<p class="look-ref">(.+?)</p>.+?</div>.+?</div>\s+<a href="(.+?)".+?>.+?<img .+?data-src="(.+?)".*?/>\s+</li>', flags=re.S)
                            for item in p.finditer(data_html):
                                i_url, i_img, s_number, i_name = self.home_url+item.group(3), item.group(4), item.group(2), re.sub(r'<.+?>','',item.group(1)).strip()
                                i_number = ''
                                m = re.search(r'<span class="look-ref-sku">\s*<span.+?>(.+?)</span>\s*</span>', s_number, flags=re.S)
                                if m:
                                    i_number = m.group(1)
                                print i_url, i_img, i_name, i_number
                                if Common.isBag(i_name):
                                    self.link_list.append((tab_name, tab_url, i_name, i_url, i_img, i_number))
                                    #i = BagItem(self.home_url, self.brand_type)
                                    #i.initItem(tab_name, '', i_name, '', '', '', i_url, i_img, i_number)
                                    #self.items.append(i.outItem())
                
    def bagItems(self):
        #for link in self.link_list: self.itemPage(link)
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#3
0
class DolcegabbanaBag():
    '''A class of dolcegabbana bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://www.dolcegabbana.com.cn'
        self.price_url  = ''
        self.refers     = None

        # 品牌type
        self.brand_type = 'dolcegabbana'

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        
    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        serie_title = '包袋'
        p = re.compile(r'<li data-position="\d+\s*" class="product isAvailable".+?data-category="(.+?)".+?>\s*<div class="prodContent"><div class="imagesContainer".+?>.+?<img.+?data-original="(.+?)".+?>.+?</div>\s*<div class="\s*productDescription\s*">\s*<a href="(.+?)".+?><h2.+?>(.+?)</h2>\s*</a>\s*<div class="price">.+?<span class="currency">(.+?)</span>.*?<span class="priceValue">(.+?)</span>.+?</li>', flags=re.S)
        for item in p.finditer(page):
            tab_name, i_img, i_url, i_name, s_unit, s_price = item.group(1).strip(),item.group(2),item.group(3),item.group(4).strip(),item.group(5),item.group(6)
            i_unit = ""
            if s_unit.find("¥") != -1:
                i_unit = "CNY"
            i_price = re.sub(r'<.+?>','',s_price).strip()
            print tab_name, i_img, self.home_url+i_url, i_name, i_unit, i_price
            
            if i_url and i_url != '':
                self.link_list.append((serie_title,tab_name,url,i_name,self.home_url+i_url,i_img,i_price,i_unit))
            else:
                i = BagItem(self.brand_type)
                i.initItem(serie_title, tab_name, i_name, i_price, i_unit, '', i_url, i_img)
                self.items.append(i.outItem())
        page_num = 2
        ajax_url = "http://www.dolcegabbana.com.cn/yeti/api/DOLCEEGABBANA_CN/searchIndented.json?page=2&sortRule=PriorityDescending&format=full&authorlocalized=&macro=1147&micro=&color=&look=&size=&gender=D&season=P%2CE&department=&brand=&heel=&heeltype=&wedge=&washtype=&washcode=&colortype=&fabric=&waist=&family=&structure=&environment=&author=&textSearch=&minPrice=&maxPrice=&occasion=&salesline=&prints=&stone=&material=&agerange=&productsPerPage=20&gallery=&macroMarchio=&modelnames=&GroupBy=&style=&site=DOLCEEGABBANA&baseurl=http://www.dolcegabbana.com.cn/searchresult.asp"
        a_url = re.sub('page=\d+&', 'page=%d&'%page_num, ajax_url)
        a_page = self.crawler.getData(a_url, url)
        result = self.ajax_item(a_page, url)
        while result: 
            page_num += 1
            a_url = re.sub('page=\d+&', 'page=%d&'%page_num, ajax_url)
            a_page = self.crawler.getData(a_url, url)
            result = self.ajax_item(a_page, url)

    def ajax_item(self, page, refers):
        if not page or page == '': return False
        try:
            result = json.loads(page)
            if result.has_key("ApiResult"):
                r_ApiResult = result["ApiResult"]
                if r_ApiResult.has_key("Items"):
                    for item in r_ApiResult["Items"]:
                        tab_name, i_img, i_url, i_name, i_price = "", "", "", "", ""
                        if item.has_key("MicroCategory"):
                            tab_name = item["MicroCategory"].strip()
                        if item.has_key("DefaultCode10"):
                            item_code10 = item["DefaultCode10"]
                            if item.has_key("ImageTypes"):
                                if "12_f" in item["ImageTypes"]:
                                    i_img = "http://cdn.yoox.biz/55/%s_%s.jpg"%(item_code10,"12_f")
                                else:
                                    i_img = "http://cdn.yoox.biz/55/%s_%s.jpg"%(item_code10,max(item["ImageTypes"]))
                        if item.has_key("SingleSelectLink"):
                            i_url = self.home_url + item["SingleSelectLink"].strip()
                        if item.has_key("TitleAttribute"):
                            i_name = item["TitleAttribute"].strip()
                        if item.has_key("FullPrice"):
                            i_price = '{0:,}'.format(int(item["FullPrice"]))
                        i_unit = "CNY"
                        print tab_name,i_name,i_url,i_img,i_price,i_unit
                        if i_url and i_url != '':
                            self.link_list.append((tab_name,refers,i_name,i_url,i_img,i_price,i_unit))
                        else:
                            i = BagItem(self.brand_type)
                            i.initItem('', tab_name, i_name, i_price, i_unit, '', i_url, i_img)
                            self.items.append(i.outItem())
            if result.has_key("Page"):
                r_Page = result["Page"]
                if r_Page.has_key("CurrentSearchPage") and r_Page.has_key("TotalPages"):
                    if int(r_Page["CurrentSearchPage"]) < int(r_Page["TotalPages"]):
                        return True
            return False
        except Exception as e:
            print e
            return False

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        item_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return
        
        m = re.search(r'<div id="itemTechSheet">\s*<h1>(.+?)</h1>', page, flags=re.S)
        if m:
            i_name = m.group(1).strip()

        m = re.search(r'<div id="itemTechSheet">.+?<div class="price">(.+?)</div>', page, flags=re.S)
        if m:
            s_price = m.group(1).strip()
            if s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = re.sub(r'<.+?>','',s_price).replace('¥','').strip()
            
        m = re.search(r'<div id="itemImagesBox".*?>\s*<img.+?class="mainImage" src="(.+?)">', page, flags=re.S)
        if m:
            i_img = m.group(1)

        i_size = ''
        m = re.search(r'<div class="scrollCnt">\s*<ul>.+?<li>(尺寸.+?)</li>', page, flags=re.S)
        if m:
            i_size = m.group(1)
        else:
            m = re.search(r'<div class="scrollCnt">\s*<ul>.+?<li>(尺码.+?)</li>', page, flags=re.S)
            if m:
                i_size = m.group(1)

        i_number = ''
        m = re.search(r'<div id="itemTechSheet">.+?<p class="prodCode">(.+?)</p>', page, flags=re.S)
        if m:
            i_number = m.group(1).split(':')[1].strip()

        i = BagItem(self.brand_type)
        i.initItem('', item_title, i_name, i_price, i_unit, i_size, i_url, i_img, i_number)
        print '# itemPage:',i.outItem()
        #self.items.append(i.outItem()) 

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#4
0
文件: TMCrawler.py 项目: xzhoutxd/tb
class TMCrawler():
    '''A class of TMall shop'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # db
        self.mysqlAccess = MysqlAccess()  # mysql access

        # 品牌官网链接
        self.home_url = 'http://www.taobao.com'
        self.refers = None

        # 抓取商品列表
        self.link_list = []
        self.items = []

        self.begin_time = Common.now()

    def getPage(self, url, shop_home_url):
        position = 1
        i = 1
        max_page = 0

        asyn_url = ''
        i_url = url
        refers = shop_home_url
        result_s = self.get_asyn_data(i_url, refers, shop_home_url)
        m = re.search(r'<b class="ui-page-s-len">\d+/(\d+)</b>',
                      result_s,
                      flags=re.S)
        if m:
            max_page = int(m.group(1))
        print '# page num:', max_page
        while i <= max_page:
            m = re.search(
                r'<div class="J_TItems">(.+?)<div class="pagination">',
                result_s,
                flags=re.S)
            if m:
                items_s = m.group(1)
                p = re.compile(
                    r'<dl class=".+?".+?data-id="(.+?)">.+?<dd class="detail">\s*<a class="item-name".+?href="(.+?)".+?>(.+?)</a>\s*<div class="attribute">\s*<div class="cprice-area">\s*<span class="symbol">(.+?)</span>\s*<span\s*class="c-price">(.+?)</span>\s*</div>.+?</dl>'
                )
                j = 1
                for item in p.finditer(items_s):
                    item_id, url_s, item_name, price_symbol, price = item.group(
                        1), item.group(2), Common.htmlDecode(
                            item.group(3).strip()), item.group(
                                4).strip(), item.group(5).strip()
                    if url_s.find('http') == -1:
                        item_url = 'http:' + url_s
                    else:
                        item_url = url_s
                    print '### item ###'
                    print '# item val:', item_id, item_name, price, item_url
                    item = Item()
                    item.parserTM((item_id, item_name, price, item_url, i_url,
                                   self.begin_time))
                    print '# item info:', item.outItemSql()
                    self.mysqlAccess.insert_parser_item_info(item.outItemSql())
                    time.sleep(2)

            refers = i_url
            if i_url.find('pageNo=') == -1:
                i_url = re.sub(r'&tsearch=y',
                               '&pageNo=%d&tsearch=y#anchor' % i, refers)
            else:
                i_url = re.sub(r'&pageNo=\d+&', '&pageNo=%d&' % i, refers)

            i += 1
            time.sleep(2)
            result_s = self.get_asyn_data(i_url, refers, shop_home_url)

    def get_asyn_data(self, i_url, refers, shop_home_url):
        result = ''
        result_s = ''
        page = self.crawler.getData(i_url, refers)
        m = re.search(r'<input id="J_ShopAsynSearchURL".+?value="(.+?)"\s*/>',
                      page,
                      flags=re.S)
        if m:
            ts = '?_ksTS=%s&callback=jsonp135&' % (str(int(
                time.time() * 1000)) + '_' + str(random.randint(100, 999)))
            a_url = shop_home_url + Common.htmlDecode(m.group(1))
            asyn_url = re.sub('\?', ts, a_url)
            result = self.crawler.getData(asyn_url, i_url)
            m = re.search(r'jsonp135\("(.+?)"\)', result, flags=re.S)
            if m:
                result_s = re.sub(r'\\"', '"', m.group(1))
        return result_s

    def getItems(self):
        #for link in self.link_list: self.itemPage(link)
        max_th = 10
        #if len(self.link_list) > max_th:
        #    m_itemsObj = BagItemM(self.home_url,self.brand_type, max_th)
        #else:
        #    m_itemsObj = BagItemM(self.home_url,self.brand_type, len(self.link_list))
        #m_itemsObj.createthread()
        #m_itemsObj.putItems(self.link_list)
        #m_itemsObj.run()
        #self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        print '# itemPage :', serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img
示例#5
0
class BottegavenetaBag():
    '''A class of bottegaveneta bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://www.bottegaveneta.com'
        self.price_url  = ''
        self.refers     = None

        # 品牌type
        self.brand_type = 'bottegaveneta'

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        
    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        p = re.compile(r'<div id="slot_\d+".+?<a.+?href="(.+?)".+?>\s*<img.+?src="(.+?)".+?/>\s*<div class="iteminfo">\s*<div class="headgroup">\s*<div class="extra">\s*<h1 class="modelname">(.+?)</h1>', flags=re.S)
        for item in p.finditer(page):
            i_url, i_img, i_name = item.group(1),item.group(2),item.group(3)
            print i_url, i_img, i_name
            if i_url and i_url != '':
                self.link_list.append(('',url,i_name,i_url,i_img))
            else:
                i = BagItem(self.brand_type)
                i.initItem('',url,i_name,i_url,i_img)
                self.items.append(i.outItem())

        p = re.compile(r'<div class="slot lazySlot".+?data-slot="(.+?)".+?>',flags=re.S)
        for item in p.finditer(page):
            data_info = item.group(1)
            data_info_str = data_info.replace('&quot;','"')
            i_url, i_img, i_name = '', '', ''
            m = re.search(r'"Link":"(.+?)"', data_info_str, flags=re.S)
            if m:
                i_url = m.group(1)

            m = re.search(r'"ModelName":"(.+?)",', data_info_str, flags=re.S)
            if m:
                i_name = m.group(1)

            print i_url, i_img, i_name
            if i_url and i_url != '':
                self.link_list.append(('',url,i_name,i_url,i_img))
            else:
                i = BagItem(self.brand_type)
                i.initItem('',url,i_name,i_url,i_img)
                self.items.append(i.outItem())
                
    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """

        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return
        
        m = re.search(r'<h1 class="producTitle".+?>\s*<div class="modelName".+?>\s*<span class="modelName">(.+?)</span>', page, flags=re.S)
        if m:
            i_name = m.group(1).strip()

        m = re.search(r'<div class="mainImage".+?>\s*<img.+?src="(.+?)".*?/>\s*</div>', page, flags=re.S)
        if m:
            i_img = m.group(1)
        else:
            m = re.search(r'<section id="bgItem">\s*<img.+?src="(.+?)".*?/>\s*</section>', page, flags=re.S)
            if m:
                i_img = m.group(1)
        
        i_size = ''
        m = re.search(r'<div class="localizedAttributes">.*?<div class="height">.+?<span class="text">(.+?)</span>\s*<span class="value">(.+?)</span>\s*</div>', page, flags=re.S)
        if m:
            i_size += m.group(1) + ":" + m.group(2) + ";"
        m = re.search(r'<div class="localizedAttributes">.*?<div class="depth">.+?<span class="text">(.+?)</span>\s*<span class="value">(.+?)</span>\s*</div>', page, flags=re.S)
        if m:
            i_size += m.group(1) + ":" + m.group(2) + ";"
        m = re.search(r'<div class="localizedAttributes">.*?<div class="length_of_strap">.+?<span class="text">(.+?)</span>\s*<span class="value">(.+?)</span>\s*</div>', page, flags=re.S)
        if m:
            i_size += m.group(1) + ":" + m.group(2) + ";"
        m = re.search(r'<div class="localizedAttributes">.*?<div class="width">.+?<span class="text">(.+?)</span>\s*<span class="value">(.+?)</span>\s*</div>\s*</div>', page, flags=re.S)
        if m:
            i_size += m.group(1) + ":" + m.group(2) + ";"

        i_number
        m = re.search(r' <div class="modelFabricColorWrapper">\s*<div class="inner".*?>\s*<span class="modelTitle">.+?</span>.+?<span.*?class="value">(.+?)</span>\s*</div>\s*</div>\s*</div>', page, flags=re.S)
        if m:
            i_number = m.group(1)

        i = BagItem(self.brand_type)
        i.initItem(serie_title, '', i_name, '', '', i_size, i_url, i_img, i_number)
        print '# itemPage:',i.outItem()
        #self.items.append(i.outItem())    
        #print '# itemPage :', serie_title, i_name, i_url, i_img, i_size

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#6
0
class BossBag():
    '''A class of boss bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://store.hugoboss.cn'
        self.price_url  = ''
        self.refers     = None

        # 品牌type
        self.brand_type = 'boss'

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        
    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        tab_name = '手袋'
        self.parse_item(page,tab_name,url)

    def parse_item(self, tab_page, tab_name, tab_url):
        items_info = ''
        m = re.search(r'<div class="productlist-widget">.+?<div class="container">\s*<ul.+?>(.+?)</ul>', tab_page, flags=re.S)
        if m:
            items_info = m.group(1)
            self.run_items(items_info, tab_name, tab_url)
 
    def run_items(self, items_info, tab_name, tab_url):
        p = re.compile(r'<li class="productlist-item ">\s*<div class="product-image".+?>\s*<a.+?><img src="(.+?)".+?/>\s*</a>\s*</div>.+?<div class="product-title">\s*<a href="(.+?)".+?>(.+?)</a>\s*</div>.+?<p>\s*<span class="product-price">(.+?)</span>\s*</p>\s*</li>', flags=re.S)
        for item in p.finditer(items_info):
            i_img, i_url, i_name, s_price = item.group(1),item.group(2),item.group(3),item.group(4)
            i_price, i_unit = '', ''
            if s_price.find("¥") != -1 or s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = s_price.replace('¥','').replace('¥','').strip()
            
            if i_url and i_url != '':
                print self.home_url+i_url, i_img, i_name, i_price, i_unit
                self.link_list.append((tab_name,tab_url,i_name,self.home_url+i_url,i_img,i_price,i_unit))
            else:
                print self.home_url+i_url, i_img, i_name, i_price, i_unit
                i = BagItem(self.brand_type)
                i.initItem(tab_name, '', i_name, i_price, i_unit, '', i_url, i_img)
                self.items.append(i.outItem())

    def isBag(self,name):
        bag_info = ["包","袋","皮夹","钱夹"]
        other_info = ["裤","T恤","衬衫","礼服","上衣"]
        for b_info in bag_info:
            if name.find(b_info) != -1:
                for o_info in other_info:
                    if name.find(o_info) != -1:
                        return False
                return True
        return False
        

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        m = re.search(r'<h1 class="product-name">(.+?)</h1>', page, flags=re.S)
        if m:
            i_name = ' '.join(m.group(1).strip().split())

        m = re.search(r'<div class="product-prices">.+?<dd class="saleprice">(.+?)</dd>.+?</div>', page, flags=re.S)
        if m:
            s_price = m.group(1).strip()
            if s_price.find("¥") != -1 or s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = s_price.replace('¥','').replace('¥','').strip()
    
        m = re.search(r'<div class="image">.+?<div class="container".*?>\s*<table.+?>\s*<tr>\s*<td>\s*<a.+?>\s*<img.+?class="thumb".+?big="(.+?)".*?/>\s*</a>\s*</td>', page, flags=re.S)
        if m:
            i_img = m.group(1)

        i_size = ''
        m = re.search(r'<div class="tabpage inc".+?>.+?<span.*?>(尺寸大小.+?)</span>', page, flags=re.S)
        if m:
            s_size = m.group(1)
            i_size = s_size.split(':')[1]
        if i_size == '':
            m = re.search(r'<span.+?>尺寸大小:</span>(.+?)</span>', page, flags=re.S)
            if m:
                i_size = re.sub(r'<.+?>','',m.group(1))

        i_number = ''
        m = re.search(r'<div class="base">\s*<div class="sku-brand">.+?<dl class="hidden"><dt>商品货号: </dt><dd>(.+?)</dd></dl>\s*</div>', page, flags=re.S)
        if m:
            i_number = m.group(1)

        i = BagItem(self.brand_type)
        i.initItem(serie_title, '', i_name, i_price, i_unit, i_size, i_url, i_img, i_number)
        print '# itemPage:',i.outItem()
        #self.items.append(i.outItem()) 

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#7
0
class Item():
    '''A class of item'''
    def __init__(self):
        # crawler
        #self.crawler         = TBCrawler()
        self.crawler = RetryCrawler()

        # shop
        self.shop_type = '1'  # 店铺类型
        self.seller_id = ''  # 卖家ID
        self.seller_name = ''  # 卖家Name
        self.shop_id = ''  # 店铺ID
        self.shop_name = ''  # 店铺Name
        self.shop_url = ''  # 店铺URL

        # refers
        self.refers = ''

        # 成交抓取参数
        self.deal_pageSize = 15
        self.deal_maxPages = 100
        self.deal_bufferdays = 3  # 往前追溯3天

        # 评价抓取参数
        self.rate_pageSize = 20
        self.rate_maxPages = 100

        # 初始化实例变量
        self.initItem()

    def initItem(self):
        # 商品抓取设置
        self.crawling_time = Common.now()
        self.crawling_begintime = ''  # 本次抓取开始时间
        self.crawling_beginDate = ''  # 本次爬取日期
        self.crawling_beginHour = ''  # 本次爬取小时

        # 商品属性
        self.item_id = ''  # 商品ID
        self.item_name = ''  # 商品名称
        self.item_price = ''  # 商品价格
        self.item_url = ''  # 商品链接
        self.item_spuId = ''  # SPU ID
        self.item_sellCount = 0  # 月销售数

        self.brand_name = ''
        self.brand_id = ''
        self.category_id = ''

        # 商品页
        self.item_page = None  # 商品首页

        # item html urls
        self.item_urls = []  # 商品链接列表

        # item html pages
        #self.item_pages      = []   # 商品网页列表
        self.item_pages = {}  # 商品网页列表

        # 成交记录
        self.deal_url = ''
        self.deal_stopCrawl = False
        self.deal_deadLine = 0.0  # 上次抓取的成交记录最晚时间
        self.deal_deadLine2 = 0.0  # 本次抓取的成交记录最早时间

    def TMItem(self):
        if self.item_url != '':
            page = self.crawler.getData(self.item_url, self.refers)
            if not page or page == '':
                raise Common.InvalidPageException(
                    "# TMItem: not find item page,itemid:%s,item_url:%s" %
                    (str(self.item_id), self.item_url))

            m = re.search(r'sellerId:"(\d+)",', page, flags=re.S)
            if m:
                self.seller_id = m.group(1)
            m = re.search(r'shopId:"(\d+)",', page, flags=re.S)
            if m:
                self.shop_id = m.group(1)
            m = re.search(
                r'<div class="slogo">\s*<a class="slogo-shopname" href="(.+?)".+?><strong>(.+?)</strong></a>',
                page,
                flags=re.S)
            if m:
                self.shop_url, self.shop_name = Common.fix_url(
                    m.group(1)), m.group(2).strip()

            m = re.search(r'TShop\.Setup\((.+?)\);', page, flags=re.S)
            if m:
                TShop_s = m.group(1).strip()
                m = re.search(r'"brand":"(.+?)",', TShop_s, flags=re.S)
                if m:
                    self.brand_name = Common.htmlDecode(m.group(1).strip())
                m = re.search(r'"brandId":"(\d+)",', TShop_s, flags=re.S)
                if m:
                    self.brand_id = m.group(1)
                m = re.search(r'"categoryId":"(\d+)",', TShop_s, flags=re.S)
                if m:
                    self.category_id = m.group(1)
                m = re.search(r'"sellerNickName":"(.+?)",',
                              TShop_s,
                              flags=re.S)
                if m:
                    self.seller_name = Common.urlDecode(m.group(1).strip())

                m = re.search(r'"initApi":"(.+?)",', TShop_s, flags=re.S)
                if m:
                    ts = "&callback=setMdskip&timestamp=%s" % str(
                        int(time.time() * 1000))
                    initapi_url = Common.fix_url(m.group(1).strip(
                    )) + ts + "&ref=%s" % Common.urlCode(self.refers)
                    init_page = self.crawler.getData(initapi_url,
                                                     self.item_url)
                    if not init_page and init_page == '':
                        print '# init page is null..'
                    else:
                        m = re.search(r'"sellCountDO":{"sellCount":(\d+),',
                                      init_page,
                                      flags=re.S)
                        if m:
                            self.item_sellCount = m.group(1)

    def parserTM(self, val):
        self.item_id, self.item_name, self.item_price, self.item_url, self.refers, self.crawling_begintime = val
        # 本次抓取开始日期
        self.crawling_beginDate = time.strftime(
            "%Y-%m-%d", time.localtime(self.crawling_begintime))
        # 本次抓取开始小时
        self.crawling_beginHour = time.strftime(
            "%H", time.localtime(self.crawling_begintime))
        self.TMItem()

    # 输出抓取的网页sql
    def outItemSql(self):
        return (Common.time_s(self.crawling_time), self.item_id,
                self.item_name, self.item_price, self.item_sellCount,
                self.item_url, self.seller_id, self.seller_name, self.shop_id,
                self.shop_name, self.shop_url, self.brand_id, self.brand_name,
                self.category_id, self.crawling_beginDate,
                self.crawling_beginHour)
示例#8
0
class DolcegabbanaBag():
    '''A class of dolcegabbana bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = 'http://www.dolcegabbana.com.cn'
        self.price_url = ''
        self.refers = None

        # 品牌type
        self.brand_type = 'dolcegabbana'

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        serie_title = '包袋'
        p = re.compile(
            r'<li data-position="\d+\s*" class="product isAvailable".+?data-category="(.+?)".+?>\s*<div class="prodContent"><div class="imagesContainer".+?>.+?<img.+?data-original="(.+?)".+?>.+?</div>\s*<div class="\s*productDescription\s*">\s*<a href="(.+?)".+?><h2.+?>(.+?)</h2>\s*</a>\s*<div class="price">.+?<span class="currency">(.+?)</span>.*?<span class="priceValue">(.+?)</span>.+?</li>',
            flags=re.S)
        for item in p.finditer(page):
            tab_name, i_img, i_url, i_name, s_unit, s_price = item.group(
                1).strip(), item.group(2), item.group(3), item.group(
                    4).strip(), item.group(5), item.group(6)
            i_unit = ""
            if s_unit.find("¥") != -1:
                i_unit = "CNY"
            i_price = re.sub(r'<.+?>', '', s_price).strip()
            print tab_name, i_img, self.home_url + i_url, i_name, i_unit, i_price

            if i_url and i_url != '':
                self.link_list.append(
                    (serie_title, tab_name, url, i_name, self.home_url + i_url,
                     i_img, i_price, i_unit))
            else:
                i = BagItem(self.brand_type)
                i.initItem(serie_title, tab_name, i_name, i_price, i_unit, '',
                           i_url, i_img)
                self.items.append(i.outItem())
        page_num = 2
        ajax_url = "http://www.dolcegabbana.com.cn/yeti/api/DOLCEEGABBANA_CN/searchIndented.json?page=2&sortRule=PriorityDescending&format=full&authorlocalized=&macro=1147&micro=&color=&look=&size=&gender=D&season=P%2CE&department=&brand=&heel=&heeltype=&wedge=&washtype=&washcode=&colortype=&fabric=&waist=&family=&structure=&environment=&author=&textSearch=&minPrice=&maxPrice=&occasion=&salesline=&prints=&stone=&material=&agerange=&productsPerPage=20&gallery=&macroMarchio=&modelnames=&GroupBy=&style=&site=DOLCEEGABBANA&baseurl=http://www.dolcegabbana.com.cn/searchresult.asp"
        a_url = re.sub('page=\d+&', 'page=%d&' % page_num, ajax_url)
        a_page = self.crawler.getData(a_url, url)
        result = self.ajax_item(a_page, url)
        while result:
            page_num += 1
            a_url = re.sub('page=\d+&', 'page=%d&' % page_num, ajax_url)
            a_page = self.crawler.getData(a_url, url)
            result = self.ajax_item(a_page, url)

    def ajax_item(self, page, refers):
        if not page or page == '': return False
        try:
            result = json.loads(page)
            if result.has_key("ApiResult"):
                r_ApiResult = result["ApiResult"]
                if r_ApiResult.has_key("Items"):
                    for item in r_ApiResult["Items"]:
                        tab_name, i_img, i_url, i_name, i_price = "", "", "", "", ""
                        if item.has_key("MicroCategory"):
                            tab_name = item["MicroCategory"].strip()
                        if item.has_key("DefaultCode10"):
                            item_code10 = item["DefaultCode10"]
                            if item.has_key("ImageTypes"):
                                if "12_f" in item["ImageTypes"]:
                                    i_img = "http://cdn.yoox.biz/55/%s_%s.jpg" % (
                                        item_code10, "12_f")
                                else:
                                    i_img = "http://cdn.yoox.biz/55/%s_%s.jpg" % (
                                        item_code10, max(item["ImageTypes"]))
                        if item.has_key("SingleSelectLink"):
                            i_url = self.home_url + item[
                                "SingleSelectLink"].strip()
                        if item.has_key("TitleAttribute"):
                            i_name = item["TitleAttribute"].strip()
                        if item.has_key("FullPrice"):
                            i_price = '{0:,}'.format(int(item["FullPrice"]))
                        i_unit = "CNY"
                        print tab_name, i_name, i_url, i_img, i_price, i_unit
                        if i_url and i_url != '':
                            self.link_list.append(
                                (tab_name, refers, i_name, i_url, i_img,
                                 i_price, i_unit))
                        else:
                            i = BagItem(self.brand_type)
                            i.initItem('', tab_name, i_name, i_price, i_unit,
                                       '', i_url, i_img)
                            self.items.append(i.outItem())
            if result.has_key("Page"):
                r_Page = result["Page"]
                if r_Page.has_key("CurrentSearchPage") and r_Page.has_key(
                        "TotalPages"):
                    if int(r_Page["CurrentSearchPage"]) < int(
                            r_Page["TotalPages"]):
                        return True
            return False
        except Exception as e:
            print e
            return False

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type,
                                  len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        item_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        m = re.search(r'<div id="itemTechSheet">\s*<h1>(.+?)</h1>',
                      page,
                      flags=re.S)
        if m:
            i_name = m.group(1).strip()

        m = re.search(
            r'<div id="itemTechSheet">.+?<div class="price">(.+?)</div>',
            page,
            flags=re.S)
        if m:
            s_price = m.group(1).strip()
            if s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = re.sub(r'<.+?>', '', s_price).replace('¥', '').strip()

        m = re.search(
            r'<div id="itemImagesBox".*?>\s*<img.+?class="mainImage" src="(.+?)">',
            page,
            flags=re.S)
        if m:
            i_img = m.group(1)

        i_size = ''
        m = re.search(r'<div class="scrollCnt">\s*<ul>.+?<li>(尺寸.+?)</li>',
                      page,
                      flags=re.S)
        if m:
            i_size = m.group(1)
        else:
            m = re.search(r'<div class="scrollCnt">\s*<ul>.+?<li>(尺码.+?)</li>',
                          page,
                          flags=re.S)
            if m:
                i_size = m.group(1)

        i_number = ''
        m = re.search(
            r'<div id="itemTechSheet">.+?<p class="prodCode">(.+?)</p>',
            page,
            flags=re.S)
        if m:
            i_number = m.group(1).split(':')[1].strip()

        i = BagItem(self.brand_type)
        i.initItem('', item_title, i_name, i_price, i_unit, i_size, i_url,
                   i_img, i_number)
        print '# itemPage:', i.outItem()
        #self.items.append(i.outItem())

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#9
0
class FerragamoBag():
    '''A class of ferragamo bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = 'http://www.ferragamo.cn'
        self.price_url = ''
        self.refers = None

        # 品牌type
        self.brand_type = 'ferragamo'

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        tab_name = '女士手袋'
        self.parse_item(page, tab_name, url)

    def parse_item(self, tab_page, tab_name, tab_url):
        m = re.search(
            r'<div class="view-content">.+?<div class="page-wrapper-product">(.+?)</script>\s*</div>\s*</div>',
            tab_page,
            flags=re.S)
        if m:
            items_info = m.group(1)
            self.run_items(items_info, tab_name, tab_url)

    def run_items(self, items_info, tab_name, tab_url):
        p = re.compile(
            r'<div class="large.+?columns.+?">\s*<a href="(.+?)">\s*<img.*?src="(.+?)".*?/><span class="prodcaption">(.+?)</br>(.+?)</span>\s*</a>',
            flags=re.S)
        for item in p.finditer(items_info):
            i_url, i_img, i_name, s_price = item.group(1), item.group(
                2), item.group(3), item.group(4)
            i_price, i_unit = '', ''
            if s_price.find("¥") != -1 or s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = s_price.replace('¥', '').replace('¥', '').strip()

            if i_url and i_url != '':
                print self.home_url + i_url, i_img, i_name, i_price, i_unit
                self.link_list.append(
                    (tab_name, tab_url, i_name, self.home_url + i_url, i_img,
                     i_price, i_unit))
            else:
                print self.home_url + i_url, i_img, i_name, i_price, i_unit
                i = BagItem(self.brand_type)
                i.initItem(tab_name, '', i_name, i_price, i_unit, '', i_url,
                           i_img)
                self.items.append(i.outItem())

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type,
                                  len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        m = re.search(r'<div class="product-title">(.+?)</div>',
                      page,
                      flags=re.S)
        if m:
            i_name = ' '.join(m.group(1).strip().split())

        m = re.search(r'<div class="product-prices">(.+?)</div>',
                      page,
                      flags=re.S)
        if m:
            s_price = m.group(1).strip()
            if s_price.find("¥") != -1 or s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = s_price.replace('¥', '').replace('¥', '').strip()

        m = re.search(
            r'<div class="item-list">\s*<ul.+?>\s*<li class="first">.+?<img.+?src="(.+?)".*?/>',
            page,
            flags=re.S)
        if m:
            i_img = m.group(1)

        i_size = ''
        i_number = ''
        m = re.search(r'<div class="product-code">(.+?)型号代码(.+?)</div>',
                      page,
                      flags=re.S)
        if m:
            i_size, i_number = m.group(1).strip(), m.group(2).strip()

        i = BagItem(self.brand_type)
        i.initItem(serie_title, '', i_name, i_price, i_unit, i_size, i_url,
                   i_img, i_number)
        print '# itemPage:', i.outItem()
        #self.items.append(i.outItem())

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#10
0
class DiorBag():
    '''A class of dior bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = 'http://www.dior.cn'
        self.price_url = ''
        self.refers = None

        # 品牌type
        self.brand_type = 'dior'

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        tab_list = []
        m = re.search(r'<ul class="tabsList collections">(.+?)</ul>',
                      page,
                      flags=re.S)
        if m:
            tabs_list_info = m.group(1)

            p = re.compile(
                r'<li class=".+?">\s+<a href="(.+?)" data-magento_call_page="(.+?)".+?>(.+?)</a>\s+</li>',
                flags=re.S)
            for tab in p.finditer(tabs_list_info):
                tab_list.append(
                    (tab.group(3).strip(), self.home_url + tab.group(2),
                     url + tab.group(1)))

        for tab in tab_list:
            tab_name, tab_data_url, tab_url = tab
            print '# tab:', tab_name, tab_data_url, tab_url
            tab_page = self.crawler.getData(tab_data_url, url)

            p = re.compile(
                r'<li class="li-product.+?>\s+<a href="(.*?)" class="linkProduct">.+?<img src="(.+?)".+?/>.+?<span class="description".+?>.+?<span class="title">(.+?)</span>.+?</span>\s+</a>\s+</li>',
                flags=re.S)
            for item in p.finditer(tab_page):
                i_url, i_img, i_name = self.home_url + item.group(
                    1), self.home_url + item.group(2), item.group(3)
                print i_url, i_img, i_name
                if i_url and i_url != '':
                    self.link_list.append(
                        (tab_name, tab_url, i_name, i_url, i_img))
                else:
                    i = BagItem(self.brand_type)
                    i.initItem(tab_name, '', i_name, '', '', '', i_url,
                               self.home_url + i_img)
                    self.items.append(i.outItem())

    def bagItems(self):
        #for link in self.link_list: self.itemPage(link)
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type,
                                  len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url = val

        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        i_title, i_img, i_size, i_price, i_unit = '', '', '', '', ''

        m = re.search(r'<h2 class="" itemprop="name">(.+?)<br />(.+?)</h2>',
                      page,
                      flags=re.S)
        if m:
            i_title = m.group(1).strip()

        m = re.search(
            r'<li class="firstThumbnails">\s+<a href="#" class="active".+?>\s+<img src="(.+?)" alt="" />\s+</a>\s+</li>',
            page,
            flags=re.S)
        if m: i_img = self.home_url + m.group(1)

        m = re.search(
            r'<div class="modText">\s+<h4.+?>说明</h4>\s+<p>(.+?)</p>\s+</div>',
            page,
            flags=re.S)
        if m:
            i_desc = m.group(1)
            m = re.search(r'尺寸:(.+?)<br />', i_desc, flags=re.S)
            if m:
                i_size = m.group(1).strip()
            else:
                m = re.search(r'尺寸:(.+?)$', i_desc, flags=re.S)
                if m: i_size = m.group(1).strip()

        i_number = ''
        m = re.search(
            r'<div class="columns-wrapper">.+?<div class="column">.*?<div class="reference">\s*<p>(.+?)</p>\s*</div>',
            page,
            flags=re.S)
        if m:
            s_number = m.group(1)
            i_number = s_number.split('-')[1].strip()

        i = BagItem()
        i.initItem(serie_title, i_title, i_name, i_price, i_unit, i_size,
                   i_url, i_img, i_number)
        self.items.append(i.outItem)
        print '# itemPage :', serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#11
0
class FerragamoBag:
    """A class of ferragamo bag"""

    def __init__(self):
        # 抓取设置
        # self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = "http://www.ferragamo.cn"
        self.price_url = ""
        self.refers = None

        # 品牌type
        self.brand_type = "ferragamo"

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == "":
            return

        tab_name = "女士手袋"
        self.parse_item(page, tab_name, url)

    def parse_item(self, tab_page, tab_name, tab_url):
        m = re.search(
            r'<div class="view-content">.+?<div class="page-wrapper-product">(.+?)</script>\s*</div>\s*</div>',
            tab_page,
            flags=re.S,
        )
        if m:
            items_info = m.group(1)
            self.run_items(items_info, tab_name, tab_url)

    def run_items(self, items_info, tab_name, tab_url):
        p = re.compile(
            r'<div class="large.+?columns.+?">\s*<a href="(.+?)">\s*<img.*?src="(.+?)".*?/><span class="prodcaption">(.+?)</br>(.+?)</span>\s*</a>',
            flags=re.S,
        )
        for item in p.finditer(items_info):
            i_url, i_img, i_name, s_price = item.group(1), item.group(2), item.group(3), item.group(4)
            i_price, i_unit = "", ""
            if s_price.find("¥") != -1 or s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = s_price.replace("¥", "").replace("¥", "").strip()

            if i_url and i_url != "":
                print self.home_url + i_url, i_img, i_name, i_price, i_unit
                self.link_list.append((tab_name, tab_url, i_name, self.home_url + i_url, i_img, i_price, i_unit))
            else:
                print self.home_url + i_url, i_img, i_name, i_price, i_unit
                i = BagItem(self.brand_type)
                i.initItem(tab_name, "", i_name, i_price, i_unit, "", i_url, i_img)
                self.items.append(i.outItem())

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == "":
            return
        page = self.crawler.getData(i_url, refers)
        if not page or page == "":
            return

        m = re.search(r'<div class="product-title">(.+?)</div>', page, flags=re.S)
        if m:
            i_name = " ".join(m.group(1).strip().split())

        m = re.search(r'<div class="product-prices">(.+?)</div>', page, flags=re.S)
        if m:
            s_price = m.group(1).strip()
            if s_price.find("¥") != -1 or s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = s_price.replace("¥", "").replace("¥", "").strip()

        m = re.search(
            r'<div class="item-list">\s*<ul.+?>\s*<li class="first">.+?<img.+?src="(.+?)".*?/>', page, flags=re.S
        )
        if m:
            i_img = m.group(1)

        i_size = ""
        i_number = ""
        m = re.search(r'<div class="product-code">(.+?)型号代码(.+?)</div>', page, flags=re.S)
        if m:
            i_size, i_number = m.group(1).strip(), m.group(2).strip()

        i = BagItem(self.brand_type)
        i.initItem(serie_title, "", i_name, i_price, i_unit, i_size, i_url, i_img, i_number)
        print "# itemPage:", i.outItem()
        # self.items.append(i.outItem())

    def outItems(self, f):
        s = "#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号"
        with open(f, "w") as f_item:
            self.items.insert(0, s)
            f_item.write("\n".join(self.items))
示例#12
0
class taobaoSearch():
    '''A class of taobao search page'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # db
        self.mysqlAccess  = MysqlAccess() # mysql access

        # 品牌官网链接
        self.home_url   = 'http://www.taobao.com'
        self.refers     = None

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        self.brand_names = ["艾沃","保宁","红太阳","常润","厨邦","大华","大同","大王","德馨斋","德阳","东古","凤球唛","福临门","高真","古龙","冠生园","广味源","广祥泰","龟甲万","国味威","海鸥","海天","好伴","禾然","和田宽","恒顺","湖西岛","湖羊","黄花园","南食召","吉成","济美","加加","金冠园","金兰","金狮","有味家","金苏","荆楚","景山","居易","聚百鲜","科沁万佳","孔膳坊","快鹿","阆中","老才臣","食味的初相","老蔡","老恒和","老潘头","李锦记","利民","六月鲜","春峰","龙菲","秋生饭堂","龙牌","隆昌","楼茂记","鲁花自然鲜","云上","禄荣","麻旺","美富达","美极","美味源","蒙古真","渔山隐","米吉真味","酿一村","盘溪","彭万春","浦源","奇峰","千禾","千鹤屋","粮赞","钱万隆","清净园","清香园","仁昌记","三不加","悦意","三和四美","博爱酵园","山古坊","膳府","膳之堂","盛田","四海","寺冈","苏美","太太乐","泰康","唐人基","唐世家","淘大","腾安","同珍","妥甸","拓东","丸天","万通","万字","味事达","五天","犀浦","仙家","先市","鲜渡","咸亨","香满园","小东字","笑厨","新宇","星湖","徐同泰","薛泰丰","扬名","尧记","肴易食","一统原创","一休屋","伊例家","宜赤必","优和","鱼味鲜","禹谟","玉堂","御酿坊","缘木记","粤香园","灶基","詹王","张家三嫂","长寿结","珍极","正信","正阳河","至味","致美斋","中邦","中冷泉","中调","珠江桥","梓山","自然集","佐参王","佐香园","中坝","天府","南吉","清湖","味华","佐餐王","一品江南","金顶","玉河","巧媳妇","齐鲁","梁山好汉","王家园子","食圣","山口","川鹰","德通","新汶","四海","德馨斋","玉兔","灯塔","仙鹤","宏林","贵族王中王","万和","口珍","同福永","威极","嘉美乐","天浩圆","铁鸟","恒裕","周太","海鸥","太阳岛","百花","小神厨","龙菲","太和","天一","美乐","三汇","通海","黑珍珠","百乐","吉鹤村","岭桥","瓦缸","味莼园","百花串","锦酿","福香居","铁石","石桥","清华","味邦","光华","罕王","营宝","非常","大有丰","沙陀","味味晓","云晓","巧妈妈","振龙","乾缘","稻香园","一品斋","孔雀","武大郎","绿芳","天赐","益彰","建洛","天口","一品江南","机轮","溢美堂","山乡","榕江","嘉乐美","万路通","肖大妈","争荣","仙源","敬义泰","昆湖","鼎兴","临江寺","迈进","玉和","通德","民天","胡玉美","楼茂记","鼎丰","古灯","槐茂","榕城","BB","汉记","松城","森江","美狮","龙华","启航","隆邦","新汶","四海","龙之味","北康","金玉兰","小二黑","吉成"]
        
    def getPage(self, url):
        position = 1
        i = 1
       
        i_url = url
        refers = self.home_url
        max_page = 10
        size_page = 48
        while i <= max_page:
            page = self.crawler.getData(i_url, refers)
            refers = i_url
            i_url = url + '&bcoffset=1&s=%s' % str(i*size_page)
            i += 1
            if not page or page == '':
                print 'not find data url:',i_url
                time.sleep(4)
                continue
            m = re.search(r'<script>\s+g_page_config = ({.+?});.+?</script>', page, flags=re.S)
            if m:
                page_config = m.group(1)
                page_config_s = re.sub(r'\n+','',page_config)
                data = json.loads(page_config_s)
                if data.has_key("mods"):
                    if data["mods"].has_key("itemlist"):
                        itemlist = data["mods"]["itemlist"]
                        if itemlist.has_key("data"):
                            itemlist_data = itemlist["data"]
                            if itemlist_data.has_key("auctions"):
                                for item in itemlist_data["auctions"]:
                                    item_id = position
                                    m = re.search(r'id=(\d+)', item["detail_url"], flags=re.S)
                                    if m:
                                        item_id = m.group(1)
                                    item_sales = item["view_sales"]
                                    m = re.search(r'(\d+)', item["view_sales"], flags=re.S)
                                    if m:
                                        item_sales = m.group(1)
                                    print Common.time_s(Common.now()), position, item_id, item["raw_title"], item["view_price"], item_sales, item["user_id"], item["nick"], "http:" + item["detail_url"], "http:" + item["shopLink"]
                                    self.mysqlAccess.insert_item((Common.time_s(Common.now()), str(item_id), str(position), str(item["raw_title"]), str(item["view_price"]), str(item_sales), "http:" + item["detail_url"], item["user_id"], str(item["nick"]), "http:" + item["shopLink"]))
                                    position += 1
            time.sleep(4)


    def getItems(self):
        #for link in self.link_list: self.itemPage(link)
        max_th = 10
        #if len(self.link_list) > max_th:
        #    m_itemsObj = BagItemM(self.home_url,self.brand_type, max_th)
        #else:
        #    m_itemsObj = BagItemM(self.home_url,self.brand_type, len(self.link_list))
        #m_itemsObj.createthread()
        #m_itemsObj.putItems(self.link_list)
        #m_itemsObj.run()
        #self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        print '# itemPage :', serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#13
0
class ArmaniBag():
    '''A class of armani bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://www.armani.cn'
        self.price_url  = ''
        self.refers     = None

        # 品牌type
        self.brand_type = 'armani'

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        
    def bagPage(self):
        tab_list = [
                    ("giorgio armani","http://www.armani.cn/cn/giorgioarmani/%E5%A5%B3%E5%A3%AB/onlinestore/%E5%8C%85%E8%A2%8B"),
                    ("emporio armani","http://www.armani.cn/cn/emporioarmani/%E5%A5%B3%E5%A3%AB/onlinestore/%E5%8C%85%E8%A2%8B"),
                    ("armani jeans","http://www.armani.cn/cn/armanijeans/%E5%A5%B3%E5%A3%AB/onlinestore/%E5%8C%85%E8%A2%8B")]
        for tab in tab_list:
            tab_name,tab_data_url = tab
            print '# tab:',tab_name,tab_data_url
            tab_page = self.crawler.getData(tab_data_url, self.home_url)

            p = re.compile(r'<div class="item hproduct".+?>.+?<a href="(.+?)".+?class="url">\s*<div class="hproductPhotoCont">\s*<img.+?(src|data-original)="(.+?)".*?/>\s*</div>\s*</a>\s*<div class="itemDesc">\s*<a.+?>\s*<h3.+?>(.+?)</h3>\s*</a>.+?<div class="itemPrice">.+?<span class="prezzoProdottoSaldo".*?>(.+?)</span>\s*</div>.+?</div>', flags=re.S)
            for item in p.finditer(tab_page):
                i_url, i_img, i_name, s_price = self.home_url+item.group(1), item.group(2), item.group(4).strip(), item.group(5)
                print i_url, i_img, i_name, s_price
                i_unit = ""
                if s_price.find("¥") != -1:
                    i_unit = "CNY"
                i_price = s_price.replace('¥','').strip()
                if i_url and i_url != '':
                    self.link_list.append((tab_name,tab_data_url,i_name,i_url,i_img,i_price,i_unit))
                else:
                    i = BagItem(self.brand_type)
                    i.initItem(tab_name, '', i_name, i_price, i_unit, '', i_url, i_img)
                    self.items.append(i.outItem())
                
    def bagItems(self):
        #for link in self.link_list:
        #    self.itemPage(link)
        #    break
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return
        
        m = re.search(r'<h2 class="productName">(.+?)</h2>', page, flags=re.S)
        if m:
            i_name = m.group(1).strip()

        m = re.search(r'<div id="zoomImageWrapper">\s*<img.+?src="(.+?)".*?/>\s*</div>', page, flags=re.S)
        if m:
            i_img = m.group(1)
        else:
            m = re.search(r'<div id="thumbsWrapper">.+?<div class="thumbElement".+?>\s*<img.+?src="(.+?)".*?/>\s*</div>', page, flags=re.S)
            if m:
                i_img = m.group(1)

        m = re.search(r'<span class="currency">(.+?)</span>.*?<span class="priceValue">(.+?)</span>', page, flags=re.S)
        if m:
            currency, i_price = m.group(1), re.sub(r'<.*>','',m.group(2))
            if currency.find("¥") != -1:
                i_unit = "CNY"
            else:
                i_unit = currency

        m = re.search(r'<div class="attributes">(.+?)</div>', page, flags=re.S)
        if m:
            size_str = re.sub(r'<.*?>','',m.group(1))
            #i_size = "".join(size_str.split())
            i_size = re.sub(r'\s*','',size_str)
            print "".join(i_size.split())

        i_number = ''
        m = re.search(r' <div class="modelFabricColorWrapper">\s*<div class="inner".*?>\s*<span class="modelTitle">.+?</span>.+?<span.*?class="value">(.+?)</span>\s*</div>\s*</div>\s*</div>', page, flags=re.S)
        if m:
            i_number = m.group(1)

        i = BagItem()
        i.initItem(serie_title, '', i_name, i_price, i_unit, i_size, i_url, i_img, i_number)
        self.items.append(i.outItem)    
        #print '# itemPage :', serie_title, i_name, i_price, i_unit, i_size, i_url, i_img

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#14
0
文件: Item.py 项目: xzhoutxd/tb
class Item():
    '''A class of item'''
    def __init__(self):
        # crawler
        #self.crawler         = TBCrawler()
        self.crawler         = RetryCrawler()

        # shop
        self.shop_type       = '1' # 店铺类型
        self.seller_id       = ''   # 卖家ID
        self.seller_name     = ''   # 卖家Name
        self.shop_id         = ''   # 店铺ID
        self.shop_name       = ''   # 店铺Name
        self.shop_url        = ''   # 店铺URL

        # refers
        self.refers          = ''

        # 成交抓取参数
        self.deal_pageSize   = 15
        self.deal_maxPages   = 100
        self.deal_bufferdays = 3   # 往前追溯3天

        # 评价抓取参数
        self.rate_pageSize   = 20
        self.rate_maxPages   = 100

        # 初始化实例变量
        self.initItem()

    def initItem(self):
        # 商品抓取设置
        self.crawling_time   = Common.now()
        self.crawling_begintime = '' # 本次抓取开始时间
        self.crawling_beginDate = '' # 本次爬取日期
        self.crawling_beginHour = '' # 本次爬取小时

        # 商品属性
        self.item_id         = ''   # 商品ID
        self.item_name       = ''   # 商品名称
        self.item_price      = ''   # 商品价格
        self.item_url        = ''   # 商品链接
        self.item_spuId      = ''   # SPU ID
        self.item_sellCount  = 0    # 月销售数

        self.brand_name      = ''
        self.brand_id        = ''
        self.category_id     = ''

        # 商品页
        self.item_page       = None # 商品首页

        # item html urls
        self.item_urls       = []   # 商品链接列表

        # item html pages
        #self.item_pages      = []   # 商品网页列表
        self.item_pages      = {}   # 商品网页列表

        # 成交记录
        self.deal_url        = ''
        self.deal_stopCrawl  = False
        self.deal_deadLine   = 0.0  # 上次抓取的成交记录最晚时间
        self.deal_deadLine2  = 0.0  # 本次抓取的成交记录最早时间

    def TMItem(self):
        if self.item_url != '':
            page = self.crawler.getData(self.item_url, self.refers)
            if not page or page == '':
                raise Common.InvalidPageException("# TMItem: not find item page,itemid:%s,item_url:%s"%(str(self.item_id), self.item_url))

            m = re.search(r'sellerId:"(\d+)",', page, flags=re.S)
            if m:
                self.seller_id = m.group(1)
            m = re.search(r'shopId:"(\d+)",', page, flags=re.S)
            if m:
                self.shop_id = m.group(1)
            m = re.search(r'<div class="slogo">\s*<a class="slogo-shopname" href="(.+?)".+?><strong>(.+?)</strong></a>', page, flags=re.S)
            if m:
                self.shop_url, self.shop_name = Common.fix_url(m.group(1)), m.group(2).strip()

            m = re.search(r'TShop\.Setup\((.+?)\);', page, flags=re.S)
            if m:
                TShop_s = m.group(1).strip()
                m = re.search(r'"brand":"(.+?)",', TShop_s, flags=re.S)
                if m:
                    self.brand_name = Common.htmlDecode(m.group(1).strip())
                m = re.search(r'"brandId":"(\d+)",', TShop_s, flags=re.S)
                if m:
                    self.brand_id = m.group(1)
                m = re.search(r'"categoryId":"(\d+)",', TShop_s, flags=re.S)
                if m:
                    self.category_id = m.group(1)
                m = re.search(r'"sellerNickName":"(.+?)",', TShop_s, flags=re.S)
                if m:
                    self.seller_name = Common.urlDecode(m.group(1).strip())

                m = re.search(r'"initApi":"(.+?)",', TShop_s, flags=re.S)
                if m:
                    ts = "&callback=setMdskip&timestamp=%s" % str(int(time.time()*1000))
                    initapi_url = Common.fix_url(m.group(1).strip()) + ts + "&ref=%s" % Common.urlCode(self.refers)
                    init_page = self.crawler.getData(initapi_url, self.item_url)
                    if not init_page and init_page == '':
                        print '# init page is null..'
                    else:
                        m = re.search(r'"sellCountDO":{"sellCount":(\d+),', init_page, flags=re.S)
                        if m:
                            self.item_sellCount = m.group(1)

    def parserTM(self, val):
        self.item_id, self.item_name, self.item_price, self.item_url, self.refers, self.crawling_begintime = val
        # 本次抓取开始日期
        self.crawling_beginDate = time.strftime("%Y-%m-%d", time.localtime(self.crawling_begintime))
        # 本次抓取开始小时
        self.crawling_beginHour = time.strftime("%H", time.localtime(self.crawling_begintime))
        self.TMItem()

    # 输出抓取的网页sql
    def outItemSql(self):
        return (Common.time_s(self.crawling_time),self.item_id,self.item_name,self.item_price,self.item_sellCount,self.item_url,self.seller_id,self.seller_name,self.shop_id,self.shop_name,self.shop_url,self.brand_id,self.brand_name,self.category_id,self.crawling_beginDate,self.crawling_beginHour)
示例#15
0
class LouisvuittonBag():
    '''A class of louisvuitton bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://www.louisvuitton.cn'
        self.price_url  = ''
        self.refers     = None

        # 品牌type
        self.brand_type = 'louisvuitton'

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        
    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        tab_list_info = ''
        m = re.search(r'<li class="mega-menu-item">\s*<div.+?data-megaMenu="women".*?>\s*<span class="onlyML">手袋</span>.+?</div>\s*<div class="mega-menu-content">.+?<ul class="level3">(.+?)</ul>', page, flags=re.S)
        if m:
            tab_list_info = m.group(1).strip()

        tab_list = []
        p = re.compile(r'<li class="mm-block">.+?<a.+?href="(.+?)">\s*<p class="mm-push-p">(.+?)</p>\s*</a>.+?</li>', flags=re.S)
        for tab_info in p.finditer(tab_list_info):
            tab_list.append((tab_info.group(2).strip(),self.home_url+tab_info.group(1)))
            print tab_info.group(2).strip(),self.home_url+tab_info.group(1)

        i = 0
        for tab in tab_list:
            refers = url
            tab_name, tab_url = tab
            print '# tab:',tab_name, tab_url
            tab_page = self.crawler.getData(tab_url, refers)
            m = re.search(r'<button.+?class="pl-next".+?data-next="(.+?)">',tab_page,flags=re.S)
            while m:
                refers = tab_url
                tab_url = re.sub(r'/to-\d+', '', tab_url)  + "/to-%s"%m.group(1)
                tab_page = self.crawler.getData(tab_url, refers)
                m = re.search(r'<button.+?class="pl-next".+?data-next="(.+?)">',tab_page,flags=re.S)

            p = re.compile(r'<a id="sku_.*?" href="(.+?)".+?>.+?<div class="description">\s*<div class="productName toMinimize">(.+?)</div>\s*<div class="productPrice.+?data-htmlContent="(.+?)">\s*</div>\s*</div>', flags=re.S)
            for item in p.finditer(tab_page):
                i_url, i_name, s_price = item.group(1),item.group(2),item.group(3)
                print self.home_url+i_url, i_name, s_price
                i_unit = ""
                if s_price.find("¥") != -1:
                    i_unit = "CNY"
                i_price = s_price.replace('¥','').strip()
                
                if i_url and i_url != '':
                    if Common.isBag(i_name):
                        self.link_list.append((tab_name,tab_url,i_name,self.home_url+i_url,i_price,i_unit))
                else:
                    if Common.isBag(i_name):
                        i = BagItem(self.brand_type)
                        i.initItem(tab_name, '', i_name, i_price, i_unit, '', i_url, '')
                        self.items.append(i.outItem())

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return
        
        m = re.search(r'<div class="productName title" id="productName">\s*<h1 itemprop="name">(.+?)</h1>\s*</div>', page, flags=re.S)
        if m:
            i_name = m.group(1).strip()

        m = re.search(r'<table class="priceButton">\s*<tr>\s*<td class="priceValue price-sheet">(.+?)</td>', page, flags=re.S)
        if m:
            i_price = m.group(1).strip()
            if i_price.find("¥") != -1:
                i_unit = "CNY"
    
        m = re.search(r'<noscript>\s*<img src="(.+?)".+?itemprop="image".*?/>\s*</noscript', page, flags=re.S)
        if m:
            i_img = m.group(1)

        i_size = ''
        m = re.search(r'<div class="textClientInfo exp_content".*?>\s*<div class="innerContent functional-text">(.+?)</div>', page, flags=re.S)
        if m:
            s_content = m.group(1).replace('&nbsp;','').strip()
            if s_content.find('宽)') != -1:
                s_size = s_content.split('宽)')[0]
                self.item_size = re.sub('<.+?>','',s_size) + "宽)"
            elif s_content.find('高)') != -1:
                s_size = s_content.split('高)')[0]
                self.item_size = re.sub('<.+?>','',s_size) + "高)"
            else:
                s_size = ''.join(s_content.split())

        i_number
        m = re.search(r'<h2 class="sku reading-and-link-text">(.+?)</h2>', page, flags=re.S)
        if m:
            i_number = m.group(1).strip()
        else:
            m = re.search(r'<meta itemprop="identifier" content="sku:(.+?)"/>', page, flags=re.S)
            if m:
                i_number = m.group(1).strip()

        i = BagItem(self.brand_type)
        i.initItem(serie_title, '', i_name, i_price, i_unit, i_size, i_url, i_img, i_number)
        print '# itemPage:',i.outItem()
        #self.items.append(i.outItem()) 

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#16
0
class BossBag():
    '''A class of boss bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = 'http://store.hugoboss.cn'
        self.price_url = ''
        self.refers = None

        # 品牌type
        self.brand_type = 'boss'

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        tab_name = '手袋'
        self.parse_item(page, tab_name, url)

    def parse_item(self, tab_page, tab_name, tab_url):
        items_info = ''
        m = re.search(
            r'<div class="productlist-widget">.+?<div class="container">\s*<ul.+?>(.+?)</ul>',
            tab_page,
            flags=re.S)
        if m:
            items_info = m.group(1)
            self.run_items(items_info, tab_name, tab_url)

    def run_items(self, items_info, tab_name, tab_url):
        p = re.compile(
            r'<li class="productlist-item ">\s*<div class="product-image".+?>\s*<a.+?><img src="(.+?)".+?/>\s*</a>\s*</div>.+?<div class="product-title">\s*<a href="(.+?)".+?>(.+?)</a>\s*</div>.+?<p>\s*<span class="product-price">(.+?)</span>\s*</p>\s*</li>',
            flags=re.S)
        for item in p.finditer(items_info):
            i_img, i_url, i_name, s_price = item.group(1), item.group(
                2), item.group(3), item.group(4)
            i_price, i_unit = '', ''
            if s_price.find("¥") != -1 or s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = s_price.replace('¥', '').replace('¥', '').strip()

            if i_url and i_url != '':
                print self.home_url + i_url, i_img, i_name, i_price, i_unit
                self.link_list.append(
                    (tab_name, tab_url, i_name, self.home_url + i_url, i_img,
                     i_price, i_unit))
            else:
                print self.home_url + i_url, i_img, i_name, i_price, i_unit
                i = BagItem(self.brand_type)
                i.initItem(tab_name, '', i_name, i_price, i_unit, '', i_url,
                           i_img)
                self.items.append(i.outItem())

    def isBag(self, name):
        bag_info = ["包", "袋", "皮夹", "钱夹"]
        other_info = ["裤", "T恤", "衬衫", "礼服", "上衣"]
        for b_info in bag_info:
            if name.find(b_info) != -1:
                for o_info in other_info:
                    if name.find(o_info) != -1:
                        return False
                return True
        return False

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type,
                                  len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        m = re.search(r'<h1 class="product-name">(.+?)</h1>', page, flags=re.S)
        if m:
            i_name = ' '.join(m.group(1).strip().split())

        m = re.search(
            r'<div class="product-prices">.+?<dd class="saleprice">(.+?)</dd>.+?</div>',
            page,
            flags=re.S)
        if m:
            s_price = m.group(1).strip()
            if s_price.find("¥") != -1 or s_price.find("¥") != -1:
                i_unit = "CNY"
            i_price = s_price.replace('¥', '').replace('¥', '').strip()

        m = re.search(
            r'<div class="image">.+?<div class="container".*?>\s*<table.+?>\s*<tr>\s*<td>\s*<a.+?>\s*<img.+?class="thumb".+?big="(.+?)".*?/>\s*</a>\s*</td>',
            page,
            flags=re.S)
        if m:
            i_img = m.group(1)

        i_size = ''
        m = re.search(
            r'<div class="tabpage inc".+?>.+?<span.*?>(尺寸大小.+?)</span>',
            page,
            flags=re.S)
        if m:
            s_size = m.group(1)
            i_size = s_size.split(':')[1]
        if i_size == '':
            m = re.search(r'<span.+?>尺寸大小:</span>(.+?)</span>',
                          page,
                          flags=re.S)
            if m:
                i_size = re.sub(r'<.+?>', '', m.group(1))

        i_number = ''
        m = re.search(
            r'<div class="base">\s*<div class="sku-brand">.+?<dl class="hidden"><dt>商品货号: </dt><dd>(.+?)</dd></dl>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_number = m.group(1)

        i = BagItem(self.brand_type)
        i.initItem(serie_title, '', i_name, i_price, i_unit, i_size, i_url,
                   i_img, i_number)
        print '# itemPage:', i.outItem()
        #self.items.append(i.outItem())

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#17
0
文件: diorBag.py 项目: xzhoutxd/brand
class DiorBag():
    '''A class of dior bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://www.dior.cn'
        self.price_url  = ''
        self.refers     = None

        # 品牌type
        self.brand_type = 'dior'

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        
    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return
       
        tab_list = []
        m = re.search(r'<ul class="tabsList collections">(.+?)</ul>', page, flags=re.S)
        if m:
            tabs_list_info = m.group(1)

            p = re.compile(r'<li class=".+?">\s+<a href="(.+?)" data-magento_call_page="(.+?)".+?>(.+?)</a>\s+</li>', flags=re.S)
            for tab in p.finditer(tabs_list_info):
                tab_list.append((tab.group(3).strip(),self.home_url+tab.group(2),url+tab.group(1)))

        for tab in tab_list:
            tab_name,tab_data_url,tab_url = tab
            print '# tab:',tab_name,tab_data_url,tab_url
            tab_page = self.crawler.getData(tab_data_url, url)

            p = re.compile(r'<li class="li-product.+?>\s+<a href="(.*?)" class="linkProduct">.+?<img src="(.+?)".+?/>.+?<span class="description".+?>.+?<span class="title">(.+?)</span>.+?</span>\s+</a>\s+</li>', flags=re.S)
            for item in p.finditer(tab_page):
                i_url, i_img, i_name = self.home_url+item.group(1), self.home_url+item.group(2), item.group(3)
                print i_url, i_img, i_name
                if i_url and i_url != '':
                    self.link_list.append((tab_name,tab_url,i_name,i_url,i_img))
                else:
                    i = BagItem(self.brand_type)
                    i.initItem(tab_name, '', i_name, '', '', '', i_url, self.home_url+i_img)
                    self.items.append(i.outItem())
                
    def bagItems(self):
        #for link in self.link_list: self.itemPage(link)
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url,self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url,self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url = val
        
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return
        
        i_title, i_img, i_size, i_price, i_unit = '', '', '', '', ''
        
        m = re.search(r'<h2 class="" itemprop="name">(.+?)<br />(.+?)</h2>', page, flags=re.S)
        if m:
            i_title = m.group(1).strip()

        m = re.search(r'<li class="firstThumbnails">\s+<a href="#" class="active".+?>\s+<img src="(.+?)" alt="" />\s+</a>\s+</li>', page, flags=re.S)
        if m: i_img  = self.home_url + m.group(1)

        m = re.search(r'<div class="modText">\s+<h4.+?>说明</h4>\s+<p>(.+?)</p>\s+</div>', page, flags=re.S)
        if m: 
            i_desc = m.group(1)
            m = re.search(r'尺寸:(.+?)<br />', i_desc, flags=re.S)
            if m:
                i_size = m.group(1).strip()
            else:
                m = re.search(r'尺寸:(.+?)$', i_desc, flags=re.S)
                if m: i_size = m.group(1).strip()

        i_number = ''
        m = re.search(r'<div class="columns-wrapper">.+?<div class="column">.*?<div class="reference">\s*<p>(.+?)</p>\s*</div>', page, flags=re.S)
        if m:
            s_number = m.group(1)
            i_number = s_number.split('-')[1].strip()
                
        i = BagItem()
        i.initItem(serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img, i_number)
        self.items.append(i.outItem)    
        print '# itemPage :', serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#18
0
class ChanelBag():
    '''A class of chanel bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://www.chanel.com'
        self.price_url  = 'http://ws.chanel.com/pricing/zh_CN/fashion/%s/?i_client_interface=fsh_v3_misc&i_locale=zh_CN&format=json&callback=localJsonpPricingCallback'
        self.refers     = None

        # 品牌type
        self.brand_type = 'chanel'

        # 抓取商品列表
        self.link_list = []
        self.items      = []
        
    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        f_url = 'http://www.chanel.com/zh_CN/fashion/products/handbags/g.metiers-d-art-paris-salzburg-2014-15.c.15A.html'
        m = re.search(r'<div class="category-container category-black".+?>.+?<ul>\s*<li>\s*<a class="no-select" target="_blank" href="(.+?)" data-cat="查看系列">', page, flags=re.S)
        if m:
            f_url = self.home_url + m.group(1)

        page = self.crawler.getData(f_url, url)
        if not page or page == '': return

        tab_list = []
        m = re.search(r'<div class="mosaic-nav">\s*<ul class="nav">(.+?)</ul>', page, flags=re.S)
        if m:
            tab_list_info = m.group(1)
            p = re.compile(r'<li class="no-select nav-item">\s*<a.+?href="(.+?)">\s*<h2>(.+?)</h2>\s*</a>', flags=re.S)
            for tab_info in p.finditer(tab_list_info):
                tab_url, tab_name = tab_info.group(1), re.sub(r'<.+?>','',tab_info.group(2).replace('&nbsp;',' ').strip())
                if tab_url == '#':
                    tab_list.append((tab_name, f_url))
                else:
                    tab_list.append((tab_name, self.home_url+tab_url))

        refers = url
        for tab in tab_list:
            tab_name, tab_url = tab
            print '# tab:',tab_name, tab_url
            tab_page = self.crawler.getData(tab_url, refers)
            refers = tab_url
            m = re.search(r'"products":\s*(\[.+?\])\s*},\s*"deeplink"', page, flags=re.S)
            if m:
                prods   = m.group(1)
                js_items= json.loads(prods)

                for js_item in js_items:
                    i_title = js_item["title"].replace('&nbsp;',' ')
                    items = js_item["items"]
                    
                    for item in items:
                        if not item.has_key("title"): continue
                        
                        i_name = item["title"].replace('&nbsp;',' ')
                        i_url   = self.home_url + item["href"]
                         
                        print tab_name, i_title, i_name, i_url
                        self.link_list.append((tab_name, i_title, tab_url, i_name, i_url))
                
    def bagItems(self):
        #for link in self.links: self.itemPage(link)
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url,self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url,self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, i_title, refers, i_name, i_url = val
        
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return
        
        i_name, i_img, ref_price, i_size, i_price, i_unit, i_number = '', '', '', '', '', '', ''
        

        m = re.search(r'<div class="product-details details".*?>.+?<p class="description info.*?">(.+?)</p>', page, flags=re.S)
        if m: i_name = re.sub(r'<.+?>', '', m.group(1)).strip()
        else:
            m = re.search(r'<title>(.+?)</title>', page, flags=re.S)
            if m: i_name = m.group(1).split('-')[0].strip()
        
        m = re.search(r'<div class="productimage.*?"><img src="(.+?)".*?/>', page, flags=re.S)
        if m: i_img  = self.home_url + m.group(1)

        p = re.compile(r'<p class="size info">(.+?)</p>', flags=re.S)
        for size in p.finditer(page):
            if self.item_size != '': i_size += '-' + size.group(1)
            else: i_size = size.group(1)
                
        #m = re.search(r'<div class="ref info">\s*<p>(.+?)</p>', page, flags=re.S)
        #if m:
        p = re.compile(r'<div class="ref info">\s*<p>(.+?)</p>', flags=re.S)
        for number in p.finditer(page):
            item_number = number.group(1)
            if self.item_number != '': self.item_number += '-' + item_number
            else: self.item_number = item_number

            refs = item_number.split(' ')[:-1]
            ref_price = ''.join(refs)

            p_url = self.price_url %ref_price
            data = self.crawler.getData(p_url, i_url)
            if not data or data == '': return
            
            # 抽取json报文
            r = re.search(r'localJsonpPricingCallback\(\[(.+?)\]\)', data, flags=re.S)
            if r:
                price, unit = '', ''
                try:
                    js_data = json.loads(r.group(1))
                    price, unit = js_data["price"]["amount"], js_data["price"]["currency-symbol"]
                except Exception as e:
                    m = re.search(r'"amount":"(.+?)"', data, flags=re.S)
                    if m: price = m.group(1)
                    m = re.search(r'"currency-symbol":"(.+?)"', data, flags=re.S)
                    if m: unit = m.group(1)
                if self.item_price != '':
                    if price: i_price += '-' + price
                else:
                    if price: i_price = price
                    if unit: i_unit  = unit
        
        i = BagItem(self.brand_type)
        i.initItem(serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img, i_number)
        #print '# itemPage :', serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#19
0
class BottegavenetaBag():
    '''A class of bottegaveneta bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = 'http://www.bottegaveneta.com'
        self.price_url = ''
        self.refers = None

        # 品牌type
        self.brand_type = 'bottegaveneta'

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        p = re.compile(
            r'<div id="slot_\d+".+?<a.+?href="(.+?)".+?>\s*<img.+?src="(.+?)".+?/>\s*<div class="iteminfo">\s*<div class="headgroup">\s*<div class="extra">\s*<h1 class="modelname">(.+?)</h1>',
            flags=re.S)
        for item in p.finditer(page):
            i_url, i_img, i_name = item.group(1), item.group(2), item.group(3)
            print i_url, i_img, i_name
            if i_url and i_url != '':
                self.link_list.append(('', url, i_name, i_url, i_img))
            else:
                i = BagItem(self.brand_type)
                i.initItem('', url, i_name, i_url, i_img)
                self.items.append(i.outItem())

        p = re.compile(r'<div class="slot lazySlot".+?data-slot="(.+?)".+?>',
                       flags=re.S)
        for item in p.finditer(page):
            data_info = item.group(1)
            data_info_str = data_info.replace('&quot;', '"')
            i_url, i_img, i_name = '', '', ''
            m = re.search(r'"Link":"(.+?)"', data_info_str, flags=re.S)
            if m:
                i_url = m.group(1)

            m = re.search(r'"ModelName":"(.+?)",', data_info_str, flags=re.S)
            if m:
                i_name = m.group(1)

            print i_url, i_img, i_name
            if i_url and i_url != '':
                self.link_list.append(('', url, i_name, i_url, i_img))
            else:
                i = BagItem(self.brand_type)
                i.initItem('', url, i_name, i_url, i_img)
                self.items.append(i.outItem())

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """

        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type,
                                  len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        m = re.search(
            r'<h1 class="producTitle".+?>\s*<div class="modelName".+?>\s*<span class="modelName">(.+?)</span>',
            page,
            flags=re.S)
        if m:
            i_name = m.group(1).strip()

        m = re.search(
            r'<div class="mainImage".+?>\s*<img.+?src="(.+?)".*?/>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_img = m.group(1)
        else:
            m = re.search(
                r'<section id="bgItem">\s*<img.+?src="(.+?)".*?/>\s*</section>',
                page,
                flags=re.S)
            if m:
                i_img = m.group(1)

        i_size = ''
        m = re.search(
            r'<div class="localizedAttributes">.*?<div class="height">.+?<span class="text">(.+?)</span>\s*<span class="value">(.+?)</span>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_size += m.group(1) + ":" + m.group(2) + ";"
        m = re.search(
            r'<div class="localizedAttributes">.*?<div class="depth">.+?<span class="text">(.+?)</span>\s*<span class="value">(.+?)</span>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_size += m.group(1) + ":" + m.group(2) + ";"
        m = re.search(
            r'<div class="localizedAttributes">.*?<div class="length_of_strap">.+?<span class="text">(.+?)</span>\s*<span class="value">(.+?)</span>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_size += m.group(1) + ":" + m.group(2) + ";"
        m = re.search(
            r'<div class="localizedAttributes">.*?<div class="width">.+?<span class="text">(.+?)</span>\s*<span class="value">(.+?)</span>\s*</div>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_size += m.group(1) + ":" + m.group(2) + ";"

        i_number
        m = re.search(
            r' <div class="modelFabricColorWrapper">\s*<div class="inner".*?>\s*<span class="modelTitle">.+?</span>.+?<span.*?class="value">(.+?)</span>\s*</div>\s*</div>\s*</div>',
            page,
            flags=re.S)
        if m:
            i_number = m.group(1)

        i = BagItem(self.brand_type)
        i.initItem(serie_title, '', i_name, '', '', i_size, i_url, i_img,
                   i_number)
        print '# itemPage:', i.outItem()
        #self.items.append(i.outItem())
        #print '# itemPage :', serie_title, i_name, i_url, i_img, i_size

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#20
0
class GivenchyBag():
    '''A class of givenchy bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = 'http://www.givenchy.com'
        self.price_url = ''
        self.refers = None

        # 品牌type
        self.brand_type = 'givenchy'

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        tab_list = []
        m = re.search(r'<p>女装配饰</p>\s+<ul class="menu-level-4">(.+?)</ul>',
                      page,
                      flags=re.S)
        if m:
            tabs_list_info = m.group(1)

            p = re.compile(
                r'<li class="submenu-item">\s+<a.+?href="(.+?)">\s+<span lang="fr" class="lang-fr">(.+?)</span>(.+?)</a>\s+</li>',
                flags=re.S)
            for tab in p.finditer(tabs_list_info):
                tab_list.append(
                    (tab.group(2) + tab.group(3).strip(), tab.group(1)))

        for tab in tab_list:
            tab_name, tab_url = tab
            print '# tab:', tab_name, tab_url
            tab_page = self.crawler.getData(tab_url, url)

            m = re.search(
                r'<div id="layer-1".+?>\s+<a href="(.+?)" class="layer-links">.+?</a>',
                tab_page,
                flags=re.S)
            if m:
                ajax_url = self.home_url + m.group(
                    1) + "?ajax=true&fragment=true"
                ajax_data = self.crawler.getData(ajax_url, tab_url)

                if ajax_data:
                    #data = json.loads(ajax_data)
                    #if data and data.has_key("html"):
                    #    print data["html"].decode("unicode-escape")
                    r_data = ajax_data.decode("unicode-escape")
                    if r_data:
                        m = re.search(r'"html":"(.+?)"}', r_data, flags=re.S)
                        if m:
                            data_html = m.group(1).replace("\/", "/")
                            #print data_html
                            #break
                            p = re.compile(
                                r'<li class="lookbook-item line" data-idlook="\d+">\s+<div class="disp-n">.+?<div class="look-info article">\s+<p>(.+?)</p>.+?<p class="look-ref">(.+?)</p>.+?</div>.+?</div>\s+<a href="(.+?)".+?>.+?<img .+?data-src="(.+?)".*?/>\s+</li>',
                                flags=re.S)
                            for item in p.finditer(data_html):
                                i_url, i_img, s_number, i_name = self.home_url + item.group(
                                    3), item.group(4), item.group(2), re.sub(
                                        r'<.+?>', '', item.group(1)).strip()
                                i_number = ''
                                m = re.search(
                                    r'<span class="look-ref-sku">\s*<span.+?>(.+?)</span>\s*</span>',
                                    s_number,
                                    flags=re.S)
                                if m:
                                    i_number = m.group(1)
                                print i_url, i_img, i_name, i_number
                                if Common.isBag(i_name):
                                    self.link_list.append(
                                        (tab_name, tab_url, i_name, i_url,
                                         i_img, i_number))
                                    #i = BagItem(self.home_url, self.brand_type)
                                    #i.initItem(tab_name, '', i_name, '', '', '', i_url, i_img, i_number)
                                    #self.items.append(i.outItem())

    def bagItems(self):
        #for link in self.link_list: self.itemPage(link)
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type,
                                  len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#21
0
class YslBag():
    '''A class of ysl bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # 品牌官网链接
        self.home_url   = 'http://www.ysl.com'
        self.price_url  = ''
        self.refers     = None

        # 品牌type
        self.brand_type = 'ysl'

        # 抓取商品列表
        self.link_list  = []
        self.items      = []
        
    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        tab_list = []
        m = re.search(r'<ul class="shopGender sep"><li class="shopWoman level1".+?>(.+?)</li><li class="shopMan level1"', page, flags=re.S)
        if m:
            tab_list_info = m.group(1)
            p = re.compile(r'<li data-dept=".+?".*?><a href="(.+?)">(.+?)</a>.*?</li>', flags=re.S)
            for tab_info in p.finditer(tab_list_info):
                tab_url, tab_name = tab_info.group(1), tab_info.group(2).strip()
                print tab_url, tab_name
                tab_list.append((tab_name, tab_url))
                if tab_name == '手提袋':
                    break


        #tab_list = [("手提袋","http://www.ysl.com/wy/shop-product/%E5%A5%B3%E5%A3%AB/%E6%89%8B%E6%8F%90%E8%A2%8B")]
        i = 0
        for tab in tab_list:
            refers = url
            tab_name, tab_url = tab
            print '# tab:',tab_name, tab_url
            tab_page = self.crawler.getData(tab_url, refers)
            self.parse_item(tab_page,tab_name,tab_url)
            m = re.search(r'<div class="pagnum".+?>(.+?)</div>', tab_page, flags=re.S)
            if m:
                pagelist_info = m.group(1)
                p = re.compile(r'<a href="(.+?)">\d+</a>', flags=re.S)
                page_list = []
                for page in p.finditer(pagelist_info):
                    page_info = self.crawler.getData(page.group(1), tab_url)
                    #print page.group(1)
                    self.parse_item(page_info,tab_name,tab_url)

    def parse_item(self, tab_page, tab_name, tab_url):
        items_info = ''
        m = re.search(r'<div id="productsContainer".+?>(.+?)</div>\s*</div>\s*</div>', tab_page, flags=re.S)
        if m:
            items_info = m.group(1)
            self.run_items(items_info, tab_name, tab_url)
        else:
            m = re.search(r'<div class="productsFromGrid.+?>(.+?)</div>\s*</div>\s*</div>\s*</div>', tab_page, flags=re.S)
            if m:
                items_info = m.group(1)
                self.run_items(items_info, tab_name, tab_url)
        
    def run_items(self, items_info, tab_name, tab_url):
        p = re.compile(r'<.+?>\s*<a.+?>\s*<img.+?src="(.+?)".*?>\s*</a><a href="(.+?)">.+?<div class="infoDescription">(.+?)</div>\s*(<div class="infoPrice">.+?</div>).+?</a>\s*<.+?>', flags=re.S)
        for item in p.finditer(items_info):
            i_img, i_url, s_name, price_info = item.group(1),item.group(2),item.group(3),item.group(4)
            i_name = re.sub(r'<.+?>','',s_name)
            i_price, i_unit = '', ''
            m = re.search(r'<div.+?class="newprice">(.+?)</div>', price_info, flags=re.S)
            if m:
                s_price = re.sub(r'<.+?>','',m.group(1))
                if s_price.find("¥") != -1:
                    i_unit = "CNY"
                i_price = s_price.replace('¥','').strip()
            
            if i_url and i_url != '':
                if Common.isBag(i_name) or Common.isBag(unquote(i_url)):
                    print self.home_url+i_url, i_img, i_name, i_price, i_unit
                    self.link_list.append((tab_name,tab_url,i_name,self.home_url+i_url,i_img,i_price,i_unit))
            else:
                if Common.isBag(i_name) or Common.isBag(unquote(i_url)):
                    print self.home_url+i_url, i_img, i_name, i_price, i_unit
                    i = BagItem(self.brand_type)
                    i.initItem(tab_name, '', i_name, i_price, i_unit, '', i_url, i_img)
                    self.items.append(i.outItem())

    def bagItems(self):
        """
        i = 0
        for link in self.link_list:
            self.itemPage(link)
            i += 1
            if i == 1:
                break
        """
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, refers, i_name, i_url, i_img, i_price, i_unit = val
        if i_url == '': return
        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        m = re.search(r'<div id="itemInfo">.+?<h1><span class="customItemDescription" itemprop="name">(.+?)</span></h1>', page, flags=re.S)
        if m:
            i_name = m.group(1).strip()

        m = re.search(r'<div id="itemPrice".+?><div.*?class="newprice">(.+?)</div>', page, flags=re.S)
        if m:
            s_price = m.group(1).strip()
            s_price = re.sub(r'<.+?>','',s_price)
            if s_price.find("¥") != -1:
                i_unit = "CNY"
                i_price = s_price.replace('¥','').strip()
            else:
                i_price = s_price
    
        m = re.search(r'<div id="mainImageContainer"><img.+?src="(.+?)".*?/></div>', page, flags=re.S)
        if m:
            i_img = m.group(1)

        i_size = ''
        m = re.search(r'<div class="itemDimensions">.+?<span class="dimensions">(.+?)</span></div>', page, flags=re.S)
        if m:
            i_size = m.group(1)

        i_number
        m = re.search(r'<div class="styleIdDescription">货号.+?<span.*?>(.+?)</span></div>', page, flags=re.S)
        if m:
            i_number = m.group(1)

        i = BagItem(self.brand_type)
        i.initItem(serie_title, '', i_name, i_price, i_unit, i_size, i_url, i_img, i_number)
        print '# itemPage:',i.outItem()
        #self.items.append(i.outItem()) 

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))
示例#22
0
文件: TMCrawler.py 项目: xzhoutxd/tb
class TMCrawler():
    '''A class of TMall shop'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler     = RetryCrawler()

        # db
        self.mysqlAccess  = MysqlAccess() # mysql access

        # 品牌官网链接
        self.home_url   = 'http://www.taobao.com'
        self.refers     = None

        # 抓取商品列表
        self.link_list  = []
        self.items      = []

        self.begin_time = Common.now()
        
    def getPage(self, url, shop_home_url):
        position = 1
        i = 1
        max_page = 0
       
        asyn_url = ''
        i_url = url
        refers = shop_home_url
        result_s = self.get_asyn_data(i_url, refers, shop_home_url)
        m = re.search(r'<b class="ui-page-s-len">\d+/(\d+)</b>', result_s, flags=re.S) 
        if m:
            max_page = int(m.group(1))
        print '# page num:', max_page
        while i <= max_page:
            m = re.search(r'<div class="J_TItems">(.+?)<div class="pagination">', result_s, flags=re.S)
            if m:
                items_s = m.group(1)
                p = re.compile(r'<dl class=".+?".+?data-id="(.+?)">.+?<dd class="detail">\s*<a class="item-name".+?href="(.+?)".+?>(.+?)</a>\s*<div class="attribute">\s*<div class="cprice-area">\s*<span class="symbol">(.+?)</span>\s*<span\s*class="c-price">(.+?)</span>\s*</div>.+?</dl>')
                j = 1
                for item in p.finditer(items_s):
                    item_id, url_s, item_name, price_symbol, price = item.group(1), item.group(2), Common.htmlDecode(item.group(3).strip()), item.group(4).strip(), item.group(5).strip()
                    if url_s.find('http') == -1:
                        item_url = 'http:' + url_s
                    else:
                        item_url = url_s
                    print '### item ###'
                    print '# item val:', item_id, item_name, price, item_url
                    item = Item()
                    item.parserTM((item_id, item_name, price, item_url, i_url, self.begin_time))
                    print '# item info:',item.outItemSql()
                    self.mysqlAccess.insert_parser_item_info(item.outItemSql())
                    time.sleep(2)
            
            refers = i_url
            if i_url.find('pageNo=') == -1:
                i_url = re.sub(r'&tsearch=y','&pageNo=%d&tsearch=y#anchor' % i, refers)
            else:
                i_url = re.sub(r'&pageNo=\d+&','&pageNo=%d&' % i, refers)

            i += 1
            time.sleep(2)
            result_s = self.get_asyn_data(i_url, refers, shop_home_url)

    def get_asyn_data(self, i_url, refers, shop_home_url):
        result = ''
        result_s = ''
        page = self.crawler.getData(i_url, refers)
        m = re.search(r'<input id="J_ShopAsynSearchURL".+?value="(.+?)"\s*/>', page, flags=re.S)
        if m:
            ts = '?_ksTS=%s&callback=jsonp135&' % (str(int(time.time()*1000)) + '_' + str(random.randint(100,999)))
            a_url = shop_home_url + Common.htmlDecode(m.group(1))
            asyn_url = re.sub('\?', ts, a_url)
            result = self.crawler.getData(asyn_url, i_url)
            m = re.search(r'jsonp135\("(.+?)"\)', result, flags=re.S)
            if m:
                result_s = re.sub(r'\\"', '"', m.group(1))
        return result_s


    def getItems(self):
        #for link in self.link_list: self.itemPage(link)
        max_th = 10
        #if len(self.link_list) > max_th:
        #    m_itemsObj = BagItemM(self.home_url,self.brand_type, max_th)
        #else:
        #    m_itemsObj = BagItemM(self.home_url,self.brand_type, len(self.link_list))
        #m_itemsObj.createthread()
        #m_itemsObj.putItems(self.link_list)
        #m_itemsObj.run()
        #self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        print '# itemPage :', serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img
示例#23
0
class ChanelBag():
    '''A class of chanel bag'''
    def __init__(self):
        # 抓取设置
        #self.crawler     = MyCrawler()
        self.crawler = RetryCrawler()

        # 品牌官网链接
        self.home_url = 'http://www.chanel.com'
        self.price_url = 'http://ws.chanel.com/pricing/zh_CN/fashion/%s/?i_client_interface=fsh_v3_misc&i_locale=zh_CN&format=json&callback=localJsonpPricingCallback'
        self.refers = None

        # 品牌type
        self.brand_type = 'chanel'

        # 抓取商品列表
        self.link_list = []
        self.items = []

    def bagPage(self, url):
        page = self.crawler.getData(url, self.home_url)
        if not page or page == '': return

        f_url = 'http://www.chanel.com/zh_CN/fashion/products/handbags/g.metiers-d-art-paris-salzburg-2014-15.c.15A.html'
        m = re.search(
            r'<div class="category-container category-black".+?>.+?<ul>\s*<li>\s*<a class="no-select" target="_blank" href="(.+?)" data-cat="查看系列">',
            page,
            flags=re.S)
        if m:
            f_url = self.home_url + m.group(1)

        page = self.crawler.getData(f_url, url)
        if not page or page == '': return

        tab_list = []
        m = re.search(r'<div class="mosaic-nav">\s*<ul class="nav">(.+?)</ul>',
                      page,
                      flags=re.S)
        if m:
            tab_list_info = m.group(1)
            p = re.compile(
                r'<li class="no-select nav-item">\s*<a.+?href="(.+?)">\s*<h2>(.+?)</h2>\s*</a>',
                flags=re.S)
            for tab_info in p.finditer(tab_list_info):
                tab_url, tab_name = tab_info.group(1), re.sub(
                    r'<.+?>', '',
                    tab_info.group(2).replace('&nbsp;', ' ').strip())
                if tab_url == '#':
                    tab_list.append((tab_name, f_url))
                else:
                    tab_list.append((tab_name, self.home_url + tab_url))

        refers = url
        for tab in tab_list:
            tab_name, tab_url = tab
            print '# tab:', tab_name, tab_url
            tab_page = self.crawler.getData(tab_url, refers)
            refers = tab_url
            m = re.search(r'"products":\s*(\[.+?\])\s*},\s*"deeplink"',
                          page,
                          flags=re.S)
            if m:
                prods = m.group(1)
                js_items = json.loads(prods)

                for js_item in js_items:
                    i_title = js_item["title"].replace('&nbsp;', ' ')
                    items = js_item["items"]

                    for item in items:
                        if not item.has_key("title"): continue

                        i_name = item["title"].replace('&nbsp;', ' ')
                        i_url = self.home_url + item["href"]

                        print tab_name, i_title, i_name, i_url
                        self.link_list.append(
                            (tab_name, i_title, tab_url, i_name, i_url))

    def bagItems(self):
        #for link in self.links: self.itemPage(link)
        max_th = 10
        if len(self.link_list) > max_th:
            m_itemsObj = BagItemM(self.home_url, self.brand_type, max_th)
        else:
            m_itemsObj = BagItemM(self.home_url, self.brand_type,
                                  len(self.link_list))
        m_itemsObj.createthread()
        m_itemsObj.putItems(self.link_list)
        m_itemsObj.run()
        self.items.extend(m_itemsObj.items)

    def itemPage(self, val):
        serie_title, i_title, refers, i_name, i_url = val

        page = self.crawler.getData(i_url, refers)
        if not page or page == '': return

        i_name, i_img, ref_price, i_size, i_price, i_unit, i_number = '', '', '', '', '', '', ''

        m = re.search(
            r'<div class="product-details details".*?>.+?<p class="description info.*?">(.+?)</p>',
            page,
            flags=re.S)
        if m: i_name = re.sub(r'<.+?>', '', m.group(1)).strip()
        else:
            m = re.search(r'<title>(.+?)</title>', page, flags=re.S)
            if m: i_name = m.group(1).split('-')[0].strip()

        m = re.search(r'<div class="productimage.*?"><img src="(.+?)".*?/>',
                      page,
                      flags=re.S)
        if m: i_img = self.home_url + m.group(1)

        p = re.compile(r'<p class="size info">(.+?)</p>', flags=re.S)
        for size in p.finditer(page):
            if self.item_size != '': i_size += '-' + size.group(1)
            else: i_size = size.group(1)

        #m = re.search(r'<div class="ref info">\s*<p>(.+?)</p>', page, flags=re.S)
        #if m:
        p = re.compile(r'<div class="ref info">\s*<p>(.+?)</p>', flags=re.S)
        for number in p.finditer(page):
            item_number = number.group(1)
            if self.item_number != '': self.item_number += '-' + item_number
            else: self.item_number = item_number

            refs = item_number.split(' ')[:-1]
            ref_price = ''.join(refs)

            p_url = self.price_url % ref_price
            data = self.crawler.getData(p_url, i_url)
            if not data or data == '': return

            # 抽取json报文
            r = re.search(r'localJsonpPricingCallback\(\[(.+?)\]\)',
                          data,
                          flags=re.S)
            if r:
                price, unit = '', ''
                try:
                    js_data = json.loads(r.group(1))
                    price, unit = js_data["price"]["amount"], js_data["price"][
                        "currency-symbol"]
                except Exception as e:
                    m = re.search(r'"amount":"(.+?)"', data, flags=re.S)
                    if m: price = m.group(1)
                    m = re.search(r'"currency-symbol":"(.+?)"',
                                  data,
                                  flags=re.S)
                    if m: unit = m.group(1)
                if self.item_price != '':
                    if price: i_price += '-' + price
                else:
                    if price: i_price = price
                    if unit: i_unit = unit

        i = BagItem(self.brand_type)
        i.initItem(serie_title, i_title, i_name, i_price, i_unit, i_size,
                   i_url, i_img, i_number)
        #print '# itemPage :', serie_title, i_title, i_name, i_price, i_unit, i_size, i_url, i_img

    def outItems(self, f):
        s = '#系列名称|商品标签|商品名称|商品价格|金额单位|商品尺寸|商品链接|商品图片|商品编号'
        with open(f, 'w') as f_item:
            self.items.insert(0, s)
            f_item.write('\n'.join(self.items))