コード例 #1
0
ファイル: glyde.py プロジェクト: dlluncor/simple-crawl
    def parse_detail(self, response):
        title = response.css('div#sku-data-container h1::attr(content)').extract_first()
        (brand, model, capacity, color, carrier) = self.extract_info_from_title(title)

        prices_by_conditions = {}
        if response.css('div.sold-out-price-container'):
            stock = "OutOfStock"
            for item in response.css('div.sold-out-price-container div.price_information'):
                condition = item.css('div.condition_description::text').extract_first()
                price = item.css('div.best_price::text').extract_first()
                condition = normalize_condition(GlydeSpider.name, condition)
                prices_by_conditions[condition] = price
        else:
            stock = None
            items = response.css('div.single-sku-container')
            if not items:
                container = response.css('div.multiple-skus-container')
                items = container.css('div.condition-button')

            for item in items:
                condition = item.css('div.font_nunito_bold::text').extract_first()
                price = item.css('div.products-show-price-container span.dollars::text,div.dollars::text').extract_first()
                condition = normalize_condition(GlydeSpider.name, condition)
                prices_by_conditions[condition] = '$%s' % price

        shipping_cost = self.get_shipping_cost(response)

        for condition in prices_by_conditions:
            device = Device()
            device.brand = brand
            device.carrier = carrier
            device.capacity = capacity
            device.color = color
            device.condition = condition
            device.used_price = prices_by_conditions[condition]
            device.model = model
            device.shipping_cost = shipping_cost
            device.source_site = GlydeSpider.name
            device.seller = GlydeSpider.name
            device.url = response.url
            if stock:
                device.stock = stock
            device.save()
            logging.info(">>>>>>>> CREATE %s" % device)
コード例 #2
0
ファイル: gazelle.py プロジェクト: dlluncor/simple-crawl
    def parse_detail(self, response):
        self.driver.get(response.url)
        time.sleep(1)
        title = self.driver.find_element_by_css_selector(self.title_selector)
        (model, capacity, connectivity, carrier) = self.extract_info_from_title(title.text)
        specs = self.extract_specs()

        colors = self.driver.find_elements_by_css_selector(self.attr_selector.format(self.color_attr_idx))
        conditions = self.driver.find_elements_by_css_selector(self.attr_selector.format(self.condition_attr_idx))
        for color in colors:
            color.click()
            time.sleep(2)
            for condition in conditions:
                condition.click()

                time.sleep(2)
                price = self.driver.find_element_by_css_selector(self.price_selector)

                device = Device()
                device.color = color.text
                device.condition = normalize_condition(GazelleSpider.name, condition.text)
                device.used_price = price.text

                device.url = response.url
                device.source_site = GazelleSpider.name
                device.seller = GazelleSpider.name
                device.stock = self.get_stock_status()

                device.model = model
                device.carrier = carrier
                device.capacity = capacity
                device.connectivity = connectivity if connectivity else specs['connectivity']
                device.processor = specs['processor']
                device.os = specs['os']
                device.memory = specs['memory']
                device.screen_size = specs['screen_size']
                device.resolution = specs['resolution']

                if re.match('iphone', model.lower()):
                    device.brand = 'Apple'
                elif re.match('ipad', model.lower()):
                    device.brand = 'Apple'
                elif re.match('galaxy', model.lower()):
                    device.brand = 'Samsung'

                device.save()
                logging.info(">>>>>>>> CREATE %s" % device)