示例#1
0
    def parse_activity(self, response):
        hxs = HtmlXPathSelector(response)

        l = ActivityItemLoader(item=ActivityItem(), response=response)
        l.add_xpath('name', '//h2[@class="itemTitle"]/text()')
        l.add_xpath('address', '//p[@class="listingAddress"]/text()')
        l.add_xpath('description', '//div[@class="overviewBottom"]/p/text()')

        l.add_value('source_url', response.url)

        type = hxs.select('//p[@class="lodgingtype"]/text()').extract()[0]

        item = l.load_item()
示例#2
0
    def parse_activity(self, response):
        hxs = HtmlXPathSelector(response)

        l = ActivityItemLoader(item=ActivityItem(), response=response)
        l.add_xpath('name', '//h2[@class="itemTitle"]/text()')
        l.add_xpath('address', '//p[@class="listingAddress"]/text()')
        l.add_xpath('description', '//div[@class="overviewBottom"]/p/text()')

        l.add_value('source_url', response.url)

        type = hxs.select('//p[@class="lodgingtype"]/text()').extract()[0]

        item = l.load_item()
示例#3
0
    def parse_activity(self, response):
        l = ActivityItemLoader(item=ActivityItem(), response=response)
        l.add_xpath('name', '//h1[@class="main-page-title"]/text()')

        # Address
        l.add_xpath('address',
                    '//div[@class="product-summary-list-box"]/div[1]/text()')

        # City
        l.add_xpath(
            'address',
            '//div[@id="product-location"]/div/div[@class="street-address"]/following-sibling::div/span[@class="locality"]/text()'
        )

        # State
        l.add_xpath(
            'address',
            '//div[@id="product-location"]/div/div[@class="street-address"]/following-sibling::div/abbr[@class="region"]/text()'
        )

        # Zip Code
        l.add_xpath(
            'address',
            '//div[@id="product-location"]/div/div[@class="street-address"]/following-sibling::div/span[@class="postal-code"]/text()'
        )
        l.add_xpath(
            'description',
            '//div[@class="product-details-description layout-module expandable collapsed"]/p/text()'
        )
        l.add_xpath(
            'time_needed',
            '//strong[contains(text(), "Tour Length: ")]/following-sibling::div/text()'
        )
        l.add_xpath('price', '//div[@class="fromprice-price"]/text()')

        l.add_value('source_url', response.url)

        referer = response.request.headers.get('Referer', None)

        if referer:
            # If available, set categories and price range
            if referer in URL_TO_CATEGORIES_MAP:
                l.add_value('categories', URL_TO_CATEGORIES_MAP[referer])

            if referer in URL_TO_PRICE_RANGE_MAP:
                l.add_value('price', URL_TO_PRICE_RANGE_MAP[referer])

        item = l.load_item()
        name = item['name']

        if self._flickr_api_key:
            images = get_images_for_term(self._flickr_api_key,
                                         search_term=name,
                                         bbox=self._nevada_bbox)
            image_urls = [item['url'] for item in images]
            l.add_value('image_urls', image_urls)
            item = l.load_item()

        return item
示例#4
0
    def parse_activity(self, response):
        l = ActivityItemLoader(item=ActivityItem(), response=response)
        l.add_xpath('name', '//h1[@class="main-page-title"]/text()')

        # Address
        l.add_xpath('address', '//div[@class="product-summary-list-box"]/div[1]/text()')

        # City
        l.add_xpath('address', '//div[@id="product-location"]/div/div[@class="street-address"]/following-sibling::div/span[@class="locality"]/text()')

        # State
        l.add_xpath('address', '//div[@id="product-location"]/div/div[@class="street-address"]/following-sibling::div/abbr[@class="region"]/text()')

        # Zip Code
        l.add_xpath('address', '//div[@id="product-location"]/div/div[@class="street-address"]/following-sibling::div/span[@class="postal-code"]/text()')
        l.add_xpath('description', '//div[@class="product-details-description layout-module expandable collapsed"]/p/text()')
        l.add_xpath('time_needed', '//strong[contains(text(), "Tour Length: ")]/following-sibling::div/text()')
        l.add_xpath('price', '//div[@class="fromprice-price"]/text()')

        l.add_value('source_url', response.url)

        referer = response.request.headers.get('Referer', None)

        if referer:
            # If available, set categories and price range
            if referer in URL_TO_CATEGORIES_MAP:
                l.add_value('categories', URL_TO_CATEGORIES_MAP[referer])

            if referer in URL_TO_PRICE_RANGE_MAP:
                l.add_value('price', URL_TO_PRICE_RANGE_MAP[referer])

        item = l.load_item()
        name = item['name']

        if self._flickr_api_key:
            images = get_images_for_term(self._flickr_api_key, search_term=name,
                                         bbox=self._nevada_bbox)
            image_urls = [item['url'] for item in images]
            l.add_value('image_urls', image_urls)
            item = l.load_item()

        return item