def test_keep_single_value(self):
"""Loaded item should contain values from the initial item"""
input_item = self.item_class(name='foo')
il = ItemLoader(item=input_item)
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(dict(loaded_item), {'name': ['foo']})
def test_add_value_list_singlevalue(self):
"""Values added after initialization should be appended"""
input_item = self.item_class(name=['foo', 'bar'])
il = ItemLoader(item=input_item)
il.add_value('name', 'qwerty')
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(dict(loaded_item), {'name': ['foo', 'bar', 'qwerty']})
def test_get_output_value_list(self):
"""Getting output value must not remove value from item"""
input_item = self.item_class(name=['foo', 'bar'])
il = ItemLoader(item=input_item)
self.assertEqual(il.get_output_value('name'), ['foo', 'bar'])
loaded_item = il.load_item()
self.assertIsInstance(loaded_item, self.item_class)
self.assertEqual(loaded_item, dict({'name': ['foo', 'bar']}))
def get_payments(self, payments) -> PaymentMethod:
if payments:
for payment in payments:
loader = ItemLoader(item=PaymentMethod())
loader.add_value('method', payment.get('method'))
loader.add_value('installments',
self.get_installments(payment))
yield loader.load_item()
def parse_single_hausnummer_page(self, response):
meta = response.meta
loader = ItemLoader(items.Street(), response)
loader.default_output_processor = TakeFirst()
loader.add_value('stadtteil', meta['stadtteil'])
loader.add_value('name', meta['adresse'])
loader.add_value('link', meta['link'])
for script in response.css("script[type='text/javascript']").getall():
if "demographicInfo" in script:
loader.add_value('demographics', self.regex_demographics.search(script).group())
break
yield loader.load_item()
def parse_application(self, response):
loader = ItemLoader(item=TraidingSsgeItem(), selector=response)
loader._add_value('statement_date', response.request.meta['appdate'])
loader.add_xpath('new_or_used',
"normalize-space(//div[@class='condition']/text())")
loader.add_xpath('location',
"//div[@class='location-time']/div[2]/p/span/text()")
loader.add_xpath(
'last_updated',
'normalize-space(//div[@class="location-time"]/div[2]/descendant::span[2]/text())'
)
loader.add_xpath('product',
"normalize-space(//h2[@class='main-title']/text())")
loader.add_xpath(
'price',
"normalize-space(//div[@class='market-item-price ']/text())")
loader.add_xpath(
'currency_symbol',
"normalize-space(//div[@class='market-item-price ']/span/text())")
loader.add_xpath(
'applicant', "normalize-space(//div[@class='author_type']/text())")
loader.add_value('current_app_url', response.url)
loader.add_xpath(
'all_apps_url',
"//div[@class='author_type']/descendant::span/a/@href")
loader.add_xpath(
'agent_or_person',
"normalize-space((//div[@class='author_type'])[1]/span/a/text())")
loader.add_xpath(
'number_of_apps',
"normalize-space(//div[@class='author_type']/descendant::span[2]/text())"
)
loader.add_xpath(
'product_description',
"normalize-space(//span[@class='details_text']/text())")
loader.add_xpath(
'product_specification',
"normalize-space(//div[@class='jobs_details']/span/text())")
loader.add_xpath(
'product_condition_description',
"normalize-space(//div[@class='jobs_details'][2]/span[2]/text())")
loader.add_xpath(
'seen',
"normalize-space(//div[@class='article_views']/span/text())")
loader.add_xpath(
'app_id',
"normalize-space(//div[@class='market-item-id']/span/text())")
loader.add_xpath(
'phone', "normalize-space(//div[@class='numbers-wrap']/a/@href)")
print(loader.item)
yield loader.load_item()
def test_get_value(self):
il = ItemLoader()
self.assertEqual('FOO',
il.get_value(['foo', 'bar'], TakeFirst(), str.upper))
self.assertEqual(['foo', 'bar'],
il.get_value(['name:foo', 'name:bar'],
re='name:(.*)$'))
self.assertEqual(
'foo',
il.get_value(['name:foo', 'name:bar'],
TakeFirst(),
re='name:(.*)$'))
il.add_value('name', ['name:foo', 'name:bar'],
TakeFirst(),
re='name:(.*)$')
self.assertEqual(['foo'], il.get_collected_values('name'))
il.replace_value('name', 'name:bar', re='name:(.*)$')
self.assertEqual(['bar'], il.get_collected_values('name'))
def parse(self, response):
current_page = response.meta['currentPage']
json_resp = json.loads(response.text)
houses = json_resp['cat1']['searchResults']['listResults']
total_pages = json_resp['cat1']['searchList']['totalPages']
for house in houses:
loader = ItemLoader(item=ZillowItem())
loader.add_value('id', house.get('id'))
loader.add_value('image_urls', house.get('imgSrc'))
loader.add_value('detail_url', house.get('detailUrl'))
loader.add_value('status_type', house.get('statusType'))
loader.add_value('status_text', house.get('statusText'))
loader.add_value('price', house.get('price'))
loader.add_value('address', house.get('address'))
loader.add_value('beds', house.get('beds'))
loader.add_value('baths', house.get('baths'))
loader.add_value('area_sqft', house.get('area'))
loader.add_value('latitude', house.get('latLong').get('latitude'))
loader.add_value('longitude', house.get('latLong').get('longitude'))
loader.add_value('broker_name', house.get('brokerName'))
loader.add_value('broker_phone', house.get('brokerPhone'))
yield loader.load_item()
print({
"houses": len(houses),
"current_page": current_page,
"total_pages": total_pages
})
if current_page <= total_pages:
current_page += 1
yield scrapy.Request(
url=parse_new_url(URL, page_number=current_page),
callback=self.parse,
cookies=get_cookie(),
meta={
'currentPage': current_page
}
)
def test_values_list(self):
"""Values from initial item must be added to loader._values"""
input_item = self.item_class(name=['foo', 'bar'])
il = ItemLoader(item=input_item)
self.assertEqual(il._values.get('name'), ['foo', 'bar'])