def detail_page(self,response):
desc1 = []
desc2 = []
for des1 in response.xpath("//div[@class='desc_content']/p/text()").extract():
desc1.append(des1.strip())
for des2 in response.xpath("//strong[text()='About the apartment:']/following::p/text()").extract():
desc2.append(des2.strip())
images = []
for img in response.xpath("//div[@class='slider_appart']/@style").extract():
images.append(img[16:-49])
aditem = AdItem(source = self.name,
title = response.xpath("//h2[@class='appName']/text()").extract_first(),
description = "".join(desc1) +"\n"+"".join(desc2),
monthly_rent = response.xpath("//p[contains(text(),'Monthly rent 6+ months:')]//following::p/text()").extract_first(),
# monthly_rent_extra_costs = monthly_rent_extra_costs,
# monthly_rent_total = response.xpath("//div[text()='Gesamtmiete:']//following::div[@class='artesia-value']/text()").extract_first(),
rooms = response.xpath("//p[contains(text(),'Bedrooms')]//following::p/a/text()").extract_first(),
# available_from = response.xpath("//div[text()='Verfügbarkeit:']//following::div[@class='artesia-value']/text()").extract_first(),
deposit = response.xpath("//p[contains(text(),'Security deposit:')]//following::p/text()").extract_first(),
size_m2 = response.xpath("//p[contains(text(),'Size')]//following::p/a/text()").extract_first(),
category = "rental_apartment",
floor = response.xpath("//p[contains(text(),'Floor')]//following::p/a/text()").extract_first(),
address = response.xpath("//div[@class='desc_content addresss']/text()").extract_first().strip(),
# reference_id =response.xpath("//td[text()='Kennung']/following::td/text()").extract_first().strip(),
detail_url = response.url,
images = images,
)
if TransformerTool.validate(aditem):
yield aditem
def detail_page(self, response):
# description = "".join(response.xpath('//div[@class="tab-content"]/div[1]/p/text()').extract())
# location = "".join(response.xpath('//div[@class="tab-content"]/div[3]/p/text()').extract())
# ref_id = response.xpath("//i[@class='obdt-id-text']/text()").extract_first()
address = response.xpath(
"//span[contains(text(),'Adresse:')]//following::div/span[@class='address']/text()"
).extract()
# address = address.replace("Etagenwohnung - -",'')
images = []
for img in response.xpath(
"//ul/li/a[contains(@href, '.jpg')]/@href").extract():
images.append(img)
aditem = AdItem(
source=self.name,
title=response.xpath(
"//h1[@class='title']/text()").extract_first().strip(),
description=response.xpath(
"//div[contains(text(),'Lageinformationen:')]//following::div[@class='field-items']/div/text()"
).extract_first(),
monthly_rent=response.xpath(
"//span[contains(text(),'Kaltmiete:')]//following::span/text()"
).extract_first(),
# monthly_rent_extra_costs = response.xpath("//dt[text()='Betriebskosten']//following::dd/p/text()").extract_first(),
monthly_rent_total=response.xpath(
"//span[contains(text(),'Warmmiete:')]//following::span/text()"
).extract_first(),
rooms=response.xpath(
"//span[contains(text(),'Zimmer:')]//following::span/text()").
extract_first() + " Zimmer",
available_from=response.xpath(
"//span[contains(text(),'Verfügbar ab')]//following::span/text()"
).extract_first(),
deposit=response.xpath(
"//span[contains(text(),'Kautionsbetrag:')]//following::span/text()"
).extract_first(),
size_m2=response.xpath(
"//span[contains(text(),'fläche:')]//following::span/text()").
extract_first(),
category="rental_apartment",
floor=response.xpath(
"//span[text()='Etage: ']//following::span/text()"
).extract_first(),
address=address[1].strip() + " " + address[0].strip(),
reference_id=response.xpath(
"//span[@class='label']//following::span/text()"
).extract_first(),
detail_url=response.url,
images=images,
)
if TransformerTool.validate(aditem):
yield aditem
def detail_page(self, response):
monthly_rent = response.xpath(
"//div[text()='Kaltmiete']/following-sibling::div/text()"
).extract_first()
monthly_rent_total = response.xpath(
"//div[text()='Warmmiete']/following-sibling::div/text()"
).extract_first()
if monthly_rent is None:
monthly_rent = monthly_rent_total
images = []
for img in response.xpath(
"//div[@id='immomakler-galleria']//a/@href").extract():
images.append(img)
aditem = AdItem(
source=self.name,
title=response.xpath(
"//h1[@class='property-title']/text()").extract_first(),
description=response.xpath(
"//h3/following-sibling::p/text()").extract_first(),
monthly_rent=monthly_rent,
# monthly_rent_extra_costs = response.xpath("//dt[text()='Betriebskosten']//following::dd/p/text()").extract_first(),
monthly_rent_total=monthly_rent_total,
rooms=response.xpath(
"//div[text()='Zimmer']/following-sibling::div/text()"
).extract_first() + " Zimmer",
available_from=response.xpath(
"//div[text()='Verfügbar ab']/following-sibling::div/text()"
).extract_first(),
deposit=response.xpath(
"//div[text()='Kaution']/following-sibling::div/text()").
extract_first(),
size_m2=response.xpath(
"//div[contains(text(),'fläche')]/following-sibling::div/text()"
).extract_first(),
category="rental_apartment",
floor=response.xpath(
"//div[text()='Etage']/following-sibling::div/text()").
extract_first(),
address=response.xpath(
"//h2[@class='property-subtitle']/text()").extract_first(),
reference_id=response.xpath(
"//div[contains(text(),'ID')]/following-sibling::div/text()"
).extract_first(),
detail_url=response.url,
images=images,
)
if TransformerTool.validate(aditem):
yield aditem
def detail_page(self, response):
images = []
for img in response.xpath(
"//div[@class='item']//img[contains(@src,'.JPG')]/@src"
).extract():
img = 'http://www.a1immobilien.de' + img
images.append(img)
aditem = AdItem(
source=self.name,
title=response.xpath("//h2/text()").extract_first().strip(),
#description = description + location,
monthly_rent=response.xpath(
"//td/b[text()='Kaltmiete:']//following::td/text()"
).extract_first(),
monthly_rent_extra_costs=response.xpath(
"//td/b[text()='Nebenkosten:']//following::td/text()"
).extract_first(),
monthly_rent_total=response.xpath(
"//td/b[text()='Warmmiete:']//following::td/text()"
).extract_first(),
rooms=response.xpath(
"//td/b[text()='Zimmer:']//following::td/text()"
).extract_first().replace('.00', ''),
available_from=response.xpath(
"//td/b[text()='Bezug ab:']//following::td/text()").
extract_first(),
deposit=response.xpath(
"//td/b[text()='Kaution:']//following::td/text()").
extract_first(),
size_m2=response.xpath(
"//td/b[text()='Wohnfläche:']//following::td/text()"
).extract_first(),
category="rental_apartment",
floor=response.xpath(
"//td/b[text()='Etage:']//following::td/text()").extract_first(
),
address=response.xpath(
"//td/b[text()='Lage:']//following::td/text()").extract_first(
).strip(),
#reference_id =response.xpath("//td[text()='Kennung']/following::td/text()").extract_first().strip(),
detail_url=response.url,
images=images,
)
if TransformerTool.validate(aditem):
yield aditem
def detail_page(self, response):
title = response.xpath("//title/text()").extract_first()
monthly_rent_extra_costs = response.xpath(
"//div[text()='Nebenkosten:']//following::div[@class='artesia-value']/text()"
).extract_first()
title = title.replace("- Artesia", "")
if monthly_rent_extra_costs is None:
monthly_rent_extra_costs = "0"
images = []
for img in response.xpath(
"//a[contains(@href,'.png')]/@href").extract():
images.append(img)
aditem = AdItem(
source=self.name,
title=title,
description=response.xpath(
"//h3[text()='Lagebeschreibung']//following::p/span/text()").
extract_first(),
monthly_rent=response.xpath(
"//div[text()='Kaltmiete:']//following::div[@class='artesia-value']/text()"
).extract_first(),
monthly_rent_extra_costs=monthly_rent_extra_costs,
monthly_rent_total=response.xpath(
"//div[text()='Gesamtmiete:']//following::div[@class='artesia-value']/text()"
).extract_first(),
rooms=response.xpath(
"//div[text()='Zimmer:']//following::div[@class='artesia-value']/text()"
).extract_first(),
available_from=response.xpath(
"//div[text()='Verfügbarkeit:']//following::div[@class='artesia-value']/text()"
).extract_first(),
# deposit = response.xpath("//td[contains(text(),'Kaution')]/following::td/text()").extract_first(),
size_m2=response.xpath(
"//div[text()='Wohnfläche:']//following::div[@class='artesia-value']/text()"
).extract_first(),
category="rental_apartment",
# floor = response.xpath("//td[text()='Etage']/following::td/text()").extract_first(),
address="Berlin",
# reference_id =response.xpath("//td[text()='Kennung']/following::td/text()").extract_first().strip(),
detail_url=response.url,
images=images,
)
if TransformerTool.validate(aditem):
yield aditem
def detail_page(self, response):
images = []
for img in response.xpath(
"//a[contains(@href,'.jpg')]/@href").extract():
images.append(img)
aditem = AdItem(
source=self.name,
title=response.xpath("//h2/text()").extract_first(),
description=response.xpath(
"//h3[text()='Beschreibung']//following::p/text()"
).extract_first(),
monthly_rent=response.xpath(
"//div[contains(text(),'Kaltmiete')]/following-sibling::div/text()"
).extract_first(),
monthly_rent_extra_costs=response.xpath(
"//div[contains(text(),'Betriebskosten ')]/following-sibling::div/text()"
).extract_first(),
monthly_rent_total=response.xpath(
"//div[contains(text(),'Gesamtbelastung ')]/following-sibling::div/text()"
).extract_first(),
rooms=response.xpath(
"//div[text()='Zimmer']/following-sibling::div/text()"
).extract_first(),
available_from=response.xpath(
"//div[text()='Verfügbar ab']/following-sibling::div/text()").
extract_first(),
deposit=response.xpath(
"//div[text()='Kaution']/following-sibling::div/text()").
extract_first(),
size_m2=response.xpath(
"//div[contains(text(),'Wohnfläche')]/following-sibling::div/text()"
).extract_first(),
category="rental_apartment",
# floor = response.xpath("//td[text()='Etage']/following::td/text()").extract_first(),
address=response.xpath("//h3/text()").extract_first(),
reference_id=response.xpath(
"//div[contains(text(),'Objekt ID')]/following::div/text()"
).extract_first().strip(),
detail_url=response.url,
images=images,
)
if TransformerTool.validate(aditem):
yield aditem
def detail_page(self, response):
images = []
for img in response.xpath("//div/@data-image-size-tall").extract():
img = img.replace("//", "")
images.append(img)
aditem = AdItem(
source=self.name,
title=response.xpath("//h1/text()").extract_first().strip(),
# description = description + location,
monthly_rent=response.xpath(
"//td[contains(text(),'Miete')]/following::td/text()"
).extract_first(),
monthly_rent_extra_costs=response.xpath(
"//td[contains(text(),'Nebenkosten')]/following::td/text()"
).extract_first(),
monthly_rent_total=response.xpath(
"//td[contains(text(),'Warmmiete')]/following::td/text()").
extract_first(),
rooms=response.xpath(
"//td[contains(text(),'Zimmer')]/following::td/text()").
extract_first(),
# available_from = response.xpath("//td[text()='Free From']/following::td/text()").extract_first(),
deposit=response.xpath(
"//td[contains(text(),'Kaution')]/following::td/text()"
).extract_first(),
size_m2=response.xpath(
"//td[contains(text(),'fläche')]/following::td/text()"
).extract_first(),
category="rental_apartment",
floor=response.xpath(
"//td[text()='Etage']/following::td/text()").extract_first(),
address=response.xpath("//td[text()='Lage']/following::td/text()"
).extract_first().strip(),
reference_id=response.xpath(
"//td[text()='Kennung']/following::td/text()").extract_first(
).strip(),
detail_url=response.url,
images=images,
)
if TransformerTool.validate(aditem):
yield aditem
def detail_page(self, response):
description = response.xpath(
"//div[@class='detailTxt']/p/text()").extract()
monthly_rent = response.xpath(
"//td[text()='Preis:']//following::td/text()").extract_first(
).replace(' €', '')
monthly_rent_extra_costs = response.xpath(
"//td[text()='Nebenkosten:']//following::td/text()").extract_first(
).replace(' €', '')
monthly_rent_total = int(monthly_rent) + int(monthly_rent_extra_costs)
images = []
for img in response.xpath("//div[@class='bilddiv']/a/@href").extract():
img = 'http://www.birnstiel-immobilien.de' + img.replace('..', '')
images.append(img)
aditem = AdItem(
source=self.name,
title=response.xpath("//h1/text()").extract_first(),
description=description[:2],
monthly_rent=monthly_rent + ' €',
monthly_rent_extra_costs=monthly_rent_extra_costs + ' €',
monthly_rent_total=str(monthly_rent_total) + ' €',
rooms=response.xpath("//td[text()='Zimmer:']//following::td/text()"
).extract_first(),
# available_from = response.xpath("//td[text()='Free From']/following::td/text()").extract_first(),
# deposit = response.xpath("//td[contains(text(),'Kaution')]/following::td/text()").extract_first(),
size_m2=response.xpath(
"//td[text()='Wohnfläche: ']//following::td/text()").
extract_first(),
category="rental_apartment",
# floor = response.xpath("//td[text()='Etage']/following::td/text()").extract_first(),
address=response.xpath(
"//td[text()='Anschrift']//following::td/text()").
extract_first().strip(),
reference_id=response.xpath(
"//td[text()='Id: ']//following::td/text()").extract_first(
).strip(),
detail_url=response.url,
images=images,
)
if TransformerTool.validate(aditem):
yield aditem