def parse_campsite_detail(self, response):
campsite_detail_item = CampsiteDetailItem()
campsite_detail_item['unknown'] = {}
campsite_detail_units = response.xpath('//div[@id="divMobileunit"]/p')
# parse unit detail
while len(campsite_detail_units):
unit = campsite_detail_units.pop()
key = unit.xpath('text()').extract_first().strip()
value = unit.xpath('b/text()').extract_first().strip()
key = strings.snake_case(key)
if key in campsite_detail_item.fields:
campsite_detail_item[key] = value
elif key in equal_campsite_detail_keys:
campsite_detail_item[equal_campsite_detail_keys[key]] = value
else:
campsite_detail_item['unknown'][key] = value
# parse Amenities
campsite_detail_amenities = response.xpath('//div[@id="pnlAmenities"]/ul/li')
while len(campsite_detail_amenities):
amenity = campsite_detail_amenities.pop()
key = amenity.xpath('text()').extract_first().strip()
value = amenity.xpath('b/text()').extract_first().strip()
key = strings.snake_case(key)
if key in campsite_detail_item.fields:
campsite_detail_item[key] = value
elif key in equal_campsite_detail_keys:
campsite_detail_item[equal_campsite_detail_keys[key]] = value
else:
campsite_detail_item['unknown'][key] = value
return campsite_detail_item
def escape_column_name(field_code: str, do_snake_case: bool = True):
res = snake_case(field_code) if do_snake_case else field_code
res = ''.join([ch.lower() if ch.isalnum() else '_' for ch in res])
res = res if len(res) > 0 and res[0].isalpha() else '_' + res
while '__' in res:
res = res.replace('__', '_')
return res
def parse_campsite_detail(self, response):
details = response.xpath('//div[@id="sitedetail"]/ul/li/text()').extract()
campsite_detail_item = CampsiteDetailItem()
while len(details):
item = details.pop()
item = re.search('^([^:]+):\s*(.+)', item)
key = strings.snake_case(item.group(1))
value = item.group(2)
if key in campsite_detail_item.fields:
campsite_detail_item[key] = value
elif key in equal_campsite_detail_keys:
campsite_detail_item[equal_campsite_detail_keys[key]] = value
else:
campsite_detail_item['unknown'][key] = value
return campsite_detail_item
