def test_all_properties_exists(self):
stub_response = HtmlResponse(
url="",
body=(
r"<html>"
r"<body>"
r"<span id='ctl00_ContentPH_GoodsName'>ice cream</span>"
r"<span id='ctl00_ContentPH_BarCodeL'>2220066000747</span>"
r"<span id='ctl00_ContentPH_KeepingTime'>13</span>"
r"<span id='ctl00_ContentPH_Comment'>comment</span>"
r"<span id='ctl00_ContentPH_Composition'>one, two, three</span>"
r"<span id='ctl00_ContentPH_Net'>1000,00 g</span>"
r"<span id='ctl00_ContentPH_Gost'>TU 919191291</span>"
r"<span id='ctl00_ContentPH_StoreCond'>+25</span>"
r"<span id='ctl00_ContentPH_ESL'>Proteins: 10,00 g</span>"
r"<span id='ctl00_ContentPH_PackingType'>test</span>"
r"</body>"
r"</html>"))
self.assertEqual(
xpath_extractor.extract_goods_properties_dict(stub_response),
{
'name': 'ice cream',
'barcode': '2220066000747',
'best_before': '13',
'comment': 'comment',
'ingredients': 'one, two, three',
'netto_weight': '1000,00 g',
'standart': 'TU 919191291',
'store_conditions': '+25',
'esl_as_string': 'Proteins: 10,00 g',
'pack_type': 'test',
})
def parse_good(self, response):
log.msg("PARSE GOOD: {0}".format(response.url), level=log.DEBUG)
if self.close_down:
raise CloseSpider(str(self.close_exception))
good = GoodItem(xpath_extractor.extract_goods_properties_dict(response))
if good:
good['goodsmatrix_url'] = response.url
return good
else:
log.msg("can't parse {0}".format(response.url, level=log.ERROR))
def parse_good(self, response):
log.msg("PARSE GOOD: {0}".format(response.url), level=log.DEBUG)
if self.close_down:
raise CloseSpider(str(self.close_exception))
good = GoodItem(
xpath_extractor.extract_goods_properties_dict(response))
if good:
good['goodsmatrix_url'] = response.url
return good
else:
log.msg("can't parse {0}".format(response.url, level=log.ERROR))
def test_no_goods_properties_in_response(self):
stub_response = HtmlResponse(
url="",
body=(
r"<html>"
r"<body>"
r"</body>"
r"</html>"))
self.assertEqual(
xpath_extractor.extract_goods_properties_dict(stub_response),
dict()
)
def test_extract_value_with_html_tag_ontly(self):
stub_response = HtmlResponse(
url="",
body=(
r"<html>"
r"<body>"
r"<span id='ctl00_ContentPH_GoodsName'><br></span>"
r"</body>"
r"</html>")
)
self.assertEqual(
xpath_extractor.extract_goods_properties_dict(stub_response),
dict())
def test_extract_esl_as_string_concatinated_with_semicolon(self):
stub_response = HtmlResponse(
url="",
body=(
r"<html>"
r"<body>"
r"<span id='ctl00_ContentPH_ESL'>Proteins: 11,20 g<br />Fats: 22,90 g<br />Carbohydrates: 25,90 g<br />Calories: 354,50 kkal<br /></span>"
r"</body>"
r"</html>")
)
self.assertEqual(
xpath_extractor.extract_goods_properties_dict(stub_response),
{
'esl_as_string': 'Proteins: 11,20 g; Fats: 22,90 g; Carbohydrates: 25,90 g; Calories: 354,50 kkal'
})
def test_extract_paired_html_dropped_but_content_saved(self):
stub_response = HtmlResponse(
url="",
body=(
r"<html>"
r"<body>"
r"<span id='ctl00_ContentPH_GoodsName'>t<b>es</b>t</span>"
r"</body>"
r"</html>")
)
self.assertEqual(
xpath_extractor.extract_goods_properties_dict(stub_response),
{
'name': 'test',
})
def test_extract_special_charcter_decoded(self):
stub_response = HtmlResponse(
url="",
body=(
r"<html>"
r"<body>"
r"<span id='ctl00_ContentPH_GoodsName'><</span>"
r"</body>"
r"</html>")
)
self.assertEqual(
xpath_extractor.extract_goods_properties_dict(stub_response),
{
'name': '<',
})
def test_if_property_does_not_exist_then_it_is_not_added_to_dict(self):
stub_response = HtmlResponse(
url="",
body=(
r"<html>"
r"<body>"
r"<span id='ctl00_ContentPH_GoodsName'>ice cream</span>"
r"<span id='ctl00_ContentPH_BarCodeL'>2220066000747</span>"
r"</body>"
r"</html>"))
self.assertEqual(
xpath_extractor.extract_goods_properties_dict(stub_response),
{
'name': 'ice cream',
'barcode': '2220066000747',
})
def test_extract_only_first_occurance(self):
stub_response = HtmlResponse(
url="",
body=(
r"<html>"
r"<body>"
r"<span id='ctl00_ContentPH_GoodsName'>test1</span>"
r"<span id='ctl00_ContentPH_GoodsName'>test2</span>"
r"</body>"
r"</html>")
)
self.assertEqual(
xpath_extractor.extract_goods_properties_dict(stub_response),
{
'name': 'test1',
})