def test_ignore_field(self):
scraper_offer: ScraperOffer = {
"price": 22,
"UnitPrice": 12,
"sku": "12334323",
"ean": "Kjempeprodukt",
"additionalProperties": [{
"key": "PriceUnit",
"value": "m2"
}],
}
field_mapping: List[MappingConfigField] = [
{
"source": "PriceUnit",
"destination": "additionalProperties.priceUnit",
"replace_type": "key",
},
{
"destination": "ean",
"replace_type": "ignore",
},
]
actual = transform_fields(scraper_offer, field_mapping)
self.assertIsInstance(actual, dict)
self.assertIsNone(actual.get("ean"))
def test_not_replace_existing_field(self):
scraper_offer: ScraperOffer = {
"price":
22,
"UnitPrice":
12,
"sku":
"12334323",
"additionalProperties": [
{
"key": "Manufacturer Number",
"value": "JSK-3434"
},
{
"key": "AltPrice",
"value": 100
},
],
}
field_mapping: List[MappingConfigField] = [
{
"replace_type": "key",
"source": "Manufacturer Number",
"destination": "mpn",
},
{
"replace_type": "key",
"source": "AltPrice",
"destination": "price",
},
]
actual = transform_fields(scraper_offer, field_mapping)
self.assertIsInstance(actual, dict)
self.assertEqual(pydash.get(actual, "price"), 22)
def test_replace_with_fixed_value(self):
scraper_offer: ScraperOffer = {
"price": 22,
"UnitPrice": 12,
"sku": "12334323"
}
field_mapping: List[MappingConfigField] = [{
"destination": "dealer",
"replace_type": "fixed",
"replace_value": "byggmax",
}]
actual = transform_fields(scraper_offer, field_mapping)
self.assertIsInstance(actual, dict)
self.assertEqual(pydash.get(actual, "dealer"), "byggmax")
def test_replace_flat_field(self):
scraper_offer: ScraperOffer = {
"price": 22,
"UnitPrice": 12,
"sku": "12334323"
}
field_mapping: List[MappingConfigField] = [{
"replace_type": "key",
"source": "sku",
"destination": "ean"
}]
actual = transform_fields(scraper_offer, field_mapping)
self.assertIsInstance(actual, dict)
self.assertEqual(pydash.get(actual, "ean"), "12334323")
def test_replace_from_additional_property_when_not_present(self):
scraper_offer: ScraperOffer = {
"price": 22,
"UnitPrice": 12,
"sku": "12334323",
"additionalProperties": [],
}
field_mapping: List[MappingConfigField] = [{
"replace_type": "key",
"source": "Manufacturer Number",
"destination": "mpn",
}]
actual = transform_fields(scraper_offer, field_mapping)
self.assertIsInstance(actual, dict)
self.assertEqual(pydash.get(actual, "mpn"), None)
def test_replace_with_additional_property_destination(self):
scraper_offer: ScraperOffer = {
"price": 22,
"UnitPrice": 12,
"sku": "12334323",
"additionalProperties": [{
"key": "PriceUnit",
"value": "m2"
}],
}
field_mapping: List[MappingConfigField] = [{
"source": "PriceUnit",
"destination": "additionalProperties.priceUnit",
"replace_type": "key",
}]
actual = transform_fields(scraper_offer, field_mapping)
self.assertIsInstance(actual, dict)
self.assertIsInstance(pydash.get(actual, ["additionalPropertyDict"]),
dict)
self.assertEqual(
pydash.get(actual,
["additionalPropertyDict", "priceUnit", "value"]), "m2")
def test_replace_flat_field_2(self):
scraper_offer: ScraperOffer = {
"price": 22,
"UnitPrice": 12,
"sku": "12334323",
"image": "//coop.no/static/lfkjgf.png",
}
field_mapping: List[MappingConfigField] = [
{
"replace_type": "key",
"source": "sku",
"destination": "ean"
},
{
"replace_type": "key",
"source": "image",
"destination": "imageUrl"
},
]
actual = transform_fields(scraper_offer, field_mapping)
self.assertIsInstance(actual, dict)
self.assertEqual(pydash.get(actual, "imageUrl"),
"//coop.no/static/lfkjgf.png")
def transform_product(offer: ScraperOffer, config: HandleConfig) -> MpnOffer:
result: MpnOffer = {}
# Still handle Shopgun offers a little differently..
if "shopgun" in config["provenance"]:
result = transform_shopgun_product(offer, config)
else:
# Start here for everything not Shopgun offer.
offer = transform_fields(offer, config["fieldMapping"])
result: MpnOffer = {}
namespace = config["namespace"]
provenanceId = get_provenance_id(offer)
result["provenanceId"] = provenanceId
result["href"] = offer["url"]
result["ahref"] = offer.get("trackingUrl")
result["uri"] = get_product_uri(namespace, provenanceId)
result["pricing"] = get_product_pricing({**offer, **result})
result["validThrough"] = time.one_week_ahead
result["validFrom"] = time.time
result["dealer"] = offer.get("dealer", namespace)
result["gtins"] = get_gtins({**offer, **result})
try:
result["imageUrl"] = result["imageUrl"].replace(
"http://", "https://")
except Exception:
pass
# Handle quantity from scraper by parsing it like a string
extra_quantity_string = " ".join(
(str(offer[key])
for key in ("rawQuantity", "quantityValue", "quantityUnit")
if offer.get(key)), )
parse_quantity_strings = list(
get_field_from_scraper_offer(offer, key)
for key in config["extractQuantityFields"])
if extra_quantity_string:
parse_quantity_strings.append(extra_quantity_string)
safe_unit_list = ["l", "kg"
] if "grocery" in config["collection_name"] else None
parsed_quantity = parse_quantity(
list(x for x in parse_quantity_strings if x), safe_unit_list)
if config["ignore_none"]:
for k, v in parsed_quantity.items():
if remove_none_fields(v):
parsed_quantity[k] = v
else:
parsed_quantity[k] = {}
parsed_explicit_quantity = parse_explicit_quantity(offer, config)
parsed_quantity = {
**parsed_quantity,
**parsed_explicit_quantity,
}
result["mpnStock"] = get_stock_status(offer)
result["categories"] = get_categories(
pydash.get(offer, "categories"),
config["categoriesLimits"],
)
result = {**result, **parsed_quantity}
result["market"] = config["market"]
result["isPartner"] = config.get("is_partner", False)
result = analyze_quantity(result)
result = standardize_quantity(result)
quantity = result.get("quantity")
final_result = {**offer, **result, **quantity}
if config["ignore_none"]:
final_result = remove_none_fields(final_result)
return final_result