def test_ignore_field(self): scraper_offer: ScraperOffer = { "price": 22, "UnitPrice": 12, "sku": "12334323", "ean": "Kjempeprodukt", "additionalProperties": [{ "key": "PriceUnit", "value": "m2" }], } field_mapping: List[MappingConfigField] = [ { "source": "PriceUnit", "destination": "additionalProperties.priceUnit", "replace_type": "key", }, { "destination": "ean", "replace_type": "ignore", }, ] actual = transform_fields(scraper_offer, field_mapping) self.assertIsInstance(actual, dict) self.assertIsNone(actual.get("ean"))
def test_not_replace_existing_field(self): scraper_offer: ScraperOffer = { "price": 22, "UnitPrice": 12, "sku": "12334323", "additionalProperties": [ { "key": "Manufacturer Number", "value": "JSK-3434" }, { "key": "AltPrice", "value": 100 }, ], } field_mapping: List[MappingConfigField] = [ { "replace_type": "key", "source": "Manufacturer Number", "destination": "mpn", }, { "replace_type": "key", "source": "AltPrice", "destination": "price", }, ] actual = transform_fields(scraper_offer, field_mapping) self.assertIsInstance(actual, dict) self.assertEqual(pydash.get(actual, "price"), 22)
def test_replace_with_fixed_value(self): scraper_offer: ScraperOffer = { "price": 22, "UnitPrice": 12, "sku": "12334323" } field_mapping: List[MappingConfigField] = [{ "destination": "dealer", "replace_type": "fixed", "replace_value": "byggmax", }] actual = transform_fields(scraper_offer, field_mapping) self.assertIsInstance(actual, dict) self.assertEqual(pydash.get(actual, "dealer"), "byggmax")
def test_replace_flat_field(self): scraper_offer: ScraperOffer = { "price": 22, "UnitPrice": 12, "sku": "12334323" } field_mapping: List[MappingConfigField] = [{ "replace_type": "key", "source": "sku", "destination": "ean" }] actual = transform_fields(scraper_offer, field_mapping) self.assertIsInstance(actual, dict) self.assertEqual(pydash.get(actual, "ean"), "12334323")
def test_replace_from_additional_property_when_not_present(self): scraper_offer: ScraperOffer = { "price": 22, "UnitPrice": 12, "sku": "12334323", "additionalProperties": [], } field_mapping: List[MappingConfigField] = [{ "replace_type": "key", "source": "Manufacturer Number", "destination": "mpn", }] actual = transform_fields(scraper_offer, field_mapping) self.assertIsInstance(actual, dict) self.assertEqual(pydash.get(actual, "mpn"), None)
def test_replace_with_additional_property_destination(self): scraper_offer: ScraperOffer = { "price": 22, "UnitPrice": 12, "sku": "12334323", "additionalProperties": [{ "key": "PriceUnit", "value": "m2" }], } field_mapping: List[MappingConfigField] = [{ "source": "PriceUnit", "destination": "additionalProperties.priceUnit", "replace_type": "key", }] actual = transform_fields(scraper_offer, field_mapping) self.assertIsInstance(actual, dict) self.assertIsInstance(pydash.get(actual, ["additionalPropertyDict"]), dict) self.assertEqual( pydash.get(actual, ["additionalPropertyDict", "priceUnit", "value"]), "m2")
def test_replace_flat_field_2(self): scraper_offer: ScraperOffer = { "price": 22, "UnitPrice": 12, "sku": "12334323", "image": "//coop.no/static/lfkjgf.png", } field_mapping: List[MappingConfigField] = [ { "replace_type": "key", "source": "sku", "destination": "ean" }, { "replace_type": "key", "source": "image", "destination": "imageUrl" }, ] actual = transform_fields(scraper_offer, field_mapping) self.assertIsInstance(actual, dict) self.assertEqual(pydash.get(actual, "imageUrl"), "//coop.no/static/lfkjgf.png")
def transform_product(offer: ScraperOffer, config: HandleConfig) -> MpnOffer: result: MpnOffer = {} # Still handle Shopgun offers a little differently.. if "shopgun" in config["provenance"]: result = transform_shopgun_product(offer, config) else: # Start here for everything not Shopgun offer. offer = transform_fields(offer, config["fieldMapping"]) result: MpnOffer = {} namespace = config["namespace"] provenanceId = get_provenance_id(offer) result["provenanceId"] = provenanceId result["href"] = offer["url"] result["ahref"] = offer.get("trackingUrl") result["uri"] = get_product_uri(namespace, provenanceId) result["pricing"] = get_product_pricing({**offer, **result}) result["validThrough"] = time.one_week_ahead result["validFrom"] = time.time result["dealer"] = offer.get("dealer", namespace) result["gtins"] = get_gtins({**offer, **result}) try: result["imageUrl"] = result["imageUrl"].replace( "http://", "https://") except Exception: pass # Handle quantity from scraper by parsing it like a string extra_quantity_string = " ".join( (str(offer[key]) for key in ("rawQuantity", "quantityValue", "quantityUnit") if offer.get(key)), ) parse_quantity_strings = list( get_field_from_scraper_offer(offer, key) for key in config["extractQuantityFields"]) if extra_quantity_string: parse_quantity_strings.append(extra_quantity_string) safe_unit_list = ["l", "kg" ] if "grocery" in config["collection_name"] else None parsed_quantity = parse_quantity( list(x for x in parse_quantity_strings if x), safe_unit_list) if config["ignore_none"]: for k, v in parsed_quantity.items(): if remove_none_fields(v): parsed_quantity[k] = v else: parsed_quantity[k] = {} parsed_explicit_quantity = parse_explicit_quantity(offer, config) parsed_quantity = { **parsed_quantity, **parsed_explicit_quantity, } result["mpnStock"] = get_stock_status(offer) result["categories"] = get_categories( pydash.get(offer, "categories"), config["categoriesLimits"], ) result = {**result, **parsed_quantity} result["market"] = config["market"] result["isPartner"] = config.get("is_partner", False) result = analyze_quantity(result) result = standardize_quantity(result) quantity = result.get("quantity") final_result = {**offer, **result, **quantity} if config["ignore_none"]: final_result = remove_none_fields(final_result) return final_result