def _optimize(self, product_batch: Dict[str, Any], language: str, _) -> int: """Runs the optimization. Fixes invalid gtin fields. See above for the definition of an invalid gtin field. Args: product_batch: A batch of product data. language: The language to use for this optimizer. Returns: The number of products affected by this optimization: int """ num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] if 'gtin' in product: gtin = product.get('gtin', '') if _gtin_passes_format_check(gtin) and _gtin_passes_checksum( gtin): continue violating_gtin = product.get('gtin', '') del product['gtin'] logging.info( 'Modified item %s: Cleared invalid gtin: %s to ' 'prevent disapproval', product.get('offerId', ''), violating_gtin) base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) num_of_products_optimized += 1 return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, _) -> int: """Runs the optimization. Removes invalid identifierExists fields. See above for the definition of an invalid identifierExists field. Args: product_batch: A batch of product data. language: The language to use for this optimizer. Returns: The number of products affected by this optimization: int """ num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] identifier_exists = product.get('identifierExists', True) brand = product.get('brand', '') gtin = product.get('gtin', '') mpn = product.get('mpn', '') if not identifier_exists and (brand or gtin or mpn): item_id = product.get('offerId', '') logging.info( 'Modified item %s: Clearing identifierExists ' 'to prevent disapproval', item_id) # Delete field from the request which defaults it to true. del product['identifierExists'] num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, country: str, currency: str) -> int: """Runs the optimization. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization. """ num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] if 'productTypes' in product: product_types = product.get('productTypes', []) optimized_product_types = ( optimization_util.cut_list_to_limit_list_length( product_types, _MAX_LIST_LENGTH)) if optimized_product_types != product_types: product['productTypes'] = optimized_product_types logging.info( 'Modified item %s: Removing the last items from productTypes: %s', product.get('offerId', ''), product_types) num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, country: str, currency: str) -> int: """Runs the optimization. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization. """ num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] original_title = product.get('title', '') _remove_unnecessary_text(product, language) if product.get('title', '') != original_title: logging.info( 'Modified item %s: Removed promo text, new title is: %s', product.get('offerId', ''), product.get('title', '')) num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) return num_of_products_optimized
def _append_attributes_to_title(self, product: Dict[str, Any], chars_to_preserve: int, language: str, country: str) -> None: """Appends mined attributes to the title. Args: product: Product data. chars_to_preserve: The num of chars to leave unchanged (used to make sure the original title is preserved). language: The language to use for this optimizer. country: The country to use for this optimizer. """ if not self._mined_attributes: return product_id = product.get('offerId', '') size_checker = size_miner.SizeMiner(language, country) fields_to_append_to_title = [] for attribute_name, mined_attribute_values in self._mined_attributes.get( product_id, {}).items(): if attribute_name == 'sizes' and size_checker.is_size_in_attribute( product, 'title'): # Does not append the size when size information is already in title. continue if isinstance(mined_attribute_values, str): fields_to_append_to_title.append(mined_attribute_values) elif isinstance(mined_attribute_values, list): fields_to_append_to_title.extend(mined_attribute_values) if fields_to_append_to_title: base_optimizer.set_optimization_tracking(product, base_optimizer.OPTIMIZED) _append_fields_to_title(product, fields_to_append_to_title, chars_to_preserve)
def _optimize(self, product_batch: Dict[str, Any], language: str) -> int: for entry in product_batch['entries']: product = entry['product'] base_optimizer.set_optimization_tracking(product, base_optimizer.SANITIZED) return len(product_batch) # Number of items Sanitized.
def _optimize(self, product_batch: Dict[str, Any], language: str, _) -> int: """Runs the optimization. Args: product_batch: A batch of product data. language: The language to use for this optimizer. Returns: The number of products affected by this optimization: int """ num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] if 'mpn' in product: mpn_value = _normalize_mpn(product.get('mpn', '')) if mpn_value in INVALID_MPN_VALUES: item_id = product.get('offerId') logging.info('Modified item %s: Removing invalid MPN [%s]', item_id, mpn_value) del product['mpn'] num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, _) -> int: """Runs the optimization. Removes invalid chars from the title and description. See above for the definition of an invalid char. Args: product_batch: A batch of product data. language: The language to use for this optimizer. Returns: The number of products affected by this optimization: int """ num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] product_was_sanitized = _sanitize_fields(product, _FIELDS_TO_SANITIZE) if product_was_sanitized: num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, _) -> int: """Runs the optimization. Fixes invalid condition values. See above for the definition of an invalid condition value. Args: product_batch: A batch of product data. language: The language to use for this optimizer. Returns: The number of products affected by this optimization. """ num_of_products_optimized = 0 self._condition_config = current_app.config.get('CONFIGS', {}).get( f'condition_optimizer_config_{language}', {}) for entry in product_batch['entries']: product = entry['product'] google_product_category = product.get('googleProductCategory', '') if self._is_google_product_category_excluded(google_product_category): logging.info( 'Product ID: %s With Category %s was flagged for exclusion ' ' of the condition check', product.get('offerId', ''), google_product_category) continue used_tokens = set( token.lower() for token in self._condition_config['used_tokens']) logging.info('Used tokens were %s', used_tokens) if product.get('condition', '') == _NEW: # Category format must follow the official spec to be converted a list. # Ref: https://support.google.com/merchants/answer/6324436?hl=en. product_categories = google_product_category.split(' > ') if isinstance(product_categories, list) and product_categories: lowest_level_category = product_categories[-1] category_specific_tokens = self._get_tokens_for_category( lowest_level_category) if category_specific_tokens: category_specific_tokens = set( token.lower() for token in category_specific_tokens) used_tokens.update(category_specific_tokens) # Search for used tokens in both title and description and reset the # condition to used if any were detected. product_title = product.get('title', '') product_description = product.get('description', '') if self._field_contains_used_tokens( product_title, used_tokens) or self._field_contains_used_tokens( product_description, used_tokens): product['condition'] = _USED logging.info('Modified item %s: Setting new product to used.', product.get('offerId', '')) num_of_products_optimized += 1 base_optimizer.set_optimization_tracking(product, base_optimizer.SANITIZED) return num_of_products_optimized
def _update_product_description(product: Dict[str, Any], optimized_description: str) -> None: """Updates the description for the given product and sets tracking/logging. Args: product: Product data. optimized_description: The new product description. """ product['description'] = optimized_description base_optimizer.set_optimization_tracking(product, base_optimizer.OPTIMIZED) logging.info('Modified item %s: Appended field values to description: %s', product['offerId'], product['description'])
def _remove_gtin(product: Dict[str, Any]) -> None: """Clears the gtin value from the product. Args: product: A dictionary representing a single shopping product. """ violating_gtin = product.get('gtin', '') del product['gtin'] logging.info( 'Modified item %s: Cleared invalid gtin: %s to ' 'prevent disapproval', product.get('offerId', ''), violating_gtin) base_optimizer.set_optimization_tracking(product, base_optimizer.SANITIZED)
def _truncate_to_max_length(product: Dict[str, Any]) -> None: """Truncate the title to the max length: 150. This function prevents a product with a 150+ title length from being disapproved. Reference: https://support.google.com/merchants/answer/6324415 Args: product: A dictionary containing product data. """ title = product.get('title', '') if len(title) > _MAX_TITLE_LENGTH: product['title'] = title[:_MAX_TITLE_LENGTH] logging.info('Modified item %s: Truncating title: %s', product.get('offerId', ''), product.get('title', '')) base_optimizer.set_optimization_tracking(product, base_optimizer.SANITIZED)
def _optimize(self, product_batch: Dict[str, Any], language: str, country: str, currency: str) -> int: """Runs title length optimization. This method optimizes the title by executing following processes: - Truncates title to the max title length if its length exceeds the max value. - Populate title with description if title is truncated from description. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization. """ num_of_products_optimized: int = 0 entry: Dict[str, Any] for entry in product_batch['entries']: product = entry['product'] item_id: str = product.get('offerId', '') title: str = product.get('title', '') trailing_dots_removed_title: str = re.sub('[.…]+$', '', title) description: str = product.get('description', '') if len(title) > _MAX_TITLE_LENGTH: product['title'] = title[:_MAX_TITLE_LENGTH] logging.info('Modified item %s: Truncating title: %s', item_id, product['title']) num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) elif trailing_dots_removed_title != description and description.startswith( trailing_dots_removed_title): product['title'] = description[:_MAX_TITLE_LENGTH] logging.info( 'Modified item %s: Populating title with ' 'description due to detected title truncation: %s', item_id, product['title']) num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.OPTIMIZED) return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, country: str, currency: str) -> int: """Runs the optimization. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization. """ num_of_products_optimized = 0 self._shopping_exclusion_config = flask.current_app.config.get( 'CONFIGS', {}).get(f'shopping_exclusion_optimizer_config_{language}', {}) self.shopping_removal_patterns_exact_match = frozenset( self._shopping_exclusion_config.get( 'shopping_exclusion_patterns_exact_match', [])) for entry in product_batch['entries']: product = entry['product'] if self._is_non_shopping_product(product.get('title', '')): if isinstance( product.get('excludedDestinations'), list ) and _SHOPPING_ADS_DESTINATION not in product['excludedDestinations']: product['excludedDestinations'].append(_SHOPPING_ADS_DESTINATION) else: product['excludedDestinations'] = [_SHOPPING_ADS_DESTINATION] if isinstance( product.get('includedDestinations'), list ) and _SHOPPING_ADS_DESTINATION in product['includedDestinations']: product['includedDestinations'].remove(_SHOPPING_ADS_DESTINATION) num_of_products_optimized += 1 base_optimizer.set_optimization_tracking(product, base_optimizer.SANITIZED) logging.info( 'Product %s was detected as not meant for Shopping Ads, so excluding from the Shopping Ads destination', product.get('offerId')) return num_of_products_optimized
def _complement_title_with_description(product: Dict[str, Any]) -> None: """Complements title with description if title is truncated from description. This is expected to improve ad performance by adding more information to the title. Args: product: Product data. """ title_without_trailing_dots = re.sub('[.…]+$', '', product['title']) description = product.get('description', '') if title_without_trailing_dots != description and description.startswith( title_without_trailing_dots): product['title'] = description[:_MAX_TITLE_LENGTH] logging.info( 'Modified item %s: Populating title with description due to detected ' 'title truncation: %s', product['offerId'], product['title']) base_optimizer.set_optimization_tracking(product, base_optimizer.OPTIMIZED)
def _optimize(self, product_batch: Dict[str, Any], language: str, country: str, currency: str) -> int: """Runs the optimization. Fixes invalid color fields. See above for the definition of an invalid color field. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization: int """ num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] if 'color' in product: original_colors = product.get('color', '') colors = original_colors.split(_SEPARATOR) optimized_colors = optimization_util.cut_list_elements_over_max_length( colors, constants.MAX_COLOR_STR_LENGTH_FOR_EACH) optimized_colors = optimization_util.cut_list_to_limit_list_length( optimized_colors, constants.MAX_COLOR_COUNT) optimized_colors = optimization_util.cut_list_to_limit_concatenated_str_length( optimized_colors, _SEPARATOR, constants.MAX_COLOR_STR_LENGTH_IN_TOTAL) optimized_colors = _SEPARATOR.join(optimized_colors) if original_colors != optimized_colors: product['color'] = optimized_colors num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, country: str, currency: str) -> int: """Runs the optimization. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization. """ num_of_products_optimized = 0 self._config = flask.current_app.config.get('CONFIGS', {}).get( f'free_shipping_optimizer_config_{language}', {}) for entry in product_batch['entries']: product = entry['product'] original_shipping = product.get('shipping') title_contains_free_shipping_pattern = self._title_contains_pattern( product, 'free_shipping_patterns') title_contains_exclusion_pattern = self._title_contains_pattern( product, 'shipping_exclusion_patterns') if title_contains_free_shipping_pattern and not title_contains_exclusion_pattern: _update_shipping_field_to_zero(product, country, currency) if product.get('shipping') != original_shipping: num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.OPTIMIZED) return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, country: str, currency: str) -> int: """Runs size length optimization. Fixes invalid size values. See above for the definition of an invalid size value. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization. """ num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] if 'sizes' in product and product.get('sizes', []): size = product['sizes'][0] if not size: continue if len(size) > _MAX_SIZE_LENGTH or len(product.get('sizes', [])) > 1: product['sizes'] = [size[:_MAX_SIZE_LENGTH]] item_id = product.get('offerId', '') logging.info( 'Modified item %s: Clearing identifierExists ' 'to prevent disapproval', item_id) num_of_products_optimized += 1 base_optimizer.set_optimization_tracking(product, base_optimizer.SANITIZED) return num_of_products_optimized
def _optimize(self, product_batch: Mapping[str, Any], language: str, country: str, currency: str) -> int: """Runs the optimization. Fixes invalid adult values. See above for the definition of an invalid adult value. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization. """ self._adult_config = current_app.config.get('CONFIGS', {}).get( f'adult_optimizer_config_{language}', {}) self._adult_types = frozenset( self._adult_config.get('adult_product_types', [])) num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] product_types = product.get('productTypes', []) product_category = product.get('googleProductCategory', '') category_specific_tokens = self._get_tokens_for_category( product_category) is_adult = product.get('adult', False) # Product Types can directly determine adult category or not. if not is_adult and self._is_product_type_adult(product_types): logging.info( 'Product ID: %s With type %s was determined to be adult', product['offerId'], product_types) product['adult'] = True num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) # Otherwise a target GPC combined with specific tokens in the title or # description can determine if the product should have the adult flag set. elif not is_adult and category_specific_tokens: product_should_be_adult = False # If only a wildcard was specified for the tokens in the config, any # product in this category should have the adult attribute set. if len(category_specific_tokens ) == 1 and category_specific_tokens[0] == '*': product_should_be_adult = True else: adult_tokens = set(token.lower() for token in category_specific_tokens) product_title = product.get('title', '') product_description = product.get('description', '') if self._field_contains_adult_tokens( product_title, adult_tokens) or self._field_contains_adult_tokens( product_description, adult_tokens): product_should_be_adult = True if product_should_be_adult: product['adult'] = True num_of_products_optimized += 1 base_optimizer.set_optimization_tracking( product, base_optimizer.SANITIZED) return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, country: str, currency: str) -> int: """Runs the optimization. Args: product_batch: A batch of product data. language: The language to use for this optimizer. country: The country to use for this optimizer. currency: The currency to use for this optimizer. Returns: The number of products affected by this optimization. """ gpc_id_to_string_mapping = current_app.config.get('CONFIGS', {}).get( f'gpc_id_to_string_mapping_{language}', {}) title_word_order_config = current_app.config.get('CONFIGS', {}).get( f'title_word_order_config_{language}', {}) num_of_products_optimized = 0 for entry in product_batch['entries']: product = entry['product'] original_title = product.get('title', None) if original_title is None or not original_title: break google_product_category = product.get('googleProductCategory', '') google_product_category_list = google_product_category.split('>') google_product_category_3_levels = '>'.join( google_product_category_list[:3]).strip() google_product_category_id = gpc_id_to_string_mapping.get( google_product_category_3_levels) if google_product_category_id is not None and google_product_category_id: keywords_for_gpc = title_word_order_config.get( str(google_product_category_id), []) sorted_keywords_for_gpc = sorted( keywords_for_gpc, key=lambda x: x['weight'], reverse=True) performance_keywords_to_prepend = [] title_to_transform = original_title for keyword_dict in sorted_keywords_for_gpc: keyword = keyword_dict.get('keyword') if keyword in title_to_transform: title_to_transform = title_to_transform.replace(keyword, '') performance_keywords_to_prepend.append(keyword) if len(performance_keywords_to_prepend) >= 3: break optimized_title = (f'{" ".join(performance_keywords_to_prepend)} ' f'{title_to_transform}') normalized_whitespace_title = ' '.join(optimized_title.split()) product['title'] = normalized_whitespace_title if product.get('title', '') != original_title: logging.info( 'Modified item %s: Moved high-performing keywords to front of title: %s', product['offerId'], product['title']) num_of_products_optimized += 1 base_optimizer.set_optimization_tracking(product, base_optimizer.OPTIMIZED) return num_of_products_optimized