def _optimize(self, product_batch: Dict[str, Any], language: str,
_) -> int:
"""Runs the optimization.
Fixes invalid gtin fields.
See above for the definition of an invalid gtin field.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
Returns:
The number of products affected by this optimization: int
"""
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
if 'gtin' in product:
gtin = product.get('gtin', '')
if _gtin_passes_format_check(gtin) and _gtin_passes_checksum(
gtin):
continue
violating_gtin = product.get('gtin', '')
del product['gtin']
logging.info(
'Modified item %s: Cleared invalid gtin: %s to '
'prevent disapproval', product.get('offerId', ''),
violating_gtin)
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
num_of_products_optimized += 1
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str,
_) -> int:
"""Runs the optimization.
Removes invalid identifierExists fields.
See above for the definition of an invalid identifierExists field.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
Returns:
The number of products affected by this optimization: int
"""
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
identifier_exists = product.get('identifierExists', True)
brand = product.get('brand', '')
gtin = product.get('gtin', '')
mpn = product.get('mpn', '')
if not identifier_exists and (brand or gtin or mpn):
item_id = product.get('offerId', '')
logging.info(
'Modified item %s: Clearing identifierExists '
'to prevent disapproval', item_id)
# Delete field from the request which defaults it to true.
del product['identifierExists']
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs the optimization.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
if 'productTypes' in product:
product_types = product.get('productTypes', [])
optimized_product_types = (
optimization_util.cut_list_to_limit_list_length(
product_types, _MAX_LIST_LENGTH))
if optimized_product_types != product_types:
product['productTypes'] = optimized_product_types
logging.info(
'Modified item %s: Removing the last items from productTypes: %s',
product.get('offerId', ''), product_types)
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs the optimization.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
original_title = product.get('title', '')
_remove_unnecessary_text(product, language)
if product.get('title', '') != original_title:
logging.info(
'Modified item %s: Removed promo text, new title is: %s',
product.get('offerId', ''), product.get('title', ''))
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
return num_of_products_optimized
def _append_attributes_to_title(self, product: Dict[str, Any],
chars_to_preserve: int, language: str,
country: str) -> None:
"""Appends mined attributes to the title.
Args:
product: Product data.
chars_to_preserve: The num of chars to leave unchanged (used to make sure
the original title is preserved).
language: The language to use for this optimizer.
country: The country to use for this optimizer.
"""
if not self._mined_attributes:
return
product_id = product.get('offerId', '')
size_checker = size_miner.SizeMiner(language, country)
fields_to_append_to_title = []
for attribute_name, mined_attribute_values in self._mined_attributes.get(
product_id, {}).items():
if attribute_name == 'sizes' and size_checker.is_size_in_attribute(
product, 'title'):
# Does not append the size when size information is already in title.
continue
if isinstance(mined_attribute_values, str):
fields_to_append_to_title.append(mined_attribute_values)
elif isinstance(mined_attribute_values, list):
fields_to_append_to_title.extend(mined_attribute_values)
if fields_to_append_to_title:
base_optimizer.set_optimization_tracking(product,
base_optimizer.OPTIMIZED)
_append_fields_to_title(product, fields_to_append_to_title,
chars_to_preserve)
def _optimize(self, product_batch: Dict[str, Any], language: str) -> int:
for entry in product_batch['entries']:
product = entry['product']
base_optimizer.set_optimization_tracking(product,
base_optimizer.SANITIZED)
return len(product_batch) # Number of items Sanitized.
def _optimize(self, product_batch: Dict[str, Any], language: str,
_) -> int:
"""Runs the optimization.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
Returns:
The number of products affected by this optimization: int
"""
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
if 'mpn' in product:
mpn_value = _normalize_mpn(product.get('mpn', ''))
if mpn_value in INVALID_MPN_VALUES:
item_id = product.get('offerId')
logging.info('Modified item %s: Removing invalid MPN [%s]',
item_id, mpn_value)
del product['mpn']
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str,
_) -> int:
"""Runs the optimization.
Removes invalid chars from the title and description.
See above for the definition of an invalid char.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
Returns:
The number of products affected by this optimization: int
"""
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
product_was_sanitized = _sanitize_fields(product,
_FIELDS_TO_SANITIZE)
if product_was_sanitized:
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str, _) -> int:
"""Runs the optimization.
Fixes invalid condition values.
See above for the definition of an invalid condition value.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
num_of_products_optimized = 0
self._condition_config = current_app.config.get('CONFIGS', {}).get(
f'condition_optimizer_config_{language}', {})
for entry in product_batch['entries']:
product = entry['product']
google_product_category = product.get('googleProductCategory', '')
if self._is_google_product_category_excluded(google_product_category):
logging.info(
'Product ID: %s With Category %s was flagged for exclusion '
' of the condition check', product.get('offerId', ''),
google_product_category)
continue
used_tokens = set(
token.lower() for token in self._condition_config['used_tokens'])
logging.info('Used tokens were %s', used_tokens)
if product.get('condition', '') == _NEW:
# Category format must follow the official spec to be converted a list.
# Ref: https://support.google.com/merchants/answer/6324436?hl=en.
product_categories = google_product_category.split(' > ')
if isinstance(product_categories, list) and product_categories:
lowest_level_category = product_categories[-1]
category_specific_tokens = self._get_tokens_for_category(
lowest_level_category)
if category_specific_tokens:
category_specific_tokens = set(
token.lower() for token in category_specific_tokens)
used_tokens.update(category_specific_tokens)
# Search for used tokens in both title and description and reset the
# condition to used if any were detected.
product_title = product.get('title', '')
product_description = product.get('description', '')
if self._field_contains_used_tokens(
product_title, used_tokens) or self._field_contains_used_tokens(
product_description, used_tokens):
product['condition'] = _USED
logging.info('Modified item %s: Setting new product to used.',
product.get('offerId', ''))
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(product,
base_optimizer.SANITIZED)
return num_of_products_optimized
def _update_product_description(product: Dict[str, Any],
optimized_description: str) -> None:
"""Updates the description for the given product and sets tracking/logging.
Args:
product: Product data.
optimized_description: The new product description.
"""
product['description'] = optimized_description
base_optimizer.set_optimization_tracking(product, base_optimizer.OPTIMIZED)
logging.info('Modified item %s: Appended field values to description: %s',
product['offerId'], product['description'])
def _remove_gtin(product: Dict[str, Any]) -> None:
"""Clears the gtin value from the product.
Args:
product: A dictionary representing a single shopping product.
"""
violating_gtin = product.get('gtin', '')
del product['gtin']
logging.info(
'Modified item %s: Cleared invalid gtin: %s to '
'prevent disapproval', product.get('offerId', ''), violating_gtin)
base_optimizer.set_optimization_tracking(product, base_optimizer.SANITIZED)
def _truncate_to_max_length(product: Dict[str, Any]) -> None:
"""Truncate the title to the max length: 150.
This function prevents a product with a 150+ title length from being
disapproved. Reference: https://support.google.com/merchants/answer/6324415
Args:
product: A dictionary containing product data.
"""
title = product.get('title', '')
if len(title) > _MAX_TITLE_LENGTH:
product['title'] = title[:_MAX_TITLE_LENGTH]
logging.info('Modified item %s: Truncating title: %s',
product.get('offerId', ''), product.get('title', ''))
base_optimizer.set_optimization_tracking(product, base_optimizer.SANITIZED)
def _optimize(self, product_batch: Dict[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs title length optimization.
This method optimizes the title by executing following processes:
- Truncates title to the max title length if its length exceeds the max
value.
- Populate title with description if title is truncated from description.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
num_of_products_optimized: int = 0
entry: Dict[str, Any]
for entry in product_batch['entries']:
product = entry['product']
item_id: str = product.get('offerId', '')
title: str = product.get('title', '')
trailing_dots_removed_title: str = re.sub('[.…]+$', '', title)
description: str = product.get('description', '')
if len(title) > _MAX_TITLE_LENGTH:
product['title'] = title[:_MAX_TITLE_LENGTH]
logging.info('Modified item %s: Truncating title: %s', item_id,
product['title'])
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
elif trailing_dots_removed_title != description and description.startswith(
trailing_dots_removed_title):
product['title'] = description[:_MAX_TITLE_LENGTH]
logging.info(
'Modified item %s: Populating title with '
'description due to detected title truncation: %s',
item_id, product['title'])
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.OPTIMIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs the optimization.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
num_of_products_optimized = 0
self._shopping_exclusion_config = flask.current_app.config.get(
'CONFIGS', {}).get(f'shopping_exclusion_optimizer_config_{language}',
{})
self.shopping_removal_patterns_exact_match = frozenset(
self._shopping_exclusion_config.get(
'shopping_exclusion_patterns_exact_match', []))
for entry in product_batch['entries']:
product = entry['product']
if self._is_non_shopping_product(product.get('title', '')):
if isinstance(
product.get('excludedDestinations'), list
) and _SHOPPING_ADS_DESTINATION not in product['excludedDestinations']:
product['excludedDestinations'].append(_SHOPPING_ADS_DESTINATION)
else:
product['excludedDestinations'] = [_SHOPPING_ADS_DESTINATION]
if isinstance(
product.get('includedDestinations'), list
) and _SHOPPING_ADS_DESTINATION in product['includedDestinations']:
product['includedDestinations'].remove(_SHOPPING_ADS_DESTINATION)
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(product,
base_optimizer.SANITIZED)
logging.info(
'Product %s was detected as not meant for Shopping Ads, so excluding from the Shopping Ads destination',
product.get('offerId'))
return num_of_products_optimized
def _complement_title_with_description(product: Dict[str, Any]) -> None:
"""Complements title with description if title is truncated from description.
This is expected to improve ad performance by adding more information to the
title.
Args:
product: Product data.
"""
title_without_trailing_dots = re.sub('[.…]+$', '', product['title'])
description = product.get('description', '')
if title_without_trailing_dots != description and description.startswith(
title_without_trailing_dots):
product['title'] = description[:_MAX_TITLE_LENGTH]
logging.info(
'Modified item %s: Populating title with description due to detected '
'title truncation: %s', product['offerId'], product['title'])
base_optimizer.set_optimization_tracking(product, base_optimizer.OPTIMIZED)
def _optimize(self, product_batch: Dict[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs the optimization.
Fixes invalid color fields.
See above for the definition of an invalid color field.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization: int
"""
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
if 'color' in product:
original_colors = product.get('color', '')
colors = original_colors.split(_SEPARATOR)
optimized_colors = optimization_util.cut_list_elements_over_max_length(
colors, constants.MAX_COLOR_STR_LENGTH_FOR_EACH)
optimized_colors = optimization_util.cut_list_to_limit_list_length(
optimized_colors, constants.MAX_COLOR_COUNT)
optimized_colors = optimization_util.cut_list_to_limit_concatenated_str_length(
optimized_colors, _SEPARATOR,
constants.MAX_COLOR_STR_LENGTH_IN_TOTAL)
optimized_colors = _SEPARATOR.join(optimized_colors)
if original_colors != optimized_colors:
product['color'] = optimized_colors
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs the optimization.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
num_of_products_optimized = 0
self._config = flask.current_app.config.get('CONFIGS', {}).get(
f'free_shipping_optimizer_config_{language}', {})
for entry in product_batch['entries']:
product = entry['product']
original_shipping = product.get('shipping')
title_contains_free_shipping_pattern = self._title_contains_pattern(
product, 'free_shipping_patterns')
title_contains_exclusion_pattern = self._title_contains_pattern(
product, 'shipping_exclusion_patterns')
if title_contains_free_shipping_pattern and not title_contains_exclusion_pattern:
_update_shipping_field_to_zero(product, country, currency)
if product.get('shipping') != original_shipping:
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.OPTIMIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs size length optimization.
Fixes invalid size values.
See above for the definition of an invalid size value.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
if 'sizes' in product and product.get('sizes', []):
size = product['sizes'][0]
if not size:
continue
if len(size) > _MAX_SIZE_LENGTH or len(product.get('sizes', [])) > 1:
product['sizes'] = [size[:_MAX_SIZE_LENGTH]]
item_id = product.get('offerId', '')
logging.info(
'Modified item %s: Clearing identifierExists '
'to prevent disapproval', item_id)
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(product,
base_optimizer.SANITIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Mapping[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs the optimization.
Fixes invalid adult values.
See above for the definition of an invalid adult value.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
self._adult_config = current_app.config.get('CONFIGS', {}).get(
f'adult_optimizer_config_{language}', {})
self._adult_types = frozenset(
self._adult_config.get('adult_product_types', []))
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
product_types = product.get('productTypes', [])
product_category = product.get('googleProductCategory', '')
category_specific_tokens = self._get_tokens_for_category(
product_category)
is_adult = product.get('adult', False)
# Product Types can directly determine adult category or not.
if not is_adult and self._is_product_type_adult(product_types):
logging.info(
'Product ID: %s With type %s was determined to be adult',
product['offerId'], product_types)
product['adult'] = True
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
# Otherwise a target GPC combined with specific tokens in the title or
# description can determine if the product should have the adult flag set.
elif not is_adult and category_specific_tokens:
product_should_be_adult = False
# If only a wildcard was specified for the tokens in the config, any
# product in this category should have the adult attribute set.
if len(category_specific_tokens
) == 1 and category_specific_tokens[0] == '*':
product_should_be_adult = True
else:
adult_tokens = set(token.lower()
for token in category_specific_tokens)
product_title = product.get('title', '')
product_description = product.get('description', '')
if self._field_contains_adult_tokens(
product_title,
adult_tokens) or self._field_contains_adult_tokens(
product_description, adult_tokens):
product_should_be_adult = True
if product_should_be_adult:
product['adult'] = True
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(
product, base_optimizer.SANITIZED)
return num_of_products_optimized
def _optimize(self, product_batch: Dict[str, Any], language: str,
country: str, currency: str) -> int:
"""Runs the optimization.
Args:
product_batch: A batch of product data.
language: The language to use for this optimizer.
country: The country to use for this optimizer.
currency: The currency to use for this optimizer.
Returns:
The number of products affected by this optimization.
"""
gpc_id_to_string_mapping = current_app.config.get('CONFIGS', {}).get(
f'gpc_id_to_string_mapping_{language}', {})
title_word_order_config = current_app.config.get('CONFIGS', {}).get(
f'title_word_order_config_{language}', {})
num_of_products_optimized = 0
for entry in product_batch['entries']:
product = entry['product']
original_title = product.get('title', None)
if original_title is None or not original_title:
break
google_product_category = product.get('googleProductCategory', '')
google_product_category_list = google_product_category.split('>')
google_product_category_3_levels = '>'.join(
google_product_category_list[:3]).strip()
google_product_category_id = gpc_id_to_string_mapping.get(
google_product_category_3_levels)
if google_product_category_id is not None and google_product_category_id:
keywords_for_gpc = title_word_order_config.get(
str(google_product_category_id), [])
sorted_keywords_for_gpc = sorted(
keywords_for_gpc, key=lambda x: x['weight'], reverse=True)
performance_keywords_to_prepend = []
title_to_transform = original_title
for keyword_dict in sorted_keywords_for_gpc:
keyword = keyword_dict.get('keyword')
if keyword in title_to_transform:
title_to_transform = title_to_transform.replace(keyword, '')
performance_keywords_to_prepend.append(keyword)
if len(performance_keywords_to_prepend) >= 3:
break
optimized_title = (f'{" ".join(performance_keywords_to_prepend)} '
f'{title_to_transform}')
normalized_whitespace_title = ' '.join(optimized_title.split())
product['title'] = normalized_whitespace_title
if product.get('title', '') != original_title:
logging.info(
'Modified item %s: Moved high-performing keywords to front of title: %s',
product['offerId'], product['title'])
num_of_products_optimized += 1
base_optimizer.set_optimization_tracking(product,
base_optimizer.OPTIMIZED)
return num_of_products_optimized
