def fix_normalized_price():
"""
Walk through all reports and fix normalized price_value
and product package
"""
keeper = get_storage()
reports = PriceReport.fetch_all(keeper)
for report in reports:
try:
correct_package_key = report.product.get_package_key()
if report.product.package.key != correct_package_key:
correct_package = ProductPackage.acquire(
correct_package_key, keeper)
product = Product.fetch(report.product.key, keeper)
print(
yellow(u'Fixing package for {}: {}-->{}'.format(
report.product, product.package, correct_package)))
product.package = correct_package
report.product = product
old_norm_price = report.normalized_price_value
new_norm_price = report._get_normalized_price(report.price_value)
if old_norm_price != new_norm_price:
print(
yellow(u'Fixing normal price {}-->{}'.format(
old_norm_price, new_norm_price)))
report.normalized_price_value = new_norm_price
except PackageLookupError, e:
print(e.message)
def fix_normalized_price():
"""
Walk through all reports and fix normalized price_value
and product package
"""
keeper = get_storage()
reports = PriceReport.fetch_all(keeper)
for report in reports:
try:
correct_package_key = report.product.get_package_key()
if report.product.package.key != correct_package_key:
correct_package = ProductPackage.acquire(correct_package_key,
keeper)
product = Product.fetch(report.product.key, keeper)
print(yellow(u'Fixing package for {}: {}-->{}'.format(
report.product, product.package, correct_package)))
product.package = correct_package
report.product = product
old_norm_price = report.normalized_price_value
new_norm_price = report._get_normalized_price(report.price_value)
if old_norm_price != new_norm_price:
print(yellow(u'Fixing normal price {}-->{}'.format(
old_norm_price, new_norm_price)))
report.normalized_price_value = new_norm_price
except PackageLookupError, e:
print(e.message)
def served_data(self, location):
"""Serve general index data optionally filter by region"""
categories = list(self.root['types'].values())
categories.sort(key=lambda x: float(x.get_data('priority')),
reverse=True)
# category list
types = list()
all_locations = set()
for type_ in categories:
category_tuples = list()
type_title_ru = type_.get_data('title_ru')
type_primary_color = type_.get_data('primary_color')
type_background_color = type_.get_data('background_color')
type_.categories.sort(
key=lambda x: float(x.get_data('priority', default=0)),
reverse=True)
for category in type_.categories:
price = category.get_price(location=location)
if price:
price = self.currency(price)
query = {'location': location} if location else None
url = self.request.resource_path(category, query=query)
title = category.get_data('keyword').split(', ')[0]
delta = int(
category.get_price_delta(self.delta_period,
location=location) * 100)
package_key = category.get_data('normal_package')
package_title = ProductPackage(package_key).get_data(
'synonyms')[0]
cat_locations = category.get_locations()
all_locations.update(cat_locations)
category_tuples.append(
(url, title, price, delta, package_title))
types.append((type_.title, type_title_ru, type_primary_color,
type_background_color, category_tuples))
time = format_datetime(datetime.datetime.now(),
format='long',
locale=self.request.locale_name)
return {
'types': types,
'time': time,
'current_location': location,
'locations': list(all_locations),
'root': True
}
def cycle(entity_class_name_, keeper):
"""Perform all needed routines on an `entity_class`"""
print(cyan('{} check...'.format(entity_class_name_)))
entity_class = globals()[entity_class_name_]
instances = entity_class.fetch_all(keeper, objects_only=False)
for key in instances.keys():
instance = instances[key]
if entity_class is ProductCategory:
if not hasattr(instance, 'category') or not instance.category:
category_key = instance.get_category_key()
category = Category.acquire(category_key, keeper)
instance.category = category
category.add(instance)
print(green(u'Added {} to {}'.format(instance, category)))
for product in instance.products:
if product.category is not instance:
instance.remove_product(product)
print(yellow(u'Removed '
u'`{}` from `{}`...'.format(product,
instance)))
if len(product.reports) == 0:
instance.remove_product(product)
print(yellow(u'Removed stale '
u'`{}` from `{}`...'.format(product,
instance)))
if product.key not in keeper[product.namespace]:
try:
instance.remove_product(product)
print(yellow(u'Removed `{}` from `{}` '
u'as its not '
u'registered...'.format(product,
instance)))
except ValueError:
pass
if entity_class is Product:
if type(instance.reports) is not list:
print(yellow(u'Fixing product report '
u'list for `{}`...'.format(instance)))
instance.reports = list(instance.reports.values())
if type(instance.merchants) is not list:
print(yellow(u'Fixing product merchant '
u'list for `{}`...'.format(instance)))
instance.merchants = list(instance.merchants.values())
if len(instance.reports) == 0:
print(yellow(u'Removing stale `{}`...'.format(instance)))
instance.delete_from(keeper)
# check category
try:
cat_key = instance.get_category_key()
category = ProductCategory.fetch(cat_key, keeper)
if instance.category is not category:
print(yellow(u'Adding `{}` to '
u'`{}`...'.format(instance, category)))
category.add_product(instance)
instance.category = category
except CategoryLookupError:
print(yellow(u'Removing `{}` as no '
u'category found...'.format(instance)))
instance.delete_from(keeper)
# check key
if key != instance.key:
print(yellow(u'Fixing key for `{}`...'.format(key)))
keeper.register(instance)
keeper.delete_key(instance.namespace, key)
if entity_class is Merchant:
if type(instance.products) is not list:
instance.products = list(instance.products.values())
for product in instance.products:
if len(product.reports) == 0:
print(yellow(u'Deleting `{}` '
u'from `{}`...'.format(product,
instance)))
instance.remove_product(product)
for report in product.reports:
if type(report) is str:
print(yellow('Removing product with str report ...'))
product.delete_from(keeper)
if entity_class is PriceReport:
if instance.product.category is None:
print(yellow(u'Removing report '
u'for {}'.format(instance.product)))
instance.delete_from(keeper)
break
try:
correct_package_key = instance.product.get_package_key()
except PackageLookupError, e:
print(e.message)
else:
if instance.product.package.key != correct_package_key:
correct_package = ProductPackage.acquire(
correct_package_key, keeper)
product = Product.fetch(instance.product.key, keeper)
print(yellow(u'Fixing package for {}: {}-->{}'.format(
instance.product, product.package, correct_package)))
product.package = correct_package
instance.product = product
old_norm_price = instance.normalized_price_value
correct_norm_price = instance._get_normalized_price(
instance.price_value)
if old_norm_price != correct_norm_price:
print(yellow(
u'Fixing normal price '
u'for {} report ({}-->{})'.format(
instance.product, old_norm_price,
correct_norm_price)))
instance.normalized_price_value = correct_norm_price
def cycle(entity_class_name_, keeper):
"""Perform all needed routines on an `entity_class`"""
print(cyan('{} check...'.format(entity_class_name_)))
entity_class = globals()[entity_class_name_]
instances = entity_class.fetch_all(keeper, objects_only=False)
for key in instances.keys():
instance = instances[key]
if entity_class is ProductCategory:
if not hasattr(instance, 'category') or not instance.category:
category_key = instance.get_category_key()
category = Category.acquire(category_key, keeper)
instance.category = category
category.add(instance)
print(green(u'Added {} to {}'.format(instance, category)))
for product in instance.products:
if product.category is not instance:
instance.remove_product(product)
print(
yellow(u'Removed '
u'`{}` from `{}`...'.format(
product, instance)))
if len(product.reports) == 0:
instance.remove_product(product)
print(
yellow(u'Removed stale '
u'`{}` from `{}`...'.format(
product, instance)))
if product.key not in keeper[product.namespace]:
try:
instance.remove_product(product)
print(
yellow(u'Removed `{}` from `{}` '
u'as its not '
u'registered...'.format(
product, instance)))
except ValueError:
pass
if entity_class is Product:
if type(instance.reports) is not list:
print(
yellow(u'Fixing product report '
u'list for `{}`...'.format(instance)))
instance.reports = list(instance.reports.values())
if type(instance.merchants) is not list:
print(
yellow(u'Fixing product merchant '
u'list for `{}`...'.format(instance)))
instance.merchants = list(instance.merchants.values())
if len(instance.reports) == 0:
print(yellow(u'Removing stale `{}`...'.format(instance)))
instance.delete_from(keeper)
# check category
try:
cat_key = instance.get_category_key()
category = ProductCategory.fetch(cat_key, keeper)
if instance.category is not category:
print(
yellow(u'Adding `{}` to '
u'`{}`...'.format(instance, category)))
category.add_product(instance)
instance.category = category
except CategoryLookupError:
print(
yellow(u'Removing `{}` as no '
u'category found...'.format(instance)))
instance.delete_from(keeper)
# check key
if key != instance.key:
print(yellow(u'Fixing key for `{}`...'.format(key)))
keeper.register(instance)
keeper.delete_key(instance.namespace, key)
if entity_class is Merchant:
if type(instance.products) is not list:
instance.products = list(instance.products.values())
for product in instance.products:
if len(product.reports) == 0:
print(
yellow(u'Deleting `{}` '
u'from `{}`...'.format(product, instance)))
instance.remove_product(product)
for report in product.reports:
if type(report) is str:
print(
yellow('Removing product with str report ...'))
product.delete_from(keeper)
if entity_class is PriceReport:
if instance.product.category is None:
print(
yellow(u'Removing report '
u'for {}'.format(instance.product)))
instance.delete_from(keeper)
break
try:
correct_package_key = instance.product.get_package_key()
except PackageLookupError, e:
print(e.message)
else:
if instance.product.package.key != correct_package_key:
correct_package = ProductPackage.acquire(
correct_package_key, keeper)
product = Product.fetch(instance.product.key, keeper)
print(
yellow(u'Fixing package for {}: {}-->{}'.format(
instance.product, product.package,
correct_package)))
product.package = correct_package
instance.product = product
old_norm_price = instance.normalized_price_value
correct_norm_price = instance._get_normalized_price(
instance.price_value)
if old_norm_price != correct_norm_price:
print(
yellow(u'Fixing normal price '
u'for {} report ({}-->{})'.format(
instance.product, old_norm_price,
correct_norm_price)))
instance.normalized_price_value = correct_norm_price
def serve_data(self, product_category, location):
"""Return prepared category data"""
category = product_category.category
category_title = category.title
category_title_ru = category.get_data('title_ru')
category_primary_color = category.get_data('primary_color')
category_background_color = category.get_data('background_color')
prod_cat_title = product_category.get_data('ru_accu_case')
median = product_category.get_price(location=location)
category_delta = int(
product_category.get_price_delta(self.delta_period,
location=location) * 100)
if not prod_cat_title:
prod_cat_title = product_category.get_data('keyword').split(
', ')[0]
package_key = product_category.get_data('normal_package')
package_title = ProductPackage(package_key).get_data('synonyms')[0]
chart_data = list()
datetimes = get_datetimes(self.display_days)
for date in datetimes:
chart_data.append([
date.strftime('%d.%m'),
product_category.get_price(date, location=location)
])
products = list()
locations = product_category.get_locations()
current_path = self.request.resource_url(product_category)
sorted_products = sorted(
product_category.get_qualified_products(location=location),
key=lambda pr: pr[1])
for num, product_tuple in enumerate(sorted_products):
try:
product, price = product_tuple
middle_num = int(len(sorted_products) / 2)
is_median = (num == middle_num)
if len(sorted_products) % 2 == 0:
is_median = (num == middle_num or num == middle_num - 1)
# construct data row as tuple
products.append(
(num + 1, product, self.request.resource_url(product),
self.currency(price),
int(product.get_price_delta(self.delta_period) * 100),
is_median))
except TypeError:
pass
return {
'price_data': json.dumps(chart_data),
'products': products,
'cat_title': prod_cat_title,
'current_location': location,
'locations': locations,
'category_title': category_title,
'category_title_ru': category_title_ru,
'category_background_color': category_background_color,
'category_primary_color': category_primary_color,
'current_path': current_path,
'package_title': package_title,
'median_price': self.currency(median) if median else None,
'category_delta': category_delta if median else None
}
def serve_data(self, product):
"""Return prepared product data"""
current_price = product.get_price()
if current_price:
current_price = self.currency(current_price)
product_delta = int(
product.get_price_delta(self.delta_period) * 100)
last_report_url = None
else:
product_delta = 0
last_price = product.get_last_reported_price()
current_price = self.currency(last_price) if last_price else None
last_report = self.context.get_last_report()
last_report_url = \
self.request.resource_url(last_report) if last_report else None
datetimes = get_datetimes(self.display_days)
chart_data = list()
for date in datetimes:
chart_data.append(
[date.strftime('%d.%m'),
product.get_price(date)])
chart_data = json.dumps(chart_data)
package_key = product.category.get_data('normal_package')
product_category_title = product.category.get_data('keyword').split(
', ')[0]
product_category_url = self.request.resource_url(product.category)
package_title = ProductPackage(package_key).get_data('synonyms')[0]
type_ = product.category.category
category_title = type_.title
category_title_ru = type_.get_data('title_ru')
category_primary_color = type_.get_data('primary_color')
category_background_color = type_.get_data('background_color')
reports = list()
for report in sorted(
product.get_reports(from_date_time=self.delta_period),
reverse=True,
key=lambda rep: rep.date_time):
url = self.request.resource_url(report)
date = format_datetime(report.date_time,
format='short',
locale=self.request.locale_name)
merchant = report.merchant.title
location = report.merchant.location
price = self.currency(report.normalized_price_value)
reports.append((url, date, merchant, location, price))
return {
'current_price': current_price,
'product_delta': product_delta,
'last_report_url': last_report_url,
'chart_data': chart_data,
'reports': reports,
'product_category_title': product_category_title,
'product_category_url': product_category_url,
'category_title': category_title,
'category_title_ru': category_title_ru,
'category_background_color': category_background_color,
'category_primary_color': category_primary_color,
'package_title': package_title
}