Пример #1
0
def fix_normalized_price():
    """
    Walk through all reports and fix normalized price_value
    and product package
    """

    keeper = get_storage()
    reports = PriceReport.fetch_all(keeper)
    for report in reports:
        try:
            correct_package_key = report.product.get_package_key()
            if report.product.package.key != correct_package_key:
                correct_package = ProductPackage.acquire(
                    correct_package_key, keeper)
                product = Product.fetch(report.product.key, keeper)
                print(
                    yellow(u'Fixing package for {}: {}-->{}'.format(
                        report.product, product.package, correct_package)))
                product.package = correct_package
                report.product = product

            old_norm_price = report.normalized_price_value
            new_norm_price = report._get_normalized_price(report.price_value)
            if old_norm_price != new_norm_price:
                print(
                    yellow(u'Fixing normal price {}-->{}'.format(
                        old_norm_price, new_norm_price)))
                report.normalized_price_value = new_norm_price
        except PackageLookupError, e:
            print(e.message)
Пример #2
0
def fix_normalized_price():
    """
    Walk through all reports and fix normalized price_value
    and product package
    """

    keeper = get_storage()
    reports = PriceReport.fetch_all(keeper)
    for report in reports:
        try:
            correct_package_key = report.product.get_package_key()
            if report.product.package.key != correct_package_key:
                correct_package = ProductPackage.acquire(correct_package_key,
                                                         keeper)
                product = Product.fetch(report.product.key, keeper)
                print(yellow(u'Fixing package for {}: {}-->{}'.format(
                    report.product, product.package, correct_package)))
                product.package = correct_package
                report.product = product

            old_norm_price = report.normalized_price_value
            new_norm_price = report._get_normalized_price(report.price_value)
            if old_norm_price != new_norm_price:
                print(yellow(u'Fixing normal price {}-->{}'.format(
                      old_norm_price, new_norm_price)))
                report.normalized_price_value = new_norm_price
        except PackageLookupError, e:
            print(e.message)
Пример #3
0
    def served_data(self, location):
        """Serve general index data optionally filter by region"""
        categories = list(self.root['types'].values())
        categories.sort(key=lambda x: float(x.get_data('priority')),
                        reverse=True)

        # category list
        types = list()
        all_locations = set()
        for type_ in categories:
            category_tuples = list()
            type_title_ru = type_.get_data('title_ru')
            type_primary_color = type_.get_data('primary_color')
            type_background_color = type_.get_data('background_color')
            type_.categories.sort(
                key=lambda x: float(x.get_data('priority', default=0)),
                reverse=True)
            for category in type_.categories:
                price = category.get_price(location=location)
                if price:
                    price = self.currency(price)
                    query = {'location': location} if location else None
                    url = self.request.resource_path(category, query=query)
                    title = category.get_data('keyword').split(', ')[0]
                    delta = int(
                        category.get_price_delta(self.delta_period,
                                                 location=location) * 100)
                    package_key = category.get_data('normal_package')
                    package_title = ProductPackage(package_key).get_data(
                        'synonyms')[0]
                    cat_locations = category.get_locations()
                    all_locations.update(cat_locations)
                    category_tuples.append(
                        (url, title, price, delta, package_title))
            types.append((type_.title, type_title_ru, type_primary_color,
                          type_background_color, category_tuples))
        time = format_datetime(datetime.datetime.now(),
                               format='long',
                               locale=self.request.locale_name)
        return {
            'types': types,
            'time': time,
            'current_location': location,
            'locations': list(all_locations),
            'root': True
        }
Пример #4
0
    def cycle(entity_class_name_, keeper):
        """Perform all needed routines on an `entity_class`"""

        print(cyan('{} check...'.format(entity_class_name_)))
        entity_class = globals()[entity_class_name_]
        instances = entity_class.fetch_all(keeper, objects_only=False)

        for key in instances.keys():
            instance = instances[key]
            if entity_class is ProductCategory:

                if not hasattr(instance, 'category') or not instance.category:
                    category_key = instance.get_category_key()
                    category = Category.acquire(category_key, keeper)
                    instance.category = category
                    category.add(instance)
                    print(green(u'Added {} to {}'.format(instance, category)))

                for product in instance.products:
                    if product.category is not instance:
                        instance.remove_product(product)
                        print(yellow(u'Removed '
                                     u'`{}` from `{}`...'.format(product,
                                                                 instance)))
                    if len(product.reports) == 0:
                        instance.remove_product(product)
                        print(yellow(u'Removed stale '
                                     u'`{}` from `{}`...'.format(product,
                                                                 instance)))
                    if product.key not in keeper[product.namespace]:
                        try:
                            instance.remove_product(product)
                            print(yellow(u'Removed `{}` from `{}` '
                                         u'as its not '
                                         u'registered...'.format(product,
                                                                 instance)))
                        except ValueError:
                            pass

            if entity_class is Product:

                if type(instance.reports) is not list:
                    print(yellow(u'Fixing product report '
                                 u'list for `{}`...'.format(instance)))
                    instance.reports = list(instance.reports.values())

                if type(instance.merchants) is not list:
                    print(yellow(u'Fixing product merchant '
                                 u'list for `{}`...'.format(instance)))
                    instance.merchants = list(instance.merchants.values())

                if len(instance.reports) == 0:
                    print(yellow(u'Removing stale `{}`...'.format(instance)))
                    instance.delete_from(keeper)

                # check category
                try:
                    cat_key = instance.get_category_key()
                    category = ProductCategory.fetch(cat_key, keeper)
                    if instance.category is not category:
                        print(yellow(u'Adding `{}` to '
                                     u'`{}`...'.format(instance, category)))
                        category.add_product(instance)
                        instance.category = category
                except CategoryLookupError:
                    print(yellow(u'Removing `{}` as no '
                                 u'category found...'.format(instance)))
                    instance.delete_from(keeper)

                # check key
                if key != instance.key:
                    print(yellow(u'Fixing key for `{}`...'.format(key)))
                    keeper.register(instance)
                    keeper.delete_key(instance.namespace, key)

            if entity_class is Merchant:
                if type(instance.products) is not list:
                    instance.products = list(instance.products.values())
                for product in instance.products:
                    if len(product.reports) == 0:
                        print(yellow(u'Deleting `{}` '
                                     u'from `{}`...'.format(product,
                                                            instance)))
                        instance.remove_product(product)
                    for report in product.reports:
                        if type(report) is str:
                            print(yellow('Removing product with str report ...'))
                            product.delete_from(keeper)

            if entity_class is PriceReport:
                if instance.product.category is None:
                    print(yellow(u'Removing report '
                                 u'for {}'.format(instance.product)))
                    instance.delete_from(keeper)
                    break
                try:
                    correct_package_key = instance.product.get_package_key()
                except PackageLookupError, e:
                    print(e.message)
                else:
                    if instance.product.package.key != correct_package_key:
                        correct_package = ProductPackage.acquire(
                            correct_package_key, keeper)
                        product = Product.fetch(instance.product.key, keeper)
                        print(yellow(u'Fixing package for {}: {}-->{}'.format(
                            instance.product, product.package, correct_package)))
                        product.package = correct_package
                        instance.product = product

                    old_norm_price = instance.normalized_price_value
                    correct_norm_price = instance._get_normalized_price(
                        instance.price_value)
                    if old_norm_price != correct_norm_price:
                        print(yellow(
                            u'Fixing normal price '
                            u'for {} report ({}-->{})'.format(
                                instance.product, old_norm_price,
                                correct_norm_price)))
                        instance.normalized_price_value = correct_norm_price
Пример #5
0
    def cycle(entity_class_name_, keeper):
        """Perform all needed routines on an `entity_class`"""

        print(cyan('{} check...'.format(entity_class_name_)))
        entity_class = globals()[entity_class_name_]
        instances = entity_class.fetch_all(keeper, objects_only=False)

        for key in instances.keys():
            instance = instances[key]
            if entity_class is ProductCategory:

                if not hasattr(instance, 'category') or not instance.category:
                    category_key = instance.get_category_key()
                    category = Category.acquire(category_key, keeper)
                    instance.category = category
                    category.add(instance)
                    print(green(u'Added {} to {}'.format(instance, category)))

                for product in instance.products:
                    if product.category is not instance:
                        instance.remove_product(product)
                        print(
                            yellow(u'Removed '
                                   u'`{}` from `{}`...'.format(
                                       product, instance)))
                    if len(product.reports) == 0:
                        instance.remove_product(product)
                        print(
                            yellow(u'Removed stale '
                                   u'`{}` from `{}`...'.format(
                                       product, instance)))
                    if product.key not in keeper[product.namespace]:
                        try:
                            instance.remove_product(product)
                            print(
                                yellow(u'Removed `{}` from `{}` '
                                       u'as its not '
                                       u'registered...'.format(
                                           product, instance)))
                        except ValueError:
                            pass

            if entity_class is Product:

                if type(instance.reports) is not list:
                    print(
                        yellow(u'Fixing product report '
                               u'list for `{}`...'.format(instance)))
                    instance.reports = list(instance.reports.values())

                if type(instance.merchants) is not list:
                    print(
                        yellow(u'Fixing product merchant '
                               u'list for `{}`...'.format(instance)))
                    instance.merchants = list(instance.merchants.values())

                if len(instance.reports) == 0:
                    print(yellow(u'Removing stale `{}`...'.format(instance)))
                    instance.delete_from(keeper)

                # check category
                try:
                    cat_key = instance.get_category_key()
                    category = ProductCategory.fetch(cat_key, keeper)
                    if instance.category is not category:
                        print(
                            yellow(u'Adding `{}` to '
                                   u'`{}`...'.format(instance, category)))
                        category.add_product(instance)
                        instance.category = category
                except CategoryLookupError:
                    print(
                        yellow(u'Removing `{}` as no '
                               u'category found...'.format(instance)))
                    instance.delete_from(keeper)

                # check key
                if key != instance.key:
                    print(yellow(u'Fixing key for `{}`...'.format(key)))
                    keeper.register(instance)
                    keeper.delete_key(instance.namespace, key)

            if entity_class is Merchant:
                if type(instance.products) is not list:
                    instance.products = list(instance.products.values())
                for product in instance.products:
                    if len(product.reports) == 0:
                        print(
                            yellow(u'Deleting `{}` '
                                   u'from `{}`...'.format(product, instance)))
                        instance.remove_product(product)
                    for report in product.reports:
                        if type(report) is str:
                            print(
                                yellow('Removing product with str report ...'))
                            product.delete_from(keeper)

            if entity_class is PriceReport:
                if instance.product.category is None:
                    print(
                        yellow(u'Removing report '
                               u'for {}'.format(instance.product)))
                    instance.delete_from(keeper)
                    break
                try:
                    correct_package_key = instance.product.get_package_key()
                except PackageLookupError, e:
                    print(e.message)
                else:
                    if instance.product.package.key != correct_package_key:
                        correct_package = ProductPackage.acquire(
                            correct_package_key, keeper)
                        product = Product.fetch(instance.product.key, keeper)
                        print(
                            yellow(u'Fixing package for {}: {}-->{}'.format(
                                instance.product, product.package,
                                correct_package)))
                        product.package = correct_package
                        instance.product = product

                    old_norm_price = instance.normalized_price_value
                    correct_norm_price = instance._get_normalized_price(
                        instance.price_value)
                    if old_norm_price != correct_norm_price:
                        print(
                            yellow(u'Fixing normal price '
                                   u'for {} report ({}-->{})'.format(
                                       instance.product, old_norm_price,
                                       correct_norm_price)))
                        instance.normalized_price_value = correct_norm_price
Пример #6
0
    def serve_data(self, product_category, location):
        """Return prepared category data"""
        category = product_category.category
        category_title = category.title
        category_title_ru = category.get_data('title_ru')
        category_primary_color = category.get_data('primary_color')
        category_background_color = category.get_data('background_color')
        prod_cat_title = product_category.get_data('ru_accu_case')
        median = product_category.get_price(location=location)
        category_delta = int(
            product_category.get_price_delta(self.delta_period,
                                             location=location) * 100)
        if not prod_cat_title:
            prod_cat_title = product_category.get_data('keyword').split(
                ', ')[0]
        package_key = product_category.get_data('normal_package')
        package_title = ProductPackage(package_key).get_data('synonyms')[0]

        chart_data = list()
        datetimes = get_datetimes(self.display_days)
        for date in datetimes:
            chart_data.append([
                date.strftime('%d.%m'),
                product_category.get_price(date, location=location)
            ])
        products = list()
        locations = product_category.get_locations()
        current_path = self.request.resource_url(product_category)
        sorted_products = sorted(
            product_category.get_qualified_products(location=location),
            key=lambda pr: pr[1])
        for num, product_tuple in enumerate(sorted_products):
            try:
                product, price = product_tuple
                middle_num = int(len(sorted_products) / 2)
                is_median = (num == middle_num)
                if len(sorted_products) % 2 == 0:
                    is_median = (num == middle_num or num == middle_num - 1)

                # construct data row as tuple
                products.append(
                    (num + 1, product, self.request.resource_url(product),
                     self.currency(price),
                     int(product.get_price_delta(self.delta_period) * 100),
                     is_median))
            except TypeError:
                pass
        return {
            'price_data': json.dumps(chart_data),
            'products': products,
            'cat_title': prod_cat_title,
            'current_location': location,
            'locations': locations,
            'category_title': category_title,
            'category_title_ru': category_title_ru,
            'category_background_color': category_background_color,
            'category_primary_color': category_primary_color,
            'current_path': current_path,
            'package_title': package_title,
            'median_price': self.currency(median) if median else None,
            'category_delta': category_delta if median else None
        }
Пример #7
0
    def serve_data(self, product):
        """Return prepared product data"""
        current_price = product.get_price()
        if current_price:
            current_price = self.currency(current_price)
            product_delta = int(
                product.get_price_delta(self.delta_period) * 100)
            last_report_url = None
        else:
            product_delta = 0
            last_price = product.get_last_reported_price()
            current_price = self.currency(last_price) if last_price else None
            last_report = self.context.get_last_report()
            last_report_url = \
                self.request.resource_url(last_report) if last_report else None
        datetimes = get_datetimes(self.display_days)
        chart_data = list()
        for date in datetimes:
            chart_data.append(
                [date.strftime('%d.%m'),
                 product.get_price(date)])
        chart_data = json.dumps(chart_data)
        package_key = product.category.get_data('normal_package')
        product_category_title = product.category.get_data('keyword').split(
            ', ')[0]
        product_category_url = self.request.resource_url(product.category)
        package_title = ProductPackage(package_key).get_data('synonyms')[0]
        type_ = product.category.category
        category_title = type_.title
        category_title_ru = type_.get_data('title_ru')
        category_primary_color = type_.get_data('primary_color')
        category_background_color = type_.get_data('background_color')
        reports = list()

        for report in sorted(
                product.get_reports(from_date_time=self.delta_period),
                reverse=True,
                key=lambda rep: rep.date_time):
            url = self.request.resource_url(report)
            date = format_datetime(report.date_time,
                                   format='short',
                                   locale=self.request.locale_name)
            merchant = report.merchant.title
            location = report.merchant.location
            price = self.currency(report.normalized_price_value)
            reports.append((url, date, merchant, location, price))

        return {
            'current_price': current_price,
            'product_delta': product_delta,
            'last_report_url': last_report_url,
            'chart_data': chart_data,
            'reports': reports,
            'product_category_title': product_category_title,
            'product_category_url': product_category_url,
            'category_title': category_title,
            'category_title_ru': category_title_ru,
            'category_background_color': category_background_color,
            'category_primary_color': category_primary_color,
            'package_title': package_title
        }