def test_delete_report(self): victim = PriceReport.fetch(self.report1_key, self.keeper) product = victim.product victim.delete_from(self.keeper) transaction.commit() self.assertNotIn(victim, product) self.assertNotIn(victim, PriceReport.fetch_all(self.keeper))
def fix_normalized_price(): """ Walk through all reports and fix normalized price_value and product package """ keeper = get_storage() reports = PriceReport.fetch_all(keeper) for report in reports: try: correct_package_key = report.product.get_package_key() if report.product.package.key != correct_package_key: correct_package = ProductPackage.acquire(correct_package_key, keeper) product = Product.fetch(report.product.key, keeper) print(yellow(u'Fixing package for {}: {}-->{}'.format( report.product, product.package, correct_package))) product.package = correct_package report.product = product old_norm_price = report.normalized_price_value new_norm_price = report._get_normalized_price(report.price_value) if old_norm_price != new_norm_price: print(yellow(u'Fixing normal price {}-->{}'.format( old_norm_price, new_norm_price))) report.normalized_price_value = new_norm_price except PackageLookupError, e: print(e.message)
def fix_normalized_price(): """ Walk through all reports and fix normalized price_value and product package """ keeper = get_storage() reports = PriceReport.fetch_all(keeper) for report in reports: try: correct_package_key = report.product.get_package_key() if report.product.package.key != correct_package_key: correct_package = ProductPackage.acquire( correct_package_key, keeper) product = Product.fetch(report.product.key, keeper) print( yellow(u'Fixing package for {}: {}-->{}'.format( report.product, product.package, correct_package))) product.package = correct_package report.product = product old_norm_price = report.normalized_price_value new_norm_price = report._get_normalized_price(report.price_value) if old_norm_price != new_norm_price: print( yellow(u'Fixing normal price {}-->{}'.format( old_norm_price, new_norm_price))) report.normalized_price_value = new_norm_price except PackageLookupError, e: print(e.message)
def recreate(): """Recreate storage from reports""" keeper = get_storage() new_keeper = get_storage('storage/new.fs') reports = PriceReport.fetch_all(keeper) print(cyan('Recreating storage from {} reports...'.format(len(reports)))) for report in reports: try: PriceReport.assemble(storage_manager=new_keeper, uuid=report.uuid, price_value=report.price_value, product_title=report.product.title, merchant_title=report.merchant.title, reporter_name=report.reporter.name, url=report.url, date_time=report.date_time) except CategoryLookupError: print(yellow(u'Dropping `{}`: ' u'no category...'.format(report.product))) transaction.commit() keeper.close() new_keeper.close()
def recreate(): """Recreate storage from reports""" keeper = get_storage() new_keeper = get_storage('storage/new.fs') reports = PriceReport.fetch_all(keeper) print(cyan('Recreating storage from {} reports...'.format(len(reports)))) for report in reports: try: PriceReport.assemble(storage_manager=new_keeper, uuid=report.uuid, price_value=report.price_value, product_title=report.product.title, merchant_title=report.merchant.title, reporter_name=report.reporter.name, url=report.url, date_time=report.date_time) except CategoryLookupError: print( yellow(u'Dropping `{}`: ' u'no category...'.format(report.product))) transaction.commit() keeper.close() new_keeper.close()
def test_report_assembly(self): raw_data1 = { 'product_title': u'Молоко Great Milk 1L', 'sku': 'ART97665', 'price_value': 42.6, 'url': 'http://scottys.com/products/milk/1', 'merchant_title': "Scotty's grocery", 'date_time': None, 'reporter_name': 'Jill' } raw_data2 = { 'product_title': u'Молоко Great Milk 0.93 L', 'price_value': 50, 'url': 'http://scottys.com/products/milk/5', 'merchant_title': "Scotty's grocery", 'date_time': None, 'reporter_name': 'Jill' } raw_data3 = { 'product_title': u'Сметана Great Sour Cream 450g', 'price_value': 60.4, 'url': 'http://scottys.com/products/sc/6', 'merchant_title': "Scotty's grocery", 'date_time': datetime.datetime(2014, 10, 5), 'reporter_name': 'Jill' } raw_data4 = { 'product_title': u'Сметана Great Sour Cream 987987g', 'price_value': 60.4, 'url': 'http://scottys.com/products/sc/8978', 'merchant_title': "Scotty's grocery", 'date_time': datetime.datetime(2014, 10, 5), 'reporter_name': 'Jill' } raw_data5 = { 'product_title': u'Картофель Вегетория для варки 3кг', 'price_value': 80.5, 'url': 'http://scottys.com/products/pot/324', 'merchant_title': "Scotty's grocery", 'reporter_name': 'Jill' } report1, stats1 = PriceReport.assemble(storage_manager=self.keeper, **raw_data1) report2, stats2 = PriceReport.assemble(storage_manager=self.keeper, **raw_data2) report3, stats3 = PriceReport.assemble(storage_manager=self.keeper, **raw_data3) report5, stats5 = PriceReport.assemble(storage_manager=self.keeper, **raw_data5) try: PriceReport.assemble(storage_manager=self.keeper, **raw_data4) except PackageLookupError: pass transaction.commit() # check category and products milk = ProductCategory.fetch('milk', self.keeper) sc = ProductCategory.fetch('sour cream', self.keeper) self.assertEqual(6, len(milk.products)) self.assertIn(42.6, milk.get_prices()) product1 = Product.fetch(u'Молоко Great Milk 1L', self.keeper) product2 = Product.fetch(u'Молоко Great Milk 0.93 L', self.keeper) self.assertIs(milk, product1.category) self.assertEqual('1 l', product1.package.title) self.assertEqual('ART97665', report1.sku) self.assertEqual('0.93 l', product2.package.title) self.assertEqual(1, product1.package_ratio) self.assertEqual(0.93, product2.package_ratio) self.assertFalse(hasattr(report2, 'sku')) self.assertIn(product1, milk.products) self.assertEqual('sour cream', report3.product.category.title) self.assertIn(u'Сметана Great Sour Cream 450g', [p.title for p in sc.products]) self.assertNotIn(u'Сметана Great Sour Cream 987987g', [p.title for p in sc.products]) # check references potato = ProductCategory.fetch('potato', self.keeper) self.assertIn(report5, PriceReport.fetch_all(self.keeper)) self.assertIn(report5, potato.products[0].reports) self.assertIn(report5, Product.fetch(u'Картофель Вегетория для варки 3кг', self.keeper).reports) # check reporter jill = Reporter.fetch('Jill', self.keeper) self.assertIs(jill, report1.reporter) self.assertIs(jill, report2.reporter) self.assertIs(jill, report3.reporter) # check price calculations self.assertEqual(42.60, round(report1.normalized_price_value, 2)) self.assertEqual(53.76, round(report2.normalized_price_value, 2)) self.assertEqual(53.69, round(report3.normalized_price_value, 2)) # check merchant merchant = Merchant.fetch("Scotty's grocery", self.keeper) self.assertIs(merchant, report1.merchant) self.assertIn(merchant, product1.merchants) self.assertIn(product1, merchant) # check datetime that is not now date_in_past = datetime.datetime(2014, 10, 5) self.assertEqual(date_in_past, report3.date_time) # check if no false product is registered false_product = Product.fetch(u'Сметана Great Sour Cream 987987g', self.keeper) self.assertIsNone(false_product)
def test_report_assembly(self): raw_data1 = { "product_title": u"Молоко Great Milk 1L", "sku": "ART97665", "price_value": 42.6, "url": "http://scottys.com/products/milk/1", "merchant_title": "Scotty's grocery", "date_time": None, "reporter_name": "Jill", } raw_data2 = { "product_title": u"Молоко Great Milk 0.93 L", "price_value": 50, "url": "http://scottys.com/products/milk/5", "merchant_title": "Scotty's grocery", "date_time": None, "reporter_name": "Jill", } raw_data3 = { "product_title": u"Сметана Great Sour Cream 450g", "price_value": 60.4, "url": "http://scottys.com/products/sc/6", "merchant_title": "Scotty's grocery", "date_time": datetime.datetime(2014, 10, 5), "reporter_name": "Jill", } raw_data4 = { "product_title": u"Сметана Great Sour Cream 987987g", "price_value": 60.4, "url": "http://scottys.com/products/sc/8978", "merchant_title": "Scotty's grocery", "date_time": datetime.datetime(2014, 10, 5), "reporter_name": "Jill", } raw_data5 = { "product_title": u"Картофель Вегетория для варки 3кг", "price_value": 80.5, "url": "http://scottys.com/products/pot/324", "merchant_title": "Scotty's grocery", "reporter_name": "Jill", } report1, stats1 = PriceReport.assemble(storage_manager=self.keeper, **raw_data1) report2, stats2 = PriceReport.assemble(storage_manager=self.keeper, **raw_data2) report3, stats3 = PriceReport.assemble(storage_manager=self.keeper, **raw_data3) report5, stats5 = PriceReport.assemble(storage_manager=self.keeper, **raw_data5) try: PriceReport.assemble(storage_manager=self.keeper, **raw_data4) except PackageLookupError: pass transaction.commit() # check category and products milk = ProductCategory.fetch("milk", self.keeper) sc = ProductCategory.fetch("sour cream", self.keeper) self.assertEqual(6, len(milk.products)) self.assertIn(42.6, milk.get_prices()) product1 = Product.fetch(u"Молоко Great Milk 1L", self.keeper) product2 = Product.fetch(u"Молоко Great Milk 0.93 L", self.keeper) self.assertIs(milk, product1.category) self.assertEqual("1 l", product1.package.title) self.assertEqual("ART97665", report1.sku) self.assertEqual("0.93 l", product2.package.title) self.assertEqual(1, product1.package_ratio) self.assertEqual(0.93, product2.package_ratio) self.assertFalse(hasattr(report2, "sku")) self.assertIn(product1, milk.products) self.assertEqual("sour cream", report3.product.category.title) self.assertIn(u"Сметана Great Sour Cream 450g", [p.title for p in sc.products]) self.assertNotIn(u"Сметана Great Sour Cream 987987g", [p.title for p in sc.products]) # check references potato = ProductCategory.fetch("potato", self.keeper) self.assertIn(report5, PriceReport.fetch_all(self.keeper)) self.assertIn(report5, potato.products[0].reports) self.assertIn(report5, Product.fetch(u"Картофель Вегетория для варки 3кг", self.keeper).reports) # check reporter jill = Reporter.fetch("Jill", self.keeper) self.assertIs(jill, report1.reporter) self.assertIs(jill, report2.reporter) self.assertIs(jill, report3.reporter) # check price calculations self.assertEqual(42.60, round(report1.normalized_price_value, 2)) self.assertEqual(53.76, round(report2.normalized_price_value, 2)) self.assertEqual(53.69, round(report3.normalized_price_value, 2)) # check merchant merchant = Merchant.fetch("Scotty's grocery", self.keeper) self.assertIs(merchant, report1.merchant) self.assertIn(merchant, product1.merchants) self.assertIn(product1, merchant) # check datetime that is not now date_in_past = datetime.datetime(2014, 10, 5) self.assertEqual(date_in_past, report3.date_time) # check if no false product is registered false_product = Product.fetch(u"Сметана Great Sour Cream 987987g", self.keeper) self.assertIsNone(false_product)