def process_item(self, item, spider):
# Check if the Product already exists
product = (self.session.query(Product).filter_by(
store=item["store"], sku=item["sku"]).first())
if product is None:
product = Product(store=item["store"], sku=item["sku"])
product.barcodes = item["barcodes"]
product.brand = item["brand"]
product.name = item["name"]
product.description = item["description"]
product.image_url = item["image_url"]
self.session.add(product)
self.session.commit()
# Check if the BranchProduct already exists
branch_product = (self.session.query(BranchProduct).filter_by(
product=product, branch=item["branch"]).first())
if branch_product is None:
branch_product = BranchProduct(product=product,
branch=item["branch"])
branch_product.stock = item["stock"]
branch_product.price = item["price"]
self.session.add(branch_product)
self.session.commit()
return item
def test_process_item_existing_product_and_existing_branch_product(self):
existing_product = Product(
store=self.item["store"],
sku=self.item["sku"],
brand=self.item["brand"],
name=self.item["name"],
description=self.item["description"],
package=self.item["package"],
image_url=self.item["image_url"],
)
self.session.add(existing_product)
self.session.commit()
existing_branch_product = BranchProduct(
product=existing_product,
branch=self.item["branch"],
stock=self.item["stock"],
price=self.item["price"],
)
self.session.add(existing_branch_product)
self.session.commit()
pipeline = StoragePipeline(db_engine=self.engine)
pipeline.open_spider(self.spider)
pipeline.process_item(self.item, self.spider)
pipeline.close_spider(self.spider)
product = (self.session.query(Product).filter_by(
store=self.item["store"], sku=self.item["sku"]).first())
self.assertEqual(product, existing_product)
branch_product = (self.session.query(BranchProduct).filter_by(
product=product, branch=self.item["branch"]).first())
self.assertEqual(branch_product, existing_branch_product)
def save(self, item):
try:
product = Product(
store='Walmart',
barcodes=item.get('barcodes'),
sku=str(item.get('sku')),
brand=item.get('brand'),
name=item.get('name'),
description=item.get('description'),
package=item.get('package'),
categories=item.get('categories'),
image_urls=item.get('image_urls'))
branch = BranchProduct(
branch=item.get('store'),
product=product,
stock=item['stock'],
price=float(item.get('price')),
)
self.session.add(product)
self.session.commit()
return item
except IntegrityError:
self.session.rollback()
return None
def saveDatabase(self, item):
try:
product = Product(store="Richart's",
barcodes=item['barcodes'],
sku=str(item['sku']),
brand=item['brand'],
name=item['name'],
description=item['description'],
package=item['package'],
categories=item['categories'],
image_urls=item['image_urls'])
branch = BranchProduct(
branch=item['store'],
product=product,
stock=item['stock'],
price=float(item['price']),
)
self.session.add(product)
self.session.commit()
return item
except IntegrityError as e:
self.session.rollback()
return None
def test_create_example_product(self):
some_product = Product(
store="Walmart",
barcodes="60538887928,1234567890,1234567891",
sku="10295446",
brand="Great Value",
name="Spring Water",
description=
"Convenient and refreshing, Great Value Spring Water is a healthy option that is Sodium-free and Non-carbonated. Bottled water is a quick and convenient way to fulfill your body's hydration needs. Zero calories, free from artifical flavors or colors, water is the right choice when it comes to your packaged beverage options. Eco-friendly, the bottle is made from 100% recycled plastic, a sustainable choice that is good for you and for the environment.",
package="24 x 500ml",
image_url=
"https://i5.walmartimages.ca/images/Large/887/928/999999-60538887928.jpg",
category="Pantry, Household & Pets|Drinks|Water|Bottled Water",
url=
"https://www.walmart.ca/en/ip/great-value-24pk-spring-water/6000143709667",
)
some_branch_product_1 = BranchProduct(
branch="BRNCH",
product=some_product,
stock=123,
price=2.27,
)
some_branch_product_2 = BranchProduct(
branch="3106",
product=some_product,
stock=0,
price=2.27,
)
self.session.add(some_product)
self.session.commit()
products = self.session.query(Product).all()
self.assertEqual(len(products), 1)
self.assertIn(some_product, products)
branch_products = self.session.query(BranchProduct).all()
self.assertEqual(len(branch_products), 2)
self.assertIn(some_branch_product_1, branch_products)
self.assertIn(some_branch_product_2, branch_products)
def products_to_db(products):
"""
It saves the products in the database
:param products: dictionary with the desired information
"""
session = load_session()
for key, item in products.items():
print('\n>>> Processing:', key, item['NAME'])
product = (session.query(Product).filter_by(store="Richart's",
sku=item["SKU"]).first())
if product is None:
product = Product(store="Richart's", sku=item["SKU"])
product.barcodes = item["BARCODES"]
product.brand = item["BRAND"].capitalize()
product.name = item["NAME"].capitalize()
description = remove_html_tags(item["DESCRIPTION"])
product.description = description.capitalize()
product.image_url = item["IMAGE_URL"]
product.category = item["FULL_CATEGORY"]
product.package = product.description.replace(product.name, '')
session.add(product)
session.commit()
# Check if the BranchProduct already exists
branch_product = (session.query(BranchProduct).filter_by(
product=product, branch=item["BRANCH"]).first())
if branch_product is None:
branch_product = BranchProduct(product=product,
branch=item["BRANCH"])
branch_product.stock = item["STOCK"]
branch_product.price = item["PRICE"]
session.add(branch_product)
session.commit()
session.close()
def process_item(self, item, spider):
session = self.Session()
# create Product object
product = Product(store="Walmart",
barcodes=item['bar_code'],
sku=item['sku'],
brand=item['brand'],
name=item['name'],
description=item['description'],
package=item['package'],
image_urls=item['image_urls'],
category=item['category'],
product_url=item['product_url'])
# create BranchProduct objects
branch_one = get_product_info_from_branch(
list(BRANCHES_INFO.keys())[0], item['bar_code'])
branch_two = get_product_info_from_branch(
list(BRANCHES_INFO.keys())[1], item['bar_code'])
branch_product_1 = BranchProduct(
branch=str(list(BRANCHES_INFO.keys())[0]),
product=product,
stock=branch_one.get('availableToSellQty', 0),
price=branch_one.get('sellPrice', 0),
)
branch_product_2 = BranchProduct(
branch=str(list(BRANCHES_INFO.keys())[1]),
product=product,
stock=branch_two.get('availableToSellQty', 0),
price=branch_two.get('sellPrice', 0),
)
session.add(product)
session.commit()
return item
def process_item(item):
Session = sessionmaker(bind=engine)
session = Session()
# Check if the Product already exists
product = (session.query(Product).filter_by(store=item["store"],
sku=item["sku"]).first())
if product is None:
product = Product(store=item["store"], sku=item["sku"])
product.barcodes = item["barcodes"]
product.brand = item["brand"]
product.name = item["name"]
product.description = item["description"]
product.image_url = item["image_url"]
product.category = item["category"]
product.package = item["package"]
session.add(product)
session.commit()
# Check if the BranchProduct already exists
branch_product = (session.query(BranchProduct).filter_by(
product=product, branch=item["branch"]).first())
if branch_product is None:
branch_product = BranchProduct(product=product, branch=item["branch"])
branch_product.stock = item["stock"]
branch_product.price = item["price"]
session.add(branch_product)
session.commit()
return item