def get_by_upc( self, upc ):
query = ProductQuery.all().filter('upc =', upc)
result = query.fetch(api_key)
if result.total > 0:
p = result.products[0]
else:
return None
product, created = self.get_or_create(sku=p.sku)
if created:
product.product_id = p.product_id
product.name = fix_signs(p.name)
product.album_title = "" if getattr(p, 'album_title', "") is None else getattr(p, 'album_title', "")
product.artist_name = "" if getattr(p, 'artist_name', "") is None else getattr(p, 'artist_name', "")
product.sku_description = p.name if p.name else p.artist_name
# get the lowest category in the hierarchy
product.category = Category.objects.get(category_id=p.category_path[-1]["id"])
product.regular_price = p.regular_price
product.sale_price = p.sale_price
product.on_sale = p.on_sale
product.new = p.new
product.thumbnail_image = p.thumbnail_image
product.medium_image = p.medium_image
product.customer_review_average = p.customer_review_average
product.customer_review_count = 0 if p.customer_review_count == None else p.customer_review_count
product.url = "" if p.url is None else p.url
product.cart_url = "" if p.add_to_cart_url is None else p.add_to_cart_url
product.save()
return product
def get_name(self):
# check if the details are already there
if self.product_id is None:
query = ProductQuery.sku(self.sku).show_all()
p = query.fetch(api_key)
if p is None:
return {}
self.product_id = p.product_id
self.name = fix_signs(p.name)
self.album_title = p.get('album_title', None)
self.artist_name = p.get('artist_name', None)
self.sku_description = p.name if p.name else p.artist_name
# get the lowest category in the hierarchy
self.category = Category.objects.get(category_id=p.category_path[-1]["id"])
self.regular_price = p.regular_price
self.sale_price = p.sale_price
self.on_sale = p.on_sale
self.new = p.new
self.thumbnail_image = p.thumbnail_image
self.medium_image = p.medium_image
self.customer_review_average = p.customer_review_average
self.customer_review_count = p.customer_review_count
self.url = p.url
self.cart_url = p.add_to_cart_url
self.save()
return self.name
def get_by_sku( self, sku ):
if self.filter(sku=sku).count() > 0:
# exists in the db
p = self.get(sku=sku)
if p.product_id is not None:
return p
query = ProductQuery.sku(sku).show_all()
p = query.fetch(api_key)
if p is None:
return None
product, created = self.get_or_create(sku=p.sku)
product.product_id = p.product_id
product.name = fix_signs(p.name)
product.album_title = "" if getattr(p, 'album_title', "") is None else getattr(p, 'album_title', "")
product.artist_name = "" if getattr(p, 'artist_name', "") is None else getattr(p, 'artist_name', "")
product.sku_description = p.name if p.name else p.artist_name
# get the lowest category in the hierarchy
product.category, created = Category.objects.get_or_create(category_id=p.category_path[-1]["id"])
product.regular_price = p.regular_price
product.sale_price = p.sale_price
product.on_sale = p.on_sale
product.new = p.new
product.thumbnail_image = "" if p.thumbnail_image is None else p.thumbnail_image
product.medium_image = "" if p.medium_image is None else p.medium_image
product.customer_review_average = 0.0 if p.customer_review_average is None else p.customer_review_average
product.customer_review_count = 0 if p.customer_review_count == None else p.customer_review_count
product.url = "" if p.url is None else p.url
product.cart_url = "" if p.add_to_cart_url is None else p.add_to_cart_url
product.save()
return product
def query(self, url):
p = {}
r = requests.get(url, headers=self.headers)
html = r.content
if url.startswith('https://clearance.bestbuy.com'):
original_price = re.findall(r'original-price">\$([\d.,]+)</span>', html, re.DOTALL)
current_price = re.findall(r'<div class="your-price(?:.*?)">\$([\d.,]+)</div>', html, re.DOTALL)
name = re.findall(r'<h1>(.*?)</h1>', html, re.DOTALL)
sku = re.findall(r'<span><strong>SKU:</strong> ([\d]+)</span>', html, re.DOTALL)
print original_price, current_price
p = {
'name': name[1],
'type': 'None',
'current_price': original_price[0],
'original_price': current_price[0],
'uuid': sku[0],
}
else:
try:
price = re.findall(r'price">(?:\$|<span class="denominator">\$</span>)([\d.,]+)(?:</span>|</div>)', html, re.DOTALL)
print price
product_query = ProductQuery.sku(self.get_uuid(url)).show_all()
api_url = product_query.url(self.api_key, pid=int(self.get_pid(url)))
print api_url, 0
r = requests.get(api_url, headers=self.headers)
j = json.loads(r.content)
p = {
'name': j.get('name', None),
'type': j.get('type', None),
'current_price': price[0].replace(',', ''),
'original_price': price[1].replace(',', ''),
'uuid': j.get('sku', None),
}
except:
product_query = ProductQuery.sku(self.get_uuid(url)).show_all()
api_url = product_query.url(self.api_key, pid=int(self.get_pid(url)))
print api_url, 1
r = requests.get(api_url, headers=self.headers)
j = json.loads(r.content)
p = {
'name': j.get('name', None),
'type': j.get('type', None),
'current_price': j.get('salePrice', None),
'original_price': j.get('regularPrice', None),
'uuid': j.get('sku', None),
}
return p
def get_by_product_id( self, product_id):
query = ProductQuery.all().filter('product_id =', product_id)
result = query.fetch(api_key)
if result.total > 0:
p = result.products[0]
else:
return None
product, created = self.get_or_create(sku=p.sku)
if created:
product.name = fix_signs(p.name)
product.album_title = "" if getattr(p, 'album_title', "") is None else getattr(p, 'album_title', "")
product.artist_name = "" if getattr(p, 'artist_name', "") is None else getattr(p, 'artist_name', "")
product.sku_description = p.name if p.name else p.artist_name
product.url = "" if p.url is None else p.url
product.cart_url = "" if p.add_to_cart_url is None else p.add_to_cart_url
product.save()
return product
def search(self, s, page=1, me=None):
results = {}
# search by name, artist_name, short description
query = ProductQuery.all().extend('name =', "%s*"%s).extend('album_title =', "%s*"%s).extend('artist_name =', "%s*"%s).extend('short_description =', "%s*"%s)
result = query.fetch(api_key, page=page)
if result.total > 0:
results['total_pages'] = result.total_pages
results['products'] = []
for p in result.products:
#print sku
if len(p.category_path) == 0:
continue
product, created = self.get_or_create(sku=p.sku)
if created:
product.product_id = p.product_id
product.name = fix_signs(p.name)
product.album_title = "" if getattr(p, 'album_title', "") is None else getattr(p, 'album_title', "")
product.artist_name = "" if getattr(p, 'artist_name', "") is None else getattr(p, 'artist_name', "")
product.sku_description = p.name if p.name else p.artist_name
# get the lowest category in the hierarchy
#print p.category_path
product.category = Category.objects.get(category_id=p.category_path[-1]["id"])
product.regular_price = p.regular_price
product.sale_price = p.sale_price
product.on_sale = p.on_sale
product.new = p.new
product.thumbnail_image = "" if p.thumbnail_image is None else p.thumbnail_image
product.medium_image = "" if p.medium_image is None else p.medium_image
product.customer_review_average = 0.0 if p.customer_review_average is None else p.customer_review_average
product.customer_review_count = 0 if p.customer_review_count == None else p.customer_review_count
product.url = "" if p.url is None else p.url
product.cart_url = "" if p.add_to_cart_url is None else p.add_to_cart_url
product.save()
results['products'].append(product.details(me))
return results
def filter_category( self, cat_id, page=1, me=None):
"""
:param cat_id: the BestBuy category ID
"""
results = {}
query = ProductQuery.all().filter('category_path.id =', cat_id)
result = query.fetch(api_key, page=page)
if result.total > 0:
results['total_pages'] = result.total_pages
results['products'] = []
for p in result.products:
product, created = self.get_or_create(sku=p.sku)
if created:
product.product_id = p.product_id
product.name = fix_signs(p.name)
product.album_title = "" if getattr(p, 'album_title', "") is None else getattr(p, 'album_title', "")
product.artist_name = "" if getattr(p, 'artist_name', "") is None else getattr(p, 'artist_name', "")
product.sku_description = p.name if p.name else p.artist_name
# get the lowest category in the hierarchy
product.category = Category.objects.get(category_id=p.category_path[-1]["id"])
product.regular_price = p.regular_price
product.sale_price = p.sale_price
product.on_sale = p.on_sale
product.new = p.new
product.thumbnail_image = "" if p.thumbnail_image is None else p.thumbnail_image
product.medium_image = "" if p.medium_image is None else p.medium_image
product.customer_review_average = 0.0 if p.customer_review_average is None else p.customer_review_average
product.customer_review_count = 0 if p.customer_review_count == None else p.customer_review_count
product.url = "" if p.url is None else p.url
product.cart_url = "" if p.add_to_cart_url is None else p.add_to_cart_url
product.save()
results['products'].append(product.details(me))
return results
#!/usr/bin/python
import os
import json
import time
import urllib2
import sys
from premix import ProductQuery
api_key='3xvhyc8utc852wrwhy34n8wk'
f2=open('products.txt','w')
product_query = ProductQuery.all().filter('search =', 'laptop')
results=product_query.fetch(api_key)
totalpages = results.total_pages
curpage = results.current_page
while (curpage < totalpages) :
try:
cur_results = product_query.fetch(api_key, page=curpage)
begin = 1
end =10
curprod=begin
time.sleep(1)
while curprod < end :
product = cur_results.products[curprod]
sku = product.sku
webURL = 'http://api.remix.bestbuy.com/v1/products(sku='+str(sku)+')?format=json&apiKey=3xvhyc8utc852wrwhy34n8wk'
try:
def details(self, me=None):
"""
me is Party object that signifies myself who's looking at info
"""
# check if the details are already there
if self.product_id is None:
query = ProductQuery.sku(self.sku).show_all()
p = query.fetch(api_key)
if p is None:
return {}
self.product_id = p.product_id
self.name = fix_signs(p.name)
product.album_title = "" if getattr(p, 'album_title', "") is None else getattr(p, 'album_title', "")
product.artist_name = "" if getattr(p, 'artist_name', "") is None else getattr(p, 'artist_name', "")
self.sku_description = p.name if p.name else p.artist_name
# get the lowest category in the hierarchy
self.category = Category.objects.get(category_id=p.category_path[-1]["id"])
self.regular_price = p.regular_price
self.sale_price = p.sale_price
self.on_sale = p.on_sale
self.new = p.new
self.thumbnail_image = "" if p.thumbnail_image is None else p.thumbnail_image
self.medium_image = "" if p.medium_image is None else p.medium_image
self.customer_review_average = 0.0 if p.customer_review_average is None else p.customer_review_average
self.customer_review_count = 0 if p.customer_review_count == None else p.customer_review_count
self.url = "" if p.url is None else p.url
self.cart_url = "" if p.add_to_cart_url is None else p.add_to_cart_url
self.save()
detail = {}
detail['sku'] = str(self.sku)
detail['product_id'] = str(self.product_id)
detail['name'] = self.name
detail['album_title'] = self.album_title
detail['artist_name'] = self.artist_name
# exclude most highlevel Best Buy category
detail['category'] = self.category.name
detail['regular_price'] = "%.2f"%self.regular_price
detail['sale_price'] = "%.2f"%self.sale_price
detail['on_sale'] = self.on_sale
detail['new'] = self.new
detail['thumbnail_image'] = self.thumbnail_image
detail['medium_image'] = self.medium_image
detail['customer_review_average'] = "%.1f"%(self.customer_review_average if self.customer_review_average else 0)
detail['customer_review_count'] = str(self.customer_review_count)
detail['wished_by'] = ""
detail['bought_by'] = ""
detail['review_request'] = ""
detail['num_wished'] = '0'
detail['num_bought'] = '0'
detail['num_requested'] = '0'
detail['url'] = self.url
detail['cart_url'] = self.cart_url
if me is not None:
# for experiment 1 and when nobody wishes or bought it
if me.experiment.id == 2:
# exclude reviews that I have requested and reviews that I have already replied
reqs = ReviewRequest.objects.exclude(requester=me).filter(product=self, requester__in=me.friends()).exclude(replies__reviewer=me)
if reqs.count() > 0:
detail['num_requested'] = str(reqs.count())
request_str = "Requested review by "
for q in reqs:
request_str += q.requester.first_name + ", "
detail['review_request'] = request_str[:-2]
wishes = Wishlist.objects.filter(product=self, party__in=me.friends())
if wishes.count() > 0:
detail['num_wished'] = str(wishes.count())
wish_str = "Wished by "
for w in wishes:
wish_str += w.party.first_name + ", "
detail['wished_by'] = wish_str[:-2]
purchases = TransactionLineItem.objects.filter(product=self, transaction__party__in=me.friends())
if purchases.count() > 0:
detail['num_bought'] = str(purchases.count())
purchase_str = "Bought by "
for b in purchases:
purchase_str += b.transaction.party.first_name + ", "
detail['bought_by'] = purchase_str[:-2]
elif me.experiment.id == 3:
# exclude reviews that I have requested and reviews that I have already replied
num_reqs = ReviewRequest.objects.exclude(requester=me).filter(product=self).exclude(replies__reviewer=me).count()
if num_reqs > 0:
detail['num_requested'] = str(num_reqs)
detail['review_request'] = "%d people requested for review"%num_reqs if num_reqs > 1 else "%d person requested for review"%num_reqs
num_wish = Wishlist.objects.filter(product=self).exclude(party=me).count()
if num_wish > 0:
detail['num_wished'] = str(num_wish)
detail['wished_by'] = "%d people want this"%num_wish if num_wish > 1 else "%d person want this"%num_wish
num_bought = TransactionLineItem.objects.filter(product=self).exclude(transaction__party=me).count()
if num_bought > 0:
detail['num_bought'] = str(num_bought)
detail['bought_by'] = "%d people bought this"%num_bought if num_bought > 1 else "%d person bought this"%num_bought
elif me.experiment.id == 4:
# exclude reviews that I have requested and reviews that I have already replied
reqs = ReviewRequest.objects.exclude(requester=me).filter(product=self, requester__in=me.friends()).exclude(replies__reviewer=me)
if reqs.count() > 0:
detail['num_requested'] = str(reqs.count())
detail['review_request'] = "%d friends requested for review"%reqs.count() if reqs.count() > 1 else "%d friend requested for review"%reqs.count()
num_wish = Wishlist.objects.filter(product=self).exclude(party=me).count()
if num_wish > 0:
detail['num_wished'] = str(wishes.count())
detail['wished_by'] = "%d friends want this"%num_wish if num_wish > 1 else "%d friend want this"%num_wish
num_bought = TransactionLineItem.objects.filter(product=self).exclude(transaction__party=me).count()
if num_bought > 0:
detail['num_bought'] = str(bought.count())
detail['bought_by'] = "%d friends bought this"%num_bought if num_bought > 1 else "%d friend bought this"%num_bought
return detail