def insert_data(self):
"""
Method that insert categories and aliments into
database with a many to many relationship
"""
with transaction.atomic():
categories = self.get_categories(30)
aliments = self.get_aliments(categories)
# keep only needed product information
for cat, elements in aliments.items():
aliments[cat] = self.cleaned_data(elements)
# insert categories in database
if Category.objects.exists():
pass
else:
for cat in categories:
category = Category(name=cat)
category.save()
# insert products in database with a many to many
# relationship with categories
for cat, elements in aliments.items():
category = Category.objects.get(name=cat)
for data in elements:
category.products.get_or_create(barcode=data[0],
name=data[1],
nutriscore=data[2],
image=data[3],
small_image=data[4],
url=data[5],
nutrition_img=data[6])
def handle(self, *args, **options):
print("**************************************************")
print("STARTING DATABASE_UPDATE - {}".format(datetime.now()))
print("**************************************************")
for nutriment in settings.NUTRIMENTS:
if not Nutriment.objects.filter(name__iexact=nutriment):
new_nutriment = Nutriment(
name=nutriment,
unit=settings.NUTRIMENTS[nutriment]["unit"])
new_nutriment.save()
print("Adding new nutriment to database :", nutriment)
else:
print("Existing nutriment :", nutriment)
print("--------------------------------------------------")
for category in settings.PRODUCTS_CATEGORIES:
if not Category.objects.filter(name__iexact=category):
new_category = Category(name=category)
new_category.save()
print("Adding new category to database :", category)
else:
print("Existing category :", category)
print("--------------------------------------------------")
for category in Category.objects.all():
self.get_products_for_category(category.name)
print("**************************************************")
print("END OF DATABASE_UPDATE - {}".format(datetime.now()))
print("**************************************************")
def get(self, request, filter = ''):
self.all_categories = Category.update_sub_category_lists()
self.categories = Category.find_main_categories(self.all_categories)
# here we use the filter to load the products accordingly!
if filter != '':
self.category_from_filter = list(Category.objects.filter(name = filter))
self.list_of_all_categories_from_filter = Category.get_all_sub_categories(self.category_from_filter[0])
self.list_of_all_categories_from_filter.append(self.category_from_filter[0])
self.all_products = Product.get_products_from_list_of_categories(self.list_of_all_categories_from_filter)
else:
self.all_products = Product.get_all_products()
#now we find all the images we need for each product
self.all_product_images = []
for product in self.all_products:
img = list(ProductImage.find_all_product_images(product.id))
self.all_product_images += img
# this should be to load the homepage, so give featured products and catalog data
return render(request, 'products/home.html', {'main_categories':self.categories, 'all_categories':self.all_categories, 'products':self.all_products, 'product_images':self.all_product_images })
def setUp(self):
# Create the model instances required for the relationship
self.new_difficulty = Difficulty(name="Very Tough")
self.new_difficulty.save()
self.new_category = Category(name="Solid Cakes")
self.new_category.save()
# Create the raw data for testing
lesson_raw_data = {
"name":"Cherry Cake",
"price": 3000,
"average_class_size": 25,
"difficulty_level": self.new_difficulty,
"description": "Good Stuff",
}
product_raw_data = {
"name": "Fairy Cake",
"price": 3000,
"category":self.new_category,
"description": "Great Cake!",
"sizes": "S"
}
self.new_lesson = Lesson(**lesson_raw_data)
self.new_lesson.save()
self.new_product = Product(**product_raw_data)
self.new_product.save()
def setUp(self):
# Create the model instances required for the relationship
self.new_difficulty = Difficulty(name="Very Tough")
self.new_difficulty.save()
self.new_category = Category(name="Solid Cakes")
self.new_category.save()
# Create the raw data for testing
raw_data = {
"name": "Cherry Cake",
"price": 3000,
"average_class_size": 25,
"difficulty_level": self.new_difficulty,
"description": "Good Stuff",
}
product_raw_data = {
"name": "Fairy Cake",
"price": 3000,
"category": self.new_category,
"description": "Great Cake!",
"sizes": "S"
}
#Pass the dictionary as named parameters to a function call
self.new_lesson = Lesson(**raw_data)
self.new_lesson.save()
self.new_product = Product(**product_raw_data)
self.new_product.save()
self.user = User.objects.create_user(username='******',
password='******')
self.client.login(username='******', password='******')
def add_product(request):
import pdb
pdb.set_trace()
if request.method == 'POST':
data = request.POST
categorySerializer = ProductSerializer(data=request.POST)
# subCategorySerializer = SubCategorySerializers(data=request.POST)
if categorySerializer.is_valid():
category = Category()
category.name = request.POST.get('name')
category.save(using='products')
subCategory = SubCategory()
subCategory.sub_name = request.POST.get('subcategory')
subCategory.category_id = category.id
subCategory.save(using='products')
message = {'message': 'Product Added Successfully'}
response = json.dumps(message)
return HttpResponse(response, content_type="application/json")
else:
content = {'message': 'Unable to add the product'}
response = json.dumps(content)
return HttpResponse(response, content_type="application/json")
else:
content = {'message': 'Something went wrong'}
response = json.dumps(content)
return HttpResponse(response, content_type="application/json")
def test_order_item_to_string_method(self):
test_category = Category(name="Kettlebells")
test_category.save()
test_discount = Discount(name="30%", rate=0.7)
test_discount.save()
test_manufacurer = Manufacturer(name="Great Sports")
test_manufacurer.save()
test_product = Product(name="Test", description="A description",
image_url="", category=test_category,
manufacturer=test_manufacurer,
prize=100,
discount_rate=test_discount,
units_sold=1)
test_user = User(username="******")
test_user.save()
date = datetime.datetime.now()
test_order = Order(date=date,
user=test_user,
amount_paid=100,
shipping_address="Street",
customer_name="Name",
customer_email="*****@*****.**")
test_order.save()
test_order_item = OrderItem(product=test_product,
order=test_order,
quantity=1)
self.assertEquals(test_order_item.__str__(), "Test")
def handle(self, *args, **options):
file_path = options.get('file')
if not file_path:
raise CommandError('File not provided.')
if not file_path.endswith('.json'):
raise CommandError('Only .json file supported.')
file_path = os.path.join('data', file_path)
try:
with open(file_path) as import_file:
categories = json.load(import_file)
except FileNotFoundError as e:
raise CommandError('File at %s was not found' %
os.path.join('data', file_path))
for category in categories:
# print(product['user_id'])
# user = AuthUserModel.objects.filter(id=product['user_id']).first()
# print(user.first_name)
db_category = Category(
name=category['name'],
image=category['image'],
)
db_category.save()
def setUp(self):
category = Category(name='maria_sharia')
category.save()
product = Product(name='name',
description='description',
offer='99.00')
product.save()
class ProductViewsTest(TestCase):
def setUp(self):
"""datas call before each funtion"""
#a cat
self.category = Category(name="jus de fruits")
self.category.save()
#a product
self.product1 = Product(
name="Pur jus d'orange sans pulpe",
image_url="https://static.openfoodfacts.org/images/products/350/211/000/9449/front_fr.80.400.jpg",
url="https://world.openfoodfacts.org/product/3502110009449/pur-jus-d-orange-sans-pulpe-tropicana",
nutriscore=3, packaging="carton")
self.product1.save()
def test_redirect_when_no_prod(self):
""" tests whether access is impossible when user is authenticated"""
response = self.client.get(reverse("products:infos"), follow=True)
self.assertEqual(response.status_code, 200)
self.assertRedirects(response, reverse('research:index'))
def test_display_page_when_prod_arg(self):
""" tests whether access is possible when user is authenticated"""
response = self.client.get("/p10/products/infos/Pur jus d'orange sans pulpe")
self.assertEqual(response.status_code, 200)
def handle(self, *args, **options):
number_categories = options['c']
number_products = options['p']
api = OpenFoodFactsAPI(number_categories, number_products)
# Adding categories
print(f"Adding {number_categories} categories", end="")
for api_category in api.categories:
category = Category(name=api_category)
category.save()
print(".", end="")
# Adding products
print(
f"\nAdding {number_categories * number_products} products, it can take a little while..."
)
for api_product in api.get_products():
api_product['category'] = Category.objects.get(
name=api_product['category'])
product = Product(**api_product)
# Taking care of duplicates if any (sanity check even with a set() in api.get_products())
try:
product.save()
except IntegrityError: # Unicity constraint
continue
print(
f"\nSuccessfully added {len(Product.objects.all())} products and "
f"{len(Category.objects.all())} categories. Differences may be duplicates."
)
def categories_processing(file, user):
tax = parse_taxonomy(file)
Category.delete_all()
return Category.store_from_taxonomy(tax,
parent=None,
created_by=user,
updated_by=user)
def setUp(self):
"""Method called to prepare the test fixture."""
self.options = ChromeOptions()
# self.options.add_experimental_option("excludeSwitches",
# ["enable-logging"])
self.options.add_argument('--headless')
self.options.add_argument('--disable-gpu')
self.options.add_argument('--remote-debugging-port=9222')
self.options.add_argument('--window-size=1920x1080')
self.browser = Chrome(chrome_options=self.options)
User = get_user_model()
self.user = User.objects.create_user(username='******',
email='*****@*****.**',
password='******')
self.category = Category(id_category="1à",
name="category_test",
products=1,
visible=True)
self.category.save()
for id_product in range(7):
self.product = Product(id_product="key%s" % id_product,
product_name_fr="test_%s" % id_product,
nutriscore_score=0,
nutriscore_grade='A',
brands='brand_test')
self.product.save()
self.product.categories.add(self.category)
def create_default_products(apps, schema_editor):
with open('products/data/products.json', 'r') as file:
context = json.load(file)
categories = []
for product in context['products']:
if product['category'] not in categories:
categories.append(product['category'])
query = Category.objects.filter(title=product['category']).count()
if not query:
new_category = Category(title=product['category'])
new_category.save()
new_product = Product(
name=product['name'],
title=product['title'],
alt=product['alt'],
description=product['description'],
characteristics=product['characteristics'],
details=product['details'],
image=Image.objects.get(title='default'),
url_key=product['url_key'],
category=Category.objects.get(title=product['category']),
)
new_product.save()
def test_product_duplicate_name(self):
url = '/product/add'
cat = Category(name='new_category')
cat.save()
response = self.client.post(url, data={'name': 'product', 'categories': [Category.objects.all()[0].pk]})
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
response = self.client.post(url, data={'name': 'product', 'categories': [Category.objects.all()[0].pk]})
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def insert_categories(self, categories_list: list):
"""Check if a category already exist in db before insertion."""
for category in categories_list:
try:
Category.objects.get(name=category)
except Category.DoesNotExist:
add = Category(name=category)
add.save()
def create_cat(cat_name, count, report, create_report=0):
""" create a category and save it"""
new_cat = Category(name=cat_name)
if create_report == 1:
print(cat_name)
report += f"{cat_name}\n"
new_cat.save()
return new_cat, report
class UnitTest(TestCase):
def setUp(self):
self.user, self.profile = create_user_and_profile("test", "test")
self.logged_in = self.client.login(username="******", password="******")
self.category = Category(name="jus de fruits")
self.category.save()
self.product1 = Product(
name="Pur jus d'orange sans pulpe",
image_url=
"https://static.openfoodfacts.org/images/products/350/211/000/9449/front_fr.80.400.jpg",
url=
"https://world.openfoodfacts.org/product/3502110009449/pur-jus-d-orange-sans-pulpe-tropicana",
nutriscore=3,
packaging="carton")
self.product1.save()
def test_get_user_and_profile(self):
""" tests wethers function returns user and profile """
#create a mock request
my_mock = mock.Mock()
my_mock.user = self.user
u, p = get_user_and_profile(my_mock)
self.assertTrue(u, self.user)
self.assertTrue(p, self.profile)
my_mock = None
def test_add_new_user_False(self):
""" checks wether function returns False when user ever exists"""
new_user = add_new_user("test", "test")
self.assertFalse(new_user[0]) #because it ever exists
def test_add_new_user_True(self):
""" checks whether function returns True when user does not ever exists"""
new_user2 = add_new_user("test2", "test")
self.assertTrue(new_user2[0])
# def test_mav_notify_db(self):
# """ tests whether function notifies the db that a code in a confirm mail has been send """
# #params user, profile, code, mail
# self.mav.notify_db(self.user, self.profile, code=123, mail="mail")
# self.assertEqual(self.profile.code, 123)
# self.assertEqual(self.user.email, "mail")
# self.assertTrue(self.profile.mail_confirm_sent)
def test_notify_db_fv(self):
""" tests wether function adds a given prod to the profile fav list"""
before = self.profile.favlist.all()
notify_db_fv(self.profile, self.product1.name)
after = self.profile.favlist.all()
self.assertTrue(len(before), len(after) - 1)
def test_logout(self):
""" Tests whether the function logouts the user """
response = self.client.get(reverse("user:logout"), follow=True)
self.assertTrue(self.user.is_authenticated)
self.assertFalse(response.wsgi_request.user.is_authenticated)
self.assertRedirects(response, reverse("research:index"))
def test_delete_from_quantity(self):
"""
Creates a test Product.
First sends a post request with the id of the product
to add the product to the bag to add_to_bag view.
Then sends another post request to delete_from_quantity to
remove one item of that product from the bag.
Ensures that the Product quantity is decreased in the bag by
displaying the correct message returned by the request.
Ensures correct redirect status code is returned.
"""
test_category = Category(name="Kettlebells")
test_category.save()
test_discount = Discount(name="30%", rate=0.7)
test_discount.save()
test_manufacurer = Manufacturer(name="Great Sports")
test_manufacurer.save()
test_product = Product(name="Test", description="A description",
image_url="", category=test_category,
manufacturer=test_manufacurer,
prize=100,
discount_rate=test_discount,
units_sold=1)
test_product.save()
self.client.post("/bag/add_to_bag/1/", {"quantity-input": 1})
test_response = self.client.post("/bag/delete_from_quantity/1/")
self.client.post("/bag/add_to_bag/1/", {"quantity-input": 1})
test_response_message = self.client.post(
"/bag/delete_from_quantity/1/", follow=True)
intended_message = "Item was deleted"
message = ""
get_stored_messages = test_response_message.context["messages"]
for message in get_stored_messages:
message = message
self.assertEquals(test_response.status_code, 302)
self.assertEquals(intended_message, message.__str__())
# Add a quantity of two
# Ensure the bag content contains one item after deleting one
self.client.post("/bag/add_to_bag/1/", {"quantity-input": 2})
test_response_message = self.client.post(
"/bag/delete_from_quantity/1/", follow=True)
intended_bag_content = {'1': {'quantity': 1}, 'total_cost': 70}
shopping_bag = self.client.session["shopping_bag"]
self.assertEquals(test_response.status_code, 302)
self.assertEquals(intended_bag_content, shopping_bag)
def test_view_cart(self):
categories = Category(name="Category Name")
categories.save()
product_detail = Product(name="Test product",
price=999.00,
description="Test product description",
image="test.jpg")
product_detail.save()
response = self.client.get("/cart/")
self.assertEqual(response.status_code, 200)
def test_view_bag(self):
categories = Category(name="Category Name")
categories.save()
service_detail = Service(name="Test service",
price=999.00,
description="Test service description",
image="test.jpg")
service_detail.save()
response = self.client.get("/bag/")
self.assertEqual(response.status_code, 200)
def fill_categories(self, cat):
for index, value in enumerate(cat):
try:
categories = Category(name=value)
if Category.objects.filter(name=value).exists():
continue
else:
categories.save()
except KeyError as e:
print(e)
def test_can_create_a_category(self):
category = Category(
title='category',
slug='catslug',
featured=False,
image='test.png',
)
category.save()
self.assertEqual(category.title, 'category')
self.assertEqual(category.slug, 'catslug')
self.assertEqual(category.featured, False)
self.assertEqual(category.image, 'test.png')
def setUp(self):
self.cli = Client()
cat = Category()
cat.name = 'conserves'
cat.save()
alim = Product()
alim.id = '3245412718649'
alim.name = 'abricots'
alim.save()
alim.categories.add(cat)
alim.save()
self.food = alim
def setUp(self):
"""datas call before each funtion"""
#a cat
self.category = Category(name="jus de fruits")
self.category.save()
#a product
self.product1 = Product(
name="Pur jus d'orange sans pulpe",
image_url="https://static.openfoodfacts.org/images/products/350/211/000/9449/front_fr.80.400.jpg",
url="https://world.openfoodfacts.org/product/3502110009449/pur-jus-d-orange-sans-pulpe-tropicana",
nutriscore=3, packaging="carton")
self.product1.save()
def product_search(request):
all_categories = Category.update_sub_category_lists()
categories = Category.find_main_categories(all_categories)
found_products = search_for_something(request)
all_product_images = []
for product in found_products:
img = list(ProductImage.find_all_product_images(product.id))
all_product_images += img
return render(request, 'products/home.html', {'main_categories':categories, 'all_categories':all_categories, 'products':found_products, 'product_images':all_product_images })
def setUp(self):
"""Sets up a category model and a stockdrop model."""
category = Category(name='clothing')
category.save()
valid_stockdrop = StockDrop(name='SD1',
description='description',
image=SimpleUploadedFile(
name='default.png',
content=open('media/default.png',
'rb').read(),
content_type='image/jpeg',
))
valid_stockdrop.save()
def setUp(self):
self.user, self.profile = create_user_and_profile("test", "test")
self.logged_in = self.client.login(username="******", password="******")
self.category = Category(name="jus de fruits")
self.category.save()
self.product1 = Product(
name="Pur jus d'orange sans pulpe",
image_url=
"https://static.openfoodfacts.org/images/products/350/211/000/9449/front_fr.80.400.jpg",
url=
"https://world.openfoodfacts.org/product/3502110009449/pur-jus-d-orange-sans-pulpe-tropicana",
nutriscore=3,
packaging="carton")
self.product1.save()
def setUp(self):
user = {
"username": "******",
"password1": "P4sswordVeryS€cure@",
"password2": "P4sswordVeryS€cure@",
"email": "*****@*****.**"
}
self.client.post('/consumer/signup', data=user)
self.token = self.client.post('/user/login',
data={
'email': '*****@*****.**',
'password': '******'
}).json()['key']
self.client.credentials(HTTP_AUTHORIZATION='Token ' + self.token)
cat = Category(name='category')
cat.save()
cat = Category(name='category2')
cat.save()
p = Product(name='product')
p.save()
p.categories.set([Category.objects.all()[0]])
p.save()
store = Store(name='store', pnt=GEOSGeometry('POINT(0 0)', srid=SRID))
store.save()
def setUp(self):
self.browser = webdriver.Firefox()
category = Category(category_name='petits déjeuners')
category.save()
nutella = Product(
product_name='nutella',
nutriscore='e',
image_url='https://static.openfoodfacts.org/images/products/301/762/042/2003/front_fr.139.400.jpg',
product_url='https://fr.openfoodfacts.org/produit/3017620422003/nutella-ferrero',
category=category
)
nutella.save()
nutella_ersatz_1 = Product(
product_name='ersatz_nutella_1',
nutriscore='c',
category=category
)
nutella_ersatz_1.save()
nutella_ersatz_2 = Product(
product_name='ersatz_nutella_2',
nutriscore='b',
category=category
)
nutella_ersatz_2.save()
nutella_ersatz_3 = Product(
product_name='ersatz_nutella_3',
nutriscore='b',
category=category
)
nutella_ersatz_3.save()
nutella_ersatz_4 = Product(
product_name='ersatz_nutella_4',
nutriscore='a',
category=category
)
nutella_ersatz_4.save()
nutella_ersatz_5 = Product(
product_name='ersatz_nutella_5',
nutriscore='d',
category=category
)
nutella_ersatz_5.save()
nutella_ersatz_6 = Product(
product_name='ersatz_nutella_6',
nutriscore='c',
category=category
)
nutella_ersatz_6.save()
def create_new_category(category,subcategory):
new_category=Category()
new_category.category_name=category
if (len(subcategory)==3):
new_category.subcategory_1=subcategory[0]
new_category.subcategory_2=subcategory[1]
new_category.subcategory_3=subcategory[2]
elif(len(subcategory)==2):
new_category.subcategory_1=subcategory[0]
new_category.subcategory_2=subcategory[1]
elif(len(subcategory)==1):
new_category.subcategory_1=subcategory[0]
else:
print "no sub category"
new_category.save()
return new_category
def setUp(self):
# création de l'admin
admin = User.objects.create_user('admin', password='******')
admin.is_staff = True
admin.save()
# création d'une catégorie
c1 = Category(name="Catégorie test 1")
c1.save()
# création des produits
f1 = File(open("media/tests/icon-test1.png", "rb"))
with f1:
p1 = Product(name="Prod1", description="desc", image=f1, category=c1)
p1.save()
f2 = File(open("media/tests/icon-test2.png", "rb"))
with f2:
p2 = Product(name="Prod2", description="desc", image=f2, category=c1)
p2.save()
def addProductPOSTHandler(post_request):
product_exists = True
try:
brand = Brand.objects.get(name=post_request['brand'])
except Brand.DoesNotExist:
brand = Brand(name=post_request['brand'])
brand.save()
product_exists = False
try:
category = Category.objects.get(name=post_request['category'])
except Category.DoesNotExist:
category = Category(name=post_request['category'])
category.save()
product_exists = False
try:
ptype = ProductType.objects.get(name=post_request['ptype'])
except ProductType.DoesNotExist:
ptype = ProductType(name=post_request['ptype'], category=category)
ptype.save()
product_exists = False
try:
product = Product.objects.get(name=post_request['name'])
product = []
except Product.DoesNotExist:
if not product_exists:
product = Product(
name=post_request['name'],
price=post_request['price'],
instock=post_request['units'],
brand=brand,
ptype=ptype)
product.save()
return product
def setUp(self):
# création de l'admin
admin = User.objects.create_user('admin', password='******')
admin.is_staff = True
admin.save()
# création des catégories
c1 = Category(name="Catégorie test 1")
c1.save()
c2 = Category(name="Catégorie test 2")
c2.save()
def parse(self, response) :
sel = Selector (response)
data = []
data1 = []
items=[]
item = MaterialtreeItem ()
self.driver.get(response.url)
time.sleep(2.5)
item['url'] = response.request.url
#import pdb;pdb.set_trace()
item['title'] = sel.xpath('//div/h1/text()').extract()
um1 = um2 = ""
try:
um1 = str(list(map(unicode.strip,sel.xpath('//span[@id="sellerPrice"]/text()').extract())))
um1 = um1.split('/')[1]
um1 = um1.replace("u'","").replace("Rs.","").replace("[","").replace("]","").replace("'","").replace('"','')
except: pass
try:
um2 = self.driver.find_element_by_xpath('//*[@id="ddContent"]/table/tbody//tr/td[6]')
um2 = um2.text
um2 = str(um2).replace("u'","")
print "<<<<",um2
except: pass
item['um'] = um2 or um1
Sku1 = Sku2 = ""
try:
Sku1 = sel.xpath('//div[@class="sku"]/span[@class="value"]/text()').extract()
except:pass
try:
Sku2 = sel.xpath('//div/h1/text()').extract()
except: pass
item['Sku'] = Sku1 or Sku2
try:
item['Product_id'] = sel.xpath('//input[@name="productid"]/@value').extract()
except: pass
price11 = price22 = ""
try:
price22 = self.driver.find_element_by_xpath('//*[@id="ddContent"]/table/tbody/tr[2]/td[7]/h5')
price22 = price22.text
price22 = str(price22).replace(".00","").replace(",","").replace("Rs","").replace("u'","")
print ">>>",price22
except: pass
try:
price11 = str(list(map(unicode.strip,sel.xpath('//span[@id="sellerPrice"]/text()').extract())))
price11 = price11.split('/')[0]
price11 = price11.replace("u'","").replace("Rs.","").replace("[","").replace("]","")
except: pass
item['price'] = price22 or price11
try:
seller = str(list(map(unicode.strip,sel.xpath('//span[@id="sold_by_name"]/a/text()').extract())))
seller = seller.replace("u'","").replace("[","").replace("]","")
except: pass
item['Saller'] = seller
#import pdb;pdb.set_trace()
category = item['url'].split('/')[-1]
category = str(category).split('-')[7:10]
category = str(category).replace("'","").replace(",","").replace("[","").replace("]","")
item['category'] = category
desc = desc_new = ""
try:
desc = str(list(map(unicode.strip,sel.xpath('//div[@class="panel"]/div[@class="std"]/text()').extract())))
desc = desc.replace("u'","").replace("[","").replace("]","")
except:pass
try:
desc_new = self.driver.find_element_by_xpath('//div[@id="category-description"]/p')
desc_new = desc_new.text
desc_new = str(desc_new).replace("u'","")
print"......",desc_new
except: pass
item['desc'] = desc_new or desc
try:
desc1 = str(list(map(unicode.strip,sel.xpath('//table[@class="data-table"]/tbody//text()').extract())))
desc1 = desc1.replace("u'","").replace("[","").replace("]","").replace("', ',","")
except: pass
item['desc1'] = desc1
brand1 = brand2 = ""
try:
brand1 = self.driver.find_element_by_xpath('//*[@id="ddContent"]/table/tbody//tr/td[3]')
brand1 = brand1.text
brand1 = str(brand1).replace("u'","")
except: pass
try:
brand2 = sel.xpath('//div[@class="brand-name attribute"]/text()').extract()
brand2 = str(brand2).replace("u'","").replace("[","").replace("]","")
except: pass
item['brand'] = brand1 or brand2
for x2 in self.driver.find_elements_by_xpath("//*[@id='ddContent']/table/tbody//tr//h4"):
price = x2.text
item['price'] = price.encode('ascii', 'ignore').replace(",","").replace("Rs","")
self.spamwriter.writerow(item['price'])
print "----", self.count, item['price']
self.count += 1
data1.append(item['price'])
row = self.driver.find_elements_by_xpath("//*[@id='ddContent']/table/tbody//tr")
for x1 in row:
variant = x1.text
Variant = str(variant).split("/")[:2]
item['Variant'] = Variant
#print item['Variant']
self.spamwriter.writerow(item['Variant'])
print "----", self.count, item['Variant']
self.count += 1
data.append(item['Variant'])
print">>>>>>>>", len(data)
# if len(data)<7:
# continue
# else:break
item1 = BaseProducts()
item1.brand_name = item['brand']
item1.title = item['title'][0]
try:
item1.Product_id = item['Product_id'][0]
except:
item1.Product_id = ""
item1.Sku = item['Sku'][0]
#item1.category_name=
item1.category_name = item['category']
item1.description=item['desc']
item1.additional_information = item['desc1']
item1.source_url = item['url']
try:
item1.unit_measurements = item['um']
except: item1.unit_measurements = ""
item1.source_id = 3
item1.save()
item2 = Subscribe_Product()
item3 = SPM()
item2.bp = item1
item2.source_id =3
#item2.region_name = item['region_name']
item2.Sku = item['Sku'][0]
# item2.Variant =
item2.save()
item3.sp = item2
item3.Sku = item['Sku'][0]
#item3.region_name = item['region_name']
try:
item3.store_price = float(item['price1'])
except: item3.store_price = 0
#item3.Delivery_time = item['Delivery_time']
try:
item3.price = float(item['price'])
except : item3.price = 0
item3.source_id = 3
item3.saller = item['Saller']
item3.save()
item4 = Category()
item4.source_id = 3
item4.category_path = 'kitchen'+item['category']
item4.category_name = item['category']
item4.level = 3
item4.save()
for y1 in range(len(data)):
print"datadata..........", data[y1]
item2 = Subscribe_Product()
item3 = SPM()
item2.bp = item1
item2.source_id =3
item2.Sku = item['Sku'][0]
item2.Variant = data[y1]
item2.save()
item3.sp = item2
item3.Sku = item['Sku'][0]
try:
item3.price = data1[y1]
except : item3.price = 0
item3.source_id = 3
item3.saller = item['Saller']
item3.save()
print item
return item
def categories_processing(file, user):
tax = parse_taxonomy(file)
Category.delete_all()
return Category.store_from_taxonomy(tax, parent=None, created_by=user, updated_by=user)
def parse(self, response) :
print ">>>>>", response.request.url
sel = Selector (response)
items=[]
data = []
data1 = []
item = SnapdealItem()
self.driver.get(response.url)
item['url'] = response.request.url
#import pdb;pdb.set_trace()
item['title'] = sel.xpath('//h1[@itemprop="name"]/text()').extract()
item['Product_id'] = str(item['url']).split('/')[-1].replace("#bcrumbLabelId:892","")
brand = sel.xpath('//i[@itemprop="name"]/text()').extract()
item['brand'] = brand
price = str(list(map(unicode.strip,sel.xpath('//p[@class="product-offer-price"]/text()').extract())))
price = price.replace("Rs.","").replace("[","").replace("]","").replace("u'","").replace(",","").replace("'","")
item['price'] = price
desc1 = str(list(map(unicode.strip,sel.xpath('//ul[@class="dtls-list clear"]//li//text()').extract())))
desc1 = desc1.replace("[","").replace("]","").replace("u'","")
item['desc1'] = desc1
desc = str(list(map(unicode.strip,sel.xpath('//div[@itemprop="description"]//text()').extract())))
desc = desc.replace("[","").replace("]","").replace("u'","")
item['desc'] = desc
try:
saller = str(list(map(unicode.strip,sel.xpath('//a[@class="pdp-e-seller-info-name reset-margin"]/text()').extract())))
saller = saller.replace("u'","").replace("[","").replace("]","")
except: pass
item['Saller'] = saller
category_path = sel.xpath('//div//span[@itemprop="title"]/text()').extract()
level = len(category_path)
item['level'] = level
category_path = str(category_path).replace("u'","").replace("Home',","").replace("[","").replace("]","").replace('"','').replace("'","")
item['category_path'] = category_path
for o1 in self.driver.find_elements_by_xpath('//*[@id="attribute-select-0"]/ul//li'):
variant1 = o1.text
print"var1",variant1
if o1:o1.click()
time.sleep(5)
for x2 in self.driver.find_elements_by_xpath('//span//ul/li[@class="rippleGrey btn btn-toggle pdpAttr attrActive"]'):
variant2 = x2.text
print"var2",variant2
if x2:x2.click()
time.sleep(5)
for x1 in self.driver.find_elements_by_xpath('//*[@id="buyPriceBox"]/div[3]/div[2]/span[2]/span'):
price = x1.text
item['price'] = price.encode('ascii', 'ignore').replace(",","")
#len(item['price'])
Variant = variant1 , variant2
item['Variant'] = str(Variant)
self.spamwriter.writerow(item["price"])
print "----", self.count, item["price"],item['Variant']
self.count += 1
data.append(item["price"])
data1.append(item['Variant'])
print len(data)
item1 = BaseProducts()
item1.brand_name = item['brand'][0]
item1.title = item['title'][0]
item1.Product_id = item['Product_id']
item1.Sku = item['title'][0]
item1.category_name= "Building Material"
item1.description=item['desc']
item1.additional_information = item['desc1']
item1.source_url = item['url']
try:
item1.unit_measurements = item['um'][0]
except: item1.unit_measurements = ""
item1.source_id = 2
item1.save()
item2 = Subscribe_Product()
item3 = SPM()
item2.bp = item1
item2.source_id =2
item2.Sku = item['title'][0]
item2.save()
item3.sp = item2
item3.Sku = item['title'][0]
try:
item3.price = float(item['price'])
except : item3.price = 0
item3.source_id = 2
item3.saller = item['Saller']
item3.save()
item4 = Category()
item4.category_name = "Building Material"
item4.category_path = item['category_path']
item4.level = item['level']
item4.source_id = 2
item4.save()
for y1 in range(len(data)):
print"datadata", data[y1]
print"datadata111",data1[y1]
item2 = Subscribe_Product()
item3 = SPM()
item2.bp = item1
item2.source_id =2
item2.Sku = item['title'][0]
item2.Variant = data1[y1]
item2.save()
item3.sp = item2
item3.Sku = item['title'][0]
try:
item3.price = data[y1]
except : item3.price = 0
item3.source_id = 2
item3.saller = item['Saller']
item3.save()
print item
return item
