def parse(self, item, html):
page = pq(html)
title = page('title').html()
print("*** TITLE " + title)
try:
price = page('.good-price strong').html()
except:
price = 0.00
print("*** PRICE " + str(price))
if not item.image:
img = page('.fotorama__img:first').attr('src')
new_image = make_upload_path()
print(make_save_name(new_image))
if img:
img_temp = NamedTemporaryFile(delete=True)
r = requests.get(img)
img_temp.write(r.content)
img_temp.flush()
item.image.save(new_image.split('/')[-1],
File(img_temp),
save=True)
if price:
item.price = Decimal(price)
for i in page('.dl-horizontal div'):
try:
name = i('dt').html()
except:
name = ""
try:
value = i('dd').html()
except:
value = ""
name = name.strip()
value = value.strip()
print("*** NAME PROP " + name)
print("*** VALUE PROP " + value)
if name and value:
cp = TypeProperty.objects.filter(category=category,
name=name).first()
if not cp:
cp = TypeProperty(category=category, name=name)
cp.save()
p = ProductProperty.objects.filter(category_property=cp,
value=value,
product=item)
if not p:
p = ProductProperty(category_property=cp,
value=value,
product=item)
p.save()
def save(self, *args, **kwargs):
if self.categ:
super(Product, self).save(*args, **kwargs)
# we create properties if not exist
for cp in CategoryProperty.objects.filter(category=self.categ):
pp = ProductProperty.objects.filter(category_property=cp,
product=self)
if not pp:
pp = ProductProperty(category_property=cp,
product=self,
value="--")
pp.save()
# we create filters if not exist
for fc in FilterCategory.objects.filter(category=self.categ):
pf = ProductFilter.objects.filter(filter_category=fc,
product=self)
if not pf:
pf = ProductFilter(filter_category=fc, product=self)
pf.save()
def save(self, *args, **kwargs):
self.update_palette_by_image()
if self.category:
super(Product, self).save(*args, **kwargs)
# create properties if not exist
for cp in TypeProperty.objects.filter(category=self.category):
pp = ProductProperty.objects.filter(category_property=cp,
product=self)
if not pp:
pp = ProductProperty(category_property=cp,
product=self,
value="--")
pp.save()
# create filters if not exist
for fc in FilterCategory.objects.filter(category=self.category):
pf = ProductFilter.objects.filter(filter_category=fc,
product=self)
if not pf:
pf = ProductFilter(filter_category=fc, product=self)
pf.save()
def start_requests(self):
print("&&&&&&&&&&&&&&&&& START &&&&&&&&&&&&&&&&&&&&&&&")
for item in Product.objects.all():
print("PRODUCT " + str(item.name))
category = item.category
if item.url:
driver = webdriver.PhantomJS(
executable_path=settings.PHANTOMJS)
driver.set_window_size(1320, 940)
driver.get(item.url)
time.sleep(5)
title = driver.find_element_by_xpath('//title').text
print("*** TITLE " + title)
try:
price = driver.find_element_by_xpath(
'//div[@class="good-price"]/strong').text
except:
price = 0.00
print("*** PRICE " + str(price))
if not item.image:
img = driver.find_elements_by_xpath(
"//*[@class='fotorama__img']")[0].get_attribute('src')
new_image = make_upload_path()
print(make_save_name(new_image))
if img:
img_temp = NamedTemporaryFile(delete=True)
r = requests.get(img)
img_temp.write(r.content)
img_temp.flush()
item.image.save(new_image.split('/')[-1],
File(img_temp),
save=True)
#urlretrieve(img,new_image)
#item.image = make_save_name(new_image)
#item.image.path = new_image
if price:
item.price = Decimal(price)
driver.find_element_by_xpath(
"//a[@href='#good-spec-tab']").click()
time.sleep(2)
for i in driver.find_elements_by_xpath(
"//*[@class='dl-horizontal']/div"):
try:
name = i.find_element_by_xpath("dt").text
except:
name = ""
try:
value = i.find_element_by_xpath("dd").text
except:
value = ""
name = name.strip()
value = value.strip()
print("*** NAME PROP " + name)
print("*** VALUE PROP " + value)
if name and value:
cp = CategoryProperty.objects.filter(
category=category, name=name).first()
if not cp:
cp = CategoryProperty(category=category, name=name)
cp.save()
p = ProductProperty.objects.filter(
category_property=cp, value=value, product=item)
if not p:
p = ProductProperty(category_property=cp,
value=value,
product=item)
p.save()
driver.quit()