def handle(self, *args, **options):
with open('electronic_products_data.csv') as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
next(csv_reader)
for row in csv_reader:
price = row[1]
brand = row[12]
image_urls = row[17]
name = row[21]
category = row[22]
stock = random.randint(5, 50)
product = Product(name=name, slug='{}-{}'.format(slugify(name), time.time()))
product.price = price
product.brand = brand
product.image_urls = image_urls
category = ProductCategory(name=category, slug='{}-{}'.format(slugify(name), time.time()))
category.save()
product.category_id = category.id
product.stock = stock
product.available = True
product.save()
self.stdout.write(self.style.SUCCESS('Successfully finished adding data'))
def parse_single_product_page(self, html, cat_id):
try:
soup = BeautifulSoup(html, 'html.parser')
except:
return
try:
article = soup.find('span', class_='product-sku-num').text.strip()
except:
article = random.randint(1000000, 9999999)
try:
title = soup.find('h1', class_='product-header').text.strip()
except:
title = 'Unknown product title'
try:
price = float(
soup.find('div', id='price-less-10').find(
'div',
class_='price-cur').find('span',
class_='num').text.replace(
',',
'.').strip().replace(' ', ''))
except:
price = 0
try:
old_price_container = soup.find('div', id='price-less-10').find(
'div', class_='price-old')
except:
old_price_container = None
discount = 0
if old_price_container:
old_price = float(
old_price_container.text.strip().split()[0].replace(
',', '.').replace(' ', ''))
percent_value = old_price - price
discount = float((percent_value / old_price) * 100)
brand = None
try:
characteristics_container = soup.find(
'div', id='product-1-inner').find(
'div', class_='collapse-list-content').find(
'div', class_='product-spec-wrap').find_all(
'div', class_='product-spec__item')
except:
characteristics_container = []
characteristics = list()
for c_item in characteristics_container:
holder = c_item.find('dl', class_='product-spec')
key = holder.find('dt', class_='product-spec__name').text.strip()
value = holder.find('dd',
class_='product-spec__value').text.strip()
if key.lower() != 'бренд':
characteristics.append(dict(key=key.lower(), value=value))
else:
brand = value
try:
description = soup.find('div', id='product-2-inner').find(
'div', class_='collapse-list-content').find('div').find(
'noindex').text
except:
description = ""
documentation_container = soup.find('div', id='product-3')
documentation = None
if documentation_container:
tab_title = documentation_container.find(
'a', class_='tab-link-collaps', recursive=False).text.strip()
if tab_title == 'документация':
documentation = documentation_container.find(
'div', class_='collapse-list',
recursive=False).find('div',
class_='collapse-list-content',
recursive=False).find(
'div', recursive=False).text
images = ()
try:
img_container = soup.find(
'div',
class_='product-summary-slider').find_all('div',
class_='slider-item')
for s in img_container:
img = self.download_image(s.find('img').get('src'))
if img:
images += img,
except:
pass
product = dict(article=article,
title=title,
price=price,
discount=discount,
brand=brand,
characteristics=characteristics,
description=description,
documentation=documentation,
images=images)
print(product)
files = list()
if len(product['images']) > 0:
for img in product['images']:
files.append(('files', open(img, 'rb')))
kwargs = dict(
data={
'title': product['title'],
'price': product['price'],
'discount': product['discount'],
'brand': product['brand'],
'characteristics': json.dumps(product['characteristics']),
'description': product['description'],
'documentation': product['documentation'],
'category_id': cat_id,
'article': product['article']
})
brand = None
if product['brand']:
try:
brand = Brand.objects.get(title=product['brand'])
except ObjectDoesNotExist:
brand = Brand.objects.create(title=product['brand'],
slug=slugify(product['brand']))
try:
cat = Category.objects.get(id=cat_id)
except ObjectDoesNotExist:
print('Category not found....')
return
_product = Product()
_product.title = product['title']
_product.price = product['price']
_product.discount = product['discount']
_product.brand = brand
_product.category = cat
_product.description = product['description']
_product.documentation = product['documentation']
_product.discount = product['discount']
_product.main_image = files[0]
_product.save()
for c in product['characteristics']:
if '?' in c['key']:
_key = c['key'].split('?')[0]
else:
_key = c['key']
try:
prop_key = AllProductProperties.objects.get(
title=_key.strip().lower())
except ObjectDoesNotExist:
prop_key = AllProductProperties.objects.create(
title=_key.strip().lower())
if slugify(_key.strip()) == slugify('вес брутто'):
value_holder = c['value'].strip().split()
ProductProperty.objects.create(
product=product,
key=prop_key,
value=float(value_holder[0].replace(',', '.')),
value_type=value_holder[1])
else:
if '?' in c['key']:
ProductProperty.objects.create(
product=product,
key=prop_key,
value=c['value'],
prop_tip=c['key'].strip().split('?')[1])
else:
ProductProperty.objects.create(product=product,
key=prop_key,
value=c['value'])
for img in files:
ProductImage.objects.create(product=product, image=img)
if os.path.exists('logs/process.log'):
open_role = 'a'
else:
open_role = 'w'
# with open('logs/process.log', open_role) as f:
# f.write(
# """
# =================================================================================================
# Product uploaded, server message: %s; \n
# Slug: %s; \n
# Category_id: %s; \n
# images: [%s]; \n
# uploading date: %s;
# =================================================================================================
# """ % (
# ,
# slugify(product['title']),
# cat_id,
# ', '.join(product['images']),
# str(datetime.datetime.now())
# )
# )
# f.close()
for f in files:
f[1].close()
for i in product['images']:
os.remove(i)
