def crawl_category(category_name, page=1):
url = get_category_url(category_name, page)
html = urlopen(url).read()
parsed_html = BeautifulSoup(html, 'html.parser')
products = parsed_html.body.find_all('div', attrs={'class': 'c-product-box'})
db = db_helper()
db.getDB()['category-' + category_name].create_index([('farsi_title', TEXT), ('en_title', TEXT)])#{'farsi_title': "text", 'en_title': 'text'})
db.getDB()['category-' + category_name].create_index([('price',ASCENDING)])#{'farsi_title': "text", 'en_title': 'text'})
db.getDB()['category-' + category_name].create_index([('product_id',ASCENDING)])#, unique = True)#{'farsi_title': "text", 'en_title': 'text'})
for product in products:
# if db.getDB()[category_name].find({'product_id': product.get("data-id")}).count() > 0:
# if db.getDB()['products'].find({'product_id': product.get("data-id")}).count() == 0:
# db.insert_one('products',
# {'category': 'category-' + category_name, 'product_id': product.get("data-id")})
# continue
product_url = "https://www.digikala.com" + product.find("div", attrs={'class': 'c-product-box__title'}).find(
"a").get("href")
data = {}
data['product_url'] = product_url
data['product_id'] = product.get("data-id")
data['farsi_title'] = product.get("data-title-fa")
data['en_title'] = product.get("data-title-en")
data['price'] = price_fa2en(product.get("data-price"))
data['image_url'] = product.find("img").get("src")
data['comments'] = crawl_comments(data['product_id'])
data['questions'] = crawl_questions(data['product_id'])
# print(str(data))
db.insert_one('category-' + category_name, data)
db.insert_one('products', {'category': 'category-' + category_name, 'product_id': data['product_id']})
def product(request, product_id):
helper = db_helper()
db = helper.getDB()
categoryName = db.products.find_one({'product_id': product_id})
product = db[categoryName['category']].find_one({'product_id': product_id})
return render(request, 'product.html', {
'product': product,
'categories': helper.getCategories()
})
def product_del(request, product_id):
helper = db_helper()
db = helper.getDB()
categoryName = db.products.find_one({'product_id': product_id})
product = db[categoryName['category']].find_one({'product_id': product_id})
if request.method == "POST":
db[categoryName['category']].remove({'product_id': product_id})
db.products.remove({'product_id': product_id})
return redirect('/category/' + categoryName['category'][9:])
return render(request, 'product_del.html', {'product': product})
def add_comment(request, product_id):
helper = db_helper()
db = helper.getDB()
categoryName = db.products.find_one({'product_id': product_id})
if request.method == "POST":
comment = {'id': int(time()), 'content': request.POST.get('comment')}
comment['by'] = "توسط " + request.POST.get('by', 'ناشناس')
db[categoryName['category']].update_one(
{'product_id': product_id}, {'$addToSet': {
'comments': comment
}})
# db.products.remove({'product_id': product_id})
return redirect('/product/' + product_id)
def remove_comment(request, product_id, comment_id):
helper = db_helper()
db = helper.getDB()
categoryName = db.products.find_one({'product_id': product_id})
db[categoryName['category']].update_one(
{'product_id': product_id},
{'$pull': {
'comments': {
'id': int(comment_id)
}
}})
# db.products.remove({'product_id': product_id})
return redirect('/product/' + product_id)
def product_edit(request, product_id):
helper = db_helper()
db = helper.getDB()
categoryName = db.products.find_one({'product_id': product_id})
product = db[categoryName['category']].find_one({'product_id': product_id})
if request.method == "POST":
db[categoryName['category']].update_one({'product_id': product_id}, {
'$set': {
'farsi_title': request.POST['farsi_title'],
'en_title': request.POST['en_title'],
'price': int(request.POST['price'])
}
})
return redirect('/product/' + product_id)
return render(request, 'product_edit.html', {'product': product})
def category(request, category_name):
filters = {}
keyword = request.GET.get("q", None)
if keyword:
reg = re.compile(".*" + keyword + ".*", re.IGNORECASE)
# filters['$or'] = [{'farsi_title':{'$regex': "/.*"+keyword+".*/i"}}, {'en_title': {'$regex': "/.*"+keyword+".*/i"}}]
filters['$or'] = [{
'farsi_title': {
'$regex': reg
}
}, {
'en_title': {
'$regex': reg
}
}]
# filters['$text'] = {'$search': keyword}
# filters['farsi_title'] = {'$regex': ".*"+keyword+".*"}
from_price = request.GET.get("from-price", None)
if from_price:
filters['price'] = {'$gte': int(from_price)}
to_price = request.GET.get("to-price", None)
if to_price:
filters['price']['$lte'] = int(to_price)
helper = db_helper()
db = helper.getDB()
products = db['category-' + category_name].find(filters)
maxPrice = db['category-' + category_name].find_one(sort=[('price', -1)])
return render(
request, 'category.html', {
'products': products,
'categories': helper.getCategories(),
'maxPrice': maxPrice['price']
})