def crawl_category(category_name, page=1): url = get_category_url(category_name, page) html = urlopen(url).read() parsed_html = BeautifulSoup(html, 'html.parser') products = parsed_html.body.find_all('div', attrs={'class': 'c-product-box'}) db = db_helper() db.getDB()['category-' + category_name].create_index([('farsi_title', TEXT), ('en_title', TEXT)])#{'farsi_title': "text", 'en_title': 'text'}) db.getDB()['category-' + category_name].create_index([('price',ASCENDING)])#{'farsi_title': "text", 'en_title': 'text'}) db.getDB()['category-' + category_name].create_index([('product_id',ASCENDING)])#, unique = True)#{'farsi_title': "text", 'en_title': 'text'}) for product in products: # if db.getDB()[category_name].find({'product_id': product.get("data-id")}).count() > 0: # if db.getDB()['products'].find({'product_id': product.get("data-id")}).count() == 0: # db.insert_one('products', # {'category': 'category-' + category_name, 'product_id': product.get("data-id")}) # continue product_url = "https://www.digikala.com" + product.find("div", attrs={'class': 'c-product-box__title'}).find( "a").get("href") data = {} data['product_url'] = product_url data['product_id'] = product.get("data-id") data['farsi_title'] = product.get("data-title-fa") data['en_title'] = product.get("data-title-en") data['price'] = price_fa2en(product.get("data-price")) data['image_url'] = product.find("img").get("src") data['comments'] = crawl_comments(data['product_id']) data['questions'] = crawl_questions(data['product_id']) # print(str(data)) db.insert_one('category-' + category_name, data) db.insert_one('products', {'category': 'category-' + category_name, 'product_id': data['product_id']})
def product(request, product_id): helper = db_helper() db = helper.getDB() categoryName = db.products.find_one({'product_id': product_id}) product = db[categoryName['category']].find_one({'product_id': product_id}) return render(request, 'product.html', { 'product': product, 'categories': helper.getCategories() })
def product_del(request, product_id): helper = db_helper() db = helper.getDB() categoryName = db.products.find_one({'product_id': product_id}) product = db[categoryName['category']].find_one({'product_id': product_id}) if request.method == "POST": db[categoryName['category']].remove({'product_id': product_id}) db.products.remove({'product_id': product_id}) return redirect('/category/' + categoryName['category'][9:]) return render(request, 'product_del.html', {'product': product})
def add_comment(request, product_id): helper = db_helper() db = helper.getDB() categoryName = db.products.find_one({'product_id': product_id}) if request.method == "POST": comment = {'id': int(time()), 'content': request.POST.get('comment')} comment['by'] = "توسط " + request.POST.get('by', 'ناشناس') db[categoryName['category']].update_one( {'product_id': product_id}, {'$addToSet': { 'comments': comment }}) # db.products.remove({'product_id': product_id}) return redirect('/product/' + product_id)
def remove_comment(request, product_id, comment_id): helper = db_helper() db = helper.getDB() categoryName = db.products.find_one({'product_id': product_id}) db[categoryName['category']].update_one( {'product_id': product_id}, {'$pull': { 'comments': { 'id': int(comment_id) } }}) # db.products.remove({'product_id': product_id}) return redirect('/product/' + product_id)
def product_edit(request, product_id): helper = db_helper() db = helper.getDB() categoryName = db.products.find_one({'product_id': product_id}) product = db[categoryName['category']].find_one({'product_id': product_id}) if request.method == "POST": db[categoryName['category']].update_one({'product_id': product_id}, { '$set': { 'farsi_title': request.POST['farsi_title'], 'en_title': request.POST['en_title'], 'price': int(request.POST['price']) } }) return redirect('/product/' + product_id) return render(request, 'product_edit.html', {'product': product})
def category(request, category_name): filters = {} keyword = request.GET.get("q", None) if keyword: reg = re.compile(".*" + keyword + ".*", re.IGNORECASE) # filters['$or'] = [{'farsi_title':{'$regex': "/.*"+keyword+".*/i"}}, {'en_title': {'$regex': "/.*"+keyword+".*/i"}}] filters['$or'] = [{ 'farsi_title': { '$regex': reg } }, { 'en_title': { '$regex': reg } }] # filters['$text'] = {'$search': keyword} # filters['farsi_title'] = {'$regex': ".*"+keyword+".*"} from_price = request.GET.get("from-price", None) if from_price: filters['price'] = {'$gte': int(from_price)} to_price = request.GET.get("to-price", None) if to_price: filters['price']['$lte'] = int(to_price) helper = db_helper() db = helper.getDB() products = db['category-' + category_name].find(filters) maxPrice = db['category-' + category_name].find_one(sort=[('price', -1)]) return render( request, 'category.html', { 'products': products, 'categories': helper.getCategories(), 'maxPrice': maxPrice['price'] })