def delete_product(product): """删除产品及产品相关信息""" product_id = product.get('_id') MongodbUtil.delete('shopping', 'product', product) logger.info('Delete a product: %s successfully!!!' % product.get('title')) #删除索引中product.id keyword_list = product.get('keywordList') for keyword in keyword_list: keyword = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword}) if not keyword: continue if keyword['invertedIndex'].__contains__(product_id.__str__()): del keyword['invertedIndex'][product_id.__str__()] MongodbUtil.save('shopping', 'keywordIndex', keyword) logger.info('Update keyword index: %s successfully!!!' % keyword.get('keyword')) #删除merchant中product.id merchant_id = product.get('merchantId') merchant = MongodbUtil.find_one('shopping', 'merchant', {'_id': merchant_id}) index_id = merchant['productIdList'].index(product_id) if isinstance(index_id, int): del merchant['productIdList'][index_id] MongodbUtil.save('shopping', 'merchant', merchant) logger.info('Update merchant productIdList successfully!!!') #删除product的image及图片文件 images = MongodbUtil.find('shopping', 'image', {'productId': product_id}) for image in images: MongodbUtil.remove(filename=image.get('fileName')) MongodbUtil.delete('shopping', 'image', {'productId': product_id})
def do_import_product(product, tag_dict): """导入商品相关信息""" try: merchant_name = product.get(tag_dict['merchant']) merchant_list = get_merchant_list() if merchant_name not in merchant_list: return merchant = MongodbUtil.find_one('shopping', 'merchant', {'name': merchant_name}) keyword_set = get_keyword_set(product, tag_dict) p = get_product_dict(product, keyword_set, merchant.get('_id'), tag_dict) # 插入或更新商品 product['_id'], update_flag = MongodbUtil.update_or_insert('shopping', 'product', p, {"productId": p.get('productId'), 'merchantId': p.get('merchantId')}) if product['_id']: if update_flag: logger.info('Update product: %s successfully!!!' % p.get('title')) else: logger.info('Save product: %s successfully!!!' % p.get('title')) # 遍历keywordList更新倒排索引 save_keyword_index(keyword_set, product) # 下载产品图片 for type, size in settings.image_size_list.items(): do_import_image(product, tag_dict, size) else: logger.error('Save product: %s failed!!!' % p.get('title')) except Exception as e: logger.error('[message: %s]; [params: %s]' % (e.message, product.__str__()))
def save_keyword_index(keyword_set, product): """遍历keywordList更新倒排索引""" try: for keyword in keyword_set: existingKeyword = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword}) if existingKeyword: if not existingKeyword['invertedIndex'].__contains__(product.get('_id').__str__()): existingKeyword['invertedIndex'][product.get('_id').__str__()] = 100.0 status_id = MongodbUtil.save('shopping', 'keywordIndex', existingKeyword) if status_id: logger.info('Update keywordIndex: %s successfully!!!' % keyword) else: save_keyword_index(keyword_set, product) logger.info('Try to update keywordIndex: %s again!!!' % keyword) else: keywordIndex = { 'keyword': keyword, 'searchTimes': 0, 'invertedIndex': {product.get('_id').__str__(): 100.0}, } status_id = MongodbUtil.insert('shopping', 'keywordIndex', keywordIndex) if status_id: logger.info('Save keywordIndex: %s successfully!!!' % keyword) else: save_keyword_index(keyword_set, product) logger.info('Try to update keywordIndex: %s again!!!' % keyword) except Exception as e: logger.error(e.message)
def get_product_dict(product, keyword_list, merchant_id, tag_dict): """拼接产品参数""" start_time = DateUtil.get_sys_date() category_mode = MongodbUtil.find_one('shopping', 'category') category_list, category, name, description = category_mode.get('category_dict'), product.get(tag_dict['category']), product.get(tag_dict['name']), product.get(tag_dict['description']) # 商品匹配定类 category_result = fit_category(category_list, category, name, description) # 填充类别关键词集合 fill_category_dict(category_list, category_result, category, name, description) product_dict = { 'productId': product.get(tag_dict['id']), 'category': category_result, 'keywordList': keyword_list, 'merchantId': merchant_id, 'title': str(product.get(tag_dict['name'])).replace('&', ' '), 'startTime': start_time, 'aliveTime': settings.alive_time, 'description': str(product.get(tag_dict['description'])).replace('&', ''), 'currency': product.get(tag_dict['currency']), 'price': float(product.get(tag_dict['price'])), 'url': product.get(tag_dict['url']), 'mpn': product.get(tag_dict.get('mpn', ''), ''), 'color': product.get(tag_dict.get('color', ''), ''), 'size': product.get(tag_dict.get('size', ''), ''), 'merchantCategory': product.get(tag_dict.get('merchantCategory', ''), ''), 'availability': product.get(tag_dict.get('availability', ''), ''), 'shippingWeight': product.get(tag_dict.get('shippingWeight', ''), ''), 'gender': product.get(tag_dict.get('gender', ''), ''), 'ageGroup': product.get(tag_dict.get('ageGroup', ''), ''), } return product_dict
def get_keyword_set(product, tag_dict): """获取关键字无重复集合""" keyword_set = list() #待分词productFeed字段 name = product.get(tag_dict['name']).lower() if product.get(tag_dict['name']) else '' category = product.get(tag_dict['category']).lower() if product.get(tag_dict['category']) else '' description = product.get(tag_dict['description']).lower() if product.get(tag_dict['description']).lower() else '' #分词模板 glossary = MongodbUtil.find_one('shopping', 'glossary') use_word_list = glossary.get('used') un_used_word_list = glossary.get('unUsed') for word in use_word_list: word = word.lower() if name.__contains__(word) or description.__contains__(word) or category.__contains__(word): keyword_set.append(word) wait_4_cut_word_list = [category, name, description] for ele in wait_4_cut_word_list.__iter__(): if not ele: continue for word in StringUtil.cut_word(ele): word = word.lower() if not un_used_word_list.__contains__(word) and not keyword_set.__contains__(word): keyword_set.append(word) return keyword_set
def __filter_product(self, product): """参数处理""" del_key_list = ['keywordList', '_id', 'merchantId', 'sellTotalCount'] merchant = MongodbUtil.find_one('shopping', 'merchant', {'_id': product.get('merchantId')}) product['merchant'] = merchant.get('name') if self._webmaster: product['url'] = '%s/cpc?s=%s&p_i=%s&a_p_i=' % ( settings.host_src, self._webmaster, product['_id'].__str__()) image_arr = MongodbUtil.find('shopping', 'image', {'productId': product['_id']}) if image_arr and len(image_arr) == 3: product['smallImageUrl'] = '%s/image?filename=%s' % ( settings.host_src, image_arr[0].get('fileName')) product['middleImageUrl'] = '%s/image?filename=%s' % ( settings.host_src, image_arr[1].get('fileName')) product['bigImageUrl'] = '%s/image?filename=%s' % ( settings.host_src, image_arr[2].get('fileName')) else: product['smallImageUrl'] = settings.default_product_pic product['middleImageUrl'] = settings.default_product_pic product['bigImageUrl'] = settings.default_product_pic else: del del_key_list[del_key_list.index('keywordList')] del product['url'] for k, v in product.items(): if k in del_key_list: del product[k] return product
def do_update_priority(product): """更新产品优先级 优先级=1/关键词数 + 商品价格*点击数 + 1/keywordList.index""" try: keyword_list = product.get('keywordList') product_price = float(product.get('price')) #获取该商品总的点击数 cpc_list = MongodbUtil.find('shopping', 'cpc', {'cpcProductId': product.get('_id')}) cpc_count = 1 cpc_active_count = 1 for cpc in cpc_list: cpc_count += int(cpc.get('cpcCount')) cpc_active_count += int(cpc.get('cpcActiveCount')) cpc_average_count = float(cpc_count + cpc_active_count) / 2.0 # 更新每个产品的keywordList的每个keyword中产品的优先级 index = 1 for keyword in keyword_list: priority = 1.0 / float(len(keyword_list)) + MathUtil.parse2percent(product_price * cpc_average_count) + 1.0 / float(index) index += 1 keyword_index = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword}) keyword_index['invertedIndex'][product.get('_id').__str__()] = priority obj_id = MongodbUtil.save('shopping', 'keywordIndex', keyword_index) if obj_id: logger.info('Update %s\'s priority successfully!!!' % keyword) else: logger.info('Update %s\'s priority failed!!!' % keyword) except Exception as e: logger.error(e.message)
def _cpsuc_effect(self, order_list): for order in order_list: # 获取校验参数 if type(order) is dict and order.has_key( 'merchant') and order.has_key('orderId') and order.has_key( 'productList') and order.has_key('merchant'): exist_order = MongodbUtil.find_one('api', 'cps', spec_or_id={ 'orderId': order.get('orderId'), 'merchant': order.get('merchant') }) if exist_order: exist_order['status'] = 'successful' exist_order['successfulTime'] = DateUtil.get_sys_time() MongodbUtil.update( 'api', 'cps', spec_or_id={'_id': exist_order.get('_id')}, document=exist_order) continue else: continue order = self._fit_params(order) # 插入cps cps_id = MongodbUtil.insert('api', 'cps', order) if cps_id: self._event('cps_insert:%s' % cps_id)
def __get_page_items(self): """排序过滤product, 获取产品列表""" page_items = list() begin_index = (self._page_index - 1) * self._page_size end_index = self._page_index * self._page_size product_id_list = self._product_id_list[begin_index:end_index] for p_id in product_id_list: product = MongodbUtil.find_one('shopping', 'product', str(p_id).encode('utf-8')) product = self.__filter_product(product) page_items.append(product) return page_items
def do_import_image(product, tag_dict, size): """导入图片相关信息""" try: exist_product = MongodbUtil.find_one('shopping', 'product', {'productId': product.get(tag_dict['id'])}) if exist_product: #下载保存图片信息 product['_id'] = exist_product.get('_id') save_product_img(product, tag_dict, size) else: logger.info('Save product\'s image: %s failed!!!' % product.get('title')) except Exception as e: logger.error('[message: %s]; [params: %s]' % (e.message, product.__str__()))
def import_category(): """导入基础词汇表""" try: exists_category = MongodbUtil.find_one('shopping', 'category') if exists_category: message = 'The category is exists!!!' logger.info(message) return id = MongodbUtil.insert('shopping', 'category', category_dict) if id: message = 'Import category(id:%s) successfully!!!' % id logger.info(message) except Exception as e: logger.error(e.message)
def get_keyword_set(keyword): """获取关键字无重复集合""" keyword_set = list() #分词模板 glossary = MongodbUtil.find_one('shopping', 'glossary') use_word_list = glossary.get('used') un_used_word_list = glossary.get('unUsed') for word in use_word_list: if keyword.__contains__(word): keyword_set.append(word) for word in StringUtil.cut_word(keyword): if not un_used_word_list.__contains__( word) and not keyword_set.__contains__(word): keyword_set.append(word) return keyword_set
def cpc_un_effect(self, cpc_id): """浏览器异常关闭有效点击数减一""" self._event('cpc_un_effect') cpc_document = MongodbUtil.find_one('api', 'cpc', spec_or_id={'_id': cpc_id}) if cpc_document: cpc_document['takeEffect'] = 0 result_id = MongodbUtil.update('api', 'cpc', spec_or_id={'_id': cpc_id}, document=cpc_document) if not result_id: self.cpc_un_effect(cpc_id) logger.info('Update cpc successful for : %s' % cpc_id)
def replenish_glossary(file_path, tag_name, obj_id=None): """根据文件补充词汇表""" try: glossary = MongodbUtil.find_one('shopping', 'glossary', obj_id) category_list = XMLUtil.read_2_list(file_path, tag_name) category_set = XMLUtil.get_obj_set(category_list) for category in category_set: clean_word_list = StringUtil.clean_list( category.get(tag_name).split(':')) for word in clean_word_list.__iter__(): word = word.lower() if not glossary.get('used').__contains__(word): glossary.get('used').append(word) id = MongodbUtil.save('shopping', 'glossary', glossary) if id: message = 'Replenish glossary(id:%s) successfully!!!' % id logger.info(message) except Exception as e: logger.error(e.message)
def import_product_id_2_merchant(merchant_name): """导入广告主下所有产品的id集合""" try: merchant = MongodbUtil.find_one('shopping', 'merchant', {'name': merchant_name}) product_list = MongodbUtil.find('shopping', 'product', {'merchantId': merchant.get('_id')}) product_id_list = list() for product in product_list: product_id_list.append(product.get('_id')) merchant['productIdList'] = product_id_list merchant_id = MongodbUtil.save('shopping', 'merchant', merchant) if merchant_id: logger.info('Import product id list to %s successfully!!!' % merchant_name) else: logger.info('Import product id list to %s failed!!!' % merchant_name) except Exception as e: logger.error(e.message)
def _cps_effect(self, merchant, order_list): """广告主订单成功,在联盟插入cps数据,确认cps生效""" cookie_value = '' for order in order_list: # 获取校验参数 if type(order) is dict and order.has_key( 'merchant') and order.has_key('orderId') and order.has_key( 'productList') and order.has_key('merchant') and ( merchant == order.get('merchant')): exist_order = MongodbUtil.find_one('api', 'cps', spec_or_id={ 'orderId': order.get('orderId'), 'merchant': order.get('merchant') }) if exist_order: exist_order['status'] = 'successful' exist_order['successfulTime'] = DateUtil.get_sys_time() MongodbUtil.update( 'api', 'cps', spec_or_id={'_id': exist_order.get('_id')}, document=exist_order) continue else: continue order = self._fit_params(order) # 插入cps cps_id = MongodbUtil.insert('api', 'cps', order) if cps_id: self._event('cps_insert:%s' % cps_id) cookie_value += '_%s' % cps_id.__str__() if cookie_value: self.set_cookie('c_i_l', cookie_value, expires_days=settings.cookie_alive_time)
def _update_cps_status(self): """更新订单状态为成功""" self._event('update_status') cps_id_str = self.get_cookie('c_i_l') cps_id_list = cps_id_str.split('_') if cps_id_str else list() for cps_id in cps_id_list: if cps_id: cps_document = MongodbUtil.find_one( 'api', 'cps', spec_or_id={'_id': ObjectId(cps_id)}) if cps_document: cps_document['status'] = 'successful' cps_document['successfulTime'] = DateUtil.get_sys_time() MongodbUtil.update( 'api', 'cps', spec_or_id={'_id': cps_document.get('_id')}, document=cps_document) self.set_cookie('c_i_l', '', expires_days=settings.cookie_alive_time) result_info = { 'status': '200', 'message': 'successful', } self.write(json_encode(result_info))
def get_product_id_list(keyword): """从mongodb中获取产品id结果集""" keyword = keyword.lower() keyword_set = get_keyword_set(keyword) p_id_list_arr = list() union_dict = {} #按照分词结果查询产品结果集 for i in xrange(len(keyword_set)): keyword_index = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword_set[i]}) p_id_dict = keyword_index.get('invertedIndex') if p_id_dict: union_dict = dict(union_dict.items() + p_id_dict.items()) p_id_list = sorted(p_id_dict, key=p_id_dict.get, reverse=True) p_id_list_arr.append(p_id_list) #结果集排序:先取交集,再取并集(取差集),分别按照优先级排序,最后合并 intersection = None union_set = None for i in xrange(len(p_id_list_arr)): if not intersection: intersection = p_id_list_arr[i] union_set = p_id_list_arr[i] continue intersection = list( set(intersection).intersection(set(p_id_list_arr[i]))) union_set = list(set(union_set).union(set(p_id_list_arr[i]))) union_set = list(set(union_set).difference(set(intersection))) #排序 intersection = OrderUtil.order_obj_dict(union_dict, intersection) union_set = OrderUtil.order_obj_dict(union_dict, union_set) product_id_list = list() for i in xrange(len(intersection)): product_id_list.append(intersection[i]) for i in xrange(len(union_set)): product_id_list.append(union_set[i]) return product_id_list