def delete_product(product):
"""删除产品及产品相关信息"""
product_id = product.get('_id')
MongodbUtil.delete('shopping', 'product', product)
logger.info('Delete a product: %s successfully!!!' % product.get('title'))
#删除索引中product.id
keyword_list = product.get('keywordList')
for keyword in keyword_list:
keyword = MongodbUtil.find_one('shopping', 'keywordIndex',
{'keyword': keyword})
if not keyword:
continue
if keyword['invertedIndex'].__contains__(product_id.__str__()):
del keyword['invertedIndex'][product_id.__str__()]
MongodbUtil.save('shopping', 'keywordIndex', keyword)
logger.info('Update keyword index: %s successfully!!!' %
keyword.get('keyword'))
#删除merchant中product.id
merchant_id = product.get('merchantId')
merchant = MongodbUtil.find_one('shopping', 'merchant',
{'_id': merchant_id})
index_id = merchant['productIdList'].index(product_id)
if isinstance(index_id, int):
del merchant['productIdList'][index_id]
MongodbUtil.save('shopping', 'merchant', merchant)
logger.info('Update merchant productIdList successfully!!!')
#删除product的image及图片文件
images = MongodbUtil.find('shopping', 'image', {'productId': product_id})
for image in images:
MongodbUtil.remove(filename=image.get('fileName'))
MongodbUtil.delete('shopping', 'image', {'productId': product_id})
def do_import_product(product, tag_dict):
"""导入商品相关信息"""
try:
merchant_name = product.get(tag_dict['merchant'])
merchant_list = get_merchant_list()
if merchant_name not in merchant_list:
return
merchant = MongodbUtil.find_one('shopping', 'merchant', {'name': merchant_name})
keyword_set = get_keyword_set(product, tag_dict)
p = get_product_dict(product, keyword_set, merchant.get('_id'), tag_dict)
# 插入或更新商品
product['_id'], update_flag = MongodbUtil.update_or_insert('shopping', 'product', p, {"productId": p.get('productId'), 'merchantId': p.get('merchantId')})
if product['_id']:
if update_flag:
logger.info('Update product: %s successfully!!!' % p.get('title'))
else:
logger.info('Save product: %s successfully!!!' % p.get('title'))
# 遍历keywordList更新倒排索引
save_keyword_index(keyword_set, product)
# 下载产品图片
for type, size in settings.image_size_list.items():
do_import_image(product, tag_dict, size)
else:
logger.error('Save product: %s failed!!!' % p.get('title'))
except Exception as e:
logger.error('[message: %s]; [params: %s]' % (e.message, product.__str__()))
def save_keyword_index(keyword_set, product):
"""遍历keywordList更新倒排索引"""
try:
for keyword in keyword_set:
existingKeyword = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword})
if existingKeyword:
if not existingKeyword['invertedIndex'].__contains__(product.get('_id').__str__()):
existingKeyword['invertedIndex'][product.get('_id').__str__()] = 100.0
status_id = MongodbUtil.save('shopping', 'keywordIndex', existingKeyword)
if status_id:
logger.info('Update keywordIndex: %s successfully!!!' % keyword)
else:
save_keyword_index(keyword_set, product)
logger.info('Try to update keywordIndex: %s again!!!' % keyword)
else:
keywordIndex = {
'keyword': keyword,
'searchTimes': 0,
'invertedIndex': {product.get('_id').__str__(): 100.0},
}
status_id = MongodbUtil.insert('shopping', 'keywordIndex', keywordIndex)
if status_id:
logger.info('Save keywordIndex: %s successfully!!!' % keyword)
else:
save_keyword_index(keyword_set, product)
logger.info('Try to update keywordIndex: %s again!!!' % keyword)
except Exception as e:
logger.error(e.message)
def get_product_dict(product, keyword_list, merchant_id, tag_dict):
"""拼接产品参数"""
start_time = DateUtil.get_sys_date()
category_mode = MongodbUtil.find_one('shopping', 'category')
category_list, category, name, description = category_mode.get('category_dict'), product.get(tag_dict['category']), product.get(tag_dict['name']), product.get(tag_dict['description'])
# 商品匹配定类
category_result = fit_category(category_list, category, name, description)
# 填充类别关键词集合
fill_category_dict(category_list, category_result, category, name, description)
product_dict = {
'productId': product.get(tag_dict['id']),
'category': category_result,
'keywordList': keyword_list,
'merchantId': merchant_id,
'title': str(product.get(tag_dict['name'])).replace('&', ' '),
'startTime': start_time,
'aliveTime': settings.alive_time,
'description': str(product.get(tag_dict['description'])).replace('&', ''),
'currency': product.get(tag_dict['currency']),
'price': float(product.get(tag_dict['price'])),
'url': product.get(tag_dict['url']),
'mpn': product.get(tag_dict.get('mpn', ''), ''),
'color': product.get(tag_dict.get('color', ''), ''),
'size': product.get(tag_dict.get('size', ''), ''),
'merchantCategory': product.get(tag_dict.get('merchantCategory', ''), ''),
'availability': product.get(tag_dict.get('availability', ''), ''),
'shippingWeight': product.get(tag_dict.get('shippingWeight', ''), ''),
'gender': product.get(tag_dict.get('gender', ''), ''),
'ageGroup': product.get(tag_dict.get('ageGroup', ''), ''),
}
return product_dict
def get_keyword_set(product, tag_dict):
"""获取关键字无重复集合"""
keyword_set = list()
#待分词productFeed字段
name = product.get(tag_dict['name']).lower() if product.get(tag_dict['name']) else ''
category = product.get(tag_dict['category']).lower() if product.get(tag_dict['category']) else ''
description = product.get(tag_dict['description']).lower() if product.get(tag_dict['description']).lower() else ''
#分词模板
glossary = MongodbUtil.find_one('shopping', 'glossary')
use_word_list = glossary.get('used')
un_used_word_list = glossary.get('unUsed')
for word in use_word_list:
word = word.lower()
if name.__contains__(word) or description.__contains__(word) or category.__contains__(word):
keyword_set.append(word)
wait_4_cut_word_list = [category, name, description]
for ele in wait_4_cut_word_list.__iter__():
if not ele:
continue
for word in StringUtil.cut_word(ele):
word = word.lower()
if not un_used_word_list.__contains__(word) and not keyword_set.__contains__(word):
keyword_set.append(word)
return keyword_set
def __filter_product(self, product):
"""参数处理"""
del_key_list = ['keywordList', '_id', 'merchantId', 'sellTotalCount']
merchant = MongodbUtil.find_one('shopping', 'merchant',
{'_id': product.get('merchantId')})
product['merchant'] = merchant.get('name')
if self._webmaster:
product['url'] = '%s/cpc?s=%s&p_i=%s&a_p_i=' % (
settings.host_src, self._webmaster, product['_id'].__str__())
image_arr = MongodbUtil.find('shopping', 'image',
{'productId': product['_id']})
if image_arr and len(image_arr) == 3:
product['smallImageUrl'] = '%s/image?filename=%s' % (
settings.host_src, image_arr[0].get('fileName'))
product['middleImageUrl'] = '%s/image?filename=%s' % (
settings.host_src, image_arr[1].get('fileName'))
product['bigImageUrl'] = '%s/image?filename=%s' % (
settings.host_src, image_arr[2].get('fileName'))
else:
product['smallImageUrl'] = settings.default_product_pic
product['middleImageUrl'] = settings.default_product_pic
product['bigImageUrl'] = settings.default_product_pic
else:
del del_key_list[del_key_list.index('keywordList')]
del product['url']
for k, v in product.items():
if k in del_key_list:
del product[k]
return product
def do_update_priority(product):
"""更新产品优先级 优先级=1/关键词数 + 商品价格*点击数 + 1/keywordList.index"""
try:
keyword_list = product.get('keywordList')
product_price = float(product.get('price'))
#获取该商品总的点击数
cpc_list = MongodbUtil.find('shopping', 'cpc', {'cpcProductId': product.get('_id')})
cpc_count = 1
cpc_active_count = 1
for cpc in cpc_list:
cpc_count += int(cpc.get('cpcCount'))
cpc_active_count += int(cpc.get('cpcActiveCount'))
cpc_average_count = float(cpc_count + cpc_active_count) / 2.0
# 更新每个产品的keywordList的每个keyword中产品的优先级
index = 1
for keyword in keyword_list:
priority = 1.0 / float(len(keyword_list)) + MathUtil.parse2percent(product_price * cpc_average_count) + 1.0 / float(index)
index += 1
keyword_index = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword})
keyword_index['invertedIndex'][product.get('_id').__str__()] = priority
obj_id = MongodbUtil.save('shopping', 'keywordIndex', keyword_index)
if obj_id:
logger.info('Update %s\'s priority successfully!!!' % keyword)
else:
logger.info('Update %s\'s priority failed!!!' % keyword)
except Exception as e:
logger.error(e.message)
def _cpsuc_effect(self, order_list):
for order in order_list:
# 获取校验参数
if type(order) is dict and order.has_key(
'merchant') and order.has_key('orderId') and order.has_key(
'productList') and order.has_key('merchant'):
exist_order = MongodbUtil.find_one('api',
'cps',
spec_or_id={
'orderId':
order.get('orderId'),
'merchant':
order.get('merchant')
})
if exist_order:
exist_order['status'] = 'successful'
exist_order['successfulTime'] = DateUtil.get_sys_time()
MongodbUtil.update(
'api',
'cps',
spec_or_id={'_id': exist_order.get('_id')},
document=exist_order)
continue
else:
continue
order = self._fit_params(order)
# 插入cps
cps_id = MongodbUtil.insert('api', 'cps', order)
if cps_id:
self._event('cps_insert:%s' % cps_id)
def __get_page_items(self):
"""排序过滤product, 获取产品列表"""
page_items = list()
begin_index = (self._page_index - 1) * self._page_size
end_index = self._page_index * self._page_size
product_id_list = self._product_id_list[begin_index:end_index]
for p_id in product_id_list:
product = MongodbUtil.find_one('shopping', 'product',
str(p_id).encode('utf-8'))
product = self.__filter_product(product)
page_items.append(product)
return page_items
def do_import_image(product, tag_dict, size):
"""导入图片相关信息"""
try:
exist_product = MongodbUtil.find_one('shopping', 'product', {'productId': product.get(tag_dict['id'])})
if exist_product:
#下载保存图片信息
product['_id'] = exist_product.get('_id')
save_product_img(product, tag_dict, size)
else:
logger.info('Save product\'s image: %s failed!!!' % product.get('title'))
except Exception as e:
logger.error('[message: %s]; [params: %s]' % (e.message, product.__str__()))
def import_category():
"""导入基础词汇表"""
try:
exists_category = MongodbUtil.find_one('shopping', 'category')
if exists_category:
message = 'The category is exists!!!'
logger.info(message)
return
id = MongodbUtil.insert('shopping', 'category', category_dict)
if id:
message = 'Import category(id:%s) successfully!!!' % id
logger.info(message)
except Exception as e:
logger.error(e.message)
def get_keyword_set(keyword):
"""获取关键字无重复集合"""
keyword_set = list()
#分词模板
glossary = MongodbUtil.find_one('shopping', 'glossary')
use_word_list = glossary.get('used')
un_used_word_list = glossary.get('unUsed')
for word in use_word_list:
if keyword.__contains__(word):
keyword_set.append(word)
for word in StringUtil.cut_word(keyword):
if not un_used_word_list.__contains__(
word) and not keyword_set.__contains__(word):
keyword_set.append(word)
return keyword_set
def cpc_un_effect(self, cpc_id):
"""浏览器异常关闭有效点击数减一"""
self._event('cpc_un_effect')
cpc_document = MongodbUtil.find_one('api',
'cpc',
spec_or_id={'_id': cpc_id})
if cpc_document:
cpc_document['takeEffect'] = 0
result_id = MongodbUtil.update('api',
'cpc',
spec_or_id={'_id': cpc_id},
document=cpc_document)
if not result_id:
self.cpc_un_effect(cpc_id)
logger.info('Update cpc successful for : %s' % cpc_id)
def replenish_glossary(file_path, tag_name, obj_id=None):
"""根据文件补充词汇表"""
try:
glossary = MongodbUtil.find_one('shopping', 'glossary', obj_id)
category_list = XMLUtil.read_2_list(file_path, tag_name)
category_set = XMLUtil.get_obj_set(category_list)
for category in category_set:
clean_word_list = StringUtil.clean_list(
category.get(tag_name).split(':'))
for word in clean_word_list.__iter__():
word = word.lower()
if not glossary.get('used').__contains__(word):
glossary.get('used').append(word)
id = MongodbUtil.save('shopping', 'glossary', glossary)
if id:
message = 'Replenish glossary(id:%s) successfully!!!' % id
logger.info(message)
except Exception as e:
logger.error(e.message)
def import_product_id_2_merchant(merchant_name):
"""导入广告主下所有产品的id集合"""
try:
merchant = MongodbUtil.find_one('shopping', 'merchant',
{'name': merchant_name})
product_list = MongodbUtil.find('shopping', 'product',
{'merchantId': merchant.get('_id')})
product_id_list = list()
for product in product_list:
product_id_list.append(product.get('_id'))
merchant['productIdList'] = product_id_list
merchant_id = MongodbUtil.save('shopping', 'merchant', merchant)
if merchant_id:
logger.info('Import product id list to %s successfully!!!' %
merchant_name)
else:
logger.info('Import product id list to %s failed!!!' %
merchant_name)
except Exception as e:
logger.error(e.message)
def _cps_effect(self, merchant, order_list):
"""广告主订单成功，在联盟插入cps数据，确认cps生效"""
cookie_value = ''
for order in order_list:
# 获取校验参数
if type(order) is dict and order.has_key(
'merchant') and order.has_key('orderId') and order.has_key(
'productList') and order.has_key('merchant') and (
merchant == order.get('merchant')):
exist_order = MongodbUtil.find_one('api',
'cps',
spec_or_id={
'orderId':
order.get('orderId'),
'merchant':
order.get('merchant')
})
if exist_order:
exist_order['status'] = 'successful'
exist_order['successfulTime'] = DateUtil.get_sys_time()
MongodbUtil.update(
'api',
'cps',
spec_or_id={'_id': exist_order.get('_id')},
document=exist_order)
continue
else:
continue
order = self._fit_params(order)
# 插入cps
cps_id = MongodbUtil.insert('api', 'cps', order)
if cps_id:
self._event('cps_insert:%s' % cps_id)
cookie_value += '_%s' % cps_id.__str__()
if cookie_value:
self.set_cookie('c_i_l',
cookie_value,
expires_days=settings.cookie_alive_time)
def _update_cps_status(self):
"""更新订单状态为成功"""
self._event('update_status')
cps_id_str = self.get_cookie('c_i_l')
cps_id_list = cps_id_str.split('_') if cps_id_str else list()
for cps_id in cps_id_list:
if cps_id:
cps_document = MongodbUtil.find_one(
'api', 'cps', spec_or_id={'_id': ObjectId(cps_id)})
if cps_document:
cps_document['status'] = 'successful'
cps_document['successfulTime'] = DateUtil.get_sys_time()
MongodbUtil.update(
'api',
'cps',
spec_or_id={'_id': cps_document.get('_id')},
document=cps_document)
self.set_cookie('c_i_l', '', expires_days=settings.cookie_alive_time)
result_info = {
'status': '200',
'message': 'successful',
}
self.write(json_encode(result_info))
def get_product_id_list(keyword):
"""从mongodb中获取产品id结果集"""
keyword = keyword.lower()
keyword_set = get_keyword_set(keyword)
p_id_list_arr = list()
union_dict = {}
#按照分词结果查询产品结果集
for i in xrange(len(keyword_set)):
keyword_index = MongodbUtil.find_one('shopping', 'keywordIndex',
{'keyword': keyword_set[i]})
p_id_dict = keyword_index.get('invertedIndex')
if p_id_dict:
union_dict = dict(union_dict.items() + p_id_dict.items())
p_id_list = sorted(p_id_dict, key=p_id_dict.get, reverse=True)
p_id_list_arr.append(p_id_list)
#结果集排序：先取交集，再取并集(取差集)，分别按照优先级排序，最后合并
intersection = None
union_set = None
for i in xrange(len(p_id_list_arr)):
if not intersection:
intersection = p_id_list_arr[i]
union_set = p_id_list_arr[i]
continue
intersection = list(
set(intersection).intersection(set(p_id_list_arr[i])))
union_set = list(set(union_set).union(set(p_id_list_arr[i])))
union_set = list(set(union_set).difference(set(intersection)))
#排序
intersection = OrderUtil.order_obj_dict(union_dict, intersection)
union_set = OrderUtil.order_obj_dict(union_dict, union_set)
product_id_list = list()
for i in xrange(len(intersection)):
product_id_list.append(intersection[i])
for i in xrange(len(union_set)):
product_id_list.append(union_set[i])
return product_id_list