def save_keyword_index(keyword_set, product):
"""遍历keywordList更新倒排索引"""
try:
for keyword in keyword_set:
existingKeyword = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword})
if existingKeyword:
if not existingKeyword['invertedIndex'].__contains__(product.get('_id').__str__()):
existingKeyword['invertedIndex'][product.get('_id').__str__()] = 100.0
status_id = MongodbUtil.save('shopping', 'keywordIndex', existingKeyword)
if status_id:
logger.info('Update keywordIndex: %s successfully!!!' % keyword)
else:
save_keyword_index(keyword_set, product)
logger.info('Try to update keywordIndex: %s again!!!' % keyword)
else:
keywordIndex = {
'keyword': keyword,
'searchTimes': 0,
'invertedIndex': {product.get('_id').__str__(): 100.0},
}
status_id = MongodbUtil.insert('shopping', 'keywordIndex', keywordIndex)
if status_id:
logger.info('Save keywordIndex: %s successfully!!!' % keyword)
else:
save_keyword_index(keyword_set, product)
logger.info('Try to update keywordIndex: %s again!!!' % keyword)
except Exception as e:
logger.error(e.message)
def do_import_product(product, tag_dict):
"""导入商品相关信息"""
try:
merchant_name = product.get(tag_dict['merchant'])
merchant_list = get_merchant_list()
if merchant_name not in merchant_list:
return
merchant = MongodbUtil.find_one('shopping', 'merchant', {'name': merchant_name})
keyword_set = get_keyword_set(product, tag_dict)
p = get_product_dict(product, keyword_set, merchant.get('_id'), tag_dict)
# 插入或更新商品
product['_id'], update_flag = MongodbUtil.update_or_insert('shopping', 'product', p, {"productId": p.get('productId'), 'merchantId': p.get('merchantId')})
if product['_id']:
if update_flag:
logger.info('Update product: %s successfully!!!' % p.get('title'))
else:
logger.info('Save product: %s successfully!!!' % p.get('title'))
# 遍历keywordList更新倒排索引
save_keyword_index(keyword_set, product)
# 下载产品图片
for type, size in settings.image_size_list.items():
do_import_image(product, tag_dict, size)
else:
logger.error('Save product: %s failed!!!' % p.get('title'))
except Exception as e:
logger.error('[message: %s]; [params: %s]' % (e.message, product.__str__()))
def do_update_priority(product):
"""更新产品优先级 优先级=1/关键词数 + 商品价格*点击数 + 1/keywordList.index"""
try:
keyword_list = product.get('keywordList')
product_price = float(product.get('price'))
#获取该商品总的点击数
cpc_list = MongodbUtil.find('shopping', 'cpc', {'cpcProductId': product.get('_id')})
cpc_count = 1
cpc_active_count = 1
for cpc in cpc_list:
cpc_count += int(cpc.get('cpcCount'))
cpc_active_count += int(cpc.get('cpcActiveCount'))
cpc_average_count = float(cpc_count + cpc_active_count) / 2.0
# 更新每个产品的keywordList的每个keyword中产品的优先级
index = 1
for keyword in keyword_list:
priority = 1.0 / float(len(keyword_list)) + MathUtil.parse2percent(product_price * cpc_average_count) + 1.0 / float(index)
index += 1
keyword_index = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword})
keyword_index['invertedIndex'][product.get('_id').__str__()] = priority
obj_id = MongodbUtil.save('shopping', 'keywordIndex', keyword_index)
if obj_id:
logger.info('Update %s\'s priority successfully!!!' % keyword)
else:
logger.info('Update %s\'s priority failed!!!' % keyword)
except Exception as e:
logger.error(e.message)
def _cpsuc_effect(self, order_list):
for order in order_list:
# 获取校验参数
if type(order) is dict and order.has_key(
'merchant') and order.has_key('orderId') and order.has_key(
'productList') and order.has_key('merchant'):
exist_order = MongodbUtil.find_one('api',
'cps',
spec_or_id={
'orderId':
order.get('orderId'),
'merchant':
order.get('merchant')
})
if exist_order:
exist_order['status'] = 'successful'
exist_order['successfulTime'] = DateUtil.get_sys_time()
MongodbUtil.update(
'api',
'cps',
spec_or_id={'_id': exist_order.get('_id')},
document=exist_order)
continue
else:
continue
order = self._fit_params(order)
# 插入cps
cps_id = MongodbUtil.insert('api', 'cps', order)
if cps_id:
self._event('cps_insert:%s' % cps_id)
def fill_category_dict(category_list, category_result, category, name, description):
"""补充类别关键词"""
c_k = category_list[category_result].get('category_keyword')
d_k = category_list[category_result].get('desc_keyword')
n_k = category_list[category_result].get('name_keyword')
c_k_list = StringUtil.cut_word(category) if category else list()
d_k_list = StringUtil.cut_word(description) if description else list()
n_k_list = StringUtil.cut_word(name) if name else list()
for keyword in c_k_list:
if not c_k.__contains__(keyword):
c_k.append(keyword)
if d_k.__contains__(keyword):
d_k.remove(keyword)
if n_k.__contains__(keyword):
n_k.remove(keyword)
for keyword in d_k_list:
if not d_k.__contains__(keyword) and not c_k.__contains__(keyword):
d_k.append(keyword)
if n_k.__contains__(keyword):
n_k.remove(keyword)
for keyword in n_k_list:
if not n_k.__contains__(keyword) and not d_k.__contains__(keyword) and not c_k.__contains__(keyword):
n_k.append(keyword)
category_list[category_result]['category_keyword'] = c_k
category_list[category_result]['desc_keyword'] = d_k
category_list[category_result]['name_keyword'] = n_k
MongodbUtil.update('shopping', 'category', {'_id': category_list.get('_id')}, category_list)
def __filter_product(self, product):
"""参数处理"""
del_key_list = ['keywordList', '_id', 'merchantId', 'sellTotalCount']
merchant = MongodbUtil.find_one('shopping', 'merchant',
{'_id': product.get('merchantId')})
product['merchant'] = merchant.get('name')
if self._webmaster:
product['url'] = '%s/cpc?s=%s&p_i=%s&a_p_i=' % (
settings.host_src, self._webmaster, product['_id'].__str__())
image_arr = MongodbUtil.find('shopping', 'image',
{'productId': product['_id']})
if image_arr and len(image_arr) == 3:
product['smallImageUrl'] = '%s/image?filename=%s' % (
settings.host_src, image_arr[0].get('fileName'))
product['middleImageUrl'] = '%s/image?filename=%s' % (
settings.host_src, image_arr[1].get('fileName'))
product['bigImageUrl'] = '%s/image?filename=%s' % (
settings.host_src, image_arr[2].get('fileName'))
else:
product['smallImageUrl'] = settings.default_product_pic
product['middleImageUrl'] = settings.default_product_pic
product['bigImageUrl'] = settings.default_product_pic
else:
del del_key_list[del_key_list.index('keywordList')]
del product['url']
for k, v in product.items():
if k in del_key_list:
del product[k]
return product
def import_category():
"""导入基础词汇表"""
try:
exists_category = MongodbUtil.find_one('shopping', 'category')
if exists_category:
message = 'The category is exists!!!'
logger.info(message)
return
id = MongodbUtil.insert('shopping', 'category', category_dict)
if id:
message = 'Import category(id:%s) successfully!!!' % id
logger.info(message)
except Exception as e:
logger.error(e.message)
def cpc_un_effect(self, cpc_id):
"""浏览器异常关闭有效点击数减一"""
self._event('cpc_un_effect')
cpc_document = MongodbUtil.find_one('api',
'cpc',
spec_or_id={'_id': cpc_id})
if cpc_document:
cpc_document['takeEffect'] = 0
result_id = MongodbUtil.update('api',
'cpc',
spec_or_id={'_id': cpc_id},
document=cpc_document)
if not result_id:
self.cpc_un_effect(cpc_id)
logger.info('Update cpc successful for : %s' % cpc_id)
def get_keyword_set(product, tag_dict):
"""获取关键字无重复集合"""
keyword_set = list()
#待分词productFeed字段
name = product.get(tag_dict['name']).lower() if product.get(tag_dict['name']) else ''
category = product.get(tag_dict['category']).lower() if product.get(tag_dict['category']) else ''
description = product.get(tag_dict['description']).lower() if product.get(tag_dict['description']).lower() else ''
#分词模板
glossary = MongodbUtil.find_one('shopping', 'glossary')
use_word_list = glossary.get('used')
un_used_word_list = glossary.get('unUsed')
for word in use_word_list:
word = word.lower()
if name.__contains__(word) or description.__contains__(word) or category.__contains__(word):
keyword_set.append(word)
wait_4_cut_word_list = [category, name, description]
for ele in wait_4_cut_word_list.__iter__():
if not ele:
continue
for word in StringUtil.cut_word(ele):
word = word.lower()
if not un_used_word_list.__contains__(word) and not keyword_set.__contains__(word):
keyword_set.append(word)
return keyword_set
def get_product_dict(product, keyword_list, merchant_id, tag_dict):
"""拼接产品参数"""
start_time = DateUtil.get_sys_date()
category_mode = MongodbUtil.find_one('shopping', 'category')
category_list, category, name, description = category_mode.get('category_dict'), product.get(tag_dict['category']), product.get(tag_dict['name']), product.get(tag_dict['description'])
# 商品匹配定类
category_result = fit_category(category_list, category, name, description)
# 填充类别关键词集合
fill_category_dict(category_list, category_result, category, name, description)
product_dict = {
'productId': product.get(tag_dict['id']),
'category': category_result,
'keywordList': keyword_list,
'merchantId': merchant_id,
'title': str(product.get(tag_dict['name'])).replace('&', ' '),
'startTime': start_time,
'aliveTime': settings.alive_time,
'description': str(product.get(tag_dict['description'])).replace('&', ''),
'currency': product.get(tag_dict['currency']),
'price': float(product.get(tag_dict['price'])),
'url': product.get(tag_dict['url']),
'mpn': product.get(tag_dict.get('mpn', ''), ''),
'color': product.get(tag_dict.get('color', ''), ''),
'size': product.get(tag_dict.get('size', ''), ''),
'merchantCategory': product.get(tag_dict.get('merchantCategory', ''), ''),
'availability': product.get(tag_dict.get('availability', ''), ''),
'shippingWeight': product.get(tag_dict.get('shippingWeight', ''), ''),
'gender': product.get(tag_dict.get('gender', ''), ''),
'ageGroup': product.get(tag_dict.get('ageGroup', ''), ''),
}
return product_dict
def get_merchant_list():
"""
获取广告主列表
"""
merchant_name_list = list()
merchant_list = MongodbUtil.find('shopping', 'merchant')
for merchant in merchant_list:
merchant_name_list.append(merchant.get('name'))
return merchant_name_list
def replenish_glossary(file_path, tag_name, obj_id=None):
"""根据文件补充词汇表"""
try:
glossary = MongodbUtil.find_one('shopping', 'glossary', obj_id)
category_list = XMLUtil.read_2_list(file_path, tag_name)
category_set = XMLUtil.get_obj_set(category_list)
for category in category_set:
clean_word_list = StringUtil.clean_list(
category.get(tag_name).split(':'))
for word in clean_word_list.__iter__():
word = word.lower()
if not glossary.get('used').__contains__(word):
glossary.get('used').append(word)
id = MongodbUtil.save('shopping', 'glossary', glossary)
if id:
message = 'Replenish glossary(id:%s) successfully!!!' % id
logger.info(message)
except Exception as e:
logger.error(e.message)
def save_overdue_product(product_list):
"""保存过期产品到overdueProduct集合中"""
update_count = 0
insert_count = 0
for product in product_list:
product_id = product.get('_id')
overdue_product_id = MongodbUtil.insert('shopping', 'overdueProduct',
product)
if overdue_product_id:
insert_count += 1
logger.info('Insert overdue product: %s successfully!!!' %
product.get('title'))
#cpc统计数据转移至备份集合overdueCpc中后删除
cpc_list = MongodbUtil.find('shopping', 'cpc',
{'productId': product_id})
save_overdue_cpc(overdue_product_id, product, cpc_list)
MongodbUtil.delete('shopping', 'cpc', {'cpcProductId': product_id})
logger.info('Update %s overdue product, Insert %s overdue product!!!' %
(update_count, insert_count))
def import_product_id_2_merchant(merchant_name):
"""导入广告主下所有产品的id集合"""
try:
merchant = MongodbUtil.find_one('shopping', 'merchant',
{'name': merchant_name})
product_list = MongodbUtil.find('shopping', 'product',
{'merchantId': merchant.get('_id')})
product_id_list = list()
for product in product_list:
product_id_list.append(product.get('_id'))
merchant['productIdList'] = product_id_list
merchant_id = MongodbUtil.save('shopping', 'merchant', merchant)
if merchant_id:
logger.info('Import product id list to %s successfully!!!' %
merchant_name)
else:
logger.info('Import product id list to %s failed!!!' %
merchant_name)
except Exception as e:
logger.error(e.message)
def do_import_image(product, tag_dict, size):
"""导入图片相关信息"""
try:
exist_product = MongodbUtil.find_one('shopping', 'product', {'productId': product.get(tag_dict['id'])})
if exist_product:
#下载保存图片信息
product['_id'] = exist_product.get('_id')
save_product_img(product, tag_dict, size)
else:
logger.info('Save product\'s image: %s failed!!!' % product.get('title'))
except Exception as e:
logger.error('[message: %s]; [params: %s]' % (e.message, product.__str__()))
def __get_page_items(self):
"""排序过滤product, 获取产品列表"""
page_items = list()
begin_index = (self._page_index - 1) * self._page_size
end_index = self._page_index * self._page_size
product_id_list = self._product_id_list[begin_index:end_index]
for p_id in product_id_list:
product = MongodbUtil.find_one('shopping', 'product',
str(p_id).encode('utf-8'))
product = self.__filter_product(product)
page_items.append(product)
return page_items
def _cps_effect(self, merchant, order_list):
"""广告主订单成功，在联盟插入cps数据，确认cps生效"""
cookie_value = ''
for order in order_list:
# 获取校验参数
if type(order) is dict and order.has_key(
'merchant') and order.has_key('orderId') and order.has_key(
'productList') and order.has_key('merchant') and (
merchant == order.get('merchant')):
exist_order = MongodbUtil.find_one('api',
'cps',
spec_or_id={
'orderId':
order.get('orderId'),
'merchant':
order.get('merchant')
})
if exist_order:
exist_order['status'] = 'successful'
exist_order['successfulTime'] = DateUtil.get_sys_time()
MongodbUtil.update(
'api',
'cps',
spec_or_id={'_id': exist_order.get('_id')},
document=exist_order)
continue
else:
continue
order = self._fit_params(order)
# 插入cps
cps_id = MongodbUtil.insert('api', 'cps', order)
if cps_id:
self._event('cps_insert:%s' % cps_id)
cookie_value += '_%s' % cps_id.__str__()
if cookie_value:
self.set_cookie('c_i_l',
cookie_value,
expires_days=settings.cookie_alive_time)
def upload_pic(cls, file_name, file_body, width=128, height=128):
"""上传logo到mangodb"""
try:
file_name = cls.get_unique_name(file_name, width, height)
content = StringIO(file_body)
img = Image.open(content)
cover = ImageCover(img)
format_size = (int(width), int(height))
# img = cover.thumbnail(format_size)
img = cover.resize(format_size)
filename = MongodbUtil.put(file_name, img)
return filename
except UploadError as e:
e.log_message(traceback.print_exc())
def _update_cps_status(self):
"""更新订单状态为成功"""
self._event('update_status')
cps_id_str = self.get_cookie('c_i_l')
cps_id_list = cps_id_str.split('_') if cps_id_str else list()
for cps_id in cps_id_list:
if cps_id:
cps_document = MongodbUtil.find_one(
'api', 'cps', spec_or_id={'_id': ObjectId(cps_id)})
if cps_document:
cps_document['status'] = 'successful'
cps_document['successfulTime'] = DateUtil.get_sys_time()
MongodbUtil.update(
'api',
'cps',
spec_or_id={'_id': cps_document.get('_id')},
document=cps_document)
self.set_cookie('c_i_l', '', expires_days=settings.cookie_alive_time)
result_info = {
'status': '200',
'message': 'successful',
}
self.write(json_encode(result_info))
def get_keyword_set(keyword):
"""获取关键字无重复集合"""
keyword_set = list()
#分词模板
glossary = MongodbUtil.find_one('shopping', 'glossary')
use_word_list = glossary.get('used')
un_used_word_list = glossary.get('unUsed')
for word in use_word_list:
if keyword.__contains__(word):
keyword_set.append(word)
for word in StringUtil.cut_word(keyword):
if not un_used_word_list.__contains__(
word) and not keyword_set.__contains__(word):
keyword_set.append(word)
return keyword_set
def export_keyword(file_path, file_name):
"""导出关键词"""
try:
keyword_index_list = MongodbUtil.find('shopping', 'keywordIndex')
file_content = ''
index = 0
for keyword_index in keyword_index_list:
keyword = keyword_index.get('keyword')
if index == 10:
file_content = str(file_content) + ', ' + str(keyword) + '\n'
index = 0
continue
file_content = str(file_content) + ', ' + str(keyword)
index += 1
FileUtil.write_2_file(file_path, file_name, file_content, 'utf-8')
except Exception as e:
logger.error(e.message)
def _get_hot_word_list(self, keyword):
"""获取热词列表"""
hot_word_list = list()
keyword_index_list = MongodbUtil.find(
'shopping',
'keywordIndex',
spec_or_id={'keyword': {
'$regex': keyword
}},
skip=0,
limit=10,
sort=[(u'searchTimes', -1)])
for keyword_index in keyword_index_list:
keyword = keyword_index.get('keyword')
if hot_word_list.__contains__(keyword):
continue
hot_word_list.append(keyword)
return hot_word_list
def clear_product():
"""清理过期产品到备份集合中"""
try:
product_list = MongodbUtil.find('shopping', 'product')
overdue_product_list = list()
today = DateUtil.get_sys_date()
for product in product_list.__iter__():
start_date = product.get('startTime')
alive_time = product.get('aliveTime')
end_date = DateUtil.get_end_date(start_date, alive_time)
if today > end_date:
#添加到备份产品集合
overdue_product_list.append(product)
#执行删除操作
delete_product(product)
save_overdue_product(overdue_product_list)
except Exception as e:
logger.error(e.message)
def save_overdue_cpc(overdue_product_id, product, cpc_list):
"""定期备份cpc跟踪点击数到overdueCpc集合中"""
for cpc in cpc_list:
overdue_cpc = {
'cpcOverdueProductId': overdue_product_id,
'webmaster': cpc.get('webmaster'),
'merchant': cpc.get('merchant'),
'adPositionId': cpc.get('adPositionId'),
'clickTime': cpc.get('clickTime'),
'clearTime': DateUtil.get_sys_time(),
'takeEffect': cpc.get('takeEffect'),
}
overdue_cpc_id = MongodbUtil.insert('shopping', 'overdueCpc',
overdue_cpc)
if overdue_cpc_id:
logger.info(
'Insert overdue cpc from:%s to:%s successfully!!!' %
(overdue_cpc.get('webmaster'), overdue_cpc.get('merchant')))
def insert_cpc(self, user_id, language, title, webmaster, merchant,
category, url, meta_info, duration):
"""插入新增点击 takeEffect: 1(successful) 0(failed)"""
cpc_document = {
'uid': user_id,
'merchant': merchant,
'clickTime': DateUtil.get_sys_time(),
'url': url,
'takeEffect': 1,
'webmaster': webmaster,
'category': category,
'language': language,
'title': title,
'metaInfo': meta_info,
'duration': duration,
}
obj_id = MongodbUtil.insert('api', 'cpc', cpc_document)
return obj_id
def import_merchant(merchant_name):
"""导入广告主"""
try:
import_flag = False
if merchant_name:
merchant_id, update_flag = MongodbUtil.update_or_insert(
'shopping', 'merchant', {'name': merchant_name},
{'name': merchant_name})
if merchant_id:
import_flag = True
if update_flag:
logger.info('Update merchant: %s successfully!!!' %
merchant_name)
else:
logger.info('Save merchant: %s successfully!!!' %
merchant_name)
else:
logger.info('Save merchant: %s failed!!!' % merchant_name)
return import_flag
except Exception as e:
logger.error(e.message)
def save_product_img(product, tag_dict, size):
"""下载产品图片并存入mongodb中"""
file_name = product.get(tag_dict['image'])
opener = urllib2.build_opener()
response = opener.open(file_name)
file_body = response.read()
file_name = UploadUtil.upload_pic(file_name, file_body, size[0], size[1])
image = {
'productId': product.get('_id'),
'version': DateUtil.get_sys_time(),
'fileName': file_name,
'width': size[0],
'height': size[1],
}
image_id = MongodbUtil.insert('shopping', 'image', image)
if image_id:
logger.info('Save product\'s image: %s successfully!!!' % product.get(tag_dict['image']))
return image_id
else:
logger.info('Save product\'s image: %s failed!!!' % product.get(tag_dict['image']))
return None
def get_product_id_list(keyword):
"""从mongodb中获取产品id结果集"""
keyword = keyword.lower()
keyword_set = get_keyword_set(keyword)
p_id_list_arr = list()
union_dict = {}
#按照分词结果查询产品结果集
for i in xrange(len(keyword_set)):
keyword_index = MongodbUtil.find_one('shopping', 'keywordIndex',
{'keyword': keyword_set[i]})
p_id_dict = keyword_index.get('invertedIndex')
if p_id_dict:
union_dict = dict(union_dict.items() + p_id_dict.items())
p_id_list = sorted(p_id_dict, key=p_id_dict.get, reverse=True)
p_id_list_arr.append(p_id_list)
#结果集排序：先取交集，再取并集(取差集)，分别按照优先级排序，最后合并
intersection = None
union_set = None
for i in xrange(len(p_id_list_arr)):
if not intersection:
intersection = p_id_list_arr[i]
union_set = p_id_list_arr[i]
continue
intersection = list(
set(intersection).intersection(set(p_id_list_arr[i])))
union_set = list(set(union_set).union(set(p_id_list_arr[i])))
union_set = list(set(union_set).difference(set(intersection)))
#排序
intersection = OrderUtil.order_obj_dict(union_dict, intersection)
union_set = OrderUtil.order_obj_dict(union_dict, union_set)
product_id_list = list()
for i in xrange(len(intersection)):
product_id_list.append(intersection[i])
for i in xrange(len(union_set)):
product_id_list.append(union_set[i])
return product_id_list
def get_param(self, begin_date, end_date):
"""获取指定时间范围内的CPC/CPS统计数据 时间由前端来限制：默认都是在一个月内的时间范围，所以天数自动加一即可"""
days = (end_date - begin_date).days + 1
cpc_list = MongodbUtil.find(
'api',
'cpc',
spec_or_id={'clickTime': {
'$gt': begin_date,
'$lt': end_date
}},
sort=[(u'clickTime', 1)])
cps_list = MongodbUtil.find(
'api',
'cps',
spec_or_id={'orderTime': {
'$gt': begin_date,
'$lt': end_date
}},
sort=[(u'orderTime', 1)])
param = list()
for i in xrange(days):
element = list()
temp_date = '%s-%s-%s' % (begin_date.year, begin_date.month,
(begin_date.day + i))
element.append(temp_date)
cpc_count = 0
for cpc in cpc_list:
click_date = '%s-%s-%s' % (cpc.get('clickTime').year,
cpc.get('clickTime').month,
cpc.get('clickTime').day)
if temp_date == click_date and cpc.get('takeEffect') == 1:
cpc_count += 1
element.append(cpc_count)
cps_count = 0
for cps in cps_list:
cps_date = '%s-%s-%s' % (cps.get('orderTime').year,
cps.get('orderTime').month,
cps.get('orderTime').day)
if temp_date == cps_date:
cps_count += 1
element.append(cps_count)
param.append(element)
return param
# class ListHandler(BaseHandler):
# """
# 列表页
#
# """
#
# def on_request(self):
# try:
# #查询数据
# page_response = self.search_product(**self.params)
# #以json形式返回数据
# self._jsonify_response(page_response)
# except Exception as e:
# logger.error('[message: %s]; [params: %s]' % (e.message, self.params.__str__()))
# error_response = BrokenResponse()
# self._jsonify_response(error_response)
#
# def search_product(self, **kwargs):
# """
# 查询产品列表
#
# """
# #查询参数校验
# if not self._check_params(**self.params):
# self._event('check_false')
# page_response = BrokenResponse()
# return page_response
# self._event('%s_list' % kwargs.get('webmaster'))
# page_response = self._get_product_list(**kwargs)
# return page_response
#
# def _check_params(self, **kwargs):
# """参数校验"""
# check_flag = False
# if kwargs.get('webmaster') and kwargs.get('pageIndex') and kwargs.get('pageSize') and (
# kwargs.get('p_s') in ('default', 'up', 'down',)) and (
# kwargs.get('s_v') in ('default', 'up', 'down')) and category_name_map.__contains__(kwargs.get('c_k')):
# page_index = int(kwargs.get('pageIndex'))
# page_size = int(kwargs.get('pageSize'))
# if page_index > 0 and page_size > 0:
# check_flag = True
# return check_flag
#
# def _filter_price(self, **kwargs):
# """获取价格筛选参数"""
# filter_price = dict()
# if kwargs.get('l_p') and kwargs.get('u_p'):
# filter_price['description'] = str(kwargs.get('l_p')) + ' - ' + (kwargs.get('u_p'))
# filter_price['lower_price'] = kwargs.get('l_p')
# filter_price['upper_price'] = kwargs.get('u_p')
# elif not kwargs.get('l_p') and kwargs.get('u_p'):
# filter_price['description'] = 'Less then ' + (kwargs.get('u_p'))
# filter_price['lower_price'] = ''
# filter_price['upper_price'] = kwargs.get('u_p')
# elif kwargs.get('l_p') and not kwargs.get('u_p'):
# filter_price['description'] = 'More then ' + str(kwargs.get('l_p'))
# filter_price['lower_price'] = kwargs.get('l_p')
# filter_price['upper_price'] = ''
# else:
# filter_price = None
# return filter_price
#
# def _get_product_list(self, **kwargs):
# """没有缓存搜索产品列表，搜索并存入缓存"""
# # TODO 查询出商品的销量添加到商品属性里面，用于界面展示及排序
# category = category_name_map.get(kwargs.get('c_k'))
# price_section = self._get_price_section(str(kwargs.get('l_p')), str(kwargs.get('u_p')))
# merchant_id = None
# if kwargs.get('m_n'):
# merchant_id = self._get_merchant_id(kwargs.get('m_n'))
# if not merchant_id:
# page_response = BrokenResponse()
# return page_response
# if category:
# if kwargs.get('p_s') == 'up':
# if price_section:
# if merchant_id:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'category': category,
# 'merchantId': merchant_id,
# '$where': price_section},
# sort=[(u'price', 1)])
# else:
# product_list = MongodbUtil.find('shopping', 'product',
# spec_or_id={'category': category, '$where': price_section},
# sort=[(u'price', 1)])
# else:
# if merchant_id:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'category': category,
# 'merchantId': merchant_id},
# sort=[(u'price', 1)])
# else:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'category': category},
# sort=[(u'price', 1)])
# elif kwargs.get('p_s') == 'down':
# if price_section:
# if merchant_id:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'category': category,
# 'merchantId': merchant_id,
# '$where': price_section},
# sort=[(u'price', -1)])
# else:
# product_list = MongodbUtil.find('shopping', 'product',
# spec_or_id={'category': category, '$where': price_section},
# sort=[(u'price', -1)])
# else:
# if merchant_id:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'category': category,
# 'merchantId': merchant_id},
# sort=[(u'price', -1)])
# else:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'category': category},
# sort=[(u'price', -1)])
# else:
# if price_section:
# if merchant_id:
# product_list = MongodbUtil.find('shopping', 'product',
# {'category': category, 'merchantId': merchant_id,
# '$where': price_section})
# else:
# product_list = MongodbUtil.find('shopping', 'product', {'category': category, '$where': price_section})
# else:
# if merchant_id:
# product_list = MongodbUtil.find('shopping', 'product',
# {'category': category, 'merchantId': merchant_id})
# else:
# product_list = MongodbUtil.find('shopping', 'product', {'category': category})
# page_response = ListPageResponse(product_list, **kwargs)
# elif kwargs.get('keyword'):
# product_id_list = self.get_product_id_list(**kwargs)
# product_list = self._get_sort_list(merchant_id, product_id_list, **kwargs)
# page_response = ListPageResponse(product_list, **kwargs)
# else:
# page_response = BrokenResponse()
# return page_response
#
# def _get_merchant_id(self, merchant_name):
# """获取商户id"""
# merchant_id = None
# if not merchant_name:
# return merchant_id
# merchant = MongodbUtil.find_one('shopping', 'merchant', {'name': merchant_name})
# if merchant:
# merchant_id = merchant.get('_id')
#
# return merchant_id
#
# def _get_price_section(self, lower_price, upper_price):
# """获得根据价格区间筛选商品"""
# price_section = None
# if lower_price and upper_price:
# price_section = 'this.price > ' + str(lower_price) + ' & this.price < ' + str(upper_price)
# elif not lower_price and upper_price:
# price_section = 'this.price < ' + str(upper_price)
# elif lower_price and not upper_price:
# price_section = 'this.price > ' + str(lower_price)
# return price_section
#
# def _get_sort_list(self, merchant_id, product_id_list, **kwargs):
# """根据产品id集合过滤排序获取商品列表"""
# today = DateUtil.get_sys_date()
# product_list = list()
# for product_id in product_id_list:
# product = MongodbUtil.find_one('shopping', 'product', spec_or_id=str(product_id))
# if merchant_id and product.get('merchantId') != merchant_id:
# continue
# if kwargs.get('f_n') == 'checked':
# if product.get('startTime') == today:
# if kwargs.get('l_p') and kwargs.get('u_p'):
# if (float(product.get('price')) > float(kwargs.get('l_p'))) and \
# (float(product.get('price')) < float(kwargs.get('u_p'))):
# product_list.append(product)
# elif not kwargs.get('l_p') and kwargs.get('u_p'):
# if float(product.get('price')) < float(kwargs.get('u_p')):
# product_list.append(product)
# elif kwargs.get('l_p') and not kwargs.get('u_p'):
# if float(product.get('price')) > float(kwargs.get('l_p')):
# product_list.append(product)
# else:
# product_list.append(product)
# else:
# if kwargs.get('l_p') and kwargs.get('u_p'):
# if (float(product.get('price')) > float(kwargs.get('l_p'))) and \
# (float(product.get('price')) < float(kwargs.get('u_p'))):
# product_list.append(product)
# elif not kwargs.get('l_p') and kwargs.get('u_p'):
# if float(product.get('price')) < float(kwargs.get('u_p')):
# product_list.append(product)
# elif kwargs.get('l_p') and not kwargs.get('u_p'):
# if float(product.get('price')) > float(kwargs.get('l_p')):
# product_list.append(product)
# else:
# product_list.append(product)
# if kwargs.get('p_s') == 'up' and len(product_list) > 0:
# product_list = sorted(product_list, key=lambda product: product['price'], reverse=False)
# elif kwargs.get('p_s') == 'down' and len(product_list) > 0:
# product_list = sorted(product_list, key=lambda product: product['price'], reverse=True)
# return product_list
#
# def _add_search_times(self, keyword):
# """关键词搜索的次数加一"""
# keyword = keyword.rstrip().lstrip().lower()
# keyword_set = self.get_keyword_set(keyword)
# #按照分词结果查询产品结果集
# for i in xrange(len(keyword_set)):
# keyword_index = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword_set[i]})
# if not keyword_index:
# continue
# #搜索次数加一
# document = {'$inc': {'searchTimes': 1}}
# keyword_index['searchTimes'] += 1
# MongodbUtil.update('shopping', 'keywordIndex', {'_id': keyword_index.get('_id')}, document)
#
# def _add_glossary(self, keyword):
# """动态填充词汇表"""
# glossary = MongodbUtil.find_one('shopping', 'glossary')
# if not glossary.get('used').__contains__(keyword):
# glossary['used'].append(keyword)
# MongodbUtil.update('shopping', 'glossary', {'_id': glossary.get('_id')}, glossary)
#
# def get_product_id_list(self, **kwargs):
# """从mongodb中获取产品id结果集"""
# keyword = kwargs.get('keyword').rstrip().lstrip().lower()
# if keyword.__contains__(' '):
# self._add_glossary(keyword)
# keyword_set = self.get_keyword_set(keyword)
# p_id_list_arr = list()
# union_dict = {}
# #按照分词结果查询产品结果集
# for i in xrange(len(keyword_set)):
# keyword_index = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword_set[i]})
# if not keyword_index:
# continue
# #搜索次数加一
# keyword_index['searchTimes'] += 1
# MongodbUtil.update('shopping', 'keywordIndex', {'_id': keyword_index.get('_id')}, keyword_index)
# p_id_dict = keyword_index.get('invertedIndex')
# if p_id_dict:
# union_dict = dict(union_dict.items() + p_id_dict.items())
# p_id_list = sorted(p_id_dict, key=p_id_dict.get, reverse=True)
# p_id_list_arr.append(p_id_list)
# #结果集排序：先取交集，再取并集(取差集)，分别按照优先级排序，最后合并
# intersection = list()
# union_set = list()
# for i in xrange(len(p_id_list_arr)):
# if not intersection:
# intersection = p_id_list_arr[i]
# union_set = p_id_list_arr[i]
# continue
# intersection = list(set(intersection).intersection(set(p_id_list_arr[i])))
# union_set = list(set(union_set).union(set(p_id_list_arr[i])))
# if intersection and union_set:
# union_set = list(set(union_set).difference(set(intersection)))
# #排序
# if intersection:
# intersection = OrderUtil.order_obj_dict(union_dict, intersection)
# if union_set:
# union_set = OrderUtil.order_obj_dict(union_dict, union_set)
#
# product_id_list = list()
# for i in xrange(len(intersection)):
# product_id_list.append(intersection[i])
# for i in xrange(len(union_set)):
# product_id_list.append(union_set[i])
# return product_id_list
#
# def get_keyword_set(self, keyword):
# """获取关键字无重复集合"""
# keyword_set = list()
# #分词模板
# glossary = MongodbUtil.find_one('shopping', 'glossary')
# use_word_list = glossary.get('used')
# un_used_word_list = glossary.get('unUsed')
# for word in use_word_list:
# if keyword.__contains__(word):
# keyword_set.append(word)
# for word in StringUtil.cut_word(keyword):
# if not un_used_word_list.__contains__(word) and not keyword_set.__contains__(word):
# keyword_set.append(word)
# return keyword_set
#
#
# class SuggestHandler(BaseHandler):
# """
# 推荐逻辑处理
#
# """
#
# def get(self):
# try:
# page = self.get_suggest_product(**self.params)
# if len(page.get('suggestItems')) != 0:
# self.render(settings.inner_ad_template, page=page)
# else:
# self.render(settings.default_inner_ad_template)
# except Exception as e:
# self.render(settings.default_inner_ad_template)
# logger.error('[message: %s]; [params: %s]' % (e.message, self.params.__str__()))
#
# def get_suggest_product(self, **kwargs):
# """
# 获取广告位推荐商品集合
#
# """
# #查询参数校验
# page = dict()
# if not self._check_params(**self.params):
# self._event('check_false')
# page['suggestItems'] = list()
# return page
# self._event('%s_suggest' % kwargs.get('w_m'))
# page = self._get_suggest_page(**kwargs)
# return page
#
# def _check_params(self, **kwargs):
# """
# 参数校验
#
# """
# check_flag = False
# # TODO 过滤校验网站主身份
# if kwargs.get('w_m') and kwargs.get('c_k') and (int(kwargs.get('p_c')) > 0):
# check_flag = True
# return check_flag
#
# def _get_suggest_page(self, **kwargs):
# """
# 查询数据库获取推荐产品集合
# 1.筛选参数：
# 网站主名称、商品分类、推荐商品个数、价格区间。
#
# """
# # 根据筛选参数查库获取推荐商品集合
# category = category_name_map.get(kwargs.get('c_k'))
# product_count = kwargs.get('p_c')
# lower_price = int(kwargs.get('l_p')) if kwargs.get('l_p') else None
# upper_price = int(kwargs.get('u_p')) if kwargs.get('u_p') else None
# price_section = None
# if lower_price or upper_price:
# price_section = self._get_price_section(lower_price, upper_price)
#
# product_list = self._get_random_product_list(str(category), price_section, int(product_count))
#
# # 创建响应对象并返回
# page_response = SuggestResponse(product_list, **kwargs)
# return page_response.to_dict()
#
# def _get_random_product_list(self, category, price_section, product_count):
# """获取网站主广告位随机推荐商品集合"""
# if category:
# if price_section:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'category': category, '$where': price_section})
# else:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'category': category})
# else:
# if price_section:
# product_list = MongodbUtil.find('shopping', 'product', spec_or_id={'$where': price_section})
# else:
# product_list = MongodbUtil.find('shopping', 'product')
# start_index, end_index = self._get_random_index(len(product_list), product_count)
# return product_list[start_index: end_index]
#
# def _get_random_index(self, product_list_length, product_count):
# """获取随机索引值"""
# start_index = random.randint(0, product_list_length - product_count)
# end_index = start_index + product_count
# return start_index, end_index
#
# def _get_price_section(self, lower_price, upper_price):
# """获得根据价格区间筛选商品"""
# price_section = None
# if lower_price and upper_price:
# price_section = 'this.price > ' + str(lower_price) + ' & this.price < ' + str(upper_price)
# elif not lower_price and upper_price:
# price_section = 'this.price < ' + str(upper_price)
# elif lower_price and not upper_price:
# price_section = 'this.price > ' + str(lower_price)
# return price_section
# class AnalysisHandler(BaseHandler):
# """数据统计分析"""
#
# # Group data by javascript template
# reduce_cpc_sum_js = """
# function (obj, cpc) {
# if(obj.takeEffect == 1){
# cpc.effectCount++;
# } else if (obj.takeEffect == 0){
# cpc.unEffectCount++;
# }
# }
# """
#
# def on_request(self):
# try:
# #身份验证
# check_auth = CheckUtil.check_auth(**self.params)
# check_params = self._check_params(**self.params)
# if not (check_auth and check_params):
# error_response = BrokenResponse()
# self._jsonify_response(error_response)
# return
# method_name = self.params.get('method')
# method_dict = {
# 'cpc': lambda: self.analysis_cpc(**self.params),
# 'cps': lambda: self.analysis_cps(**self.params),
# }
# method_dict[method_name]()
# except Exception as e:
# logger.error('[message: %s]; [host: %s]; [ip: %s]; [params: %s]' % (
# e.message, self.request.host, self.request.remote_ip, self.params.__str__()))
# error_response = BrokenResponse()
# self._jsonify_response(error_response)
#
# def _check_params(self, **kwargs):
# """参数校验"""
# check_flag = False
# page_index = int(kwargs.get('pageIndex'))
# page_size = int(kwargs.get('pageSize'))
# if page_index > 0 and page_size > 0 and kwargs.get('f') and kwargs.get('t'):
# check_flag = True
# return check_flag
#
# def analysis_cps(self, **kwargs):
# """cps实时统计接口"""
# self._event('cps_list')
# page_index = int(kwargs.get('pageIndex'))
# page_size = int(kwargs.get('pageSize'))
# product_list, page_count = self._get_cps_item_list(page_index, page_size, **kwargs)
# page_response = PageResponse(page_index, page_size, page_count, product_list)
# #返回结果列表信息
# self._jsonify_response(page_response)
#
# def _get_cps_item_list(self, page_index, page_size, **kwargs):
# """查询数据列表"""
# # 变量声明＆参数赋值
# item_list, cps_from, cps_to, end_date = list(), kwargs.get('f').lstrip().rstrip(), kwargs.get(
# 't').lstrip().rstrip(), DateUtil.get_sys_date()
# # 筛选条件
# spec_or_id, merchant = self._get_condition(cps_from, cps_to)
# # 查询监控列表
# cps_list = MongodbUtil.find('shopping', 'cps', spec_or_id=spec_or_id, sort=[(u'cpsTime', 1)])
# for cps in cps_list:
# cps['_id'] = cps.get('_id').__str__()
# cps['webmaster'] = cps_from or cps.get('webmaster')
# cps['merchant'] = cps_to or merchant.get('name')
# cps['cpcId'] = cps.get('cpcId').__str__()
# item_list.append(cps)
# page_count = MathUtil.round(len(item_list), page_size)
# item_list = self._filter_cps(item_list, page_index, page_size)
# return item_list, page_count
#
# def _filter_cps(self, cps_list, page_index, page_size):
# """排序过滤监测产品信息"""
# if page_size > len(cps_list) - (page_index - 1) * page_size:
# length = len(cps_list) - (page_index - 1) * page_size
# else:
# length = page_size
# return cps_list[(page_index - 1) * page_size: ((page_index - 1) * page_size + length)]
#
# def analysis_cpc(self, **kwargs):
# """cpc实时统计接口"""
# self._event('cpc_list')
# page_index = int(kwargs.get('pageIndex'))
# page_size = int(kwargs.get('pageSize'))
# product_list, page_count = self._get_cpc_item_list(page_index, page_size, **kwargs)
# page_response = PageResponse(page_index, page_size, page_count, product_list)
# #返回结果列表信息
# self._jsonify_response(page_response)
#
# def _get_cpc_item_list(self, page_index, page_size, **kwargs):
# """查询数据列表"""
# # 变量声明＆参数赋值
# item_list, cpc_from, cpc_to, end_date = list(), kwargs.get('f').lstrip().rstrip(), kwargs.get(
# 't').lstrip().rstrip(), DateUtil.get_sys_date()
# # 筛选条件
# condition, merchant = self._get_condition(cpc_from, cpc_to)
# # 查询监控列表
# cpc_list = MongodbUtil.group('shopping', collection='cpc', key=['productId', 'webmaster', 'merchant', 'adPositionId', ],
# condition=condition, initial={'effectCount': 0, 'unEffectCount': 0},
# reduce=self.reduce_cpc_sum_js)
# page_count = MathUtil.round(len(cpc_list), page_size)
# for cpc in cpc_list:
# product_id = cpc.get('productId')
# product = MongodbUtil.find_one('shopping', 'product', {'_id': product_id})
# start_date = product.get('startTime')
# product_cpc = {
# 'title': product.get('title'),
# 'webmaster': cpc_from,
# 'merchant': merchant.get('name') or cpc_to,
# 'startDate': start_date,
# 'endDate': end_date,
# 'effectCount': cpc.get('effectCount'),
# 'unEffectCount': cpc.get('unEffectCount'),
# }
# item_list.append(product_cpc)
# item_list = self._filter_product_cpc(item_list, page_index, page_size)
# return item_list, page_count
#
# def _get_condition(self, webmaster_name, merchant_name):
# """获取筛选条件"""
# merchant = MongodbUtil.find_one('shopping', 'merchant', {'name': merchant_name})
# condition = dict()
# if webmaster_name:
# condition['webmaster'] = webmaster_name
# elif merchant:
# condition['merchant'] = merchant.get('_id')
# else:
# condition = None
# return condition, merchant
#
# def _filter_product_cpc(self, product_cpc_list, page_index, page_size):
# """排序过滤监测产品信息"""
# result_list = list()
# for i in xrange(len(product_cpc_list)):
# temp_cpc = None
# for product_cpc in product_cpc_list:
# if not temp_cpc:
# temp_cpc = product_cpc
# continue
# if temp_cpc.get('startTime') > product_cpc.get('startTime'):
# temp_cpc = product_cpc
# continue
# result_list.append(temp_cpc)
# if product_cpc_list:
# del product_cpc_list[product_cpc_list.index(temp_cpc)]
# if page_size > len(result_list) - (page_index - 1) * page_size:
# length = len(result_list) - (page_index - 1) * page_size
# else:
# length = page_size
# return result_list[(page_index - 1) * page_size: ((page_index - 1) * page_size + length)]
def delete_product(product):
"""删除产品及产品相关信息"""
product_id = product.get('_id')
MongodbUtil.delete('shopping', 'product', product)
logger.info('Delete a product: %s successfully!!!' % product.get('title'))
#删除索引中product.id
keyword_list = product.get('keywordList')
for keyword in keyword_list:
keyword = MongodbUtil.find_one('shopping', 'keywordIndex',
{'keyword': keyword})
if not keyword:
continue
if keyword['invertedIndex'].__contains__(product_id.__str__()):
del keyword['invertedIndex'][product_id.__str__()]
MongodbUtil.save('shopping', 'keywordIndex', keyword)
logger.info('Update keyword index: %s successfully!!!' %
keyword.get('keyword'))
#删除merchant中product.id
merchant_id = product.get('merchantId')
merchant = MongodbUtil.find_one('shopping', 'merchant',
{'_id': merchant_id})
index_id = merchant['productIdList'].index(product_id)
if isinstance(index_id, int):
del merchant['productIdList'][index_id]
MongodbUtil.save('shopping', 'merchant', merchant)
logger.info('Update merchant productIdList successfully!!!')
#删除product的image及图片文件
images = MongodbUtil.find('shopping', 'image', {'productId': product_id})
for image in images:
MongodbUtil.remove(filename=image.get('fileName'))
MongodbUtil.delete('shopping', 'image', {'productId': product_id})
