def save_keyword_index(keyword_set, product):
"""遍历keywordList更新倒排索引"""
try:
for keyword in keyword_set:
existingKeyword = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword})
if existingKeyword:
if not existingKeyword['invertedIndex'].__contains__(product.get('_id').__str__()):
existingKeyword['invertedIndex'][product.get('_id').__str__()] = 100.0
status_id = MongodbUtil.save('shopping', 'keywordIndex', existingKeyword)
if status_id:
logger.info('Update keywordIndex: %s successfully!!!' % keyword)
else:
save_keyword_index(keyword_set, product)
logger.info('Try to update keywordIndex: %s again!!!' % keyword)
else:
keywordIndex = {
'keyword': keyword,
'searchTimes': 0,
'invertedIndex': {product.get('_id').__str__(): 100.0},
}
status_id = MongodbUtil.insert('shopping', 'keywordIndex', keywordIndex)
if status_id:
logger.info('Save keywordIndex: %s successfully!!!' % keyword)
else:
save_keyword_index(keyword_set, product)
logger.info('Try to update keywordIndex: %s again!!!' % keyword)
except Exception as e:
logger.error(e.message)
def _cpsuc_effect(self, order_list):
for order in order_list:
# 获取校验参数
if type(order) is dict and order.has_key(
'merchant') and order.has_key('orderId') and order.has_key(
'productList') and order.has_key('merchant'):
exist_order = MongodbUtil.find_one('api',
'cps',
spec_or_id={
'orderId':
order.get('orderId'),
'merchant':
order.get('merchant')
})
if exist_order:
exist_order['status'] = 'successful'
exist_order['successfulTime'] = DateUtil.get_sys_time()
MongodbUtil.update(
'api',
'cps',
spec_or_id={'_id': exist_order.get('_id')},
document=exist_order)
continue
else:
continue
order = self._fit_params(order)
# 插入cps
cps_id = MongodbUtil.insert('api', 'cps', order)
if cps_id:
self._event('cps_insert:%s' % cps_id)
def import_category():
"""导入基础词汇表"""
try:
exists_category = MongodbUtil.find_one('shopping', 'category')
if exists_category:
message = 'The category is exists!!!'
logger.info(message)
return
id = MongodbUtil.insert('shopping', 'category', category_dict)
if id:
message = 'Import category(id:%s) successfully!!!' % id
logger.info(message)
except Exception as e:
logger.error(e.message)
def save_overdue_cpc(overdue_product_id, product, cpc_list):
"""定期备份cpc跟踪点击数到overdueCpc集合中"""
for cpc in cpc_list:
overdue_cpc = {
'cpcOverdueProductId': overdue_product_id,
'webmaster': cpc.get('webmaster'),
'merchant': cpc.get('merchant'),
'adPositionId': cpc.get('adPositionId'),
'clickTime': cpc.get('clickTime'),
'clearTime': DateUtil.get_sys_time(),
'takeEffect': cpc.get('takeEffect'),
}
overdue_cpc_id = MongodbUtil.insert('shopping', 'overdueCpc',
overdue_cpc)
if overdue_cpc_id:
logger.info(
'Insert overdue cpc from:%s to:%s successfully!!!' %
(overdue_cpc.get('webmaster'), overdue_cpc.get('merchant')))
def insert_cpc(self, user_id, language, title, webmaster, merchant,
category, url, meta_info, duration):
"""插入新增点击 takeEffect: 1(successful) 0(failed)"""
cpc_document = {
'uid': user_id,
'merchant': merchant,
'clickTime': DateUtil.get_sys_time(),
'url': url,
'takeEffect': 1,
'webmaster': webmaster,
'category': category,
'language': language,
'title': title,
'metaInfo': meta_info,
'duration': duration,
}
obj_id = MongodbUtil.insert('api', 'cpc', cpc_document)
return obj_id
def save_overdue_product(product_list):
"""保存过期产品到overdueProduct集合中"""
update_count = 0
insert_count = 0
for product in product_list:
product_id = product.get('_id')
overdue_product_id = MongodbUtil.insert('shopping', 'overdueProduct',
product)
if overdue_product_id:
insert_count += 1
logger.info('Insert overdue product: %s successfully!!!' %
product.get('title'))
#cpc统计数据转移至备份集合overdueCpc中后删除
cpc_list = MongodbUtil.find('shopping', 'cpc',
{'productId': product_id})
save_overdue_cpc(overdue_product_id, product, cpc_list)
MongodbUtil.delete('shopping', 'cpc', {'cpcProductId': product_id})
logger.info('Update %s overdue product, Insert %s overdue product!!!' %
(update_count, insert_count))
def save_product_img(product, tag_dict, size):
"""下载产品图片并存入mongodb中"""
file_name = product.get(tag_dict['image'])
opener = urllib2.build_opener()
response = opener.open(file_name)
file_body = response.read()
file_name = UploadUtil.upload_pic(file_name, file_body, size[0], size[1])
image = {
'productId': product.get('_id'),
'version': DateUtil.get_sys_time(),
'fileName': file_name,
'width': size[0],
'height': size[1],
}
image_id = MongodbUtil.insert('shopping', 'image', image)
if image_id:
logger.info('Save product\'s image: %s successfully!!!' % product.get(tag_dict['image']))
return image_id
else:
logger.info('Save product\'s image: %s failed!!!' % product.get(tag_dict['image']))
return None
def _cps_effect(self, merchant, order_list):
"""广告主订单成功，在联盟插入cps数据，确认cps生效"""
cookie_value = ''
for order in order_list:
# 获取校验参数
if type(order) is dict and order.has_key(
'merchant') and order.has_key('orderId') and order.has_key(
'productList') and order.has_key('merchant') and (
merchant == order.get('merchant')):
exist_order = MongodbUtil.find_one('api',
'cps',
spec_or_id={
'orderId':
order.get('orderId'),
'merchant':
order.get('merchant')
})
if exist_order:
exist_order['status'] = 'successful'
exist_order['successfulTime'] = DateUtil.get_sys_time()
MongodbUtil.update(
'api',
'cps',
spec_or_id={'_id': exist_order.get('_id')},
document=exist_order)
continue
else:
continue
order = self._fit_params(order)
# 插入cps
cps_id = MongodbUtil.insert('api', 'cps', order)
if cps_id:
self._event('cps_insert:%s' % cps_id)
cookie_value += '_%s' % cps_id.__str__()
if cookie_value:
self.set_cookie('c_i_l',
cookie_value,
expires_days=settings.cookie_alive_time)
