def save_keyword_index(keyword_set, product): """遍历keywordList更新倒排索引""" try: for keyword in keyword_set: existingKeyword = MongodbUtil.find_one('shopping', 'keywordIndex', {'keyword': keyword}) if existingKeyword: if not existingKeyword['invertedIndex'].__contains__(product.get('_id').__str__()): existingKeyword['invertedIndex'][product.get('_id').__str__()] = 100.0 status_id = MongodbUtil.save('shopping', 'keywordIndex', existingKeyword) if status_id: logger.info('Update keywordIndex: %s successfully!!!' % keyword) else: save_keyword_index(keyword_set, product) logger.info('Try to update keywordIndex: %s again!!!' % keyword) else: keywordIndex = { 'keyword': keyword, 'searchTimes': 0, 'invertedIndex': {product.get('_id').__str__(): 100.0}, } status_id = MongodbUtil.insert('shopping', 'keywordIndex', keywordIndex) if status_id: logger.info('Save keywordIndex: %s successfully!!!' % keyword) else: save_keyword_index(keyword_set, product) logger.info('Try to update keywordIndex: %s again!!!' % keyword) except Exception as e: logger.error(e.message)
def _cpsuc_effect(self, order_list): for order in order_list: # 获取校验参数 if type(order) is dict and order.has_key( 'merchant') and order.has_key('orderId') and order.has_key( 'productList') and order.has_key('merchant'): exist_order = MongodbUtil.find_one('api', 'cps', spec_or_id={ 'orderId': order.get('orderId'), 'merchant': order.get('merchant') }) if exist_order: exist_order['status'] = 'successful' exist_order['successfulTime'] = DateUtil.get_sys_time() MongodbUtil.update( 'api', 'cps', spec_or_id={'_id': exist_order.get('_id')}, document=exist_order) continue else: continue order = self._fit_params(order) # 插入cps cps_id = MongodbUtil.insert('api', 'cps', order) if cps_id: self._event('cps_insert:%s' % cps_id)
def import_category(): """导入基础词汇表""" try: exists_category = MongodbUtil.find_one('shopping', 'category') if exists_category: message = 'The category is exists!!!' logger.info(message) return id = MongodbUtil.insert('shopping', 'category', category_dict) if id: message = 'Import category(id:%s) successfully!!!' % id logger.info(message) except Exception as e: logger.error(e.message)
def save_overdue_cpc(overdue_product_id, product, cpc_list): """定期备份cpc跟踪点击数到overdueCpc集合中""" for cpc in cpc_list: overdue_cpc = { 'cpcOverdueProductId': overdue_product_id, 'webmaster': cpc.get('webmaster'), 'merchant': cpc.get('merchant'), 'adPositionId': cpc.get('adPositionId'), 'clickTime': cpc.get('clickTime'), 'clearTime': DateUtil.get_sys_time(), 'takeEffect': cpc.get('takeEffect'), } overdue_cpc_id = MongodbUtil.insert('shopping', 'overdueCpc', overdue_cpc) if overdue_cpc_id: logger.info( 'Insert overdue cpc from:%s to:%s successfully!!!' % (overdue_cpc.get('webmaster'), overdue_cpc.get('merchant')))
def insert_cpc(self, user_id, language, title, webmaster, merchant, category, url, meta_info, duration): """插入新增点击 takeEffect: 1(successful) 0(failed)""" cpc_document = { 'uid': user_id, 'merchant': merchant, 'clickTime': DateUtil.get_sys_time(), 'url': url, 'takeEffect': 1, 'webmaster': webmaster, 'category': category, 'language': language, 'title': title, 'metaInfo': meta_info, 'duration': duration, } obj_id = MongodbUtil.insert('api', 'cpc', cpc_document) return obj_id
def save_overdue_product(product_list): """保存过期产品到overdueProduct集合中""" update_count = 0 insert_count = 0 for product in product_list: product_id = product.get('_id') overdue_product_id = MongodbUtil.insert('shopping', 'overdueProduct', product) if overdue_product_id: insert_count += 1 logger.info('Insert overdue product: %s successfully!!!' % product.get('title')) #cpc统计数据转移至备份集合overdueCpc中后删除 cpc_list = MongodbUtil.find('shopping', 'cpc', {'productId': product_id}) save_overdue_cpc(overdue_product_id, product, cpc_list) MongodbUtil.delete('shopping', 'cpc', {'cpcProductId': product_id}) logger.info('Update %s overdue product, Insert %s overdue product!!!' % (update_count, insert_count))
def save_product_img(product, tag_dict, size): """下载产品图片并存入mongodb中""" file_name = product.get(tag_dict['image']) opener = urllib2.build_opener() response = opener.open(file_name) file_body = response.read() file_name = UploadUtil.upload_pic(file_name, file_body, size[0], size[1]) image = { 'productId': product.get('_id'), 'version': DateUtil.get_sys_time(), 'fileName': file_name, 'width': size[0], 'height': size[1], } image_id = MongodbUtil.insert('shopping', 'image', image) if image_id: logger.info('Save product\'s image: %s successfully!!!' % product.get(tag_dict['image'])) return image_id else: logger.info('Save product\'s image: %s failed!!!' % product.get(tag_dict['image'])) return None
def _cps_effect(self, merchant, order_list): """广告主订单成功,在联盟插入cps数据,确认cps生效""" cookie_value = '' for order in order_list: # 获取校验参数 if type(order) is dict and order.has_key( 'merchant') and order.has_key('orderId') and order.has_key( 'productList') and order.has_key('merchant') and ( merchant == order.get('merchant')): exist_order = MongodbUtil.find_one('api', 'cps', spec_or_id={ 'orderId': order.get('orderId'), 'merchant': order.get('merchant') }) if exist_order: exist_order['status'] = 'successful' exist_order['successfulTime'] = DateUtil.get_sys_time() MongodbUtil.update( 'api', 'cps', spec_or_id={'_id': exist_order.get('_id')}, document=exist_order) continue else: continue order = self._fit_params(order) # 插入cps cps_id = MongodbUtil.insert('api', 'cps', order) if cps_id: self._event('cps_insert:%s' % cps_id) cookie_value += '_%s' % cps_id.__str__() if cookie_value: self.set_cookie('c_i_l', cookie_value, expires_days=settings.cookie_alive_time)