Esempio n. 1
0
def gettags(kw):
	"""标签缓存key: biz:prod:targ
	test: 	
     curl -l -H "Content-Type: application/json" -X POST -d  '{"biznum":"tags.corp","tag_id":101,"lv2rows":13,"lv3rows":1}'     http://192.168.10.126:6000/biz_tags/gettags
	参数说明:
		biznum:业务配置标识
		tag_id:父标签
		lv2rows:二级标签提取数量
		lv3rows:三级标签提取数量
	"""
	biznum = kw.get('biznum','')

	sort = ""
	tag_id = str(kw.get('tag_id',""))
	tag_id_len = 0 if len(tag_id) < 3 else len(tag_id)
	lv2rows = kw.get('lv2rows',0)
	lv3rows = kw.get('lv3rows',0)
		
	if not biznum:
		return [-6,'parameter biznum not set.']
	bizcnf = get_cnf_val(biznum,sconf.BIZ)
	#取业务配置
	if not bizcnf:
		return sconf.err_handle.biznum_not_config

	ttl     = bizcnf.get('ttl',600)
	isort   = bizcnf.get('sort','total_num')
	#取数据
	rdb = rediswrap.get_redis('cache')
	ckey = "%s.%s%s%s%s" %(biznum,sort,tag_id,lv2rows,lv3rows)
	res = rdb.get(ckey)
	if res:
		return 0,json.loads(res)
		
	res = rdb.get("biz:targ.%s"%bizcnf['table'])
	if not res:
		s,res = tagsclass.prod_targs(biznum)
	else:
		res = json.loads(res)
	#组织数据
	#tag_id of level 1
	data={}
	if tag_id_len ==3 and tag_id in res and 'extn' in res[tag_id]:
		data = res[tag_id]
	elif tag_id_len ==6:
		data = res[str(tag_id)[:3]]['extn'][tag_id]
	#tag_id of all
	elif not tag_id:
		data={'info':{'tag_id':0}}
		data['extn'] = res
	if data and 'extn' in data:
		res = tag_recursion(data,lv2rows,lv3rows)
		rdb.setex(ckey,json.dumps(res),ttl)
		return 0,res 
	else:
		return 0,{}
Esempio n. 2
0
def prod_targs(key):
	bizcnf = get_cnf_val(key,sconf.BIZ)
	#取业务配置
	if not bizcnf:
		return sconf.err_handle.biznum_not_config
	ttl     = bizcnf.get('ttl',600)
	#取数据库配置
	dbinfo = get_host_by_data(bizcnf['source'])
	if not dbinfo:
		return sconf.err_handle.db_not_config
	dbinfo['dbname']=bizcnf['source'].split('.')[-1]
	rdb = rediswrap.get_redis('cache')
	rkey = "biz:targ.%s"%bizcnf['table']
	db = dbclass(dbinfo)
	res,desc = db.connect()
	
	if res == -1:
		return sconf.err_handle.db_err
	sql_item={}
	sql_item['table'] = bizcnf['table']
	sql_item['fields'] = bizcnf['fields']
	sql_item['order'] = "tag_id asc"

	res,desc = db.query(sql_item)

	if res == -1:
		return sconf.err_handle.db_err
	if desc:
		obj = {}
		for row in desc:
			tid = str(row['tag_id'])
			if len(tid) ==3:
				obj[row['tag_id']]={"info":row}
			elif len(tid) ==6:
				pid = int(tid[:3])
				if 'extn' not in  obj[pid]:
					obj[pid]['extn']={}
				obj[pid]['extn'][row['tag_id']]={"info":row}
			elif len(tid)==9:

				ppid = int(tid[:3])
				pid =  int(tid[:6])
				if 'extn' not in  obj[ppid]['extn'][pid]:
					obj[ppid]['extn'][pid]['extn']={}
				obj[ppid]['extn'][pid]['extn'][row['tag_id']]={"info":row}
		#try:
		#rdb.set(rkey,gzip.compress(json.dumps(res).encode()))
		rdb.set(rkey,json.dumps(obj))
		#except:
		#	pass
		return 0,obj
Esempio n. 3
0
def sphinx2redis(cnfkey):
	"""通过配置文件,获取统计数据
	"""
	#取业务配置
	bizcnf = get_cnf_val(cnfkey,sconf.BIZ)	
	if not bizcnf:
		return sconf.err_handle.biznum_not_config
	rdb = rediswrap.get_redis('cache')

	for i in range(0,len(bizcnf['prama'])):
		itm = bizcnf['prama'][i]
		source = itm['source'].split('.')
		if source[1] == 'sphinx':
			host_info = get_host_by_data(itm['source'])
			if  not host_info :
				return sconf.sphinx_index_not_found
			
			sp = sphinx(host_info['host'],host_info['port'])
			expression = itm['expression']
			expression['index'] = source[2]
			total_found = 0
			while True:
				#if total_found >0:
				#	if expression['pageSize'] * expression['page'] >=total_found:
				#		break
				#	expression['page'] +=1
					
				sp.initQuery(itm['expression'])
				rs = sp.RunQueries()
				pprint(rs)
				if rs and rs[0]['status']==0:
					total_found = rs[0]['total_found']
					_items = {}
					for row in rs[0]['matches']:
						if itm['key'] in row['attrs'] and itm['value'] in row['attrs']:
							_items[row['attrs'][itm['key']]]=row['attrs'][itm['value']]
					if _items:
						print(_items)
						#res = rdb.mset(cnfkey,_items)
						
				else:
					print(sp._error)
					break
				break
Esempio n. 4
0
def prodlist(kw):
	"""
	test: 	
     curl -l -H "Content-Type: application/json" -X POST -d  '{"biznum":"tags.corplist","tag_id":[101]}'     http://192.168.10.126:6000/biz_tags/prodlist
	"""	
	tag_id = kw.get('tag_id',[])
	biznum = kw.get('biznum','')

	if not biznum:
		return [-6,'parameter biznum not set.']
	bizcnf = get_cnf_val(biznum,sconf.BIZ)
			
	#取业务配置
	if not bizcnf:
		return sconf.err_handle.biznum_not_config
		
	pageSize = kw.get('pageSize',bizcnf['search']['expression']['pageSize'])
	page = kw.get('page',bizcnf['search']['expression']['page'])
	#连接搜索引擎
	host_info = get_host_by_data(bizcnf['search']['source'])
	if  not host_info :
		return sconf.sphinx_index_not_found
	sp = sphinx(host_info['host'],host_info['port'])
	
	expression = bizcnf['search']['expression']
	expression['index'] = bizcnf['search']['source'].split('.')[-1]
	expression['pageSize']=pageSize
	expression['page']=page
	st = int(time.time())
	for tid in tag_id:
		tag_len = len(str(tid))
		if tag_len == 3:
			f = "r_tag_id"
		elif tag_len == 6:
			f = "c_tag_id"
		elif tag_len == 9:
			f = "_tag_id"
		#exp = expression.copy()
		expression['intType'][f]=str(tid)
		sp.initQuery(expression)
	rs = sp.RunQueries()
	print("search_time:",time.time()-st)
	result = []
	st = int(time.time())
	if rs:
		#连接数据库
		dbinfo = get_host_by_data(bizcnf['data']['source'])
		dbinfo['dbname'] = bizcnf['data']['source'].split('.')[-1]
		db = dbclass(dbinfo)
		db.connect()
		for row in rs:
			sql_item =  bizcnf['data'].copy()
			ids = [str(k['id']) for k in row['matches']]
			print("ids:",ids)
			sql_item['where'] = sql_item['where'] % ",".join(ids)
			res,desc = db.query(sql_item)
			if res ==0 and desc:
				result.append(desc)
	else:
		result.append([])
	#print("search_time:",time.time()-st)
	#pprint(result)	
	#print(sp._error)
	return [0,result]
Esempio n. 5
0
def gettags_base(kw):
	"""标签缓存key: biz:prod:targ
	"""
	biznum = kw.get('biznum','')
	sort = kw.get('sort','pro')
	tag_id = int(kw.get('tag_id',0))
	tag_id_len = 0 if len(str(tag_id)) < 3 else len(str(tag_id))
	if not biznum:
		return [-6,'parameter biznum not set.']
	bizcnf = get_cnf_val(biznum,sconf.BIZ)
	#取业务配置
	if not bizcnf:
		return sconf.err_handle.biznum_not_config
	lv2rows = kw.get('lv2rows',bizcnf['lv2rows'])
	lv3rows = kw.get('lv3rows',bizcnf['lv3rows'])
	ttl     = kw.get(bizcnf['ttl'],600)
	isort   = kw.get(bizcnf['sort'],'total_num')
	#取数据库配置
	dbinfo = get_host_by_data(bizcnf['source'])
	if not dbinfo:
		return sconf.err_handle.db_not_config
	dbinfo['dbname']=bizcnf['source'].split('.')[-1]
	rdb = rediswrap.get_redis('cache')
	
	res = rdb.get("%s.%s%s%s%s" %(biznum,sort,tag_id,lv2rows,lv3rows))
	if res:
		return gzip.decompress(res)
	db = dbclass(dbinfo)
	res,desc = db.connect()
	
	if res == -1:
		return sconf.err_handle.db_err
	sql_item={}
	sql_item['table'] = bizcnf['table']
	sql_item['fields'] = bizcnf['fields']
	if tag_id_len ==3:
		sql_item['where'] = "tag_id = %s or (tag_id>%s000 and tag_id <%s000) or (tag_id>%s000000 and tag_id<%s000000) " % (tag_id,tag_id,(tag_id+1),tag_id,(tag_id+1))
	elif tag_id_len >=6:
		sql_item['where'] = "tag_id = %s or (tag_id>%s000 and tag_id <%s000)" % (tag_id,tag_id,(tag_id+1))

	sql_item['order'] = "tag_id asc"
	pprint.pprint(kw)
	pprint.pprint(sql_item)
	res,desc = db.query(sql_item)

	if res == -1:
		return sconf.err_handle.db_err
	if desc:
		tag_id_len = 3 if tag_id_len==0 else tag_id_len
		obj = {}
		for row in desc:
			tid = str(row['tag_id'])
			if len(tid)-tag_id_len ==0:
				obj[row['tag_id']]={"info":row}
			elif len(tid) - tag_id_len ==3:
				pid = int(tid[:tag_id_len])
				if 'extn' not in  obj[pid]:
					obj[pid].update({'extn':{"info":row}})
				obj[pid]['extn'].update({row['tag_id']:row})
			elif len(tid) - tag_id_len==6:

				ppid = int(tid[:tag_id_len])
				pid =  int(tid[:tag_id_len+3])
				if 'extn' not in  obj[ppid]['extn'][pid]:
					obj[ppid]['extn'][pid].update({'extn':{"info":row}})
				obj[ppid]['extn'][pid].update({row['tag_id']:row})
		#pprint.pprint(obj)
		return 0,obj
Esempio n. 6
0
def get_index(kw):
	
    data = get_cnf_val('base.sphinx',sconf.DATA_SOURC)
    return [0,data]
Esempio n. 7
0
def sphinx2redis_ex(cnfkey):
	"""通过配置文件,获取统计数据
	"""
	#取业务配置
	bizcnf = get_cnf_val(cnfkey,sconf.BIZ)	
	if not bizcnf:
		return sconf.err_handle.biznum_not_config
	rdb = rediswrap.get_redis('cache')
	dbinfo = get_host_by_data("base.mysql.biz72_product")
	dbinfo['dbname']="biz72_product"
	db  = mysqlwrap.dbclass(dbinfo)
	db.connect()
	sql = "select tag_id from pro_tag where tag_id>100 and tag_id<200"
	res,desc = db.query(sql)
	print("rows:",len(desc))
	source="base.sphinx.IDX_com_corp_dist"
	host_info = get_host_by_data(source)
	sp = sphinx(host_info['host'],host_info['port'])
	prama = {
					"querymod":"SPH_MATCH_EXTENDED2",
					"pageSize":1,
					"page":1,
					"intType":{
            		},
            		"index":"IDX_com_corp_dist"
			}
	itm1 = {}
	itm2 = {}
	itm3 = {}
	x=0
	st = int(time.time())
	for row in desc:
		x+=1
		if x%100==0:
			print(x,time.time()-st)
		tag_len = len(str(row['tag_id']))
		if tag_len == 3:
			f = "r_com"
		elif tag_len == 6:
			f = "c_com"
		elif tag_len == 9:
			f = "com"
		prama['intType'][f]=str(row['tag_id'])
		#prama['intType']["com"]='101105105'
		sp.initQuery(prama)
		rs = sp.RunQueries()

		if rs and rs[0]['status']==0:
			total_found = rs[0]['total_found']
			print("tag_id",row['tag_id'],"total_found",total_found)
			if total_found >0:
				rdb.hset('stat2cach.tags_corp',row['tag_id'],total_found)
			if tag_len == 3:
				itm1[row['tag_id']]=total_found
			elif tag_len == 6:
				itm2[row['tag_id']]=total_found
			elif tag_len == 9:
				itm3[row['tag_id']]=total_found
		else:
			print(sp._error)

	print("time:",time.time()-st)
	print("itm1","len:",len(itm1),"sum:",sum(list(itm1.values())))		
	print("itm2","len:",len(itm2),"sum:",sum(list(itm2.values())))	
	print("itm3","len:",len(itm3),"sum:",sum(list(itm3.values())))