def run(self):
while True:
try:
task = self.que_manager.getQue(self.get_que_name)
if task is None:
time.sleep(0.5)
continue
image_dir = task['image_dir']
for f in os.listdir(image_dir):
img_path = os.path.join(image_dir, f)
md5 = utils.calc_md5(img_path)
if utils.get_key(md5, rcli=utils.g_md5_redis) is Not None:
# md5 duplicated, remove image and process next
utils.remove_file(img_path)
continue
# md5 unique, record to redis
utils.set_key(md5, 1, rcli=utils.g_md5_redis)
# image quality check
score = image_util.calc_quality(img_path)
if score < quality_thres:
utils.remove_file(img_path)
continue
# put to feature detect que
task = {}
task['img_path'] = img_path
self.que_manager.putQue(json.dumps(task), utils.msg_queues['feature_detect'])
except Exception as e:
log_util.error("ImageETL run error: %s", str(e))
time.sleep(0.5)
def getWeixinData(openID, accessToken): #获得微信信息
url = "https://api.weixin.qq.com/sns/userinfo?access_token=%s&openid=%s" % (
accessToken, openID)
message = getUrlMessage(url)
if 'errcode' in message:
log_util.error('weixin get player data error: %s' % message)
return False
else:
return message
def checkWeixinCode4fish(code): #捕鱼校验微信code
url = "https://api.weixin.qq.com/sns/oauth2/access_token?appid=%s&secret=%s&code=%s&grant_type=authorization_code"\
%(APPID_FISH, SECRET_FISH, code)
message = getUrlMessage(url)
if 'errcode' in message:
log_util.error('weixin check code error:[%s]' % message)
return False
else:
return message
def getUrlMessage(getUrl):
socket.setdefaulttimeout(WAIT_WEB_TIME)
try:
url = urllib.urlopen(getUrl)
data = url.read()
message = json.loads(data)
except Exception, e:
log_util.error('get weixin message error: %s' % e)
message = {'errcode': 123}
def checkAccessToken(openID, access_token): #校验accessToken
url = "https://api.weixin.qq.com/sns/auth?access_token=%s&openid=%s" % (
access_token, openID)
message = getUrlMessage(url)
if message['errcode']:
log_util.error('weixin check access token error: %s' % message)
return False
else:
return True
def connect(self):
try:
return pymysql.connect(host='rm-j6c4883wd5345860apo.mysql.rds.aliyuncs.com',
user='******',
password='******',
db='image_db',
cursorclass=pymysql.cursors.DictCursor)
except Exception as e:
log_util.error("DBOperation::connect fail: %s", str(e))
def onRefreshToken(refreshToken): #刷新accessToken
url = "https://api.weixin.qq.com/sns/oauth2/refresh_token?appid=%s&grant_type=refresh_token&refresh_token=%s" % (
APPID, refreshToken)
message = getUrlMessage(url)
if 'access_token' in message:
return message['access_token']
else:
log_util.error('weixin refresh token error: %s' % message)
return False
def execute(self, sql):
try:
# check connection alive (if not, ping will reconnect)
self.connection.ping()
cursor = self.connection.cursor()
log_util.debug("DBOperation::execute [%s]", sql)
cursor.execute(sql)
self.connection.commit()
except Exception as e:
log_util.error("DBOperation::execute fail: %s", str(e))
def deleteMsg(redis,msgId,memberId):
"""
从玩家列表中移除
"""
userMsgBox = FORMAT_USER_MESSAGE_LIST%(memberId)
readList = redis.smembers(FORMAT_MSG_READ_SET%(msgId))
pipe = redis.pipeline()
try:
pipe.lrem(userMsgBox,msgId)
except Exception,e:
log_util.error('[try deleteMsg] ERROR msgId[%s] memberId[%s] errReason[%s]'%(msgId,memberId,e))
return
def do_take_benefit(redis,session):
""" 救济金补领接口 """
fields = ('sid','token')
for field in fields:
exec(consts.FORMAT_PARAMS_POST_STR%(field,field))
SessionTable,account,uid,verfiySid = getInfoBySid(redis,sid)
check_code,check_msg,user_table,osid = check_session_verfiy(redis,'/fish/getRewardInfo/',SessionTable,account,sid,verfiySid)
# ----------暂时可无限领取弹头救济金-----------------
'''
warhead = redis.hget('users:%s'%uid,'warhead')
warhead = int(warhead or 0)
if warhead < 1000:
redis.hincrby('users:%s'%uid,20000)
sendProtocol2AllGameService(redis, HEAD_SERVICE_PROTOCOL_MEMBER_REFRESH%(account),game="FISH")
return {'code':0,'msg':'领取救济金成功'}
'''
# ----------暂时可无限领取弹头救济金-----------------
log_util.debug('[try do_refresh] check_code[%s] check_msg[%s]'%(check_code,check_msg))
if int(check_code)<0:
if check_code == -4:
return {'code':check_code,'msg':check_msg,'osid':osid}
return {'code':check_code,'msg':check_msg}
server_token = redis.get(USER_BENEFIT_TOKEN%(uid))
if token != server_token:
log_util.info('[do_take_benefit] client_token[%s] server_token[%s]'%(token,server_token))
return {'code':-9003,'msg':'不能重复领取!'}
code,msg,benefit_info = can_take_benefit(redis,uid,user_table)
if code < 0:
log_util.debug('[do_take_benefit] can\'t take benefit code[%s] msg[%s]'%(code,msg))
return {'code':code,'msg':msg}
_datetime = time.strftime("%Y-%m-%d", time.localtime())
try:
#领取救济金
pipe = redis.pipeline()
pipe.hincrby(FISH_BENEFIT_COIN_TABLE % _datetime ,uid,1)
pipe.hincrby(user_table,'coin',benefit_info['benefit_coin'])
except Exception,e:
log_util.error('[do_take_benefit] take benefit error[%s]'%(e))
return {'code':-9000,'msg':'领取救济金失败!'}
def before_request_check():
req_path = request.path
req_ip = web_util.get_ip()
if req_path in self.check_urls:
sid = request.params.sid
log_util.debug('before req_ip[%s] request_path[%s] sid[%s] redis[%s]'%(req_ip,req_path,sid,self.redis_inst))
# if not sid:
# """ 在接口请求中不携带SID的都是非法 """
# log_util.error('req_ip[%s] req_path[%s] sid[%s] is illg request...'%(req_ip,req_path,sid))
# bottle.abort(503,'Invalid Request')
if not web_util.api_limit_checker(self.redis_inst,sid,req_ip,req_path):
""" 检查是否恶意访问 """
log_util.error('req_ip[%s] req_path[%s] sid[%s] is attack...'%(req_ip,req_path,sid))
bottle.abort(503,'Invalid Request')
def create_fish_room(redis,roomInfo):
"""
创建捕鱼房间
:params: redis,
:params: roomInfo(创建的房间信息)
"""
channel_id = roomInfo['room_id']
room_table = FISH_ROOM_TABLE%(channel_id)
log_util.debug('[try create_fish_room] room_id[%s] room_table[%s]'%(channel_id,room_table))
pipe = redis.pipeline()
try:
pipe.hmset(room_table,roomInfo)
pipe.sadd(FISH_ROOM_ID_SETS,channel_id)
pipe.sadd(GAMEID_SET,channel_id)
pipe.lpush(FISH_ROOM_LIST,channel_id)
except Exception,e:
log_util.error('[try create_fish_room] room_id[%s] create error.reason[%s]'%(channel_id,e))
return
def ingest_one(self, img_path, im_type):
try:
log_util.debug("start to ingest one image: %s", img_path)
ret = False
retry = 0
# upload to oss. img in table is a relative oss storage path
# retry 3 times
target = oss2util.get_oss_path(img_path)
while ret is False and retry < 3:
ret = oss2util.uploadFile(img_path, target)
retry += 1
if ret:
# move file to correct place, named by md5
file_name = target.split('/')[-1]
sub_folder = self.get_sub_folder(img_path)
dest_file_path = os.path.join(utils.MEDIA_ROOT, sub_folder,
file_name)
utils.move_file(img_path, dest_file_path)
relative_path = os.path.join(sub_folder, file_name)
# insert image meta to database
meta = image_util.get_image_meta(img_path)
# insert into db
table = 'static_image'
if im_type == 'gif':
table = 'animated_image'
params = []
values = []
for k, v in meta.items():
params.append(k)
values.append(v)
params.append('create_time')
values.append(
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
#TODO add iamge original name to description
sql = "insert into %s %s values %s" % (table, str(params),
str(values))
self.db_con.execute(sql)
except Exception as e:
log_util.error("ingest image[%s] fail: %s", img_path, str(e))
def do_create_game(redis,game_id,gameInfo):
"""
创建新游戏
@param:
redis redis链接实例
gameInfo 游戏信息
"""
if game_id in GAMEID_SET:
return None
game_table = GAME_TABLE%(game_id)
pipe = redis.pipeline()
try:
pipe.hmset(game_table,gameInfo)
pipe.sadd(GAMEID_SET,game_id)
pipe.lpush(GAME_LIST,game_id)
except Exception,e:
log_util.error('[try do_create_game] reason[%s]'%(e))
return None
def delete_fishroom(redis,room_id):
"""
删除捕鱼房间
:params redis 数据库实例
:params room_id 捕鱼房间ID
"""
room_table = FISH_ROOM_TABLE%(room_id)
if not redis.exists(room_table):
log_util.debug('[try delete_fishroom] room_table[%s] already delete.'%(room_table))
return
pipe = redis.pipeline()
try:
pipe.delete(room_table)
pipe.delete(FISH_BET_DATA4ROOM%(room_table))
pipe.lrem(FISH_ROOM_LIST,room_id)
pipe.srem(GAMEID_SET,room_id)
for day_key in redis.keys(FISH_BET_DATA4DAY4ROOM%(room_id,'*')):
pipe.delete(day_key)
except Exception,e:
log_util.error('[try delete_fishroom] delete room[%s] error.reason[%s]'%(room_id,e))
return
def do_reward_deleteCall(redis,session):
"""
删除奖品信息接口
:params reward_id 兑换奖品ID
"""
fields = {
('reward_id','奖品ID','')
}
for field in fields:
exec('%s = web_util.get_form("%s","%s","%s")'%(field[0],field[0],field[1],field[2]))
reward_table = FISH_REWARD_TABLE%(reward_id)
if not redis.exists(reward_table):
return web_util.do_response(1,'商品不存在.')
reward_status = convert_util.to_int(redis.hget(reward_table,'reward_status'))
if reward_status == REWARD_ONLINE:
return web_util.do_response(1,'不能删除已上架的商品.')
try:
do_reward_delete(redis,reward_id)
except Exception,e:
log_util.error('[do_reward_delete] reward_id[%s] delete error.reason[%s]'%(reward_id,e))
return web_util.do_response(1,msg="删除奖品失败!")
def run(self):
while True:
try:
# get task from finish que
task = self.que_manager.getQue(self.get_que_name)
if task is None:
time.sleep(0.5)
continue
log_util.debug("get one task from ingest queue: %s", task)
# ingest or delete image
task = json.loads(task.decode())
if task['ingest']:
# ingest image
im_type = 'jpg'
if image_util.is_gif(task['img_path']):
im_type = 'gif'
self.ingest_one(task['img_path'], im_type)
else:
# delete image
utils.remove_file(task['img_path'])
except Exception as e:
log_util.error("Ingestor run error: %s", str(e))
import os
import sys
import time
sys.path.append(os.path.join(os.path.dirname(__file__)))
sys.path.append(os.path.join(os.path.dirname(__file__), os.pardir))
from common import utils, log_util
from common.msg_que import MsgQueue
from . import image_etl, ingestor
log_util.debug("Starting ingest main service...")
# declare msg queues
mq = MsgQueue()
for v in utils.msg_queues.values():
mq.createQueue(v)
# crawler notify queue handler
img_etl = image_etl.ImageETL(mq, utils.msg_queues['crawler_notify'], utils.msg_queues['feature_detect'])
img_etl.start()
# feature detect handler
ingestor_service = ingestor.Ingestor(mq, utils.msg_queues['detect_finish'])
ingestor_service.start()
img_etl.join()
ingestor_service.join()
log_util.error("!!! Ingest main service exited.")
def remove_file(file_path):
try:
cmd = "rm " + "\"" + file_path + "\""
os.system(cmd)
except Exception as e:
log_util.error("remove file[%s] error: %s", file_path, str(e))
def move_file(src_file, dest_file):
try:
cmd = "mv " + "\"" + src_file + "\"" + dest_file
os.system(cmd)
except Exception as e:
log_util.error("move file[%s] error: %s", cmd, str(e))