def getHttpReq(self, type, actid):
if responseDict.has_key(str(actid) + ',' + type):
return responseDict[str(actid) + ',' + type]
else:
host = self.host
port = self.port
url = '/vote/themetotal/getnum?pid=heyinliang&tid=' + type + '&actid=' + str(
actid) + '&orderby=hot&desc=1&start=0'
httpClient = None
try:
httpClient = httplib.HTTPConnection(host, port, timeout=30)
httpClient.request('GET', url)
#logging.info('the request is %s','http://'+host+':'+port+url)
#response是HTTPResponse对象
response = httpClient.getresponse()
res = response.read()
logging.info('the request is %s and the response info is %s',
'http://' + host + ':' + port + url, res)
responseDict[str(actid) + ',' + type] = res
except Exception, e:
logging.error('http request error,the error info is %s', e)
finally:
def __init__(self):
try:
redisconfig = config['redisconf']
self.pool = redis.ConnectionPool(host=redisconfig['host'],
port=redisconfig['port'],
db=0)
self.Redis = redis.Redis(connection_pool=self.pool)
except Exception, e:
logging.error('conn redis error,the error info is %s', e)
sys.exit(1)
def __getConn(dbconfig):
"""
@summary: 静态方法，从连接池中取出连接
@return MySQLdb.connection
"""
if Mysql.__pool is None:
try:
__pool = PooledDB(creator=MySQLdb, mincached=1 , maxcached=20,host=dbconfig['host'] , port=dbconfig['port'] , user=dbconfig['user'] , passwd=dbconfig['password'] ,db=dbconfig['db'],use_unicode=False,charset=dbconfig['charset'],cursorclass=DictCursor,setsession=['SET AUTOCOMMIT = 1'])
except Exception, e:
logging.error('conn db error,the error info is %s',e)
sys.exit(1)
def __query(self,sql,param=None):
count = 0
try:
if param is None:
count = self._cursor.execute(sql)
else:
count = self._cursor.execute(sql,param)
self._conn.commit()
except Exception, e:
logging.error('exec db command error,the error info is [%s]',e)
sys.exit(1)
WORK_NODE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# print CURRENT_DIR
os.sys.path.insert(0, os.path.dirname(WORK_NODE_DIR))
from src.cores import dbutils
from src.cores import my_logger as logging
if __name__ == '__main__':
params = len(sys.argv)
logging.info('the len of the params is %s',params)
if params <= 1:
logging.error('the py script params is error')
sys.exit(0)
db = dbutils.dbutils()
actids = db.getAct()
for id in actids:
cmd = 'sh user_act.sh '+ sys.argv[1] +' '+ str(id)
logging.info('begin to exec cmd [%s]',cmd)
status,output = commands.getstatusoutput(cmd)
if status == 0 :
logging.info('the cmd [%s] exec finished,the result is [%s]',cmd,output)
else:
logging.error('the cmd [%s] exec error,the status is %s,and the result is [%s]',cmd,status,output)
key = redis.lindex('default_' + str(actid) + '_' + str(num), index)
#logging.debug('the refvalue of the redis (key:%s) is %s','default_'+str(actid)+'_'+str(num),key)
for sid in songids:
pipe.linsert('default_' + str(actid) + '_' + str(num),
'before', key, sid)
#logging.debug('begin to insert data %s to redis ,the key is %s',sid,'default_'+str(actid)+'_'+str(num))
#logging.info('insert data to redis end ,the key is %s','default_'+str(actid)+'_'+str(num))
for actid, songids in delDict.items():
songids = list(set(songids))
for num in range(1, 1001):
#logging.info('begin to delete from redis ,the key is %s','default_'+str(actid)+'_'+str(num))
for sid in songids:
pipe.lrem('default_' + str(actid) + '_' + str(num), sid, 0)
#logging.debug('begin to delete %s from redis ,the key is %s',sid,'default_'+str(actid)+'_'+str(num))
pipe.execute()
logging.info('begin to update the task status ,the taskid is %s', taskids)
r = 1
if len(taskids) > 0:
for tid in taskids:
i = db.updateTaskStatus(tid)
r = r & i
if r == 1:
logging.info('update task status success !')
else:
logging.error('upate task status failed !')
else:
logging.info('no task to update status ')
def parselog():
u1 = r"?P<u1>\S+"
u2 = r"?P<u2>\S+"
u3 = r"?P<u3>\S+"
u4 = r"?P<u4>\S+"
u5 = r"?P<u5>\S+"
u6 = r"?P<u6>\S+"
u7 = r"?P<u7>\S+"
u8 = r"?P<u8>\S+"
u9 = r"?P<u9>\S+"
u10 = r"?P<u10>\S+"
u11 = r"?P<u11>\S+"
u12 = r"?P<u12>\S+"
u13 = r"?P<u13>\S+"
info = r"?P<info>\S+"
p = re.compile(
r"(%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s)\ (%s).*?"
% (u1, u2, u3, u4, u5, u6, u7, u8, u9, u10, u11, u12, u13, info),
re.VERBOSE)
listen_log_path = config['listen_log_path']
lasthour = ((datetime.datetime.now() -
datetime.timedelta(hours=1)).strftime("%Y%m%d%H"))
listen_log_path = listen_log_path % lasthour
if os.path.isfile(listen_log_path) == False:
logging.error('the listen_log_path [%s] no exist', listen_log_path)
sys.exit(1)
return
else:
logging.info('begin to parse log info %s ', listen_log_path)
logfile = open(listen_log_path)
listendict = {}
while True:
line = logfile.readline()
if not line:
break
if 'log_song_play' not in line:
continue
info = line.split(' ')[15]
#logging.debug('parse log , the info is %s' , info)
info = info.replace('log[', '').replace(']', '')
logs = info.split('_')
#logging.debug('parse ,the parsed log info is %s',logs)
topicid = logs[0]
actid = logs[1]
songid = logs[2]
userid = logs[3]
userlevel = logs[4]
key = topicid + ',' + actid + ',' + songid
if listendict.has_key(key):
listendict[key] = listendict[key] + 1
else:
listendict[key] = 1
#print 'not contains'
logging.info('parse log %s finish,the result of the listen num is %s',
listen_log_path, listendict)
return listendict