コード例 #1
0
ファイル: fileconsumer.py プロジェクト: firren123/log
class FileConsumer(object):
	"""
	读取指定目录的文件
	"""
	def __init__(self):
		# zip 文件 path
		config = json.load(open(u'../config/config.json'))
		self.redisConf = config['redis']
		self.data = []
		self.lines = []
		self.redis = redis.Redis(host='127.0.0.1',port=6379,db=0)
		self.storage = Storage(config['db'])
		# self.initSelf()

	def progress(self):

		f = self.redis.rpop('logfile')
		if f and zipfile.is_zipfile(f):
			self.read_zip_file(f)
			if len(self.lines) > 0:
				try:
					# 分析每行数据
					#import pdb; pdb.set_trace()
					pubData = json.loads(self.lines.pop(0).decode('utf-8'),encoding="utf-8")
					for evt  in self.lines:
						evt = json.loads(evt.decode('utf-8'),encoding='utf-8')
						param = json.dumps(evt['param'],encoding="utf-8", ensure_ascii=False)
						_row = (evt['event'],param,str(pubData['uid']),pubData['deviceid'],pubData['devicemodel'],str(pubData['appid']),pubData['appversion'],pubData['os'],pubData['osversion'],time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(int(evt['timestamp']))),pubData['appname'],pubData['appbuildversion'])
						self.data.append(_row)

					if len(self.data) > 0 :
						self.storage.dbConnect()
						self.saved()
						self.storage.dbClose()
					os.remove(f)
				except OSError:
					pass

	def saved(self):
		if self.data is None or len(self.data) == 0:
			return False
		try:
			self.storage.save(self.data)
		except Exception as e:
			print("执行Mysql: 时出错:%s data is:%s" % ( e ,self.data))


	def read_zip_file(self,filepath):
		zfile = zipfile.ZipFile(filepath)
		for finfo in zfile.infolist():
			ifile = zfile.open(finfo)
			self.lines = ifile.readlines()

# if __name__ == "__main__":
# 	f = FileConsumer()
# 	f.progress()
コード例 #2
0
ファイル: consumer.py プロジェクト: firren123/log
class Consumer:
    """
    获取redis的数据,取出后做处理
    """
    def __init__(self):
        self.config = json.load(open(u'../config/config.json'))
        self.redisConf = self.config['redis']
        self.queue_key = self.redisConf['queue_key']
        # 链接redis
        # self.redis = redis.Redis(host=str(self.redisConf['server']),port=int(self.redisConf['port']),db=int(self.redisConf['db']))
        self.redis = redis.Redis(host='127.0.0.1',port=6379,db=0)
        logging.basicConfig(filename = self.config['error_log'], level = logging.DEBUG)
        # self.db = None
        self.storage = Storage(self.config['db'])
        self.num = 50
        self.data = []

    def progress(self):
        #import pdb; pdb.set_trace()
        start = 0
        len_events = self.redis.llen('log')
        if int(len_events) > 0:
            self.data = []
            while start < min(len_events,self.num):
                _data = self.redis.rpop('log')
                self.parseData(_data)
                start+=1

            if len(self.data) > 0 :
                self.storage.dbConnect()
                self.saved()
                self.storage.dbClose()
        else:
            return


    def parseData(self,dataStr):
        """
        分析数据
        :return:
        """

        from urllib import unquote
        test = unquote(dataStr)
        dataStr = test
        if dataStr is None:
            return None
        try:
            data = json.loads(dataStr.decode('utf-8'),encoding="utf-8")
            events = data['events']
            if not isinstance(events,list):
                events = [events]
            for dict in events:
                param = json.dumps(dict['param'],encoding="utf-8", ensure_ascii=False)
                _row = (dict['event'],param,str(data['uid']),data['deviceid'],data['devicemodel'],str(data['appid']),data['appversion'],data['os'],data['osversion'],time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(int(dict['timestamp']))),data['appname'],data['appbuildversion'])
                #_row = (str(data['event']),param,str(data['uid']),data['deviceid'],data['deviceModel'])
                self.data.append(_row)
        except Exception as e:
            logging.error("convert data to json object error! --data is :" + dataStr + "; \nerror is " + str(e)  + ";\n===============================================================\n")
            return None
        # finally:



    def saved(self):
        """
        :return : bool
        """
        #连接数据库

        if self.data is None or len(self.data) == 0:
            return False
        if isinstance(self.data,type({})):
            self.singleSave()
            print("执行Mysql: 时出错:%s data is:%s" % ( e ,self.data))
        elif isinstance(self.data,(tuple, list)):
            """
            批量更新数据
            """
            self.mutilSave()

    def singleSave(self,row):
        """
        保存数据到数据库中
        :param data:
        :return:
        """
        if row is None or row =='':
            return
        print (row)

    def mutilSave(self):
        """
        批量插入数据
        :param data:
        :return:
        """
        # cursor = self.db.cursor()
        try:
            self.storage.save(self.data)
        except Exception as e:
            print("执行Mysql: 时出错:%s data is:%s" % ( e ,self.data))