コード例 #1
0
ファイル: HiveWorker.py プロジェクト: x100up/analystics
    def run(self):
        print 'run HiveWorker'
        self.logger.debug('worker [' + self.getName() + '] run')
        hiveClient = None
        try:
            hiveClient = HiveService(self.host, self.port)
            processor = HiveResponseProcessor(task=self.task)
            data = []
            for query in self.workerService.querys:
                print 'run {}'.format(query)
                dataPart = hiveClient.execute(query)
                print 'Data responsed'
                #print dataPart
                data += dataPart

            #print data
            print 'process data'
            data = {'result' : processor.prepareData(data)}
            print 'end process data'
            self.logger.debug('worker [' + self.getName() + '] end job')
            status = Worker.STATUS_SUCCESS
        except Thrift.TException as tx:
            data = {'exception' : {'Thrift.TException' : tx.message }}
            status = Worker.STATUS_ERROR
            self.logger.error(tx.message)
        except TTransportException as tx:
            data = {'exception' : {u'Ошибка связи с Hive' : tx.message }}
            status = Worker.STATUS_ERROR
            self.logger.error(tx.message)
        except HiveServerException as tx:
            data = {'exception' : {'HiveServerException': tx.message }}
            status = Worker.STATUS_ERROR
            self.logger.error(tx.message)
        except Exception as tx:
            print type(tx)
            print tx.message
            print tx.args
            data = {'exception': {'Exception': str(tx.message)}}
            status = Worker.STATUS_ERROR
            self.logger.error(tx.message)
        finally:
            if hiveClient:
                hiveClient.close()

        worker = self.workerService.getWorker()
        session = self.mysqlSessionMaker()
        worker.endDate = datetime.now()
        worker.status = status
        session.merge(worker)
        session.commit()
        session.close()
        self.workerService.saveResult(data, self.version, worker.startDate, worker.endDate)
コード例 #2
0
 def query(self, query):
     host = self.getConfigValue(Config.HIVE_HOST)
     port = self.getConfigValue(Config.HIVE_PORT)
     hiveClient = HiveService(host, port)
     return hiveClient.execute(query)