def run(self):
print 'run HiveWorker'
self.logger.debug('worker [' + self.getName() + '] run')
hiveClient = None
try:
hiveClient = HiveService(self.host, self.port)
processor = HiveResponseProcessor(task=self.task)
data = []
for query in self.workerService.querys:
print 'run {}'.format(query)
dataPart = hiveClient.execute(query)
print 'Data responsed'
#print dataPart
data += dataPart
#print data
print 'process data'
data = {'result' : processor.prepareData(data)}
print 'end process data'
self.logger.debug('worker [' + self.getName() + '] end job')
status = Worker.STATUS_SUCCESS
except Thrift.TException as tx:
data = {'exception' : {'Thrift.TException' : tx.message }}
status = Worker.STATUS_ERROR
self.logger.error(tx.message)
except TTransportException as tx:
data = {'exception' : {u'Ошибка связи с Hive' : tx.message }}
status = Worker.STATUS_ERROR
self.logger.error(tx.message)
except HiveServerException as tx:
data = {'exception' : {'HiveServerException': tx.message }}
status = Worker.STATUS_ERROR
self.logger.error(tx.message)
except Exception as tx:
print type(tx)
print tx.message
print tx.args
data = {'exception': {'Exception': str(tx.message)}}
status = Worker.STATUS_ERROR
self.logger.error(tx.message)
finally:
if hiveClient:
hiveClient.close()
worker = self.workerService.getWorker()
session = self.mysqlSessionMaker()
worker.endDate = datetime.now()
worker.status = status
session.merge(worker)
session.commit()
session.close()
self.workerService.saveResult(data, self.version, worker.startDate, worker.endDate)
#!/usr/bin/python
from components.HiveResponseProcessor import HiveResponseProcessor
from services.WorkerService import WorkerService
from models.Worker import Worker
import os, inspect, re
task = 1
_file = '/../TestHiveResponse'
thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
resultPath = os.path.abspath(thisPath + '/../result/')
worker = Worker()
worker.workerId = task
workerService = WorkerService(resultPath, worker)
task = workerService.getTask()
r = re.compile('\s+')
f = open(os.path.abspath(thisPath + _file), 'r')
data = f.readlines()
data = [r.split(line) for line in data]
processor = HiveResponseProcessor(task)
processor.prepareData(data)
