def get(self, *args, **kwargs):
app = self.checkAppAccess(args)
workerId = self.get_argument("workerId")
workerId = int(workerId)
dbSession = self.getDBSession()
# загружаем воркер из базы
worker = dbSession.query(Worker).filter_by(workerId=workerId).first()
worker.status = Worker.STATUS_ALIVE
worker.startDate = datetime.now()
worker.endDate = None
dbSession.add(worker)
dbSession.commit()
# создаем WorkerService и загружаем его данные
workerService = WorkerService(self.application.getResultPath(), worker)
workerService.load()
workerService.version += 1
workerService.init()
# создаем и запускаем тред
workerThread = self.createHiveWorker(workerService)
workerThread.start()
self.redirect("/dashboard/app/" + app.code + "/")
def get(self, *args, **kwargs):
appcode, workerId = args
aliveThreadNames = ThredService.getAliveThreads()
if not "worker-" + workerId in aliveThreadNames:
self.renderJSON({"redirect": "status/" + workerId})
else:
dbSession = self.getDBSession()
# загружаем воркер из базы
worker = dbSession.query(Worker).filter_by(workerId=int(workerId)).first()
workerService = WorkerService(self.application.getResultPath(), worker)
workerService.load()
self.renderJSON(
{"html": self.render("/dashboard/result/taskAlive.jinja2", query=workerService.query, _return=True)}
)
def get(self, *args, **kwargs):
app = self.checkAppAccess(args)
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(app.code)
dbSession = self.getDBSession()
baseTask = self.get_argument("baseOn", False)
template = self.get_argument("template", False)
nameConstructor = NameConstructor(appConfig)
if baseTask:
# задача основана на другой задаче
worker = dbSession.query(Worker).filter_by(workerId=baseTask).first()
workerService = WorkerService(self.application.getResultPath(), worker)
task = workerService.getTask()
# fix indexes
for index, taskItem in enumerate(task.getTaskItems()):
taskItem.index = index
if not taskItem.name:
taskItem.name = nameConstructor.getTaskItemName(taskItem)
elif template:
templateId = int(template)
userId = self.get_current_user().userId
taskTemplate = (
dbSession.query(TaskTemplate)
.filter(
and_(
TaskTemplate.taskTemplateId == templateId,
or_(TaskTemplate.userId == userId, TaskTemplate.shared == TaskTemplate.SHARED_YES),
)
)
.first()
)
if taskTemplate:
taskTemplateFile = TaskTemplateFile.read(self.application.getTemplatePath(), taskTemplate)
task = taskTemplateFile.getTask()
else:
self.showFatalError(u"Ошибка при доступе к шаблону")
return
else:
task = Task(appname=app.code)
eventsLoaded = len(task.items)
html = self.render("dashboard/new.jinja2", {"task": task, "app": app, "appConfig": appConfig}, _return=True)
self.renderJSON({"html": html, "vars": {"eventLoaded": eventsLoaded}})
def get(self, *args, **kwargs):
appCode, taskId = args
worker = self.getWorker(int(taskId))
service = WorkerService(self.application.getResultPath(), worker)
data = service.getResultData()["data"]["result"]
task = service.getTask()
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(appCode)
nameService = NameConstructor(appConfig, task)
tableService = TableConstructor(data, nameService, task)
tableHeaders, tableData = tableService.getVerticalData()
self.render(
"dashboard/result/table2.jinja2", tableData=tableData, tableHeaders=tableHeaders, interval=task.interval
)
def get(self, *args, **kwargs):
appCode, workerId = args
worker = self.getWorker(workerId)
if worker.status == Worker.STATUS_ALIVE:
self.renderJSON({"html": self.render("dashboard/result/taskAlive.jinja2", _return=True)})
elif (worker.status == Worker.STATUS_ERROR) or (worker.status == Worker.STATUS_DIED):
service = WorkerService(self.application.getResultPath(), worker)
self.renderJSON(
{
"html": self.render(
"dashboard/result/taskFailed.jinja2", {"errors": [service.getError()]}, _return=True
)
}
)
elif worker.status == Worker.STATUS_SUCCESS:
self.renderJSON({"redirect": "result/job=" + workerId})
else:
raise Exception("Unknown worker state")
def getWorkerList(self, app, page = 1):
db_session = self.getDBSession()
user = self.get_current_user()
perPage = 5
# получаем количество
count, = db_session.query(func.count(Worker.workerId)).filter(Worker.userId == user.userId, Worker.appId == app.appId).first()
pageCount = int(math.ceil(float(count) / perPage))
# получаем список последних запросов
lastWorkers = db_session.query(Worker).filter(Worker.userId == user.userId, Worker.appId == app.appId).order_by(desc(Worker.startDate)).offset(perPage * (page - 1)).limit(perPage)
# определяем, какие воркеры уже умерли
workers = []
alivedWorkers= []
for worker in lastWorkers:
if worker.status == Worker.STATUS_ALIVE:
alivedWorkers.append(worker)
workers.append(worker)
aliveThreadNames = ThredService.getAliveThreads()
for worker in alivedWorkers:
# определяем мертвые воркеры
if not 'worker-' + str(worker.workerId) in aliveThreadNames:
worker.status = Worker.STATUS_DIED
db_session.add(worker)
alivedWorkers.remove(worker)
db_session.commit()
# для живых воркеров загружаем таски
if alivedWorkers:
workerService = WorkerService(self.application.getResultPath())
for worker in alivedWorkers:
workerService.setWorker(worker)
worker.task = workerService.getTask()
return workers, pageCount
def get(self, *args, **kwargs):
# get app code
if len(args) == 0:
raise RuntimeError("Cant find app code in dashboard")
appCode = args[0]
user = self.get_current_user()
# get app
app = self.dbSession.query(App).filter(App.code == appCode).first()
if app is None:
raise RuntimeError("Cant find app by code " + appCode)
# check access
ruleService = RuleService(self.dbSession)
if not ruleService.isAllow(user.userId, app.appId):
raise RuntimeError("Access denied")
workerId = self.get_argument("jobId")
worker = self.getWorker(workerId)
service = WorkerService(self.application.getResultPath(), worker)
if worker.status != Worker.STATUS_SUCCESS:
self.renderJSON({"redirect": "status/" + str(workerId)})
return
# configuration name services
task = service.getTask()
startDates = []
endDates = []
for i in task.items:
taskItem = task.items[i]
startDates.append(taskItem.start)
endDates.append(taskItem.end)
minStartDate = min(startDates)
maxEndDate = max(endDates)
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(app.code)
nameService = NameConstructor(appConfig, task)
try:
data = service.getResultData()["data"]["result"]
if len(data.items()) == 0:
self.render("dashboard/result_no_data.jinja2", {"app": app})
return
# chart data
chartService = ChartConstructor(data, nameService, task)
tableService = TableConstructor(data, nameService, task)
except IOError as ioerr:
self.render(
"dashboard/result.jinja2", {"errors": [u"Ошибка чтения результатов выполнения работы"], "app": app}
)
else:
html = self.render(
"dashboard/result.jinja2",
{
"app": app,
"data": data,
"tablesdata": tableService.getData(),
"nameService": nameService,
"chartService": chartService,
"name": worker.name,
"interval": task.interval,
"tagCloud": chartService.getTagCloud(),
"workerId": workerId,
},
_return=True,
)
self.renderJSON(
{
"html": html,
"vars": {
"chartdata": chartService.getResult(),
"interval": task.interval,
"minStartDate": time.mktime(minStartDate.timetuple()),
"maxEndDate": time.mktime(maxEndDate.timetuple()),
"taskId": workerId,
},
}
)
#!/usr/bin/python
from components.HiveResponseProcessor import HiveResponseProcessor
from services.WorkerService import WorkerService
from models.Worker import Worker
import os, inspect, re
task = 1
_file = '/../TestHiveResponse'
thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
resultPath = os.path.abspath(thisPath + '/../result/')
worker = Worker()
worker.workerId = task
workerService = WorkerService(resultPath, worker)
task = workerService.getTask()
r = re.compile('\s+')
f = open(os.path.abspath(thisPath + _file), 'r')
data = f.readlines()
data = [r.split(line) for line in data]
processor = HiveResponseProcessor(task)
processor.prepareData(data)
def post(self, *args, **kwargs):
app = self.checkAppAccess(args)
saveAsTemplate = str(self.get_argument("saveAsTemplate", default=""))
# создаем задачу из аргументов
task = createTaskFromRequestArguments(self.request.arguments)
session = self.getDBSession()
if saveAsTemplate:
# сохраняем шаблон
taskTemplate = TaskTemplate()
taskTemplate.appId = app.appId
taskTemplate.name = self.get_argument("taskName", default=None)
taskTemplate.userId = self.get_current_user().userId
taskTemplate.shared = TaskTemplate.SHARED_NO
session.add(taskTemplate)
session.commit()
# скидываем на диск
taskTemplateFile = TaskTemplateFile(
self.application.getTemplatePath(), task=task, taskTemplate=taskTemplate
)
taskTemplateFile.baseDate = datetime.now()
taskTemplateFile.save()
self.redirect("/dashboard/app/{}/#templates".format(app.code))
return
user = self.get_current_user()
# объект для записи в базу
worker = Worker()
worker.userId = user.userId
worker.startDate = datetime.now()
worker.status = Worker.STATUS_ALIVE
worker.appId = app.appId
worker.name = self.get_argument("taskName", default=None)
# генерируем имя запроса
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getAppConfig(app.code)
nameConstructor = NameConstructor(appConfig, task)
if not worker.name:
worker.name = nameConstructor.generateTaskName()
session.add(worker)
session.commit()
# конструирем запрос
constructor = HiveQueryConstructor(task, appConfig)
querys = constructor.getHiveQuerys(worker.workerId)
# task.stageCount = constructor.getStageCount()
# создаем WorkerService - он будет связывать тред с файловой системой
workerService = WorkerService(self.application.getResultPath(), worker)
workerService.setQuery(querys)
workerService.setTask(task)
workerService.init()
# создаем и запускаем тред
workerThread = self.createHiveWorker(workerService)
workerThread.start()
self.redirect("/dashboard/app/" + app.code + "/#new_task/" + str(worker.workerId))