Esempio n. 1
0
    def get(self, *args, **kwargs):
        app = self.checkAppAccess(args)
        workerId = self.get_argument("workerId")
        workerId = int(workerId)
        dbSession = self.getDBSession()

        # загружаем воркер из базы
        worker = dbSession.query(Worker).filter_by(workerId=workerId).first()
        worker.status = Worker.STATUS_ALIVE
        worker.startDate = datetime.now()
        worker.endDate = None
        dbSession.add(worker)
        dbSession.commit()

        # создаем WorkerService и загружаем его данные
        workerService = WorkerService(self.application.getResultPath(), worker)
        workerService.load()
        workerService.version += 1
        workerService.init()

        # создаем и запускаем тред
        workerThread = self.createHiveWorker(workerService)
        workerThread.start()

        self.redirect("/dashboard/app/" + app.code + "/")
Esempio n. 2
0
    def get(self, *args, **kwargs):
        appcode, workerId = args
        aliveThreadNames = ThredService.getAliveThreads()
        if not "worker-" + workerId in aliveThreadNames:
            self.renderJSON({"redirect": "status/" + workerId})
        else:
            dbSession = self.getDBSession()
            # загружаем воркер из базы
            worker = dbSession.query(Worker).filter_by(workerId=int(workerId)).first()
            workerService = WorkerService(self.application.getResultPath(), worker)
            workerService.load()

            self.renderJSON(
                {"html": self.render("/dashboard/result/taskAlive.jinja2", query=workerService.query, _return=True)}
            )
Esempio n. 3
0
    def get(self, *args, **kwargs):
        app = self.checkAppAccess(args)
        appService = AppService(self.application.getAppConfigPath())
        appConfig = appService.getNewAppConfig(app.code)
        dbSession = self.getDBSession()
        baseTask = self.get_argument("baseOn", False)
        template = self.get_argument("template", False)
        nameConstructor = NameConstructor(appConfig)
        if baseTask:
            # задача основана на другой задаче
            worker = dbSession.query(Worker).filter_by(workerId=baseTask).first()
            workerService = WorkerService(self.application.getResultPath(), worker)
            task = workerService.getTask()
            # fix indexes
            for index, taskItem in enumerate(task.getTaskItems()):
                taskItem.index = index
                if not taskItem.name:
                    taskItem.name = nameConstructor.getTaskItemName(taskItem)

        elif template:
            templateId = int(template)
            userId = self.get_current_user().userId
            taskTemplate = (
                dbSession.query(TaskTemplate)
                .filter(
                    and_(
                        TaskTemplate.taskTemplateId == templateId,
                        or_(TaskTemplate.userId == userId, TaskTemplate.shared == TaskTemplate.SHARED_YES),
                    )
                )
                .first()
            )

            if taskTemplate:
                taskTemplateFile = TaskTemplateFile.read(self.application.getTemplatePath(), taskTemplate)
                task = taskTemplateFile.getTask()
            else:
                self.showFatalError(u"Ошибка при доступе к шаблону")
                return

        else:
            task = Task(appname=app.code)

        eventsLoaded = len(task.items)
        html = self.render("dashboard/new.jinja2", {"task": task, "app": app, "appConfig": appConfig}, _return=True)

        self.renderJSON({"html": html, "vars": {"eventLoaded": eventsLoaded}})
Esempio n. 4
0
    def get(self, *args, **kwargs):
        appCode, taskId = args
        worker = self.getWorker(int(taskId))
        service = WorkerService(self.application.getResultPath(), worker)
        data = service.getResultData()["data"]["result"]

        task = service.getTask()

        appService = AppService(self.application.getAppConfigPath())
        appConfig = appService.getNewAppConfig(appCode)
        nameService = NameConstructor(appConfig, task)

        tableService = TableConstructor(data, nameService, task)
        tableHeaders, tableData = tableService.getVerticalData()
        self.render(
            "dashboard/result/table2.jinja2", tableData=tableData, tableHeaders=tableHeaders, interval=task.interval
        )
Esempio n. 5
0
    def get(self, *args, **kwargs):
        appCode, workerId = args
        worker = self.getWorker(workerId)

        if worker.status == Worker.STATUS_ALIVE:
            self.renderJSON({"html": self.render("dashboard/result/taskAlive.jinja2", _return=True)})
        elif (worker.status == Worker.STATUS_ERROR) or (worker.status == Worker.STATUS_DIED):
            service = WorkerService(self.application.getResultPath(), worker)
            self.renderJSON(
                {
                    "html": self.render(
                        "dashboard/result/taskFailed.jinja2", {"errors": [service.getError()]}, _return=True
                    )
                }
            )
        elif worker.status == Worker.STATUS_SUCCESS:
            self.renderJSON({"redirect": "result/job=" + workerId})
        else:
            raise Exception("Unknown worker state")
Esempio n. 6
0
    def getWorkerList(self, app, page = 1):
        db_session = self.getDBSession()
        user = self.get_current_user()
        perPage = 5
        # получаем количество
        count, = db_session.query(func.count(Worker.workerId)).filter(Worker.userId == user.userId, Worker.appId == app.appId).first()
        pageCount = int(math.ceil(float(count) / perPage))
        # получаем список последних запросов
        lastWorkers = db_session.query(Worker).filter(Worker.userId == user.userId, Worker.appId == app.appId).order_by(desc(Worker.startDate)).offset(perPage * (page - 1)).limit(perPage)

        # определяем, какие воркеры уже умерли
        workers = []
        alivedWorkers= []
        for worker in lastWorkers:
            if worker.status == Worker.STATUS_ALIVE:
                alivedWorkers.append(worker)

            workers.append(worker)

        aliveThreadNames = ThredService.getAliveThreads()
        for worker in alivedWorkers:
            # определяем мертвые воркеры
            if not 'worker-' + str(worker.workerId) in aliveThreadNames:
                worker.status = Worker.STATUS_DIED
                db_session.add(worker)
                alivedWorkers.remove(worker)

        db_session.commit()

        # для живых воркеров загружаем таски
        if alivedWorkers:
            workerService = WorkerService(self.application.getResultPath())
            for worker in alivedWorkers:
                workerService.setWorker(worker)
                worker.task = workerService.getTask()
        return workers, pageCount
Esempio n. 7
0
    def get(self, *args, **kwargs):
        # get app code
        if len(args) == 0:
            raise RuntimeError("Cant find app code in dashboard")
        appCode = args[0]

        user = self.get_current_user()

        # get app
        app = self.dbSession.query(App).filter(App.code == appCode).first()
        if app is None:
            raise RuntimeError("Cant find app by code " + appCode)

        # check access
        ruleService = RuleService(self.dbSession)
        if not ruleService.isAllow(user.userId, app.appId):
            raise RuntimeError("Access denied")

        workerId = self.get_argument("jobId")

        worker = self.getWorker(workerId)
        service = WorkerService(self.application.getResultPath(), worker)

        if worker.status != Worker.STATUS_SUCCESS:
            self.renderJSON({"redirect": "status/" + str(workerId)})
            return

        # configuration name services
        task = service.getTask()

        startDates = []
        endDates = []
        for i in task.items:
            taskItem = task.items[i]
            startDates.append(taskItem.start)
            endDates.append(taskItem.end)

        minStartDate = min(startDates)
        maxEndDate = max(endDates)

        appService = AppService(self.application.getAppConfigPath())
        appConfig = appService.getNewAppConfig(app.code)
        nameService = NameConstructor(appConfig, task)

        try:
            data = service.getResultData()["data"]["result"]
            if len(data.items()) == 0:
                self.render("dashboard/result_no_data.jinja2", {"app": app})
                return
            # chart data
            chartService = ChartConstructor(data, nameService, task)
            tableService = TableConstructor(data, nameService, task)
        except IOError as ioerr:
            self.render(
                "dashboard/result.jinja2", {"errors": [u"Ошибка чтения результатов выполнения работы"], "app": app}
            )
        else:
            html = self.render(
                "dashboard/result.jinja2",
                {
                    "app": app,
                    "data": data,
                    "tablesdata": tableService.getData(),
                    "nameService": nameService,
                    "chartService": chartService,
                    "name": worker.name,
                    "interval": task.interval,
                    "tagCloud": chartService.getTagCloud(),
                    "workerId": workerId,
                },
                _return=True,
            )

            self.renderJSON(
                {
                    "html": html,
                    "vars": {
                        "chartdata": chartService.getResult(),
                        "interval": task.interval,
                        "minStartDate": time.mktime(minStartDate.timetuple()),
                        "maxEndDate": time.mktime(maxEndDate.timetuple()),
                        "taskId": workerId,
                    },
                }
            )
Esempio n. 8
0
#!/usr/bin/python
from components.HiveResponseProcessor import HiveResponseProcessor
from services.WorkerService import WorkerService
from models.Worker import Worker
import os, inspect, re

task = 1
_file = '/../TestHiveResponse'

thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
resultPath = os.path.abspath(thisPath + '/../result/')

worker = Worker()
worker.workerId = task
workerService = WorkerService(resultPath, worker)
task = workerService.getTask()

r = re.compile('\s+')

f = open(os.path.abspath(thisPath + _file), 'r')
data = f.readlines()
data = [r.split(line) for line in data]

processor = HiveResponseProcessor(task)
processor.prepareData(data)
Esempio n. 9
0
    def post(self, *args, **kwargs):
        app = self.checkAppAccess(args)

        saveAsTemplate = str(self.get_argument("saveAsTemplate", default=""))

        # создаем задачу из аргументов
        task = createTaskFromRequestArguments(self.request.arguments)
        session = self.getDBSession()

        if saveAsTemplate:
            # сохраняем шаблон
            taskTemplate = TaskTemplate()
            taskTemplate.appId = app.appId
            taskTemplate.name = self.get_argument("taskName", default=None)
            taskTemplate.userId = self.get_current_user().userId
            taskTemplate.shared = TaskTemplate.SHARED_NO

            session.add(taskTemplate)
            session.commit()

            # скидываем на диск
            taskTemplateFile = TaskTemplateFile(
                self.application.getTemplatePath(), task=task, taskTemplate=taskTemplate
            )
            taskTemplateFile.baseDate = datetime.now()
            taskTemplateFile.save()

            self.redirect("/dashboard/app/{}/#templates".format(app.code))
            return

        user = self.get_current_user()

        # объект для записи в базу
        worker = Worker()
        worker.userId = user.userId
        worker.startDate = datetime.now()
        worker.status = Worker.STATUS_ALIVE
        worker.appId = app.appId
        worker.name = self.get_argument("taskName", default=None)

        # генерируем имя запроса
        appService = AppService(self.application.getAppConfigPath())
        appConfig = appService.getAppConfig(app.code)
        nameConstructor = NameConstructor(appConfig, task)
        if not worker.name:
            worker.name = nameConstructor.generateTaskName()

        session.add(worker)
        session.commit()

        # конструирем запрос
        constructor = HiveQueryConstructor(task, appConfig)
        querys = constructor.getHiveQuerys(worker.workerId)

        # task.stageCount = constructor.getStageCount()
        # создаем WorkerService - он будет связывать тред с файловой системой
        workerService = WorkerService(self.application.getResultPath(), worker)
        workerService.setQuery(querys)
        workerService.setTask(task)
        workerService.init()

        # создаем и запускаем тред
        workerThread = self.createHiveWorker(workerService)
        workerThread.start()

        self.redirect("/dashboard/app/" + app.code + "/#new_task/" + str(worker.workerId))