def post(self, *args, **kwargs): appCode = self.get_argument('appCode') appService = AppService(self.application.getAppConfigPath()) appConfig = appService.getNewAppConfig(appCode) data = { 'index': self.get_argument('index'), 'bunch': AppTagBunch(), 'tags': appConfig.getGeneralTags() } self.render('admin/appConfig/editOneBunch.jinja2', data)
def post(self, *args, **kwargs): copy_index = self.get_argument('copy_key_index') new_index = self.get_argument('new_index') appCode = self.get_argument('appname') task = createTaskFromRequestArguments(self.request.arguments) taskItem = task.getTaskItem(copy_index) taskItem.index = new_index appService = AppService(self.application.getAppConfigPath()) appConfig = appService.getNewAppConfig(appCode) self.render('dashboard/taskForm/eventFormItem.jinja2', {'taskItem': taskItem, 'appConfig':appConfig})
def loadAppList(self): db = self.getDBSession() self.apps = db.query(App).filter_by(status = App.STATUS_ACTIVE).all() appService = AppService(self.application.getAppConfigPath()) self.app_files = appService.getKnowAppList() exist_app_code = [] for app in self.apps: exist_app_code.append(app.code) self.new_app_code = [app_code for app_code in self.app_files if app_code not in set(exist_app_code)]
class IndexAction(AdminAction): def _prepare(self, appCode): self.appService = AppService(self.application.getAppConfigPath()) self.appConfig = self.appService.getNewAppConfig(appCode=appCode) def get(self, *args, **kwargs): app = self.checkAppAccess(args) self._prepare(app.code) self.renderPage() def renderPage(self): self.render('admin/groups/app_groups.jinja2', { 'appConfig': self.appConfig, 'js_vars': { 'groupsCount': 0 } }) def post(self, *args, **kwargs): app = self.checkAppAccess(args) self._prepare(app.code) # events eventGroupIterator = self.get_argument('eventGroupIterator', default=None) groups = {} if eventGroupIterator: for index in range(0, int(eventGroupIterator) + 1): groupName = self.get_argument('group_event_' + str(index), default=None) if groupName: groups[index] = groupName eventGroups = {} for event in self.appConfig.getEvents(): groupIDs = self.get_arguments('group_event_' + event.code) if groupIDs: for groupId in groupIDs: groupId = int(groupId) if not eventGroups.has_key(groupId): eventGroups[groupId] = [] eventGroups[groupId].append(event.code) self.appConfig.clearEventsGroups() for index in groups: i = self.appConfig.addEventGroup(groups[index]) if eventGroups.has_key(index): for eventCode in eventGroups[index]: self.appConfig.setEventGroup(i, eventCode) self.appService.newSaveConfig(self.appConfig) self.renderPage()
def post(self, *args, **kwargs): jsonData = self.get_argument('data', False) if jsonData: try: data = json.loads(jsonData) service = AppService(self.application.getAppConfigPath()) service.saveConfig(data) except BaseException: self.renderJSON({'error': 'exception in process data'}) else: self.renderJSON({'error': 'cant find data'}) self.renderJSON({'success': 'success'})
def __init__(self, options): self.scoped_session = self.analyticsWebHDFS = self.hiveService = None # set logger logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') # set configurations rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory self.config = Config() self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg'))) self.options = options self.appService = AppService(rootPath + '/../../app_configs/') self.availableApps = self.appService.getAppConfigList() if options['appname'] is None and options['all_apps'] is None: print 'App name or --all is not set' print 'Availale app names: ' + str(self.availableApps) self.terminate() self.appCodes = [] if options['appname']: self.appCodes.append(options['appname']) else: self.appCodes = self.availableApps
def get(self, *args, **kwargs): app = self.checkAppAccess(args) appService = AppService(self.application.getAppConfigPath()) appConfig = appService.getNewAppConfig(app.code) dbSession = self.getDBSession() baseTask = self.get_argument("baseOn", False) template = self.get_argument("template", False) nameConstructor = NameConstructor(appConfig) if baseTask: # задача основана на другой задаче worker = dbSession.query(Worker).filter_by(workerId=baseTask).first() workerService = WorkerService(self.application.getResultPath(), worker) task = workerService.getTask() # fix indexes for index, taskItem in enumerate(task.getTaskItems()): taskItem.index = index if not taskItem.name: taskItem.name = nameConstructor.getTaskItemName(taskItem) elif template: templateId = int(template) userId = self.get_current_user().userId taskTemplate = ( dbSession.query(TaskTemplate) .filter( and_( TaskTemplate.taskTemplateId == templateId, or_(TaskTemplate.userId == userId, TaskTemplate.shared == TaskTemplate.SHARED_YES), ) ) .first() ) if taskTemplate: taskTemplateFile = TaskTemplateFile.read(self.application.getTemplatePath(), taskTemplate) task = taskTemplateFile.getTask() else: self.showFatalError(u"Ошибка при доступе к шаблону") return else: task = Task(appname=app.code) eventsLoaded = len(task.items) html = self.render("dashboard/new.jinja2", {"task": task, "app": app, "appConfig": appConfig}, _return=True) self.renderJSON({"html": html, "vars": {"eventLoaded": eventsLoaded}})
def get(self, *args, **kwargs): appCode, taskId = args worker = self.getWorker(int(taskId)) service = WorkerService(self.application.getResultPath(), worker) data = service.getResultData()["data"]["result"] task = service.getTask() appService = AppService(self.application.getAppConfigPath()) appConfig = appService.getNewAppConfig(appCode) nameService = NameConstructor(appConfig, task) tableService = TableConstructor(data, nameService, task) tableHeaders, tableData = tableService.getVerticalData() self.render( "dashboard/result/table2.jinja2", tableData=tableData, tableHeaders=tableHeaders, interval=task.interval )
class BaseEditConfigAction(AdminAction): def prepare(self): super(BaseEditConfigAction, self).prepare() self.errors = [] self.appService = AppService(self.application.getAppConfigPath()) def getAppConfig(self, appCode): return self.appService.getNewAppConfig(appCode)
def post(self, *args, **kwargs): eventCode = self.get_argument('eventCode', None) appCode = self.get_argument('appCode', None) index = self.get_argument('index', 1) taskItem = TaskItem(index = index, key = eventCode) appService = AppService(self.application.getAppConfigPath()) appConfig = appService.getNewAppConfig(appCode) nameConstructor = NameConstructor(appConfig) taskItem.name = nameConstructor.getTaskItemName(taskItem) self.render('dashboard/taskForm/eventFormItem.jinja2', { 'taskItem': taskItem, 'appConfig': appConfig, 'values':{ } })
def get(self, *args, **kwargs): # get app code if len(args) == 0: raise RuntimeError("Cant find app code in dashboard") appCode = args[0] user = self.get_current_user() # get app app = self.dbSession.query(App).filter(App.code == appCode).first() if app is None: raise RuntimeError("Cant find app by code " + appCode) # check access ruleService = RuleService(self.dbSession) if not ruleService.isAllow(user.userId, app.appId): raise RuntimeError("Access denied") workerId = self.get_argument("jobId") worker = self.getWorker(workerId) service = WorkerService(self.application.getResultPath(), worker) if worker.status != Worker.STATUS_SUCCESS: self.renderJSON({"redirect": "status/" + str(workerId)}) return # configuration name services task = service.getTask() startDates = [] endDates = [] for i in task.items: taskItem = task.items[i] startDates.append(taskItem.start) endDates.append(taskItem.end) minStartDate = min(startDates) maxEndDate = max(endDates) appService = AppService(self.application.getAppConfigPath()) appConfig = appService.getNewAppConfig(app.code) nameService = NameConstructor(appConfig, task) try: data = service.getResultData()["data"]["result"] if len(data.items()) == 0: self.render("dashboard/result_no_data.jinja2", {"app": app}) return # chart data chartService = ChartConstructor(data, nameService, task) tableService = TableConstructor(data, nameService, task) except IOError as ioerr: self.render( "dashboard/result.jinja2", {"errors": [u"Ошибка чтения результатов выполнения работы"], "app": app} ) else: html = self.render( "dashboard/result.jinja2", { "app": app, "data": data, "tablesdata": tableService.getData(), "nameService": nameService, "chartService": chartService, "name": worker.name, "interval": task.interval, "tagCloud": chartService.getTagCloud(), "workerId": workerId, }, _return=True, ) self.renderJSON( { "html": html, "vars": { "chartdata": chartService.getResult(), "interval": task.interval, "minStartDate": time.mktime(minStartDate.timetuple()), "maxEndDate": time.mktime(maxEndDate.timetuple()), "taskId": workerId, }, } )
def _prepare(self, appCode): self.appService = AppService(self.application.getAppConfigPath()) self.appConfig = self.appService.getNewAppConfig(appCode=appCode)
class BaseAnalyticsScript(): def __init__(self, options): self.scoped_session = self.analyticsWebHDFS = self.hiveService = None # set logger logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') # set configurations rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory self.config = Config() self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg'))) self.options = options self.appService = AppService(rootPath + '/../../app_configs/') self.availableApps = self.appService.getAppConfigList() if options['appname'] is None and options['all_apps'] is None: print 'App name or --all is not set' print 'Availale app names: ' + str(self.availableApps) self.terminate() self.appCodes = [] if options['appname']: self.appCodes.append(options['appname']) else: self.appCodes = self.availableApps def getAppCodes(self): return self.appCodes def getAppConfig(self, appCode): """ -> AppConfig """ return self.appService.getAppConfig(appCode) def getApp(self, appCode): return self.getDBSession().query(App).filter_by(code = appCode).first() def getWebHDFSClient(self): if not self.analyticsWebHDFS: host = self.config.get(Config.HDFS_HOST) port = int(self.config.get(Config.HDFS_PORT)) username = self.config.get(Config.HDFS_USERNAME) statRoot = self.config.get(Config.HDFS_STAT_ROOT) self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot) return self.analyticsWebHDFS def getHiveClient(self): if not self.hiveService: self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT))) return self.hiveService def getDBSession(self): if not self.scoped_session: mysql_user = self.config.get(Config.MYSQL_USER) mysql_password = self.config.get(Config.MYSQL_PASSWORD) mysql_host = self.config.get(Config.MYSQL_HOST) mysql_dbname = self.config.get(Config.MYSQL_DBNAME) conn_str = 'mysql://' if mysql_user: conn_str += mysql_user if mysql_password: conn_str += ':' + mysql_password conn_str += '@' conn_str += mysql_host + '/' + mysql_dbname engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22', encoding = 'utf8', convert_unicode = True, pool_recycle = 3600) engine.execute('SET NAMES utf8') self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False)) return self.scoped_session() def terminate(self): exit()
def _prepare(self, appCode): appService = AppService(self.application.getAppConfigPath()) self.appconfig = appService.getAppConfig(appCode=appCode) self.spell = Spell() self.spellService = SpellService(self.application.spellFolder, appCode)
def prepare(self): super(BaseEditConfigAction, self).prepare() self.errors = [] self.appService = AppService(self.application.getAppConfigPath())
def post(self, *args, **kwargs): app = self.checkAppAccess(args) saveAsTemplate = str(self.get_argument("saveAsTemplate", default="")) # создаем задачу из аргументов task = createTaskFromRequestArguments(self.request.arguments) session = self.getDBSession() if saveAsTemplate: # сохраняем шаблон taskTemplate = TaskTemplate() taskTemplate.appId = app.appId taskTemplate.name = self.get_argument("taskName", default=None) taskTemplate.userId = self.get_current_user().userId taskTemplate.shared = TaskTemplate.SHARED_NO session.add(taskTemplate) session.commit() # скидываем на диск taskTemplateFile = TaskTemplateFile( self.application.getTemplatePath(), task=task, taskTemplate=taskTemplate ) taskTemplateFile.baseDate = datetime.now() taskTemplateFile.save() self.redirect("/dashboard/app/{}/#templates".format(app.code)) return user = self.get_current_user() # объект для записи в базу worker = Worker() worker.userId = user.userId worker.startDate = datetime.now() worker.status = Worker.STATUS_ALIVE worker.appId = app.appId worker.name = self.get_argument("taskName", default=None) # генерируем имя запроса appService = AppService(self.application.getAppConfigPath()) appConfig = appService.getAppConfig(app.code) nameConstructor = NameConstructor(appConfig, task) if not worker.name: worker.name = nameConstructor.generateTaskName() session.add(worker) session.commit() # конструирем запрос constructor = HiveQueryConstructor(task, appConfig) querys = constructor.getHiveQuerys(worker.workerId) # task.stageCount = constructor.getStageCount() # создаем WorkerService - он будет связывать тред с файловой системой workerService = WorkerService(self.application.getResultPath(), worker) workerService.setQuery(querys) workerService.setTask(task) workerService.init() # создаем и запускаем тред workerThread = self.createHiveWorker(workerService) workerThread.start() self.redirect("/dashboard/app/" + app.code + "/#new_task/" + str(worker.workerId))