def _prepare(self, appCode): appService = AppService(self.application.getAppConfigPath()) self.appconfig = appService.getAppConfig(appCode=appCode) self.spell = Spell() self.spellService = SpellService(self.application.spellFolder, appCode)
class BaseAnalyticsScript(): def __init__(self, options): self.scoped_session = self.analyticsWebHDFS = self.hiveService = None # set logger logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') # set configurations rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory self.config = Config() self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg'))) self.options = options self.appService = AppService(rootPath + '/../../app_configs/') self.availableApps = self.appService.getAppConfigList() if options['appname'] is None and options['all_apps'] is None: print 'App name or --all is not set' print 'Availale app names: ' + str(self.availableApps) self.terminate() self.appCodes = [] if options['appname']: self.appCodes.append(options['appname']) else: self.appCodes = self.availableApps def getAppCodes(self): return self.appCodes def getAppConfig(self, appCode): """ -> AppConfig """ return self.appService.getAppConfig(appCode) def getApp(self, appCode): return self.getDBSession().query(App).filter_by(code = appCode).first() def getWebHDFSClient(self): if not self.analyticsWebHDFS: host = self.config.get(Config.HDFS_HOST) port = int(self.config.get(Config.HDFS_PORT)) username = self.config.get(Config.HDFS_USERNAME) statRoot = self.config.get(Config.HDFS_STAT_ROOT) self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot) return self.analyticsWebHDFS def getHiveClient(self): if not self.hiveService: self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT))) return self.hiveService def getDBSession(self): if not self.scoped_session: mysql_user = self.config.get(Config.MYSQL_USER) mysql_password = self.config.get(Config.MYSQL_PASSWORD) mysql_host = self.config.get(Config.MYSQL_HOST) mysql_dbname = self.config.get(Config.MYSQL_DBNAME) conn_str = 'mysql://' if mysql_user: conn_str += mysql_user if mysql_password: conn_str += ':' + mysql_password conn_str += '@' conn_str += mysql_host + '/' + mysql_dbname engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22', encoding = 'utf8', convert_unicode = True, pool_recycle = 3600) engine.execute('SET NAMES utf8') self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False)) return self.scoped_session() def terminate(self): exit()
def post(self, *args, **kwargs): app = self.checkAppAccess(args) saveAsTemplate = str(self.get_argument("saveAsTemplate", default="")) # создаем задачу из аргументов task = createTaskFromRequestArguments(self.request.arguments) session = self.getDBSession() if saveAsTemplate: # сохраняем шаблон taskTemplate = TaskTemplate() taskTemplate.appId = app.appId taskTemplate.name = self.get_argument("taskName", default=None) taskTemplate.userId = self.get_current_user().userId taskTemplate.shared = TaskTemplate.SHARED_NO session.add(taskTemplate) session.commit() # скидываем на диск taskTemplateFile = TaskTemplateFile( self.application.getTemplatePath(), task=task, taskTemplate=taskTemplate ) taskTemplateFile.baseDate = datetime.now() taskTemplateFile.save() self.redirect("/dashboard/app/{}/#templates".format(app.code)) return user = self.get_current_user() # объект для записи в базу worker = Worker() worker.userId = user.userId worker.startDate = datetime.now() worker.status = Worker.STATUS_ALIVE worker.appId = app.appId worker.name = self.get_argument("taskName", default=None) # генерируем имя запроса appService = AppService(self.application.getAppConfigPath()) appConfig = appService.getAppConfig(app.code) nameConstructor = NameConstructor(appConfig, task) if not worker.name: worker.name = nameConstructor.generateTaskName() session.add(worker) session.commit() # конструирем запрос constructor = HiveQueryConstructor(task, appConfig) querys = constructor.getHiveQuerys(worker.workerId) # task.stageCount = constructor.getStageCount() # создаем WorkerService - он будет связывать тред с файловой системой workerService = WorkerService(self.application.getResultPath(), worker) workerService.setQuery(querys) workerService.setTask(task) workerService.init() # создаем и запускаем тред workerThread = self.createHiveWorker(workerService) workerThread.start() self.redirect("/dashboard/app/" + app.code + "/#new_task/" + str(worker.workerId))