def _prepare(self, appCode):
appService = AppService(self.application.getAppConfigPath())
self.appconfig = appService.getAppConfig(appCode=appCode)
self.spell = Spell()
self.spellService = SpellService(self.application.spellFolder, appCode)
class BaseAnalyticsScript():
def __init__(self, options):
self.scoped_session = self.analyticsWebHDFS = self.hiveService = None
# set logger
logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.config = Config()
self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg')))
self.options = options
self.appService = AppService(rootPath + '/../../app_configs/')
self.availableApps = self.appService.getAppConfigList()
if options['appname'] is None and options['all_apps'] is None:
print 'App name or --all is not set'
print 'Availale app names: ' + str(self.availableApps)
self.terminate()
self.appCodes = []
if options['appname']:
self.appCodes.append(options['appname'])
else:
self.appCodes = self.availableApps
def getAppCodes(self):
return self.appCodes
def getAppConfig(self, appCode):
"""
-> AppConfig
"""
return self.appService.getAppConfig(appCode)
def getApp(self, appCode):
return self.getDBSession().query(App).filter_by(code = appCode).first()
def getWebHDFSClient(self):
if not self.analyticsWebHDFS:
host = self.config.get(Config.HDFS_HOST)
port = int(self.config.get(Config.HDFS_PORT))
username = self.config.get(Config.HDFS_USERNAME)
statRoot = self.config.get(Config.HDFS_STAT_ROOT)
self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot)
return self.analyticsWebHDFS
def getHiveClient(self):
if not self.hiveService:
self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT)))
return self.hiveService
def getDBSession(self):
if not self.scoped_session:
mysql_user = self.config.get(Config.MYSQL_USER)
mysql_password = self.config.get(Config.MYSQL_PASSWORD)
mysql_host = self.config.get(Config.MYSQL_HOST)
mysql_dbname = self.config.get(Config.MYSQL_DBNAME)
conn_str = 'mysql://'
if mysql_user:
conn_str += mysql_user
if mysql_password:
conn_str += ':' + mysql_password
conn_str += '@'
conn_str += mysql_host + '/' + mysql_dbname
engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
encoding = 'utf8',
convert_unicode = True,
pool_recycle = 3600)
engine.execute('SET NAMES utf8')
self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))
return self.scoped_session()
def terminate(self):
exit()
def post(self, *args, **kwargs):
app = self.checkAppAccess(args)
saveAsTemplate = str(self.get_argument("saveAsTemplate", default=""))
# создаем задачу из аргументов
task = createTaskFromRequestArguments(self.request.arguments)
session = self.getDBSession()
if saveAsTemplate:
# сохраняем шаблон
taskTemplate = TaskTemplate()
taskTemplate.appId = app.appId
taskTemplate.name = self.get_argument("taskName", default=None)
taskTemplate.userId = self.get_current_user().userId
taskTemplate.shared = TaskTemplate.SHARED_NO
session.add(taskTemplate)
session.commit()
# скидываем на диск
taskTemplateFile = TaskTemplateFile(
self.application.getTemplatePath(), task=task, taskTemplate=taskTemplate
)
taskTemplateFile.baseDate = datetime.now()
taskTemplateFile.save()
self.redirect("/dashboard/app/{}/#templates".format(app.code))
return
user = self.get_current_user()
# объект для записи в базу
worker = Worker()
worker.userId = user.userId
worker.startDate = datetime.now()
worker.status = Worker.STATUS_ALIVE
worker.appId = app.appId
worker.name = self.get_argument("taskName", default=None)
# генерируем имя запроса
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getAppConfig(app.code)
nameConstructor = NameConstructor(appConfig, task)
if not worker.name:
worker.name = nameConstructor.generateTaskName()
session.add(worker)
session.commit()
# конструирем запрос
constructor = HiveQueryConstructor(task, appConfig)
querys = constructor.getHiveQuerys(worker.workerId)
# task.stageCount = constructor.getStageCount()
# создаем WorkerService - он будет связывать тред с файловой системой
workerService = WorkerService(self.application.getResultPath(), worker)
workerService.setQuery(querys)
workerService.setTask(task)
workerService.init()
# создаем и запускаем тред
workerThread = self.createHiveWorker(workerService)
workerThread.start()
self.redirect("/dashboard/app/" + app.code + "/#new_task/" + str(worker.workerId))
