def post(self, *args, **kwargs):
appCode = self.get_argument('appCode')
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(appCode)
data = {
'index': self.get_argument('index'),
'bunch': AppTagBunch(),
'tags': appConfig.getGeneralTags()
}
self.render('admin/appConfig/editOneBunch.jinja2', data)
def post(self, *args, **kwargs):
copy_index = self.get_argument('copy_key_index')
new_index = self.get_argument('new_index')
appCode = self.get_argument('appname')
task = createTaskFromRequestArguments(self.request.arguments)
taskItem = task.getTaskItem(copy_index)
taskItem.index = new_index
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(appCode)
self.render('dashboard/taskForm/eventFormItem.jinja2', {'taskItem': taskItem, 'appConfig':appConfig})
def loadAppList(self):
db = self.getDBSession()
self.apps = db.query(App).filter_by(status = App.STATUS_ACTIVE).all()
appService = AppService(self.application.getAppConfigPath())
self.app_files = appService.getKnowAppList()
exist_app_code = []
for app in self.apps:
exist_app_code.append(app.code)
self.new_app_code = [app_code for app_code in self.app_files if app_code not in set(exist_app_code)]
class IndexAction(AdminAction):
def _prepare(self, appCode):
self.appService = AppService(self.application.getAppConfigPath())
self.appConfig = self.appService.getNewAppConfig(appCode=appCode)
def get(self, *args, **kwargs):
app = self.checkAppAccess(args)
self._prepare(app.code)
self.renderPage()
def renderPage(self):
self.render('admin/groups/app_groups.jinja2', {
'appConfig': self.appConfig,
'js_vars': {
'groupsCount': 0
}
})
def post(self, *args, **kwargs):
app = self.checkAppAccess(args)
self._prepare(app.code)
# events
eventGroupIterator = self.get_argument('eventGroupIterator', default=None)
groups = {}
if eventGroupIterator:
for index in range(0, int(eventGroupIterator) + 1):
groupName = self.get_argument('group_event_' + str(index), default=None)
if groupName:
groups[index] = groupName
eventGroups = {}
for event in self.appConfig.getEvents():
groupIDs = self.get_arguments('group_event_' + event.code)
if groupIDs:
for groupId in groupIDs:
groupId = int(groupId)
if not eventGroups.has_key(groupId):
eventGroups[groupId] = []
eventGroups[groupId].append(event.code)
self.appConfig.clearEventsGroups()
for index in groups:
i = self.appConfig.addEventGroup(groups[index])
if eventGroups.has_key(index):
for eventCode in eventGroups[index]:
self.appConfig.setEventGroup(i, eventCode)
self.appService.newSaveConfig(self.appConfig)
self.renderPage()
def post(self, *args, **kwargs):
jsonData = self.get_argument('data', False)
if jsonData:
try:
data = json.loads(jsonData)
service = AppService(self.application.getAppConfigPath())
service.saveConfig(data)
except BaseException:
self.renderJSON({'error': 'exception in process data'})
else:
self.renderJSON({'error': 'cant find data'})
self.renderJSON({'success': 'success'})
def __init__(self, options):
self.scoped_session = self.analyticsWebHDFS = self.hiveService = None
# set logger
logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.config = Config()
self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg')))
self.options = options
self.appService = AppService(rootPath + '/../../app_configs/')
self.availableApps = self.appService.getAppConfigList()
if options['appname'] is None and options['all_apps'] is None:
print 'App name or --all is not set'
print 'Availale app names: ' + str(self.availableApps)
self.terminate()
self.appCodes = []
if options['appname']:
self.appCodes.append(options['appname'])
else:
self.appCodes = self.availableApps
def get(self, *args, **kwargs):
app = self.checkAppAccess(args)
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(app.code)
dbSession = self.getDBSession()
baseTask = self.get_argument("baseOn", False)
template = self.get_argument("template", False)
nameConstructor = NameConstructor(appConfig)
if baseTask:
# задача основана на другой задаче
worker = dbSession.query(Worker).filter_by(workerId=baseTask).first()
workerService = WorkerService(self.application.getResultPath(), worker)
task = workerService.getTask()
# fix indexes
for index, taskItem in enumerate(task.getTaskItems()):
taskItem.index = index
if not taskItem.name:
taskItem.name = nameConstructor.getTaskItemName(taskItem)
elif template:
templateId = int(template)
userId = self.get_current_user().userId
taskTemplate = (
dbSession.query(TaskTemplate)
.filter(
and_(
TaskTemplate.taskTemplateId == templateId,
or_(TaskTemplate.userId == userId, TaskTemplate.shared == TaskTemplate.SHARED_YES),
)
)
.first()
)
if taskTemplate:
taskTemplateFile = TaskTemplateFile.read(self.application.getTemplatePath(), taskTemplate)
task = taskTemplateFile.getTask()
else:
self.showFatalError(u"Ошибка при доступе к шаблону")
return
else:
task = Task(appname=app.code)
eventsLoaded = len(task.items)
html = self.render("dashboard/new.jinja2", {"task": task, "app": app, "appConfig": appConfig}, _return=True)
self.renderJSON({"html": html, "vars": {"eventLoaded": eventsLoaded}})
def get(self, *args, **kwargs):
appCode, taskId = args
worker = self.getWorker(int(taskId))
service = WorkerService(self.application.getResultPath(), worker)
data = service.getResultData()["data"]["result"]
task = service.getTask()
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(appCode)
nameService = NameConstructor(appConfig, task)
tableService = TableConstructor(data, nameService, task)
tableHeaders, tableData = tableService.getVerticalData()
self.render(
"dashboard/result/table2.jinja2", tableData=tableData, tableHeaders=tableHeaders, interval=task.interval
)
class BaseEditConfigAction(AdminAction):
def prepare(self):
super(BaseEditConfigAction, self).prepare()
self.errors = []
self.appService = AppService(self.application.getAppConfigPath())
def getAppConfig(self, appCode):
return self.appService.getNewAppConfig(appCode)
def post(self, *args, **kwargs):
eventCode = self.get_argument('eventCode', None)
appCode = self.get_argument('appCode', None)
index = self.get_argument('index', 1)
taskItem = TaskItem(index = index, key = eventCode)
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(appCode)
nameConstructor = NameConstructor(appConfig)
taskItem.name = nameConstructor.getTaskItemName(taskItem)
self.render('dashboard/taskForm/eventFormItem.jinja2', {
'taskItem': taskItem,
'appConfig': appConfig,
'values':{
}
})
def get(self, *args, **kwargs):
# get app code
if len(args) == 0:
raise RuntimeError("Cant find app code in dashboard")
appCode = args[0]
user = self.get_current_user()
# get app
app = self.dbSession.query(App).filter(App.code == appCode).first()
if app is None:
raise RuntimeError("Cant find app by code " + appCode)
# check access
ruleService = RuleService(self.dbSession)
if not ruleService.isAllow(user.userId, app.appId):
raise RuntimeError("Access denied")
workerId = self.get_argument("jobId")
worker = self.getWorker(workerId)
service = WorkerService(self.application.getResultPath(), worker)
if worker.status != Worker.STATUS_SUCCESS:
self.renderJSON({"redirect": "status/" + str(workerId)})
return
# configuration name services
task = service.getTask()
startDates = []
endDates = []
for i in task.items:
taskItem = task.items[i]
startDates.append(taskItem.start)
endDates.append(taskItem.end)
minStartDate = min(startDates)
maxEndDate = max(endDates)
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getNewAppConfig(app.code)
nameService = NameConstructor(appConfig, task)
try:
data = service.getResultData()["data"]["result"]
if len(data.items()) == 0:
self.render("dashboard/result_no_data.jinja2", {"app": app})
return
# chart data
chartService = ChartConstructor(data, nameService, task)
tableService = TableConstructor(data, nameService, task)
except IOError as ioerr:
self.render(
"dashboard/result.jinja2", {"errors": [u"Ошибка чтения результатов выполнения работы"], "app": app}
)
else:
html = self.render(
"dashboard/result.jinja2",
{
"app": app,
"data": data,
"tablesdata": tableService.getData(),
"nameService": nameService,
"chartService": chartService,
"name": worker.name,
"interval": task.interval,
"tagCloud": chartService.getTagCloud(),
"workerId": workerId,
},
_return=True,
)
self.renderJSON(
{
"html": html,
"vars": {
"chartdata": chartService.getResult(),
"interval": task.interval,
"minStartDate": time.mktime(minStartDate.timetuple()),
"maxEndDate": time.mktime(maxEndDate.timetuple()),
"taskId": workerId,
},
}
)
def _prepare(self, appCode):
self.appService = AppService(self.application.getAppConfigPath())
self.appConfig = self.appService.getNewAppConfig(appCode=appCode)
class BaseAnalyticsScript():
def __init__(self, options):
self.scoped_session = self.analyticsWebHDFS = self.hiveService = None
# set logger
logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.config = Config()
self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg')))
self.options = options
self.appService = AppService(rootPath + '/../../app_configs/')
self.availableApps = self.appService.getAppConfigList()
if options['appname'] is None and options['all_apps'] is None:
print 'App name or --all is not set'
print 'Availale app names: ' + str(self.availableApps)
self.terminate()
self.appCodes = []
if options['appname']:
self.appCodes.append(options['appname'])
else:
self.appCodes = self.availableApps
def getAppCodes(self):
return self.appCodes
def getAppConfig(self, appCode):
"""
-> AppConfig
"""
return self.appService.getAppConfig(appCode)
def getApp(self, appCode):
return self.getDBSession().query(App).filter_by(code = appCode).first()
def getWebHDFSClient(self):
if not self.analyticsWebHDFS:
host = self.config.get(Config.HDFS_HOST)
port = int(self.config.get(Config.HDFS_PORT))
username = self.config.get(Config.HDFS_USERNAME)
statRoot = self.config.get(Config.HDFS_STAT_ROOT)
self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot)
return self.analyticsWebHDFS
def getHiveClient(self):
if not self.hiveService:
self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT)))
return self.hiveService
def getDBSession(self):
if not self.scoped_session:
mysql_user = self.config.get(Config.MYSQL_USER)
mysql_password = self.config.get(Config.MYSQL_PASSWORD)
mysql_host = self.config.get(Config.MYSQL_HOST)
mysql_dbname = self.config.get(Config.MYSQL_DBNAME)
conn_str = 'mysql://'
if mysql_user:
conn_str += mysql_user
if mysql_password:
conn_str += ':' + mysql_password
conn_str += '@'
conn_str += mysql_host + '/' + mysql_dbname
engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
encoding = 'utf8',
convert_unicode = True,
pool_recycle = 3600)
engine.execute('SET NAMES utf8')
self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))
return self.scoped_session()
def terminate(self):
exit()
def _prepare(self, appCode):
appService = AppService(self.application.getAppConfigPath())
self.appconfig = appService.getAppConfig(appCode=appCode)
self.spell = Spell()
self.spellService = SpellService(self.application.spellFolder, appCode)
def prepare(self):
super(BaseEditConfigAction, self).prepare()
self.errors = []
self.appService = AppService(self.application.getAppConfigPath())
def post(self, *args, **kwargs):
app = self.checkAppAccess(args)
saveAsTemplate = str(self.get_argument("saveAsTemplate", default=""))
# создаем задачу из аргументов
task = createTaskFromRequestArguments(self.request.arguments)
session = self.getDBSession()
if saveAsTemplate:
# сохраняем шаблон
taskTemplate = TaskTemplate()
taskTemplate.appId = app.appId
taskTemplate.name = self.get_argument("taskName", default=None)
taskTemplate.userId = self.get_current_user().userId
taskTemplate.shared = TaskTemplate.SHARED_NO
session.add(taskTemplate)
session.commit()
# скидываем на диск
taskTemplateFile = TaskTemplateFile(
self.application.getTemplatePath(), task=task, taskTemplate=taskTemplate
)
taskTemplateFile.baseDate = datetime.now()
taskTemplateFile.save()
self.redirect("/dashboard/app/{}/#templates".format(app.code))
return
user = self.get_current_user()
# объект для записи в базу
worker = Worker()
worker.userId = user.userId
worker.startDate = datetime.now()
worker.status = Worker.STATUS_ALIVE
worker.appId = app.appId
worker.name = self.get_argument("taskName", default=None)
# генерируем имя запроса
appService = AppService(self.application.getAppConfigPath())
appConfig = appService.getAppConfig(app.code)
nameConstructor = NameConstructor(appConfig, task)
if not worker.name:
worker.name = nameConstructor.generateTaskName()
session.add(worker)
session.commit()
# конструирем запрос
constructor = HiveQueryConstructor(task, appConfig)
querys = constructor.getHiveQuerys(worker.workerId)
# task.stageCount = constructor.getStageCount()
# создаем WorkerService - он будет связывать тред с файловой системой
workerService = WorkerService(self.application.getResultPath(), worker)
workerService.setQuery(querys)
workerService.setTask(task)
workerService.init()
# создаем и запускаем тред
workerThread = self.createHiveWorker(workerService)
workerThread.start()
self.redirect("/dashboard/app/" + app.code + "/#new_task/" + str(worker.workerId))
