# coding=utf-8
__author__ = '*****@*****.**'
from models.Config import Config
from components.webhdfs import WebHDFS
import os, inspect
from datetime import datetime
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
config = Config()
config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../server.cfg')))
STATISTICS_ROOT = config.get('hdfs_statistic_path') + 'topface/'
HDFS_HOST = config.get('hdfs_host')
HDFS_PORT = int(config.get('hdfs_port'))
HDFS_USER = config.get('hdfs_username')
webHDFS = WebHDFS(HDFS_HOST, HDFS_PORT, HDFS_USER)
dirs = webHDFS.listdir(STATISTICS_ROOT)
def scanDir(dirname, eventcode, skip):
years = webHDFS.listdir(dirname)
for year in years:
months = webHDFS.listdir(dirname + '/' + year)
for month in months:
days = webHDFS.listdir(dirname + '/' + year + '/' + month)
for day in days:
if (int(year), int(month), int(day)) in skip:
continue
class BaseAnalyticsScript():
def __init__(self, options):
self.scoped_session = self.analyticsWebHDFS = self.hiveService = None
# set logger
logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.config = Config()
self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg')))
self.options = options
self.appService = AppService(rootPath + '/../../app_configs/')
self.availableApps = self.appService.getAppConfigList()
if options['appname'] is None and options['all_apps'] is None:
print 'App name or --all is not set'
print 'Availale app names: ' + str(self.availableApps)
self.terminate()
self.appCodes = []
if options['appname']:
self.appCodes.append(options['appname'])
else:
self.appCodes = self.availableApps
def getAppCodes(self):
return self.appCodes
def getAppConfig(self, appCode):
"""
-> AppConfig
"""
return self.appService.getAppConfig(appCode)
def getApp(self, appCode):
return self.getDBSession().query(App).filter_by(code = appCode).first()
def getWebHDFSClient(self):
if not self.analyticsWebHDFS:
host = self.config.get(Config.HDFS_HOST)
port = int(self.config.get(Config.HDFS_PORT))
username = self.config.get(Config.HDFS_USERNAME)
statRoot = self.config.get(Config.HDFS_STAT_ROOT)
self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot)
return self.analyticsWebHDFS
def getHiveClient(self):
if not self.hiveService:
self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT)))
return self.hiveService
def getDBSession(self):
if not self.scoped_session:
mysql_user = self.config.get(Config.MYSQL_USER)
mysql_password = self.config.get(Config.MYSQL_PASSWORD)
mysql_host = self.config.get(Config.MYSQL_HOST)
mysql_dbname = self.config.get(Config.MYSQL_DBNAME)
conn_str = 'mysql://'
if mysql_user:
conn_str += mysql_user
if mysql_password:
conn_str += ':' + mysql_password
conn_str += '@'
conn_str += mysql_host + '/' + mysql_dbname
engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
encoding = 'utf8',
convert_unicode = True,
pool_recycle = 3600)
engine.execute('SET NAMES utf8')
self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))
return self.scoped_session()
def terminate(self):
exit()
class AnalyticsServer(tornado.web.Application):
spellFolder = None
def __init__(self, debug = False):
# define app root path
thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.appRoot = os.path.abspath(thisPath + '/../../')
self.spellFolder = self.appRoot + '/app_configs/spells/'
level = logging.WARNING
if debug:
level = logging.INFO
formatter = logging.Formatter('%(asctime)s %(filename)s(%(lineno)s)[%(funcName)s] %(threadName)s %(levelname)-8s %(message)s')
self.logger = logging.getLogger('AnalyticsServer')
self.logger.setLevel(level)
self.internalData = {}
# sterr out
if debug:
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(formatter)
handler.setLevel(level)
self.logger.addHandler(handler)
handler = logging.FileHandler(self.appRoot + '/log/analyticsServer_' + datetime.now().strftime('%d-%m-%y') + '.log', 'a+')
handler.setLevel(level)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.logger.info('AnalyticsServer started')
# define app setting
settings = {
"cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
"login_url": "/user/login",
"static_url_prefix": "/static",
"debug" : debug,
}
# call parent constuctor
super(AnalyticsServer, self).__init__(handlers = self.getHandlers(), default_host = "", transforms = None,
wsgi = False, **settings)
self.loadConfiguration()
self.determineIsInstall()
self.jinjaEnvironment = Environment(loader = PackageLoader('static', 'template'))
self.jinjaEnvironment.filters['datetofiled'] = datetofiled
self.jinjaEnvironment.filters['smartDate'] = smartDate
self.jinjaEnvironment.filters['smartDatePeriod'] = smartDatePeriod
self.jinjaEnvironment.filters['smartDateInterval'] = smartDateInterval
self.jinjaEnvironment.filters['excelDate'] = excelDate
self.jinjaEnvironment.filters['excelTime'] = excelTime
self.jinjaEnvironment.filters['dateFromTS'] = dateFromTS
self.jinjaEnvironment.filters['toJsVar'] = toJsVar
self.jinjaEnvironment.filters['minInt'] = minInt
self.jinjaEnvironment.filters['numberFormat'] = numberFormat
self.jinjaEnvironment.filters['showMonth'] = showMonth
self.scoped_session = None
self.loopInstance = None
if self.isInstalled:
self.initDBScopedSession()
def initDBScopedSession(self):
if not self.scoped_session:
mysql_user = self.config.get(Config.MYSQL_USER)
mysql_password = self.config.get(Config.MYSQL_PASSWORD)
mysql_host = self.config.get(Config.MYSQL_HOST)
mysql_dbname = self.config.get(Config.MYSQL_DBNAME)
conn_str = 'mysql://'
if mysql_user:
conn_str += mysql_user
if mysql_password:
conn_str += ':' + mysql_password
conn_str += '@'
conn_str += mysql_host + '/' + mysql_dbname
engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
encoding = 'utf8',
convert_unicode = True,
pool_recycle = 3600)
engine.execute('SET NAMES utf8')
self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))
def start(self):
port = int(self.config.get('app_port', 48888))
address = self.config.get('app_host', "")
self.listen(port = port, address = address )
self.loopInstance = tornado.ioloop.IOLoop.instance()
self.logger.info('application run {}:{}'.format(address, port))
self.loopInstance.start()
def loadConfiguration(self):
self.config = Config()
self.config.readConfigFile(os.path.abspath(self.appRoot + '/server.cfg'))
def getAppConfigPath(self):
return self.appRoot + '/app_configs/'
def getResultPath(self):
return self.appRoot + '/result/'
def getLogPath(self):
return self.appRoot + '/log/'
def getTemplatePath(self):
return self.appRoot + '/template/'
def determineIsInstall(self):
"""
Простое определение - установлено ли приложение
"""
self.isInstalled = os.path.exists(self.appRoot + '/server.cfg')
def setData(self, key, value):
self.internalData[key] = value
def getData(self, key):
return self.internalData[key]
def deleteData(self, key):
del self.internalData[key]
def getHandlers(self):
thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
return [
(r"/", IndexController.IndexAction),
# --------- DASHBOARD -----------
(r"/dashboard/?", DashboardController.SwitchApp),
(r"/dashboard/app/([^/]+)/?", DashboardController.IndexAction),
(r"/dashboard/app/([^/]+)/first/?", DashboardController.FirstAction),
(r"/dashboard/app/([^/]+)/result/?", ResultController.ResultAction),
(r"/dashboard/app/([^/]+)/result/table/(\d+)/?", ResultController.TableAction),
(r"/dashboard/app/([^/]+)/new/?", CreateTaskController.CreateAction),
(r"/dashboard/app/([^/]+)/recalculate/?", CreateTaskController.RecalculateAction),
(r"/dashboard/app/([^/]+)/templates/?", TemplateController.IndexAction),
(r"/dashboard/empty/?", DashboardController.EmptyAppAction),
(r"/dashboard/selectapp/?", DashboardController.SelectAppAction),
(r"/dashboard/app/([^/]+)/new_task/([^/]+)/?", CreateTaskController.ShowNewTaskAction),
(r"/dashboard/app/([^/]+)/status/([^/]+)/?", ResultController.ShowTaskStatus),
(r"/ajax/dashboard/app/([^/]+)/workers/([^/]+)/??", DashboardController.GetWorkers),
# --------- LOGIN & LOGOUT ---------
(r"/user/login/?", UserController.AuthAction),
(r"/user/logout/?", UserController.LogoutAction),
# --------- ADMIN -----------
(r"/admin/?", AdminIndexController.IndexAction),
(r"/admin/users/?", AdminUserController.UserAction),
(r"/admin/users/edit?", AdminUserController.EditUserAction),
(r"/admin/apps/?", AdminAppController.IndexAction),
(r"/admin/app/edit/?", AdminAppController.EditAction),
(r"/admin/rules/?", AdminRulesController.IndexAction),
(r"/admin/rules/switch?", AdminRulesController.SwitchAjaxAction),
(r'/admin/settings/?', AdminSettingsController.IndexAction),
(r'/admin/cluster?', AdminClusterController.IndexAction),
(r'/admin/historyServer/?', AdminProxy.HistoryServerView),
(r'/admin/resourceManager/?', AdminProxy.ResourceManagerView),
(r'/admin/proxy/([^/]+)/([^/]+)/(.+)?', AdminProxy.CoreProxy),
(r'/admin/app/([^/]+)/settings/?', AdminAppConfigController.IndexAction),
(r'/admin/app/([^/]+)/eventsList/?', AdminAppConfigController.EventListAction),
(r"/admin/app/([^/]+)/editEvent/?", AdminAppConfigController.EditEvent),
(r'/admin/app/([^/]+)/editTags/?', AdminAppConfigController.TagListAction),
(r"/ajax/editTag/?", AdminAppConfigController.EditTag),
(r'/admin/app/([^/]+)/editBunches/?', AdminAppConfigController.BunchListAction),
(r'/ajax/editBunch/?', AdminAppConfigController.EditBunchAction),
(r'/admin/app/([^/]+)/spell/?', AdminAppSpellController.IndexAction),
(r'/admin/app/([^/]+)/groups/?', AdminAppGroupsController.IndexAction),
# --------- CLUSTER -----------
(r'/cluster/namenode/?', NameNodeController.IndexAction),
# --------- AJAX -----------
(r"/ajax/key_configuration/?", AjaxController.KeyConfigurationAction),
(r"/ajax/get_key_form/?", AjaxController.GetKeyForm),
(r"/ajax/getTasksProgress/?", AjaxController.GatTasksProgress),
(r"/ajax/copyTaskKey/?", AjaxController.CopyTaskKey),
(r"/ajax/saveWorkerName?", AjaxController.SaveWorkerName),
#(r"/ajax/add_new_key?", AjaxController.AddNewKey),
(r"/ajax/add_new_tag?", AjaxController.AddNewTag),
(r"/ajax/add_new_bunch?", AjaxController.AddNewBunch),
(r"/ajax/getDateSelector/?", AjaxController.GetDateSelector),
(r"/ajax/getTagUniqueValues/?", HiveAJAXController.getTagUniqueValues),
(r"/ajax/downloadCSV/?", AjaxController.DownloadCSVAction),
(r"/ajax/admin/getClusterState/?", AdminClusterController.ClusterStateAction),
# ----------- TEMPLATE ---------------
#(r"/template/create/([^/]+)/?", TemplateController.CreateTemplateAction),
# ----------- HDFS ---------------
(r"/hdfs/?", HDFSController.IndexAction),
(r"/hdfs/ajax/getPath/?", HDFSAJAXController.GetPathAction),
(r"/hdfs/ajax/getPathStat/?", HDFSAJAXController.GetPathStat),
(r"/hdfs/ajax/getHiveStat/?", HDFSAJAXController.GetHiveStatus),
(r"/api/putConfig/?", APIController.PutConfigAction),
(r"/install/?", InstallController.InstallAction),
(r"/install/final/?", InstallController.FinalInstallAction),
(r"/static/(.*)", tornado.web.StaticFileHandler, {"path": os.path.abspath(thisPath + '/../static')}),
]