示例#1
0
# coding=utf-8
__author__ = '*****@*****.**'
from models.Config import Config
from components.webhdfs import WebHDFS
import os, inspect
from datetime import datetime

# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory

config = Config()
config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../server.cfg')))

STATISTICS_ROOT = config.get('hdfs_statistic_path') + 'topface/'
HDFS_HOST = config.get('hdfs_host')
HDFS_PORT = int(config.get('hdfs_port'))
HDFS_USER = config.get('hdfs_username')
webHDFS = WebHDFS(HDFS_HOST, HDFS_PORT, HDFS_USER)

dirs = webHDFS.listdir(STATISTICS_ROOT)

def scanDir(dirname, eventcode, skip):
    years = webHDFS.listdir(dirname)
    for year in years:
        months = webHDFS.listdir(dirname + '/' + year)
        for month in months:
            days = webHDFS.listdir(dirname + '/' + year + '/' + month)
            for day in days:
                if (int(year), int(month), int(day)) in skip:
                    continue
示例#2
0
class BaseAnalyticsScript():

    def __init__(self, options):
        self.scoped_session = self.analyticsWebHDFS = self.hiveService = None
        # set logger
        logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')

        # set configurations
        rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory

        self.config = Config()
        self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg')))

        self.options = options
        self.appService = AppService(rootPath + '/../../app_configs/')
        self.availableApps = self.appService.getAppConfigList()

        if options['appname'] is None and options['all_apps'] is None:
            print 'App name or --all is not set'
            print 'Availale app names: ' + str(self.availableApps)
            self.terminate()

        self.appCodes = []
        if options['appname']:
            self.appCodes.append(options['appname'])
        else:
            self.appCodes = self.availableApps



    def getAppCodes(self):
        return self.appCodes

    def getAppConfig(self, appCode):
        """
        -> AppConfig
        """
        return self.appService.getAppConfig(appCode)

    def getApp(self, appCode):
        return self.getDBSession().query(App).filter_by(code = appCode).first()

    def getWebHDFSClient(self):
        if not self.analyticsWebHDFS:
            host = self.config.get(Config.HDFS_HOST)
            port = int(self.config.get(Config.HDFS_PORT))
            username = self.config.get(Config.HDFS_USERNAME)
            statRoot = self.config.get(Config.HDFS_STAT_ROOT)
            self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot)
        return self.analyticsWebHDFS

    def getHiveClient(self):
        if not self.hiveService:
            self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT)))
        return self.hiveService

    def getDBSession(self):
        if not self.scoped_session:
            mysql_user = self.config.get(Config.MYSQL_USER)
            mysql_password = self.config.get(Config.MYSQL_PASSWORD)
            mysql_host = self.config.get(Config.MYSQL_HOST)
            mysql_dbname = self.config.get(Config.MYSQL_DBNAME)

            conn_str = 'mysql://'
            if mysql_user:
                conn_str += mysql_user
                if mysql_password:
                    conn_str += ':' + mysql_password
                conn_str += '@'
            conn_str += mysql_host + '/' + mysql_dbname

            engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
                encoding = 'utf8',
                convert_unicode = True,
                pool_recycle = 3600)
            engine.execute('SET NAMES utf8')
            self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))
        return self.scoped_session()

    def terminate(self):
        exit()
示例#3
0
class AnalyticsServer(tornado.web.Application):

    spellFolder = None

    def __init__(self, debug = False):
        # define app root path
        thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
        self.appRoot = os.path.abspath(thisPath + '/../../')

        self.spellFolder = self.appRoot + '/app_configs/spells/'

        level = logging.WARNING
        if debug:
            level = logging.INFO

        formatter = logging.Formatter('%(asctime)s %(filename)s(%(lineno)s)[%(funcName)s] %(threadName)s %(levelname)-8s %(message)s')
        self.logger = logging.getLogger('AnalyticsServer')
        self.logger.setLevel(level)
        self.internalData = {}

        # sterr out
        if debug:
            handler = logging.StreamHandler(sys.stderr)
            handler.setFormatter(formatter)
            handler.setLevel(level)
            self.logger.addHandler(handler)
        handler = logging.FileHandler(self.appRoot + '/log/analyticsServer_' + datetime.now().strftime('%d-%m-%y') + '.log', 'a+')
        handler.setLevel(level)
        handler.setFormatter(formatter)
        self.logger.addHandler(handler)

        self.logger.info('AnalyticsServer started')

        # define app setting
        settings = {
            "cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
            "login_url": "/user/login",
            "static_url_prefix": "/static",
            "debug" : debug,
        }

        # call parent constuctor
        super(AnalyticsServer, self).__init__(handlers = self.getHandlers(), default_host = "", transforms = None,
            wsgi = False, **settings)

        self.loadConfiguration()
        self.determineIsInstall()
        self.jinjaEnvironment = Environment(loader = PackageLoader('static', 'template'))
        self.jinjaEnvironment.filters['datetofiled'] = datetofiled
        self.jinjaEnvironment.filters['smartDate'] = smartDate
        self.jinjaEnvironment.filters['smartDatePeriod'] = smartDatePeriod
        self.jinjaEnvironment.filters['smartDateInterval'] = smartDateInterval
        self.jinjaEnvironment.filters['excelDate'] = excelDate
        self.jinjaEnvironment.filters['excelTime'] = excelTime
        self.jinjaEnvironment.filters['dateFromTS'] = dateFromTS
        self.jinjaEnvironment.filters['toJsVar'] = toJsVar
        self.jinjaEnvironment.filters['minInt'] = minInt
        self.jinjaEnvironment.filters['numberFormat'] = numberFormat
        self.jinjaEnvironment.filters['showMonth'] = showMonth

        self.scoped_session = None
        self.loopInstance = None
        if self.isInstalled:
            self.initDBScopedSession()

    def initDBScopedSession(self):
        if not self.scoped_session:
            mysql_user = self.config.get(Config.MYSQL_USER)
            mysql_password = self.config.get(Config.MYSQL_PASSWORD)
            mysql_host = self.config.get(Config.MYSQL_HOST)
            mysql_dbname = self.config.get(Config.MYSQL_DBNAME)

            conn_str = 'mysql://'
            if mysql_user:
                conn_str += mysql_user
                if mysql_password:
                    conn_str += ':' + mysql_password
                conn_str += '@'
            conn_str += mysql_host + '/' + mysql_dbname

            engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
                                   encoding = 'utf8',
                                   convert_unicode = True,
                                   pool_recycle = 3600)
            engine.execute('SET NAMES utf8')
            self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))


    def start(self):
        port = int(self.config.get('app_port', 48888))
        address = self.config.get('app_host', "")
        self.listen(port = port, address = address )
        self.loopInstance = tornado.ioloop.IOLoop.instance()
        self.logger.info('application run {}:{}'.format(address, port))
        self.loopInstance.start()



    def loadConfiguration(self):
        self.config = Config()
        self.config.readConfigFile(os.path.abspath(self.appRoot + '/server.cfg'))

    def getAppConfigPath(self):
        return self.appRoot + '/app_configs/'

    def getResultPath(self):
        return self.appRoot + '/result/'

    def getLogPath(self):
        return self.appRoot + '/log/'

    def getTemplatePath(self):
        return self.appRoot + '/template/'

    def determineIsInstall(self):
        """
        Простое определение - установлено ли приложение
        """
        self.isInstalled =  os.path.exists(self.appRoot + '/server.cfg')

    def setData(self, key, value):
        self.internalData[key] = value

    def getData(self, key):
        return self.internalData[key]

    def deleteData(self, key):
        del self.internalData[key]

    def getHandlers(self):
        thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
        return [
            (r"/", IndexController.IndexAction),

            # --------- DASHBOARD -----------

            (r"/dashboard/?", DashboardController.SwitchApp),
            (r"/dashboard/app/([^/]+)/?", DashboardController.IndexAction),
            (r"/dashboard/app/([^/]+)/first/?", DashboardController.FirstAction),
            (r"/dashboard/app/([^/]+)/result/?", ResultController.ResultAction),
            (r"/dashboard/app/([^/]+)/result/table/(\d+)/?", ResultController.TableAction),
            (r"/dashboard/app/([^/]+)/new/?", CreateTaskController.CreateAction),
            (r"/dashboard/app/([^/]+)/recalculate/?", CreateTaskController.RecalculateAction),
            (r"/dashboard/app/([^/]+)/templates/?", TemplateController.IndexAction),
            (r"/dashboard/empty/?", DashboardController.EmptyAppAction),
            (r"/dashboard/selectapp/?", DashboardController.SelectAppAction),
            (r"/dashboard/app/([^/]+)/new_task/([^/]+)/?", CreateTaskController.ShowNewTaskAction),
            (r"/dashboard/app/([^/]+)/status/([^/]+)/?", ResultController.ShowTaskStatus),
            (r"/ajax/dashboard/app/([^/]+)/workers/([^/]+)/??", DashboardController.GetWorkers),

            # --------- LOGIN & LOGOUT ---------
            (r"/user/login/?", UserController.AuthAction),
            (r"/user/logout/?", UserController.LogoutAction),

            # --------- ADMIN -----------
            (r"/admin/?", AdminIndexController.IndexAction),
            (r"/admin/users/?", AdminUserController.UserAction),
            (r"/admin/users/edit?", AdminUserController.EditUserAction),
            (r"/admin/apps/?", AdminAppController.IndexAction),
            (r"/admin/app/edit/?", AdminAppController.EditAction),
            (r"/admin/rules/?", AdminRulesController.IndexAction),
            (r"/admin/rules/switch?", AdminRulesController.SwitchAjaxAction),
            (r'/admin/settings/?', AdminSettingsController.IndexAction),
            (r'/admin/cluster?', AdminClusterController.IndexAction),
            (r'/admin/historyServer/?', AdminProxy.HistoryServerView),
            (r'/admin/resourceManager/?', AdminProxy.ResourceManagerView),
            (r'/admin/proxy/([^/]+)/([^/]+)/(.+)?', AdminProxy.CoreProxy),






            (r'/admin/app/([^/]+)/settings/?', AdminAppConfigController.IndexAction),
            (r'/admin/app/([^/]+)/eventsList/?', AdminAppConfigController.EventListAction),
            (r"/admin/app/([^/]+)/editEvent/?", AdminAppConfigController.EditEvent),

            (r'/admin/app/([^/]+)/editTags/?', AdminAppConfigController.TagListAction),
            (r"/ajax/editTag/?", AdminAppConfigController.EditTag),


            (r'/admin/app/([^/]+)/editBunches/?', AdminAppConfigController.BunchListAction),
            (r'/ajax/editBunch/?', AdminAppConfigController.EditBunchAction),


            (r'/admin/app/([^/]+)/spell/?', AdminAppSpellController.IndexAction),
            (r'/admin/app/([^/]+)/groups/?', AdminAppGroupsController.IndexAction),
            # --------- CLUSTER -----------
            (r'/cluster/namenode/?', NameNodeController.IndexAction),

            # --------- AJAX -----------
            (r"/ajax/key_configuration/?", AjaxController.KeyConfigurationAction),
            (r"/ajax/get_key_form/?", AjaxController.GetKeyForm),
            (r"/ajax/getTasksProgress/?", AjaxController.GatTasksProgress),
            (r"/ajax/copyTaskKey/?", AjaxController.CopyTaskKey),
            (r"/ajax/saveWorkerName?", AjaxController.SaveWorkerName),


            #(r"/ajax/add_new_key?", AjaxController.AddNewKey),
            (r"/ajax/add_new_tag?", AjaxController.AddNewTag),
            (r"/ajax/add_new_bunch?", AjaxController.AddNewBunch),
            (r"/ajax/getDateSelector/?", AjaxController.GetDateSelector),


            (r"/ajax/getTagUniqueValues/?", HiveAJAXController.getTagUniqueValues),
            (r"/ajax/downloadCSV/?", AjaxController.DownloadCSVAction),

            (r"/ajax/admin/getClusterState/?", AdminClusterController.ClusterStateAction),

            # ----------- TEMPLATE ---------------
            #(r"/template/create/([^/]+)/?", TemplateController.CreateTemplateAction),

            # ----------- HDFS ---------------
            (r"/hdfs/?", HDFSController.IndexAction),
            (r"/hdfs/ajax/getPath/?", HDFSAJAXController.GetPathAction),
            (r"/hdfs/ajax/getPathStat/?", HDFSAJAXController.GetPathStat),
            (r"/hdfs/ajax/getHiveStat/?", HDFSAJAXController.GetHiveStatus),

            (r"/api/putConfig/?", APIController.PutConfigAction),

            (r"/install/?", InstallController.InstallAction),
            (r"/install/final/?", InstallController.FinalInstallAction),





            (r"/static/(.*)", tornado.web.StaticFileHandler, {"path": os.path.abspath(thisPath + '/../static')}),
        ]