class BaseAnalyticsScript(): def __init__(self, options): self.scoped_session = self.analyticsWebHDFS = self.hiveService = None # set logger logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') # set configurations rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory self.config = Config() self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg'))) self.options = options self.appService = AppService(rootPath + '/../../app_configs/') self.availableApps = self.appService.getAppConfigList() if options['appname'] is None and options['all_apps'] is None: print 'App name or --all is not set' print 'Availale app names: ' + str(self.availableApps) self.terminate() self.appCodes = [] if options['appname']: self.appCodes.append(options['appname']) else: self.appCodes = self.availableApps def getAppCodes(self): return self.appCodes def getAppConfig(self, appCode): """ -> AppConfig """ return self.appService.getAppConfig(appCode) def getApp(self, appCode): return self.getDBSession().query(App).filter_by(code = appCode).first() def getWebHDFSClient(self): if not self.analyticsWebHDFS: host = self.config.get(Config.HDFS_HOST) port = int(self.config.get(Config.HDFS_PORT)) username = self.config.get(Config.HDFS_USERNAME) statRoot = self.config.get(Config.HDFS_STAT_ROOT) self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot) return self.analyticsWebHDFS def getHiveClient(self): if not self.hiveService: self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT))) return self.hiveService def getDBSession(self): if not self.scoped_session: mysql_user = self.config.get(Config.MYSQL_USER) mysql_password = self.config.get(Config.MYSQL_PASSWORD) mysql_host = self.config.get(Config.MYSQL_HOST) mysql_dbname = self.config.get(Config.MYSQL_DBNAME) conn_str = 'mysql://' if mysql_user: conn_str += mysql_user if mysql_password: conn_str += ':' + mysql_password conn_str += '@' conn_str += mysql_host + '/' + mysql_dbname engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22', encoding = 'utf8', convert_unicode = True, pool_recycle = 3600) engine.execute('SET NAMES utf8') self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False)) return self.scoped_session() def terminate(self): exit()
class InstallAction(InstallController): def prepare(self): super(InstallAction, self).prepare() self.errors = [] def post(self, *args, **kwargs): self.config = Config( dict([(key, value.pop()) for key, value in self.request.arguments.items()]) ) # mysql settings if not self.request.arguments.has_key(Config.MYSQL_HOST): self.errors.append(u'Вы должны определить хост для MySQL') elif not self.request.arguments.has_key(Config.MYSQL_DBNAME): self.errors.append(u'Вы должны определить имя базы данных') else: conn_str = 'mysql://' if self.config.get(Config.MYSQL_USER): conn_str += self.config.get(Config.MYSQL_USER) if self.config.get(Config.MYSQL_PASSWORD): conn_str += ':' + self.config.get(Config.MYSQL_PASSWORD) conn_str += '@' conn_str += self.config.get(Config.MYSQL_HOST) engine = create_engine(conn_str + '/?init_command=set%20names%20%22utf8%22', encoding='utf8', convert_unicode=True) try: connection = engine.connect() except OperationalError as op_error: self.errors.append(u'Ошибка соединения с MySQL: ' + op_error.message) except BaseException as ex: self.errors.append(u'Исключение SQLALCHEMY: ' + ex.message) else: try: connection.execute('CREATE DATABASE IF NOT EXISTS `{0}` CHARACTER SET utf8 COLLATE utf8_general_ci'. format(self.config.get(Config.MYSQL_DBNAME))) except BaseException as ex: self.errors.append(u'Исключение при создании БД: ' + ex.message) finally: connection.close() if not self.errors: # save config file raw_config = self.config.getRawConfig() try: configfile = open(self.application.appRoot + '/server.cfg', 'wb') except BaseException as exception: self.errors.append(u'Ошибка при записи файла конфигурации: ' + exception.message) else: raw_config.write(configfile) configfile.close() self.redirect('/install/final') return self.run() def get(self, *args, **kwargs): self.config = Config() # create folders if not os.access(self.application.appRoot, os.W_OK): self.errors.append(u'Корневая директория закрыта на запись! ('+self.application.appRoot + ')') if not os.path.exists(self.application.getLogPath()): try : os.mkdir(self.application.getLogPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для логов:' + ex.message) elif not os.access(self.application.getLogPath(), os.W_OK): self.errors.append(u'Директория для логов создана, но закрыта на запись! (' + self.application.getLogPath() + ')') if not os.path.exists(self.application.getAppConfigPath()): try : os.mkdir(self.application.getAppConfigPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для конфигураций:' + ex.message) elif not os.access(self.application.getAppConfigPath(), os.W_OK): self.errors.append(u'Директория для конфигураций создана, но закрыта на запись! (' + self.application.getAppConfigPath() + ')') if not os.path.exists(self.application.getResultPath()): try : os.mkdir(self.application.getResultPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для результатов:' + ex.message) elif not os.access(self.application.getResultPath(), os.W_OK): self.errors.append(u'Директория для результатов создана, но закрыта на запись! (' + self.application.getResultPath() + ')') self.run() def run(self): self.render('install/install.jinja2', {'config': self.config, 'curdir': os.path.abspath(os.curdir), 'errors':self.errors}) def createFolder(self, folder): path = os.path.abspath(folder) if not os.path.exists(path): os.makedirs(path, mode=0644) return path
# coding=utf-8 __author__ = '*****@*****.**' from models.Config import Config from components.webhdfs import WebHDFS import os, inspect from datetime import datetime # set configurations rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory config = Config() config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../server.cfg'))) STATISTICS_ROOT = config.get('hdfs_statistic_path') + 'topface/' HDFS_HOST = config.get('hdfs_host') HDFS_PORT = int(config.get('hdfs_port')) HDFS_USER = config.get('hdfs_username') webHDFS = WebHDFS(HDFS_HOST, HDFS_PORT, HDFS_USER) dirs = webHDFS.listdir(STATISTICS_ROOT) def scanDir(dirname, eventcode, skip): years = webHDFS.listdir(dirname) for year in years: months = webHDFS.listdir(dirname + '/' + year) for month in months: days = webHDFS.listdir(dirname + '/' + year + '/' + month) for day in days: if (int(year), int(month), int(day)) in skip: continue