class BaseAnalyticsScript():
def __init__(self, options):
self.scoped_session = self.analyticsWebHDFS = self.hiveService = None
# set logger
logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.config = Config()
self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg')))
self.options = options
self.appService = AppService(rootPath + '/../../app_configs/')
self.availableApps = self.appService.getAppConfigList()
if options['appname'] is None and options['all_apps'] is None:
print 'App name or --all is not set'
print 'Availale app names: ' + str(self.availableApps)
self.terminate()
self.appCodes = []
if options['appname']:
self.appCodes.append(options['appname'])
else:
self.appCodes = self.availableApps
def getAppCodes(self):
return self.appCodes
def getAppConfig(self, appCode):
"""
-> AppConfig
"""
return self.appService.getAppConfig(appCode)
def getApp(self, appCode):
return self.getDBSession().query(App).filter_by(code = appCode).first()
def getWebHDFSClient(self):
if not self.analyticsWebHDFS:
host = self.config.get(Config.HDFS_HOST)
port = int(self.config.get(Config.HDFS_PORT))
username = self.config.get(Config.HDFS_USERNAME)
statRoot = self.config.get(Config.HDFS_STAT_ROOT)
self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot)
return self.analyticsWebHDFS
def getHiveClient(self):
if not self.hiveService:
self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT)))
return self.hiveService
def getDBSession(self):
if not self.scoped_session:
mysql_user = self.config.get(Config.MYSQL_USER)
mysql_password = self.config.get(Config.MYSQL_PASSWORD)
mysql_host = self.config.get(Config.MYSQL_HOST)
mysql_dbname = self.config.get(Config.MYSQL_DBNAME)
conn_str = 'mysql://'
if mysql_user:
conn_str += mysql_user
if mysql_password:
conn_str += ':' + mysql_password
conn_str += '@'
conn_str += mysql_host + '/' + mysql_dbname
engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
encoding = 'utf8',
convert_unicode = True,
pool_recycle = 3600)
engine.execute('SET NAMES utf8')
self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))
return self.scoped_session()
def terminate(self):
exit()
class InstallAction(InstallController):
def prepare(self):
super(InstallAction, self).prepare()
self.errors = []
def post(self, *args, **kwargs):
self.config = Config( dict([(key, value.pop()) for key, value in self.request.arguments.items()]) )
# mysql settings
if not self.request.arguments.has_key(Config.MYSQL_HOST):
self.errors.append(u'Вы должны определить хост для MySQL')
elif not self.request.arguments.has_key(Config.MYSQL_DBNAME):
self.errors.append(u'Вы должны определить имя базы данных')
else:
conn_str = 'mysql://'
if self.config.get(Config.MYSQL_USER):
conn_str += self.config.get(Config.MYSQL_USER)
if self.config.get(Config.MYSQL_PASSWORD):
conn_str += ':' + self.config.get(Config.MYSQL_PASSWORD)
conn_str += '@'
conn_str += self.config.get(Config.MYSQL_HOST)
engine = create_engine(conn_str + '/?init_command=set%20names%20%22utf8%22', encoding='utf8', convert_unicode=True)
try:
connection = engine.connect()
except OperationalError as op_error:
self.errors.append(u'Ошибка соединения с MySQL: ' + op_error.message)
except BaseException as ex:
self.errors.append(u'Исключение SQLALCHEMY: ' + ex.message)
else:
try:
connection.execute('CREATE DATABASE IF NOT EXISTS `{0}` CHARACTER SET utf8 COLLATE utf8_general_ci'.
format(self.config.get(Config.MYSQL_DBNAME)))
except BaseException as ex:
self.errors.append(u'Исключение при создании БД: ' + ex.message)
finally:
connection.close()
if not self.errors:
# save config file
raw_config = self.config.getRawConfig()
try:
configfile = open(self.application.appRoot + '/server.cfg', 'wb')
except BaseException as exception:
self.errors.append(u'Ошибка при записи файла конфигурации: ' + exception.message)
else:
raw_config.write(configfile)
configfile.close()
self.redirect('/install/final')
return
self.run()
def get(self, *args, **kwargs):
self.config = Config()
# create folders
if not os.access(self.application.appRoot, os.W_OK):
self.errors.append(u'Корневая директория закрыта на запись! ('+self.application.appRoot + ')')
if not os.path.exists(self.application.getLogPath()):
try :
os.mkdir(self.application.getLogPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для логов:' + ex.message)
elif not os.access(self.application.getLogPath(), os.W_OK):
self.errors.append(u'Директория для логов создана, но закрыта на запись! (' + self.application.getLogPath() + ')')
if not os.path.exists(self.application.getAppConfigPath()):
try :
os.mkdir(self.application.getAppConfigPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для конфигураций:' + ex.message)
elif not os.access(self.application.getAppConfigPath(), os.W_OK):
self.errors.append(u'Директория для конфигураций создана, но закрыта на запись! (' + self.application.getAppConfigPath() + ')')
if not os.path.exists(self.application.getResultPath()):
try :
os.mkdir(self.application.getResultPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для результатов:' + ex.message)
elif not os.access(self.application.getResultPath(), os.W_OK):
self.errors.append(u'Директория для результатов создана, но закрыта на запись! (' + self.application.getResultPath() + ')')
self.run()
def run(self):
self.render('install/install.jinja2', {'config': self.config, 'curdir': os.path.abspath(os.curdir),
'errors':self.errors})
def createFolder(self, folder):
path = os.path.abspath(folder)
if not os.path.exists(path):
os.makedirs(path, mode=0644)
return path
# coding=utf-8
__author__ = '*****@*****.**'
from models.Config import Config
from components.webhdfs import WebHDFS
import os, inspect
from datetime import datetime
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
config = Config()
config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../server.cfg')))
STATISTICS_ROOT = config.get('hdfs_statistic_path') + 'topface/'
HDFS_HOST = config.get('hdfs_host')
HDFS_PORT = int(config.get('hdfs_port'))
HDFS_USER = config.get('hdfs_username')
webHDFS = WebHDFS(HDFS_HOST, HDFS_PORT, HDFS_USER)
dirs = webHDFS.listdir(STATISTICS_ROOT)
def scanDir(dirname, eventcode, skip):
years = webHDFS.listdir(dirname)
for year in years:
months = webHDFS.listdir(dirname + '/' + year)
for month in months:
days = webHDFS.listdir(dirname + '/' + year + '/' + month)
for day in days:
if (int(year), int(month), int(day)) in skip:
continue
