def main():
while True:
now = datetime.now()
if (Config.is_business_hours(now) and now.minute == 0) or Config.debug:
print "assigning! is_business_hours: " + str(Config.is_business_hours(now)) + " " + str(now)
for data in Config.channels:
channel = Channel(data)
buffbot = BuffBot(channel)
buffbot.assign_exercise()
time.sleep(3)
else:
print now
time.sleep(59)
class CSVJoiner:
def __init__(self):
self.requester = Config()
def joinReport(self, json):
string_token_split = self.requester.load_auth_token()
print(string_token_split)
def get(self, *args, **kwargs):
self.config = Config()
# create folders
if not os.access(self.application.appRoot, os.W_OK):
self.errors.append(u'Корневая директория закрыта на запись! ('+self.application.appRoot + ')')
if not os.path.exists(self.application.getLogPath()):
try :
os.mkdir(self.application.getLogPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для логов:' + ex.message)
elif not os.access(self.application.getLogPath(), os.W_OK):
self.errors.append(u'Директория для логов создана, но закрыта на запись! (' + self.application.getLogPath() + ')')
if not os.path.exists(self.application.getAppConfigPath()):
try :
os.mkdir(self.application.getAppConfigPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для конфигураций:' + ex.message)
elif not os.access(self.application.getAppConfigPath(), os.W_OK):
self.errors.append(u'Директория для конфигураций создана, но закрыта на запись! (' + self.application.getAppConfigPath() + ')')
if not os.path.exists(self.application.getResultPath()):
try :
os.mkdir(self.application.getResultPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для результатов:' + ex.message)
elif not os.access(self.application.getResultPath(), os.W_OK):
self.errors.append(u'Директория для результатов создана, но закрыта на запись! (' + self.application.getResultPath() + ')')
self.run()
def __init__(self, options):
self.scoped_session = self.analyticsWebHDFS = self.hiveService = None
# set logger
logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.config = Config()
self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg')))
self.options = options
self.appService = AppService(rootPath + '/../../app_configs/')
self.availableApps = self.appService.getAppConfigList()
if options['appname'] is None and options['all_apps'] is None:
print 'App name or --all is not set'
print 'Availale app names: ' + str(self.availableApps)
self.terminate()
self.appCodes = []
if options['appname']:
self.appCodes.append(options['appname'])
else:
self.appCodes = self.availableApps
def main():
print("-----", os.getenv("account_list_file"), os.getenv("scraping_id"),
os.getenv("scraper_type"))
config = Config()
scraper = AccountScrapper(config)
while True:
scraper.scrap_one_day_in_each_account()
def post(self, *args, **kwargs):
self.config = Config( dict([(key, value.pop()) for key, value in self.request.arguments.items()]) )
# mysql settings
if not self.request.arguments.has_key(Config.MYSQL_HOST):
self.errors.append(u'Вы должны определить хост для MySQL')
elif not self.request.arguments.has_key(Config.MYSQL_DBNAME):
self.errors.append(u'Вы должны определить имя базы данных')
else:
conn_str = 'mysql://'
if self.config.get(Config.MYSQL_USER):
conn_str += self.config.get(Config.MYSQL_USER)
if self.config.get(Config.MYSQL_PASSWORD):
conn_str += ':' + self.config.get(Config.MYSQL_PASSWORD)
conn_str += '@'
conn_str += self.config.get(Config.MYSQL_HOST)
engine = create_engine(conn_str + '/?init_command=set%20names%20%22utf8%22', encoding='utf8', convert_unicode=True)
try:
connection = engine.connect()
except OperationalError as op_error:
self.errors.append(u'Ошибка соединения с MySQL: ' + op_error.message)
except BaseException as ex:
self.errors.append(u'Исключение SQLALCHEMY: ' + ex.message)
else:
try:
connection.execute('CREATE DATABASE IF NOT EXISTS `{0}` CHARACTER SET utf8 COLLATE utf8_general_ci'.
format(self.config.get(Config.MYSQL_DBNAME)))
except BaseException as ex:
self.errors.append(u'Исключение при создании БД: ' + ex.message)
finally:
connection.close()
if not self.errors:
# save config file
raw_config = self.config.getRawConfig()
try:
configfile = open(self.application.appRoot + '/server.cfg', 'wb')
except BaseException as exception:
self.errors.append(u'Ошибка при записи файла конфигурации: ' + exception.message)
else:
raw_config.write(configfile)
configfile.close()
self.redirect('/install/final')
return
self.run()
class InstallAction(InstallController):
def prepare(self):
super(InstallAction, self).prepare()
self.errors = []
def post(self, *args, **kwargs):
self.config = Config( dict([(key, value.pop()) for key, value in self.request.arguments.items()]) )
# mysql settings
if not self.request.arguments.has_key(Config.MYSQL_HOST):
self.errors.append(u'Вы должны определить хост для MySQL')
elif not self.request.arguments.has_key(Config.MYSQL_DBNAME):
self.errors.append(u'Вы должны определить имя базы данных')
else:
conn_str = 'mysql://'
if self.config.get(Config.MYSQL_USER):
conn_str += self.config.get(Config.MYSQL_USER)
if self.config.get(Config.MYSQL_PASSWORD):
conn_str += ':' + self.config.get(Config.MYSQL_PASSWORD)
conn_str += '@'
conn_str += self.config.get(Config.MYSQL_HOST)
engine = create_engine(conn_str + '/?init_command=set%20names%20%22utf8%22', encoding='utf8', convert_unicode=True)
try:
connection = engine.connect()
except OperationalError as op_error:
self.errors.append(u'Ошибка соединения с MySQL: ' + op_error.message)
except BaseException as ex:
self.errors.append(u'Исключение SQLALCHEMY: ' + ex.message)
else:
try:
connection.execute('CREATE DATABASE IF NOT EXISTS `{0}` CHARACTER SET utf8 COLLATE utf8_general_ci'.
format(self.config.get(Config.MYSQL_DBNAME)))
except BaseException as ex:
self.errors.append(u'Исключение при создании БД: ' + ex.message)
finally:
connection.close()
if not self.errors:
# save config file
raw_config = self.config.getRawConfig()
try:
configfile = open(self.application.appRoot + '/server.cfg', 'wb')
except BaseException as exception:
self.errors.append(u'Ошибка при записи файла конфигурации: ' + exception.message)
else:
raw_config.write(configfile)
configfile.close()
self.redirect('/install/final')
return
self.run()
def get(self, *args, **kwargs):
self.config = Config()
# create folders
if not os.access(self.application.appRoot, os.W_OK):
self.errors.append(u'Корневая директория закрыта на запись! ('+self.application.appRoot + ')')
if not os.path.exists(self.application.getLogPath()):
try :
os.mkdir(self.application.getLogPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для логов:' + ex.message)
elif not os.access(self.application.getLogPath(), os.W_OK):
self.errors.append(u'Директория для логов создана, но закрыта на запись! (' + self.application.getLogPath() + ')')
if not os.path.exists(self.application.getAppConfigPath()):
try :
os.mkdir(self.application.getAppConfigPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для конфигураций:' + ex.message)
elif not os.access(self.application.getAppConfigPath(), os.W_OK):
self.errors.append(u'Директория для конфигураций создана, но закрыта на запись! (' + self.application.getAppConfigPath() + ')')
if not os.path.exists(self.application.getResultPath()):
try :
os.mkdir(self.application.getResultPath(), 0644)
except BaseException as ex:
self.errors.append(u'Невозможно создать директорию для результатов:' + ex.message)
elif not os.access(self.application.getResultPath(), os.W_OK):
self.errors.append(u'Директория для результатов создана, но закрыта на запись! (' + self.application.getResultPath() + ')')
self.run()
def run(self):
self.render('install/install.jinja2', {'config': self.config, 'curdir': os.path.abspath(os.curdir),
'errors':self.errors})
def createFolder(self, folder):
path = os.path.abspath(folder)
if not os.path.exists(path):
os.makedirs(path, mode=0644)
return path
def test_load_config(self):
config = Config()
self.assertEqual('foo'.upper(), 'FOO')
def __init__(self):
self.requester = Config()
# coding=utf-8
__author__ = '*****@*****.**'
from models.Config import Config
from components.webhdfs import WebHDFS
import os, inspect
from datetime import datetime
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
config = Config()
config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../server.cfg')))
STATISTICS_ROOT = config.get('hdfs_statistic_path') + 'topface/'
HDFS_HOST = config.get('hdfs_host')
HDFS_PORT = int(config.get('hdfs_port'))
HDFS_USER = config.get('hdfs_username')
webHDFS = WebHDFS(HDFS_HOST, HDFS_PORT, HDFS_USER)
dirs = webHDFS.listdir(STATISTICS_ROOT)
def scanDir(dirname, eventcode, skip):
years = webHDFS.listdir(dirname)
for year in years:
months = webHDFS.listdir(dirname + '/' + year)
for month in months:
days = webHDFS.listdir(dirname + '/' + year + '/' + month)
for day in days:
if (int(year), int(month), int(day)) in skip:
continue
class BaseAnalyticsScript():
def __init__(self, options):
self.scoped_session = self.analyticsWebHDFS = self.hiveService = None
# set logger
logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
# set configurations
rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.config = Config()
self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg')))
self.options = options
self.appService = AppService(rootPath + '/../../app_configs/')
self.availableApps = self.appService.getAppConfigList()
if options['appname'] is None and options['all_apps'] is None:
print 'App name or --all is not set'
print 'Availale app names: ' + str(self.availableApps)
self.terminate()
self.appCodes = []
if options['appname']:
self.appCodes.append(options['appname'])
else:
self.appCodes = self.availableApps
def getAppCodes(self):
return self.appCodes
def getAppConfig(self, appCode):
"""
-> AppConfig
"""
return self.appService.getAppConfig(appCode)
def getApp(self, appCode):
return self.getDBSession().query(App).filter_by(code = appCode).first()
def getWebHDFSClient(self):
if not self.analyticsWebHDFS:
host = self.config.get(Config.HDFS_HOST)
port = int(self.config.get(Config.HDFS_PORT))
username = self.config.get(Config.HDFS_USERNAME)
statRoot = self.config.get(Config.HDFS_STAT_ROOT)
self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot)
return self.analyticsWebHDFS
def getHiveClient(self):
if not self.hiveService:
self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT)))
return self.hiveService
def getDBSession(self):
if not self.scoped_session:
mysql_user = self.config.get(Config.MYSQL_USER)
mysql_password = self.config.get(Config.MYSQL_PASSWORD)
mysql_host = self.config.get(Config.MYSQL_HOST)
mysql_dbname = self.config.get(Config.MYSQL_DBNAME)
conn_str = 'mysql://'
if mysql_user:
conn_str += mysql_user
if mysql_password:
conn_str += ':' + mysql_password
conn_str += '@'
conn_str += mysql_host + '/' + mysql_dbname
engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
encoding = 'utf8',
convert_unicode = True,
pool_recycle = 3600)
engine.execute('SET NAMES utf8')
self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))
return self.scoped_session()
def terminate(self):
exit()
class AnalyticsServer(tornado.web.Application):
spellFolder = None
def __init__(self, debug = False):
# define app root path
thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
self.appRoot = os.path.abspath(thisPath + '/../../')
self.spellFolder = self.appRoot + '/app_configs/spells/'
level = logging.WARNING
if debug:
level = logging.INFO
formatter = logging.Formatter('%(asctime)s %(filename)s(%(lineno)s)[%(funcName)s] %(threadName)s %(levelname)-8s %(message)s')
self.logger = logging.getLogger('AnalyticsServer')
self.logger.setLevel(level)
self.internalData = {}
# sterr out
if debug:
handler = logging.StreamHandler(sys.stderr)
handler.setFormatter(formatter)
handler.setLevel(level)
self.logger.addHandler(handler)
handler = logging.FileHandler(self.appRoot + '/log/analyticsServer_' + datetime.now().strftime('%d-%m-%y') + '.log', 'a+')
handler.setLevel(level)
handler.setFormatter(formatter)
self.logger.addHandler(handler)
self.logger.info('AnalyticsServer started')
# define app setting
settings = {
"cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
"login_url": "/user/login",
"static_url_prefix": "/static",
"debug" : debug,
}
# call parent constuctor
super(AnalyticsServer, self).__init__(handlers = self.getHandlers(), default_host = "", transforms = None,
wsgi = False, **settings)
self.loadConfiguration()
self.determineIsInstall()
self.jinjaEnvironment = Environment(loader = PackageLoader('static', 'template'))
self.jinjaEnvironment.filters['datetofiled'] = datetofiled
self.jinjaEnvironment.filters['smartDate'] = smartDate
self.jinjaEnvironment.filters['smartDatePeriod'] = smartDatePeriod
self.jinjaEnvironment.filters['smartDateInterval'] = smartDateInterval
self.jinjaEnvironment.filters['excelDate'] = excelDate
self.jinjaEnvironment.filters['excelTime'] = excelTime
self.jinjaEnvironment.filters['dateFromTS'] = dateFromTS
self.jinjaEnvironment.filters['toJsVar'] = toJsVar
self.jinjaEnvironment.filters['minInt'] = minInt
self.jinjaEnvironment.filters['numberFormat'] = numberFormat
self.jinjaEnvironment.filters['showMonth'] = showMonth
self.scoped_session = None
self.loopInstance = None
if self.isInstalled:
self.initDBScopedSession()
def initDBScopedSession(self):
if not self.scoped_session:
mysql_user = self.config.get(Config.MYSQL_USER)
mysql_password = self.config.get(Config.MYSQL_PASSWORD)
mysql_host = self.config.get(Config.MYSQL_HOST)
mysql_dbname = self.config.get(Config.MYSQL_DBNAME)
conn_str = 'mysql://'
if mysql_user:
conn_str += mysql_user
if mysql_password:
conn_str += ':' + mysql_password
conn_str += '@'
conn_str += mysql_host + '/' + mysql_dbname
engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22',
encoding = 'utf8',
convert_unicode = True,
pool_recycle = 3600)
engine.execute('SET NAMES utf8')
self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False))
def start(self):
port = int(self.config.get('app_port', 48888))
address = self.config.get('app_host', "")
self.listen(port = port, address = address )
self.loopInstance = tornado.ioloop.IOLoop.instance()
self.logger.info('application run {}:{}'.format(address, port))
self.loopInstance.start()
def loadConfiguration(self):
self.config = Config()
self.config.readConfigFile(os.path.abspath(self.appRoot + '/server.cfg'))
def getAppConfigPath(self):
return self.appRoot + '/app_configs/'
def getResultPath(self):
return self.appRoot + '/result/'
def getLogPath(self):
return self.appRoot + '/log/'
def getTemplatePath(self):
return self.appRoot + '/template/'
def determineIsInstall(self):
"""
Простое определение - установлено ли приложение
"""
self.isInstalled = os.path.exists(self.appRoot + '/server.cfg')
def setData(self, key, value):
self.internalData[key] = value
def getData(self, key):
return self.internalData[key]
def deleteData(self, key):
del self.internalData[key]
def getHandlers(self):
thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory
return [
(r"/", IndexController.IndexAction),
# --------- DASHBOARD -----------
(r"/dashboard/?", DashboardController.SwitchApp),
(r"/dashboard/app/([^/]+)/?", DashboardController.IndexAction),
(r"/dashboard/app/([^/]+)/first/?", DashboardController.FirstAction),
(r"/dashboard/app/([^/]+)/result/?", ResultController.ResultAction),
(r"/dashboard/app/([^/]+)/result/table/(\d+)/?", ResultController.TableAction),
(r"/dashboard/app/([^/]+)/new/?", CreateTaskController.CreateAction),
(r"/dashboard/app/([^/]+)/recalculate/?", CreateTaskController.RecalculateAction),
(r"/dashboard/app/([^/]+)/templates/?", TemplateController.IndexAction),
(r"/dashboard/empty/?", DashboardController.EmptyAppAction),
(r"/dashboard/selectapp/?", DashboardController.SelectAppAction),
(r"/dashboard/app/([^/]+)/new_task/([^/]+)/?", CreateTaskController.ShowNewTaskAction),
(r"/dashboard/app/([^/]+)/status/([^/]+)/?", ResultController.ShowTaskStatus),
(r"/ajax/dashboard/app/([^/]+)/workers/([^/]+)/??", DashboardController.GetWorkers),
# --------- LOGIN & LOGOUT ---------
(r"/user/login/?", UserController.AuthAction),
(r"/user/logout/?", UserController.LogoutAction),
# --------- ADMIN -----------
(r"/admin/?", AdminIndexController.IndexAction),
(r"/admin/users/?", AdminUserController.UserAction),
(r"/admin/users/edit?", AdminUserController.EditUserAction),
(r"/admin/apps/?", AdminAppController.IndexAction),
(r"/admin/app/edit/?", AdminAppController.EditAction),
(r"/admin/rules/?", AdminRulesController.IndexAction),
(r"/admin/rules/switch?", AdminRulesController.SwitchAjaxAction),
(r'/admin/settings/?', AdminSettingsController.IndexAction),
(r'/admin/cluster?', AdminClusterController.IndexAction),
(r'/admin/historyServer/?', AdminProxy.HistoryServerView),
(r'/admin/resourceManager/?', AdminProxy.ResourceManagerView),
(r'/admin/proxy/([^/]+)/([^/]+)/(.+)?', AdminProxy.CoreProxy),
(r'/admin/app/([^/]+)/settings/?', AdminAppConfigController.IndexAction),
(r'/admin/app/([^/]+)/eventsList/?', AdminAppConfigController.EventListAction),
(r"/admin/app/([^/]+)/editEvent/?", AdminAppConfigController.EditEvent),
(r'/admin/app/([^/]+)/editTags/?', AdminAppConfigController.TagListAction),
(r"/ajax/editTag/?", AdminAppConfigController.EditTag),
(r'/admin/app/([^/]+)/editBunches/?', AdminAppConfigController.BunchListAction),
(r'/ajax/editBunch/?', AdminAppConfigController.EditBunchAction),
(r'/admin/app/([^/]+)/spell/?', AdminAppSpellController.IndexAction),
(r'/admin/app/([^/]+)/groups/?', AdminAppGroupsController.IndexAction),
# --------- CLUSTER -----------
(r'/cluster/namenode/?', NameNodeController.IndexAction),
# --------- AJAX -----------
(r"/ajax/key_configuration/?", AjaxController.KeyConfigurationAction),
(r"/ajax/get_key_form/?", AjaxController.GetKeyForm),
(r"/ajax/getTasksProgress/?", AjaxController.GatTasksProgress),
(r"/ajax/copyTaskKey/?", AjaxController.CopyTaskKey),
(r"/ajax/saveWorkerName?", AjaxController.SaveWorkerName),
#(r"/ajax/add_new_key?", AjaxController.AddNewKey),
(r"/ajax/add_new_tag?", AjaxController.AddNewTag),
(r"/ajax/add_new_bunch?", AjaxController.AddNewBunch),
(r"/ajax/getDateSelector/?", AjaxController.GetDateSelector),
(r"/ajax/getTagUniqueValues/?", HiveAJAXController.getTagUniqueValues),
(r"/ajax/downloadCSV/?", AjaxController.DownloadCSVAction),
(r"/ajax/admin/getClusterState/?", AdminClusterController.ClusterStateAction),
# ----------- TEMPLATE ---------------
#(r"/template/create/([^/]+)/?", TemplateController.CreateTemplateAction),
# ----------- HDFS ---------------
(r"/hdfs/?", HDFSController.IndexAction),
(r"/hdfs/ajax/getPath/?", HDFSAJAXController.GetPathAction),
(r"/hdfs/ajax/getPathStat/?", HDFSAJAXController.GetPathStat),
(r"/hdfs/ajax/getHiveStat/?", HDFSAJAXController.GetHiveStatus),
(r"/api/putConfig/?", APIController.PutConfigAction),
(r"/install/?", InstallController.InstallAction),
(r"/install/final/?", InstallController.FinalInstallAction),
(r"/static/(.*)", tornado.web.StaticFileHandler, {"path": os.path.abspath(thisPath + '/../static')}),
]
def loadConfiguration(self):
self.config = Config()
self.config.readConfigFile(os.path.abspath(self.appRoot + '/server.cfg'))
def run_sensors():
config = Config()
print("running sensors")
camera = Camera(config)
camera.shot_periodically()
def test_load_image(self):
config = Config()
image = Image(config,"Config.py")
image.upload_image()
self.assertEqual('foo'.upper(), 'FOO')
torch.backends.cudnn.deterministic = True
dataset = args.datasetdir
outputdir = args.outputdir
embedding = ''
if args.embedding == 'random':
embedding = 'random'
else:
embedding = args.embedding
model_name = args.model
print(model_name)
x = import_module('models.' + model_name)
config = Config(dataset, outputdir, embedding)
# reset config
config.model_name = args.model
config.save_path = os.path.join(outputdir, args.model + '.ckpt')
config.log_path = os.path.join(outputdir, args.model + '.log')
config.dropout = float(args.dropout)
config.require_improvement = int(args.require_improvement)
config.num_epochs = int(args.num_epochs)
config.batch_size = int(args.batch_size)
config.max_length = int(args.max_length)
config.learning_rate = float(args.learning_rate)
config.embed = int(args.embed_dim)
config.bucket = int(args.bucket)
config.wordNgrams = int(args.wordNgrams)
config.lr_decay_rate = float(args.lr_decay_rate)
