def main(): while True: now = datetime.now() if (Config.is_business_hours(now) and now.minute == 0) or Config.debug: print "assigning! is_business_hours: " + str(Config.is_business_hours(now)) + " " + str(now) for data in Config.channels: channel = Channel(data) buffbot = BuffBot(channel) buffbot.assign_exercise() time.sleep(3) else: print now time.sleep(59)
class CSVJoiner: def __init__(self): self.requester = Config() def joinReport(self, json): string_token_split = self.requester.load_auth_token() print(string_token_split)
def get(self, *args, **kwargs): self.config = Config() # create folders if not os.access(self.application.appRoot, os.W_OK): self.errors.append(u'Корневая директория закрыта на запись! ('+self.application.appRoot + ')') if not os.path.exists(self.application.getLogPath()): try : os.mkdir(self.application.getLogPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для логов:' + ex.message) elif not os.access(self.application.getLogPath(), os.W_OK): self.errors.append(u'Директория для логов создана, но закрыта на запись! (' + self.application.getLogPath() + ')') if not os.path.exists(self.application.getAppConfigPath()): try : os.mkdir(self.application.getAppConfigPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для конфигураций:' + ex.message) elif not os.access(self.application.getAppConfigPath(), os.W_OK): self.errors.append(u'Директория для конфигураций создана, но закрыта на запись! (' + self.application.getAppConfigPath() + ')') if not os.path.exists(self.application.getResultPath()): try : os.mkdir(self.application.getResultPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для результатов:' + ex.message) elif not os.access(self.application.getResultPath(), os.W_OK): self.errors.append(u'Директория для результатов создана, но закрыта на запись! (' + self.application.getResultPath() + ')') self.run()
def __init__(self, options): self.scoped_session = self.analyticsWebHDFS = self.hiveService = None # set logger logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') # set configurations rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory self.config = Config() self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg'))) self.options = options self.appService = AppService(rootPath + '/../../app_configs/') self.availableApps = self.appService.getAppConfigList() if options['appname'] is None and options['all_apps'] is None: print 'App name or --all is not set' print 'Availale app names: ' + str(self.availableApps) self.terminate() self.appCodes = [] if options['appname']: self.appCodes.append(options['appname']) else: self.appCodes = self.availableApps
def main(): print("-----", os.getenv("account_list_file"), os.getenv("scraping_id"), os.getenv("scraper_type")) config = Config() scraper = AccountScrapper(config) while True: scraper.scrap_one_day_in_each_account()
def post(self, *args, **kwargs): self.config = Config( dict([(key, value.pop()) for key, value in self.request.arguments.items()]) ) # mysql settings if not self.request.arguments.has_key(Config.MYSQL_HOST): self.errors.append(u'Вы должны определить хост для MySQL') elif not self.request.arguments.has_key(Config.MYSQL_DBNAME): self.errors.append(u'Вы должны определить имя базы данных') else: conn_str = 'mysql://' if self.config.get(Config.MYSQL_USER): conn_str += self.config.get(Config.MYSQL_USER) if self.config.get(Config.MYSQL_PASSWORD): conn_str += ':' + self.config.get(Config.MYSQL_PASSWORD) conn_str += '@' conn_str += self.config.get(Config.MYSQL_HOST) engine = create_engine(conn_str + '/?init_command=set%20names%20%22utf8%22', encoding='utf8', convert_unicode=True) try: connection = engine.connect() except OperationalError as op_error: self.errors.append(u'Ошибка соединения с MySQL: ' + op_error.message) except BaseException as ex: self.errors.append(u'Исключение SQLALCHEMY: ' + ex.message) else: try: connection.execute('CREATE DATABASE IF NOT EXISTS `{0}` CHARACTER SET utf8 COLLATE utf8_general_ci'. format(self.config.get(Config.MYSQL_DBNAME))) except BaseException as ex: self.errors.append(u'Исключение при создании БД: ' + ex.message) finally: connection.close() if not self.errors: # save config file raw_config = self.config.getRawConfig() try: configfile = open(self.application.appRoot + '/server.cfg', 'wb') except BaseException as exception: self.errors.append(u'Ошибка при записи файла конфигурации: ' + exception.message) else: raw_config.write(configfile) configfile.close() self.redirect('/install/final') return self.run()
class InstallAction(InstallController): def prepare(self): super(InstallAction, self).prepare() self.errors = [] def post(self, *args, **kwargs): self.config = Config( dict([(key, value.pop()) for key, value in self.request.arguments.items()]) ) # mysql settings if not self.request.arguments.has_key(Config.MYSQL_HOST): self.errors.append(u'Вы должны определить хост для MySQL') elif not self.request.arguments.has_key(Config.MYSQL_DBNAME): self.errors.append(u'Вы должны определить имя базы данных') else: conn_str = 'mysql://' if self.config.get(Config.MYSQL_USER): conn_str += self.config.get(Config.MYSQL_USER) if self.config.get(Config.MYSQL_PASSWORD): conn_str += ':' + self.config.get(Config.MYSQL_PASSWORD) conn_str += '@' conn_str += self.config.get(Config.MYSQL_HOST) engine = create_engine(conn_str + '/?init_command=set%20names%20%22utf8%22', encoding='utf8', convert_unicode=True) try: connection = engine.connect() except OperationalError as op_error: self.errors.append(u'Ошибка соединения с MySQL: ' + op_error.message) except BaseException as ex: self.errors.append(u'Исключение SQLALCHEMY: ' + ex.message) else: try: connection.execute('CREATE DATABASE IF NOT EXISTS `{0}` CHARACTER SET utf8 COLLATE utf8_general_ci'. format(self.config.get(Config.MYSQL_DBNAME))) except BaseException as ex: self.errors.append(u'Исключение при создании БД: ' + ex.message) finally: connection.close() if not self.errors: # save config file raw_config = self.config.getRawConfig() try: configfile = open(self.application.appRoot + '/server.cfg', 'wb') except BaseException as exception: self.errors.append(u'Ошибка при записи файла конфигурации: ' + exception.message) else: raw_config.write(configfile) configfile.close() self.redirect('/install/final') return self.run() def get(self, *args, **kwargs): self.config = Config() # create folders if not os.access(self.application.appRoot, os.W_OK): self.errors.append(u'Корневая директория закрыта на запись! ('+self.application.appRoot + ')') if not os.path.exists(self.application.getLogPath()): try : os.mkdir(self.application.getLogPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для логов:' + ex.message) elif not os.access(self.application.getLogPath(), os.W_OK): self.errors.append(u'Директория для логов создана, но закрыта на запись! (' + self.application.getLogPath() + ')') if not os.path.exists(self.application.getAppConfigPath()): try : os.mkdir(self.application.getAppConfigPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для конфигураций:' + ex.message) elif not os.access(self.application.getAppConfigPath(), os.W_OK): self.errors.append(u'Директория для конфигураций создана, но закрыта на запись! (' + self.application.getAppConfigPath() + ')') if not os.path.exists(self.application.getResultPath()): try : os.mkdir(self.application.getResultPath(), 0644) except BaseException as ex: self.errors.append(u'Невозможно создать директорию для результатов:' + ex.message) elif not os.access(self.application.getResultPath(), os.W_OK): self.errors.append(u'Директория для результатов создана, но закрыта на запись! (' + self.application.getResultPath() + ')') self.run() def run(self): self.render('install/install.jinja2', {'config': self.config, 'curdir': os.path.abspath(os.curdir), 'errors':self.errors}) def createFolder(self, folder): path = os.path.abspath(folder) if not os.path.exists(path): os.makedirs(path, mode=0644) return path
def test_load_config(self): config = Config() self.assertEqual('foo'.upper(), 'FOO')
def __init__(self): self.requester = Config()
# coding=utf-8 __author__ = '*****@*****.**' from models.Config import Config from components.webhdfs import WebHDFS import os, inspect from datetime import datetime # set configurations rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory config = Config() config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../server.cfg'))) STATISTICS_ROOT = config.get('hdfs_statistic_path') + 'topface/' HDFS_HOST = config.get('hdfs_host') HDFS_PORT = int(config.get('hdfs_port')) HDFS_USER = config.get('hdfs_username') webHDFS = WebHDFS(HDFS_HOST, HDFS_PORT, HDFS_USER) dirs = webHDFS.listdir(STATISTICS_ROOT) def scanDir(dirname, eventcode, skip): years = webHDFS.listdir(dirname) for year in years: months = webHDFS.listdir(dirname + '/' + year) for month in months: days = webHDFS.listdir(dirname + '/' + year + '/' + month) for day in days: if (int(year), int(month), int(day)) in skip: continue
class BaseAnalyticsScript(): def __init__(self, options): self.scoped_session = self.analyticsWebHDFS = self.hiveService = None # set logger logging.basicConfig(level = logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s') # set configurations rootPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory self.config = Config() self.config.readConfigFile(os.path.abspath(os.path.abspath(rootPath + '/../../server.cfg'))) self.options = options self.appService = AppService(rootPath + '/../../app_configs/') self.availableApps = self.appService.getAppConfigList() if options['appname'] is None and options['all_apps'] is None: print 'App name or --all is not set' print 'Availale app names: ' + str(self.availableApps) self.terminate() self.appCodes = [] if options['appname']: self.appCodes.append(options['appname']) else: self.appCodes = self.availableApps def getAppCodes(self): return self.appCodes def getAppConfig(self, appCode): """ -> AppConfig """ return self.appService.getAppConfig(appCode) def getApp(self, appCode): return self.getDBSession().query(App).filter_by(code = appCode).first() def getWebHDFSClient(self): if not self.analyticsWebHDFS: host = self.config.get(Config.HDFS_HOST) port = int(self.config.get(Config.HDFS_PORT)) username = self.config.get(Config.HDFS_USERNAME) statRoot = self.config.get(Config.HDFS_STAT_ROOT) self.analyticsWebHDFS = AnalyticsWebHDFS(host, port, username, statRoot) return self.analyticsWebHDFS def getHiveClient(self): if not self.hiveService: self.hiveService = HiveService(self.config.get(Config.HIVE_HOST), int(self.config.get(Config.HIVE_PORT))) return self.hiveService def getDBSession(self): if not self.scoped_session: mysql_user = self.config.get(Config.MYSQL_USER) mysql_password = self.config.get(Config.MYSQL_PASSWORD) mysql_host = self.config.get(Config.MYSQL_HOST) mysql_dbname = self.config.get(Config.MYSQL_DBNAME) conn_str = 'mysql://' if mysql_user: conn_str += mysql_user if mysql_password: conn_str += ':' + mysql_password conn_str += '@' conn_str += mysql_host + '/' + mysql_dbname engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22', encoding = 'utf8', convert_unicode = True, pool_recycle = 3600) engine.execute('SET NAMES utf8') self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False)) return self.scoped_session() def terminate(self): exit()
class AnalyticsServer(tornado.web.Application): spellFolder = None def __init__(self, debug = False): # define app root path thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory self.appRoot = os.path.abspath(thisPath + '/../../') self.spellFolder = self.appRoot + '/app_configs/spells/' level = logging.WARNING if debug: level = logging.INFO formatter = logging.Formatter('%(asctime)s %(filename)s(%(lineno)s)[%(funcName)s] %(threadName)s %(levelname)-8s %(message)s') self.logger = logging.getLogger('AnalyticsServer') self.logger.setLevel(level) self.internalData = {} # sterr out if debug: handler = logging.StreamHandler(sys.stderr) handler.setFormatter(formatter) handler.setLevel(level) self.logger.addHandler(handler) handler = logging.FileHandler(self.appRoot + '/log/analyticsServer_' + datetime.now().strftime('%d-%m-%y') + '.log', 'a+') handler.setLevel(level) handler.setFormatter(formatter) self.logger.addHandler(handler) self.logger.info('AnalyticsServer started') # define app setting settings = { "cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", "login_url": "/user/login", "static_url_prefix": "/static", "debug" : debug, } # call parent constuctor super(AnalyticsServer, self).__init__(handlers = self.getHandlers(), default_host = "", transforms = None, wsgi = False, **settings) self.loadConfiguration() self.determineIsInstall() self.jinjaEnvironment = Environment(loader = PackageLoader('static', 'template')) self.jinjaEnvironment.filters['datetofiled'] = datetofiled self.jinjaEnvironment.filters['smartDate'] = smartDate self.jinjaEnvironment.filters['smartDatePeriod'] = smartDatePeriod self.jinjaEnvironment.filters['smartDateInterval'] = smartDateInterval self.jinjaEnvironment.filters['excelDate'] = excelDate self.jinjaEnvironment.filters['excelTime'] = excelTime self.jinjaEnvironment.filters['dateFromTS'] = dateFromTS self.jinjaEnvironment.filters['toJsVar'] = toJsVar self.jinjaEnvironment.filters['minInt'] = minInt self.jinjaEnvironment.filters['numberFormat'] = numberFormat self.jinjaEnvironment.filters['showMonth'] = showMonth self.scoped_session = None self.loopInstance = None if self.isInstalled: self.initDBScopedSession() def initDBScopedSession(self): if not self.scoped_session: mysql_user = self.config.get(Config.MYSQL_USER) mysql_password = self.config.get(Config.MYSQL_PASSWORD) mysql_host = self.config.get(Config.MYSQL_HOST) mysql_dbname = self.config.get(Config.MYSQL_DBNAME) conn_str = 'mysql://' if mysql_user: conn_str += mysql_user if mysql_password: conn_str += ':' + mysql_password conn_str += '@' conn_str += mysql_host + '/' + mysql_dbname engine = create_engine(conn_str + '?init_command=set%20names%20%22utf8%22', encoding = 'utf8', convert_unicode = True, pool_recycle = 3600) engine.execute('SET NAMES utf8') self.scoped_session = scoped_session(sessionmaker(bind = engine, autoflush = False)) def start(self): port = int(self.config.get('app_port', 48888)) address = self.config.get('app_host', "") self.listen(port = port, address = address ) self.loopInstance = tornado.ioloop.IOLoop.instance() self.logger.info('application run {}:{}'.format(address, port)) self.loopInstance.start() def loadConfiguration(self): self.config = Config() self.config.readConfigFile(os.path.abspath(self.appRoot + '/server.cfg')) def getAppConfigPath(self): return self.appRoot + '/app_configs/' def getResultPath(self): return self.appRoot + '/result/' def getLogPath(self): return self.appRoot + '/log/' def getTemplatePath(self): return self.appRoot + '/template/' def determineIsInstall(self): """ Простое определение - установлено ли приложение """ self.isInstalled = os.path.exists(self.appRoot + '/server.cfg') def setData(self, key, value): self.internalData[key] = value def getData(self, key): return self.internalData[key] def deleteData(self, key): del self.internalData[key] def getHandlers(self): thisPath = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))) # script directory return [ (r"/", IndexController.IndexAction), # --------- DASHBOARD ----------- (r"/dashboard/?", DashboardController.SwitchApp), (r"/dashboard/app/([^/]+)/?", DashboardController.IndexAction), (r"/dashboard/app/([^/]+)/first/?", DashboardController.FirstAction), (r"/dashboard/app/([^/]+)/result/?", ResultController.ResultAction), (r"/dashboard/app/([^/]+)/result/table/(\d+)/?", ResultController.TableAction), (r"/dashboard/app/([^/]+)/new/?", CreateTaskController.CreateAction), (r"/dashboard/app/([^/]+)/recalculate/?", CreateTaskController.RecalculateAction), (r"/dashboard/app/([^/]+)/templates/?", TemplateController.IndexAction), (r"/dashboard/empty/?", DashboardController.EmptyAppAction), (r"/dashboard/selectapp/?", DashboardController.SelectAppAction), (r"/dashboard/app/([^/]+)/new_task/([^/]+)/?", CreateTaskController.ShowNewTaskAction), (r"/dashboard/app/([^/]+)/status/([^/]+)/?", ResultController.ShowTaskStatus), (r"/ajax/dashboard/app/([^/]+)/workers/([^/]+)/??", DashboardController.GetWorkers), # --------- LOGIN & LOGOUT --------- (r"/user/login/?", UserController.AuthAction), (r"/user/logout/?", UserController.LogoutAction), # --------- ADMIN ----------- (r"/admin/?", AdminIndexController.IndexAction), (r"/admin/users/?", AdminUserController.UserAction), (r"/admin/users/edit?", AdminUserController.EditUserAction), (r"/admin/apps/?", AdminAppController.IndexAction), (r"/admin/app/edit/?", AdminAppController.EditAction), (r"/admin/rules/?", AdminRulesController.IndexAction), (r"/admin/rules/switch?", AdminRulesController.SwitchAjaxAction), (r'/admin/settings/?', AdminSettingsController.IndexAction), (r'/admin/cluster?', AdminClusterController.IndexAction), (r'/admin/historyServer/?', AdminProxy.HistoryServerView), (r'/admin/resourceManager/?', AdminProxy.ResourceManagerView), (r'/admin/proxy/([^/]+)/([^/]+)/(.+)?', AdminProxy.CoreProxy), (r'/admin/app/([^/]+)/settings/?', AdminAppConfigController.IndexAction), (r'/admin/app/([^/]+)/eventsList/?', AdminAppConfigController.EventListAction), (r"/admin/app/([^/]+)/editEvent/?", AdminAppConfigController.EditEvent), (r'/admin/app/([^/]+)/editTags/?', AdminAppConfigController.TagListAction), (r"/ajax/editTag/?", AdminAppConfigController.EditTag), (r'/admin/app/([^/]+)/editBunches/?', AdminAppConfigController.BunchListAction), (r'/ajax/editBunch/?', AdminAppConfigController.EditBunchAction), (r'/admin/app/([^/]+)/spell/?', AdminAppSpellController.IndexAction), (r'/admin/app/([^/]+)/groups/?', AdminAppGroupsController.IndexAction), # --------- CLUSTER ----------- (r'/cluster/namenode/?', NameNodeController.IndexAction), # --------- AJAX ----------- (r"/ajax/key_configuration/?", AjaxController.KeyConfigurationAction), (r"/ajax/get_key_form/?", AjaxController.GetKeyForm), (r"/ajax/getTasksProgress/?", AjaxController.GatTasksProgress), (r"/ajax/copyTaskKey/?", AjaxController.CopyTaskKey), (r"/ajax/saveWorkerName?", AjaxController.SaveWorkerName), #(r"/ajax/add_new_key?", AjaxController.AddNewKey), (r"/ajax/add_new_tag?", AjaxController.AddNewTag), (r"/ajax/add_new_bunch?", AjaxController.AddNewBunch), (r"/ajax/getDateSelector/?", AjaxController.GetDateSelector), (r"/ajax/getTagUniqueValues/?", HiveAJAXController.getTagUniqueValues), (r"/ajax/downloadCSV/?", AjaxController.DownloadCSVAction), (r"/ajax/admin/getClusterState/?", AdminClusterController.ClusterStateAction), # ----------- TEMPLATE --------------- #(r"/template/create/([^/]+)/?", TemplateController.CreateTemplateAction), # ----------- HDFS --------------- (r"/hdfs/?", HDFSController.IndexAction), (r"/hdfs/ajax/getPath/?", HDFSAJAXController.GetPathAction), (r"/hdfs/ajax/getPathStat/?", HDFSAJAXController.GetPathStat), (r"/hdfs/ajax/getHiveStat/?", HDFSAJAXController.GetHiveStatus), (r"/api/putConfig/?", APIController.PutConfigAction), (r"/install/?", InstallController.InstallAction), (r"/install/final/?", InstallController.FinalInstallAction), (r"/static/(.*)", tornado.web.StaticFileHandler, {"path": os.path.abspath(thisPath + '/../static')}), ]
def loadConfiguration(self): self.config = Config() self.config.readConfigFile(os.path.abspath(self.appRoot + '/server.cfg'))
def run_sensors(): config = Config() print("running sensors") camera = Camera(config) camera.shot_periodically()
def test_load_image(self): config = Config() image = Image(config,"Config.py") image.upload_image() self.assertEqual('foo'.upper(), 'FOO')
torch.backends.cudnn.deterministic = True dataset = args.datasetdir outputdir = args.outputdir embedding = '' if args.embedding == 'random': embedding = 'random' else: embedding = args.embedding model_name = args.model print(model_name) x = import_module('models.' + model_name) config = Config(dataset, outputdir, embedding) # reset config config.model_name = args.model config.save_path = os.path.join(outputdir, args.model + '.ckpt') config.log_path = os.path.join(outputdir, args.model + '.log') config.dropout = float(args.dropout) config.require_improvement = int(args.require_improvement) config.num_epochs = int(args.num_epochs) config.batch_size = int(args.batch_size) config.max_length = int(args.max_length) config.learning_rate = float(args.learning_rate) config.embed = int(args.embed_dim) config.bucket = int(args.bucket) config.wordNgrams = int(args.wordNgrams) config.lr_decay_rate = float(args.lr_decay_rate)