def get_redis_client(system_config, decode_response=True):
assert "redis" in system_config, "Can't read redis config from system config"
redis_config = system_config["redis"]
redis_password = None
if "password" in redis_config:
redis_password = redis_config["password"]
redis_client = RedisDatabase(redis_config["host"],
redis_config["port"],
redis_config["db"],
redis_password, decode_response)
return redis_client
from nmtwizard import config, task
from nmtwizard.redis_database import RedisDatabase
from nmtwizard.worker import Worker
cfg = configparser.ConfigParser()
cfg.read('settings.ini')
logging.basicConfig(stream=sys.stdout, level=cfg.get('default', 'log_level'))
logger = logging.getLogger()
redis_password = None
if cfg.has_option('redis', 'password'):
redis_password = cfg.get('redis', 'password')
redis = RedisDatabase(cfg.get('redis', 'host'),
cfg.getint('redis', 'port'),
cfg.get('redis', 'db'),
redis_password)
retry = 0
while retry < 10:
try:
# make sure notify events are set
redis.config_set('notify-keyspace-events', 'Klgx')
break
except ConnectionError as e:
retry += 1
logger.warn("cannot connect to redis DB - retrying (%d)" % retry)
time.sleep(1)
assert retry < 10, "Cannot connect to redis DB - aborting"
with open(default_file) as default_fh:
default_config = default_fh.read()
base_config = json.loads(default_config)
assert 'storages' in base_config, "incomplete configuration - missing " \
"`storages` in %s" % default_file
app.iniconfig = FlaskIni()
with app.app_context():
app.iniconfig.read(config_file)
app.logger.setLevel(
logging.getLevelName(
app.iniconfig.get('default', 'log_level', fallback='ERROR')))
redis_db = RedisDatabase(app.iniconfig.get('redis', 'host'),
app.iniconfig.get('redis', 'port', fallback=6379),
app.iniconfig.get('redis', 'db', fallback=0),
app.iniconfig.get('redis', 'password', fallback=None))
redis_db_without_decode = RedisDatabase(
app.iniconfig.get('redis', 'host'),
app.iniconfig.get('redis', 'port', fallback=6379),
app.iniconfig.get('redis', 'db', fallback=0),
app.iniconfig.get('redis', 'password', fallback=None), False)
assert app.iniconfig.get(
'default', 'taskfile_dir'), "missing taskfile_dir from settings.ini"
taskfile_dir = app.iniconfig.get('default', 'taskfile_dir')
assert os.path.isdir(
taskfile_dir), "taskfile_dir (%s) must be a directory" % taskfile_dir
retry = 0
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
config.add_log_handler(ch)
common.add_log_handler(ch)
cfg = configparser.ConfigParser()
cfg.read('settings.ini')
MODE = os.getenv('LAUNCHER_MODE', 'Production')
redis_password = None
if cfg.has_option(MODE, 'redis_password'):
redis_password = cfg.get(MODE, 'redis_password')
redis = RedisDatabase(cfg.get(MODE, 'redis_host'),
cfg.getint(MODE, 'redis_port'),
cfg.get(MODE, 'redis_db'),
redis_password)
services = config.load_services(cfg.get(MODE, 'config_dir'))
def _get_service(service):
"""Wrapper to fail on invalid service."""
if service not in services:
response = flask.jsonify(message="invalid service name: %s" % service)
flask.abort(flask.make_response(response, 404))
return services[service]
app = flask.Flask(__name__)
@app.route("/list_services", methods=["GET"])
1:-5]
current_configuration_md5 = md5file(args.config)
if current_configuration_md5 in config_service_md5:
current_configuration = config_service_md5[current_configuration_md5]
cfg = configparser.ConfigParser()
cfg.read('settings.ini')
logging.config.fileConfig('logging.conf')
logger = logging.getLogger('worker')
redis_password = None
if cfg.has_option('redis', 'password'):
redis_password = cfg.get('redis', 'password')
redis = RedisDatabase(cfg.get('redis', 'host'), cfg.getint('redis', 'port'),
cfg.get('redis', 'db'), redis_password)
retry = 0
while retry < 10:
try:
# make sure notify events are set
redis.config_set('notify-keyspace-events', 'Klgx')
break
except ConnectionError as e:
retry += 1
logger.warn("cannot connect to redis DB - retrying (%d)" % retry)
time.sleep(1)
assert retry < 10, "Cannot connect to redis DB - aborting"
redis.set('admin:storages', json.dumps(base_config['storages']))
from nmtwizard import config, common
import logging
app = Flask(__name__)
ch = logging.StreamHandler()
ch.setLevel(logging.ERROR)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
config.add_log_handler(ch)
common.add_log_handler(ch)
config_file = os.getenv('LAUNCHER_CONFIG')
assert config_file is not None and os.path.isfile(
config_file), "invalid LAUNCHER_CONFIG"
app.iniconfig = FlaskIni()
with app.app_context():
app.iniconfig.read(config_file)
redis = RedisDatabase(app.iniconfig.get('redis', 'host'),
app.iniconfig.get('redis', 'port', fallback=6379),
app.iniconfig.get('redis', 'db', fallback=0),
app.iniconfig.get('redis', 'password', fallback=None))
services, base_config = config.load_services(
app.iniconfig.get('default', 'config_dir'))
from app import routes
import signal
import logging
# from redis.exceptions import ConnectionError
from six.moves import configparser
from nmtwizard.redis_database import RedisDatabase
# connecting to redis to monitor the worker process
cfg = configparser.ConfigParser()
cfg.read('settings.ini')
redis_password = None
if cfg.has_option('redis', 'password'):
redis_password = cfg.get('redis', 'password')
redis = RedisDatabase(cfg.get('redis', 'host'),
cfg.getint('redis', 'port'),
cfg.get('redis', 'db'),
redis_password)
retry = 0
while retry < 10:
try:
redis.get('notify-keyspace-events')
break
except ConnectionError as e:
retry += 1
logging.warning("cannot connect to redis DB - retrying (%d)", retry)
time.sleep(1)
assert retry < 10, "Cannot connect to redis DB - aborting"
for l in f.readlines():
m.update(l)
return m.hexdigest()
cfg = configparser.ConfigParser()
cfg.read('settings.ini')
logging.config.fileConfig('logging.conf')
logger = logging.getLogger('worker')
redis_password = None
if cfg.has_option('redis', 'password'):
redis_password = cfg.get('redis', 'password')
redis = RedisDatabase(cfg.get('redis', 'host'), cfg.getint('redis', 'port'),
cfg.get('redis', 'db'), redis_password)
retry = 0
while retry < 10:
try:
# make sure notify events are set
redis.config_set('notify-keyspace-events', 'Klgx')
break
except ConnectionError as e:
retry += 1
logger.warn("cannot connect to redis DB - retrying (%d)" % retry)
time.sleep(1)
assert retry < 10, "Cannot connect to redis DB - aborting"
# load default configuration from database
assert is_float(worker_cycle_config) and float(
worker_cycle_config) > 0, "worker_cycle must be numeric and greater than 0"
worker_cycle = float(worker_cycle_config)
if cfg.has_option('worker', 'worker_butler_cycle'):
worker_butler_cycle_config = cfg.get('worker', 'worker_butler_cycle')
assert is_float(worker_butler_cycle_config) and float(
worker_butler_cycle_config) > 0, "worker_butler_cycle must be numeric and greater than 0"
worker_butler_cycle = float(worker_butler_cycle_config)
redis_password = None
if cfg.has_option('redis', 'password'):
redis_password = cfg.get('redis', 'password')
redis = RedisDatabase(cfg.get('redis', 'host'),
cfg.getint('redis', 'port'),
cfg.get('redis', 'db'),
redis_password)
redis2 = RedisDatabase(cfg.get('redis', 'host'),
cfg.getint('redis', 'port'),
cfg.get('redis', 'db'),
redis_password, False)
retry = 0
while retry < 10:
try:
# make sure notify events are set
redis.config_set('notify-keyspace-events', 'Klgx')
break
except ConnectionError as e:
retry += 1
assert os.path.isfile(default_file), "Cannot find default.json: %s" % default_file
with open(default_file) as default_fh:
default_config = default_fh.read()
base_config = json.loads(default_config)
assert 'storages' in base_config, "incomplete configuration - missing `storages` in %s" % default_file
app.iniconfig = FlaskIni()
with app.app_context():
app.iniconfig.read(config_file)
app.logger.setLevel(logging.getLevelName(
app.iniconfig.get('default', 'log_level', fallback='ERROR')))
redis = RedisDatabase(app.iniconfig.get('redis', 'host'),
app.iniconfig.get('redis', 'port', fallback=6379),
app.iniconfig.get('redis', 'db', fallback=0),
app.iniconfig.get('redis', 'password', fallback=None))
assert app.iniconfig.get('default', 'taskfile_dir'), "missing taskfile_dir from settings.ini"
taskfile_dir = app.iniconfig.get('default', 'taskfile_dir')
assert os.path.isdir(taskfile_dir), "taskfile_dir (%s) must be a directory" % taskfile_dir
retry = 0
while retry < 10:
try:
current_default_config = redis.exists("default") and redis.hget("default", "configuration")
break
except (ConnectionError, AssertionError) as e:
retry += 1
time.sleep(1)
from nmtwizard import config, task
from nmtwizard.redis_database import RedisDatabase
from nmtwizard.worker import Worker
cfg = configparser.ConfigParser()
cfg.read('settings.ini')
logging.basicConfig(stream=sys.stdout, level=cfg.get('default', 'log_level'))
logger = logging.getLogger()
redis_password = None
if cfg.has_option('redis', 'password'):
redis_password = cfg.get('redis', 'password')
redis = RedisDatabase(cfg.get('redis', 'host'), cfg.getint('redis', 'port'),
cfg.get('redis', 'db'), redis_password)
# make sure notify events are set
redis.config_set('notify-keyspace-events', 'Klgx')
services, base_config = config.load_services(cfg.get('default', 'config_dir'))
# remove busy state from resources
for key in redis.keys('busy:*'):
redis.delete(key)
# remove reserved state from resources
for key in redis.keys('reserved:*'):
redis.delete(key)
# On startup, add all active tasks in the work queue.
for task_id in task.list_active(redis):
common.add_log_handler(ch)
config_file = os.getenv('LAUNCHER_CONFIG', 'settings.ini')
assert config_file is not None and os.path.isfile(
config_file), "invalid LAUNCHER_CONFIG"
app.iniconfig = FlaskIni()
with app.app_context():
app.iniconfig.read(config_file)
app.logger.setLevel(
logging.getLevelName(
app.iniconfig.get('default', 'log_level', fallback='ERROR')))
redis = RedisDatabase(app.iniconfig.get('redis', 'host'),
app.iniconfig.get('redis', 'port', fallback=6379),
app.iniconfig.get('redis', 'db', fallback=0),
app.iniconfig.get('redis', 'password', fallback=None))
assert app.iniconfig.get(
'default', 'taskfile_dir'), "missing taskfile_dir from settings.ini"
taskfile_dir = app.iniconfig.get('default', 'taskfile_dir')
assert os.path.isdir(
taskfile_dir), "taskfile_dir (%s) must be a directory" % taskfile_dir
retry = 0
while retry < 10:
try:
storages_list = redis.get("admin:storages")
assert storages_list, "ERROR: cannot get storages from worker db"
break
except (ConnectionError, AssertionError) as e: