def run(self):
hosts = []
for host in self.rabbitmq_config.hosts:
(ip, port) = host.split(":")
hosts.append((ip, int(port)))
self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password,
self.rabbitmq_config.virtual_host)
self.rabbitmq.setup_database()
for config_name in os.listdir(self.path_config.worker_configs):
if not config_name.endswith(".yml") and not config_name.endswith(".yaml"):
continue
config_path = self.path_config.worker_configs + "/" + config_name
try:
worker_config = parse_config(config_path)
except YAMLError as e:
self.logger.error("Could load worker config " + config_name + " " + str(e))
continue
if 'module' not in worker_config:
self.logger.error("Worker config " + config_name + " does not have a module to load.")
continue
modules = worker_config['module'].split(".")
try:
module = __import__(worker_config['module'])
modules.pop(0)
for m in modules:
module = getattr(module, m)
worker = getattr(module, 'Worker')
worker = worker()
self.workers.append(worker)
except:
self.logger.exception("Error loading worker module " + worker_config['module'])
continue
if not worker.register(self.rabbitmq, config_path):
self.logger.error("Worker " + worker.framework_name + " could not register")
if len(self.workers) == 0:
self.logger.warning("No workers loaded")
return False
return True
class ManagerDaemon(Daemon):
def __init__(self, args):
super(ManagerDaemon, self).__init__("Manager")
self.args = args
self.base_config = None
self.path_config = None
self.sql_config = None
self.ldap_config = None
self.rabbitmq_config = None
self.rabbitmq = None
def get_pid_file(self):
return self.path_config.pid
def get_log_path(self):
return self.path_config.logs
def setup(self):
try:
self.base_config = parse_config(self.args.config)
except YAMLError as e:
self.logger.error("Could not load base config" + str(e))
return False
except IOError as e:
self.logger.error("Could not load worker config " + e.message)
return False
try:
self.base_config = BaseConfig(self.base_config, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create base config " + json.dumps(e.message))
return False
try:
self.path_config = PathConfig(self.base_config.paths, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create path config " + json.dumps(e.message))
return False
try:
self.path_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate path config " + json.dumps(e.message))
return False
try:
self.sql_config = SQLConfig(self.base_config.sql, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create sql config " + json.dumps(e.message))
return False
try:
self.sql_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate sql config " + json.dumps(e.message))
return False
if self.base_config.ldap is not None:
try:
self.ldap_config = LDAPConfig(self.base_config.ldap, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create ldap config " + json.dumps(e.message))
return False
try:
self.ldap_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate ldap config " + json.dumps(e.message))
return False
try:
self.rabbitmq_config = RabbitMQConfig(self.base_config.rabbitmq, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create rabbitmq config " + json.dumps(e.message))
return False
try:
self.rabbitmq_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate rabbitmq config " + json.dumps(e.message))
return False
return True
def run(self):
database = SQLDB(self.sql_config.driver, self.sql_config.host, self.sql_config.port, self.sql_config.database,
self.sql_config.username, self.sql_config.password, self.sql_config.pool_size)
database.connect()
with database.session() as session:
Base.metadata.create_all(bind=session.get_bind())
ldap = None
if self.ldap_config is not None:
ldap = LDAP(self.ldap_config.host, self.ldap_config.domain, self.ldap_config.base_dn,
self.ldap_config.bind_username, self.ldap_config.bind_password)
hosts = []
for host in self.rabbitmq_config.hosts:
(ip, port) = host.split(":")
hosts.append((ip, int(port)))
self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password,
self.rabbitmq_config.virtual_host)
self.rabbitmq.setup_database()
if 'driver' not in self.base_config.identity:
self.logger.error("Identity Config does not have a driver")
return False
identity_module = self.base_config.identity['driver'].split(".")
module = __import__(self.base_config.identity['driver'])
identity_module.pop(0)
for m in identity_module:
module = getattr(module, m)
identity = getattr(module, 'IdentityDriver')
identity = identity()
if not identity.validate_config(self.base_config.identity):
self.logger.error("Error validating identity config")
return False
identity.db_connections(database=database, rabbitmq=self.rabbitmq, ldap=ldap)
if 'driver' not in self.base_config.assignment:
self.logger.error("Assignment Config does not have a driver set")
return
assignment_modules = self.base_config.assignment['driver'].split(".")
module = __import__(self.base_config.assignment['driver'])
assignment_modules.pop(0)
for m in assignment_modules:
module = getattr(module, m)
assignment = getattr(module, 'AssignmentDriver')
assignment = assignment()
if not assignment.validate_config(self.base_config.assignment):
self.logger.error("Error validating assignment config")
return False
assignment.db_connections(database=database, rabbitmq=self.rabbitmq, ldap=ldap)
RegisterFrameworkSubscriber(self.rabbitmq).start()
TaskStatusSubscriber(self.rabbitmq, database).start()
TaskLaunchSubscriber(self.rabbitmq, database).start()
WorkerRegister(self.rabbitmq, database).start()
WorkerReload(self.rabbitmq, database).start()
WorkerGet(self.rabbitmq, database).start()
Validate(self.rabbitmq, assignment).start()
hqmanager.api.setup(self.rabbitmq, database, identity, assignment)
return True
def on_shutdown(self, signum=None, frame=None):
cherrypy.engine.exit()
for subscriber in list(self.rabbitmq.active_subscribers):
subscriber.stop()
def on_reload(self, signum=None, frame=None):
pass
def run(self):
database = SQLDB(self.sql_config.driver, self.sql_config.host, self.sql_config.port, self.sql_config.database,
self.sql_config.username, self.sql_config.password, self.sql_config.pool_size)
database.connect()
with database.session() as session:
Base.metadata.create_all(bind=session.get_bind())
ldap = None
if self.ldap_config is not None:
ldap = LDAP(self.ldap_config.host, self.ldap_config.domain, self.ldap_config.base_dn,
self.ldap_config.bind_username, self.ldap_config.bind_password)
hosts = []
for host in self.rabbitmq_config.hosts:
(ip, port) = host.split(":")
hosts.append((ip, int(port)))
self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password,
self.rabbitmq_config.virtual_host)
self.rabbitmq.setup_database()
if 'driver' not in self.base_config.identity:
self.logger.error("Identity Config does not have a driver")
return False
identity_module = self.base_config.identity['driver'].split(".")
module = __import__(self.base_config.identity['driver'])
identity_module.pop(0)
for m in identity_module:
module = getattr(module, m)
identity = getattr(module, 'IdentityDriver')
identity = identity()
if not identity.validate_config(self.base_config.identity):
self.logger.error("Error validating identity config")
return False
identity.db_connections(database=database, rabbitmq=self.rabbitmq, ldap=ldap)
if 'driver' not in self.base_config.assignment:
self.logger.error("Assignment Config does not have a driver set")
return
assignment_modules = self.base_config.assignment['driver'].split(".")
module = __import__(self.base_config.assignment['driver'])
assignment_modules.pop(0)
for m in assignment_modules:
module = getattr(module, m)
assignment = getattr(module, 'AssignmentDriver')
assignment = assignment()
if not assignment.validate_config(self.base_config.assignment):
self.logger.error("Error validating assignment config")
return False
assignment.db_connections(database=database, rabbitmq=self.rabbitmq, ldap=ldap)
RegisterFrameworkSubscriber(self.rabbitmq).start()
TaskStatusSubscriber(self.rabbitmq, database).start()
TaskLaunchSubscriber(self.rabbitmq, database).start()
WorkerRegister(self.rabbitmq, database).start()
WorkerReload(self.rabbitmq, database).start()
WorkerGet(self.rabbitmq, database).start()
Validate(self.rabbitmq, assignment).start()
hqmanager.api.setup(self.rabbitmq, database, identity, assignment)
return True
def run(self):
database = SQLDB(self.sql_config.driver, self.sql_config.host, self.sql_config.port, self.sql_config.database,
self.sql_config.username, self.sql_config.password, self.sql_config.pool_size)
database.connect()
hosts = []
for host in self.rabbitmq_config.hosts:
(ip, port) = host.split(":")
hosts.append((ip, int(port)))
self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password,
self.rabbitmq_config.virtual_host)
self.rabbitmq.setup_database()
framework_utils = FrameworkUtils(self.rabbitmq)
cherrypy.tools.auth = cherrypy.Tool("on_start_resource", framework_utils.auth)
self.logger.info("Starting Framework")
for config_name in os.listdir(self.path_config.framework_configs):
if not config_name.endswith(".yml") and not config_name.endswith(".yaml"):
continue
config_path = self.path_config.framework_configs + "/" + config_name
try:
framework_config = parse_config(config_path)
except YAMLError as e:
self.logger.error("Could load framework config " + config_name + " " + str(e))
continue
if 'module' not in framework_config:
self.logger.error("Framework config " + config_name + " does not have a module to load.")
continue
modules = framework_config['module'].split(".")
try:
module = __import__(framework_config['module'])
modules.pop(0)
for m in modules:
module = getattr(module, m)
api = getattr(module, 'FrameworkAPI')
framework = getattr(module, 'Framework')
framework = framework()
self.frameworks.append(framework)
api = api(framework)
except:
self.logger.exception("Error loading framework module " + framework_config['module'])
continue
if not framework.register_framework(self.rabbitmq, database, config_path):
self.logger.error("Framework " + framework.name + " could not register")
continue
api.register()
if len(self.frameworks) == 0:
self.logger.warning("No frameworks loaded")
return False
cherrypy.config.update({'engine.autoreload.on': False,
'engine.timeout_monitor.on': False,
'error_page.default': framework_utils.jsonify_error,
'server.socket_port': 8081})
cherrypy.engine.start()
return True
class FrameworkDaemon(Daemon):
def __init__(self, args):
super(FrameworkDaemon, self).__init__("Framework")
self.args = args
self.path_config = None
self.sql_config = None
self.rabbitmq_config = None
self.rabbitmq = None
self.frameworks = []
def setup(self):
try:
base_config = parse_config(self.args.config)
except YAMLError as e:
self.logger.error("Could not load worker config "+str(e))
return False
except IOError as e:
self.logger.error("Could not load worker config "+e.message)
return False
try:
base_config = BaseConfig(base_config, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create base config " + json.dumps(e.message))
return False
try:
base_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate base config " + json.dumps(e.message))
return False
try:
self.path_config = PathConfig(base_config.paths, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create path config " + json.dumps(e.message))
return False
try:
self.path_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate path config " + json.dumps(e.message))
return False
try:
self.sql_config = SQLConfig(base_config.sql, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create sql config " + json.dumps(e.message))
return False
try:
self.sql_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate sql config " + json.dumps(e.message))
return False
try:
self.rabbitmq_config = RabbitMQConfig(base_config.rabbitmq, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create rabbitmq config " + json.dumps(e.message))
return False
try:
self.rabbitmq_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate rabbitmq config " + json.dumps(e.message))
return False
return True
def run(self):
database = SQLDB(self.sql_config.driver, self.sql_config.host, self.sql_config.port, self.sql_config.database,
self.sql_config.username, self.sql_config.password, self.sql_config.pool_size)
database.connect()
hosts = []
for host in self.rabbitmq_config.hosts:
(ip, port) = host.split(":")
hosts.append((ip, int(port)))
self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password,
self.rabbitmq_config.virtual_host)
self.rabbitmq.setup_database()
framework_utils = FrameworkUtils(self.rabbitmq)
cherrypy.tools.auth = cherrypy.Tool("on_start_resource", framework_utils.auth)
self.logger.info("Starting Framework")
for config_name in os.listdir(self.path_config.framework_configs):
if not config_name.endswith(".yml") and not config_name.endswith(".yaml"):
continue
config_path = self.path_config.framework_configs + "/" + config_name
try:
framework_config = parse_config(config_path)
except YAMLError as e:
self.logger.error("Could load framework config " + config_name + " " + str(e))
continue
if 'module' not in framework_config:
self.logger.error("Framework config " + config_name + " does not have a module to load.")
continue
modules = framework_config['module'].split(".")
try:
module = __import__(framework_config['module'])
modules.pop(0)
for m in modules:
module = getattr(module, m)
api = getattr(module, 'FrameworkAPI')
framework = getattr(module, 'Framework')
framework = framework()
self.frameworks.append(framework)
api = api(framework)
except:
self.logger.exception("Error loading framework module " + framework_config['module'])
continue
if not framework.register_framework(self.rabbitmq, database, config_path):
self.logger.error("Framework " + framework.name + " could not register")
continue
api.register()
if len(self.frameworks) == 0:
self.logger.warning("No frameworks loaded")
return False
cherrypy.config.update({'engine.autoreload.on': False,
'engine.timeout_monitor.on': False,
'error_page.default': framework_utils.jsonify_error,
'server.socket_port': 8081})
cherrypy.engine.start()
return True
def on_shutdown(self, signum=None, frame=None):
cherrypy.engine.exit()
for subscriber in list(self.rabbitmq.active_subscribers):
subscriber.stop()
for framework in list(self.frameworks):
framework.stop()
def on_reload(self, signum=None, frame=None):
pass
def get_pid_file(self):
return self.path_config.pid
def get_log_path(self):
return self.path_config.logs
class WorkerDaemon(Daemon):
def __init__(self, args):
super(WorkerDaemon, self).__init__("Worker")
self.args = args
self.path_config = None
self.rabbitmq_config = None
self.rabbitmq = None
self.workers = []
def setup(self):
try:
base_config = parse_config(self.args.config)
except YAMLError as e:
self.logger.error("Could not load worker config " + str(e))
return False
except IOError as e:
self.logger.error("Could not load worker config " + e.message)
return False
try:
base_config = BaseConfig(base_config, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create base config " + json.dumps(e.message))
return False
try:
base_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate base config " + json.dumps(e.message))
return False
try:
self.path_config = PathConfig(base_config.paths, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create path config " + json.dumps(e.message))
return False
try:
self.path_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate path config " + json.dumps(e.message))
return False
try:
self.rabbitmq_config = RabbitMQConfig(base_config.rabbitmq, strict=False)
except ModelConversionError as e:
self.logger.error("Could not create rabbitmq config " + json.dumps(e.message))
return False
try:
self.rabbitmq_config.validate()
except ModelValidationError as e:
self.logger.error("Could not validate rabbitmq config " + json.dumps(e.message))
return False
return True
def run(self):
hosts = []
for host in self.rabbitmq_config.hosts:
(ip, port) = host.split(":")
hosts.append((ip, int(port)))
self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password,
self.rabbitmq_config.virtual_host)
self.rabbitmq.setup_database()
for config_name in os.listdir(self.path_config.worker_configs):
if not config_name.endswith(".yml") and not config_name.endswith(".yaml"):
continue
config_path = self.path_config.worker_configs + "/" + config_name
try:
worker_config = parse_config(config_path)
except YAMLError as e:
self.logger.error("Could load worker config " + config_name + " " + str(e))
continue
if 'module' not in worker_config:
self.logger.error("Worker config " + config_name + " does not have a module to load.")
continue
modules = worker_config['module'].split(".")
try:
module = __import__(worker_config['module'])
modules.pop(0)
for m in modules:
module = getattr(module, m)
worker = getattr(module, 'Worker')
worker = worker()
self.workers.append(worker)
except:
self.logger.exception("Error loading worker module " + worker_config['module'])
continue
if not worker.register(self.rabbitmq, config_path):
self.logger.error("Worker " + worker.framework_name + " could not register")
if len(self.workers) == 0:
self.logger.warning("No workers loaded")
return False
return True
def on_shutdown(self, signum=None, frame=None):
for subscriber in list(self.rabbitmq.active_subscribers):
subscriber.stop()
def on_reload(self, signum=None, frame=None):
self.logger.info("Reloading worker config files")
for worker in self.workers:
worker.load_config(worker.config_path, True)
def get_pid_file(self):
return self.path_config.pid
def get_log_path(self):
return self.path_config.logs
