class FrameworkDaemon(Daemon): def __init__(self, args): super(FrameworkDaemon, self).__init__("Framework") self.args = args self.path_config = None self.sql_config = None self.rabbitmq_config = None self.rabbitmq = None self.frameworks = [] def setup(self): try: base_config = parse_config(self.args.config) except YAMLError as e: self.logger.error("Could not load worker config "+str(e)) return False except IOError as e: self.logger.error("Could not load worker config "+e.message) return False try: base_config = BaseConfig(base_config, strict=False) except ModelConversionError as e: self.logger.error("Could not create base config " + json.dumps(e.message)) return False try: base_config.validate() except ModelValidationError as e: self.logger.error("Could not validate base config " + json.dumps(e.message)) return False try: self.path_config = PathConfig(base_config.paths, strict=False) except ModelConversionError as e: self.logger.error("Could not create path config " + json.dumps(e.message)) return False try: self.path_config.validate() except ModelValidationError as e: self.logger.error("Could not validate path config " + json.dumps(e.message)) return False try: self.sql_config = SQLConfig(base_config.sql, strict=False) except ModelConversionError as e: self.logger.error("Could not create sql config " + json.dumps(e.message)) return False try: self.sql_config.validate() except ModelValidationError as e: self.logger.error("Could not validate sql config " + json.dumps(e.message)) return False try: self.rabbitmq_config = RabbitMQConfig(base_config.rabbitmq, strict=False) except ModelConversionError as e: self.logger.error("Could not create rabbitmq config " + json.dumps(e.message)) return False try: self.rabbitmq_config.validate() except ModelValidationError as e: self.logger.error("Could not validate rabbitmq config " + json.dumps(e.message)) return False return True def run(self): database = SQLDB(self.sql_config.driver, self.sql_config.host, self.sql_config.port, self.sql_config.database, self.sql_config.username, self.sql_config.password, self.sql_config.pool_size) database.connect() hosts = [] for host in self.rabbitmq_config.hosts: (ip, port) = host.split(":") hosts.append((ip, int(port))) self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password, self.rabbitmq_config.virtual_host) self.rabbitmq.setup_database() framework_utils = FrameworkUtils(self.rabbitmq) cherrypy.tools.auth = cherrypy.Tool("on_start_resource", framework_utils.auth) self.logger.info("Starting Framework") for config_name in os.listdir(self.path_config.framework_configs): if not config_name.endswith(".yml") and not config_name.endswith(".yaml"): continue config_path = self.path_config.framework_configs + "/" + config_name try: framework_config = parse_config(config_path) except YAMLError as e: self.logger.error("Could load framework config " + config_name + " " + str(e)) continue if 'module' not in framework_config: self.logger.error("Framework config " + config_name + " does not have a module to load.") continue modules = framework_config['module'].split(".") try: module = __import__(framework_config['module']) modules.pop(0) for m in modules: module = getattr(module, m) api = getattr(module, 'FrameworkAPI') framework = getattr(module, 'Framework') framework = framework() self.frameworks.append(framework) api = api(framework) except: self.logger.exception("Error loading framework module " + framework_config['module']) continue if not framework.register_framework(self.rabbitmq, database, config_path): self.logger.error("Framework " + framework.name + " could not register") continue api.register() if len(self.frameworks) == 0: self.logger.warning("No frameworks loaded") return False cherrypy.config.update({'engine.autoreload.on': False, 'engine.timeout_monitor.on': False, 'error_page.default': framework_utils.jsonify_error, 'server.socket_port': 8081}) cherrypy.engine.start() return True def on_shutdown(self, signum=None, frame=None): cherrypy.engine.exit() for subscriber in list(self.rabbitmq.active_subscribers): subscriber.stop() for framework in list(self.frameworks): framework.stop() def on_reload(self, signum=None, frame=None): pass def get_pid_file(self): return self.path_config.pid def get_log_path(self): return self.path_config.logs
class ManagerDaemon(Daemon): def __init__(self, args): super(ManagerDaemon, self).__init__("Manager") self.args = args self.base_config = None self.path_config = None self.sql_config = None self.ldap_config = None self.rabbitmq_config = None self.rabbitmq = None def get_pid_file(self): return self.path_config.pid def get_log_path(self): return self.path_config.logs def setup(self): try: self.base_config = parse_config(self.args.config) except YAMLError as e: self.logger.error("Could not load base config" + str(e)) return False except IOError as e: self.logger.error("Could not load worker config " + e.message) return False try: self.base_config = BaseConfig(self.base_config, strict=False) except ModelConversionError as e: self.logger.error("Could not create base config " + json.dumps(e.message)) return False try: self.path_config = PathConfig(self.base_config.paths, strict=False) except ModelConversionError as e: self.logger.error("Could not create path config " + json.dumps(e.message)) return False try: self.path_config.validate() except ModelValidationError as e: self.logger.error("Could not validate path config " + json.dumps(e.message)) return False try: self.sql_config = SQLConfig(self.base_config.sql, strict=False) except ModelConversionError as e: self.logger.error("Could not create sql config " + json.dumps(e.message)) return False try: self.sql_config.validate() except ModelValidationError as e: self.logger.error("Could not validate sql config " + json.dumps(e.message)) return False if self.base_config.ldap is not None: try: self.ldap_config = LDAPConfig(self.base_config.ldap, strict=False) except ModelConversionError as e: self.logger.error("Could not create ldap config " + json.dumps(e.message)) return False try: self.ldap_config.validate() except ModelValidationError as e: self.logger.error("Could not validate ldap config " + json.dumps(e.message)) return False try: self.rabbitmq_config = RabbitMQConfig(self.base_config.rabbitmq, strict=False) except ModelConversionError as e: self.logger.error("Could not create rabbitmq config " + json.dumps(e.message)) return False try: self.rabbitmq_config.validate() except ModelValidationError as e: self.logger.error("Could not validate rabbitmq config " + json.dumps(e.message)) return False return True def run(self): database = SQLDB(self.sql_config.driver, self.sql_config.host, self.sql_config.port, self.sql_config.database, self.sql_config.username, self.sql_config.password, self.sql_config.pool_size) database.connect() with database.session() as session: Base.metadata.create_all(bind=session.get_bind()) ldap = None if self.ldap_config is not None: ldap = LDAP(self.ldap_config.host, self.ldap_config.domain, self.ldap_config.base_dn, self.ldap_config.bind_username, self.ldap_config.bind_password) hosts = [] for host in self.rabbitmq_config.hosts: (ip, port) = host.split(":") hosts.append((ip, int(port))) self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password, self.rabbitmq_config.virtual_host) self.rabbitmq.setup_database() if 'driver' not in self.base_config.identity: self.logger.error("Identity Config does not have a driver") return False identity_module = self.base_config.identity['driver'].split(".") module = __import__(self.base_config.identity['driver']) identity_module.pop(0) for m in identity_module: module = getattr(module, m) identity = getattr(module, 'IdentityDriver') identity = identity() if not identity.validate_config(self.base_config.identity): self.logger.error("Error validating identity config") return False identity.db_connections(database=database, rabbitmq=self.rabbitmq, ldap=ldap) if 'driver' not in self.base_config.assignment: self.logger.error("Assignment Config does not have a driver set") return assignment_modules = self.base_config.assignment['driver'].split(".") module = __import__(self.base_config.assignment['driver']) assignment_modules.pop(0) for m in assignment_modules: module = getattr(module, m) assignment = getattr(module, 'AssignmentDriver') assignment = assignment() if not assignment.validate_config(self.base_config.assignment): self.logger.error("Error validating assignment config") return False assignment.db_connections(database=database, rabbitmq=self.rabbitmq, ldap=ldap) RegisterFrameworkSubscriber(self.rabbitmq).start() TaskStatusSubscriber(self.rabbitmq, database).start() TaskLaunchSubscriber(self.rabbitmq, database).start() WorkerRegister(self.rabbitmq, database).start() WorkerReload(self.rabbitmq, database).start() WorkerGet(self.rabbitmq, database).start() Validate(self.rabbitmq, assignment).start() hqmanager.api.setup(self.rabbitmq, database, identity, assignment) return True def on_shutdown(self, signum=None, frame=None): cherrypy.engine.exit() for subscriber in list(self.rabbitmq.active_subscribers): subscriber.stop() def on_reload(self, signum=None, frame=None): pass
class WorkerDaemon(Daemon): def __init__(self, args): super(WorkerDaemon, self).__init__("Worker") self.args = args self.path_config = None self.rabbitmq_config = None self.rabbitmq = None self.workers = [] def setup(self): try: base_config = parse_config(self.args.config) except YAMLError as e: self.logger.error("Could not load worker config " + str(e)) return False except IOError as e: self.logger.error("Could not load worker config " + e.message) return False try: base_config = BaseConfig(base_config, strict=False) except ModelConversionError as e: self.logger.error("Could not create base config " + json.dumps(e.message)) return False try: base_config.validate() except ModelValidationError as e: self.logger.error("Could not validate base config " + json.dumps(e.message)) return False try: self.path_config = PathConfig(base_config.paths, strict=False) except ModelConversionError as e: self.logger.error("Could not create path config " + json.dumps(e.message)) return False try: self.path_config.validate() except ModelValidationError as e: self.logger.error("Could not validate path config " + json.dumps(e.message)) return False try: self.rabbitmq_config = RabbitMQConfig(base_config.rabbitmq, strict=False) except ModelConversionError as e: self.logger.error("Could not create rabbitmq config " + json.dumps(e.message)) return False try: self.rabbitmq_config.validate() except ModelValidationError as e: self.logger.error("Could not validate rabbitmq config " + json.dumps(e.message)) return False return True def run(self): hosts = [] for host in self.rabbitmq_config.hosts: (ip, port) = host.split(":") hosts.append((ip, int(port))) self.rabbitmq = RabbitMQ(hosts, self.rabbitmq_config.username, self.rabbitmq_config.password, self.rabbitmq_config.virtual_host) self.rabbitmq.setup_database() for config_name in os.listdir(self.path_config.worker_configs): if not config_name.endswith(".yml") and not config_name.endswith(".yaml"): continue config_path = self.path_config.worker_configs + "/" + config_name try: worker_config = parse_config(config_path) except YAMLError as e: self.logger.error("Could load worker config " + config_name + " " + str(e)) continue if 'module' not in worker_config: self.logger.error("Worker config " + config_name + " does not have a module to load.") continue modules = worker_config['module'].split(".") try: module = __import__(worker_config['module']) modules.pop(0) for m in modules: module = getattr(module, m) worker = getattr(module, 'Worker') worker = worker() self.workers.append(worker) except: self.logger.exception("Error loading worker module " + worker_config['module']) continue if not worker.register(self.rabbitmq, config_path): self.logger.error("Worker " + worker.framework_name + " could not register") if len(self.workers) == 0: self.logger.warning("No workers loaded") return False return True def on_shutdown(self, signum=None, frame=None): for subscriber in list(self.rabbitmq.active_subscribers): subscriber.stop() def on_reload(self, signum=None, frame=None): self.logger.info("Reloading worker config files") for worker in self.workers: worker.load_config(worker.config_path, True) def get_pid_file(self): return self.path_config.pid def get_log_path(self): return self.path_config.logs