def start_state_manager_watches(self): """ Receive updates to the packing plan from the statemgrs and update processes as needed. """ statemgr_config = StateMgrConfig() statemgr_config.set_state_locations(configloader.load_state_manager_locations(self.cluster)) self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config) # pylint: disable=unused-argument def on_packing_plan_watch(state_manager, new_packing_plan): Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" % (self.shard, str(self.packing_plan), str(new_packing_plan))) if self.packing_plan != new_packing_plan: Log.info("PackingPlan change detected on shard %s, relaunching effected processes." % self.shard) self.update_packing_plan(new_packing_plan) Log.info("Updating executor processes") self.launch() else: Log.info( "State watch triggered for PackingPlan update but plan not changed so not relaunching.") for state_manager in self.state_managers: # The callback function with the bound # state_manager as first variable. onPackingPlanWatch = partial(on_packing_plan_watch, state_manager) state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch) Log.info("Registered state watch for packing plan changes with state manager %s." % str(state_manager))
def test_all_zk_supports_comma_separated_hostports(self): """Verify that a comma separated list of host ports is ok""" conf = Config() conf.set_state_locations([{'type':'zookeeper', 'name':'zk', 'hostport':'127.0.0.1:2181,127.0.0.1:2281', 'rootpath':'/heron', 'tunnelhost':'127.0.0.1'}]) statemanagers = statemanagerfactory.get_all_zk_state_managers(conf) # 1 state_location should result in 1 state manager self.assertEquals(1, len(statemanagers)) statemanager = statemanagers[0] # statemanager.hostportlist should contain both host port pairs self.assertTrue(('127.0.0.1', 2181) in statemanager.hostportlist) self.assertTrue(('127.0.0.1', 2281) in statemanager.hostportlist)
def test_all_zk_supports_comma_separated_hostports(self): """Verify that a comma separated list of host ports is ok""" conf = Config() conf.set_state_locations([{'type':'zookeeper', 'name':'zk', 'hostport':'127.0.0.1:2181,127.0.0.1:2281', 'rootpath':'/heron', 'tunnelhost':'127.0.0.1'}]) statemanagers = statemanagerfactory.get_all_zk_state_managers(conf) # 1 state_location should result in 1 state manager self.assertEqual(1, len(statemanagers)) statemanager = statemanagers[0] # statemanager.hostportlist should contain both host port pairs self.assertTrue(('127.0.0.1', 2181) in statemanager.hostportlist) self.assertTrue(('127.0.0.1', 2281) in statemanager.hostportlist)
class Config: """ Responsible for reading the yaml config file and exposing various tracker configs. """ FORMATTER_PARAMETERS = {"CLUSTER", "ENVIRON", "TOPOLOGY", "ROLE", "USER"} def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() self.statemgr_config.set_state_locations(configs[STATEMGRS_KEY]) self.extra_links = configs.get(EXTRA_LINKS_KEY, []) for link in self.extra_links: self.validate_extra_link(link) @classmethod def validate_extra_link(cls, extra_link: dict) -> None: """validate extra link""" if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link: raise Exception("Invalid extra.links format. " + "Extra link must include a 'name' and 'formatter' field") cls.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY]) @classmethod def validated_formatter(cls, url_format: str) -> None: """Check visualization url format has no unrecongnised parameters.""" # collect the parameters which would be interpolated formatter_variables = set() class ValidationHelper: def __getitem__(self, key): formatter_variables.add(key) return "" string.Template(url_format).safe_substitute(ValidationHelper()) if not formatter_variables <= cls.FORMATTER_PARAMETERS: raise Exception(f"Invalid viz.url.format: {url_format!r}") def __str__(self): return "".join(self.config_str(c) for c in self.configs[STATEMGRS_KEY]) @staticmethod def config_str(config): keys = ("type", "name", "hostport", "rootpath", "tunnelhost") # pylint: disable=consider-using-f-string return "".join("\t{}: {}\n".format(k, config[k]) for k in keys if k in config).rstrip()
def start_state_manager_watches(self): """ Receive updates to the packing plan from the statemgrs and update processes as needed. """ Log.info("Start state manager watches") with open(self.override_config_file, 'r') as stream: overrides = yaml.load(stream) if overrides is None: overrides = {} overrides["heron.statemgr.connection.string"] = self.state_manager_connection statemgr_config = StateMgrConfig() statemgr_config.set_state_locations(configloader.load_state_manager_locations( self.cluster, state_manager_config_file=self.state_manager_config_file, **overrides)) try: self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config) for state_manager in self.state_managers: state_manager.start() except Exception as ex: Log.error("Found exception while initializing state managers: %s. Bailing out..." % ex) traceback.print_exc() sys.exit(1) # pylint: disable=unused-argument def on_packing_plan_watch(state_manager, new_packing_plan): Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" % (self.shard, str(self.packing_plan), str(new_packing_plan))) if self.packing_plan != new_packing_plan: Log.info("PackingPlan change detected on shard %s, relaunching effected processes." % self.shard) self.update_packing_plan(new_packing_plan) Log.info("Updating executor processes") self.launch() else: Log.info( "State watch triggered for PackingPlan update but plan not changed so not relaunching.") for state_manager in self.state_managers: # The callback function with the bound # state_manager as first variable. onPackingPlanWatch = functools.partial(on_packing_plan_watch, state_manager) state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch) Log.info("Registered state watch for packing plan changes with state manager %s." % str(state_manager))
def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() self.statemgr_config.set_state_locations(configs[STATEMGRS_KEY]) self.extra_links = configs.get(EXTRA_LINKS_KEY, []) for link in self.extra_links: self.validate_extra_link(link)
def start_state_manager_watches(self): """ Receive updates to the packing plan from the statemgrs and update processes as needed. """ statemgr_config = StateMgrConfig() statemgr_config.set_state_locations(configloader.load_state_manager_locations( self.cluster, state_manager_config_file=self.state_manager_config_file, overrides={"heron.statemgr.connection.string": self.state_manager_connection})) try: self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config) for state_manager in self.state_managers: state_manager.start() except Exception as ex: Log.error("Found exception while initializing state managers: %s. Bailing out..." % ex) traceback.print_exc() sys.exit(1) # pylint: disable=unused-argument def on_packing_plan_watch(state_manager, new_packing_plan): Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" % (self.shard, str(self.packing_plan), str(new_packing_plan))) if self.packing_plan != new_packing_plan: Log.info("PackingPlan change detected on shard %s, relaunching effected processes." % self.shard) self.update_packing_plan(new_packing_plan) Log.info("Updating executor processes") self.launch() else: Log.info( "State watch triggered for PackingPlan update but plan not changed so not relaunching.") for state_manager in self.state_managers: # The callback function with the bound # state_manager as first variable. onPackingPlanWatch = functools.partial(on_packing_plan_watch, state_manager) state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch) Log.info("Registered state watch for packing plan changes with state manager %s." % str(state_manager))
class Config(object): """ Responsible for reading the yaml config file and exposing various tracker configs. """ def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() self.extra_links = [] self.load_configs() def load_configs(self): """load config files""" self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY]) if EXTRA_LINKS_KEY in self.configs: for extra_link in self.configs[EXTRA_LINKS_KEY]: self.extra_links.append(self.validate_extra_link(extra_link)) def validate_extra_link(self, extra_link): """validate extra link""" if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link: raise Exception( "Invalid extra.links format. " + "Extra link must include a 'name' and 'formatter' field") self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY]) return extra_link # pylint: disable=no-self-use def validated_formatter(self, url_format): """validate visualization url format""" # We try to create a string by substituting all known # parameters. If an unknown parameter is present, an error # will be thrown valid_parameters = { "${CLUSTER}": "cluster", "${ENVIRON}": "environ", "${TOPOLOGY}": "topology", "${ROLE}": "role", "${USER}": "user", } dummy_formatted_url = url_format for key, value in list(valid_parameters.items()): dummy_formatted_url = dummy_formatted_url.replace(key, value) # All $ signs must have been replaced if '$' in dummy_formatted_url: raise Exception("Invalid viz.url.format: %s" % (url_format)) # No error is thrown, so the format is valid. return url_format def get_formatted_url(self, formatter, execution_state): """ @param formatter: The template string to interpolate @param execution_state: The python dict representing JSON execution_state @return Formatted viz url """ # Create the parameters based on execution state common_parameters = { "${CLUSTER}": execution_state.get("cluster", "${CLUSTER}"), "${ENVIRON}": execution_state.get("environ", "${ENVIRON}"), "${TOPOLOGY}": execution_state.get("jobname", "${TOPOLOGY}"), "${ROLE}": execution_state.get("role", "${ROLE}"), "${USER}": execution_state.get("submission_user", "${USER}"), } formatted_url = formatter for key, value in list(common_parameters.items()): formatted_url = formatted_url.replace(key, value) return formatted_url def __str__(self): return "".join( (self.config_str(c) for c in self.configs[STATEMGRS_KEY])) def config_str(self, config): keys = ("type", "name", "hostport", "rootpath", "tunnelhost") return "".join("\t{}: {}\n".format(k, config[k]) for k in keys if k in config).rstrip()
def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() self.extra_links = [] self.load_configs()
class Config(object): """ Responsible for reading the yaml config file and exposing various tracker configs. """ def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() self.viz_url_format = None self.load_configs() def load_configs(self): """load config files""" self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY]) if VIZ_URL_FORMAT_KEY in self.configs: self.viz_url_format = self.validated_viz_url_format(self.configs[VIZ_URL_FORMAT_KEY]) else: self.viz_url_format = "" # pylint: disable=no-self-use def validated_viz_url_format(self, viz_url_format): """validate visualization url format""" # We try to create a string by substituting all known # parameters. If an unknown parameter is present, an error # will be thrown valid_parameters = { "${CLUSTER}": "cluster", "${ENVIRON}": "environ", "${TOPOLOGY}": "topology", "${ROLE}": "role", "${USER}": "user", } dummy_formatted_viz_url = viz_url_format for key, value in valid_parameters.items(): dummy_formatted_viz_url = dummy_formatted_viz_url.replace(key, value) # All $ signs must have been replaced if '$' in dummy_formatted_viz_url: raise Exception("Invalid viz.url.format: %s" % (viz_url_format)) # No error is thrown, so the format is valid. return viz_url_format def get_formatted_viz_url(self, execution_state): """ @param execution_state: The python dict representing JSON execution_state @return Formatted viz url """ # Create the parameters based on execution state valid_parameters = { "${CLUSTER}": execution_state["cluster"], "${ENVIRON}": execution_state["environ"], "${TOPOLOGY}": execution_state["jobname"], "${ROLE}": execution_state["role"], "${USER}": execution_state["submission_user"], } formatted_viz_url = self.viz_url_format for key, value in valid_parameters.items(): formatted_viz_url = formatted_viz_url.replace(key, value) return formatted_viz_url def __str__(self): return "".join((self.config_str(c) for c in self.configs[STATEMGRS_KEY])) def config_str(self, config): keys = ("type", "name", "hostport", "rootpath", "tunnelhost") return "".join("\t{}: {}\n".format(k, config[k]) for k in keys if k in config).rstrip()
class Config(object): """ Responsible for reading the yaml config file and exposing various tracker configs. """ def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() self.extra_links = [] self.load_configs() def load_configs(self): """load config files""" self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY]) if EXTRA_LINKS_KEY in self.configs: for extra_link in self.configs[EXTRA_LINKS_KEY]: self.extra_links.append(self.validate_extra_link(extra_link)) def validate_extra_link(self, extra_link): """validate extra link""" if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link: raise Exception("Invalid extra.links format. " + "Extra link must include a 'name' and 'formatter' field") self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY]) return extra_link # pylint: disable=no-self-use def validated_formatter(self, url_format): """validate visualization url format""" # We try to create a string by substituting all known # parameters. If an unknown parameter is present, an error # will be thrown valid_parameters = { "${CLUSTER}": "cluster", "${ENVIRON}": "environ", "${TOPOLOGY}": "topology", "${ROLE}": "role", "${USER}": "user", } dummy_formatted_url = url_format for key, value in valid_parameters.items(): dummy_formatted_url = dummy_formatted_url.replace(key, value) # All $ signs must have been replaced if '$' in dummy_formatted_url: raise Exception("Invalid viz.url.format: %s" % (url_format)) # No error is thrown, so the format is valid. return url_format def get_formatted_url(self, execution_state, formatter): """ @param execution_state: The python dict representing JSON execution_state @return Formatted viz url """ # Create the parameters based on execution state valid_parameters = { "${CLUSTER}": execution_state["cluster"], "${ENVIRON}": execution_state["environ"], "${TOPOLOGY}": execution_state["jobname"], "${ROLE}": execution_state["role"], "${USER}": execution_state["submission_user"], } formatted_url = formatter for key, value in valid_parameters.items(): formatted_url = formatted_url.replace(key, value) return formatted_url def __str__(self): return "".join((self.config_str(c) for c in self.configs[STATEMGRS_KEY])) def config_str(self, config): keys = ("type", "name", "hostport", "rootpath", "tunnelhost") return "".join("\t{}: {}\n".format(k, config[k]) for k in keys if k in config).rstrip()
def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() self.viz_url_format = None self.load_configs()
class Config: """ Responsible for reading the yaml config file and exposing various tracker configs. """ FORMATTER_PARAMETERS = {"CLUSTER", "ENVIRON", "TOPOLOGY", "ROLE", "USER"} def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() self.extra_links = [] self.load_configs() def load_configs(self): """load config files""" self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY]) if EXTRA_LINKS_KEY in self.configs: for extra_link in self.configs[EXTRA_LINKS_KEY]: self.extra_links.append(self.validate_extra_link(extra_link)) def validate_extra_link(self, extra_link: dict) -> None: """validate extra link""" if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link: raise Exception( "Invalid extra.links format. " + "Extra link must include a 'name' and 'formatter' field") self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY]) def validated_formatter(self, url_format: str) -> None: """Check visualization url format has no unrecongnised parameters.""" # collect the parameters which would be interpolated formatter_variables = set() class ValidationHelper: def __getitem__(self, key): formatter_variables.add(key) return "" string.Template(url_format).safe_substitute(ValidationHelper()) if not formatter_variables <= self.FORMATTER_PARAMETERS: raise Exception(f"Invalid viz.url.format: {url_format!r}") @staticmethod def get_formatted_url(formatter: str, execution_state: dict) -> str: """ Format a url string using values from the execution state. """ subs = { var: execution_state[prop] for prop, var in (("cluster", "CLUSTER"), ("environ", "ENVIRON"), ("jobname", "TOPOLOGY"), ("role", "ROLE"), ("submission_user", "USER")) if prop in execution_state } return string.Template(formatter).substitute(subs) def __str__(self): return "".join( (self.config_str(c) for c in self.configs[STATEMGRS_KEY])) @staticmethod def config_str(config): keys = ("type", "name", "hostport", "rootpath", "tunnelhost") return "".join("\t{k}: {config[k]}\n" for k in keys if k in config).rstrip()
def __init__(self, conf_file): self.configs = None self.statemgr_config = StateMgrConfig() self.viz_url_format = None self.parse_config_file(conf_file)
class Config(object): """ Responsible for reading the yaml config file and exposing various tracker configs. """ def __init__(self, conf_file): self.configs = None self.statemgr_config = StateMgrConfig() self.viz_url_format = None self.parse_config_file(conf_file) def parse_config_file(self, conf_file): """parse config files""" expanded_conf_file_path = os.path.expanduser(conf_file) assert os.path.lexists(expanded_conf_file_path), "Config file does not exists: %s" % (conf_file) # Read the configuration file with open(expanded_conf_file_path, 'r') as f: self.configs = yaml.load(f) self.load_configs() def load_configs(self): """load config files""" self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY]) if VIZ_URL_FORMAT_KEY in self.configs: self.viz_url_format = self.validated_viz_url_format(self.configs[VIZ_URL_FORMAT_KEY]) else: self.viz_url_format = "" # pylint: disable=no-self-use def validated_viz_url_format(self, viz_url_format): """validate visualization url format""" # We try to create a string by substituting all known # parameters. If an unknown parameter is present, an error # will be thrown valid_parameters = { "${CLUSTER}": "cluster", "${ENVIRON}": "environ", "${TOPOLOGY}": "topology", "${ROLE}": "role", "${USER}": "user", } dummy_formatted_viz_url = viz_url_format for key, value in valid_parameters.iteritems(): dummy_formatted_viz_url = dummy_formatted_viz_url.replace(key, value) # All $ signs must have been replaced if '$' in dummy_formatted_viz_url: raise Exception("Invalid viz.url.format: %s" % (viz_url_format)) # No error is thrown, so the format is valid. return viz_url_format def get_formatted_viz_url(self, execution_state): """ @param execution_state: The python dict representing JSON execution_state @return Formatted viz url """ # Create the parameters based on execution state valid_parameters = { "${CLUSTER}": execution_state["cluster"], "${ENVIRON}": execution_state["environ"], "${TOPOLOGY}": execution_state["jobname"], "${ROLE}": execution_state["role"], "${USER}": execution_state["submission_user"], } formatted_viz_url = self.viz_url_format for key, value in valid_parameters.iteritems(): formatted_viz_url = formatted_viz_url.replace(key, value) return formatted_viz_url
class Config(object): """ Responsible for reading the yaml config file and exposing various tracker configs. """ def __init__(self, conf_file): self.configs = None self.statemgr_config = StateMgrConfig() self.viz_url_format = None self.parse_config_file(conf_file) def parse_config_file(self, conf_file): """parse config files""" expanded_conf_file_path = os.path.expanduser(conf_file) assert os.path.lexists( expanded_conf_file_path), "Config file does not exists: %s" % ( conf_file) # Read the configuration file with open(expanded_conf_file_path, 'r') as f: self.configs = yaml.load(f) self.load_configs() def load_configs(self): """load config files""" self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY]) if VIZ_URL_FORMAT_KEY in self.configs: self.viz_url_format = self.validated_viz_url_format( self.configs[VIZ_URL_FORMAT_KEY]) else: self.viz_url_format = "" # pylint: disable=no-self-use def validated_viz_url_format(self, viz_url_format): """validate visualization url format""" # We try to create a string by substituting all known # parameters. If an unknown parameter is present, an error # will be thrown valid_parameters = { "${CLUSTER}": "cluster", "${ENVIRON}": "environ", "${TOPOLOGY}": "topology", "${ROLE}": "role", "${USER}": "user", } dummy_formatted_viz_url = viz_url_format for key, value in valid_parameters.iteritems(): dummy_formatted_viz_url = dummy_formatted_viz_url.replace( key, value) # All $ signs must have been replaced if '$' in dummy_formatted_viz_url: raise Exception("Invalid viz.url.format: %s" % (viz_url_format)) # No error is thrown, so the format is valid. return viz_url_format def get_formatted_viz_url(self, execution_state): """ @param execution_state: The python dict representing JSON execution_state @return Formatted viz url """ # Create the parameters based on execution state valid_parameters = { "${CLUSTER}": execution_state["cluster"], "${ENVIRON}": execution_state["environ"], "${TOPOLOGY}": execution_state["jobname"], "${ROLE}": execution_state["role"], "${USER}": execution_state["submission_user"], } formatted_viz_url = self.viz_url_format for key, value in valid_parameters.iteritems(): formatted_viz_url = formatted_viz_url.replace(key, value) return formatted_viz_url