def start_state_manager_watches(self):
"""
Receive updates to the packing plan from the statemgrs and update processes as needed.
"""
statemgr_config = StateMgrConfig()
statemgr_config.set_state_locations(configloader.load_state_manager_locations(self.cluster))
self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)
# pylint: disable=unused-argument
def on_packing_plan_watch(state_manager, new_packing_plan):
Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" %
(self.shard, str(self.packing_plan), str(new_packing_plan)))
if self.packing_plan != new_packing_plan:
Log.info("PackingPlan change detected on shard %s, relaunching effected processes."
% self.shard)
self.update_packing_plan(new_packing_plan)
Log.info("Updating executor processes")
self.launch()
else:
Log.info(
"State watch triggered for PackingPlan update but plan not changed so not relaunching.")
for state_manager in self.state_managers:
# The callback function with the bound
# state_manager as first variable.
onPackingPlanWatch = partial(on_packing_plan_watch, state_manager)
state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch)
Log.info("Registered state watch for packing plan changes with state manager %s." %
str(state_manager))
def start_state_manager_watches(self):
"""
Receive updates to the packing plan from the statemgrs and update processes as needed.
"""
statemgr_config = StateMgrConfig()
statemgr_config.set_state_locations(configloader.load_state_manager_locations(self.cluster))
self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)
# pylint: disable=unused-argument
def on_packing_plan_watch(state_manager, new_packing_plan):
Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" %
(self.shard, str(self.packing_plan), str(new_packing_plan)))
if self.packing_plan != new_packing_plan:
Log.info("PackingPlan change detected on shard %s, relaunching effected processes."
% self.shard)
self.update_packing_plan(new_packing_plan)
Log.info("Updating executor processes")
self.launch()
else:
Log.info(
"State watch triggered for PackingPlan update but plan not changed so not relaunching.")
for state_manager in self.state_managers:
# The callback function with the bound
# state_manager as first variable.
onPackingPlanWatch = partial(on_packing_plan_watch, state_manager)
state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch)
Log.info("Registered state watch for packing plan changes with state manager %s." %
str(state_manager))
def test_all_zk_supports_comma_separated_hostports(self):
"""Verify that a comma separated list of host ports is ok"""
conf = Config()
conf.set_state_locations([{'type':'zookeeper', 'name':'zk', 'hostport':'127.0.0.1:2181,127.0.0.1:2281',
'rootpath':'/heron', 'tunnelhost':'127.0.0.1'}])
statemanagers = statemanagerfactory.get_all_zk_state_managers(conf)
# 1 state_location should result in 1 state manager
self.assertEquals(1, len(statemanagers))
statemanager = statemanagers[0]
# statemanager.hostportlist should contain both host port pairs
self.assertTrue(('127.0.0.1', 2181) in statemanager.hostportlist)
self.assertTrue(('127.0.0.1', 2281) in statemanager.hostportlist)
def test_all_zk_supports_comma_separated_hostports(self):
"""Verify that a comma separated list of host ports is ok"""
conf = Config()
conf.set_state_locations([{'type':'zookeeper', 'name':'zk', 'hostport':'127.0.0.1:2181,127.0.0.1:2281',
'rootpath':'/heron', 'tunnelhost':'127.0.0.1'}])
statemanagers = statemanagerfactory.get_all_zk_state_managers(conf)
# 1 state_location should result in 1 state manager
self.assertEqual(1, len(statemanagers))
statemanager = statemanagers[0]
# statemanager.hostportlist should contain both host port pairs
self.assertTrue(('127.0.0.1', 2181) in statemanager.hostportlist)
self.assertTrue(('127.0.0.1', 2281) in statemanager.hostportlist)
class Config:
"""
Responsible for reading the yaml config file and
exposing various tracker configs.
"""
FORMATTER_PARAMETERS = {"CLUSTER", "ENVIRON", "TOPOLOGY", "ROLE", "USER"}
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.statemgr_config.set_state_locations(configs[STATEMGRS_KEY])
self.extra_links = configs.get(EXTRA_LINKS_KEY, [])
for link in self.extra_links:
self.validate_extra_link(link)
@classmethod
def validate_extra_link(cls, extra_link: dict) -> None:
"""validate extra link"""
if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link:
raise Exception("Invalid extra.links format. " +
"Extra link must include a 'name' and 'formatter' field")
cls.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY])
@classmethod
def validated_formatter(cls, url_format: str) -> None:
"""Check visualization url format has no unrecongnised parameters."""
# collect the parameters which would be interpolated
formatter_variables = set()
class ValidationHelper:
def __getitem__(self, key):
formatter_variables.add(key)
return ""
string.Template(url_format).safe_substitute(ValidationHelper())
if not formatter_variables <= cls.FORMATTER_PARAMETERS:
raise Exception(f"Invalid viz.url.format: {url_format!r}")
def __str__(self):
return "".join(self.config_str(c) for c in self.configs[STATEMGRS_KEY])
@staticmethod
def config_str(config):
keys = ("type", "name", "hostport", "rootpath", "tunnelhost")
# pylint: disable=consider-using-f-string
return "".join("\t{}: {}\n".format(k, config[k]) for k in keys if k in config).rstrip()
def start_state_manager_watches(self):
"""
Receive updates to the packing plan from the statemgrs and update processes as needed.
"""
Log.info("Start state manager watches")
with open(self.override_config_file, 'r') as stream:
overrides = yaml.load(stream)
if overrides is None:
overrides = {}
overrides["heron.statemgr.connection.string"] = self.state_manager_connection
statemgr_config = StateMgrConfig()
statemgr_config.set_state_locations(configloader.load_state_manager_locations(
self.cluster, state_manager_config_file=self.state_manager_config_file,
**overrides))
try:
self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)
for state_manager in self.state_managers:
state_manager.start()
except Exception as ex:
Log.error("Found exception while initializing state managers: %s. Bailing out..." % ex)
traceback.print_exc()
sys.exit(1)
# pylint: disable=unused-argument
def on_packing_plan_watch(state_manager, new_packing_plan):
Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" %
(self.shard, str(self.packing_plan), str(new_packing_plan)))
if self.packing_plan != new_packing_plan:
Log.info("PackingPlan change detected on shard %s, relaunching effected processes."
% self.shard)
self.update_packing_plan(new_packing_plan)
Log.info("Updating executor processes")
self.launch()
else:
Log.info(
"State watch triggered for PackingPlan update but plan not changed so not relaunching.")
for state_manager in self.state_managers:
# The callback function with the bound
# state_manager as first variable.
onPackingPlanWatch = functools.partial(on_packing_plan_watch, state_manager)
state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch)
Log.info("Registered state watch for packing plan changes with state manager %s." %
str(state_manager))
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.statemgr_config.set_state_locations(configs[STATEMGRS_KEY])
self.extra_links = configs.get(EXTRA_LINKS_KEY, [])
for link in self.extra_links:
self.validate_extra_link(link)
def start_state_manager_watches(self):
"""
Receive updates to the packing plan from the statemgrs and update processes as needed.
"""
statemgr_config = StateMgrConfig()
statemgr_config.set_state_locations(configloader.load_state_manager_locations(
self.cluster, state_manager_config_file=self.state_manager_config_file,
overrides={"heron.statemgr.connection.string": self.state_manager_connection}))
try:
self.state_managers = statemanagerfactory.get_all_state_managers(statemgr_config)
for state_manager in self.state_managers:
state_manager.start()
except Exception as ex:
Log.error("Found exception while initializing state managers: %s. Bailing out..." % ex)
traceback.print_exc()
sys.exit(1)
# pylint: disable=unused-argument
def on_packing_plan_watch(state_manager, new_packing_plan):
Log.debug("State watch triggered for PackingPlan update on shard %s. Existing: %s, New: %s" %
(self.shard, str(self.packing_plan), str(new_packing_plan)))
if self.packing_plan != new_packing_plan:
Log.info("PackingPlan change detected on shard %s, relaunching effected processes."
% self.shard)
self.update_packing_plan(new_packing_plan)
Log.info("Updating executor processes")
self.launch()
else:
Log.info(
"State watch triggered for PackingPlan update but plan not changed so not relaunching.")
for state_manager in self.state_managers:
# The callback function with the bound
# state_manager as first variable.
onPackingPlanWatch = functools.partial(on_packing_plan_watch, state_manager)
state_manager.get_packing_plan(self.topology_name, onPackingPlanWatch)
Log.info("Registered state watch for packing plan changes with state manager %s." %
str(state_manager))
class Config(object):
"""
Responsible for reading the yaml config file and
exposing various tracker configs.
"""
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.extra_links = []
self.load_configs()
def load_configs(self):
"""load config files"""
self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])
if EXTRA_LINKS_KEY in self.configs:
for extra_link in self.configs[EXTRA_LINKS_KEY]:
self.extra_links.append(self.validate_extra_link(extra_link))
def validate_extra_link(self, extra_link):
"""validate extra link"""
if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link:
raise Exception(
"Invalid extra.links format. " +
"Extra link must include a 'name' and 'formatter' field")
self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY])
return extra_link
# pylint: disable=no-self-use
def validated_formatter(self, url_format):
"""validate visualization url format"""
# We try to create a string by substituting all known
# parameters. If an unknown parameter is present, an error
# will be thrown
valid_parameters = {
"${CLUSTER}": "cluster",
"${ENVIRON}": "environ",
"${TOPOLOGY}": "topology",
"${ROLE}": "role",
"${USER}": "user",
}
dummy_formatted_url = url_format
for key, value in list(valid_parameters.items()):
dummy_formatted_url = dummy_formatted_url.replace(key, value)
# All $ signs must have been replaced
if '$' in dummy_formatted_url:
raise Exception("Invalid viz.url.format: %s" % (url_format))
# No error is thrown, so the format is valid.
return url_format
def get_formatted_url(self, formatter, execution_state):
"""
@param formatter: The template string to interpolate
@param execution_state: The python dict representing JSON execution_state
@return Formatted viz url
"""
# Create the parameters based on execution state
common_parameters = {
"${CLUSTER}": execution_state.get("cluster", "${CLUSTER}"),
"${ENVIRON}": execution_state.get("environ", "${ENVIRON}"),
"${TOPOLOGY}": execution_state.get("jobname", "${TOPOLOGY}"),
"${ROLE}": execution_state.get("role", "${ROLE}"),
"${USER}": execution_state.get("submission_user", "${USER}"),
}
formatted_url = formatter
for key, value in list(common_parameters.items()):
formatted_url = formatted_url.replace(key, value)
return formatted_url
def __str__(self):
return "".join(
(self.config_str(c) for c in self.configs[STATEMGRS_KEY]))
def config_str(self, config):
keys = ("type", "name", "hostport", "rootpath", "tunnelhost")
return "".join("\t{}: {}\n".format(k, config[k]) for k in keys
if k in config).rstrip()
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.extra_links = []
self.load_configs()
class Config(object):
"""
Responsible for reading the yaml config file and
exposing various tracker configs.
"""
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.viz_url_format = None
self.load_configs()
def load_configs(self):
"""load config files"""
self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])
if VIZ_URL_FORMAT_KEY in self.configs:
self.viz_url_format = self.validated_viz_url_format(self.configs[VIZ_URL_FORMAT_KEY])
else:
self.viz_url_format = ""
# pylint: disable=no-self-use
def validated_viz_url_format(self, viz_url_format):
"""validate visualization url format"""
# We try to create a string by substituting all known
# parameters. If an unknown parameter is present, an error
# will be thrown
valid_parameters = {
"${CLUSTER}": "cluster",
"${ENVIRON}": "environ",
"${TOPOLOGY}": "topology",
"${ROLE}": "role",
"${USER}": "user",
}
dummy_formatted_viz_url = viz_url_format
for key, value in valid_parameters.items():
dummy_formatted_viz_url = dummy_formatted_viz_url.replace(key, value)
# All $ signs must have been replaced
if '$' in dummy_formatted_viz_url:
raise Exception("Invalid viz.url.format: %s" % (viz_url_format))
# No error is thrown, so the format is valid.
return viz_url_format
def get_formatted_viz_url(self, execution_state):
"""
@param execution_state: The python dict representing JSON execution_state
@return Formatted viz url
"""
# Create the parameters based on execution state
valid_parameters = {
"${CLUSTER}": execution_state["cluster"],
"${ENVIRON}": execution_state["environ"],
"${TOPOLOGY}": execution_state["jobname"],
"${ROLE}": execution_state["role"],
"${USER}": execution_state["submission_user"],
}
formatted_viz_url = self.viz_url_format
for key, value in valid_parameters.items():
formatted_viz_url = formatted_viz_url.replace(key, value)
return formatted_viz_url
def __str__(self):
return "".join((self.config_str(c) for c in self.configs[STATEMGRS_KEY]))
def config_str(self, config):
keys = ("type", "name", "hostport", "rootpath", "tunnelhost")
return "".join("\t{}: {}\n".format(k, config[k]) for k in keys if k in config).rstrip()
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.extra_links = []
self.load_configs()
class Config(object):
"""
Responsible for reading the yaml config file and
exposing various tracker configs.
"""
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.extra_links = []
self.load_configs()
def load_configs(self):
"""load config files"""
self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])
if EXTRA_LINKS_KEY in self.configs:
for extra_link in self.configs[EXTRA_LINKS_KEY]:
self.extra_links.append(self.validate_extra_link(extra_link))
def validate_extra_link(self, extra_link):
"""validate extra link"""
if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link:
raise Exception("Invalid extra.links format. " +
"Extra link must include a 'name' and 'formatter' field")
self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY])
return extra_link
# pylint: disable=no-self-use
def validated_formatter(self, url_format):
"""validate visualization url format"""
# We try to create a string by substituting all known
# parameters. If an unknown parameter is present, an error
# will be thrown
valid_parameters = {
"${CLUSTER}": "cluster",
"${ENVIRON}": "environ",
"${TOPOLOGY}": "topology",
"${ROLE}": "role",
"${USER}": "user",
}
dummy_formatted_url = url_format
for key, value in valid_parameters.items():
dummy_formatted_url = dummy_formatted_url.replace(key, value)
# All $ signs must have been replaced
if '$' in dummy_formatted_url:
raise Exception("Invalid viz.url.format: %s" % (url_format))
# No error is thrown, so the format is valid.
return url_format
def get_formatted_url(self, execution_state, formatter):
"""
@param execution_state: The python dict representing JSON execution_state
@return Formatted viz url
"""
# Create the parameters based on execution state
valid_parameters = {
"${CLUSTER}": execution_state["cluster"],
"${ENVIRON}": execution_state["environ"],
"${TOPOLOGY}": execution_state["jobname"],
"${ROLE}": execution_state["role"],
"${USER}": execution_state["submission_user"],
}
formatted_url = formatter
for key, value in valid_parameters.items():
formatted_url = formatted_url.replace(key, value)
return formatted_url
def __str__(self):
return "".join((self.config_str(c) for c in self.configs[STATEMGRS_KEY]))
def config_str(self, config):
keys = ("type", "name", "hostport", "rootpath", "tunnelhost")
return "".join("\t{}: {}\n".format(k, config[k]) for k in keys if k in config).rstrip()
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.viz_url_format = None
self.load_configs()
class Config:
"""
Responsible for reading the yaml config file and
exposing various tracker configs.
"""
FORMATTER_PARAMETERS = {"CLUSTER", "ENVIRON", "TOPOLOGY", "ROLE", "USER"}
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.extra_links = []
self.load_configs()
def load_configs(self):
"""load config files"""
self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])
if EXTRA_LINKS_KEY in self.configs:
for extra_link in self.configs[EXTRA_LINKS_KEY]:
self.extra_links.append(self.validate_extra_link(extra_link))
def validate_extra_link(self, extra_link: dict) -> None:
"""validate extra link"""
if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link:
raise Exception(
"Invalid extra.links format. " +
"Extra link must include a 'name' and 'formatter' field")
self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY])
def validated_formatter(self, url_format: str) -> None:
"""Check visualization url format has no unrecongnised parameters."""
# collect the parameters which would be interpolated
formatter_variables = set()
class ValidationHelper:
def __getitem__(self, key):
formatter_variables.add(key)
return ""
string.Template(url_format).safe_substitute(ValidationHelper())
if not formatter_variables <= self.FORMATTER_PARAMETERS:
raise Exception(f"Invalid viz.url.format: {url_format!r}")
@staticmethod
def get_formatted_url(formatter: str, execution_state: dict) -> str:
"""
Format a url string using values from the execution state.
"""
subs = {
var: execution_state[prop]
for prop, var in (("cluster", "CLUSTER"), ("environ", "ENVIRON"),
("jobname", "TOPOLOGY"), ("role", "ROLE"),
("submission_user", "USER"))
if prop in execution_state
}
return string.Template(formatter).substitute(subs)
def __str__(self):
return "".join(
(self.config_str(c) for c in self.configs[STATEMGRS_KEY]))
@staticmethod
def config_str(config):
keys = ("type", "name", "hostport", "rootpath", "tunnelhost")
return "".join("\t{k}: {config[k]}\n" for k in keys
if k in config).rstrip()
def __init__(self, conf_file):
self.configs = None
self.statemgr_config = StateMgrConfig()
self.viz_url_format = None
self.parse_config_file(conf_file)
class Config(object):
"""
Responsible for reading the yaml config file and
exposing various tracker configs.
"""
def __init__(self, conf_file):
self.configs = None
self.statemgr_config = StateMgrConfig()
self.viz_url_format = None
self.parse_config_file(conf_file)
def parse_config_file(self, conf_file):
"""parse config files"""
expanded_conf_file_path = os.path.expanduser(conf_file)
assert os.path.lexists(expanded_conf_file_path), "Config file does not exists: %s" % (conf_file)
# Read the configuration file
with open(expanded_conf_file_path, 'r') as f:
self.configs = yaml.load(f)
self.load_configs()
def load_configs(self):
"""load config files"""
self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])
if VIZ_URL_FORMAT_KEY in self.configs:
self.viz_url_format = self.validated_viz_url_format(self.configs[VIZ_URL_FORMAT_KEY])
else:
self.viz_url_format = ""
# pylint: disable=no-self-use
def validated_viz_url_format(self, viz_url_format):
"""validate visualization url format"""
# We try to create a string by substituting all known
# parameters. If an unknown parameter is present, an error
# will be thrown
valid_parameters = {
"${CLUSTER}": "cluster",
"${ENVIRON}": "environ",
"${TOPOLOGY}": "topology",
"${ROLE}": "role",
"${USER}": "user",
}
dummy_formatted_viz_url = viz_url_format
for key, value in valid_parameters.iteritems():
dummy_formatted_viz_url = dummy_formatted_viz_url.replace(key, value)
# All $ signs must have been replaced
if '$' in dummy_formatted_viz_url:
raise Exception("Invalid viz.url.format: %s" % (viz_url_format))
# No error is thrown, so the format is valid.
return viz_url_format
def get_formatted_viz_url(self, execution_state):
"""
@param execution_state: The python dict representing JSON execution_state
@return Formatted viz url
"""
# Create the parameters based on execution state
valid_parameters = {
"${CLUSTER}": execution_state["cluster"],
"${ENVIRON}": execution_state["environ"],
"${TOPOLOGY}": execution_state["jobname"],
"${ROLE}": execution_state["role"],
"${USER}": execution_state["submission_user"],
}
formatted_viz_url = self.viz_url_format
for key, value in valid_parameters.iteritems():
formatted_viz_url = formatted_viz_url.replace(key, value)
return formatted_viz_url
def __init__(self, configs):
self.configs = configs
self.statemgr_config = StateMgrConfig()
self.viz_url_format = None
self.load_configs()
def __init__(self, conf_file):
self.configs = None
self.statemgr_config = StateMgrConfig()
self.viz_url_format = None
self.parse_config_file(conf_file)
class Config(object):
"""
Responsible for reading the yaml config file and
exposing various tracker configs.
"""
def __init__(self, conf_file):
self.configs = None
self.statemgr_config = StateMgrConfig()
self.viz_url_format = None
self.parse_config_file(conf_file)
def parse_config_file(self, conf_file):
"""parse config files"""
expanded_conf_file_path = os.path.expanduser(conf_file)
assert os.path.lexists(
expanded_conf_file_path), "Config file does not exists: %s" % (
conf_file)
# Read the configuration file
with open(expanded_conf_file_path, 'r') as f:
self.configs = yaml.load(f)
self.load_configs()
def load_configs(self):
"""load config files"""
self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY])
if VIZ_URL_FORMAT_KEY in self.configs:
self.viz_url_format = self.validated_viz_url_format(
self.configs[VIZ_URL_FORMAT_KEY])
else:
self.viz_url_format = ""
# pylint: disable=no-self-use
def validated_viz_url_format(self, viz_url_format):
"""validate visualization url format"""
# We try to create a string by substituting all known
# parameters. If an unknown parameter is present, an error
# will be thrown
valid_parameters = {
"${CLUSTER}": "cluster",
"${ENVIRON}": "environ",
"${TOPOLOGY}": "topology",
"${ROLE}": "role",
"${USER}": "user",
}
dummy_formatted_viz_url = viz_url_format
for key, value in valid_parameters.iteritems():
dummy_formatted_viz_url = dummy_formatted_viz_url.replace(
key, value)
# All $ signs must have been replaced
if '$' in dummy_formatted_viz_url:
raise Exception("Invalid viz.url.format: %s" % (viz_url_format))
# No error is thrown, so the format is valid.
return viz_url_format
def get_formatted_viz_url(self, execution_state):
"""
@param execution_state: The python dict representing JSON execution_state
@return Formatted viz url
"""
# Create the parameters based on execution state
valid_parameters = {
"${CLUSTER}": execution_state["cluster"],
"${ENVIRON}": execution_state["environ"],
"${TOPOLOGY}": execution_state["jobname"],
"${ROLE}": execution_state["role"],
"${USER}": execution_state["submission_user"],
}
formatted_viz_url = self.viz_url_format
for key, value in valid_parameters.iteritems():
formatted_viz_url = formatted_viz_url.replace(key, value)
return formatted_viz_url
