def resolve_options(cli_options,
env_config,
topology_class,
topology_name,
local_only=False):
"""Resolve potentially conflicting Storm options from three sources:
CLI options > Topology options > config.json options
:param local_only: Whether or not we should talk to Nimbus to get Storm
workers and other info.
"""
storm_options = {}
# Start with environment options
storm_options.update(env_config.get("options", {}))
# Set topology.python.path
if env_config.get("use_virtualenv", True):
python_path = "/".join(
[env_config["virtualenv_root"], topology_name, "bin", "python"])
# This setting is for information purposes only, and is not actually
# read by any pystorm or streamparse code.
storm_options["topology.python.path"] = python_path
# Set logging options based on environment config
log_config = env_config.get("log", {})
log_path = log_config.get("path") or env_config.get("log_path")
log_file = log_config.get("file") or env_config.get("log_file")
if log_path:
storm_options["pystorm.log.path"] = log_path
if log_file:
storm_options["pystorm.log.file"] = log_file
if isinstance(log_config.get("max_bytes"), integer_types):
storm_options["pystorm.log.max_bytes"] = log_config["max_bytes"]
if isinstance(log_config.get("backup_count"), integer_types):
storm_options["pystorm.log.backup_count"] = log_config["backup_count"]
if isinstance(log_config.get("level"), string_types):
storm_options["pystorm.log.level"] = log_config["level"].lower()
# Make sure virtualenv options are present here
for venv_option in VIRTUALENV_OPTIONS:
if venv_option in env_config:
storm_options[venv_option] = env_config[venv_option]
# Override options with topology options
storm_options.update(topology_class.config)
# Override options with CLI options
storm_options.update(cli_options or {})
# Set log level to debug if topology.debug is set
if storm_options.get("topology.debug", False):
storm_options["pystorm.log.level"] = "debug"
# If ackers and executors still aren't set, use number of worker nodes
if not local_only:
if not storm_options.get("storm.workers.list"):
storm_options["storm.workers.list"] = get_storm_workers(env_config)
elif isinstance(storm_options["storm.workers.list"], string_types):
storm_options["storm.workers.list"] = storm_options[
"storm.workers.list"].split(",")
num_storm_workers = len(storm_options["storm.workers.list"])
else:
storm_options["storm.workers.list"] = []
num_storm_workers = 1
if storm_options.get("topology.acker.executors") is None:
storm_options["topology.acker.executors"] = num_storm_workers
if storm_options.get("topology.workers") is None:
storm_options["topology.workers"] = num_storm_workers
return storm_options
def resolve_options(
cli_options, env_config, topology_class, topology_name, local_only=False
):
"""Resolve potentially conflicting Storm options from three sources:
CLI options > Topology options > config.json options
:param local_only: Whether or not we should talk to Nimbus to get Storm
workers and other info.
"""
storm_options = {}
# Start with environment options
storm_options.update(env_config.get("options", {}))
# Set topology.python.path
if env_config.get("use_virtualenv", True):
python_path = "/".join(
[env_config["virtualenv_root"], topology_name, "bin", "python"]
)
# This setting is for information purposes only, and is not actually
# read by any pystorm or streamparse code.
storm_options["topology.python.path"] = python_path
# Set logging options based on environment config
log_config = env_config.get("log", {})
log_path = log_config.get("path") or env_config.get("log_path")
log_file = log_config.get("file") or env_config.get("log_file")
if log_path:
storm_options["pystorm.log.path"] = log_path
if log_file:
storm_options["pystorm.log.file"] = log_file
if isinstance(log_config.get("max_bytes"), int):
storm_options["pystorm.log.max_bytes"] = log_config["max_bytes"]
if isinstance(log_config.get("backup_count"), int):
storm_options["pystorm.log.backup_count"] = log_config["backup_count"]
if isinstance(log_config.get("level"), str):
storm_options["pystorm.log.level"] = log_config["level"].lower()
# Make sure virtualenv options are present here
for venv_option in VIRTUALENV_OPTIONS:
if venv_option in env_config:
storm_options[venv_option] = env_config[venv_option]
# Set sudo_user default to SSH user if we have one
storm_options["sudo_user"] = env_config.get("user", None)
# Override options with topology options
storm_options.update(topology_class.config)
# Override options with CLI options
storm_options.update(cli_options or {})
# Set log level to debug if topology.debug is set
if storm_options.get("topology.debug", False):
storm_options["pystorm.log.level"] = "debug"
# If ackers and executors still aren't set, use number of worker nodes
if not local_only:
if not storm_options.get("storm.workers.list"):
storm_options["storm.workers.list"] = get_storm_workers(env_config)
elif isinstance(storm_options["storm.workers.list"], str):
storm_options["storm.workers.list"] = storm_options[
"storm.workers.list"
].split(",")
num_storm_workers = len(storm_options["storm.workers.list"])
else:
storm_options["storm.workers.list"] = []
num_storm_workers = 1
if storm_options.get("topology.acker.executors") is None:
storm_options["topology.acker.executors"] = num_storm_workers
if storm_options.get("topology.workers") is None:
storm_options["topology.workers"] = num_storm_workers
# If sudo_user was not present anywhere, set it to "root"
storm_options.setdefault("sudo_user", "root")
return storm_options