def get_storage_set_keys(self) -> Set[StorageSetKey]:
all_storage_sets = set(key.value for key in StorageSetKey)
storage_set_keys = set()
for storage_set in self.__storage_sets:
# We ignore invalid storage set keys since new storage sets will
# need to be registered to configuration before they can be used
# in Snuba.
if storage_set in all_storage_sets:
storage_set_keys.add(StorageSetKey(storage_set))
return storage_set_keys
def add_node(
node_type: str,
storage_set_names: Sequence[str],
host_name: str,
port: int,
database: str,
) -> None:
"""
Runs all migrations on a brand new ClickHouse node. This should be performed
before a new node is added to an existing ClickHouse cluster.
All of the SQL operations for the provided storage sets will be run. Any non
SQL (Python) operations will be skipped.
This operation does not change the migration status in the migrations_local
/ migrations_dist tables, since it is designed to bring a new node up to
the same state as existing ones already added to the cluster.
"""
user = os.environ.get("CLICKHOUSE_USER", "default")
password = os.environ.get("CLICKHOUSE_PASSWORD", "")
storage_set_keys = [StorageSetKey(name) for name in storage_set_names]
cluster = next(
(
c
for c in CLUSTERS
if all(ss in c.get_storage_set_keys() for ss in storage_set_keys)
),
None,
)
if not cluster:
raise click.ClickException("Storage sets should be in the same cluster")
if cluster.is_single_node():
raise click.ClickException("You cannot add a node to a single node cluster")
Runner.add_node(
node_type=ClickhouseNodeType(node_type),
storage_sets=storage_set_keys,
host_name=host_name,
port=port,
user=user,
password=password,
database=database,
)
def _validate_settings(locals: Mapping[str, Any]) -> None:
if locals.get("QUERIES_TOPIC"):
raise ValueError(
"QUERIES_TOPIC is deprecated. Use KAFKA_TOPIC_MAP instead.")
if locals.get("STORAGE_TOPICS"):
raise ValueError(
"STORAGE_TOPICS is deprecated. Use KAFKA_TOPIC_MAP instead.")
if locals.get("STORAGE_BROKER_CONFIG"):
raise ValueError(
"DEPRECATED: STORAGE_BROKER_CONFIG is deprecated. Use KAFKA_BROKER_CONFIG instead."
)
if locals.get("DEFAULT_STORAGE_BROKERS"):
raise ValueError(
"DEFAULT_STORAGE_BROKERS is deprecated. Use KAFKA_BROKER_CONFIG instead."
)
topic_names = {
"events",
"event-replacements",
"event-replacements-legacy",
"snuba-commit-log",
"cdc",
"ingest-metrics",
"outcomes",
"ingest-sessions",
"snuba-queries",
"events-subscription-results",
"transactions-subscription-results",
}
for key in locals["KAFKA_TOPIC_MAP"].keys():
if key not in topic_names:
raise ValueError(f"Invalid topic value: {key}")
for key in locals["KAFKA_BROKER_CONFIG"].keys():
if key not in topic_names:
raise ValueError(f"Invalid topic value {key}")
# Validate cluster configuration
from snuba.clusters.storage_sets import JOINABLE_STORAGE_SETS, StorageSetKey
storage_set_to_cluster: MutableMapping[StorageSetKey, Any] = {}
for cluster in locals["CLUSTERS"]:
for cluster_storage_set in cluster["storage_sets"]:
storage_set_to_cluster[StorageSetKey(
cluster_storage_set)] = cluster
for group in JOINABLE_STORAGE_SETS:
clusters = [
storage_set_to_cluster[storage_set] for storage_set in group
]
first = clusters[0]
for cluster in clusters[1:]:
if first != cluster:
for property in [
"host",
"port",
"user",
"password",
"database",
"http_port",
"single_node",
"distributed_cluster_name",
]:
assert first.get(property) == cluster.get(
property), f"Invalid property: {property}"