def list() -> None:
"""
Lists migrations and their statuses
"""
check_clickhouse_connections()
runner = Runner()
for group, group_migrations in runner.show_all():
click.echo(group.value)
for migration_id, status, blocking in group_migrations:
symbol = {
Status.COMPLETED: "X",
Status.NOT_STARTED: " ",
Status.IN_PROGRESS: "-",
}[status]
in_progress_text = " (IN PROGRESS)" if status == Status.IN_PROGRESS else ""
blocking_text = ""
if status != Status.COMPLETED and blocking:
blocking_text = " (blocking)"
click.echo(
f"[{symbol}] {migration_id}{in_progress_text}{blocking_text}")
click.echo()
def run(group: str, migration_id: str, force: bool, fake: bool,
dry_run: bool) -> None:
"""
Runs a single migration.
--force must be passed in order to run blocking migrations.
--fake marks a migration as completed without running anything.
Migrations that are already in an in-progress or completed status will not be run.
"""
if not dry_run:
check_clickhouse_connections()
runner = Runner()
migration_group = MigrationGroup(group)
migration_key = MigrationKey(migration_group, migration_id)
if dry_run:
runner.run_migration(migration_key, dry_run=True)
return
try:
if fake:
click.confirm(
"This will mark the migration as completed without actually running it. Your database may be in an invalid state. Are you sure?",
abort=True,
)
runner.run_migration(migration_key, force=force, fake=fake)
except MigrationError as e:
raise click.ClickException(str(e))
click.echo(f"Finished running migration {migration_key}")
def reverse(group: str, migration_id: str, force: bool, fake: bool,
dry_run: bool) -> None:
"""
Reverses a single migration.
--force is required to reverse an already completed migration.
--fake marks a migration as reversed without doing anything.
"""
if not dry_run:
check_clickhouse_connections()
runner = Runner()
migration_group = MigrationGroup(group)
migration_key = MigrationKey(migration_group, migration_id)
if dry_run:
runner.reverse_migration(migration_key, dry_run=True)
return
try:
if fake:
click.confirm(
"This will mark the migration as not started without actually reversing it. Your database may be in an invalid state. Are you sure?",
abort=True,
)
runner.reverse_migration(migration_key, force=force, fake=fake)
except MigrationError as e:
raise click.ClickException(str(e))
click.echo(f"Finished reversing migration {migration_key}")
def migrate(*, log_level: Optional[str] = None) -> None:
click.echo("Warning: The migrate command is deprecated and will be removed soon\n")
setup_logging(log_level)
check_clickhouse_connections()
run()
def migrate(force: bool) -> None:
"""
Runs all migrations. Blocking migrations will not be run unless --force is passed.
"""
check_clickhouse_connections()
runner = Runner()
try:
runner.run_all(force=force)
except MigrationError as e:
raise click.ClickException(str(e))
click.echo("Finished running migrations")
def bootstrap(
*,
bootstrap_server: Sequence[str],
kafka: bool,
migrate: bool,
force: bool,
log_level: Optional[str] = None,
) -> None:
"""
Warning: Not intended to be used in production yet.
"""
if not force:
raise click.ClickException("Must use --force to run")
setup_logging(log_level)
logger = logging.getLogger("snuba.bootstrap")
import time
if kafka:
logger.debug("Using Kafka with %r", bootstrap_server)
from confluent_kafka.admin import AdminClient, NewTopic
attempts = 0
while True:
try:
logger.debug("Attempting to connect to Kafka (attempt %d)",
attempts)
client = AdminClient({
"bootstrap.servers":
",".join(bootstrap_server),
"socket.timeout.ms":
1000,
})
client.list_topics(timeout=1)
break
except Exception as e:
logger.error("Connection to Kafka failed (attempt %d)",
attempts,
exc_info=e)
attempts += 1
if attempts == 60:
raise
time.sleep(1)
topics = {}
for name in ACTIVE_DATASET_NAMES:
dataset = get_dataset(name)
for entity in dataset.get_all_entities():
writable_storage = entity.get_writable_storage()
if writable_storage:
table_writer = writable_storage.get_table_writer()
stream_loader = table_writer.get_stream_loader()
for topic_spec in stream_loader.get_all_topic_specs():
if topic_spec.topic_name in topics:
continue
logger.debug("Adding topic %s to creation list",
topic_spec.topic_name)
topics[topic_spec.topic_name] = NewTopic(
topic_spec.topic_name,
num_partitions=topic_spec.partitions_number,
replication_factor=topic_spec.replication_factor,
)
logger.debug("Initiating topic creation")
for topic, future in client.create_topics(list(topics.values()),
operation_timeout=1).items():
try:
future.result()
logger.info("Topic %s created", topic)
except Exception as e:
logger.error("Failed to create topic %s", topic, exc_info=e)
if migrate:
check_clickhouse_connections()
Runner().run_all(force=True)
def bootstrap(
*,
bootstrap_server: Sequence[str],
kafka: bool,
migrate: bool,
force: bool,
log_level: Optional[str] = None,
) -> None:
"""
Warning: Not intended to be used in production yet.
"""
if not force:
raise click.ClickException("Must use --force to run")
setup_logging(log_level)
logger = logging.getLogger("snuba.bootstrap")
import time
if kafka:
logger.debug("Using Kafka with %r", bootstrap_server)
from confluent_kafka.admin import AdminClient
override_params = {
# Same as above: override socket timeout as we expect Kafka
# to not getting ready for a while
"socket.timeout.ms": 1000,
}
if logger.getEffectiveLevel() != logging.DEBUG:
# Override rdkafka loglevel to be critical unless we are
# debugging as we expect failures when trying to connect
# (Kafka may not be up yet)
override_params["log_level"] = LOG_CRIT
attempts = 0
while True:
try:
logger.info("Attempting to connect to Kafka (attempt %d)...",
attempts)
client = AdminClient(
get_default_kafka_configuration(
bootstrap_servers=bootstrap_server,
override_params=override_params,
))
client.list_topics(timeout=1)
break
except KafkaException as err:
logger.debug("Connection to Kafka failed (attempt %d)",
attempts,
exc_info=err)
attempts += 1
if attempts == 60:
raise
time.sleep(1)
logger.info("Connected to Kafka on attempt %d", attempts)
create_topics(client, [t for t in Topic])
if migrate:
check_clickhouse_connections()
Runner().run_all(force=True)
def bootstrap(
*,
bootstrap_server: Sequence[str],
kafka: bool,
migrate: bool,
force: bool,
log_level: Optional[str] = None,
) -> None:
"""
Warning: Not intended to be used in production yet.
"""
if not force:
raise click.ClickException("Must use --force to run")
setup_logging(log_level)
logger = logging.getLogger("snuba.bootstrap")
import time
if kafka:
logger.debug("Using Kafka with %r", bootstrap_server)
from confluent_kafka.admin import AdminClient, NewTopic
override_params = {
# Same as above: override socket timeout as we expect Kafka
# to not getting ready for a while
"socket.timeout.ms": 1000,
}
if logger.getEffectiveLevel() != logging.DEBUG:
# Override rdkafka loglevel to be critical unless we are
# debugging as we expect failures when trying to connect
# (Kafka may not be up yet)
override_params["log_level"] = LOG_CRIT
attempts = 0
while True:
try:
logger.info("Attempting to connect to Kafka (attempt %d)...", attempts)
client = AdminClient(
get_default_kafka_configuration(
bootstrap_servers=bootstrap_server,
override_params=override_params,
)
)
client.list_topics(timeout=1)
break
except KafkaException as err:
logger.debug(
"Connection to Kafka failed (attempt %d)", attempts, exc_info=err
)
attempts += 1
if attempts == 60:
raise
time.sleep(1)
logger.info("Connected to Kafka on attempt %d", attempts)
topics = {}
for topic in Topic:
topic_spec = KafkaTopicSpec(topic)
logger.debug("Adding topic %s to creation list", topic_spec.topic_name)
topics[topic_spec.topic_name] = NewTopic(
topic_spec.topic_name,
num_partitions=topic_spec.partitions_number,
replication_factor=topic_spec.replication_factor,
config=topic_spec.topic_creation_config,
)
logger.info("Creating Kafka topics...")
for topic, future in client.create_topics(
list(topics.values()), operation_timeout=1
).items():
try:
future.result()
logger.info("Topic %s created", topic)
except KafkaException as err:
if err.args[0].code() != KafkaError.TOPIC_ALREADY_EXISTS:
logger.error("Failed to create topic %s", topic, exc_info=err)
if migrate:
check_clickhouse_connections()
Runner().run_all(force=True)
def migrations() -> None:
check_clickhouse_connections()