def trigger(self, request, **kwargs):
if len(get_all_running_async_migrations()
) >= MAX_CONCURRENT_ASYNC_MIGRATIONS:
return response.Response(
{
"success":
False,
"error":
f"No more than {MAX_CONCURRENT_ASYNC_MIGRATIONS} async migration can run at once.",
},
status=400,
)
migration_instance = self.get_object()
if not is_posthog_version_compatible(
migration_instance.posthog_min_version,
migration_instance.posthog_max_version):
return response.Response(
{
"success":
False,
"error":
f"Can't run migration. Minimum PostHog version: {migration_instance.posthog_min_version}. Maximum PostHog version: {migration_instance.posthog_max_version}",
},
status=400,
)
migration_instance.status = MigrationStatus.Starting
migration_instance.save()
trigger_migration(migration_instance)
return response.Response({"success": True}, status=200)
def run_next_migration(candidate: str):
migration_instance = AsyncMigration.objects.get(name=candidate)
migration_in_range = is_posthog_version_compatible(
migration_instance.posthog_min_version,
migration_instance.posthog_max_version)
dependency_ok, _ = is_migration_dependency_fulfilled(candidate)
if dependency_ok and migration_in_range and migration_instance.status == MigrationStatus.NotStarted:
trigger_migration(migration_instance)
def resume(self, request, **kwargs):
migration_instance = self.get_object()
if migration_instance.status != MigrationStatus.Errored:
return response.Response(
{
"success":
False,
"error":
"Can't resume a migration that isn't in errored state",
},
status=400,
)
migration_instance.status = MigrationStatus.Running
migration_instance.save()
trigger_migration(migration_instance, fresh_start=False)
return response.Response({"success": True}, status=200)
def check_async_migration_health() -> None:
from posthog.models.async_migration import AsyncMigration, MigrationStatus
try:
migration_instance: AsyncMigration = AsyncMigration.objects.get(status=MigrationStatus.Running)
except AsyncMigration.DoesNotExist:
return
migration_task_celery_state = AsyncResult(migration_instance.celery_task_id).state
# we only care about "supposedly running" tasks here
# failures and successes are handled elsewhere
# pending means we haven't picked up the task yet
# retry is not possible as max_retries == 0
if migration_task_celery_state != states.STARTED:
return
inspector = app.control.inspect()
active_tasks_per_node = inspector.active()
active_task_ids = []
if active_tasks_per_node:
for _, tasks in active_tasks_per_node.items():
active_task_ids += [task["id"] for task in tasks]
# the worker crashed - this is how we find out and process the error
if migration_instance.celery_task_id not in active_task_ids:
if getattr(config, "ASYNC_MIGRATIONS_AUTO_CONTINUE"):
trigger_migration(migration_instance, fresh_start=False)
else:
process_error(migration_instance, "Celery worker crashed while running migration.")
return
ok, error = run_migration_healthcheck(migration_instance)
if not ok:
force_stop_migration(migration_instance, f"Healthcheck failed with error: {error}")
return
update_migration_progress(migration_instance)
def test_trigger_migration(self, mock_run_async_migration):
sm = create_async_migration()
trigger_migration(sm)
mock_run_async_migration.assert_called_once()