def clickhouse_auto_sync():
"""
Plans syncing models
:return: None
"""
# Import all model modules
for app in settings.INSTALLED_APPS:
import_submodules("%s.%s" % (app, config.MODELS_MODULE))
# Start
for cls in get_subclasses(ClickHouseModel, recursive=True):
if cls.need_sync():
# Даже если синхронизация вдруг не выполнится, не страшно, что мы установили период синхронизации
# Она выполнится следующей таской через интервал.
sync_clickhouse_converter.delay(cls)
def get_async_migration_definition(migration_name: str) -> AsyncMigrationDefinition:
if TEST:
test_migrations = import_submodules(ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH)
if migration_name in test_migrations:
return test_migrations[migration_name].Migration()
return ALL_ASYNC_MIGRATIONS[migration_name]
def migrate_app(app_label, db_alias, up_to=9999, database=None):
# type: (str, str, int, Optional[Database]) -> None
"""
Migrates given django app
:param app_label: App label to migrate
:param db_alias: Database alias to migrate
:param up_to: Migration number to migrate to
:param database: Sometimes I want to pass db object directly for testing purposes
:return: None
"""
database = database or connections[db_alias]
migrations_package = "%s.%s" % (app_label, config.MIGRATIONS_PACKAGE)
if module_exists(migrations_package):
applied_migrations = database._get_applied_migrations(
migrations_package)
modules = import_submodules(migrations_package)
unapplied_migrations = set(modules.keys()) - applied_migrations
for name in sorted(unapplied_migrations):
migration = modules[name].Migration()
migration.apply(db_alias, database=database)
database.insert([
MigrationHistory(package_name=migrations_package,
module_name=name,
applied=datetime.date.today())
])
if int(name[:4]) >= up_to:
break
def get_migrations(self, database, upto):
modules = import_submodules(MIGRATIONS_PACKAGE_NAME)
applied_migrations = self.get_applied_migrations(database)
unapplied_migrations = set(modules.keys()) - applied_migrations
for migration_name in sorted(unapplied_migrations):
yield migration_name, modules[migration_name].operations
if int(migration_name[:4]) >= upto:
break
def get_migrations(self, database, upto):
applied_migrations = database._get_applied_migrations(
MIGRATIONS_PACKAGE_NAME, replicated=CLICKHOUSE_REPLICATION)
modules = import_submodules(MIGRATIONS_PACKAGE_NAME)
unapplied_migrations = set(modules.keys()) - applied_migrations
for migration_name in sorted(unapplied_migrations):
yield migration_name, modules[migration_name].operations
if int(migration_name[:4]) >= upto:
break
def clickhouse_auto_sync() -> None:
"""
Plans syncing models
:return: None
"""
# Import all model modules
for app in django_apps.get_app_configs():
package_name = "%s.%s" % (app.name, config.MODELS_MODULE)
try:
module = importlib.import_module(package_name)
if hasattr(module, '__path__'):
import_submodules(package_name)
except ImportError:
pass
for cls in get_subclasses(ClickHouseModel, recursive=True):
if cls.need_sync():
# I pass class as a string in order to make it JSON serializable
cls_path = "%s.%s" % (cls.__module__, cls.__name__)
sync_clickhouse_model.delay(cls_path)
def clickhouse_auto_sync():
"""
Plans syncing models
:return: None
"""
# Import all model modules
for app in settings.INSTALLED_APPS:
package_name = "%s.%s" % (app, config.MODELS_MODULE)
try:
module = importlib.import_module(package_name)
if hasattr(module, '__path__'):
import_submodules(package_name)
except ImportError:
pass
# Start
for cls in get_subclasses(ClickHouseModel, recursive=True):
if cls.need_sync():
# Даже если синхронизация вдруг не выполнится, не страшно, что мы установили период синхронизации
# Она выполнится следующей таской через интервал.
sync_clickhouse_model.delay(cls)
def migrate_app(app_label: str,
db_alias: str,
up_to: int = 9999,
database: Optional[Database] = None) -> None:
"""
Migrates given django app
:param app_label: App label to migrate
:param db_alias: Database alias to migrate
:param up_to: Migration number to migrate to
:param database: Sometimes I want to pass db object directly for testing purposes
:return: None
"""
# Can't migrate such connection, just skip it
if config.DATABASES[db_alias].get('readonly', False):
return
# Ignore force not migrated databases
if not config.DATABASES[db_alias].get('migrate', True):
return
migrations_package = "%s.%s" % (app_label, config.MIGRATIONS_PACKAGE)
if module_exists(migrations_package):
database = database or connections[db_alias]
migration_history_model = lazy_class_import(
config.MIGRATION_HISTORY_MODEL)
applied_migrations = migration_history_model.get_applied_migrations(
db_alias, migrations_package)
modules = import_submodules(migrations_package)
unapplied_migrations = set(modules.keys()) - applied_migrations
for name in sorted(unapplied_migrations):
print(
'Applying ClickHouse migration %s for app %s in database %s' %
(name, app_label, db_alias))
migration = modules[name].Migration()
migration.apply(db_alias, database=database)
migration_history_model.set_migration_applied(
db_alias, migrations_package, name)
if int(name[:4]) >= up_to:
break
def test_get_async_migration_definition(self):
from posthog.async_migrations.examples.example import example_fn, example_rollback_fn
modules = import_submodules(ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH)
example_migration = modules["example"].Migration()
self.assertTrue(isinstance(example_migration,
AsyncMigrationDefinition))
self.assertTrue(
isinstance(example_migration.operations[0],
AsyncMigrationOperation))
self.assertEqual(example_migration.description,
"An example async migration.")
self.assertEqual(example_migration.posthog_min_version, "1.29.0")
self.assertEqual(example_migration.posthog_max_version, "1.30.0")
self.assertEqual(example_migration.operations[-1].fn, example_fn)
self.assertEqual(example_migration.operations[-1].rollback_fn,
example_rollback_fn)
self.assertTrue(
isinstance(example_migration.service_version_requirements[0],
ServiceVersionRequirement))
def migrate_app(app_label: str,
db_alias: str,
up_to: int = 9999,
database: Optional[Database] = None,
verbosity: int = 1) -> bool:
"""
Migrates given django app
:param app_label: App label to migrate
:param db_alias: Database alias to migrate
:param up_to: Migration number to migrate to
:param database: Sometimes I want to pass db object directly for testing purposes
:param verbosity: 0-4, уровень verbosity вывода
:return: True if any migration has been applied
"""
# Can't migrate such connection, just skip it
if config.DATABASES[db_alias].get('readonly', False):
if verbosity > 1:
print('Skipping database "%s": marked as readonly' % db_alias)
return False
# Ignore force not migrated databases
if not config.DATABASES[db_alias].get('migrate', True):
if verbosity > 1:
print(
'Skipping database "%s": migrations are restricted in configuration'
% db_alias)
return False
migrations_package = "%s.%s" % (app_label, config.MIGRATIONS_PACKAGE)
if not module_exists(migrations_package):
if verbosity > 1:
print(
'Skipping migrations for app "%s": no migration_package "%s"' %
(app_label, migrations_package))
return False
database = database or connections[db_alias]
migration_history_model = lazy_class_import(config.MIGRATION_HISTORY_MODEL)
applied_migrations = migration_history_model.get_applied_migrations(
db_alias, migrations_package)
modules = import_submodules(migrations_package)
unapplied_migrations = set(modules.keys()) - applied_migrations
any_applied = False
for name in sorted(unapplied_migrations):
if int(name[:4]) > up_to:
break
if verbosity > 0:
print(
'Applying ClickHouse migration %s for app %s in database %s' %
(name, app_label, db_alias))
migration = modules[name].Migration()
migration.apply(db_alias, database=database)
migration_history_model.set_migration_applied(db_alias,
migrations_package, name)
any_applied = True
if not any_applied:
if verbosity > 1:
print('No migrations to apply for app "%s" does not exist' %
app_label)
return False
return True
from posthog.version import VERSION
ALL_ASYNC_MIGRATIONS: Dict[str, AsyncMigrationDefinition] = {}
ASYNC_MIGRATION_TO_DEPENDENCY: Dict[str, Optional[str]] = {}
# inverted mapping of ASYNC_MIGRATION_TO_DEPENDENCY
DEPENDENCY_TO_ASYNC_MIGRATION: Dict[Optional[str], str] = {}
POSTHOG_VERSION = Version(VERSION)
ASYNC_MIGRATIONS_MODULE_PATH = "posthog.async_migrations.migrations"
ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH = "posthog.async_migrations.examples"
all_migrations = import_submodules(ASYNC_MIGRATIONS_MODULE_PATH)
for name, module in all_migrations.items():
ALL_ASYNC_MIGRATIONS[name] = module.Migration()
def setup_async_migrations(ignore_posthog_version: bool = False):
"""
Execute the necessary setup for async migrations to work:
1. Import all the migration definitions
2. Create a database record for each
3. Check if all migrations necessary for this PostHog version have completed (else don't start)
4. Populate a dependencies map and in-memory record of migration definitions
"""
applied_migrations = set(instance.name for instance in get_all_completed_async_migrations())
