def clickhouse_auto_sync(): """ Plans syncing models :return: None """ # Import all model modules for app in settings.INSTALLED_APPS: import_submodules("%s.%s" % (app, config.MODELS_MODULE)) # Start for cls in get_subclasses(ClickHouseModel, recursive=True): if cls.need_sync(): # Даже если синхронизация вдруг не выполнится, не страшно, что мы установили период синхронизации # Она выполнится следующей таской через интервал. sync_clickhouse_converter.delay(cls)
def get_async_migration_definition(migration_name: str) -> AsyncMigrationDefinition: if TEST: test_migrations = import_submodules(ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH) if migration_name in test_migrations: return test_migrations[migration_name].Migration() return ALL_ASYNC_MIGRATIONS[migration_name]
def migrate_app(app_label, db_alias, up_to=9999, database=None): # type: (str, str, int, Optional[Database]) -> None """ Migrates given django app :param app_label: App label to migrate :param db_alias: Database alias to migrate :param up_to: Migration number to migrate to :param database: Sometimes I want to pass db object directly for testing purposes :return: None """ database = database or connections[db_alias] migrations_package = "%s.%s" % (app_label, config.MIGRATIONS_PACKAGE) if module_exists(migrations_package): applied_migrations = database._get_applied_migrations( migrations_package) modules = import_submodules(migrations_package) unapplied_migrations = set(modules.keys()) - applied_migrations for name in sorted(unapplied_migrations): migration = modules[name].Migration() migration.apply(db_alias, database=database) database.insert([ MigrationHistory(package_name=migrations_package, module_name=name, applied=datetime.date.today()) ]) if int(name[:4]) >= up_to: break
def get_migrations(self, database, upto): modules = import_submodules(MIGRATIONS_PACKAGE_NAME) applied_migrations = self.get_applied_migrations(database) unapplied_migrations = set(modules.keys()) - applied_migrations for migration_name in sorted(unapplied_migrations): yield migration_name, modules[migration_name].operations if int(migration_name[:4]) >= upto: break
def get_migrations(self, database, upto): applied_migrations = database._get_applied_migrations( MIGRATIONS_PACKAGE_NAME, replicated=CLICKHOUSE_REPLICATION) modules = import_submodules(MIGRATIONS_PACKAGE_NAME) unapplied_migrations = set(modules.keys()) - applied_migrations for migration_name in sorted(unapplied_migrations): yield migration_name, modules[migration_name].operations if int(migration_name[:4]) >= upto: break
def clickhouse_auto_sync() -> None: """ Plans syncing models :return: None """ # Import all model modules for app in django_apps.get_app_configs(): package_name = "%s.%s" % (app.name, config.MODELS_MODULE) try: module = importlib.import_module(package_name) if hasattr(module, '__path__'): import_submodules(package_name) except ImportError: pass for cls in get_subclasses(ClickHouseModel, recursive=True): if cls.need_sync(): # I pass class as a string in order to make it JSON serializable cls_path = "%s.%s" % (cls.__module__, cls.__name__) sync_clickhouse_model.delay(cls_path)
def clickhouse_auto_sync(): """ Plans syncing models :return: None """ # Import all model modules for app in settings.INSTALLED_APPS: package_name = "%s.%s" % (app, config.MODELS_MODULE) try: module = importlib.import_module(package_name) if hasattr(module, '__path__'): import_submodules(package_name) except ImportError: pass # Start for cls in get_subclasses(ClickHouseModel, recursive=True): if cls.need_sync(): # Даже если синхронизация вдруг не выполнится, не страшно, что мы установили период синхронизации # Она выполнится следующей таской через интервал. sync_clickhouse_model.delay(cls)
def migrate_app(app_label: str, db_alias: str, up_to: int = 9999, database: Optional[Database] = None) -> None: """ Migrates given django app :param app_label: App label to migrate :param db_alias: Database alias to migrate :param up_to: Migration number to migrate to :param database: Sometimes I want to pass db object directly for testing purposes :return: None """ # Can't migrate such connection, just skip it if config.DATABASES[db_alias].get('readonly', False): return # Ignore force not migrated databases if not config.DATABASES[db_alias].get('migrate', True): return migrations_package = "%s.%s" % (app_label, config.MIGRATIONS_PACKAGE) if module_exists(migrations_package): database = database or connections[db_alias] migration_history_model = lazy_class_import( config.MIGRATION_HISTORY_MODEL) applied_migrations = migration_history_model.get_applied_migrations( db_alias, migrations_package) modules = import_submodules(migrations_package) unapplied_migrations = set(modules.keys()) - applied_migrations for name in sorted(unapplied_migrations): print( 'Applying ClickHouse migration %s for app %s in database %s' % (name, app_label, db_alias)) migration = modules[name].Migration() migration.apply(db_alias, database=database) migration_history_model.set_migration_applied( db_alias, migrations_package, name) if int(name[:4]) >= up_to: break
def test_get_async_migration_definition(self): from posthog.async_migrations.examples.example import example_fn, example_rollback_fn modules = import_submodules(ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH) example_migration = modules["example"].Migration() self.assertTrue(isinstance(example_migration, AsyncMigrationDefinition)) self.assertTrue( isinstance(example_migration.operations[0], AsyncMigrationOperation)) self.assertEqual(example_migration.description, "An example async migration.") self.assertEqual(example_migration.posthog_min_version, "1.29.0") self.assertEqual(example_migration.posthog_max_version, "1.30.0") self.assertEqual(example_migration.operations[-1].fn, example_fn) self.assertEqual(example_migration.operations[-1].rollback_fn, example_rollback_fn) self.assertTrue( isinstance(example_migration.service_version_requirements[0], ServiceVersionRequirement))
def migrate_app(app_label: str, db_alias: str, up_to: int = 9999, database: Optional[Database] = None, verbosity: int = 1) -> bool: """ Migrates given django app :param app_label: App label to migrate :param db_alias: Database alias to migrate :param up_to: Migration number to migrate to :param database: Sometimes I want to pass db object directly for testing purposes :param verbosity: 0-4, уровень verbosity вывода :return: True if any migration has been applied """ # Can't migrate such connection, just skip it if config.DATABASES[db_alias].get('readonly', False): if verbosity > 1: print('Skipping database "%s": marked as readonly' % db_alias) return False # Ignore force not migrated databases if not config.DATABASES[db_alias].get('migrate', True): if verbosity > 1: print( 'Skipping database "%s": migrations are restricted in configuration' % db_alias) return False migrations_package = "%s.%s" % (app_label, config.MIGRATIONS_PACKAGE) if not module_exists(migrations_package): if verbosity > 1: print( 'Skipping migrations for app "%s": no migration_package "%s"' % (app_label, migrations_package)) return False database = database or connections[db_alias] migration_history_model = lazy_class_import(config.MIGRATION_HISTORY_MODEL) applied_migrations = migration_history_model.get_applied_migrations( db_alias, migrations_package) modules = import_submodules(migrations_package) unapplied_migrations = set(modules.keys()) - applied_migrations any_applied = False for name in sorted(unapplied_migrations): if int(name[:4]) > up_to: break if verbosity > 0: print( 'Applying ClickHouse migration %s for app %s in database %s' % (name, app_label, db_alias)) migration = modules[name].Migration() migration.apply(db_alias, database=database) migration_history_model.set_migration_applied(db_alias, migrations_package, name) any_applied = True if not any_applied: if verbosity > 1: print('No migrations to apply for app "%s" does not exist' % app_label) return False return True
from posthog.version import VERSION ALL_ASYNC_MIGRATIONS: Dict[str, AsyncMigrationDefinition] = {} ASYNC_MIGRATION_TO_DEPENDENCY: Dict[str, Optional[str]] = {} # inverted mapping of ASYNC_MIGRATION_TO_DEPENDENCY DEPENDENCY_TO_ASYNC_MIGRATION: Dict[Optional[str], str] = {} POSTHOG_VERSION = Version(VERSION) ASYNC_MIGRATIONS_MODULE_PATH = "posthog.async_migrations.migrations" ASYNC_MIGRATIONS_EXAMPLE_MODULE_PATH = "posthog.async_migrations.examples" all_migrations = import_submodules(ASYNC_MIGRATIONS_MODULE_PATH) for name, module in all_migrations.items(): ALL_ASYNC_MIGRATIONS[name] = module.Migration() def setup_async_migrations(ignore_posthog_version: bool = False): """ Execute the necessary setup for async migrations to work: 1. Import all the migration definitions 2. Create a database record for each 3. Check if all migrations necessary for this PostHog version have completed (else don't start) 4. Populate a dependencies map and in-memory record of migration definitions """ applied_migrations = set(instance.name for instance in get_all_completed_async_migrations())