def migrate_tables(self, engine, diffs, table_names, adapters_by_table): pillow_logging.debug("[rebuild] Application migrations to tables: %s", table_names) migration_diffs = [diff for diff in diffs if diff.table_name in table_names] changes = migrate_tables(engine, migration_diffs) for table, diffs in changes.items(): adapter = adapters_by_table[table] adapter.log_table_migrate(source='pillowtop', diffs=diffs)
def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) all_adapters = [] for adapter in adapters: if getattr(adapter, 'all_adapters', None): all_adapters.extend(adapter.all_adapters) else: all_adapters.append(adapter) for adapter in all_adapters: tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): table_names = list(table_map) engine = connection_manager.get_engine(engine_id) diffs = get_table_diffs(engine, table_names, get_metadata(engine_id)) tables_to_act_on = get_tables_rebuild_migrate(diffs) for table_name in tables_to_act_on.rebuild: pillow_logging.debug("[rebuild] Rebuilding table: %s", table_name) sql_adapter = table_map[table_name] table_diffs = [diff for diff in diffs if diff.table_name == table_name] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter, table_diffs) except TableRebuildError as e: _notify_rebuild(str(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter, table_diffs) self.migrate_tables(engine, diffs, tables_to_act_on.migrate, table_map)
def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) all_adapters = [] for adapter in adapters: if getattr(adapter, 'all_adapters', None): all_adapters.extend(adapter.all_adapters) else: all_adapters.append(adapter) for adapter in all_adapters: try: tables_by_engine[adapter.engine_id][ adapter.get_table().name] = adapter except BadSpecError: _soft_assert = soft_assert( to='{}@{}'.format('jemord', 'dimagi.com')) _soft_assert( False, "Broken data source {}".format(adapter.config.get_id)) _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): table_names = list(table_map) engine = connection_manager.get_engine(engine_id) diffs = get_table_diffs(engine, table_names, get_metadata(engine_id)) tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names) for table_name in tables_to_act_on.rebuild: pillow_logging.debug("[rebuild] Rebuilding table: %s", table_name) sql_adapter = table_map[table_name] table_diffs = [ diff for diff in diffs.formatted if diff.table_name == table_name ] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter, table_diffs) except TableRebuildError as e: _notify_rebuild(six.text_type(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter, table_diffs) migration_diffs = diffs.filter(tables_to_act_on.migrate) self.migrate_tables(engine, migration_diffs.raw, tables_to_act_on.migrate, table_map)
def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) for adapter in adapters: try: tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter except BadSpecError: _soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com')) _soft_assert(False, "Broken data source {}".format(adapter.config.get_id)) _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): table_names = list(table_map) engine = connection_manager.get_engine(engine_id) # Temporary measure necessary during the process of renaming tables # - Configs point to new tables which may just be views and not real tables # - The global metadata contains references to the new table names legacy_tables = {} table_names_for_diff = [] diff_metadata = sqlalchemy.MetaData() with engine.begin() as connection: for table_name in table_names: sql_adapter = table_map[table_name] legacy_table_name = get_legacy_table_name( sql_adapter.config.domain, sql_adapter.config.table_id ) if not table_exists(connection, table_name) and table_exists(connection, legacy_table_name): legacy_tables[legacy_table_name] = table_name pillow_logging.debug("[rebuild] Using legacy table: %s", legacy_table_name) # popultate metadata with the table schema get_indicator_table( sql_adapter.config, metadata=diff_metadata, override_table_name=legacy_table_name ) table_names_for_diff.append(legacy_table_name) else: # popultate metadata with the table schema get_indicator_table(sql_adapter.config, metadata=diff_metadata) table_names_for_diff.append(table_name) diffs = get_table_diffs(engine, table_names_for_diff, diff_metadata) tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names_for_diff) for real_table_name in tables_to_act_on.rebuild: table_name = legacy_tables.get(real_table_name, real_table_name) pillow_logging.debug("[rebuild] Rebuilding table: %s (%s)", real_table_name, table_name) sql_adapter = table_map[table_name] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter) except TableRebuildError as e: _notify_rebuild(six.text_type(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter) pillow_logging.debug("[rebuild] Application migrations to tables: %s", tables_to_act_on.migrate) migrate_tables(engine, diffs.raw, tables_to_act_on.migrate)
def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) all_adapters = [] for adapter in adapters: if getattr(adapter, 'all_adapters', None): all_adapters.extend(adapter.all_adapters) else: all_adapters.append(adapter) for adapter in all_adapters: try: tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter except BadSpecError: _soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com')) _soft_assert(False, "Broken data source {}".format(adapter.config.get_id)) _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): table_names = list(table_map) engine = connection_manager.get_engine(engine_id) diffs = get_table_diffs(engine, table_names, get_metadata(engine_id)) tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names) for table_name in tables_to_act_on.rebuild: pillow_logging.debug("[rebuild] Rebuilding table: %s", table_name) sql_adapter = table_map[table_name] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter) except TableRebuildError as e: _notify_rebuild(six.text_type(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter) self.migrate_tables(engine, diffs.raw, tables_to_act_on.migrate, table_map)
def migrate_tables(self, engine, diffs, table_names, adapters_by_table): pillow_logging.debug("[rebuild] Application migrations to tables: %s", table_names) changes = migrate_tables(engine, diffs, table_names) for table, diffs in changes.items(): adapter = adapters_by_table[table] adapter.log_table_migrate(source='pillowtop', diffs=diffs)