def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) all_adapters = [] for adapter in adapters: if getattr(adapter, 'all_adapters', None): all_adapters.extend(adapter.all_adapters) else: all_adapters.append(adapter) for adapter in all_adapters: tables_by_engine[adapter.engine_id][ adapter.get_table().name] = adapter _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): table_names = list(table_map) engine = connection_manager.get_engine(engine_id) diffs = get_table_diffs(engine, table_names, get_metadata(engine_id)) tables_to_act_on = get_tables_rebuild_migrate(diffs) for table_name in tables_to_act_on.rebuild: sql_adapter = table_map[table_name] pillow_logging.info( "[rebuild] Rebuilding table: %s, from config %s at rev %s", table_name, sql_adapter.config._id, sql_adapter.config._rev) pillow_logging.info("[rebuild] Using config: %r", sql_adapter.config) pillow_logging.info("[rebuild] sqlalchemy metadata: %r", get_metadata(engine_id).tables[table_name]) pillow_logging.info("[rebuild] sqlalchemy table: %r", sql_adapter.get_table()) table_diffs = [ diff for diff in diffs if diff.table_name == table_name ] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter, table_diffs) except TableRebuildError as e: _notify_rebuild(str(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter, table_diffs) self.migrate_tables(engine, diffs, tables_to_act_on.migrate, table_map)
def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) for adapter in adapters: sql_adapter = get_indicator_adapter(adapter.config) try: tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter except BadSpecError: _soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com')) _soft_assert(False, "Broken data source {}".format(adapter.config.get_id)) _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): table_names = list(table_map) engine = connection_manager.get_engine(engine_id) diffs = get_table_diffs(engine, table_names, get_metadata(engine_id)) tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names) for table_name in tables_to_act_on.rebuild: sql_adapter = table_map[table_name] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter) except TableRebuildError as e: _notify_rebuild(six.text_type(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter) migrate_tables(engine, diffs.raw, tables_to_act_on.migrate)
def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) all_adapters = [] for adapter in adapters: if getattr(adapter, 'all_adapters', None): all_adapters.extend(adapter.all_adapters) else: all_adapters.append(adapter) for adapter in all_adapters: try: tables_by_engine[adapter.engine_id][ adapter.get_table().name] = adapter except BadSpecError: _soft_assert = soft_assert( to='{}@{}'.format('jemord', 'dimagi.com')) _soft_assert( False, "Broken data source {}".format(adapter.config.get_id)) _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): table_names = list(table_map) engine = connection_manager.get_engine(engine_id) diffs = get_table_diffs(engine, table_names, get_metadata(engine_id)) tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names) for table_name in tables_to_act_on.rebuild: pillow_logging.debug("[rebuild] Rebuilding table: %s", table_name) sql_adapter = table_map[table_name] table_diffs = [ diff for diff in diffs.formatted if diff.table_name == table_name ] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter, table_diffs) except TableRebuildError as e: _notify_rebuild(six.text_type(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter, table_diffs) migration_diffs = diffs.filter(tables_to_act_on.migrate) self.migrate_tables(engine, migration_diffs.raw, tables_to_act_on.migrate, table_map)
def migrate_tables(self, engine, diffs, table_names, adapters_by_table): migration_diffs = [diff for diff in diffs if diff.table_name in table_names] for table in table_names: adapter = adapters_by_table[table] pillow_logging.info("[rebuild] Using config: %r", adapter.config) pillow_logging.info("[rebuild] sqlalchemy metadata: %r", get_metadata(adapter.engine_id).tables[table]) pillow_logging.info("[rebuild] sqlalchemy table: %r", adapter.get_table()) changes = migrate_tables(engine, migration_diffs) for table, diffs in changes.items(): adapter = adapters_by_table[table] pillow_logging.info( "[rebuild] Migrating table: %s, from config %s at rev %s", table, adapter.config._id, adapter.config._rev ) adapter.log_table_migrate(source='pillowtop', diffs=diffs)
def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) all_adapters = [] for adapter in adapters: if getattr(adapter, 'all_adapters', None): all_adapters.extend(adapter.all_adapters) else: all_adapters.append(adapter) for adapter in all_adapters: try: tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter except BadSpecError: _soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com')) _soft_assert(False, "Broken data source {}".format(adapter.config.get_id)) _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): table_names = list(table_map) engine = connection_manager.get_engine(engine_id) diffs = get_table_diffs(engine, table_names, get_metadata(engine_id)) tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names) for table_name in tables_to_act_on.rebuild: pillow_logging.debug("[rebuild] Rebuilding table: %s", table_name) sql_adapter = table_map[table_name] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter) except TableRebuildError as e: _notify_rebuild(six.text_type(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter) self.migrate_tables(engine, diffs.raw, tables_to_act_on.migrate, table_map)