def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) for adapter in adapters: sql_adapter = get_indicator_adapter(adapter.config) try: tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter except BadSpecError: _soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com')) _soft_assert(False, "Broken data source {}".format(adapter.config.get_id)) _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) table_names = list(table_map) with engine.begin() as connection: migration_context = get_migration_context(connection, table_names) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) tables_to_rebuild = get_tables_to_rebuild(diffs, table_names) for table_name in tables_to_rebuild: sql_adapter = table_map[table_name] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter) except TableRebuildError as e: _notify_rebuild(six.text_type(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter) tables_to_migrate = get_tables_to_migrate(diffs, table_names) tables_to_migrate -= tables_to_rebuild migrate_tables(engine, raw_diffs, tables_to_migrate)
def _rebuild_sql_tables(self, adapters): tables_by_engine = defaultdict(dict) for adapter in adapters: tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter _assert = soft_assert(to='@'.join(['czue', 'dimagi.com'])) _notify_cory = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context(connection, table_map.keys()) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: sql_adapter = table_map[table_name] if not sql_adapter.config.is_static: try: rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id) self.rebuild_table(sql_adapter) except TableRebuildError, e: _notify_cory(unicode(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter)
def _test_diffs(self, metadata, expected_diffs, table_names=None): migration_context = get_migration_context( self.engine, table_names or [self.table_name, 'new_table']) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) self.assertEqual(set(diffs), expected_diffs) return diffs
def test_table_filter(self): migration_context = get_migration_context(self.engine, [self.table_name]) sqlalchemy.Table('new_table', self.metadata) raw_diffs = compare_metadata(migration_context, self.metadata) _, diffs = reformat_alembic_diffs(raw_diffs) self.assertEqual(0, len(diffs))
def _rebuild_sql_tables(self, adapters): # todo move this code to sql adapter rebuild_if_necessary tables_by_engine = defaultdict(dict) for adapter in adapters: sql_adapter = get_indicator_adapter(adapter.config) tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter _assert = soft_assert(to='@'.join(['czue', 'dimagi.com'])) _notify_cory = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context(connection, table_map.keys()) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: sql_adapter = table_map[table_name] if not sql_adapter.config.is_static: try: rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id) self.rebuild_table(sql_adapter) except TableRebuildError, e: _notify_cory(unicode(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter)
def handle(self, **options): data_sources = list(DataSourceConfiguration.all()) data_sources.extend(list(StaticDataSourceConfiguration.all())) engine_ids = self._get_engine_ids(data_sources, options.get('engine_id')) tables_to_remove_by_engine = defaultdict(list) for engine_id in engine_ids: engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context( connection, include_object=_include_object) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) tables_to_remove_by_engine[engine_id] = [ diff.table_name for diff in diffs if diff.type == 'remove_table' ] for engine_id, tablenames in tables_to_remove_by_engine.items(): engine = connection_manager.get_engine(engine_id) for tablename in tablenames: with engine.begin() as connection: try: result = connection.execute( 'SELECT COUNT(*), MAX(inserted_at) FROM "{tablename}"' .format(tablename=tablename)) except Exception: print(tablename, "no inserted_at column, probably not UCR") else: print(tablename, result.fetchone())
def _rebuild_sql_tables(self, adapters): # todo move this code to sql adapter rebuild_if_necessary tables_by_engine = defaultdict(dict) for adapter in adapters: sql_adapter = get_indicator_adapter(adapter.config) tables_by_engine[sql_adapter.engine_id][ sql_adapter.get_table().name] = sql_adapter _assert = soft_assert(notify_admins=True) _notify_rebuild = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) table_names = list(table_map) with engine.begin() as connection: migration_context = get_migration_context( connection, table_names) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) tables_to_rebuild = get_tables_to_rebuild(diffs, table_names) for table_name in tables_to_rebuild: sql_adapter = table_map[table_name] if not sql_adapter.config.is_static: try: self.rebuild_table(sql_adapter) except TableRebuildError as e: _notify_rebuild(six.text_type(e), sql_adapter.config.to_json()) else: self.rebuild_table(sql_adapter) tables_to_migrate = get_tables_to_migrate(diffs, table_names) tables_to_migrate -= tables_to_rebuild migrate_tables(engine, raw_diffs, tables_to_migrate)
def handle(self, **options): data_sources = list(DataSourceConfiguration.all()) data_sources.extend(list(StaticDataSourceConfiguration.all())) engine_ids = self._get_engine_ids(data_sources, options.get('engine_id')) tables_to_remove_by_engine = defaultdict(list) for engine_id in engine_ids: engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context(connection, include_object=_include_object) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) tables_to_remove_by_engine[engine_id] = [ diff.table_name for diff in diffs if diff.type == 'remove_table' ] for engine_id, tablenames in tables_to_remove_by_engine.items(): engine = connection_manager.get_engine(engine_id) for tablename in tablenames: with engine.begin() as connection: try: result = connection.execute( 'SELECT COUNT(*), MAX(inserted_at) FROM "{tablename}"'.format(tablename=tablename) ) except Exception: print(tablename, "no inserted_at column, probably not UCR") else: print(tablename, result.fetchone())
def get_table_diffs(engine, table_names, metadata): with engine.begin() as connection: migration_context = get_migration_context(connection, table_names) raw_diffs = compare_metadata(migration_context, metadata) return [ diff for diff in reformat_alembic_diffs(raw_diffs) if diff.table_name in table_names ]
def rebuild_tables_if_necessary(self): table_map = {t.get_table().name: t for t in self.tables} engine = self.get_sql_engine() with engine.begin() as connection: migration_context = get_migration_context(connection, table_map.keys()) diffs = compare_metadata(migration_context, metadata) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: table = table_map[table_name] self.rebuild_table(table)
def add_columns(engine, raw_diffs, table_names): with engine.begin() as conn: ctx = get_migration_context(conn, table_names) op = Operations(ctx) columns = _get_columns_to_add(raw_diffs, table_names) for col in columns: table_name = col.table.name # the column has a reference to a table definition that already # has the column defined, so remove that and add the column col.table = None op.add_column(table_name, col)
def add_columns(engine, diffs): with engine.begin() as conn: ctx = get_migration_context(conn) op = Operations(ctx) col_diffs = _filter_diffs(diffs, [DiffTypes.ADD_NULLABLE_COLUMN]) for diff in col_diffs: col = diff.column table_name = col.table.name # the column has a reference to a table definition that already # has the column defined, so remove that and add the column col.table = None op.add_column(table_name, col) return col_diffs
def rebuild_tables_if_necessary(self): tables_by_engine = defaultdict(dict) for adapter in self.table_adapters: tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context(connection, table_map.keys()) diffs = compare_metadata(migration_context, metadata) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: table = table_map[table_name] try: self.rebuild_table(table) except TableRebuildError, e: notify_error(unicode(e))
def add_columns(engine, raw_diffs, table_names): changes = defaultdict(list) with engine.begin() as conn: ctx = get_migration_context(conn, table_names) op = Operations(ctx) columns = _get_columns_to_add(raw_diffs, table_names) for col in columns: table_name = col.table.name # the column has a reference to a table definition that already # has the column defined, so remove that and add the column col.table = None changes[table_name].append({ 'type': DiffTypes.ADD_COLUMN, 'item_name': col.name }) op.add_column(table_name, col) return dict(changes)
def add_columns(engine, raw_diffs, table_names): changes = defaultdict(list) with engine.begin() as conn: ctx = get_migration_context(conn, table_names) op = Operations(ctx) columns = _get_columns_to_add(raw_diffs, table_names) for col in columns: table_name = col.table.name # the column has a reference to a table definition that already # has the column defined, so remove that and add the column col.table = None changes[table_name].append({ 'type': DiffTypes.ADD_COLUMN, 'value': col.name }) op.add_column(table_name, col) return dict(changes)
def rebuild_tables_if_necessary(self): tables_by_engine = defaultdict(dict) for adapter in self.table_adapters: tables_by_engine[adapter.engine_id][ adapter.get_table().name] = adapter for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context( connection, table_map.keys()) diffs = compare_metadata(migration_context, metadata) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: table = table_map[table_name] try: self.rebuild_table(table) except TableRebuildError, e: notify_error(unicode(e))
def rebuild_tables_if_necessary(self): tables_by_engine = defaultdict(dict) for adapter in self.table_adapters: tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter _assert = soft_assert(to='@'.join(['czue', 'dimagi.com'])) _notify_cory = lambda msg, obj: _assert(False, msg, obj) for engine_id, table_map in tables_by_engine.items(): engine = connection_manager.get_engine(engine_id) with engine.begin() as connection: migration_context = get_migration_context(connection, table_map.keys()) diffs = compare_metadata(migration_context, metadata) tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys()) for table_name in tables_to_rebuild: sql_adapter = table_map[table_name] if not is_static(sql_adapter.config._id): try: rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id) self.rebuild_table(sql_adapter) except TableRebuildError, e: _notify_cory(unicode(e), sql_adapter.config.to_json()) else: # note: this fancy logging can be removed as soon as we get to the # bottom of http://manage.dimagi.com/default.asp?211297 # if no signs of it popping back up by april 2016, should remove this rev_after_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id) _notify_cory( u'rebuilt table {} ({}) because {}. rev before: {}, rev after: {}'.format( table_name, u'{} [{}]'.format(sql_adapter.config.display_name, sql_adapter.config._id), diffs, rev_before_rebuild, rev_after_rebuild, ), sql_adapter.config.to_json(), ) else: self.rebuild_table(sql_adapter)
def _test_diffs(self, metadata, expected_diffs, table_names=None): migration_context = get_migration_context(self.engine, table_names or [self.table_name, 'new_table']) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) self.assertEqual(set(diffs), expected_diffs)
def test_table_filter(self): migration_context = get_migration_context(self.engine, [self.table_name]) sqlalchemy.Table('new_table', self.metadata) raw_diffs = compare_metadata(migration_context, self.metadata) diffs = reformat_alembic_diffs(raw_diffs) self.assertEqual(0, len(diffs))
def get_table_diffs(engine, table_names, metadata): with engine.begin() as connection: migration_context = get_migration_context(connection, table_names) raw_diffs = compare_metadata(migration_context, metadata) diffs = reformat_alembic_diffs(raw_diffs) return TableDiffs(raw=raw_diffs, formatted=diffs)