Exemplo n.º 1
0
    def _rebuild_sql_tables(self, adapters):
        tables_by_engine = defaultdict(dict)
        for adapter in adapters:
            sql_adapter = get_indicator_adapter(adapter.config)
            try:
                tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter
            except BadSpecError:
                _soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com'))
                _soft_assert(False, "Broken data source {}".format(adapter.config.get_id))

        _assert = soft_assert(notify_admins=True)
        _notify_rebuild = lambda msg, obj: _assert(False, msg, obj)

        for engine_id, table_map in tables_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            table_names = list(table_map)
            with engine.begin() as connection:
                migration_context = get_migration_context(connection, table_names)
                raw_diffs = compare_metadata(migration_context, metadata)
                diffs = reformat_alembic_diffs(raw_diffs)

            tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
            for table_name in tables_to_rebuild:
                sql_adapter = table_map[table_name]
                if not sql_adapter.config.is_static:
                    try:
                        self.rebuild_table(sql_adapter)
                    except TableRebuildError as e:
                        _notify_rebuild(six.text_type(e), sql_adapter.config.to_json())
                else:
                    self.rebuild_table(sql_adapter)

            tables_to_migrate = get_tables_to_migrate(diffs, table_names)
            tables_to_migrate -= tables_to_rebuild
            migrate_tables(engine, raw_diffs, tables_to_migrate)
Exemplo n.º 2
0
    def _rebuild_sql_tables(self, adapters):
        # todo move this code to sql adapter rebuild_if_necessary
        tables_by_engine = defaultdict(dict)
        for adapter in adapters:
            sql_adapter = get_indicator_adapter(adapter.config)
            tables_by_engine[sql_adapter.engine_id][
                sql_adapter.get_table().name] = sql_adapter

        _assert = soft_assert(notify_admins=True)
        _notify_rebuild = lambda msg, obj: _assert(False, msg, obj)

        for engine_id, table_map in tables_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            table_names = list(table_map)
            with engine.begin() as connection:
                migration_context = get_migration_context(
                    connection, table_names)
                raw_diffs = compare_metadata(migration_context, metadata)
                diffs = reformat_alembic_diffs(raw_diffs)

            tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
            for table_name in tables_to_rebuild:
                sql_adapter = table_map[table_name]
                if not sql_adapter.config.is_static:
                    try:
                        self.rebuild_table(sql_adapter)
                    except TableRebuildError as e:
                        _notify_rebuild(six.text_type(e),
                                        sql_adapter.config.to_json())
                else:
                    self.rebuild_table(sql_adapter)

            tables_to_migrate = get_tables_to_migrate(diffs, table_names)
            tables_to_migrate -= tables_to_rebuild
            migrate_tables(engine, raw_diffs, tables_to_migrate)
Exemplo n.º 3
0
    def _rebuild_sql_tables(self, adapters):
        tables_by_engine = defaultdict(dict)
        for adapter in adapters:
            sql_adapter = get_indicator_adapter(adapter.config)
            try:
                tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter
            except BadSpecError:
                _soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com'))
                _soft_assert(False, "Broken data source {}".format(adapter.config.get_id))

        _assert = soft_assert(notify_admins=True)
        _notify_rebuild = lambda msg, obj: _assert(False, msg, obj)

        for engine_id, table_map in tables_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            table_names = list(table_map)
            with engine.begin() as connection:
                migration_context = get_migration_context(connection, table_names)
                raw_diffs = compare_metadata(migration_context, metadata)
                diffs = reformat_alembic_diffs(raw_diffs)

            tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
            for table_name in tables_to_rebuild:
                sql_adapter = table_map[table_name]
                if not sql_adapter.config.is_static:
                    try:
                        self.rebuild_table(sql_adapter)
                    except TableRebuildError as e:
                        _notify_rebuild(six.text_type(e), sql_adapter.config.to_json())
                else:
                    self.rebuild_table(sql_adapter)

            tables_to_migrate = get_tables_to_migrate(diffs, table_names)
            tables_to_migrate -= tables_to_rebuild
            migrate_tables(engine, raw_diffs, tables_to_migrate)
Exemplo n.º 4
0
 def test_filter_by_type(self):
     diffs = {
         SimpleDiff(type_, type_, None, None)
         for type_ in DiffTypes.ALL
     }
     tables = get_tables_to_rebuild(diffs)
     self.assertEqual(tables, set(DiffTypes.TYPES_FOR_REBUILD))
Exemplo n.º 5
0
    def _rebuild_sql_tables(self, adapters):
        tables_by_engine = defaultdict(dict)
        for adapter in adapters:
            tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter

        _assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
        _notify_cory = lambda msg, obj: _assert(False, msg, obj)

        for engine_id, table_map in tables_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            with engine.begin() as connection:
                migration_context = get_migration_context(connection, table_map.keys())
                raw_diffs = compare_metadata(migration_context, metadata)
                diffs = reformat_alembic_diffs(raw_diffs)

            tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
            for table_name in tables_to_rebuild:
                sql_adapter = table_map[table_name]
                if not sql_adapter.config.is_static:
                    try:
                        rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
                        self.rebuild_table(sql_adapter)
                    except TableRebuildError, e:
                        _notify_cory(unicode(e), sql_adapter.config.to_json())
                else:
                    self.rebuild_table(sql_adapter)
Exemplo n.º 6
0
    def _rebuild_sql_tables(self, adapters):
        # todo move this code to sql adapter rebuild_if_necessary
        tables_by_engine = defaultdict(dict)
        for adapter in adapters:
            sql_adapter = get_indicator_adapter(adapter.config)
            tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter

        _assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
        _notify_cory = lambda msg, obj: _assert(False, msg, obj)

        for engine_id, table_map in tables_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            with engine.begin() as connection:
                migration_context = get_migration_context(connection, table_map.keys())
                raw_diffs = compare_metadata(migration_context, metadata)
                diffs = reformat_alembic_diffs(raw_diffs)

            tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
            for table_name in tables_to_rebuild:
                sql_adapter = table_map[table_name]
                if not sql_adapter.config.is_static:
                    try:
                        rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
                        self.rebuild_table(sql_adapter)
                    except TableRebuildError, e:
                        _notify_cory(unicode(e), sql_adapter.config.to_json())
                else:
                    self.rebuild_table(sql_adapter)
Exemplo n.º 7
0
 def test_filter_by_table(self):
     diffs = {
         SimpleDiff(DiffTypes.ADD_TABLE, 't1', None),
         SimpleDiff(DiffTypes.ADD_TABLE, 't2', None),
         SimpleDiff(DiffTypes.ADD_COLUMN, 't1', None),
     }
     tables = get_tables_to_rebuild(diffs, {'t1'})
     self.assertEqual(tables, {'t1'})
Exemplo n.º 8
0
 def test_filter_by_table(self):
     diffs = {
         SimpleDiff(DiffTypes.ADD_TABLE, 't1', None),
         SimpleDiff(DiffTypes.ADD_TABLE, 't2', None),
         SimpleDiff(DiffTypes.ADD_COLUMN, 't1', None),
     }
     tables = get_tables_to_rebuild(diffs, {'t1'})
     self.assertEqual(tables, {'t1'})
Exemplo n.º 9
0
 def test_filter_by_type(self):
     diffs = {
         SimpleDiff(type_, type_, None)
         for type_ in DiffTypes.ALL
     }
     tables = {diff.table_name for diff in diffs}
     tables = get_tables_to_rebuild(diffs, tables)
     self.assertEqual(
         tables,
         set(DiffTypes.TYPES_FOR_REBUILD)
     )
Exemplo n.º 10
0
    def rebuild_tables_if_necessary(self):
        table_map = {t.get_table().name: t for t in self.tables}
        engine = self.get_sql_engine()
        with engine.begin() as connection:
            migration_context = get_migration_context(connection, table_map.keys())
            diffs = compare_metadata(migration_context, metadata)

        tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
        for table_name in tables_to_rebuild:
            table = table_map[table_name]
            self.rebuild_table(table)
Exemplo n.º 11
0
    def rebuild_tables_if_necessary(self):
        tables_by_engine = defaultdict(dict)
        for adapter in self.table_adapters:
            tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter

        for engine_id, table_map in tables_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            with engine.begin() as connection:
                migration_context = get_migration_context(connection, table_map.keys())
                diffs = compare_metadata(migration_context, metadata)

            tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
            for table_name in tables_to_rebuild:
                table = table_map[table_name]
                try:
                    self.rebuild_table(table)
                except TableRebuildError, e:
                    notify_error(unicode(e))
Exemplo n.º 12
0
    def rebuild_tables_if_necessary(self):
        tables_by_engine = defaultdict(dict)
        for adapter in self.table_adapters:
            tables_by_engine[adapter.engine_id][
                adapter.get_table().name] = adapter

        for engine_id, table_map in tables_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            with engine.begin() as connection:
                migration_context = get_migration_context(
                    connection, table_map.keys())
                diffs = compare_metadata(migration_context, metadata)

            tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
            for table_name in tables_to_rebuild:
                table = table_map[table_name]
                try:
                    self.rebuild_table(table)
                except TableRebuildError, e:
                    notify_error(unicode(e))
Exemplo n.º 13
0
    def rebuild_tables_if_necessary(self):
        tables_by_engine = defaultdict(dict)
        for adapter in self.table_adapters:
            tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter

        _assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
        _notify_cory = lambda msg, obj: _assert(False, msg, obj)

        for engine_id, table_map in tables_by_engine.items():
            engine = connection_manager.get_engine(engine_id)
            with engine.begin() as connection:
                migration_context = get_migration_context(connection, table_map.keys())
                diffs = compare_metadata(migration_context, metadata)

            tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
            for table_name in tables_to_rebuild:
                sql_adapter = table_map[table_name]
                if not is_static(sql_adapter.config._id):
                    try:
                        rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
                        self.rebuild_table(sql_adapter)
                    except TableRebuildError, e:
                        _notify_cory(unicode(e), sql_adapter.config.to_json())
                    else:
                        # note: this fancy logging can be removed as soon as we get to the
                        # bottom of http://manage.dimagi.com/default.asp?211297
                        # if no signs of it popping back up by april 2016, should remove this
                        rev_after_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
                        _notify_cory(
                            u'rebuilt table {} ({}) because {}. rev before: {}, rev after: {}'.format(
                                table_name,
                                u'{} [{}]'.format(sql_adapter.config.display_name, sql_adapter.config._id),
                                diffs,
                                rev_before_rebuild,
                                rev_after_rebuild,
                            ),
                            sql_adapter.config.to_json(),
                        )
                else:
                    self.rebuild_table(sql_adapter)
Exemplo n.º 14
0
def get_tables_rebuild_migrate(diffs):
    tables_to_rebuild = get_tables_to_rebuild(diffs)
    tables_to_migrate = get_tables_to_migrate(diffs)
    tables_to_migrate -= tables_to_rebuild
    return MigrateRebuildTables(migrate=tables_to_migrate,
                                rebuild=tables_to_rebuild)
Exemplo n.º 15
0
def get_tables_rebuild_migrate(diffs, table_names):
    tables_to_rebuild = get_tables_to_rebuild(diffs.formatted, table_names)
    tables_to_migrate = get_tables_to_migrate(diffs.formatted, table_names)
    tables_to_migrate -= tables_to_rebuild
    return MigrateRebuildTables(migrate=tables_to_migrate, rebuild=tables_to_rebuild)
Exemplo n.º 16
0
def get_tables_rebuild_migrate(diffs, table_names):
    tables_to_rebuild = get_tables_to_rebuild(diffs.formatted, table_names)
    tables_to_migrate = get_tables_to_migrate(diffs.formatted, table_names)
    tables_to_migrate -= tables_to_rebuild
    return MigrateRebuildTables(migrate=tables_to_migrate, rebuild=tables_to_rebuild)