def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
for adapter in adapters:
sql_adapter = get_indicator_adapter(adapter.config)
try:
tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter
except BadSpecError:
_soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com'))
_soft_assert(False, "Broken data source {}".format(adapter.config.get_id))
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
table_names = list(table_map)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
tables_to_migrate = get_tables_to_migrate(diffs, table_names)
tables_to_migrate -= tables_to_rebuild
migrate_tables(engine, raw_diffs, tables_to_migrate)
def _rebuild_sql_tables(self, adapters):
# todo move this code to sql adapter rebuild_if_necessary
tables_by_engine = defaultdict(dict)
for adapter in adapters:
sql_adapter = get_indicator_adapter(adapter.config)
tables_by_engine[sql_adapter.engine_id][
sql_adapter.get_table().name] = sql_adapter
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
table_names = list(table_map)
with engine.begin() as connection:
migration_context = get_migration_context(
connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e),
sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
tables_to_migrate = get_tables_to_migrate(diffs, table_names)
tables_to_migrate -= tables_to_rebuild
migrate_tables(engine, raw_diffs, tables_to_migrate)
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
for adapter in adapters:
sql_adapter = get_indicator_adapter(adapter.config)
try:
tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter
except BadSpecError:
_soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com'))
_soft_assert(False, "Broken data source {}".format(adapter.config.get_id))
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
table_names = list(table_map)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
tables_to_migrate = get_tables_to_migrate(diffs, table_names)
tables_to_migrate -= tables_to_rebuild
migrate_tables(engine, raw_diffs, tables_to_migrate)
def test_filter_by_type(self):
diffs = {
SimpleDiff(type_, type_, None, None)
for type_ in DiffTypes.ALL
}
tables = get_tables_to_rebuild(diffs)
self.assertEqual(tables, set(DiffTypes.TYPES_FOR_REBUILD))
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
for adapter in adapters:
tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter
_assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
_notify_cory = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_map.keys())
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
self.rebuild_table(sql_adapter)
except TableRebuildError, e:
_notify_cory(unicode(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
def _rebuild_sql_tables(self, adapters):
# todo move this code to sql adapter rebuild_if_necessary
tables_by_engine = defaultdict(dict)
for adapter in adapters:
sql_adapter = get_indicator_adapter(adapter.config)
tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter
_assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
_notify_cory = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_map.keys())
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
self.rebuild_table(sql_adapter)
except TableRebuildError, e:
_notify_cory(unicode(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
def test_filter_by_table(self):
diffs = {
SimpleDiff(DiffTypes.ADD_TABLE, 't1', None),
SimpleDiff(DiffTypes.ADD_TABLE, 't2', None),
SimpleDiff(DiffTypes.ADD_COLUMN, 't1', None),
}
tables = get_tables_to_rebuild(diffs, {'t1'})
self.assertEqual(tables, {'t1'})
def test_filter_by_table(self):
diffs = {
SimpleDiff(DiffTypes.ADD_TABLE, 't1', None),
SimpleDiff(DiffTypes.ADD_TABLE, 't2', None),
SimpleDiff(DiffTypes.ADD_COLUMN, 't1', None),
}
tables = get_tables_to_rebuild(diffs, {'t1'})
self.assertEqual(tables, {'t1'})
def test_filter_by_type(self):
diffs = {
SimpleDiff(type_, type_, None)
for type_ in DiffTypes.ALL
}
tables = {diff.table_name for diff in diffs}
tables = get_tables_to_rebuild(diffs, tables)
self.assertEqual(
tables,
set(DiffTypes.TYPES_FOR_REBUILD)
)
def rebuild_tables_if_necessary(self):
table_map = {t.get_table().name: t for t in self.tables}
engine = self.get_sql_engine()
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_map.keys())
diffs = compare_metadata(migration_context, metadata)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
for table_name in tables_to_rebuild:
table = table_map[table_name]
self.rebuild_table(table)
def rebuild_tables_if_necessary(self):
tables_by_engine = defaultdict(dict)
for adapter in self.table_adapters:
tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_map.keys())
diffs = compare_metadata(migration_context, metadata)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
for table_name in tables_to_rebuild:
table = table_map[table_name]
try:
self.rebuild_table(table)
except TableRebuildError, e:
notify_error(unicode(e))
def rebuild_tables_if_necessary(self):
tables_by_engine = defaultdict(dict)
for adapter in self.table_adapters:
tables_by_engine[adapter.engine_id][
adapter.get_table().name] = adapter
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(
connection, table_map.keys())
diffs = compare_metadata(migration_context, metadata)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
for table_name in tables_to_rebuild:
table = table_map[table_name]
try:
self.rebuild_table(table)
except TableRebuildError, e:
notify_error(unicode(e))
def rebuild_tables_if_necessary(self):
tables_by_engine = defaultdict(dict)
for adapter in self.table_adapters:
tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter
_assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
_notify_cory = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_map.keys())
diffs = compare_metadata(migration_context, metadata)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not is_static(sql_adapter.config._id):
try:
rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
self.rebuild_table(sql_adapter)
except TableRebuildError, e:
_notify_cory(unicode(e), sql_adapter.config.to_json())
else:
# note: this fancy logging can be removed as soon as we get to the
# bottom of http://manage.dimagi.com/default.asp?211297
# if no signs of it popping back up by april 2016, should remove this
rev_after_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
_notify_cory(
u'rebuilt table {} ({}) because {}. rev before: {}, rev after: {}'.format(
table_name,
u'{} [{}]'.format(sql_adapter.config.display_name, sql_adapter.config._id),
diffs,
rev_before_rebuild,
rev_after_rebuild,
),
sql_adapter.config.to_json(),
)
else:
self.rebuild_table(sql_adapter)
def get_tables_rebuild_migrate(diffs):
tables_to_rebuild = get_tables_to_rebuild(diffs)
tables_to_migrate = get_tables_to_migrate(diffs)
tables_to_migrate -= tables_to_rebuild
return MigrateRebuildTables(migrate=tables_to_migrate,
rebuild=tables_to_rebuild)
def get_tables_rebuild_migrate(diffs, table_names):
tables_to_rebuild = get_tables_to_rebuild(diffs.formatted, table_names)
tables_to_migrate = get_tables_to_migrate(diffs.formatted, table_names)
tables_to_migrate -= tables_to_rebuild
return MigrateRebuildTables(migrate=tables_to_migrate, rebuild=tables_to_rebuild)
def get_tables_rebuild_migrate(diffs, table_names):
tables_to_rebuild = get_tables_to_rebuild(diffs.formatted, table_names)
tables_to_migrate = get_tables_to_migrate(diffs.formatted, table_names)
tables_to_migrate -= tables_to_rebuild
return MigrateRebuildTables(migrate=tables_to_migrate, rebuild=tables_to_rebuild)