def _rebuild_sql_tables(self, adapters):
# todo move this code to sql adapter rebuild_if_necessary
tables_by_engine = defaultdict(dict)
for adapter in adapters:
sql_adapter = get_indicator_adapter(adapter.config)
tables_by_engine[sql_adapter.engine_id][
sql_adapter.get_table().name] = sql_adapter
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
table_names = list(table_map)
with engine.begin() as connection:
migration_context = get_migration_context(
connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e),
sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
tables_to_migrate = get_tables_to_migrate(diffs, table_names)
tables_to_migrate -= tables_to_rebuild
migrate_tables(engine, raw_diffs, tables_to_migrate)
def test_table_filter(self):
migration_context = get_migration_context(self.engine,
[self.table_name])
sqlalchemy.Table('new_table', self.metadata)
raw_diffs = compare_metadata(migration_context, self.metadata)
_, diffs = reformat_alembic_diffs(raw_diffs)
self.assertEqual(0, len(diffs))
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
for adapter in adapters:
sql_adapter = get_indicator_adapter(adapter.config)
try:
tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter
except BadSpecError:
_soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com'))
_soft_assert(False, "Broken data source {}".format(adapter.config.get_id))
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
table_names = list(table_map)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
tables_to_migrate = get_tables_to_migrate(diffs, table_names)
tables_to_migrate -= tables_to_rebuild
migrate_tables(engine, raw_diffs, tables_to_migrate)
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
for adapter in adapters:
tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter
_assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
_notify_cory = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_map.keys())
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
self.rebuild_table(sql_adapter)
except TableRebuildError, e:
_notify_cory(unicode(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
for adapter in adapters:
sql_adapter = get_indicator_adapter(adapter.config)
try:
tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter
except BadSpecError:
_soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com'))
_soft_assert(False, "Broken data source {}".format(adapter.config.get_id))
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
table_names = list(table_map)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_names)
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
tables_to_migrate = get_tables_to_migrate(diffs, table_names)
tables_to_migrate -= tables_to_rebuild
migrate_tables(engine, raw_diffs, tables_to_migrate)
def _test_diffs(self, metadata, expected_diffs, table_names=None):
migration_context = get_migration_context(
self.engine, table_names or [self.table_name, 'new_table'])
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
self.assertEqual(set(diffs), expected_diffs)
return diffs
def handle(self, **options):
data_sources = list(DataSourceConfiguration.all())
data_sources.extend(list(StaticDataSourceConfiguration.all()))
engine_ids = self._get_engine_ids(data_sources, options.get('engine_id'))
tables_to_remove_by_engine = defaultdict(list)
for engine_id in engine_ids:
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(connection, include_object=_include_object)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_remove_by_engine[engine_id] = [
diff.table_name for diff in diffs
if diff.type == 'remove_table'
]
for engine_id, tablenames in tables_to_remove_by_engine.items():
engine = connection_manager.get_engine(engine_id)
for tablename in tablenames:
with engine.begin() as connection:
try:
result = connection.execute(
'SELECT COUNT(*), MAX(inserted_at) FROM "{tablename}"'.format(tablename=tablename)
)
except Exception:
print(tablename, "no inserted_at column, probably not UCR")
else:
print(tablename, result.fetchone())
def _rebuild_sql_tables(self, adapters):
# todo move this code to sql adapter rebuild_if_necessary
tables_by_engine = defaultdict(dict)
for adapter in adapters:
sql_adapter = get_indicator_adapter(adapter.config)
tables_by_engine[sql_adapter.engine_id][sql_adapter.get_table().name] = sql_adapter
_assert = soft_assert(to='@'.join(['czue', 'dimagi.com']))
_notify_cory = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_map.keys())
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_rebuild = get_tables_to_rebuild(diffs, table_map.keys())
for table_name in tables_to_rebuild:
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
rev_before_rebuild = sql_adapter.config.get_db().get_rev(sql_adapter.config._id)
self.rebuild_table(sql_adapter)
except TableRebuildError, e:
_notify_cory(unicode(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
def handle(self, **options):
data_sources = list(DataSourceConfiguration.all())
data_sources.extend(list(StaticDataSourceConfiguration.all()))
engine_ids = self._get_engine_ids(data_sources,
options.get('engine_id'))
tables_to_remove_by_engine = defaultdict(list)
for engine_id in engine_ids:
engine = connection_manager.get_engine(engine_id)
with engine.begin() as connection:
migration_context = get_migration_context(
connection, include_object=_include_object)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
tables_to_remove_by_engine[engine_id] = [
diff.table_name for diff in diffs
if diff.type == 'remove_table'
]
for engine_id, tablenames in tables_to_remove_by_engine.items():
engine = connection_manager.get_engine(engine_id)
for tablename in tablenames:
with engine.begin() as connection:
try:
result = connection.execute(
'SELECT COUNT(*), MAX(inserted_at) FROM "{tablename}"'
.format(tablename=tablename))
except Exception:
print(tablename,
"no inserted_at column, probably not UCR")
else:
print(tablename, result.fetchone())
def get_table_diffs(engine, table_names, metadata):
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
return [
diff for diff in reformat_alembic_diffs(raw_diffs)
if diff.table_name in table_names
]
def test_flatten_raw_diffs():
raw_diffs = [
[('diff1', None)],
[('diff2', None)],
('diff3', None),
]
flattened, _ = reformat_alembic_diffs(raw_diffs)
assert_list_equal(flattened, [
('diff1', None),
('diff2', None),
('diff3', None),
])
def _test_diffs(self, metadata, expected_diffs, table_names=None):
migration_context = get_migration_context(self.engine, table_names or [self.table_name, 'new_table'])
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
self.assertEqual(set(diffs), expected_diffs)
def test_table_filter(self):
migration_context = get_migration_context(self.engine, [self.table_name])
sqlalchemy.Table('new_table', self.metadata)
raw_diffs = compare_metadata(migration_context, self.metadata)
diffs = reformat_alembic_diffs(raw_diffs)
self.assertEqual(0, len(diffs))
def get_table_diffs(engine, table_names, metadata):
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
return TableDiffs(raw=raw_diffs, formatted=diffs)
def get_table_diffs(engine, table_names, metadata):
with engine.begin() as connection:
migration_context = get_migration_context(connection, table_names)
raw_diffs = compare_metadata(migration_context, metadata)
diffs = reformat_alembic_diffs(raw_diffs)
return TableDiffs(raw=raw_diffs, formatted=diffs)