def migrate_tables(self, engine, diffs, table_names, adapters_by_table):
pillow_logging.debug("[rebuild] Application migrations to tables: %s", table_names)
migration_diffs = [diff for diff in diffs if diff.table_name in table_names]
changes = migrate_tables(engine, migration_diffs)
for table, diffs in changes.items():
adapter = adapters_by_table[table]
adapter.log_table_migrate(source='pillowtop', diffs=diffs)
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
all_adapters = []
for adapter in adapters:
if getattr(adapter, 'all_adapters', None):
all_adapters.extend(adapter.all_adapters)
else:
all_adapters.append(adapter)
for adapter in all_adapters:
tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
table_names = list(table_map)
engine = connection_manager.get_engine(engine_id)
diffs = get_table_diffs(engine, table_names, get_metadata(engine_id))
tables_to_act_on = get_tables_rebuild_migrate(diffs)
for table_name in tables_to_act_on.rebuild:
pillow_logging.debug("[rebuild] Rebuilding table: %s", table_name)
sql_adapter = table_map[table_name]
table_diffs = [diff for diff in diffs if diff.table_name == table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter, table_diffs)
except TableRebuildError as e:
_notify_rebuild(str(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter, table_diffs)
self.migrate_tables(engine, diffs, tables_to_act_on.migrate, table_map)
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
all_adapters = []
for adapter in adapters:
if getattr(adapter, 'all_adapters', None):
all_adapters.extend(adapter.all_adapters)
else:
all_adapters.append(adapter)
for adapter in all_adapters:
try:
tables_by_engine[adapter.engine_id][
adapter.get_table().name] = adapter
except BadSpecError:
_soft_assert = soft_assert(
to='{}@{}'.format('jemord', 'dimagi.com'))
_soft_assert(
False,
"Broken data source {}".format(adapter.config.get_id))
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
table_names = list(table_map)
engine = connection_manager.get_engine(engine_id)
diffs = get_table_diffs(engine, table_names,
get_metadata(engine_id))
tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names)
for table_name in tables_to_act_on.rebuild:
pillow_logging.debug("[rebuild] Rebuilding table: %s",
table_name)
sql_adapter = table_map[table_name]
table_diffs = [
diff for diff in diffs.formatted
if diff.table_name == table_name
]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter, table_diffs)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e),
sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter, table_diffs)
migration_diffs = diffs.filter(tables_to_act_on.migrate)
self.migrate_tables(engine, migration_diffs.raw,
tables_to_act_on.migrate, table_map)
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
for adapter in adapters:
try:
tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter
except BadSpecError:
_soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com'))
_soft_assert(False, "Broken data source {}".format(adapter.config.get_id))
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
table_names = list(table_map)
engine = connection_manager.get_engine(engine_id)
# Temporary measure necessary during the process of renaming tables
# - Configs point to new tables which may just be views and not real tables
# - The global metadata contains references to the new table names
legacy_tables = {}
table_names_for_diff = []
diff_metadata = sqlalchemy.MetaData()
with engine.begin() as connection:
for table_name in table_names:
sql_adapter = table_map[table_name]
legacy_table_name = get_legacy_table_name(
sql_adapter.config.domain, sql_adapter.config.table_id
)
if not table_exists(connection, table_name) and table_exists(connection, legacy_table_name):
legacy_tables[legacy_table_name] = table_name
pillow_logging.debug("[rebuild] Using legacy table: %s", legacy_table_name)
# popultate metadata with the table schema
get_indicator_table(
sql_adapter.config,
metadata=diff_metadata,
override_table_name=legacy_table_name
)
table_names_for_diff.append(legacy_table_name)
else:
# popultate metadata with the table schema
get_indicator_table(sql_adapter.config, metadata=diff_metadata)
table_names_for_diff.append(table_name)
diffs = get_table_diffs(engine, table_names_for_diff, diff_metadata)
tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names_for_diff)
for real_table_name in tables_to_act_on.rebuild:
table_name = legacy_tables.get(real_table_name, real_table_name)
pillow_logging.debug("[rebuild] Rebuilding table: %s (%s)", real_table_name, table_name)
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
pillow_logging.debug("[rebuild] Application migrations to tables: %s", tables_to_act_on.migrate)
migrate_tables(engine, diffs.raw, tables_to_act_on.migrate)
def _rebuild_sql_tables(self, adapters):
tables_by_engine = defaultdict(dict)
all_adapters = []
for adapter in adapters:
if getattr(adapter, 'all_adapters', None):
all_adapters.extend(adapter.all_adapters)
else:
all_adapters.append(adapter)
for adapter in all_adapters:
try:
tables_by_engine[adapter.engine_id][adapter.get_table().name] = adapter
except BadSpecError:
_soft_assert = soft_assert(to='{}@{}'.format('jemord', 'dimagi.com'))
_soft_assert(False, "Broken data source {}".format(adapter.config.get_id))
_assert = soft_assert(notify_admins=True)
_notify_rebuild = lambda msg, obj: _assert(False, msg, obj)
for engine_id, table_map in tables_by_engine.items():
table_names = list(table_map)
engine = connection_manager.get_engine(engine_id)
diffs = get_table_diffs(engine, table_names, get_metadata(engine_id))
tables_to_act_on = get_tables_rebuild_migrate(diffs, table_names)
for table_name in tables_to_act_on.rebuild:
pillow_logging.debug("[rebuild] Rebuilding table: %s", table_name)
sql_adapter = table_map[table_name]
if not sql_adapter.config.is_static:
try:
self.rebuild_table(sql_adapter)
except TableRebuildError as e:
_notify_rebuild(six.text_type(e), sql_adapter.config.to_json())
else:
self.rebuild_table(sql_adapter)
self.migrate_tables(engine, diffs.raw, tables_to_act_on.migrate, table_map)
def migrate_tables(self, engine, diffs, table_names, adapters_by_table):
pillow_logging.debug("[rebuild] Application migrations to tables: %s", table_names)
changes = migrate_tables(engine, diffs, table_names)
for table, diffs in changes.items():
adapter = adapters_by_table[table]
adapter.log_table_migrate(source='pillowtop', diffs=diffs)