def make_update_script_for_model(cls, engine, oldmodel, model, repository,
**opts):
"""Create a migration script"""
# Compute differences.
if isinstance(repository, basestring):
from migrate.versioning.repository import Repository # oh dear, an import cycle!
repository = Repository(repository)
oldmodel = loadModel(oldmodel)
model = loadModel(model)
diff = schemadiff.getDiffOfModelAgainstModel(
oldmodel, model, engine, excludeTables=[repository.version_table])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(
diff).toUpgradeDowngradePython()
# Store differences into file.
template_file = None
src = template.get_script(template_file)
contents = open(src).read()
search = 'def upgrade():'
contents = contents.replace(search, decls + '\n\n' + search, 1)
if upgradeCommands:
contents = contents.replace(' pass', upgradeCommands, 1)
if downgradeCommands:
contents = contents.replace(' pass', downgradeCommands, 1)
return contents
def create_model(cls, engine, repository, declarative=False):
"""
Dump the current database as a Python model.
"""
if isinstance(repository, basestring):
repository = Repository(repository)
diff = schemadiff.getDiffOfModelAgainstDatabase(
MetaData(), engine, excludeTables=[repository.version_table])
return genmodel.ModelGenerator(diff, declarative).toPython()
def update_db_from_model(self, model):
"""
Modify the database to match the structure of the current Python model.
"""
model = load_model(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(
model, self.engine, excludeTables=[self.repository.version_table])
genmodel.ModelGenerator(diff, self.engine).runB2A()
self.update_repository_table(self.version, int(self.repository.latest))
self.load()
def update_db_from_model(self, model):
"""
Modify the database to match the structure of the current Python model.
"""
if isinstance(self.repository, basestring):
self.repository = Repository(self.repository)
model = loadModel(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(
model, self.engine, excludeTables=[self.repository.version_table])
genmodel.ModelGenerator(diff).applyModel()
update = self.table.update(
self.table.c.repository_id == str(self.repository.id))
self.engine.execute(update, version=int(self.repository.latest))
def make_update_script_for_model(cls, engine, oldmodel, model, repository,
**opts):
"""Create a migration script based on difference between two SA models.
:param repository: path to migrate repository
:param oldmodel: dotted.module.name:SAClass or SAClass object
:param model: dotted.module.name:SAClass or SAClass object
:param engine: SQLAlchemy engine
:type repository: string or :class:`Repository instance <migrate.versioning.repository.Repository>`
:type oldmodel: string or Class
:type model: string or Class
:type engine: Engine instance
:returns: Upgrade / Downgrade script
:rtype: string
"""
if isinstance(repository, six.string_types):
# oh dear, an import cycle!
from migrate.versioning.repository import Repository
repository = Repository(repository)
oldmodel = load_model(oldmodel)
model = load_model(model)
# Compute differences.
diff = schemadiff.getDiffOfModelAgainstModel(
model, oldmodel, excludeTables=[repository.version_table])
# TODO: diff can be False (there is no difference?)
decls, upgradeCommands, downgradeCommands = \
genmodel.ModelGenerator(diff,engine).genB2AMigration()
# Store differences into file.
src = Template(opts.pop('templates_path', None)).get_script(
opts.pop('templates_theme', None))
f = open(src)
contents = f.read()
f.close()
# generate source
search = 'def upgrade(migrate_engine):'
contents = contents.replace(search, '\n\n'.join((decls, search)), 1)
if upgradeCommands:
contents = contents.replace(' pass', upgradeCommands, 1)
if downgradeCommands:
contents = contents.replace(' pass', downgradeCommands, 1)
return contents
def test_functional(self):
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel,
tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
eq_((diff.tables_missing_from_B, diff.tables_missing_from_A,
diff.tables_different.keys(), bool(diff)),
(tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff,
isDiff))
# Model is defined but database is empty.
assertDiff(True, [self.table_name], [], [])
# Check Python upgrade and downgrade of database from updated model.
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(
diff, self.engine).genB2AMigration()
# Feature test for a recent SQLa feature;
# expect different output in that case.
if repr(String()) == 'String()':
self.assertEqualsIgnoreWhitespace(
decls, '''
from migrate.changeset import schema
pre_meta = MetaData()
post_meta = MetaData()
tmp_schemadiff = Table('tmp_schemadiff', post_meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('name', UnicodeText),
Column('data', UnicodeText),
)
''')
else:
self.assertEqualsIgnoreWhitespace(
decls, '''
from migrate.changeset import schema
pre_meta = MetaData()
post_meta = MetaData()
tmp_schemadiff = Table('tmp_schemadiff', post_meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('name', UnicodeText(length=None)),
Column('data', UnicodeText(length=None)),
)
''')
# Create table in database, now model should match database.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Check Python code gen from database.
diff = schemadiff.getDiffOfModelAgainstDatabase(
MetaData(), self.engine, excludeTables=['migrate_version'])
src = genmodel.ModelGenerator(diff, self.engine).genBDefinition()
exec src in locals()
c1 = Table('tmp_schemadiff', self.meta, autoload=True).c
c2 = tmp_schemadiff.c
self.compare_columns_equal(c1, c2, ['type'])
# TODO: get rid of ignoring type
if not self.engine.name == 'oracle':
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(),
id=1,
name=u'mydata')
dataId = result.inserted_primary_key[0]
# Modify table in model (by removing it and adding it back to model)
# Drop column data, add columns data2 and data3.
self.meta.remove(self.table)
self.table = Table(
self.table_name,
self.meta,
Column('id', Integer(), primary_key=True),
Column('name', UnicodeText(length=None)),
Column('data2', Integer(), nullable=True),
Column('data3', Integer(), nullable=True),
)
assertDiff(True, [], [], [self.table_name])
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Drop column data3, add data4
self.meta.remove(self.table)
self.table = Table(
self.table_name,
self.meta,
Column('id', Integer(), primary_key=True),
Column('name', UnicodeText(length=None)),
Column('data2', Integer(), nullable=True),
Column('data4', Float(), nullable=True),
)
assertDiff(True, [], [], [self.table_name])
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(
diff, self.engine).genB2AMigration(indent='')
# decls have changed since genBDefinition
exec decls in locals()
# migration commands expect a namespace containing migrate_engine
migrate_engine = self.engine
# run the migration up and down
exec upgradeCommands in locals()
assertDiff(False, [], [], [])
exec decls in locals()
exec downgradeCommands in locals()
assertDiff(True, [], [], [self.table_name])
exec decls in locals()
exec upgradeCommands in locals()
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
# Make sure data is still present.
result = self.engine.execute(
self.table.select(self.table.c.id == dataId))
rows = result.fetchall()
eq_(len(rows), 1)
eq_(rows[0].name, 'mydata')
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(),
id=2,
name=u'mydata2',
data2=123)
dataId2 = result.inserted_primary_key[0]
# Change column type in model.
self.meta.remove(self.table)
self.table = Table(
self.table_name,
self.meta,
Column('id', Integer(), primary_key=True),
Column('name', UnicodeText(length=None)),
Column('data2', String(255), nullable=True),
)
# XXX test type diff
return
assertDiff(True, [], [], [self.table_name])
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
# Make sure data is still present.
result = self.engine.execute(
self.table.select(self.table.c.id == dataId2))
rows = result.fetchall()
self.assertEquals(len(rows), 1)
self.assertEquals(rows[0].name, 'mydata2')
self.assertEquals(rows[0].data2, '123')
# Delete data, since we're about to make a required column.
# Not even using sqlalchemy.PassiveDefault helps because we're doing explicit column select.
self.engine.execute(self.table.delete(), id=dataId)
if not self.engine.name == 'firebird':
# Change column nullable in model.
self.meta.remove(self.table)
self.table = Table(
self.table_name,
self.meta,
Column('id', Integer(), primary_key=True),
Column('name', UnicodeText(length=None)),
Column('data2', String(255), nullable=False),
)
assertDiff(True, [], [],
[self.table_name]) # TODO test nullable diff
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Remove table from model.
self.meta.remove(self.table)
assertDiff(True, [], [self.table_name], [])
def _applyLatestModel(self):
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
genmodel.ModelGenerator(diff, self.engine).runB2A()
def make_table(metadata, *cols):
primary_col = Column('user_id',
UnicodeText,
ForeignKey("user.id"),
primary_key=True)
auto_load = False
create = False
extend = False
try:
# test table exists?
# setting keep_existing to False here causes the second exception
# to throw, indicating that the table is already in the metadata.
_ = Table(table_name,
metadata,
primary_col,
autoload=True,
keep_existing=False)
auto_load = True
extend = True
except NoSuchTableError:
# don't autoload existing table
# don't extend existing table
create = True
except InvalidRequestError:
# don't create new table
auto_load = True
extend = True
except Exception as e:
log.error(str(e))
raise e
try:
user_ext = Table(table_name,
metadata,
primary_col,
*cols,
extend_existing=extend,
mustexist=(not create),
autoload=auto_load)
except Exception as e:
log.error(str(e))
raise e
if create:
metadata.create_all(bind=metadata.bind, tables=[user_ext])
else:
d = strip_diff(get_table_diff(user_ext))
if d:
g = genmodel.ModelGenerator(d, metadata.bind)
g.runB2A()
try:
_map = class_mapper(UserExt)
assert _map.class_ == UserExt
assert _map.local_table is not None
assert _map.local_table.name == table_name
except UnmappedClassError:
_map = mapper(UserExt, user_ext)
except AssertionError as e:
log.error("New mapping on UserExt didn't match existing mapping")
log.error(str(e))
return user_ext
from dtg.forms import *
db.create_all()
if not SystemInfo.query.first():
initialize_db(db)
# setup secret key, do DB migrations
sys_info = SystemInfo.query.first()
app.secret_key = str(sys_info.secret_app_key)
if sys_info.db_rev != APP_DB_REV:
if not sys.dtg_do_upgrade:
raise Exception("Old database version, please backup your database and enable DB migrations.")
if sys_info.db_rev == 3:
dict(db.metadata.tables)["flash_message"].drop(db.engine)
db.create_all()
diff = schemadiff.getDiffOfModelAgainstDatabase(db.metadata, db.engine)
genmodel.ModelGenerator(diff, db.engine).runB2A()
sys_info = SystemInfo.query.first()
sys_info.db_rev = APP_DB_REV
db.session.commit()
@app.before_first_request
def init_background_jobs():
# setup background jobs
app.background_jobs = []
for JOB in JOBS:
job = JOB(app, NIGHTLY_RUNTIME)
app.background_jobs.append(job)
job.start()
