def load_environment(global_conf, app_conf):
"""Configure the Pylons environment via the ``pylons.config``
object
"""
config = PylonsConfig()
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[os.path.join(root, 'templates')])
# import our private.ini that holds keys, etc
imp = global_conf.get('import')
if imp:
cp = ConfigParser()
cp.read(imp)
global_conf.update(cp.defaults())
if cp.has_section('APP'):
app_conf.update(cp.items('APP'))
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='linkdrop', paths=paths)
config['routes.map'] = make_map(config)
config['pylons.app_globals'] = app_globals.Globals(config)
import linkdrop.lib.helpers as h
config['pylons.h'] = h
# Setup cache object as early as possible
import pylons
pylons.cache._push_object(config['pylons.app_globals'].cache)
# Create the Mako TemplateLookup, with the default auto-escaping
config['pylons.app_globals'].mako_lookup = TemplateLookup(
directories=paths['templates'],
error_handler=handle_mako_error,
module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
input_encoding='utf-8', default_filters=['escape'],
imports=['from webhelpers.html import escape'])
# Setup the SQLAlchemy database engine
engine = engine_from_config(config, 'sqlalchemy.')
init_model(engine)
# sqlalchemy auto migration
if asbool(config.get('migrate.auto')):
try:
# managed upgrades
cschema = schema.ControlledSchema.create(engine, config['migrate.repository'])
cschema.update_db_from_model(meta.Base.metadata)
except exceptions.InvalidRepositoryError, e:
# unmanaged upgrades
diff = schemadiff.getDiffOfModelAgainstDatabase(
meta.Base.metadata, engine, excludeTables=None)
genmodel.ModelGenerator(diff).applyModel()
def create_model(cls, engine, repository, declarative=False):
"""
Dump the current database as a Python model.
"""
if isinstance(repository, basestring):
repository = Repository(repository)
diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), engine, excludeTables=[repository.version_table])
return genmodel.ModelGenerator(diff, engine, declarative).genBDefinition()
def compare_model_to_db(cls, engine, model, repository):
"""
Compare the current model against the current database.
"""
if isinstance(repository, basestring):
repository = Repository(repository)
model = load_model(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(model, engine, excludeTables=[repository.version_table])
return diff
def create_model(cls, engine, repository, declarative=False):
"""
Dump the current database as a Python model.
"""
if isinstance(repository, six.string_types):
repository = Repository(repository)
diff = schemadiff.getDiffOfModelAgainstDatabase(
MetaData(), engine, excludeTables=[repository.version_table])
return genmodel.ModelGenerator(diff, engine,
declarative).genBDefinition()
def compare_model_to_db(cls, engine, model, repository):
"""
Compare the current model against the current database.
"""
if isinstance(repository, six.string_types):
repository = Repository(repository)
model = load_model(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(
model, engine, excludeTables=[repository.version_table])
return diff
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
eq_(bool(diff), isDiff)
eq_(
(diff.tables_missing_from_B,
diff.tables_missing_from_A,
diff.tables_different.keys()),
(tablesMissingInDatabase,
tablesMissingInModel,
tablesWithDiff)
)
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
eq_(
(diff.tables_missing_from_B,
diff.tables_missing_from_A,
diff.tables_different.keys(),
bool(diff)),
(tablesMissingInDatabase,
tablesMissingInModel,
tablesWithDiff,
isDiff)
)
def update_db_from_model(self, model):
"""
Modify the database to match the structure of the current Python model.
"""
if isinstance(self.repository, basestring):
self.repository=Repository(self.repository)
model = load_model(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(
model, self.engine, excludeTables=[self.repository.version_table])
genmodel.ModelGenerator(diff).applyModel()
update = self.table.update(
self.table.c.repository_id == str(self.repository.id))
self.engine.execute(update, version=int(self.repository.latest))
def update_db_from_model(self, model):
"""
Modify the database to match the structure of the current Python model.
"""
model = load_model(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(
model, self.engine, excludeTables=[self.repository.version_table])
genmodel.ModelGenerator(diff).applyModel()
self.update_repository_table(self.version, int(self.repository.latest))
self.load()
def update_db_from_model(self, model):
"""
Modify the database to match the structure of the current Python model.
"""
model = load_model(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(
model, self.engine, excludeTables=[self.repository.version_table])
genmodel.ModelGenerator(diff, self.engine).runB2A()
self.update_repository_table(self.version, int(self.repository.latest))
self.load()
def validate_database(database):
diff = schemadiff.getDiffOfModelAgainstDatabase(database.metadata,
database.engine)
all_diff = []
tables_diff = check_tables(database, diff)
fields_diff = check_fields(database, diff)
all_diff.extend(tables_diff)
all_diff.extend(fields_diff)
return all_diff
def update_db_from_model(self, model):
"""
Modify the database to match the structure of the current Python model.
"""
if isinstance(self.repository, basestring):
self.repository = Repository(self.repository)
model = loadModel(model)
diff = schemadiff.getDiffOfModelAgainstDatabase(
model, self.engine, excludeTables=[self.repository.version_table])
genmodel.ModelGenerator(diff).applyModel()
update = self.table.update(
self.table.c.repository_id == str(self.repository.id))
self.engine.execute(update, version=int(self.repository.latest))
def test_z_add_drop_existing_table(self):
print self.Donkey.tables.keys()
self.Donkey.add_table(tables.Table("moo%s" % self.p, Text("moo%s" % self.p)), drop = True)
self.Donkey.persist()
print validate_database(self.Donkey)
diff = schemadiff.getDiffOfModelAgainstDatabase(self.Donkey.metadata,
self.Donkey.engine)
print diff
assert validate_database(self.Donkey)[0] == []
assert validate_database(self.Donkey)[1] == []
assert validate_database(self.Donkey)[2] == []
def test_functional(self):
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
eq_(
(diff.tables_missing_from_B,
diff.tables_missing_from_A,
diff.tables_different.keys(),
bool(diff)),
(tablesMissingInDatabase,
tablesMissingInModel,
tablesWithDiff,
isDiff)
)
# Model is defined but database is empty.
assertDiff(True, [self.table_name], [], [])
# Check Python upgrade and downgrade of database from updated model.
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(diff,self.engine).genB2AMigration()
# Feature test for a recent SQLa feature;
# expect different output in that case.
if repr(String()) == 'String()':
self.assertEqualsIgnoreWhitespace(decls, '''
from migrate.changeset import schema
pre_meta = MetaData()
post_meta = MetaData()
tmp_schemadiff = Table('tmp_schemadiff', post_meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('name', UnicodeText),
Column('data', UnicodeText),
)
''')
else:
self.assertEqualsIgnoreWhitespace(decls, '''
from migrate.changeset import schema
pre_meta = MetaData()
post_meta = MetaData()
tmp_schemadiff = Table('tmp_schemadiff', post_meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('name', UnicodeText(length=None)),
Column('data', UnicodeText(length=None)),
)
''')
# Create table in database, now model should match database.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Check Python code gen from database.
diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), self.engine, excludeTables=['migrate_version'])
src = genmodel.ModelGenerator(diff,self.engine).genBDefinition()
exec src in locals()
c1 = Table('tmp_schemadiff', self.meta, autoload=True).c
c2 = tmp_schemadiff.c
self.compare_columns_equal(c1, c2, ['type'])
# TODO: get rid of ignoring type
if not self.engine.name == 'oracle':
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(), id=1, name=u'mydata')
dataId = result.inserted_primary_key[0]
# Modify table in model (by removing it and adding it back to model)
# Drop column data, add columns data2 and data3.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',Integer(),nullable=True),
Column('data3',Integer(),nullable=True),
)
assertDiff(True, [], [], [self.table_name])
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Drop column data3, add data4
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',Integer(),nullable=True),
Column('data4',Float(),nullable=True),
)
assertDiff(True, [], [], [self.table_name])
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(diff,self.engine).genB2AMigration(indent='')
# decls have changed since genBDefinition
exec decls in locals()
# migration commands expect a namespace containing migrate_engine
migrate_engine = self.engine
# run the migration up and down
exec upgradeCommands in locals()
assertDiff(False, [], [], [])
exec decls in locals()
exec downgradeCommands in locals()
assertDiff(True, [], [], [self.table_name])
exec decls in locals()
exec upgradeCommands in locals()
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
# Make sure data is still present.
result = self.engine.execute(self.table.select(self.table.c.id==dataId))
rows = result.fetchall()
eq_(len(rows), 1)
eq_(rows[0].name, 'mydata')
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(), id=2, name=u'mydata2', data2=123)
dataId2 = result.inserted_primary_key[0]
# Change column type in model.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',String(255),nullable=True),
)
# XXX test type diff
return
assertDiff(True, [], [], [self.table_name])
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
# Make sure data is still present.
result = self.engine.execute(self.table.select(self.table.c.id==dataId2))
rows = result.fetchall()
self.assertEquals(len(rows), 1)
self.assertEquals(rows[0].name, 'mydata2')
self.assertEquals(rows[0].data2, '123')
# Delete data, since we're about to make a required column.
# Not even using sqlalchemy.PassiveDefault helps because we're doing explicit column select.
self.engine.execute(self.table.delete(), id=dataId)
if not self.engine.name == 'firebird':
# Change column nullable in model.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',String(255),nullable=False),
)
assertDiff(True, [], [], [self.table_name]) # TODO test nullable diff
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Remove table from model.
self.meta.remove(self.table)
assertDiff(True, [], [self.table_name], [])
def _applyLatestModel(self):
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
genmodel.ModelGenerator(diff,self.engine).runB2A()
def schemadiff(self, excludeTables=None):
from migrate.versioning import schemadiff
return schemadiff.getDiffOfModelAgainstDatabase(
self.metadata, self.engine, excludeTables=excludeTables)
def _run_diff(self,**kw):
return schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, **kw
)
def _applyLatestModel(self):
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
genmodel.ModelGenerator(diff, self.engine).runB2A()
def update_pylons_db_from_model(url, model_str, commit=False):
"""update_pylons_db_from_model URL MODEL_STR COMMIT
Modify the database to match the structure of the current Pylons
model.
NOTE: This is EXPERIMENTAL.
""" # TODO: get rid of EXPERIMENTAL label
global _debug_messages
# engine = opts.pop('engine') # relies on @with_engine, and url being first argument
engine = construct_engine(url) # direct approach
if _debug_messages:
print "engine= ", engine
if _debug_messages:
print "sqlalchemy-migrate-pylons : engine created"
model_metadata = load_model(model_str)
if _debug_messages:
print "sqlalchemy-migrate-pylons : model metadata loaded from model_str"
#schema.MetaData(engine).bind=model_metadata
#print " Meta Internal: ", model.metadata, "\n"
#print " Meta Migrate : ", sqlalchemy.MetaData(engine), "\n"
#model.metadata=sqlalchemy.MetaData(engine)
#model_metadata = declarative_base(metadata=sqlalchemy.MetaData(engine))
#model_metadata.DBsession.configure(bind=engine, autocommit=False)
#model_metadata.engine = engine
#model_metadata.reflect(bind=engine)
#declarative_base(metadata=model_metadata)
#sqlalchemy.MetaData(engine).bind = engine
diff = schemadiff.getDiffOfModelAgainstDatabase(model_metadata, engine)
print "\n====== sqlalchemy-migrate-pylons : Model vs. Database ======\n", diff, "\n"
if commit:
# genmodel.ModelGenerator(diff).applyModel()
# print "\n Engine in Diffs : ", diff.conn.engine, "\n"
#print " Meta Internal: ", model.session, "\n"
#print " Meta Migrate : ", sqlalchemy.MetaData(diff.conn.engine), "\n"
#meta = model.metadata
#sqlalchemy.MetaData(diff.conn.engine).bind = engine
#meta.DBsession.configure(bind=engine, autocommit=False)
if hasattr(diff, 'tablesMissingInDatabase'):
print "MISSING : ", diff.tablesMissingInDatabase
print "MISSING : ", sqlalchemy.sql.util.sort_tables(diff.tablesMissingInDatabase)
diff.tablesMissingInDatabase = sqlalchemy.sql.util.sort_tables(diff.tablesMissingInDatabase)
print "====== sqlalchemy-migrate-pylons : Table Creation order : ======\n", diff, "\n"
#sqlalchemy.MetaData(engine).bind = engine
if hasattr(diff, 'tablesMissingInDatabase'):
## Oldrelease...
newmodel = genmodel.ModelGenerator(diff, declarative=True)
#newmodel.reflect(bind=engine)
else:
# This is a change for new release...
newmodel = genmodel.ModelGenerator(diff, engine, declarative=True)
if hasattr(diff, 'tablesMissingInDatabase'):
## Oldrelease...
print "====== sqlalchemy-migrate-pylons : Different Model created ======\n"
print " New Model (in Python): \n\n", newmodel.toUpgradeDowngradePython()[0], "\n" # Show the Declarations
print "# Upgrade Code :\n", newmodel.toUpgradeDowngradePython()[1], "\n" # Show the Upgrade code
newmodel.applyModel()
print "====== sqlalchemy-migrate-pylons : Database Migrated ======\n"
diff = schemadiff.getDiffOfModelAgainstDatabase(model_metadata, engine)
print "\n====== sqlalchemy-migrate-pylons : Model vs. Database (after migration) ======\n", diff, "\n"
if isinstance(engine, Engine):
engine.dispose()
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
eq_(bool(diff), isDiff)
eq_( ([t.name for t in diff.tablesMissingInDatabase], [t.name for t in diff.tablesMissingInModel], [t.name for t in diff.tablesWithDiff]),
(tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff) )
def schemadiff(self, excludeTables=None):
from migrate.versioning import schemadiff
return schemadiff.getDiffOfModelAgainstDatabase(self.metadata,
self.engine,
excludeTables=excludeTables)
print 'The sqlalchemy-migrate Python module was not found.\nThis is required for the dbupdate functionallity'
traceback.print_exc()
return
from migrate.versioning.util import load_model
from migrate.versioning import genmodel, schemadiff
from migrate.changeset import schema
# Setup minimalistic, offline HF environment
hf.configtools.readConfigurationAndEnv()
hf.configtools.setupLogging('acquire_logging_cfg')
hf.module.importModuleClasses()
hf.database.connect(implicit_execution=True)
# calculate diff using sqlalchemy-migrate magic
diff = schemadiff.getDiffOfModelAgainstDatabase(hf.database.metadata, hf.database.engine)
if args.dry:
print "Dry run! Database will be unchanged"
# compatibility with newer versions of sqlalchemy-migrate
if not hasattr(diff, "tablesMissingInDatabase"):
diff.tablesMissingInDatabase = diff.tables_missing_from_B
if not hasattr(diff, "tablesMissingInModel"):
diff.tablesMissingInModel = diff.tables_missing_from_A
if not hasattr(diff, "tablesWithDiff"):
diff.tablesWithDiff = diff.tables_different.values()
#import pdb; pdb.set_trace()
# create missing tables
if len(diff.tablesMissingInDatabase) > 0:
generate_forms(app, db)
from dtg.forms import *
db.create_all()
if not SystemInfo.query.first():
initialize_db(db)
# setup secret key, do DB migrations
sys_info = SystemInfo.query.first()
app.secret_key = str(sys_info.secret_app_key)
if sys_info.db_rev != APP_DB_REV:
if not sys.dtg_do_upgrade:
raise Exception("Old database version, please backup your database and enable DB migrations.")
if sys_info.db_rev == 3:
dict(db.metadata.tables)["flash_message"].drop(db.engine)
db.create_all()
diff = schemadiff.getDiffOfModelAgainstDatabase(db.metadata, db.engine)
genmodel.ModelGenerator(diff, db.engine).runB2A()
sys_info = SystemInfo.query.first()
sys_info.db_rev = APP_DB_REV
db.session.commit()
@app.before_first_request
def init_background_jobs():
# setup background jobs
app.background_jobs = []
for JOB in JOBS:
job = JOB(app, NIGHTLY_RUNTIME)
app.background_jobs.append(job)
job.start()
def test_functional(self):
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel,
tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
eq_((diff.tables_missing_from_B, diff.tables_missing_from_A,
diff.tables_different.keys(), bool(diff)),
(tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff,
isDiff))
# Model is defined but database is empty.
assertDiff(True, [self.table_name], [], [])
# Check Python upgrade and downgrade of database from updated model.
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(
diff, self.engine).genB2AMigration()
# Feature test for a recent SQLa feature;
# expect different output in that case.
if repr(String()) == 'String()':
self.assertEqualsIgnoreWhitespace(
decls, '''
from migrate.changeset import schema
pre_meta = MetaData()
post_meta = MetaData()
tmp_schemadiff = Table('tmp_schemadiff', post_meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('name', UnicodeText),
Column('data', UnicodeText),
)
''')
else:
self.assertEqualsIgnoreWhitespace(
decls, '''
from migrate.changeset import schema
pre_meta = MetaData()
post_meta = MetaData()
tmp_schemadiff = Table('tmp_schemadiff', post_meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('name', UnicodeText(length=None)),
Column('data', UnicodeText(length=None)),
)
''')
# Create table in database, now model should match database.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Check Python code gen from database.
diff = schemadiff.getDiffOfModelAgainstDatabase(
MetaData(), self.engine, excludeTables=['migrate_version'])
src = genmodel.ModelGenerator(diff, self.engine).genBDefinition()
exec src in locals()
c1 = Table('tmp_schemadiff', self.meta, autoload=True).c
c2 = tmp_schemadiff.c
self.compare_columns_equal(c1, c2, ['type'])
# TODO: get rid of ignoring type
if not self.engine.name == 'oracle':
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(),
id=1,
name=u'mydata')
dataId = result.inserted_primary_key[0]
# Modify table in model (by removing it and adding it back to model)
# Drop column data, add columns data2 and data3.
self.meta.remove(self.table)
self.table = Table(
self.table_name,
self.meta,
Column('id', Integer(), primary_key=True),
Column('name', UnicodeText(length=None)),
Column('data2', Integer(), nullable=True),
Column('data3', Integer(), nullable=True),
)
assertDiff(True, [], [], [self.table_name])
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Drop column data3, add data4
self.meta.remove(self.table)
self.table = Table(
self.table_name,
self.meta,
Column('id', Integer(), primary_key=True),
Column('name', UnicodeText(length=None)),
Column('data2', Integer(), nullable=True),
Column('data4', Float(), nullable=True),
)
assertDiff(True, [], [], [self.table_name])
diff = schemadiff.getDiffOfModelAgainstDatabase(
self.meta, self.engine, excludeTables=['migrate_version'])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(
diff, self.engine).genB2AMigration(indent='')
# decls have changed since genBDefinition
exec decls in locals()
# migration commands expect a namespace containing migrate_engine
migrate_engine = self.engine
# run the migration up and down
exec upgradeCommands in locals()
assertDiff(False, [], [], [])
exec decls in locals()
exec downgradeCommands in locals()
assertDiff(True, [], [], [self.table_name])
exec decls in locals()
exec upgradeCommands in locals()
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
# Make sure data is still present.
result = self.engine.execute(
self.table.select(self.table.c.id == dataId))
rows = result.fetchall()
eq_(len(rows), 1)
eq_(rows[0].name, 'mydata')
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(),
id=2,
name=u'mydata2',
data2=123)
dataId2 = result.inserted_primary_key[0]
# Change column type in model.
self.meta.remove(self.table)
self.table = Table(
self.table_name,
self.meta,
Column('id', Integer(), primary_key=True),
Column('name', UnicodeText(length=None)),
Column('data2', String(255), nullable=True),
)
# XXX test type diff
return
assertDiff(True, [], [], [self.table_name])
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
# Make sure data is still present.
result = self.engine.execute(
self.table.select(self.table.c.id == dataId2))
rows = result.fetchall()
self.assertEquals(len(rows), 1)
self.assertEquals(rows[0].name, 'mydata2')
self.assertEquals(rows[0].data2, '123')
# Delete data, since we're about to make a required column.
# Not even using sqlalchemy.PassiveDefault helps because we're doing explicit column select.
self.engine.execute(self.table.delete(), id=dataId)
if not self.engine.name == 'firebird':
# Change column nullable in model.
self.meta.remove(self.table)
self.table = Table(
self.table_name,
self.meta,
Column('id', Integer(), primary_key=True),
Column('name', UnicodeText(length=None)),
Column('data2', String(255), nullable=False),
)
assertDiff(True, [], [],
[self.table_name]) # TODO test nullable diff
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Remove table from model.
self.meta.remove(self.table)
assertDiff(True, [], [self.table_name], [])
def test_rundiffs(self):
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
eq_(bool(diff), isDiff)
eq_( ([t.name for t in diff.tablesMissingInDatabase], [t.name for t in diff.tablesMissingInModel], [t.name for t in diff.tablesWithDiff]),
(tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff) )
# Model is defined but database is empty.
assertDiff(True, [self.table_name], [], [])
# Check Python upgrade and downgrade of database from updated model.
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(diff).toUpgradeDowngradePython()
self.assertEqualsIgnoreWhitespace(decls, '''
from migrate.changeset import schema
meta = MetaData()
tmp_schemadiff = Table('tmp_schemadiff', meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', UnicodeText(length=None)),
Column('data', UnicodeText(length=None)),
)
''')
self.assertEqualsIgnoreWhitespace(upgradeCommands,
'''meta.bind = migrate_engine
tmp_schemadiff.create()''')
self.assertEqualsIgnoreWhitespace(downgradeCommands,
'''meta.bind = migrate_engine
tmp_schemadiff.drop()''')
# Create table in database, now model should match database.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Check Python code gen from database.
diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), self.engine, excludeTables=['migrate_version'])
src = genmodel.ModelGenerator(diff).toPython()
exec src in locals()
c1 = Table('tmp_schemadiff', self.meta, autoload=True).c
c2 = tmp_schemadiff.c
self.compare_columns_equal(c1, c2, ['type'])
# TODO: get rid of ignoring type
if not self.engine.name == 'oracle':
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(), id=1, name=u'mydata')
if SQLA_06:
dataId = result.inserted_primary_key[0]
else:
dataId = result.last_inserted_ids()[0]
# Modify table in model (by removing it and adding it back to model) -- drop column data and add column data2.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',Integer(),nullable=True),
)
assertDiff(True, [], [], [self.table_name])
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
# Make sure data is still present.
result = self.engine.execute(self.table.select(self.table.c.id==dataId))
rows = result.fetchall()
eq_(len(rows), 1)
eq_(rows[0].name, 'mydata')
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(), id=2, name=u'mydata2', data2=123)
if SQLA_06:
dataId2 = result.inserted_primary_key[0]
else:
dataId2 = result.last_inserted_ids()[0]
# Change column type in model.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',String(255),nullable=True),
)
assertDiff(True, [], [], [self.table_name]) # TODO test type diff
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
if not self.engine.name == 'oracle':
# Make sure data is still present.
result = self.engine.execute(self.table.select(self.table.c.id==dataId2))
rows = result.fetchall()
self.assertEquals(len(rows), 1)
self.assertEquals(rows[0].name, 'mydata2')
self.assertEquals(rows[0].data2, '123')
# Delete data, since we're about to make a required column.
# Not even using sqlalchemy.PassiveDefault helps because we're doing explicit column select.
self.engine.execute(self.table.delete(), id=dataId)
if not self.engine.name == 'firebird':
# Change column nullable in model.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',String(255),nullable=False),
)
assertDiff(True, [], [], [self.table_name]) # TODO test nullable diff
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Remove table from model.
self.meta.remove(self.table)
assertDiff(True, [], [self.table_name], [])
def test_rundiffs(self):
# Yuck! We have to import from changeset to apply the monkey-patch to allow column adding/dropping.
from migrate.changeset import schema
def assertDiff(isDiff, tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff):
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
self.assertEquals(bool(diff), isDiff)
self.assertEquals( ([t.name for t in diff.tablesMissingInDatabase], [t.name for t in diff.tablesMissingInModel], [t.name for t in diff.tablesWithDiff]),
(tablesMissingInDatabase, tablesMissingInModel, tablesWithDiff) )
# Model is defined but database is empty.
assertDiff(True, [self.table_name], [], [])
# Check Python upgrade and downgrade of database from updated model.
diff = schemadiff.getDiffOfModelAgainstDatabase(self.meta, self.engine, excludeTables=['migrate_version'])
decls, upgradeCommands, downgradeCommands = genmodel.ModelGenerator(diff).toUpgradeDowngradePython()
self.assertEqualsIgnoreWhitespace(decls, '''
meta = MetaData(migrate_engine)
tmp_schemadiff = Table('tmp_schemadiff',meta,
Column('id',Integer(),primary_key=True,nullable=False),
Column('name',UnicodeText(length=None)),
Column('data',UnicodeText(length=None)),
)
''')
self.assertEqualsIgnoreWhitespace(upgradeCommands, '''tmp_schemadiff.create()''')
self.assertEqualsIgnoreWhitespace(downgradeCommands, '''tmp_schemadiff.drop()''')
# Create table in database, now model should match database.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Check Python code gen from database.
diff = schemadiff.getDiffOfModelAgainstDatabase(MetaData(), self.engine, excludeTables=['migrate_version'])
src = genmodel.ModelGenerator(diff).toPython()
src = src.replace(genmodel.HEADER, '')
self.assertEqualsIgnoreWhitespace(src, '''
tmp_schemadiff = Table('tmp_schemadiff',meta,
Column('id',Integer(),primary_key=True,nullable=False),
Column('name',Text(length=None,convert_unicode=False,assert_unicode=None)),
Column('data',Text(length=None,convert_unicode=False,assert_unicode=None)),
)
''')
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(), id=1, name=u'mydata')
dataId = result.last_inserted_ids()[0]
# Modify table in model (by removing it and adding it back to model) -- drop column data and add column data2.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',Integer(),nullable=True),
)
assertDiff(True, [], [], [self.table_name])
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Make sure data is still present.
result = self.engine.execute(self.table.select(self.table.c.id==dataId))
rows = result.fetchall()
self.assertEquals(len(rows), 1)
self.assertEquals(rows[0].name, 'mydata')
# Add data, later we'll make sure it's still present.
result = self.engine.execute(self.table.insert(), id=2, name=u'mydata2', data2=123)
dataId2 = result.last_inserted_ids()[0]
# Change column type in model.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',UnicodeText(),nullable=True),
)
assertDiff(True, [], [], [self.table_name]) # TODO test type diff
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Make sure data is still present.
result = self.engine.execute(self.table.select(self.table.c.id==dataId2))
rows = result.fetchall()
self.assertEquals(len(rows), 1)
self.assertEquals(rows[0].name, 'mydata2')
self.assertEquals(rows[0].data2, '123')
# Delete data, since we're about to make a required column.
# Not even using sqlalchemy.PassiveDefault helps because we're doing explicit column select.
self.engine.execute(self.table.delete(), id=dataId)
# Change column nullable in model.
self.meta.remove(self.table)
self.table = Table(self.table_name,self.meta,
Column('id',Integer(),primary_key=True),
Column('name',UnicodeText(length=None)),
Column('data2',UnicodeText(),nullable=False),
)
assertDiff(True, [], [], [self.table_name]) # TODO test nullable diff
# Apply latest model changes and find no more diffs.
self._applyLatestModel()
assertDiff(False, [], [], [])
# Remove table from model.
self.meta.remove(self.table)
assertDiff(True, [], [self.table_name], [])