def test_upgrade_imipTokens(self):
"""
Old-style canonical CUAs (urn:uuid:) are converted to new style (urn:x-uid:)
"""
# Load old schema and populate with data
yield self._loadOldSchema(self.upgradePath.child("v56.sql"))
# Add two tokens records crafted to simulate conflicting old-style and
# new style CUAs -- the result should be only the new-style copy.
txn = self.store.newTransaction("loadData")
yield iMIPTokenRecord.create(txn,
token="123",
organizer="urn:uuid:PLUGH",
attendee="mailto:[email protected]",
icaluid="XYZZY")
yield iMIPTokenRecord.create(txn,
token="456",
organizer="urn:x-uid:PLUGH",
attendee="mailto:[email protected]",
icaluid="XYZZY")
yield txn.commit()
upgrader = UpgradeDatabaseSchemaStep(self.store)
yield upgrader.databaseUpgrade()
txn = self.store.newTransaction("loadData")
tokens = yield iMIPTokenRecord.all(txn)
self.assertEqual(len(tokens), 1)
token = list(tokens)[0]
self.assertEqual(token.token, "456")
self.assertEqual(token.organizer, "urn:x-uid:PLUGH")
yield txn.commit()
def test_upgradeAvailability(self):
"""
Make sure that each old schema has a valid upgrade path to the current one.
"""
for dialect in (POSTGRES_DIALECT, ORACLE_DIALECT,):
upgrader = UpgradeDatabaseSchemaStep(None)
files = upgrader.scanForUpgradeFiles(dialect)
current_version = self._getSchemaVersion(upgrader.schemaLocation.child("current.sql"), "VERSION")
for child in upgrader.schemaLocation.child("old").child(dialect).globChildren("*.sql"):
old_version = self._getSchemaVersion(child, "VERSION")
upgrades = upgrader.determineUpgradeSequence(old_version, current_version, files, dialect)
self.assertNotEqual(len(upgrades), 0)
def test_scanUpgradeFiles(self):
upgrader = UpgradeDatabaseSchemaStep(None)
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema1")
files = upgrader.scanForUpgradeFiles("fake_dialect")
self.assertEqual(files,
[(3, 4, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql"))],
)
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema2")
files = upgrader.scanForUpgradeFiles("fake_dialect")
self.assertEqual(files,
[
(3, 4, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql")),
(3, 5, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_5.sql")),
(4, 5, upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql")),
]
)
def setUp(self):
TestCase.setUp(self)
test_upgrader = UpgradeDatabaseSchemaStep(None)
self.upgradePath = test_upgrader.schemaLocation.child("old").child(
POSTGRES_DIALECT)
self.currentVersion = self._getSchemaVersion(
test_upgrader.schemaLocation.child("current.sql"), "VERSION")
self.store = yield theStoreBuilder.buildStore(
self, {"push": StubNotifierFactory()}, enableJobProcessing=False)
def test_upgradeAvailability(self):
"""
Make sure that each old schema has a valid upgrade path to the current one.
"""
for dialect in (
POSTGRES_DIALECT,
ORACLE_DIALECT,
):
upgrader = UpgradeDatabaseSchemaStep(None)
files = upgrader.scanForUpgradeFiles(dialect)
current_version = self._getSchemaVersion(
upgrader.schemaLocation.child("current.sql"), "VERSION")
for child in upgrader.schemaLocation.child("old").child(
dialect).globChildren("*.sql"):
old_version = self._getSchemaVersion(child, "VERSION")
upgrades = upgrader.determineUpgradeSequence(
old_version, current_version, files, dialect)
self.assertNotEqual(len(upgrades), 0)
def test_upgrade_imipTokens(self):
"""
Old-style canonical CUAs (urn:uuid:) are converted to new style (urn:x-uid:)
"""
# Load old schema and populate with data
yield self._loadOldSchema(self.upgradePath.child("v56.sql"))
# Add two tokens records crafted to simulate conflicting old-style and
# new style CUAs -- the result should be only the new-style copy.
txn = self.store.newTransaction("loadData")
yield iMIPTokenRecord.create(
txn,
token="123",
organizer="urn:uuid:PLUGH",
attendee="mailto:[email protected]",
icaluid="XYZZY"
)
yield iMIPTokenRecord.create(
txn,
token="456",
organizer="urn:x-uid:PLUGH",
attendee="mailto:[email protected]",
icaluid="XYZZY"
)
yield txn.commit()
upgrader = UpgradeDatabaseSchemaStep(self.store)
yield upgrader.databaseUpgrade()
txn = self.store.newTransaction("loadData")
tokens = yield iMIPTokenRecord.all(txn)
self.assertEqual(len(tokens), 1)
token = list(tokens)[0]
self.assertEqual(token.token, "456")
self.assertEqual(token.organizer, "urn:x-uid:PLUGH")
yield txn.commit()
def test_scanUpgradeFiles(self):
upgrader = UpgradeDatabaseSchemaStep(None)
upgrader.schemaLocation = getModule(__name__).filePath.sibling(
"fake_schema1")
files = upgrader.scanForUpgradeFiles("fake_dialect")
self.assertEqual(
files,
[(3, 4, upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_3_to_4.sql"))],
)
upgrader.schemaLocation = getModule(__name__).filePath.sibling(
"fake_schema2")
files = upgrader.scanForUpgradeFiles("fake_dialect")
self.assertEqual(files, [
(3, 4, upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_3_to_4.sql")),
(3, 5, upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_3_to_5.sql")),
(4, 5, upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_4_to_5.sql")),
])
def test_upgrade_SCHEDULE_REPLY(self):
cal1 = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
BEGIN:VEVENT
UID:1234-5678
DTSTART:20071114T010000Z
DURATION:PT1H
DTSTAMP:20071114T000000Z
ATTENDEE:mailto:[email protected]
ATTENDEE:mailto:[email protected]
ORGANIZER:mailto:[email protected]
SUMMARY:Test
END:VEVENT
END:VCALENDAR
"""
# Load old schema and populate with data
schema = yield self._loadOldSchema(self.upgradePath.child("v49.sql"))
txn = self.store.newTransaction("loadData")
yield Insert({
schema.CALENDAR_HOME.RESOURCE_ID: 1,
schema.CALENDAR_HOME.OWNER_UID: "abcdefg",
}).on(txn)
yield Insert({
schema.CALENDAR.RESOURCE_ID: 2,
}).on(txn)
yield Insert({
schema.CALENDAR_OBJECT.RESOURCE_ID: 3,
schema.CALENDAR_OBJECT.CALENDAR_RESOURCE_ID: 2,
schema.CALENDAR_OBJECT.RESOURCE_NAME: "1.ics",
schema.CALENDAR_OBJECT.ICALENDAR_TEXT: cal1,
schema.CALENDAR_OBJECT.ICALENDAR_UID: "1234-5678",
schema.CALENDAR_OBJECT.ICALENDAR_TYPE: "VEVENT",
schema.CALENDAR_OBJECT.MD5: "md5-1234567890",
}).on(txn)
yield Insert({
schema.JOB.JOB_ID: 1,
schema.JOB.WORK_TYPE: "SCHEDULE_REPLY_WORK",
schema.JOB.NOT_BEFORE: datetime.utcnow(),
}).on(txn)
yield Insert({
schema.SCHEDULE_WORK.WORK_ID: 1,
schema.SCHEDULE_WORK.JOB_ID: 1,
schema.SCHEDULE_WORK.ICALENDAR_UID: "1234-5678",
schema.SCHEDULE_WORK.WORK_TYPE: "SCHEDULE_REPLY_WORK",
}).on(txn)
yield Insert({
schema.SCHEDULE_REPLY_WORK.WORK_ID: 1,
schema.SCHEDULE_REPLY_WORK.HOME_RESOURCE_ID: 1,
schema.SCHEDULE_REPLY_WORK.RESOURCE_ID: 3,
schema.SCHEDULE_REPLY_WORK.CHANGED_RIDS: None,
}).on(txn)
yield txn.commit()
# Try to upgrade and verify new version afterwards
upgrader = UpgradeDatabaseSchemaStep(self.store)
yield upgrader.databaseUpgrade()
new_version = yield self._loadVersion()
self.assertEqual(new_version, self.currentVersion)
txn = self.store.newTransaction("loadData")
jobs = yield Select(From=schema.JOB, ).on(txn)
schedules = yield Select(From=schema.SCHEDULE_WORK, ).on(txn)
replies = yield Select(From=schema.SCHEDULE_REPLY_WORK, ).on(txn)
self.assertEqual(len(jobs), 1)
self.assertEqual(len(schedules), 1)
self.assertEqual(len(replies), 1)
self.assertEqual(list(replies[0]), [
1,
1,
3,
None,
])
jobs = yield JobItem.all(txn)
self.assertEqual(len(jobs), 1)
work = yield jobs[0].workItem()
self.assertTrue(isinstance(work, ScheduleReplyWork))
workers = yield ScheduleWork.all(txn)
self.assertEqual(len(workers), 1)
self.assertEqual(workers[0].workType, "SCHEDULE_REPLY_WORK")
yield txn.commit()
def test_determineUpgradeSequence(self):
upgrader = UpgradeDatabaseSchemaStep(None)
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema1")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 4, files, "fake_dialect")
self.assertEqual(upgrades,
[upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql")],
)
self.assertRaises(RuntimeError, upgrader.determineUpgradeSequence, 3, 5, files, "fake_dialect")
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema2")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 5, files, "fake_dialect")
self.assertEqual(upgrades,
[upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_5.sql")]
)
upgrades = upgrader.determineUpgradeSequence(4, 5, files, "fake_dialect")
self.assertEqual(upgrades,
[upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql")]
)
upgrader.schemaLocation = getModule(__name__).filePath.sibling("fake_schema3")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 5, files, "fake_dialect")
self.assertEqual(upgrades,
[
upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_3_to_4.sql"),
upgrader.schemaLocation.child("upgrades").child("fake_dialect").child("upgrade_from_4_to_5.sql"),
]
)
def _dbSchemaUpgrades(self, child):
"""
This does a full DB test of all possible upgrade paths. For each old schema, it loads it into the DB
then runs the upgrade service. This ensures all the upgrade.sql files work correctly - at least for
postgres.
"""
store = yield theStoreBuilder.buildStore(
self, {"push": StubNotifierFactory()}
)
@inlineCallbacks
def _loadOldSchema(path):
"""
Use the postgres schema mechanism to do tests under a separate "namespace"
in postgres that we can quickly wipe clean afterwards.
"""
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("create schema test_dbUpgrades;")
yield startTxn.execSQL("set search_path to test_dbUpgrades;")
yield startTxn.execSQL(path.getContent())
yield startTxn.commit()
@inlineCallbacks
def _loadVersion():
startTxn = store.newTransaction("test_dbUpgrades")
new_version = yield startTxn.execSQL("select value from calendarserver where name = 'VERSION';")
yield startTxn.commit()
returnValue(int(new_version[0][0]))
@inlineCallbacks
def _loadSchemaFromDatabase():
startTxn = store.newTransaction("test_dbUpgrades")
schema = yield dumpSchema(startTxn, "Upgraded from %s" % (child.basename(),), "test_dbUpgrades")
yield startTxn.commit()
returnValue(schema)
@inlineCallbacks
def _unloadOldSchema():
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade;")
yield startTxn.commit()
@inlineCallbacks
def _cleanupOldSchema():
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade;")
yield startTxn.commit()
self.addCleanup(_cleanupOldSchema)
test_upgrader = UpgradeDatabaseSchemaStep(None)
expected_version = self._getSchemaVersion(test_upgrader.schemaLocation.child("current.sql"), "VERSION")
# Upgrade allowed
upgrader = UpgradeDatabaseSchemaStep(store)
yield _loadOldSchema(child)
yield upgrader.databaseUpgrade()
new_version = yield _loadVersion()
# Compare the upgraded schema with the expected current schema
new_schema = yield _loadSchemaFromDatabase()
currentSchema = schemaFromPath(test_upgrader.schemaLocation.child("current.sql"))
mismatched = currentSchema.compare(new_schema)
self.assertEqual(len(mismatched), 0, "Schema mismatch:\n" + "\n".join(mismatched))
yield _unloadOldSchema()
self.assertEqual(new_version, expected_version)
# Upgrade disallowed
upgrader = UpgradeDatabaseSchemaStep(store, failIfUpgradeNeeded=True)
yield _loadOldSchema(child)
old_version = yield _loadVersion()
try:
yield upgrader.databaseUpgrade()
except RuntimeError:
pass
except Exception:
self.fail("RuntimeError not raised")
else:
self.fail("RuntimeError not raised")
new_version = yield _loadVersion()
yield _unloadOldSchema()
self.assertEqual(old_version, new_version)
def _dbSchemaUpgrades(self, child):
"""
This does a full DB test of all possible upgrade paths. For each old schema, it loads it into the DB
then runs the upgrade service. This ensures all the upgrade.sql files work correctly - at least for
postgres.
"""
store = yield self.testStoreBuilder.buildStore(
self, {"push": StubNotifierFactory()}, enableJobProcessing=False
)
@inlineCallbacks
def _loadOldSchema(path):
"""
Use the postgres schema mechanism to do tests under a separate "namespace"
in postgres that we can quickly wipe clean afterwards.
"""
startTxn = store.newTransaction("test_dbUpgrades")
if startTxn.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("create schema test_dbUpgrades")
yield startTxn.execSQL("set search_path to test_dbUpgrades")
yield startTxn.execSQLBlock(path.getContent())
yield startTxn.commit()
@inlineCallbacks
def _loadVersion():
startTxn = store.newTransaction("test_dbUpgrades")
new_version = yield startTxn.execSQL("select value from calendarserver where name = 'VERSION'")
yield startTxn.commit()
returnValue(int(new_version[0][0]))
@inlineCallbacks
def _loadSchemaFromDatabase():
startTxn = store.newTransaction("test_dbUpgrades")
schema = yield dumpSchema(startTxn, "Upgraded from %s" % (child.basename(),), "test_dbUpgrades")
yield startTxn.commit()
returnValue(schema)
@inlineCallbacks
def _unloadOldSchema():
startTxn = store.newTransaction("test_dbUpgrades")
if startTxn.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("set search_path to public")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade")
elif startTxn.dialect == ORACLE_DIALECT:
yield cleanDatabase(startTxn)
yield startTxn.commit()
@inlineCallbacks
def _cleanupOldSchema():
startTxn = store.newTransaction("test_dbUpgrades")
if startTxn.dialect == POSTGRES_DIALECT:
yield startTxn.execSQL("set search_path to public")
yield startTxn.execSQL("drop schema if exists test_dbUpgrades cascade")
elif startTxn.dialect == ORACLE_DIALECT:
yield cleanDatabase(startTxn)
yield startTxn.commit()
self.addCleanup(_cleanupOldSchema)
test_upgrader = UpgradeDatabaseSchemaStep(None)
expected_version = self._getSchemaVersion(test_upgrader.schemaLocation.child(DB_TYPE[2]), "VERSION")
# Upgrade allowed
upgrader = UpgradeDatabaseSchemaStep(store)
yield _loadOldSchema(child)
yield upgrader.databaseUpgrade()
new_version = yield _loadVersion()
# Compare the upgraded schema with the expected current schema
new_schema = yield _loadSchemaFromDatabase()
currentSchema = schemaFromPath(test_upgrader.schemaLocation.child(DB_TYPE[2]))
mismatched = currentSchema.compare(new_schema)
# These are special case exceptions
for i in (
"Table: CALENDAR_HOME, column name DATAVERSION default mismatch",
"Table: CALENDAR_HOME, mismatched constraints: set([<Constraint: (NOT NULL ('DATAVERSION',) None)>])",
"Table: ADDRESSBOOK_HOME, column name DATAVERSION default mismatch",
"Table: ADDRESSBOOK_HOME, mismatched constraints: set([<Constraint: (NOT NULL ('DATAVERSION',) None)>])",
"Table: PUSH_NOTIFICATION_WORK, column name PUSH_PRIORITY default mismatch",
):
try:
mismatched.remove(i)
except ValueError:
pass
self.assertEqual(len(mismatched), 0, "Schema mismatch:\n" + "\n".join(mismatched))
yield _unloadOldSchema()
self.assertEqual(new_version, expected_version)
# Upgrade disallowed
upgrader = UpgradeDatabaseSchemaStep(store, failIfUpgradeNeeded=True)
yield _loadOldSchema(child)
old_version = yield _loadVersion()
try:
yield upgrader.databaseUpgrade()
except NotAllowedToUpgrade:
pass
except Exception:
self.fail("NotAllowedToUpgrade not raised")
else:
self.fail("NotAllowedToUpgrade not raised")
new_version = yield _loadVersion()
yield _unloadOldSchema()
self.assertEqual(old_version, new_version)
def test_upgrade_SCHEDULE_REPLY(self):
cal1 = """BEGIN:VCALENDAR
VERSION:2.0
PRODID:-//CALENDARSERVER.ORG//NONSGML Version 1//EN
BEGIN:VEVENT
UID:1234-5678
DTSTART:20071114T010000Z
DURATION:PT1H
DTSTAMP:20071114T000000Z
ATTENDEE:mailto:[email protected]
ATTENDEE:mailto:[email protected]
ORGANIZER:mailto:[email protected]
SUMMARY:Test
END:VEVENT
END:VCALENDAR
"""
# Load old schema and populate with data
schema = yield self._loadOldSchema(self.upgradePath.child("v49.sql"))
txn = self.store.newTransaction("loadData")
yield Insert(
{
schema.CALENDAR_HOME.RESOURCE_ID: 1,
schema.CALENDAR_HOME.OWNER_UID: "abcdefg",
}
).on(txn)
yield Insert(
{
schema.CALENDAR.RESOURCE_ID: 2,
}
).on(txn)
yield Insert(
{
schema.CALENDAR_OBJECT.RESOURCE_ID: 3,
schema.CALENDAR_OBJECT.CALENDAR_RESOURCE_ID: 2,
schema.CALENDAR_OBJECT.RESOURCE_NAME: "1.ics",
schema.CALENDAR_OBJECT.ICALENDAR_TEXT: cal1,
schema.CALENDAR_OBJECT.ICALENDAR_UID: "1234-5678",
schema.CALENDAR_OBJECT.ICALENDAR_TYPE: "VEVENT",
schema.CALENDAR_OBJECT.MD5: "md5-1234567890",
}
).on(txn)
yield Insert(
{
schema.JOB.JOB_ID: 1,
schema.JOB.WORK_TYPE: "SCHEDULE_REPLY_WORK",
schema.JOB.NOT_BEFORE: datetime.utcnow(),
}
).on(txn)
yield Insert(
{
schema.SCHEDULE_WORK.WORK_ID: 1,
schema.SCHEDULE_WORK.JOB_ID: 1,
schema.SCHEDULE_WORK.ICALENDAR_UID: "1234-5678",
schema.SCHEDULE_WORK.WORK_TYPE: "SCHEDULE_REPLY_WORK",
}
).on(txn)
yield Insert(
{
schema.SCHEDULE_REPLY_WORK.WORK_ID: 1,
schema.SCHEDULE_REPLY_WORK.HOME_RESOURCE_ID: 1,
schema.SCHEDULE_REPLY_WORK.RESOURCE_ID: 3,
schema.SCHEDULE_REPLY_WORK.CHANGED_RIDS: None,
}
).on(txn)
yield txn.commit()
# Try to upgrade and verify new version afterwards
upgrader = UpgradeDatabaseSchemaStep(self.store)
yield upgrader.databaseUpgrade()
new_version = yield self._loadVersion()
self.assertEqual(new_version, self.currentVersion)
txn = self.store.newTransaction("loadData")
jobs = yield Select(
From=schema.JOB,
).on(txn)
schedules = yield Select(
From=schema.SCHEDULE_WORK,
).on(txn)
replies = yield Select(
From=schema.SCHEDULE_REPLY_WORK,
).on(txn)
self.assertEqual(len(jobs), 1)
self.assertEqual(len(schedules), 1)
self.assertEqual(len(replies), 1)
self.assertEqual(list(replies[0]), [1, 1, 3, None, ])
jobs = yield JobItem.all(txn)
self.assertEqual(len(jobs), 1)
work = yield jobs[0].workItem()
self.assertTrue(isinstance(work, ScheduleReplyWork))
workers = yield ScheduleWork.all(txn)
self.assertEqual(len(workers), 1)
self.assertEqual(workers[0].workType, "SCHEDULE_REPLY_WORK")
yield txn.commit()
def test_determineUpgradeSequence(self):
upgrader = UpgradeDatabaseSchemaStep(None)
upgrader.schemaLocation = getModule(__name__).filePath.sibling(
"fake_schema1")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 4, files,
"fake_dialect")
self.assertEqual(
upgrades,
[
upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_3_to_4.sql")
],
)
self.assertRaises(RuntimeError, upgrader.determineUpgradeSequence, 3,
5, files, "fake_dialect")
upgrader.schemaLocation = getModule(__name__).filePath.sibling(
"fake_schema2")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 5, files,
"fake_dialect")
self.assertEqual(upgrades, [
upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_3_to_5.sql")
])
upgrades = upgrader.determineUpgradeSequence(4, 5, files,
"fake_dialect")
self.assertEqual(upgrades, [
upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_4_to_5.sql")
])
upgrader.schemaLocation = getModule(__name__).filePath.sibling(
"fake_schema3")
files = upgrader.scanForUpgradeFiles("fake_dialect")
upgrades = upgrader.determineUpgradeSequence(3, 5, files,
"fake_dialect")
self.assertEqual(upgrades, [
upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_3_to_4.sql"),
upgrader.schemaLocation.child("upgrades").child(
"fake_dialect").child("upgrade_from_4_to_5.sql"),
])
def _dbDataUpgrades(self, version, versionKey, upgraderClass):
"""
This does a full DB test of all possible data upgrade paths. For each old schema, it loads it into the DB
then runs the data upgrade service. This ensures all the upgrade_XX.py files work correctly - at least for
postgres.
TODO: this currently does not create any data to test with. It simply runs the upgrade on an empty
store.
"""
store = yield theStoreBuilder.buildStore(
self, {"push": StubNotifierFactory()}, enableJobProcessing=False)
@inlineCallbacks
def _loadOldData(path, oldVersion):
"""
Use the postgres schema mechanism to do tests under a separate "namespace"
in postgres that we can quickly wipe clean afterwards.
"""
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("create schema test_dbUpgrades;")
yield startTxn.execSQL("set search_path to test_dbUpgrades;")
yield startTxn.execSQL(path.getContent())
yield startTxn.execSQL(
"update CALENDARSERVER set VALUE = '%s' where NAME = '%s';" % (
oldVersion,
versionKey,
))
yield startTxn.commit()
@inlineCallbacks
def _loadVersion():
startTxn = store.newTransaction("test_dbUpgrades")
new_version = yield startTxn.execSQL(
"select value from calendarserver where name = '%s';" %
(versionKey, ))
yield startTxn.commit()
returnValue(int(new_version[0][0]))
@inlineCallbacks
def _unloadOldData():
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade;")
yield startTxn.commit()
@inlineCallbacks
def _cleanupOldData():
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL(
"drop schema if exists test_dbUpgrades cascade;")
yield startTxn.commit()
self.addCleanup(_cleanupOldData)
test_upgrader = UpgradeDatabaseSchemaStep(None)
expected_version = self._getSchemaVersion(
test_upgrader.schemaLocation.child("current.sql"), versionKey)
oldVersion = version
upgrader = upgraderClass(store)
yield _loadOldData(test_upgrader.schemaLocation.child("current.sql"),
oldVersion)
yield upgrader.databaseUpgrade()
new_version = yield _loadVersion()
yield _unloadOldData()
self.assertEqual(new_version, expected_version)
def _dbSchemaUpgrades(self, child):
"""
This does a full DB test of all possible upgrade paths. For each old schema, it loads it into the DB
then runs the upgrade service. This ensures all the upgrade.sql files work correctly - at least for
postgres.
"""
store = yield theStoreBuilder.buildStore(
self, {"push": StubNotifierFactory()}, enableJobProcessing=False)
@inlineCallbacks
def _loadOldSchema(path):
"""
Use the postgres schema mechanism to do tests under a separate "namespace"
in postgres that we can quickly wipe clean afterwards.
"""
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("create schema test_dbUpgrades;")
yield startTxn.execSQL("set search_path to test_dbUpgrades;")
yield startTxn.execSQL(path.getContent())
yield startTxn.commit()
@inlineCallbacks
def _loadVersion():
startTxn = store.newTransaction("test_dbUpgrades")
new_version = yield startTxn.execSQL(
"select value from calendarserver where name = 'VERSION';")
yield startTxn.commit()
returnValue(int(new_version[0][0]))
@inlineCallbacks
def _loadSchemaFromDatabase():
startTxn = store.newTransaction("test_dbUpgrades")
schema = yield dumpSchema(
startTxn, "Upgraded from %s" % (child.basename(), ),
"test_dbUpgrades")
yield startTxn.commit()
returnValue(schema)
@inlineCallbacks
def _unloadOldSchema():
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL("drop schema test_dbUpgrades cascade;")
yield startTxn.commit()
@inlineCallbacks
def _cleanupOldSchema():
startTxn = store.newTransaction("test_dbUpgrades")
yield startTxn.execSQL("set search_path to public;")
yield startTxn.execSQL(
"drop schema if exists test_dbUpgrades cascade;")
yield startTxn.commit()
self.addCleanup(_cleanupOldSchema)
test_upgrader = UpgradeDatabaseSchemaStep(None)
expected_version = self._getSchemaVersion(
test_upgrader.schemaLocation.child("current.sql"), "VERSION")
# Upgrade allowed
upgrader = UpgradeDatabaseSchemaStep(store)
yield _loadOldSchema(child)
yield upgrader.databaseUpgrade()
new_version = yield _loadVersion()
# Compare the upgraded schema with the expected current schema
new_schema = yield _loadSchemaFromDatabase()
currentSchema = schemaFromPath(
test_upgrader.schemaLocation.child("current.sql"))
mismatched = currentSchema.compare(new_schema)
# These are special case exceptions
for i in (
"Table: CALENDAR_HOME, column name DATAVERSION default mismatch",
"Table: ADDRESSBOOK_HOME, column name DATAVERSION default mismatch",
"Table: PUSH_NOTIFICATION_WORK, column name PUSH_PRIORITY default mismatch",
):
try:
mismatched.remove(i)
except ValueError:
pass
self.assertEqual(len(mismatched), 0,
"Schema mismatch:\n" + "\n".join(mismatched))
yield _unloadOldSchema()
self.assertEqual(new_version, expected_version)
# Upgrade disallowed
upgrader = UpgradeDatabaseSchemaStep(store, failIfUpgradeNeeded=True)
yield _loadOldSchema(child)
old_version = yield _loadVersion()
try:
yield upgrader.databaseUpgrade()
except NotAllowedToUpgrade:
pass
except Exception:
self.fail("NotAllowedToUpgrade not raised")
else:
self.fail("NotAllowedToUpgrade not raised")
new_version = yield _loadVersion()
yield _unloadOldSchema()
self.assertEqual(old_version, new_version)
test_upgrader = UpgradeDatabaseSchemaStep(None)
expected_version = self._getSchemaVersion(
test_upgrader.schemaLocation.child("current.sql"), versionKey)
oldVersion = version
upgrader = upgraderClass(store)
yield _loadOldData(test_upgrader.schemaLocation.child("current.sql"),
oldVersion)
yield upgrader.databaseUpgrade()
new_version = yield _loadVersion()
yield _unloadOldData()
self.assertEqual(new_version, expected_version)
test_upgrader = UpgradeDatabaseSchemaStep(None)
# Bind test methods for each schema version
for child in test_upgrader.schemaLocation.child("old").child(
POSTGRES_DIALECT).globChildren("*.sql"):
def f(self, lchild=child):
return self._dbSchemaUpgrades(lchild)
setattr(SchemaUpgradeTests,
"test_dbSchemaUpgrades_%s" % (child.basename().split(".", 1)[0], ),
f)
# Bind test methods for each addressbook data version
versions = set()
for child in test_upgrader.schemaLocation.child("old").child(