def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
def setUp(self):
"""
Set up two stores to migrate between.
"""
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, TestStoreDirectoryService(), True, True
)
self.sqlStore = yield theStoreBuilder.buildStore(
self, StubNotifierFactory()
)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
def setUp(self):
"""
Set up two stores to migrate between.
"""
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, TestStoreDirectoryService(), True, True
)
self.sqlStore = yield theStoreBuilder.buildStore(
self, StubNotifierFactory()
)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
# Add some properties we want to check get migrated over
txn = self.fileStore.newTransaction()
home = yield txn.calendarHomeWithUID("home_defaults")
cal = yield home.calendarWithName("calendar_1")
props = cal.properties()
props[PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)] = caldavxml.SupportedCalendarComponentSet(
caldavxml.CalendarComponent(name="VEVENT"),
caldavxml.CalendarComponent(name="VTODO"),
)
props[PropertyName.fromElement(element.ResourceType)] = element.ResourceType(
element.Collection(),
caldavxml.Calendar(),
)
props[PropertyName.fromElement(customxml.GETCTag)] = customxml.GETCTag.fromString("foobar")
inbox = yield home.calendarWithName("inbox")
props = inbox.properties()
props[PropertyName.fromElement(customxml.CalendarAvailability)] = customxml.CalendarAvailability.fromString(str(self.av1))
props[PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)] = caldavxml.ScheduleDefaultCalendarURL(
element.HRef.fromString("/calendars/__uids__/home_defaults/calendar_1"),
)
yield txn.commit()
def setUp(self):
"""
Set up two stores to migrate between.
"""
yield super(HomeMigrationTests, self).setUp()
yield self.buildStoreAndDirectory(extraUids=(
u"home1",
u"home2",
u"home3",
u"home_defaults",
u"home_no_splits",
u"home_splits",
u"home_splits_shared",
))
self.sqlStore = self.store
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, self.directory,
True, True)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child("__uids__").child("ho").child(
"me").child("home1").child(".some-extra-data").setContent(
"some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child("__uids__").child(
"ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
# Add some properties we want to check get migrated over
txn = self.fileStore.newTransaction()
home = yield txn.calendarHomeWithUID("home_defaults")
cal = yield home.calendarWithName("calendar_1")
props = cal.properties()
props[PropertyName.fromElement(
caldavxml.SupportedCalendarComponentSet
)] = caldavxml.SupportedCalendarComponentSet(
caldavxml.CalendarComponent(name="VEVENT"),
caldavxml.CalendarComponent(name="VTODO"),
)
props[PropertyName.fromElement(
element.ResourceType)] = element.ResourceType(
element.Collection(),
caldavxml.Calendar(),
)
props[PropertyName.fromElement(
customxml.GETCTag)] = customxml.GETCTag.fromString("foobar")
inbox = yield home.calendarWithName("inbox")
props = inbox.properties()
props[PropertyName.fromElement(
customxml.CalendarAvailability
)] = customxml.CalendarAvailability.fromString(str(self.av1))
props[PropertyName.fromElement(
caldavxml.ScheduleDefaultCalendarURL
)] = caldavxml.ScheduleDefaultCalendarURL(
element.HRef.fromString(
"/calendars/__uids__/home_defaults/calendar_1"), )
yield txn.commit()
class HomeMigrationTests(CommonCommonTests, TestCase):
"""
Tests for L{UpgradeToDatabaseStep}.
"""
av1 = Component.fromString("""BEGIN:VCALENDAR
VERSION:2.0
CALSCALE:GREGORIAN
PRODID:-//calendarserver.org//Zonal//EN
BEGIN:VAVAILABILITY
ORGANIZER:mailto:[email protected]
UID:[email protected]
DTSTAMP:20061005T133225Z
DTEND:20140101T000000Z
BEGIN:AVAILABLE
UID:[email protected]
DTSTAMP:20061005T133225Z
SUMMARY:Monday to Friday from 9:00 to 17:00
DTSTART:20130101T090000Z
DTEND:20130101T170000Z
RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
END:AVAILABLE
END:VAVAILABILITY
END:VCALENDAR
""")
@inlineCallbacks
def setUp(self):
"""
Set up two stores to migrate between.
"""
yield super(HomeMigrationTests, self).setUp()
yield self.buildStoreAndDirectory(extraUids=(
u"home1",
u"home2",
u"home3",
u"home_defaults",
u"home_no_splits",
u"home_splits",
u"home_splits_shared",
))
self.sqlStore = self.store
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, self.directory,
True, True)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child("__uids__").child("ho").child(
"me").child("home1").child(".some-extra-data").setContent(
"some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child("__uids__").child(
"ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
# Add some properties we want to check get migrated over
txn = self.fileStore.newTransaction()
home = yield txn.calendarHomeWithUID("home_defaults")
cal = yield home.calendarWithName("calendar_1")
props = cal.properties()
props[PropertyName.fromElement(
caldavxml.SupportedCalendarComponentSet
)] = caldavxml.SupportedCalendarComponentSet(
caldavxml.CalendarComponent(name="VEVENT"),
caldavxml.CalendarComponent(name="VTODO"),
)
props[PropertyName.fromElement(
element.ResourceType)] = element.ResourceType(
element.Collection(),
caldavxml.Calendar(),
)
props[PropertyName.fromElement(
customxml.GETCTag)] = customxml.GETCTag.fromString("foobar")
inbox = yield home.calendarWithName("inbox")
props = inbox.properties()
props[PropertyName.fromElement(
customxml.CalendarAvailability
)] = customxml.CalendarAvailability.fromString(str(self.av1))
props[PropertyName.fromElement(
caldavxml.ScheduleDefaultCalendarURL
)] = caldavxml.ScheduleDefaultCalendarURL(
element.HRef.fromString(
"/calendars/__uids__/home_defaults/calendar_1"), )
yield txn.commit()
def mergeRequirements(self, a, b):
"""
Merge two requirements dictionaries together, modifying C{a} and
returning it.
@param a: Some requirements, in the format of
L{CommonTests.requirements}.
@type a: C{dict}
@param b: Some additional requirements, to be merged into C{a}.
@type b: C{dict}
@return: C{a}
@rtype: C{dict}
"""
for homeUID in b:
homereq = a.setdefault(homeUID, {})
homeExtras = b[homeUID]
for calendarUID in homeExtras:
calreq = homereq.setdefault(calendarUID, {})
calendarExtras = homeExtras[calendarUID]
calreq.update(calendarExtras)
return a
@withSpecialValue(
"extraRequirements", {
"home1": {
"calendar_1": {
"bogus.ics":
(getModule("twistedcaldav").filePath.sibling("zoneinfo").
child("EST.ics").getContent(), CommonTests.metadata1)
}
}
})
@inlineCallbacks
def test_unknownTypeNotMigrated(self):
"""
The only types of calendar objects that should get migrated are VEVENTs
and VTODOs. Other component types, such as free-standing VTIMEZONEs,
don't have a UID and can't be stored properly in the database, so they
should not be migrated.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
self.assertIdentical(
None, (yield
(yield
(yield
(yield txn.calendarHomeWithUID("home1")).calendarWithName(
"calendar_1"))).calendarObjectWithName("bogus.ics")))
@inlineCallbacks
def test_upgradeCalendarHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
# Create a fake directory in the same place as a home, but with a non-existent uid
fake_dir = self.filesPath.child("calendars").child("__uids__").child(
"ho").child("me").child("foobar")
fake_dir.makedirs()
# Create a fake file in the same place as a home,with a name that matches the hash uid prefix
fake_file = self.filesPath.child("calendars").child("__uids__").child(
"ho").child("me").child("home_file")
fake_file.setContent("")
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(None,
(yield txn.calendarHomeWithUID(uid)))
# Successfully migrated calendar homes are deleted
self.assertFalse(
self.filesPath.child("calendars").child("__uids__").child(
"ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
for name, metadata, md5 in (
("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
):
object = (yield calendar.calendarObjectWithName(name))
self.assertEquals(object.getMetadata(), metadata)
self.assertEquals(object.md5(), md5)
@withSpecialValue("extraRequirements", {"nonexistent": {"calendar_1": {}}})
@inlineCallbacks
def test_upgradeCalendarHomesMissingDirectoryRecord(self):
"""
Test an upgrade where a directory record is missing for a home;
the original home directory will remain on disk.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(None,
(yield txn.calendarHomeWithUID(uid)))
self.assertIdentical(None,
(yield txn.calendarHomeWithUID(u"nonexistent")))
# Skipped calendar homes are not deleted
self.assertTrue(
self.filesPath.child("calendars").child("__uids__").child(
"no").child("ne").child("nonexistent").exists())
@inlineCallbacks
def test_upgradeExistingHome(self):
"""
L{UpgradeToDatabaseService.startService} will skip migrating existing
homes.
"""
startTxn = self.sqlStore.newTransaction("populate empty sample")
yield startTxn.calendarHomeWithUID("home1", create=True)
yield startTxn.commit()
yield self.upgrader.stepWithResult(None)
vrfyTxn = self.sqlStore.newTransaction("verify sample still empty")
self.addCleanup(vrfyTxn.commit)
home = yield vrfyTxn.calendarHomeWithUID("home1")
# The default calendar is still there.
self.assertNotIdentical(None,
(yield home.calendarWithName("calendar")))
# The migrated calendar isn't.
self.assertIdentical(None, (yield home.calendarWithName("calendar_1")))
@inlineCallbacks
def test_upgradeAttachments(self):
"""
L{UpgradeToDatabaseService.startService} upgrades calendar attachments
as well.
"""
# Need to tweak config and settings to setup dropbox to work
self.patch(config, "EnableDropBox", True)
self.patch(config, "EnableManagedAttachments", False)
self.sqlStore.enableManagedAttachments = False
txn = self.sqlStore.newTransaction()
cs = schema.CALENDARSERVER
yield Delete(From=cs, Where=cs.NAME == "MANAGED-ATTACHMENTS").on(txn)
yield txn.commit()
txn = self.fileStore.newTransaction()
committed = []
def maybeCommit():
if not committed:
committed.append(True)
return txn.commit()
self.addCleanup(maybeCommit)
@inlineCallbacks
def getSampleObj():
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
object = (yield calendar.calendarObjectWithName("1.ics"))
returnValue(object)
inObject = yield getSampleObj()
someAttachmentName = "some-attachment"
someAttachmentType = MimeType.fromString("application/x-custom-type")
attachment = yield inObject.createAttachmentWithName(
someAttachmentName, )
transport = attachment.store(someAttachmentType)
someAttachmentData = "Here is some data for your attachment, enjoy."
transport.write(someAttachmentData)
yield transport.loseConnection()
yield maybeCommit()
yield self.upgrader.stepWithResult(None)
committed = []
txn = self.sqlStore.newTransaction()
outObject = yield getSampleObj()
outAttachment = yield outObject.attachmentWithName(someAttachmentName)
allDone = Deferred()
class SimpleProto(Protocol):
data = ''
def dataReceived(self, data):
self.data += data
def connectionLost(self, reason):
allDone.callback(self.data)
self.assertEquals(outAttachment.contentType(), someAttachmentType)
outAttachment.retrieve(SimpleProto())
allData = yield allDone
self.assertEquals(allData, someAttachmentData)
@inlineCallbacks
def test_upgradeAddressBookHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in ABCommonTests.requirements:
if ABCommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.addressbookHomeWithUID(uid)))
# Successfully migrated addressbook homes are deleted
self.assertFalse(
self.filesPath.child("addressbooks").child("__uids__").child(
"ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.addressbookHomeWithUID("home1"))
adbk = (yield home.addressbookWithName("addressbook"))
for name, md5 in (
("1.vcf", ABCommonTests.md5Values[0]),
("2.vcf", ABCommonTests.md5Values[1]),
("3.vcf", ABCommonTests.md5Values[2]),
):
object = (yield adbk.addressbookObjectWithName(name))
self.assertEquals(object.md5(), md5)
@inlineCallbacks
def test_upgradeProperties(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home_defaults"))
cal = (yield home.calendarWithName("calendar_1"))
inbox = (yield home.calendarWithName("inbox"))
# Supported components
self.assertEqual(cal.getSupportedComponents(), "VEVENT")
self.assertTrue(cal.properties().get(
PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet))
is None)
# Resource type removed
self.assertTrue(cal.properties().get(
PropertyName.fromElement(element.ResourceType)) is None)
# Ctag removed
self.assertTrue(cal.properties().get(
PropertyName.fromElement(customxml.GETCTag)) is None)
# Availability
self.assertEquals(str(home.getAvailability()), str(self.av1))
self.assertTrue(inbox.properties().get(
PropertyName.fromElement(customxml.CalendarAvailability)) is None)
# Default calendar
self.assertTrue(home.isDefaultCalendar(cal))
self.assertTrue(inbox.properties().get(
PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)) is
None)
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
class HomeMigrationTests(CommonCommonTests, TestCase):
"""
Tests for L{UpgradeToDatabaseStep}.
"""
av1 = Component.fromString("""BEGIN:VCALENDAR
VERSION:2.0
CALSCALE:GREGORIAN
PRODID:-//calendarserver.org//Zonal//EN
BEGIN:VAVAILABILITY
ORGANIZER:mailto:[email protected]
UID:[email protected]
DTSTAMP:20061005T133225Z
DTEND:20140101T000000Z
BEGIN:AVAILABLE
UID:[email protected]
DTSTAMP:20061005T133225Z
SUMMARY:Monday to Friday from 9:00 to 17:00
DTSTART:20130101T090000Z
DTEND:20130101T170000Z
RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
END:AVAILABLE
END:VAVAILABILITY
END:VCALENDAR
""")
@inlineCallbacks
def setUp(self):
"""
Set up two stores to migrate between.
"""
yield super(HomeMigrationTests, self).setUp()
yield self.buildStoreAndDirectory(
extraUids=(
u"home1",
u"home2",
u"home3",
u"home_defaults",
u"home_no_splits",
u"home_splits",
u"home_splits_shared",
)
)
self.sqlStore = self.store
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, self.directory, True, True
)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
# Add some properties we want to check get migrated over
txn = self.fileStore.newTransaction()
home = yield txn.calendarHomeWithUID("home_defaults")
cal = yield home.calendarWithName("calendar_1")
props = cal.properties()
props[PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)] = caldavxml.SupportedCalendarComponentSet(
caldavxml.CalendarComponent(name="VEVENT"),
caldavxml.CalendarComponent(name="VTODO"),
)
props[PropertyName.fromElement(element.ResourceType)] = element.ResourceType(
element.Collection(),
caldavxml.Calendar(),
)
props[PropertyName.fromElement(customxml.GETCTag)] = customxml.GETCTag.fromString("foobar")
inbox = yield home.calendarWithName("inbox")
props = inbox.properties()
props[PropertyName.fromElement(customxml.CalendarAvailability)] = customxml.CalendarAvailability.fromString(str(self.av1))
props[PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)] = caldavxml.ScheduleDefaultCalendarURL(
element.HRef.fromString("/calendars/__uids__/home_defaults/calendar_1"),
)
yield txn.commit()
def mergeRequirements(self, a, b):
"""
Merge two requirements dictionaries together, modifying C{a} and
returning it.
@param a: Some requirements, in the format of
L{CommonTests.requirements}.
@type a: C{dict}
@param b: Some additional requirements, to be merged into C{a}.
@type b: C{dict}
@return: C{a}
@rtype: C{dict}
"""
for homeUID in b:
homereq = a.setdefault(homeUID, {})
homeExtras = b[homeUID]
for calendarUID in homeExtras:
calreq = homereq.setdefault(calendarUID, {})
calendarExtras = homeExtras[calendarUID]
calreq.update(calendarExtras)
return a
@withSpecialValue(
"extraRequirements",
{
"home1": {
"calendar_1": {
"bogus.ics": (
getModule("twistedcaldav").filePath.sibling("zoneinfo")
.child("EST.ics").getContent(),
CommonTests.metadata1
)
}
}
}
)
@inlineCallbacks
def test_unknownTypeNotMigrated(self):
"""
The only types of calendar objects that should get migrated are VEVENTs
and VTODOs. Other component types, such as free-standing VTIMEZONEs,
don't have a UID and can't be stored properly in the database, so they
should not be migrated.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
self.assertIdentical(
None,
(yield (yield (yield (
yield txn.calendarHomeWithUID("home1")
).calendarWithName("calendar_1"))
).calendarObjectWithName("bogus.ics"))
)
@inlineCallbacks
def test_upgradeCalendarHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
# Create a fake directory in the same place as a home, but with a non-existent uid
fake_dir = self.filesPath.child("calendars").child("__uids__").child("ho").child("me").child("foobar")
fake_dir.makedirs()
# Create a fake file in the same place as a home,with a name that matches the hash uid prefix
fake_file = self.filesPath.child("calendars").child("__uids__").child("ho").child("me").child("home_file")
fake_file.setContent("")
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.calendarHomeWithUID(uid))
)
# Successfully migrated calendar homes are deleted
self.assertFalse(self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
for name, metadata, md5 in (
("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
):
object = (yield calendar.calendarObjectWithName(name))
self.assertEquals(object.getMetadata(), metadata)
self.assertEquals(object.md5(), md5)
@withSpecialValue(
"extraRequirements",
{
"nonexistent": {
"calendar_1": {
}
}
}
)
@inlineCallbacks
def test_upgradeCalendarHomesMissingDirectoryRecord(self):
"""
Test an upgrade where a directory record is missing for a home;
the original home directory will remain on disk.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.calendarHomeWithUID(uid))
)
self.assertIdentical(None, (yield txn.calendarHomeWithUID(u"nonexistent")))
# Skipped calendar homes are not deleted
self.assertTrue(self.filesPath.child("calendars").child(
"__uids__").child("no").child("ne").child("nonexistent").exists())
@inlineCallbacks
def test_upgradeExistingHome(self):
"""
L{UpgradeToDatabaseService.startService} will skip migrating existing
homes.
"""
startTxn = self.sqlStore.newTransaction("populate empty sample")
yield startTxn.calendarHomeWithUID("home1", create=True)
yield startTxn.commit()
yield self.upgrader.stepWithResult(None)
vrfyTxn = self.sqlStore.newTransaction("verify sample still empty")
self.addCleanup(vrfyTxn.commit)
home = yield vrfyTxn.calendarHomeWithUID("home1")
# The default calendar is still there.
self.assertNotIdentical(None, (yield home.calendarWithName("calendar")))
# The migrated calendar isn't.
self.assertIdentical(None, (yield home.calendarWithName("calendar_1")))
@inlineCallbacks
def test_upgradeAttachments(self):
"""
L{UpgradeToDatabaseService.startService} upgrades calendar attachments
as well.
"""
# Need to tweak config and settings to setup dropbox to work
self.patch(config, "EnableDropBox", True)
self.patch(config, "EnableManagedAttachments", False)
self.sqlStore.enableManagedAttachments = False
txn = self.sqlStore.newTransaction()
cs = schema.CALENDARSERVER
yield Delete(
From=cs,
Where=cs.NAME == "MANAGED-ATTACHMENTS"
).on(txn)
yield txn.commit()
txn = self.fileStore.newTransaction()
committed = []
def maybeCommit():
if not committed:
committed.append(True)
return txn.commit()
self.addCleanup(maybeCommit)
@inlineCallbacks
def getSampleObj():
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
object = (yield calendar.calendarObjectWithName("1.ics"))
returnValue(object)
inObject = yield getSampleObj()
someAttachmentName = "some-attachment"
someAttachmentType = MimeType.fromString("application/x-custom-type")
attachment = yield inObject.createAttachmentWithName(
someAttachmentName,
)
transport = attachment.store(someAttachmentType)
someAttachmentData = "Here is some data for your attachment, enjoy."
transport.write(someAttachmentData)
yield transport.loseConnection()
yield maybeCommit()
yield self.upgrader.stepWithResult(None)
committed = []
txn = self.sqlStore.newTransaction()
outObject = yield getSampleObj()
outAttachment = yield outObject.attachmentWithName(someAttachmentName)
allDone = Deferred()
class SimpleProto(Protocol):
data = ''
def dataReceived(self, data):
self.data += data
def connectionLost(self, reason):
allDone.callback(self.data)
self.assertEquals(outAttachment.contentType(), someAttachmentType)
outAttachment.retrieve(SimpleProto())
allData = yield allDone
self.assertEquals(allData, someAttachmentData)
@inlineCallbacks
def test_upgradeAddressBookHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in ABCommonTests.requirements:
if ABCommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.addressbookHomeWithUID(uid))
)
# Successfully migrated addressbook homes are deleted
self.assertFalse(self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.addressbookHomeWithUID("home1"))
adbk = (yield home.addressbookWithName("addressbook"))
for name, md5 in (
("1.vcf", ABCommonTests.md5Values[0]),
("2.vcf", ABCommonTests.md5Values[1]),
("3.vcf", ABCommonTests.md5Values[2]),
):
object = (yield adbk.addressbookObjectWithName(name))
self.assertEquals(object.md5(), md5)
@inlineCallbacks
def test_upgradeProperties(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home_defaults"))
cal = (yield home.calendarWithName("calendar_1"))
inbox = (yield home.calendarWithName("inbox"))
# Supported components
self.assertEqual(cal.getSupportedComponents(), "VEVENT")
self.assertTrue(cal.properties().get(PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)) is None)
# Resource type removed
self.assertTrue(cal.properties().get(PropertyName.fromElement(element.ResourceType)) is None)
# Ctag removed
self.assertTrue(cal.properties().get(PropertyName.fromElement(customxml.GETCTag)) is None)
# Availability
self.assertEquals(str(home.getAvailability()), str(self.av1))
self.assertTrue(inbox.properties().get(PropertyName.fromElement(customxml.CalendarAvailability)) is None)
# Default calendar
self.assertTrue(home.isDefaultCalendar(cal))
self.assertTrue(inbox.properties().get(PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)) is None)
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
class HomeMigrationTests(TestCase):
"""
Tests for L{UpgradeToDatabaseStep}.
"""
@inlineCallbacks
def setUp(self):
"""
Set up two stores to migrate between.
"""
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, TestStoreDirectoryService(), True, True
)
self.sqlStore = yield theStoreBuilder.buildStore(
self, StubNotifierFactory()
)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
def mergeRequirements(self, a, b):
"""
Merge two requirements dictionaries together, modifying C{a} and
returning it.
@param a: Some requirements, in the format of
L{CommonTests.requirements}.
@type a: C{dict}
@param b: Some additional requirements, to be merged into C{a}.
@type b: C{dict}
@return: C{a}
@rtype: C{dict}
"""
for homeUID in b:
homereq = a.setdefault(homeUID, {})
homeExtras = b[homeUID]
for calendarUID in homeExtras:
calreq = homereq.setdefault(calendarUID, {})
calendarExtras = homeExtras[calendarUID]
calreq.update(calendarExtras)
return a
@withSpecialValue(
"extraRequirements",
{
"home1": {
"calendar_1": {
"bogus.ics": (
getModule("twistedcaldav").filePath.sibling("zoneinfo")
.child("EST.ics").getContent(),
CommonTests.metadata1
)
}
}
}
)
@inlineCallbacks
def test_unknownTypeNotMigrated(self):
"""
The only types of calendar objects that should get migrated are VEVENTs
and VTODOs. Other component types, such as free-standing VTIMEZONEs,
don't have a UID and can't be stored properly in the database, so they
should not be migrated.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
self.assertIdentical(
None,
(yield (yield (yield
(yield txn.calendarHomeWithUID("home1"))
.calendarWithName("calendar_1")))
.calendarObjectWithName("bogus.ics"))
)
@inlineCallbacks
def test_upgradeCalendarHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.calendarHomeWithUID(uid))
)
# Successfully migrated calendar homes are deleted
self.assertFalse(self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
for name, metadata, md5 in (
("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
):
object = (yield calendar.calendarObjectWithName(name))
self.assertEquals(object.getMetadata(), metadata)
self.assertEquals(object.md5(), md5)
@inlineCallbacks
def test_upgradeExistingHome(self):
"""
L{UpgradeToDatabaseService.startService} will skip migrating existing
homes.
"""
startTxn = self.sqlStore.newTransaction("populate empty sample")
yield startTxn.calendarHomeWithUID("home1", create=True)
yield startTxn.commit()
yield self.upgrader.stepWithResult(None)
vrfyTxn = self.sqlStore.newTransaction("verify sample still empty")
self.addCleanup(vrfyTxn.commit)
home = yield vrfyTxn.calendarHomeWithUID("home1")
# The default calendar is still there.
self.assertNotIdentical(None, (yield home.calendarWithName("calendar")))
# The migrated calendar isn't.
self.assertIdentical(None, (yield home.calendarWithName("calendar_1")))
@inlineCallbacks
def test_upgradeAttachments(self):
"""
L{UpgradeToDatabaseService.startService} upgrades calendar attachments
as well.
"""
# Need to tweak config and settings to setup dropbox to work
self.patch(config, "EnableDropBox", True)
self.patch(config, "EnableManagedAttachments", False)
self.sqlStore.enableManagedAttachments = False
txn = self.sqlStore.newTransaction()
cs = schema.CALENDARSERVER
yield Delete(
From=cs,
Where=cs.NAME == "MANAGED-ATTACHMENTS"
).on(txn)
yield txn.commit()
txn = self.fileStore.newTransaction()
committed = []
def maybeCommit():
if not committed:
committed.append(True)
return txn.commit()
self.addCleanup(maybeCommit)
@inlineCallbacks
def getSampleObj():
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
object = (yield calendar.calendarObjectWithName("1.ics"))
returnValue(object)
inObject = yield getSampleObj()
someAttachmentName = "some-attachment"
someAttachmentType = MimeType.fromString("application/x-custom-type")
attachment = yield inObject.createAttachmentWithName(
someAttachmentName,
)
transport = attachment.store(someAttachmentType)
someAttachmentData = "Here is some data for your attachment, enjoy."
transport.write(someAttachmentData)
yield transport.loseConnection()
yield maybeCommit()
yield self.upgrader.stepWithResult(None)
committed = []
txn = self.sqlStore.newTransaction()
outObject = yield getSampleObj()
outAttachment = yield outObject.attachmentWithName(someAttachmentName)
allDone = Deferred()
class SimpleProto(Protocol):
data = ''
def dataReceived(self, data):
self.data += data
def connectionLost(self, reason):
allDone.callback(self.data)
self.assertEquals(outAttachment.contentType(), someAttachmentType)
outAttachment.retrieve(SimpleProto())
allData = yield allDone
self.assertEquals(allData, someAttachmentData)
@inlineCallbacks
def test_upgradeAddressBookHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in ABCommonTests.requirements:
if ABCommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.addressbookHomeWithUID(uid))
)
# Successfully migrated addressbook homes are deleted
self.assertFalse(self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.addressbookHomeWithUID("home1"))
adbk = (yield home.addressbookWithName("addressbook"))
for name, md5 in (
("1.vcf", ABCommonTests.md5Values[0]),
("2.vcf", ABCommonTests.md5Values[1]),
("3.vcf", ABCommonTests.md5Values[2]),
):
object = (yield adbk.addressbookObjectWithName(name))
self.assertEquals(object.md5(), md5)
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
