def _defaultCalendarUpgrade_setup(self):
# Set dead property on inbox
for user in (
"user01",
"user02",
):
inbox = (yield self.calendarUnderTest(name="inbox", home=user))
inbox.properties()[PropertyName.fromElement(
ScheduleDefaultCalendarURL)] = ScheduleDefaultCalendarURL(
HRef.fromString("/calendars/__uids__/%s/calendar_1" %
(user, )))
# Force current default to null
home = (yield self.homeUnderTest(name=user))
chm = home._homeMetaDataSchema
yield Update(
{
chm.DEFAULT_EVENTS: None
},
Where=chm.RESOURCE_ID == home._resourceID,
).on(self.transactionUnderTest())
# Force data version to previous
ch = home._homeSchema
yield Update(
{
ch.DATAVERSION: 3
},
Where=ch.RESOURCE_ID == home._resourceID,
).on(self.transactionUnderTest())
yield self.commit()
def doUpgrade(sqlStore):
"""
Do the required upgrade steps.
"""
sqlTxn = sqlStore.newTransaction(
label="calendar_upgrade_from_5_to_6.doUpgrade")
cb = schema.CALENDAR_BIND
# Fix shared calendar alarms which should default to "empty"
yield Update(
{
cb.ALARM_VEVENT_TIMED: "empty",
cb.ALARM_VEVENT_ALLDAY: "empty",
cb.ALARM_VTODO_TIMED: "empty",
cb.ALARM_VTODO_ALLDAY: "empty",
},
Where=(cb.BIND_MODE != _BIND_MODE_OWN)).on(sqlTxn)
# Fix inbox transparency which should always be True
yield Update({
cb.TRANSP: _TRANSP_TRANSPARENT,
},
Where=(cb.CALENDAR_RESOURCE_NAME == "inbox")).on(sqlTxn)
yield sqlTxn.commit()
# Always bump the DB value
yield updateAllCalendarHomeDataVersions(sqlStore, UPGRADE_TO_VERSION)
yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
def _normalizeColumnUUIDs(txn, column):
"""
Upper-case the UUIDs in the given SQL DAL column.
@param txn: The transaction.
@type txn: L{CommonStoreTransaction}
@param column: the column, which may contain UIDs, to normalize.
@type column: L{ColumnSyntax}
@return: A L{Deferred} that will fire when the UUID normalization of the
given column has completed.
"""
tableModel = column.model.table
# Get a primary key made of column syntax objects for querying and
# comparison later.
pkey = [ColumnSyntax(columnModel) for columnModel in tableModel.primaryKey]
for row in (yield Select([column] + pkey,
From=TableSyntax(tableModel)).on(txn)):
before = row[0]
pkeyparts = row[1:]
after = normalizeUUIDOrNot(before)
if after != before:
where = _AndNothing
# Build a where clause out of the primary key and the parts of the
# primary key that were found.
for pkeycol, pkeypart in zip(pkeyparts, pkey):
where = where.And(pkeycol == pkeypart)
yield Update({column: after}, Where=where).on(txn)
def _renameHome(txn, table, oldUID, newUID):
"""
Rename a calendar, addressbook, or notification home. Note that this
function is only safe in transactions that have had caching disabled, and
more specifically should only ever be used during upgrades. Running this
in a normal transaction will have unpredictable consequences, especially
with respect to memcache.
@param txn: an SQL transaction to use for this update
@type txn: L{twext.enterprise.ienterprise.IAsyncTransaction}
@param table: the storage table of the desired home type
@type table: L{TableSyntax}
@param oldUID: the old UID, the existing home's UID
@type oldUID: L{str}
@param newUID: the new UID, to change the UID to
@type newUID: L{str}
@return: a L{Deferred} which fires when the home is renamed.
"""
return Update({
table.OWNER_UID: newUID
}, Where=table.OWNER_UID == oldUID).on(txn)
def test_old_queued(self):
"""
Verify that old inbox items are removed
"""
# Patch to force remove work items
self.patch(config.InboxCleanup, "InboxRemoveWorkThreshold", 0)
# Predate some inbox items
inbox = yield self.calendarUnderTest(home="user01", name="inbox")
oldDate = datetime.datetime.utcnow() - datetime.timedelta(days=float(config.InboxCleanup.ItemLifetimeDays), seconds=10)
itemsToPredate = ["cal2.ics", "cal3.ics"]
co = schema.CALENDAR_OBJECT
yield Update(
{co.CREATED: oldDate},
Where=co.RESOURCE_NAME.In(Parameter("itemsToPredate", len(itemsToPredate))).And(
co.CALENDAR_RESOURCE_ID == inbox._resourceID)
).on(self.transactionUnderTest(), itemsToPredate=itemsToPredate)
# do cleanup
yield self.transactionUnderTest().enqueue(CleanupOneInboxWork, homeID=inbox.ownerHome()._resourceID, notBefore=datetime.datetime.utcnow())
yield self.commit()
yield JobItem.waitEmpty(self.storeUnderTest().newTransaction, reactor, 60)
# check that old items are deleted
inbox = yield self.calendarUnderTest(home="user01", name="inbox")
items = yield inbox.objectResources()
names = [item.name() for item in items]
self.assertEqual(set(names), set(["cal1.ics"]))
def doDataUpgradeSteps(self):
"""
Do SQL store data upgrades to make sure properties etc that were moved from the property store
into columns also get migrated to the current schema.
"""
# First force each home to v1 data format so the upgrades will be triggered
self.log.warn("Migration extra steps.")
txn = self.sqlStore.newTransaction(
label="UpgradeToDatabaseStep.doDataUpgradeSteps")
for storeType in (ECALENDARTYPE, EADDRESSBOOKTYPE):
schema = txn._homeClass[storeType]._homeSchema
yield Update(
{
schema.DATAVERSION: 1
},
Where=None,
).on(txn)
yield txn.commit()
# Now apply each required data upgrade
self.sqlStore.setUpgrading(True)
for upgrade, description in (
(doCalendarUpgrade_1_to_2, "Calendar data upgrade from v1 to v2"),
(doCalendarUpgrade_2_to_3, "Calendar data upgrade from v2 to v3"),
(doCalendarUpgrade_3_to_4, "Calendar data upgrade from v3 to v4"),
(doCalendarUpgrade_4_to_5, "Calendar data upgrade from v4 to v5"),
(doAddressbookUpgrade_1_to_2,
"Addressbook data upgrade from v1 to v2"),
):
self.log.warn("Migration extra step: {desc}", desc=description)
yield upgrade(self.sqlStore)
self.sqlStore.setUpgrading(False)
def test_referenceOldEvent(self):
"""
Verify that inbox items references old events are removed
"""
# events are already too old, so make one event end now
calendar = yield self.calendarUnderTest(home="user01", name="calendar")
cal3Event = yield calendar.objectResourceWithName("cal3.ics")
tr = schema.TIME_RANGE
yield Update(
{
tr.END_DATE: datetime.datetime.utcnow()
},
Where=tr.CALENDAR_OBJECT_RESOURCE_ID == cal3Event._resourceID).on(
self.transactionUnderTest())
# do cleanup
yield self.transactionUnderTest().enqueue(
CleanupOneInboxWork,
homeID=calendar.ownerHome()._resourceID,
notBefore=datetime.datetime.utcnow())
yield self.commit()
yield JobItem.waitEmpty(self.storeUnderTest().newTransaction, reactor,
60)
# check that old items are deleted
inbox = yield self.calendarUnderTest(home="user01", name="inbox")
items = yield inbox.objectResources()
names = [item.name() for item in items]
self.assertEqual(set(names), set(["cal3.ics"]))
def doIt(txn, homeResourceID):
"""
KIND is set to person by schema upgrade.
To upgrade MEMBERS and FOREIGN_MEMBERS:
1. Set group KIND (avoids assert)
2. Write groups. Write logic will fill in MEMBERS and FOREIGN_MEMBERS
(Remember that all members resource IDs must already be in the address book).
"""
home = yield txn.addressbookHomeWithResourceID(homeResourceID)
abObjectResources = yield home.addressbook().objectResources()
for abObject in abObjectResources:
component = yield abObject.component()
lcResourceKind = component.resourceKind().lower(
) if component.resourceKind() else component.resourceKind()
if lcResourceKind == "group":
# update kind
abo = schema.ADDRESSBOOK_OBJECT
yield Update(
{
abo.KIND: _ABO_KIND_GROUP
},
Where=abo.RESOURCE_ID == abObject._resourceID,
).on(txn)
abObject._kind = _ABO_KIND_GROUP
#update rest
yield abObject.setComponent(component)
def doToEachHomeNotAtVersion(store, homeSchema, version, doIt, logStr, filterOwnerUID=None, processExternal=False):
"""
Do something to each home whose version column indicates it is older
than the specified version. Do this in batches as there may be a lot of work to do. Also,
allow the GUID to be filtered to support a parallel mode of operation.
"""
txn = store.newTransaction("updateDataVersion")
where = homeSchema.DATAVERSION < version
if filterOwnerUID:
where = where.And(homeSchema.OWNER_UID.StartsWith(filterOwnerUID))
total = (yield Select(
[Count(homeSchema.RESOURCE_ID), ],
From=homeSchema,
Where=where,
).on(txn))[0][0]
yield txn.commit()
count = 0
while True:
logUpgradeStatus(logStr, count, total)
# Get the next home with an old version
txn = store.newTransaction("updateDataVersion")
try:
rows = yield Select(
[homeSchema.RESOURCE_ID, homeSchema.OWNER_UID, homeSchema.STATUS, ],
From=homeSchema,
Where=where,
OrderBy=homeSchema.OWNER_UID,
Limit=1,
).on(txn)
if len(rows) == 0:
yield txn.commit()
logUpgradeStatus("End {}".format(logStr), count, total)
returnValue(None)
# Apply to the home if not external
homeResourceID, _ignore_owner_uid, homeStatus = rows[0]
if homeStatus != _HOME_STATUS_EXTERNAL or processExternal:
yield doIt(txn, homeResourceID)
# Update the home to the current version
yield Update(
{homeSchema.DATAVERSION: version},
Where=homeSchema.RESOURCE_ID == homeResourceID,
).on(txn)
yield txn.commit()
except RuntimeError, e:
f = Failure()
logUpgradeError(
logStr,
"Failed to upgrade {} to {}: {}".format(homeSchema, version, e)
)
yield txn.abort()
f.raiseException()
count += 1
def updateAllAddressBookHomeDataVersions(store, version):
txn = store.newTransaction("updateAllAddressBookHomeDataVersions")
ah = schema.ADDRESSBOOK_HOME
yield Update(
{ah.DATAVERSION: version},
).on(txn)
yield txn.commit()
def updatesome(cls, transaction, where, **kw):
"""
Update rows matching the where expression from the table that corresponds to C{cls}.
"""
colmap = {}
for k, v in kw.iteritems():
colmap[cls.__attrmap__[k]] = v
return Update(colmap, Where=where).on(transaction)
def updateAllCalendarHomeDataVersions(store, version):
txn = store.newTransaction("updateAllCalendarHomeDataVersions")
ch = schema.CALENDAR_HOME
yield Update(
{ch.DATAVERSION: version},
Where=None,
).on(txn)
yield txn.commit()
def _updateDataVersion(store, key, version):
txn = store.newTransaction("updateDataVersion")
cs = schema.CALENDARSERVER
yield Update(
{cs.VALUE: version},
Where=cs.NAME == key,
).on(txn)
yield txn.commit()
def _updateBumpTokenQuery(cls):
rev = cls._revisionsSchema
return Update(
{
rev.REVISION: schema.REVISION_SEQ,
rev.MODIFIED: utcNowSQL,
},
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == Parameter("name")),
Return=rev.REVISION)
def setStatus(self, newStatus):
"""
Mark this home as being purged.
"""
# Only if different
if self._status != newStatus:
yield Update(
{self._homeSchema.STATUS: newStatus},
Where=(self._homeSchema.RESOURCE_ID == self._resourceID),
).on(self._txn)
self._status = newStatus
def _updateNotificationQuery(cls):
no = cls._objectSchema
return Update(
{
no.NOTIFICATION_TYPE: Parameter("notificationType"),
no.NOTIFICATION_DATA: Parameter("notificationData"),
no.MD5: Parameter("md5"),
},
Where=(
no.NOTIFICATION_HOME_RESOURCE_ID == Parameter("homeID")).And(
no.NOTIFICATION_UID == Parameter("uid")),
Return=no.MODIFIED)
def _bumpSyncTokenQuery(cls):
"""
DAL query to change collection sync token. Note this can impact multiple rows if the
collection is shared.
"""
rev = cls._revisionsSchema
return Update(
{
rev.REVISION: schema.REVISION_SEQ,
rev.MODIFIED: utcNowSQL,
},
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == None))
def update(cls, txn, oldManagedID, ownerHomeID, referencedBy,
oldAttachmentID):
"""
Update an Attachment object. This creates a new one and adjusts the reference to the old
one to point to the new one. If the old one is no longer referenced at all, it is deleted.
@param txn: The transaction to use
@type txn: L{CommonStoreTransaction}
@param oldManagedID: the identifier for the original attachment
@type oldManagedID: C{str}
@param ownerHomeID: the resource-id of the home collection of the attachment owner
@type ownerHomeID: C{int}
@param referencedBy: the resource-id of the calendar object referencing the attachment
@type referencedBy: C{int}
@param oldAttachmentID: the attachment-id of the existing attachment being updated
@type oldAttachmentID: C{int}
"""
# Now create the DB entry with a new managed-ID
managed_id = str(uuid.uuid4())
attachment = (yield cls._create(txn, managed_id, ownerHomeID))
attachment._objectResourceID = referencedBy
# Update the attachment<->calendar object relationship for managed attachments
attco = cls._attachmentLinkSchema
yield Update(
{
attco.ATTACHMENT_ID: attachment._attachmentID,
attco.MANAGED_ID: attachment._managedID,
},
Where=(attco.MANAGED_ID == oldManagedID).And(
attco.CALENDAR_OBJECT_RESOURCE_ID ==
attachment._objectResourceID),
).on(txn)
# Now check whether old attachmentID is still referenced - if not delete it
rows = (yield Select(
[
attco.ATTACHMENT_ID,
],
From=attco,
Where=(attco.ATTACHMENT_ID == oldAttachmentID),
).on(txn))
aids = [row[0] for row in rows] if rows is not None else ()
if len(aids) == 0:
oldattachment = ManagedAttachment(txn, oldAttachmentID, None, None)
oldattachment = (yield oldattachment.initFromStore())
yield oldattachment.remove()
returnValue(attachment)
def _renameSyncTokenQuery(cls):
"""
DAL query to change sync token for a rename (increment and adjust
resource name).
"""
rev = cls._revisionsSchema
return Update(
{
rev.REVISION: schema.REVISION_SEQ,
rev.COLLECTION_NAME: Parameter("name"),
rev.MODIFIED: utcNowSQL,
},
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == None),
Return=rev.REVISION)
def _orphanAttachment(self, home, calendar, event):
txn = self._sqlCalendarStore.newTransaction()
# Reset dropbox id in calendar_object
home = (yield txn.calendarHomeWithUID(home))
calendar = (yield home.calendarWithName(calendar))
event = (yield calendar.calendarObjectWithName(event))
co = schema.CALENDAR_OBJECT
Update(
{co.DROPBOX_ID: None, },
Where=co.RESOURCE_ID == event._resourceID,
).on(txn)
(yield txn.commit())
def _unsharedRemovalQuery(cls):
"""
DAL query to indicate an owned collection has been deleted.
"""
rev = cls._revisionsSchema
return Update(
{
rev.RESOURCE_ID: None,
rev.REVISION: schema.REVISION_SEQ,
rev.DELETED: True,
rev.MODIFIED: utcNowSQL,
},
Where=(rev.RESOURCE_ID == Parameter("resourceID")).And(
rev.RESOURCE_NAME == None),
)
def update(self, **kw):
"""
Modify the given attributes in the database.
@return: a L{Deferred} that fires when the updates have been sent to
the database.
"""
colmap = {}
for k, v in kw.iteritems():
colmap[self.__attrmap__[k]] = v
yield Update(colmap,
Where=self._primaryKeyComparison(
self._primaryKeyValue())).on(self.transaction)
self.__dict__.update(kw)
def moveSupportedComponentSetProperties(sqlStore):
"""
Need to move all the CalDAV:supported-component-set properties in the
RESOURCE_PROPERTY table to the new CALENDAR_METADATA table column,
extracting the new format value from the XML property.
"""
logUpgradeStatus("Starting Move supported-component-set")
sqlTxn = sqlStore.newTransaction(
label="calendar_upgrade_from_1_to_2.moveSupportedComponentSetProperties"
)
try:
# Do not move the properties if migrating, as migration will do a split and set supported-components,
# however we still need to remove the old properties.
if not sqlStore._migrating:
calendar_rid = None
rows = (yield
rowsForProperty(sqlTxn,
caldavxml.SupportedCalendarComponentSet))
total = len(rows)
count = 0
for calendar_rid, value in rows:
prop = WebDAVDocument.fromString(value).root_element
supported_components = ",".join(
sorted([
comp.attributes["name"].upper()
for comp in prop.children
]))
meta = schema.CALENDAR_METADATA
yield Update(
{
meta.SUPPORTED_COMPONENTS: supported_components
},
Where=(meta.RESOURCE_ID == calendar_rid)).on(sqlTxn)
count += 1
logUpgradeStatus("Move supported-component-set", count, total)
yield removeProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet)
yield sqlTxn.commit()
logUpgradeStatus("End Move supported-component-set")
except RuntimeError:
yield sqlTxn.abort()
logUpgradeError("Move supported-component-set",
"Last calendar: {}".format(calendar_rid))
raise
def _calendarTranspUpgrade_setup(self):
# Set dead property on inbox
for user in ("user01", "user02",):
inbox = (yield self.calendarUnderTest(name="inbox", home=user))
inbox.properties()[PropertyName.fromElement(CalendarFreeBusySet)] = CalendarFreeBusySet(HRef.fromString("/calendars/__uids__/%s/calendar_1" % (user,)))
# Force current to transparent
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
yield calendar.setUsedForFreeBusy(False)
calendar.properties()[PropertyName.fromElement(ScheduleCalendarTransp)] = ScheduleCalendarTransp(Opaque() if user == "user01" else Transparent())
# Force data version to previous
home = (yield self.homeUnderTest(name=user))
ch = home._homeSchema
yield Update(
{ch.DATAVERSION: 3},
Where=ch.RESOURCE_ID == home._resourceID,
).on(self.transactionUnderTest())
yield self.commit()
for user in ("user01", "user02",):
calendar = (yield self.calendarUnderTest(name="calendar_1", home=user))
self.assertFalse(calendar.isUsedForFreeBusy())
self.assertTrue(PropertyName.fromElement(ScheduleCalendarTransp) in calendar.properties())
inbox = (yield self.calendarUnderTest(name="inbox", home=user))
self.assertTrue(PropertyName.fromElement(CalendarFreeBusySet) in inbox.properties())
yield self.commit()
# Create "fake" entry for non-existent share
txn = self.transactionUnderTest()
calendar = (yield self.calendarUnderTest(name="calendar_1", home="user01"))
rp = schema.RESOURCE_PROPERTY
yield Insert(
{
rp.RESOURCE_ID: calendar._resourceID,
rp.NAME: PropertyName.fromElement(ScheduleCalendarTransp).toString(),
rp.VALUE: ScheduleCalendarTransp(Opaque()).toxml(),
rp.VIEWER_UID: "user03",
}
).on(txn)
yield self.commit()
def convertToManaged(self):
"""
Convert this dropbox attachment into a managed attachment by updating the
database and returning a new ManagedAttachment object that does not reference
any calendar object. Referencing will be added later.
@return: the managed attachment object
@rtype: L{ManagedAttachment}
"""
# Change the DROPBOX_ID to a single "." to indicate a managed attachment.
att = self._attachmentSchema
(yield Update(
{
att.DROPBOX_ID: ".",
},
Where=(att.ATTACHMENT_ID == self._attachmentID),
).on(self._txn))
# Create an "orphaned" ManagedAttachment that points to the updated data but without
# an actual managed-id (which only exists when there is a reference to a calendar object).
mattach = (yield
ManagedAttachment.load(self._txn,
None,
None,
attachmentID=self._attachmentID))
mattach._managedID = str(uuid.uuid4())
if mattach is None:
raise AttachmentMigrationFailed
# Then move the file on disk from the old path to the new one
try:
mattach._path.parent().makedirs()
except Exception:
# OK to fail if it already exists, otherwise must raise
if not mattach._path.parent().exists():
raise
oldpath = self._path
newpath = mattach._path
oldpath.moveTo(newpath)
self.removeParentPaths()
returnValue(mattach)
def changed(self, contentType, dispositionName, md5, size):
"""
Dropbox attachments never change their path - ignore dispositionName.
"""
self._contentType = contentType
self._md5 = md5
self._size = size
att = self._attachmentSchema
self._created, self._modified = map(parseSQLTimestamp, (yield Update(
{
att.CONTENT_TYPE: generateContentType(self._contentType),
att.SIZE: self._size,
att.MD5: self._md5,
att.MODIFIED: utcNowSQL,
},
Where=(att.ATTACHMENT_ID == self._attachmentID),
Return=(att.CREATED, att.MODIFIED)).on(self._txn))[0])
def changed(self, contentType, dispositionName, md5, size):
"""
Always update name to current disposition name.
"""
self._contentType = contentType
self._name = dispositionName
self._md5 = md5
self._size = size
att = self._attachmentSchema
self._created, self._modified = map(parseSQLTimestamp, (yield Update(
{
att.CONTENT_TYPE: generateContentType(self._contentType),
att.SIZE: self._size,
att.MD5: self._md5,
att.MODIFIED: utcNowSQL,
att.PATH: self._name,
},
Where=(att.ATTACHMENT_ID == self._attachmentID),
Return=(att.CREATED, att.MODIFIED)).on(self._txn))[0])
class PropertyStore(AbstractPropertyStore):
"""
We are going to use memcache to cache properties per-resource/per-user. However, we
need to be able to invalidate on a per-resource basis, in addition to per-resource/per-user.
So we will also track in memcache which resource/uid tokens are valid. That way we can remove
the tracking entry to completely invalidate all the per-resource/per-user pairs.
"""
_cacher = Memcacher("SQL.props", pickle=True, key_normalization=False)
def __init__(self, *a, **kw):
raise NotImplementedError(
"do not construct directly, call PropertyStore.load()")
_allWithID = Select([prop.NAME, prop.VIEWER_UID, prop.VALUE],
From=prop,
Where=prop.RESOURCE_ID == Parameter("resourceID"))
_allWithIDViewer = Select(
[prop.NAME, prop.VALUE],
From=prop,
Where=(prop.RESOURCE_ID == Parameter("resourceID")).And(
prop.VIEWER_UID == Parameter("viewerID")))
def _cacheToken(self, userid):
return "{0!s}/{1}".format(self._resourceID, userid)
@inlineCallbacks
def _refresh(self, txn):
"""
Load, or re-load, this object with the given transaction; first from
memcache, then pulling from the database again.
"""
# Cache existing properties in this object
# Look for memcache entry first
@inlineCallbacks
def _cache_user_props(uid):
# First check whether uid already has a valid cached entry
rows = None
if self._cacher is not None:
valid_cached_users = yield self._cacher.get(
str(self._resourceID))
if valid_cached_users is None:
valid_cached_users = set()
# Fetch cached user data if valid and present
if uid in valid_cached_users:
rows = yield self._cacher.get(self._cacheToken(uid))
# If no cached data, fetch from SQL DB and cache
if rows is None:
rows = yield self._allWithIDViewer.on(
txn,
resourceID=self._resourceID,
viewerID=uid,
)
if self._cacher is not None:
yield self._cacher.set(self._cacheToken(uid),
rows if rows is not None else ())
# Mark this uid as valid
valid_cached_users.add(uid)
yield self._cacher.set(str(self._resourceID),
valid_cached_users)
for name, value in rows:
self._cached[(name, uid)] = value
# Cache for the owner first, then the sharee if different
yield _cache_user_props(self._defaultUser)
if self._perUser != self._defaultUser:
yield _cache_user_props(self._perUser)
if self._proxyUser != self._perUser:
yield _cache_user_props(self._proxyUser)
@classmethod
@inlineCallbacks
def load(cls,
defaultuser,
shareUser,
proxyUser,
txn,
resourceID,
created=False,
notifyCallback=None):
"""
@param notifyCallback: a callable used to trigger notifications when the
property store changes.
"""
self = cls.__new__(cls)
super(PropertyStore, self).__init__(defaultuser, shareUser, proxyUser)
self._txn = txn
self._resourceID = resourceID
if not self._txn.store().queryCachingEnabled():
self._cacher = None
self._cached = {}
if not created:
yield self._refresh(txn)
self._notifyCallback = notifyCallback
returnValue(self)
@classmethod
@inlineCallbacks
def forMultipleResources(cls, defaultUser, shareeUser, proxyUser, txn,
childColumn, parentColumn, parentID):
"""
Load all property stores for all objects in a collection. This is used
to optimize Depth:1 operations on that collection, by loading all
relevant properties in a single query.
@param defaultUser: the UID of the user who owns / is requesting the
property stores; the ones whose per-user properties will be exposed.
@type defaultUser: C{str}
@param txn: the transaction within which to fetch the rows.
@type txn: L{IAsyncTransaction}
@param childColumn: The resource ID column for the child resources, i.e.
the resources of the type for which this method will loading the
property stores.
@param parentColumn: The resource ID column for the parent resources.
e.g. if childColumn is addressbook object's resource ID, then this
should be addressbook's resource ID.
@return: a L{Deferred} that fires with a C{dict} mapping resource ID (a
value taken from C{childColumn}) to a L{PropertyStore} for that ID.
"""
childTable = TableSyntax(childColumn.model.table)
query = Select(
[
childColumn,
# XXX is that column necessary? as per the 'on' clause it has to be
# the same as prop.RESOURCE_ID anyway.
prop.RESOURCE_ID,
prop.NAME,
prop.VIEWER_UID,
prop.VALUE
],
From=prop.join(childTable, prop.RESOURCE_ID == childColumn,
'right'),
Where=parentColumn == parentID)
rows = yield query.on(txn)
stores = cls._createMultipleStores(defaultUser, shareeUser, proxyUser,
txn, rows)
returnValue(stores)
@classmethod
@inlineCallbacks
def forMultipleResourcesWithResourceIDs(cls, defaultUser, shareeUser,
proxyUser, txn, resourceIDs):
"""
Load all property stores for all specified resources. This is used
to optimize Depth:1 operations on that collection, by loading all
relevant properties in a single query. Note that the caller of this
method must make sure that the number of items being queried for is
within a reasonable batch size. If the caller is itself batching
related queries, that will take care of itself.
@param defaultUser: the UID of the user who owns / is requesting the
property stores; the ones whose per-user properties will be exposed.
@type defaultUser: C{str}
@param txn: the transaction within which to fetch the rows.
@type txn: L{IAsyncTransaction}
@param resourceIDs: The set of resource ID's to query.
@return: a L{Deferred} that fires with a C{dict} mapping resource ID (a
value taken from C{childColumn}) to a L{PropertyStore} for that ID.
"""
query = Select(
[prop.RESOURCE_ID, prop.NAME, prop.VIEWER_UID, prop.VALUE],
From=prop,
Where=prop.RESOURCE_ID.In(
Parameter("resourceIDs", len(resourceIDs))))
rows = yield query.on(txn, resourceIDs=resourceIDs)
stores = cls._createMultipleStores(defaultUser, shareeUser, proxyUser,
txn, rows)
# Make sure we have a store for each resourceID even if no properties exist
for resourceID in resourceIDs:
if resourceID not in stores:
store = cls.__new__(cls)
super(PropertyStore, store).__init__(defaultUser, shareeUser,
proxyUser)
store._txn = txn
store._resourceID = resourceID
store._cached = {}
stores[resourceID] = store
returnValue(stores)
@classmethod
def _createMultipleStores(cls, defaultUser, shareeUser, proxyUser, txn,
rows):
"""
Create a set of stores for the set of rows passed in.
"""
createdStores = {}
for row in rows:
if len(row) == 5:
object_resource_id, resource_id, name, view_uid, value = row
else:
object_resource_id = None
resource_id, name, view_uid, value = row
if resource_id:
if resource_id not in createdStores:
store = cls.__new__(cls)
super(PropertyStore,
store).__init__(defaultUser, shareeUser, proxyUser)
store._txn = txn
store._resourceID = resource_id
store._cached = {}
createdStores[resource_id] = store
createdStores[resource_id]._cached[(name, view_uid)] = value
elif object_resource_id:
store = cls.__new__(cls)
super(PropertyStore, store).__init__(defaultUser, shareeUser,
proxyUser)
store._txn = txn
store._resourceID = object_resource_id
store._cached = {}
createdStores[object_resource_id] = store
return createdStores
def _getitem_uid(self, key, uid):
validKey(key)
try:
value = self._cached[(key.toString(), uid)]
except KeyError:
raise KeyError(key)
return WebDAVDocument.fromString(value).root_element
_updateQuery = Update(
{prop.VALUE: Parameter("value")},
Where=(prop.RESOURCE_ID == Parameter("resourceID")).And(
prop.NAME == Parameter("name")).And(
prop.VIEWER_UID == Parameter("uid")))
_insertQuery = Insert({
prop.VALUE: Parameter("value"),
prop.RESOURCE_ID: Parameter("resourceID"),
prop.NAME: Parameter("name"),
prop.VIEWER_UID: Parameter("uid")
})
def _setitem_uid(self, key, value, uid):
validKey(key)
key_str = key.toString()
value_str = value.toxml()
tried = []
wasCached = [(key_str, uid) in self._cached]
self._cached[(key_str, uid)] = value_str
@inlineCallbacks
def trySetItem(txn):
if tried:
yield self._refresh(txn)
wasCached[:] = [(key_str, uid) in self._cached]
tried.append(True)
if wasCached[0]:
yield self._updateQuery.on(txn,
resourceID=self._resourceID,
value=value_str,
name=key_str,
uid=uid)
else:
yield self._insertQuery.on(txn,
resourceID=self._resourceID,
value=value_str,
name=key_str,
uid=uid)
if self._cacher is not None:
self._cacher.delete(self._cacheToken(uid))
# Call the registered notification callback - we need to do this as a preCommit since it involves
# a bunch of deferred operations, but this propstore api is not deferred. preCommit will execute
# the deferreds properly, and it is fine to wait until everything else is done before sending the
# notifications.
if hasattr(self,
"_notifyCallback") and self._notifyCallback is not None:
self._txn.preCommit(self._notifyCallback)
def justLogIt(f):
f.trap(AllRetriesFailed)
self.log.error("setting a property failed; probably nothing.")
self._txn.subtransaction(trySetItem).addErrback(justLogIt)
_deleteQuery = Delete(
prop,
Where=(prop.RESOURCE_ID == Parameter("resourceID")).And(
prop.NAME == Parameter("name")).And(
prop.VIEWER_UID == Parameter("uid")))
def _delitem_uid(self, key, uid):
validKey(key)
key_str = key.toString()
del self._cached[(key_str, uid)]
@inlineCallbacks
def doIt(txn):
yield self._deleteQuery.on(txn,
lambda: KeyError(key),
resourceID=self._resourceID,
name=key_str,
uid=uid)
if self._cacher is not None:
self._cacher.delete(self._cacheToken(uid))
# Call the registered notification callback - we need to do this as a preCommit since it involves
# a bunch of deferred operations, but this propstore api is not deferred. preCommit will execute
# the deferreds properly, and it is fine to wait until everything else is done before sending the
# notifications.
if hasattr(self,
"_notifyCallback") and self._notifyCallback is not None:
self._txn.preCommit(self._notifyCallback)
def justLogIt(f):
f.trap(AllRetriesFailed)
self.log.error("setting a property failed; probably nothing.")
self._txn.subtransaction(doIt).addErrback(justLogIt)
def _keys_uid(self, uid):
for cachedKey, cachedUID in self._cached.keys():
if cachedUID == uid:
yield PropertyName.fromString(cachedKey)
_deleteResourceQuery = Delete(
prop, Where=(prop.RESOURCE_ID == Parameter("resourceID")))
@inlineCallbacks
def _removeResource(self):
self._cached = {}
yield self._deleteResourceQuery.on(self._txn,
resourceID=self._resourceID)
# Invalidate entire set of cached per-user data for this resource
if self._cacher is not None:
self._cacher.delete(str(self._resourceID))
@inlineCallbacks
def copyAllProperties(self, other):
"""
Copy all the properties from another store into this one. This needs to be done
independently of the UID.
"""
rows = yield other._allWithID.on(other._txn,
resourceID=other._resourceID)
for key_str, uid, value_str in rows:
wasCached = [(key_str, uid) in self._cached]
if wasCached[0]:
yield self._updateQuery.on(self._txn,
resourceID=self._resourceID,
value=value_str,
name=key_str,
uid=uid)
else:
yield self._insertQuery.on(self._txn,
resourceID=self._resourceID,
value=value_str,
name=key_str,
uid=uid)
# Invalidate entire set of cached per-user data for this resource and reload
self._cached = {}
if self._cacher is not None:
self._cacher.delete(str(self._resourceID))
yield self._refresh(self._txn)
def _calendarTimezoneUpgrade_setup(self):
TimezoneCache.create()
self.addCleanup(TimezoneCache.clear)
tz1 = Component(None, pycalendar=readVTZ("Etc/GMT+1"))
tz2 = Component(None, pycalendar=readVTZ("Etc/GMT+2"))
tz3 = Component(None, pycalendar=readVTZ("Etc/GMT+3"))
# Share user01 calendar with user03
calendar = (yield self.calendarUnderTest(name="calendar_1",
home="user01"))
home3 = yield self.homeUnderTest(name="user03")
shared_name = yield calendar.shareWith(home3, _BIND_MODE_WRITE)
user_details = (
("user01", "calendar_1", tz1),
("user02", "calendar_1", tz2),
("user03", "calendar_1", None),
("user03", shared_name, tz3),
)
# Set dead properties on calendars
for user, calname, tz in user_details:
calendar = (yield self.calendarUnderTest(name=calname, home=user))
if tz:
calendar.properties()[PropertyName.fromElement(
caldavxml.CalendarTimeZone
)] = caldavxml.CalendarTimeZone.fromString(str(tz))
# Force data version to previous
home = (yield self.homeUnderTest(name=user))
ch = home._homeSchema
yield Update(
{
ch.DATAVERSION: 4
},
Where=ch.RESOURCE_ID == home._resourceID,
).on(self.transactionUnderTest())
yield self.commit()
for user, calname, tz in user_details:
calendar = (yield self.calendarUnderTest(name=calname, home=user))
self.assertEqual(calendar.getTimezone(), None)
self.assertEqual(
PropertyName.fromElement(caldavxml.CalendarTimeZone)
in calendar.properties(), tz is not None)
yield self.commit()
# Create "fake" entry for non-existent share
txn = self.transactionUnderTest()
calendar = (yield self.calendarUnderTest(name="calendar_1",
home="user01"))
rp = schema.RESOURCE_PROPERTY
yield Insert({
rp.RESOURCE_ID:
calendar._resourceID,
rp.NAME:
PropertyName.fromElement(caldavxml.CalendarTimeZone).toString(),
rp.VALUE:
caldavxml.CalendarTimeZone.fromString(str(tz3)).toxml(),
rp.VIEWER_UID:
"user04",
}).on(txn)
yield self.commit()
returnValue(user_details)
def _calendarAvailabilityUpgrade_setup(self):
av1 = Component.fromString("""BEGIN:VCALENDAR
VERSION:2.0
CALSCALE:GREGORIAN
PRODID:-//calendarserver.org//Zonal//EN
BEGIN:VAVAILABILITY
ORGANIZER:mailto:[email protected]
UID:[email protected]
DTSTAMP:20061005T133225Z
DTEND:20140101T000000Z
BEGIN:AVAILABLE
UID:[email protected]
DTSTAMP:20061005T133225Z
SUMMARY:Monday to Friday from 9:00 to 17:00
DTSTART:20130101T090000Z
DTEND:20130101T170000Z
RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
END:AVAILABLE
END:VAVAILABILITY
END:VCALENDAR
""")
av2 = Component.fromString("""BEGIN:VCALENDAR
VERSION:2.0
CALSCALE:GREGORIAN
PRODID:-//calendarserver.org//Zonal//EN
BEGIN:VAVAILABILITY
ORGANIZER:mailto:[email protected]
UID:[email protected]
DTSTAMP:20061005T133225Z
DTEND:20140101T000000Z
BEGIN:AVAILABLE
UID:[email protected]
DTSTAMP:20061005T133225Z
SUMMARY:Monday to Friday from 12:00 to 17:00
DTSTART:20130101T120000Z
DTEND:20130101T170000Z
RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
END:AVAILABLE
END:VAVAILABILITY
END:VCALENDAR
""")
user_details = (
("user01", av1),
("user02", av2),
("user03", None),
)
# Set dead properties on calendars
for user, av in user_details:
calendar = (yield self.calendarUnderTest(name="inbox", home=user))
if av:
calendar.properties()[PropertyName.fromElement(
customxml.CalendarAvailability
)] = customxml.CalendarAvailability.fromString(str(av))
# Force data version to previous
home = (yield self.homeUnderTest(name=user))
ch = home._homeSchema
yield Update(
{
ch.DATAVERSION: 4
},
Where=ch.RESOURCE_ID == home._resourceID,
).on(self.transactionUnderTest())
yield self.commit()
for user, av in user_details:
home = (yield self.homeUnderTest(name=user))
calendar = (yield self.calendarUnderTest(name="inbox", home=user))
self.assertEqual(home.getAvailability(), None)
self.assertEqual(
PropertyName.fromElement(customxml.CalendarAvailability)
in calendar.properties(), av is not None)
yield self.commit()
returnValue(user_details)
