def _defaultCalendarUpgrade_setup(self): # Set dead property on inbox for user in ( "user01", "user02", ): inbox = (yield self.calendarUnderTest(name="inbox", home=user)) inbox.properties()[PropertyName.fromElement( ScheduleDefaultCalendarURL)] = ScheduleDefaultCalendarURL( HRef.fromString("/calendars/__uids__/%s/calendar_1" % (user, ))) # Force current default to null home = (yield self.homeUnderTest(name=user)) chm = home._homeMetaDataSchema yield Update( { chm.DEFAULT_EVENTS: None }, Where=chm.RESOURCE_ID == home._resourceID, ).on(self.transactionUnderTest()) # Force data version to previous ch = home._homeSchema yield Update( { ch.DATAVERSION: 3 }, Where=ch.RESOURCE_ID == home._resourceID, ).on(self.transactionUnderTest()) yield self.commit()
def doUpgrade(sqlStore): """ Do the required upgrade steps. """ sqlTxn = sqlStore.newTransaction( label="calendar_upgrade_from_5_to_6.doUpgrade") cb = schema.CALENDAR_BIND # Fix shared calendar alarms which should default to "empty" yield Update( { cb.ALARM_VEVENT_TIMED: "empty", cb.ALARM_VEVENT_ALLDAY: "empty", cb.ALARM_VTODO_TIMED: "empty", cb.ALARM_VTODO_ALLDAY: "empty", }, Where=(cb.BIND_MODE != _BIND_MODE_OWN)).on(sqlTxn) # Fix inbox transparency which should always be True yield Update({ cb.TRANSP: _TRANSP_TRANSPARENT, }, Where=(cb.CALENDAR_RESOURCE_NAME == "inbox")).on(sqlTxn) yield sqlTxn.commit() # Always bump the DB value yield updateAllCalendarHomeDataVersions(sqlStore, UPGRADE_TO_VERSION) yield updateCalendarDataVersion(sqlStore, UPGRADE_TO_VERSION)
def _normalizeColumnUUIDs(txn, column): """ Upper-case the UUIDs in the given SQL DAL column. @param txn: The transaction. @type txn: L{CommonStoreTransaction} @param column: the column, which may contain UIDs, to normalize. @type column: L{ColumnSyntax} @return: A L{Deferred} that will fire when the UUID normalization of the given column has completed. """ tableModel = column.model.table # Get a primary key made of column syntax objects for querying and # comparison later. pkey = [ColumnSyntax(columnModel) for columnModel in tableModel.primaryKey] for row in (yield Select([column] + pkey, From=TableSyntax(tableModel)).on(txn)): before = row[0] pkeyparts = row[1:] after = normalizeUUIDOrNot(before) if after != before: where = _AndNothing # Build a where clause out of the primary key and the parts of the # primary key that were found. for pkeycol, pkeypart in zip(pkeyparts, pkey): where = where.And(pkeycol == pkeypart) yield Update({column: after}, Where=where).on(txn)
def _renameHome(txn, table, oldUID, newUID): """ Rename a calendar, addressbook, or notification home. Note that this function is only safe in transactions that have had caching disabled, and more specifically should only ever be used during upgrades. Running this in a normal transaction will have unpredictable consequences, especially with respect to memcache. @param txn: an SQL transaction to use for this update @type txn: L{twext.enterprise.ienterprise.IAsyncTransaction} @param table: the storage table of the desired home type @type table: L{TableSyntax} @param oldUID: the old UID, the existing home's UID @type oldUID: L{str} @param newUID: the new UID, to change the UID to @type newUID: L{str} @return: a L{Deferred} which fires when the home is renamed. """ return Update({ table.OWNER_UID: newUID }, Where=table.OWNER_UID == oldUID).on(txn)
def test_old_queued(self): """ Verify that old inbox items are removed """ # Patch to force remove work items self.patch(config.InboxCleanup, "InboxRemoveWorkThreshold", 0) # Predate some inbox items inbox = yield self.calendarUnderTest(home="user01", name="inbox") oldDate = datetime.datetime.utcnow() - datetime.timedelta(days=float(config.InboxCleanup.ItemLifetimeDays), seconds=10) itemsToPredate = ["cal2.ics", "cal3.ics"] co = schema.CALENDAR_OBJECT yield Update( {co.CREATED: oldDate}, Where=co.RESOURCE_NAME.In(Parameter("itemsToPredate", len(itemsToPredate))).And( co.CALENDAR_RESOURCE_ID == inbox._resourceID) ).on(self.transactionUnderTest(), itemsToPredate=itemsToPredate) # do cleanup yield self.transactionUnderTest().enqueue(CleanupOneInboxWork, homeID=inbox.ownerHome()._resourceID, notBefore=datetime.datetime.utcnow()) yield self.commit() yield JobItem.waitEmpty(self.storeUnderTest().newTransaction, reactor, 60) # check that old items are deleted inbox = yield self.calendarUnderTest(home="user01", name="inbox") items = yield inbox.objectResources() names = [item.name() for item in items] self.assertEqual(set(names), set(["cal1.ics"]))
def doDataUpgradeSteps(self): """ Do SQL store data upgrades to make sure properties etc that were moved from the property store into columns also get migrated to the current schema. """ # First force each home to v1 data format so the upgrades will be triggered self.log.warn("Migration extra steps.") txn = self.sqlStore.newTransaction( label="UpgradeToDatabaseStep.doDataUpgradeSteps") for storeType in (ECALENDARTYPE, EADDRESSBOOKTYPE): schema = txn._homeClass[storeType]._homeSchema yield Update( { schema.DATAVERSION: 1 }, Where=None, ).on(txn) yield txn.commit() # Now apply each required data upgrade self.sqlStore.setUpgrading(True) for upgrade, description in ( (doCalendarUpgrade_1_to_2, "Calendar data upgrade from v1 to v2"), (doCalendarUpgrade_2_to_3, "Calendar data upgrade from v2 to v3"), (doCalendarUpgrade_3_to_4, "Calendar data upgrade from v3 to v4"), (doCalendarUpgrade_4_to_5, "Calendar data upgrade from v4 to v5"), (doAddressbookUpgrade_1_to_2, "Addressbook data upgrade from v1 to v2"), ): self.log.warn("Migration extra step: {desc}", desc=description) yield upgrade(self.sqlStore) self.sqlStore.setUpgrading(False)
def test_referenceOldEvent(self): """ Verify that inbox items references old events are removed """ # events are already too old, so make one event end now calendar = yield self.calendarUnderTest(home="user01", name="calendar") cal3Event = yield calendar.objectResourceWithName("cal3.ics") tr = schema.TIME_RANGE yield Update( { tr.END_DATE: datetime.datetime.utcnow() }, Where=tr.CALENDAR_OBJECT_RESOURCE_ID == cal3Event._resourceID).on( self.transactionUnderTest()) # do cleanup yield self.transactionUnderTest().enqueue( CleanupOneInboxWork, homeID=calendar.ownerHome()._resourceID, notBefore=datetime.datetime.utcnow()) yield self.commit() yield JobItem.waitEmpty(self.storeUnderTest().newTransaction, reactor, 60) # check that old items are deleted inbox = yield self.calendarUnderTest(home="user01", name="inbox") items = yield inbox.objectResources() names = [item.name() for item in items] self.assertEqual(set(names), set(["cal3.ics"]))
def doIt(txn, homeResourceID): """ KIND is set to person by schema upgrade. To upgrade MEMBERS and FOREIGN_MEMBERS: 1. Set group KIND (avoids assert) 2. Write groups. Write logic will fill in MEMBERS and FOREIGN_MEMBERS (Remember that all members resource IDs must already be in the address book). """ home = yield txn.addressbookHomeWithResourceID(homeResourceID) abObjectResources = yield home.addressbook().objectResources() for abObject in abObjectResources: component = yield abObject.component() lcResourceKind = component.resourceKind().lower( ) if component.resourceKind() else component.resourceKind() if lcResourceKind == "group": # update kind abo = schema.ADDRESSBOOK_OBJECT yield Update( { abo.KIND: _ABO_KIND_GROUP }, Where=abo.RESOURCE_ID == abObject._resourceID, ).on(txn) abObject._kind = _ABO_KIND_GROUP #update rest yield abObject.setComponent(component)
def doToEachHomeNotAtVersion(store, homeSchema, version, doIt, logStr, filterOwnerUID=None, processExternal=False): """ Do something to each home whose version column indicates it is older than the specified version. Do this in batches as there may be a lot of work to do. Also, allow the GUID to be filtered to support a parallel mode of operation. """ txn = store.newTransaction("updateDataVersion") where = homeSchema.DATAVERSION < version if filterOwnerUID: where = where.And(homeSchema.OWNER_UID.StartsWith(filterOwnerUID)) total = (yield Select( [Count(homeSchema.RESOURCE_ID), ], From=homeSchema, Where=where, ).on(txn))[0][0] yield txn.commit() count = 0 while True: logUpgradeStatus(logStr, count, total) # Get the next home with an old version txn = store.newTransaction("updateDataVersion") try: rows = yield Select( [homeSchema.RESOURCE_ID, homeSchema.OWNER_UID, homeSchema.STATUS, ], From=homeSchema, Where=where, OrderBy=homeSchema.OWNER_UID, Limit=1, ).on(txn) if len(rows) == 0: yield txn.commit() logUpgradeStatus("End {}".format(logStr), count, total) returnValue(None) # Apply to the home if not external homeResourceID, _ignore_owner_uid, homeStatus = rows[0] if homeStatus != _HOME_STATUS_EXTERNAL or processExternal: yield doIt(txn, homeResourceID) # Update the home to the current version yield Update( {homeSchema.DATAVERSION: version}, Where=homeSchema.RESOURCE_ID == homeResourceID, ).on(txn) yield txn.commit() except RuntimeError, e: f = Failure() logUpgradeError( logStr, "Failed to upgrade {} to {}: {}".format(homeSchema, version, e) ) yield txn.abort() f.raiseException() count += 1
def updateAllAddressBookHomeDataVersions(store, version): txn = store.newTransaction("updateAllAddressBookHomeDataVersions") ah = schema.ADDRESSBOOK_HOME yield Update( {ah.DATAVERSION: version}, ).on(txn) yield txn.commit()
def updatesome(cls, transaction, where, **kw): """ Update rows matching the where expression from the table that corresponds to C{cls}. """ colmap = {} for k, v in kw.iteritems(): colmap[cls.__attrmap__[k]] = v return Update(colmap, Where=where).on(transaction)
def updateAllCalendarHomeDataVersions(store, version): txn = store.newTransaction("updateAllCalendarHomeDataVersions") ch = schema.CALENDAR_HOME yield Update( {ch.DATAVERSION: version}, Where=None, ).on(txn) yield txn.commit()
def _updateDataVersion(store, key, version): txn = store.newTransaction("updateDataVersion") cs = schema.CALENDARSERVER yield Update( {cs.VALUE: version}, Where=cs.NAME == key, ).on(txn) yield txn.commit()
def _updateBumpTokenQuery(cls): rev = cls._revisionsSchema return Update( { rev.REVISION: schema.REVISION_SEQ, rev.MODIFIED: utcNowSQL, }, Where=(rev.RESOURCE_ID == Parameter("resourceID")).And( rev.RESOURCE_NAME == Parameter("name")), Return=rev.REVISION)
def setStatus(self, newStatus): """ Mark this home as being purged. """ # Only if different if self._status != newStatus: yield Update( {self._homeSchema.STATUS: newStatus}, Where=(self._homeSchema.RESOURCE_ID == self._resourceID), ).on(self._txn) self._status = newStatus
def _updateNotificationQuery(cls): no = cls._objectSchema return Update( { no.NOTIFICATION_TYPE: Parameter("notificationType"), no.NOTIFICATION_DATA: Parameter("notificationData"), no.MD5: Parameter("md5"), }, Where=( no.NOTIFICATION_HOME_RESOURCE_ID == Parameter("homeID")).And( no.NOTIFICATION_UID == Parameter("uid")), Return=no.MODIFIED)
def _bumpSyncTokenQuery(cls): """ DAL query to change collection sync token. Note this can impact multiple rows if the collection is shared. """ rev = cls._revisionsSchema return Update( { rev.REVISION: schema.REVISION_SEQ, rev.MODIFIED: utcNowSQL, }, Where=(rev.RESOURCE_ID == Parameter("resourceID")).And( rev.RESOURCE_NAME == None))
def update(cls, txn, oldManagedID, ownerHomeID, referencedBy, oldAttachmentID): """ Update an Attachment object. This creates a new one and adjusts the reference to the old one to point to the new one. If the old one is no longer referenced at all, it is deleted. @param txn: The transaction to use @type txn: L{CommonStoreTransaction} @param oldManagedID: the identifier for the original attachment @type oldManagedID: C{str} @param ownerHomeID: the resource-id of the home collection of the attachment owner @type ownerHomeID: C{int} @param referencedBy: the resource-id of the calendar object referencing the attachment @type referencedBy: C{int} @param oldAttachmentID: the attachment-id of the existing attachment being updated @type oldAttachmentID: C{int} """ # Now create the DB entry with a new managed-ID managed_id = str(uuid.uuid4()) attachment = (yield cls._create(txn, managed_id, ownerHomeID)) attachment._objectResourceID = referencedBy # Update the attachment<->calendar object relationship for managed attachments attco = cls._attachmentLinkSchema yield Update( { attco.ATTACHMENT_ID: attachment._attachmentID, attco.MANAGED_ID: attachment._managedID, }, Where=(attco.MANAGED_ID == oldManagedID).And( attco.CALENDAR_OBJECT_RESOURCE_ID == attachment._objectResourceID), ).on(txn) # Now check whether old attachmentID is still referenced - if not delete it rows = (yield Select( [ attco.ATTACHMENT_ID, ], From=attco, Where=(attco.ATTACHMENT_ID == oldAttachmentID), ).on(txn)) aids = [row[0] for row in rows] if rows is not None else () if len(aids) == 0: oldattachment = ManagedAttachment(txn, oldAttachmentID, None, None) oldattachment = (yield oldattachment.initFromStore()) yield oldattachment.remove() returnValue(attachment)
def _renameSyncTokenQuery(cls): """ DAL query to change sync token for a rename (increment and adjust resource name). """ rev = cls._revisionsSchema return Update( { rev.REVISION: schema.REVISION_SEQ, rev.COLLECTION_NAME: Parameter("name"), rev.MODIFIED: utcNowSQL, }, Where=(rev.RESOURCE_ID == Parameter("resourceID")).And( rev.RESOURCE_NAME == None), Return=rev.REVISION)
def _orphanAttachment(self, home, calendar, event): txn = self._sqlCalendarStore.newTransaction() # Reset dropbox id in calendar_object home = (yield txn.calendarHomeWithUID(home)) calendar = (yield home.calendarWithName(calendar)) event = (yield calendar.calendarObjectWithName(event)) co = schema.CALENDAR_OBJECT Update( {co.DROPBOX_ID: None, }, Where=co.RESOURCE_ID == event._resourceID, ).on(txn) (yield txn.commit())
def _unsharedRemovalQuery(cls): """ DAL query to indicate an owned collection has been deleted. """ rev = cls._revisionsSchema return Update( { rev.RESOURCE_ID: None, rev.REVISION: schema.REVISION_SEQ, rev.DELETED: True, rev.MODIFIED: utcNowSQL, }, Where=(rev.RESOURCE_ID == Parameter("resourceID")).And( rev.RESOURCE_NAME == None), )
def update(self, **kw): """ Modify the given attributes in the database. @return: a L{Deferred} that fires when the updates have been sent to the database. """ colmap = {} for k, v in kw.iteritems(): colmap[self.__attrmap__[k]] = v yield Update(colmap, Where=self._primaryKeyComparison( self._primaryKeyValue())).on(self.transaction) self.__dict__.update(kw)
def moveSupportedComponentSetProperties(sqlStore): """ Need to move all the CalDAV:supported-component-set properties in the RESOURCE_PROPERTY table to the new CALENDAR_METADATA table column, extracting the new format value from the XML property. """ logUpgradeStatus("Starting Move supported-component-set") sqlTxn = sqlStore.newTransaction( label="calendar_upgrade_from_1_to_2.moveSupportedComponentSetProperties" ) try: # Do not move the properties if migrating, as migration will do a split and set supported-components, # however we still need to remove the old properties. if not sqlStore._migrating: calendar_rid = None rows = (yield rowsForProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet)) total = len(rows) count = 0 for calendar_rid, value in rows: prop = WebDAVDocument.fromString(value).root_element supported_components = ",".join( sorted([ comp.attributes["name"].upper() for comp in prop.children ])) meta = schema.CALENDAR_METADATA yield Update( { meta.SUPPORTED_COMPONENTS: supported_components }, Where=(meta.RESOURCE_ID == calendar_rid)).on(sqlTxn) count += 1 logUpgradeStatus("Move supported-component-set", count, total) yield removeProperty(sqlTxn, caldavxml.SupportedCalendarComponentSet) yield sqlTxn.commit() logUpgradeStatus("End Move supported-component-set") except RuntimeError: yield sqlTxn.abort() logUpgradeError("Move supported-component-set", "Last calendar: {}".format(calendar_rid)) raise
def _calendarTranspUpgrade_setup(self): # Set dead property on inbox for user in ("user01", "user02",): inbox = (yield self.calendarUnderTest(name="inbox", home=user)) inbox.properties()[PropertyName.fromElement(CalendarFreeBusySet)] = CalendarFreeBusySet(HRef.fromString("/calendars/__uids__/%s/calendar_1" % (user,))) # Force current to transparent calendar = (yield self.calendarUnderTest(name="calendar_1", home=user)) yield calendar.setUsedForFreeBusy(False) calendar.properties()[PropertyName.fromElement(ScheduleCalendarTransp)] = ScheduleCalendarTransp(Opaque() if user == "user01" else Transparent()) # Force data version to previous home = (yield self.homeUnderTest(name=user)) ch = home._homeSchema yield Update( {ch.DATAVERSION: 3}, Where=ch.RESOURCE_ID == home._resourceID, ).on(self.transactionUnderTest()) yield self.commit() for user in ("user01", "user02",): calendar = (yield self.calendarUnderTest(name="calendar_1", home=user)) self.assertFalse(calendar.isUsedForFreeBusy()) self.assertTrue(PropertyName.fromElement(ScheduleCalendarTransp) in calendar.properties()) inbox = (yield self.calendarUnderTest(name="inbox", home=user)) self.assertTrue(PropertyName.fromElement(CalendarFreeBusySet) in inbox.properties()) yield self.commit() # Create "fake" entry for non-existent share txn = self.transactionUnderTest() calendar = (yield self.calendarUnderTest(name="calendar_1", home="user01")) rp = schema.RESOURCE_PROPERTY yield Insert( { rp.RESOURCE_ID: calendar._resourceID, rp.NAME: PropertyName.fromElement(ScheduleCalendarTransp).toString(), rp.VALUE: ScheduleCalendarTransp(Opaque()).toxml(), rp.VIEWER_UID: "user03", } ).on(txn) yield self.commit()
def convertToManaged(self): """ Convert this dropbox attachment into a managed attachment by updating the database and returning a new ManagedAttachment object that does not reference any calendar object. Referencing will be added later. @return: the managed attachment object @rtype: L{ManagedAttachment} """ # Change the DROPBOX_ID to a single "." to indicate a managed attachment. att = self._attachmentSchema (yield Update( { att.DROPBOX_ID: ".", }, Where=(att.ATTACHMENT_ID == self._attachmentID), ).on(self._txn)) # Create an "orphaned" ManagedAttachment that points to the updated data but without # an actual managed-id (which only exists when there is a reference to a calendar object). mattach = (yield ManagedAttachment.load(self._txn, None, None, attachmentID=self._attachmentID)) mattach._managedID = str(uuid.uuid4()) if mattach is None: raise AttachmentMigrationFailed # Then move the file on disk from the old path to the new one try: mattach._path.parent().makedirs() except Exception: # OK to fail if it already exists, otherwise must raise if not mattach._path.parent().exists(): raise oldpath = self._path newpath = mattach._path oldpath.moveTo(newpath) self.removeParentPaths() returnValue(mattach)
def changed(self, contentType, dispositionName, md5, size): """ Dropbox attachments never change their path - ignore dispositionName. """ self._contentType = contentType self._md5 = md5 self._size = size att = self._attachmentSchema self._created, self._modified = map(parseSQLTimestamp, (yield Update( { att.CONTENT_TYPE: generateContentType(self._contentType), att.SIZE: self._size, att.MD5: self._md5, att.MODIFIED: utcNowSQL, }, Where=(att.ATTACHMENT_ID == self._attachmentID), Return=(att.CREATED, att.MODIFIED)).on(self._txn))[0])
def changed(self, contentType, dispositionName, md5, size): """ Always update name to current disposition name. """ self._contentType = contentType self._name = dispositionName self._md5 = md5 self._size = size att = self._attachmentSchema self._created, self._modified = map(parseSQLTimestamp, (yield Update( { att.CONTENT_TYPE: generateContentType(self._contentType), att.SIZE: self._size, att.MD5: self._md5, att.MODIFIED: utcNowSQL, att.PATH: self._name, }, Where=(att.ATTACHMENT_ID == self._attachmentID), Return=(att.CREATED, att.MODIFIED)).on(self._txn))[0])
class PropertyStore(AbstractPropertyStore): """ We are going to use memcache to cache properties per-resource/per-user. However, we need to be able to invalidate on a per-resource basis, in addition to per-resource/per-user. So we will also track in memcache which resource/uid tokens are valid. That way we can remove the tracking entry to completely invalidate all the per-resource/per-user pairs. """ _cacher = Memcacher("SQL.props", pickle=True, key_normalization=False) def __init__(self, *a, **kw): raise NotImplementedError( "do not construct directly, call PropertyStore.load()") _allWithID = Select([prop.NAME, prop.VIEWER_UID, prop.VALUE], From=prop, Where=prop.RESOURCE_ID == Parameter("resourceID")) _allWithIDViewer = Select( [prop.NAME, prop.VALUE], From=prop, Where=(prop.RESOURCE_ID == Parameter("resourceID")).And( prop.VIEWER_UID == Parameter("viewerID"))) def _cacheToken(self, userid): return "{0!s}/{1}".format(self._resourceID, userid) @inlineCallbacks def _refresh(self, txn): """ Load, or re-load, this object with the given transaction; first from memcache, then pulling from the database again. """ # Cache existing properties in this object # Look for memcache entry first @inlineCallbacks def _cache_user_props(uid): # First check whether uid already has a valid cached entry rows = None if self._cacher is not None: valid_cached_users = yield self._cacher.get( str(self._resourceID)) if valid_cached_users is None: valid_cached_users = set() # Fetch cached user data if valid and present if uid in valid_cached_users: rows = yield self._cacher.get(self._cacheToken(uid)) # If no cached data, fetch from SQL DB and cache if rows is None: rows = yield self._allWithIDViewer.on( txn, resourceID=self._resourceID, viewerID=uid, ) if self._cacher is not None: yield self._cacher.set(self._cacheToken(uid), rows if rows is not None else ()) # Mark this uid as valid valid_cached_users.add(uid) yield self._cacher.set(str(self._resourceID), valid_cached_users) for name, value in rows: self._cached[(name, uid)] = value # Cache for the owner first, then the sharee if different yield _cache_user_props(self._defaultUser) if self._perUser != self._defaultUser: yield _cache_user_props(self._perUser) if self._proxyUser != self._perUser: yield _cache_user_props(self._proxyUser) @classmethod @inlineCallbacks def load(cls, defaultuser, shareUser, proxyUser, txn, resourceID, created=False, notifyCallback=None): """ @param notifyCallback: a callable used to trigger notifications when the property store changes. """ self = cls.__new__(cls) super(PropertyStore, self).__init__(defaultuser, shareUser, proxyUser) self._txn = txn self._resourceID = resourceID if not self._txn.store().queryCachingEnabled(): self._cacher = None self._cached = {} if not created: yield self._refresh(txn) self._notifyCallback = notifyCallback returnValue(self) @classmethod @inlineCallbacks def forMultipleResources(cls, defaultUser, shareeUser, proxyUser, txn, childColumn, parentColumn, parentID): """ Load all property stores for all objects in a collection. This is used to optimize Depth:1 operations on that collection, by loading all relevant properties in a single query. @param defaultUser: the UID of the user who owns / is requesting the property stores; the ones whose per-user properties will be exposed. @type defaultUser: C{str} @param txn: the transaction within which to fetch the rows. @type txn: L{IAsyncTransaction} @param childColumn: The resource ID column for the child resources, i.e. the resources of the type for which this method will loading the property stores. @param parentColumn: The resource ID column for the parent resources. e.g. if childColumn is addressbook object's resource ID, then this should be addressbook's resource ID. @return: a L{Deferred} that fires with a C{dict} mapping resource ID (a value taken from C{childColumn}) to a L{PropertyStore} for that ID. """ childTable = TableSyntax(childColumn.model.table) query = Select( [ childColumn, # XXX is that column necessary? as per the 'on' clause it has to be # the same as prop.RESOURCE_ID anyway. prop.RESOURCE_ID, prop.NAME, prop.VIEWER_UID, prop.VALUE ], From=prop.join(childTable, prop.RESOURCE_ID == childColumn, 'right'), Where=parentColumn == parentID) rows = yield query.on(txn) stores = cls._createMultipleStores(defaultUser, shareeUser, proxyUser, txn, rows) returnValue(stores) @classmethod @inlineCallbacks def forMultipleResourcesWithResourceIDs(cls, defaultUser, shareeUser, proxyUser, txn, resourceIDs): """ Load all property stores for all specified resources. This is used to optimize Depth:1 operations on that collection, by loading all relevant properties in a single query. Note that the caller of this method must make sure that the number of items being queried for is within a reasonable batch size. If the caller is itself batching related queries, that will take care of itself. @param defaultUser: the UID of the user who owns / is requesting the property stores; the ones whose per-user properties will be exposed. @type defaultUser: C{str} @param txn: the transaction within which to fetch the rows. @type txn: L{IAsyncTransaction} @param resourceIDs: The set of resource ID's to query. @return: a L{Deferred} that fires with a C{dict} mapping resource ID (a value taken from C{childColumn}) to a L{PropertyStore} for that ID. """ query = Select( [prop.RESOURCE_ID, prop.NAME, prop.VIEWER_UID, prop.VALUE], From=prop, Where=prop.RESOURCE_ID.In( Parameter("resourceIDs", len(resourceIDs)))) rows = yield query.on(txn, resourceIDs=resourceIDs) stores = cls._createMultipleStores(defaultUser, shareeUser, proxyUser, txn, rows) # Make sure we have a store for each resourceID even if no properties exist for resourceID in resourceIDs: if resourceID not in stores: store = cls.__new__(cls) super(PropertyStore, store).__init__(defaultUser, shareeUser, proxyUser) store._txn = txn store._resourceID = resourceID store._cached = {} stores[resourceID] = store returnValue(stores) @classmethod def _createMultipleStores(cls, defaultUser, shareeUser, proxyUser, txn, rows): """ Create a set of stores for the set of rows passed in. """ createdStores = {} for row in rows: if len(row) == 5: object_resource_id, resource_id, name, view_uid, value = row else: object_resource_id = None resource_id, name, view_uid, value = row if resource_id: if resource_id not in createdStores: store = cls.__new__(cls) super(PropertyStore, store).__init__(defaultUser, shareeUser, proxyUser) store._txn = txn store._resourceID = resource_id store._cached = {} createdStores[resource_id] = store createdStores[resource_id]._cached[(name, view_uid)] = value elif object_resource_id: store = cls.__new__(cls) super(PropertyStore, store).__init__(defaultUser, shareeUser, proxyUser) store._txn = txn store._resourceID = object_resource_id store._cached = {} createdStores[object_resource_id] = store return createdStores def _getitem_uid(self, key, uid): validKey(key) try: value = self._cached[(key.toString(), uid)] except KeyError: raise KeyError(key) return WebDAVDocument.fromString(value).root_element _updateQuery = Update( {prop.VALUE: Parameter("value")}, Where=(prop.RESOURCE_ID == Parameter("resourceID")).And( prop.NAME == Parameter("name")).And( prop.VIEWER_UID == Parameter("uid"))) _insertQuery = Insert({ prop.VALUE: Parameter("value"), prop.RESOURCE_ID: Parameter("resourceID"), prop.NAME: Parameter("name"), prop.VIEWER_UID: Parameter("uid") }) def _setitem_uid(self, key, value, uid): validKey(key) key_str = key.toString() value_str = value.toxml() tried = [] wasCached = [(key_str, uid) in self._cached] self._cached[(key_str, uid)] = value_str @inlineCallbacks def trySetItem(txn): if tried: yield self._refresh(txn) wasCached[:] = [(key_str, uid) in self._cached] tried.append(True) if wasCached[0]: yield self._updateQuery.on(txn, resourceID=self._resourceID, value=value_str, name=key_str, uid=uid) else: yield self._insertQuery.on(txn, resourceID=self._resourceID, value=value_str, name=key_str, uid=uid) if self._cacher is not None: self._cacher.delete(self._cacheToken(uid)) # Call the registered notification callback - we need to do this as a preCommit since it involves # a bunch of deferred operations, but this propstore api is not deferred. preCommit will execute # the deferreds properly, and it is fine to wait until everything else is done before sending the # notifications. if hasattr(self, "_notifyCallback") and self._notifyCallback is not None: self._txn.preCommit(self._notifyCallback) def justLogIt(f): f.trap(AllRetriesFailed) self.log.error("setting a property failed; probably nothing.") self._txn.subtransaction(trySetItem).addErrback(justLogIt) _deleteQuery = Delete( prop, Where=(prop.RESOURCE_ID == Parameter("resourceID")).And( prop.NAME == Parameter("name")).And( prop.VIEWER_UID == Parameter("uid"))) def _delitem_uid(self, key, uid): validKey(key) key_str = key.toString() del self._cached[(key_str, uid)] @inlineCallbacks def doIt(txn): yield self._deleteQuery.on(txn, lambda: KeyError(key), resourceID=self._resourceID, name=key_str, uid=uid) if self._cacher is not None: self._cacher.delete(self._cacheToken(uid)) # Call the registered notification callback - we need to do this as a preCommit since it involves # a bunch of deferred operations, but this propstore api is not deferred. preCommit will execute # the deferreds properly, and it is fine to wait until everything else is done before sending the # notifications. if hasattr(self, "_notifyCallback") and self._notifyCallback is not None: self._txn.preCommit(self._notifyCallback) def justLogIt(f): f.trap(AllRetriesFailed) self.log.error("setting a property failed; probably nothing.") self._txn.subtransaction(doIt).addErrback(justLogIt) def _keys_uid(self, uid): for cachedKey, cachedUID in self._cached.keys(): if cachedUID == uid: yield PropertyName.fromString(cachedKey) _deleteResourceQuery = Delete( prop, Where=(prop.RESOURCE_ID == Parameter("resourceID"))) @inlineCallbacks def _removeResource(self): self._cached = {} yield self._deleteResourceQuery.on(self._txn, resourceID=self._resourceID) # Invalidate entire set of cached per-user data for this resource if self._cacher is not None: self._cacher.delete(str(self._resourceID)) @inlineCallbacks def copyAllProperties(self, other): """ Copy all the properties from another store into this one. This needs to be done independently of the UID. """ rows = yield other._allWithID.on(other._txn, resourceID=other._resourceID) for key_str, uid, value_str in rows: wasCached = [(key_str, uid) in self._cached] if wasCached[0]: yield self._updateQuery.on(self._txn, resourceID=self._resourceID, value=value_str, name=key_str, uid=uid) else: yield self._insertQuery.on(self._txn, resourceID=self._resourceID, value=value_str, name=key_str, uid=uid) # Invalidate entire set of cached per-user data for this resource and reload self._cached = {} if self._cacher is not None: self._cacher.delete(str(self._resourceID)) yield self._refresh(self._txn)
def _calendarTimezoneUpgrade_setup(self): TimezoneCache.create() self.addCleanup(TimezoneCache.clear) tz1 = Component(None, pycalendar=readVTZ("Etc/GMT+1")) tz2 = Component(None, pycalendar=readVTZ("Etc/GMT+2")) tz3 = Component(None, pycalendar=readVTZ("Etc/GMT+3")) # Share user01 calendar with user03 calendar = (yield self.calendarUnderTest(name="calendar_1", home="user01")) home3 = yield self.homeUnderTest(name="user03") shared_name = yield calendar.shareWith(home3, _BIND_MODE_WRITE) user_details = ( ("user01", "calendar_1", tz1), ("user02", "calendar_1", tz2), ("user03", "calendar_1", None), ("user03", shared_name, tz3), ) # Set dead properties on calendars for user, calname, tz in user_details: calendar = (yield self.calendarUnderTest(name=calname, home=user)) if tz: calendar.properties()[PropertyName.fromElement( caldavxml.CalendarTimeZone )] = caldavxml.CalendarTimeZone.fromString(str(tz)) # Force data version to previous home = (yield self.homeUnderTest(name=user)) ch = home._homeSchema yield Update( { ch.DATAVERSION: 4 }, Where=ch.RESOURCE_ID == home._resourceID, ).on(self.transactionUnderTest()) yield self.commit() for user, calname, tz in user_details: calendar = (yield self.calendarUnderTest(name=calname, home=user)) self.assertEqual(calendar.getTimezone(), None) self.assertEqual( PropertyName.fromElement(caldavxml.CalendarTimeZone) in calendar.properties(), tz is not None) yield self.commit() # Create "fake" entry for non-existent share txn = self.transactionUnderTest() calendar = (yield self.calendarUnderTest(name="calendar_1", home="user01")) rp = schema.RESOURCE_PROPERTY yield Insert({ rp.RESOURCE_ID: calendar._resourceID, rp.NAME: PropertyName.fromElement(caldavxml.CalendarTimeZone).toString(), rp.VALUE: caldavxml.CalendarTimeZone.fromString(str(tz3)).toxml(), rp.VIEWER_UID: "user04", }).on(txn) yield self.commit() returnValue(user_details)
def _calendarAvailabilityUpgrade_setup(self): av1 = Component.fromString("""BEGIN:VCALENDAR VERSION:2.0 CALSCALE:GREGORIAN PRODID:-//calendarserver.org//Zonal//EN BEGIN:VAVAILABILITY ORGANIZER:mailto:[email protected] UID:[email protected] DTSTAMP:20061005T133225Z DTEND:20140101T000000Z BEGIN:AVAILABLE UID:[email protected] DTSTAMP:20061005T133225Z SUMMARY:Monday to Friday from 9:00 to 17:00 DTSTART:20130101T090000Z DTEND:20130101T170000Z RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR END:AVAILABLE END:VAVAILABILITY END:VCALENDAR """) av2 = Component.fromString("""BEGIN:VCALENDAR VERSION:2.0 CALSCALE:GREGORIAN PRODID:-//calendarserver.org//Zonal//EN BEGIN:VAVAILABILITY ORGANIZER:mailto:[email protected] UID:[email protected] DTSTAMP:20061005T133225Z DTEND:20140101T000000Z BEGIN:AVAILABLE UID:[email protected] DTSTAMP:20061005T133225Z SUMMARY:Monday to Friday from 12:00 to 17:00 DTSTART:20130101T120000Z DTEND:20130101T170000Z RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR END:AVAILABLE END:VAVAILABILITY END:VCALENDAR """) user_details = ( ("user01", av1), ("user02", av2), ("user03", None), ) # Set dead properties on calendars for user, av in user_details: calendar = (yield self.calendarUnderTest(name="inbox", home=user)) if av: calendar.properties()[PropertyName.fromElement( customxml.CalendarAvailability )] = customxml.CalendarAvailability.fromString(str(av)) # Force data version to previous home = (yield self.homeUnderTest(name=user)) ch = home._homeSchema yield Update( { ch.DATAVERSION: 4 }, Where=ch.RESOURCE_ID == home._resourceID, ).on(self.transactionUnderTest()) yield self.commit() for user, av in user_details: home = (yield self.homeUnderTest(name=user)) calendar = (yield self.calendarUnderTest(name="inbox", home=user)) self.assertEqual(home.getAvailability(), None) self.assertEqual( PropertyName.fromElement(customxml.CalendarAvailability) in calendar.properties(), av is not None) yield self.commit() returnValue(user_details)