class UpgradeDatabaseCoreStep(object): """ Base class for either schema or data upgrades on the database. upgrade files in sql syntax that we can execute against the database to accomplish the upgrade. @ivar sqlStore: The store to operate on. @type sqlStore: L{txdav.idav.IDataStore} """ log = Logger() def __init__(self, sqlStore, uid=None, gid=None, failIfUpgradeNeeded=False, checkExistingSchema=False): """ Initialize the service. """ self.sqlStore = sqlStore self.uid = uid self.gid = gid self.failIfUpgradeNeeded = failIfUpgradeNeeded self.checkExistingSchema = checkExistingSchema self.schemaLocation = getModule(__name__).filePath.parent().parent().sibling("sql_schema") self.pyLocation = getModule(__name__).filePath.parent() self.versionKey = None self.versionDescriptor = "" self.upgradeFilePrefix = "" self.upgradeFileSuffix = "" self.defaultKeyValue = None def stepWithResult(self, result): """ Start the service. """ return self.databaseUpgrade() @inlineCallbacks def databaseUpgrade(self): """ Do a database schema upgrade. """ self.log.warn("Beginning database {vers} check.", vers=self.versionDescriptor) # Retrieve information from schema and database dialect, required_version, actual_version = yield self.getVersions() if required_version == actual_version: self.log.warn("{vers} version check complete: no upgrade needed.", vers=self.versionDescriptor.capitalize()) if self.checkExistingSchema: if dialect == "postgres-dialect": expected_schema = self.schemaLocation.child("current.sql") schema_name = "public" else: expected_schema = self.schemaLocation.child("current-oracle-dialect.sql") schema_name = config.DatabaseConnection.user yield self.sqlStore.checkSchema(schemaFromPath(expected_schema), schema_name) elif required_version < actual_version: msg = "Actual %s version %s is more recent than the expected version %s. The service cannot be started" % ( self.versionDescriptor, actual_version, required_version, ) self.log.error(msg) raise RuntimeError(msg) elif self.failIfUpgradeNeeded: if self.checkExistingSchema: expected_schema = self.schemaLocation.child("old").child(dialect).child("v{}.sql".format(actual_version)) if dialect == "postgres-dialect": schema_name = "public" else: schema_name = config.DatabaseConnection.user yield self.sqlStore.checkSchema(schemaFromPath(expected_schema), schema_name) raise NotAllowedToUpgrade() else: self.sqlStore.setUpgrading(True) yield self.upgradeVersion(actual_version, required_version, dialect) self.sqlStore.setUpgrading(False) self.log.warn("Database {vers} check complete.", vers=self.versionDescriptor) returnValue(None) @inlineCallbacks def getVersions(self): """ Extract the expected version from the database schema and get the actual version in the current database, along with the DB dialect. """ # Retrieve the version number from the schema file current_schema = self.schemaLocation.child("current.sql").getContent() found = re.search("insert into CALENDARSERVER values \('%s', '(\d+)'\);" % (self.versionKey,), current_schema) if found is None: msg = "Schema is missing required database key %s insert statement: %s" % (self.versionKey, current_schema,) self.log.error(msg) raise RuntimeError(msg) else: required_version = int(found.group(1)) self.log.warn("Required database key {key}: {vers}.", key=self.versionKey, vers=required_version) # Get the schema version in the current database sqlTxn = self.sqlStore.newTransaction(label="UpgradeDatabaseCoreStep.getVersions") dialect = sqlTxn.dbtype.dialect try: actual_version = yield sqlTxn.calendarserverValue(self.versionKey) actual_version = int(actual_version) yield sqlTxn.commit() except (RuntimeError, ValueError): f = Failure() self.log.error("Database key {key} cannot be determined.", key=self.versionKey) yield sqlTxn.abort() if self.defaultKeyValue is None: f.raiseException() else: actual_version = self.defaultKeyValue self.log.warn("Actual database key {key}: {vers}.", key=self.versionKey, vers=actual_version) returnValue((dialect, required_version, actual_version,)) @inlineCallbacks def upgradeVersion(self, fromVersion, toVersion, dialect): """ Update the database from one version to another (the current one). Do this by looking for upgrade_from_X_to_Y.sql files that cover the full range of upgrades. """ self.log.warn("Starting {vers} upgrade from version {fr} to {to}.", vers=self.versionDescriptor, fr=fromVersion, to=toVersion) # Scan for all possible upgrade files - returned sorted files = self.scanForUpgradeFiles(dialect) # Determine upgrade sequence and run each upgrade upgrades = self.determineUpgradeSequence(fromVersion, toVersion, files, dialect) # Use one transaction for the entire set of upgrades try: for fp in upgrades: yield self.applyUpgrade(fp) except RuntimeError: self.log.error("Database {vers} upgrade failed using: {path}", vers=self.versionDescriptor, path=fp.basename()) raise self.log.warn("{vers} upgraded from version {fr} to {to}.", vers=self.versionDescriptor.capitalize(), fr=fromVersion, to=toVersion) def getPathToUpgrades(self, dialect): """ Return the path where appropriate upgrade files can be found. """ raise NotImplementedError def scanForUpgradeFiles(self, dialect): """ Scan for upgrade files with the require name. """ fp = self.getPathToUpgrades(dialect) upgrades = [] regex = re.compile("%supgrade_from_(\d+)_to_(\d+)%s" % (self.upgradeFilePrefix, self.upgradeFileSuffix,)) for child in fp.globChildren("%supgrade_*%s" % (self.upgradeFilePrefix, self.upgradeFileSuffix,)): matched = regex.match(child.basename()) if matched is not None: fromV = int(matched.group(1)) toV = int(matched.group(2)) upgrades.append((fromV, toV, child)) upgrades.sort(key=lambda x: (x[0], x[1])) return upgrades def determineUpgradeSequence(self, fromVersion, toVersion, files, dialect): """ Determine the upgrade_from_X_to_Y(.sql|.py) files that cover the full range of upgrades. Note that X and Y may not be consecutive, e.g., we might have an upgrade from 3 to 4, 4 to 5, and 3 to 5 - the later because it is more efficient to jump over the intermediate step. As a result we will always try and pick the upgrade file that gives the biggest jump from one version to another at each step. """ # Now find the path from the old version to the current one filesByFromVersion = {} for fromV, toV, fp in files: if fromV not in filesByFromVersion or filesByFromVersion[fromV][1] < toV: filesByFromVersion[fromV] = fromV, toV, fp upgrades = [] nextVersion = fromVersion while nextVersion != toVersion: if nextVersion not in filesByFromVersion: msg = "Missing upgrade file from version %d with dialect %s" % (nextVersion, dialect,) self.log.error(msg) raise RuntimeError(msg) else: upgrades.append(filesByFromVersion[nextVersion][2]) nextVersion = filesByFromVersion[nextVersion][1] return upgrades def applyUpgrade(self, fp): """ Apply the supplied upgrade to the database. Always return an L{Deferred" """ raise NotImplementedError
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. ## from __future__ import print_function from calendarserver.tools.cmdline import utilityMain, WorkerService from argparse import ArgumentParser from twext.python.log import Logger from twisted.internet.defer import inlineCallbacks from twext.who.idirectory import RecordType import time log = Logger() class DisplayAPNSubscriptions(WorkerService): users = [] def doWork(self): rootResource = self.rootResource() directory = rootResource.getDirectory() return displayAPNSubscriptions(self.store, directory, rootResource, self.users) def main():
class DataStoreTransaction(object): """ In-memory implementation of a data store transaction. """ log = Logger() def __init__(self, dataStore, name): """ Initialize a transaction; do not call this directly, instead call L{CalendarStore.newTransaction}. @param calendarStore: The store that created this transaction. @type calendarStore: L{CalendarStore} """ self._dataStore = dataStore self._termination = None self._operations = [] self._postCommitOperations = [] self._postAbortOperations = [] self._tracker = _CommitTracker(name) def store(self): return self._dataStore def addOperation(self, operation, name): self._operations.append(operation) self._tracker.info.append(name) def _terminate(self, mode): """ Check to see if this transaction has already been terminated somehow, either via committing or aborting, and if not, note that it has been terminated. @param mode: The manner of the termination of this transaction. @type mode: C{str} @raise AlreadyFinishedError: This transaction has already been terminated. """ if self._termination is not None: raise AlreadyFinishedError("already %s" % (self._termination, )) self._termination = mode self._tracker.done = True def abort(self): self._terminate("aborted") for operation in self._postAbortOperations: operation() def commit(self): self._terminate("committed") self.committed = True undos = [] for operation in self._operations: try: undo = operation() if undo is not None: undos.append(undo) except: self.log.debug("Undoing DataStoreTransaction") for undo in undos: try: undo() except: self.log.error("Cannot undo DataStoreTransaction") raise for operation in self._postCommitOperations: operation() def postCommit(self, operation): self._postCommitOperations.append(operation) def postAbort(self, operation): self._postAbortOperations.append(operation)
class NotificationsDatabase(AbstractSQLDatabase): log = Logger() db_basename = db_prefix + "notifications" schema_version = "1" db_type = "notifications" def __init__(self, resource): """ @param resource: the L{CalDAVResource} resource for the notifications collection.) """ self.resource = resource db_filename = os.path.join(self.resource.fp.path, NotificationsDatabase.db_basename) super(NotificationsDatabase, self).__init__(db_filename, True, autocommit=True) self.resource._txn.postCommit(self._db_close) self.resource._txn.postAbort(self._db_close) def allRecords(self): records = self._db_execute("select * from NOTIFICATIONS") return [ self._makeRecord(row) for row in (records if records is not None else ()) ] def recordForUID(self, uid): row = self._db_execute("select * from NOTIFICATIONS where UID = :1", uid) return self._makeRecord(row[0]) if row else None def addOrUpdateRecord(self, record): self._db_execute( """insert or replace into NOTIFICATIONS (UID, NAME, TYPE) values (:1, :2, :3) """, record.uid, record.name, json.dumps(record.notificationtype), ) self._db_execute( """ insert or replace into REVISIONS (NAME, REVISION, DELETED) values (:1, :2, :3) """, record.name, self.bumpRevision(fast=True), 'N', ) def removeRecordForUID(self, uid): record = self.recordForUID(uid) self.removeRecordForName(record.name) def removeRecordForName(self, rname): self._db_execute("delete from NOTIFICATIONS where NAME = :1", rname) self._db_execute( """ update REVISIONS SET REVISION = :1, DELETED = :2 where NAME = :3 """, self.bumpRevision(fast=True), 'Y', rname) def whatchanged(self, revision): results = [ (name.encode("utf-8"), deleted) for name, deleted in self._db_execute( "select NAME, DELETED from REVISIONS where REVISION > :1", revision) ] results.sort(key=lambda x: x[1]) changed = [] deleted = [] for name, wasdeleted in results: if name: if wasdeleted == 'Y': if revision: deleted.append(name) else: changed.append(name) else: raise SyncTokenValidException return changed, deleted, def lastRevision(self): return self._db_value_for_sql("select REVISION from REVISION_SEQUENCE") def bumpRevision(self, fast=False): self._db_execute( """ update REVISION_SEQUENCE set REVISION = REVISION + 1 """, ) self._db_commit() return self._db_value_for_sql( """ select REVISION from REVISION_SEQUENCE """, ) def _db_version(self): """ @return: the schema version assigned to this index. """ return NotificationsDatabase.schema_version def _db_type(self): """ @return: the collection type assigned to this index. """ return NotificationsDatabase.db_type def _db_init_data_tables(self, q): """ Initialise the underlying database tables. @param q: a database cursor to use. """ # # NOTIFICATIONS table is the primary table # UID: UID for this notification # NAME: child resource name # TYPE: type of notification # q.execute(""" create table NOTIFICATIONS ( UID text unique, NAME text unique, TYPE text ) """) q.execute(""" create index UID on NOTIFICATIONS (UID) """) # # REVISIONS table tracks changes # NAME: Last URI component (eg. <uid>.ics, RESOURCE primary key) # REVISION: revision number # WASDELETED: Y if revision deleted, N if added or changed # q.execute(""" create table REVISION_SEQUENCE ( REVISION integer ) """) q.execute(""" insert into REVISION_SEQUENCE (REVISION) values (0) """) q.execute(""" create table REVISIONS ( NAME text unique, REVISION integer, DELETED text(1) ) """) q.execute(""" create index REVISION on REVISIONS (REVISION) """) def _db_upgrade_data_tables(self, q, old_version): """ Upgrade the data from an older version of the DB. """ # Nothing to do as we have not changed the schema pass def _makeRecord(self, row): return NotificationRecord( * [str(item) if isinstance(item, unicode) else item for item in row])
class Memcacher(CachePoolUserMixIn): log = Logger() MEMCACHE_KEY_LIMIT = 250 # the memcached key length limit NAMESPACE_MAX_LENGTH = 32 # max size of namespace we will allow HASH_LENGTH = 32 # length of hash we will generate TRUNCATED_KEY_LENGTH = MEMCACHE_KEY_LIMIT - NAMESPACE_MAX_LENGTH - HASH_LENGTH - 2 # 2 accounts for delimiters MEMCACHE_VALUE_LIMIT = 1024 * 1024 # the memcached default value length limit # Translation table: all ctrls (0x00 - 0x1F) and space and 0x7F mapped to _ keyNormalizeTranslateTable = string.maketrans( "".join([chr(i) for i in range(33)]) + chr(0x7F), "_" * 33 + "_") allowTestCache = False memoryCacheInstance = { True: None, False: None, } class memoryCacher(): """ A class implementing the memcache client API we care about but using a dict to store the results in memory. This can be used for caching on a single instance server, and for tests, where memcached may not be running. """ def __init__(self, pickle=False): self._cache = {} # (value, expireTime, check-and-set identifier) self._clock = 0 self._pickle = pickle def _check_key(self, key): if not isinstance(key, str): raise ValueError("memcache keys must be str type") def _check_value(self, value): if not self._pickle and not isinstance(value, str): raise ValueError("memcache values must be str type") def add(self, key, value, expireTime=0): self._check_key(key) self._check_value(value) if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len( str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT: return succeed(False) if key not in self._cache: if not expireTime: expireTime = 99999 self._cache[key] = (value, self._clock + expireTime, 0) return succeed(True) else: return succeed(False) def set(self, key, value, expireTime=0): self._check_key(key) self._check_value(value) if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len( str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT: return succeed(False) if not expireTime: expireTime = 99999 if key in self._cache: identifier = self._cache[key][2] identifier += 1 else: identifier = 0 self._cache[key] = (value, self._clock + expireTime, identifier) return succeed(True) def checkAndSet(self, key, value, cas, flags=0, expireTime=0): self._check_key(key) self._check_value(value) if len(key) > Memcacher.MEMCACHE_KEY_LIMIT or len( str(value)) > Memcacher.MEMCACHE_VALUE_LIMIT: return succeed(False) if not expireTime: expireTime = 99999 if key in self._cache: identifier = self._cache[key][2] if cas != str(identifier): return succeed(False) identifier += 1 else: return succeed(False) self._cache[key] = (value, self._clock + expireTime, identifier) return succeed(True) def get(self, key, withIdentifier=False): self._check_key(key) if len(key) > Memcacher.MEMCACHE_KEY_LIMIT: value, expires, identifier = (None, 0, "") else: value, expires, identifier = self._cache.get( key, (None, 0, "")) if self._clock >= expires: value = None identifier = "" if withIdentifier: return succeed((0, value, str(identifier))) else: return succeed(( 0, value, )) def delete(self, key): self._check_key(key) if len(key) > Memcacher.MEMCACHE_KEY_LIMIT: return succeed(False) try: del self._cache[key] return succeed(True) except KeyError: return succeed(False) def incr(self, key, delta=1): self._check_key(key) if len(key) > Memcacher.MEMCACHE_KEY_LIMIT: return succeed(False) value = self._cache.get(key, None) if value is not None: value, expire, identifier = value try: value = int(value) except ValueError: value = None else: value += delta self._cache[key] = ( str(value), expire, identifier, ) return succeed(value) def decr(self, key, delta=1): self._check_key(key) if len(key) > Memcacher.MEMCACHE_KEY_LIMIT: return succeed(False) value = self._cache.get(key, None) if value is not None: value, expire, identifier = value try: value = int(value) except ValueError: value = None else: value -= delta if value < 0: value = 0 self._cache[key] = ( str(value), expire, identifier, ) return succeed(value) def flushAll(self): self._cache = {} return succeed(True) def advanceClock(self, seconds): self._clock += seconds # TODO: an sqlite based cacher that can be used for multiple instance servers # in the absence of memcached. This is not ideal and we may want to not implement # this, but it is being documented for completeness. # # For now we implement a cacher that does not cache. class nullCacher(): """ A class implementing the memcache client API we care about but does not actually cache anything. """ def add(self, key, value, expireTime=0): return succeed(True) def set(self, key, value, expireTime=0): return succeed(True) def checkAndSet(self, key, value, cas, flags=0, expireTime=0): return succeed(True) def get(self, key, withIdentifier=False): return succeed(( 0, None, )) def delete(self, key): return succeed(True) def incr(self, key, delta=1): return succeed(None) def decr(self, key, delta=1): return succeed(None) def flushAll(self): return succeed(True) def __init__(self, namespace, pickle=False, no_invalidation=False, key_normalization=True): """ @param namespace: a unique namespace for this cache's keys @type namespace: C{str} @param pickle: if C{True} values will be pickled/unpickled when stored/read from the cache, if C{False} values will be stored directly (and therefore must be strings) @type pickle: C{bool} @param no_invalidation: if C{True} the cache is static - there will be no invalidations. This allows Memcacher to use the memoryCacher cache instead of nullCacher for the multi-instance case when memcached is not present,as there is no issue with caches in each instance getting out of sync. If C{False} the nullCacher will be used for the multi-instance case when memcached is not configured. @type no_invalidation: C{bool} @param key_normalization: if C{True} the key is assumed to possibly be longer than the Memcache key size and so additional work is done to truncate and append a hash. @type key_normalization: C{bool} """ assert len( namespace ) <= Memcacher.NAMESPACE_MAX_LENGTH, "Memcacher namespace must be less than or equal to %s characters long" % ( Memcacher.NAMESPACE_MAX_LENGTH, ) self._memcacheProtocol = None self._cachePoolHandle = namespace self._namespace = namespace self._pickle = pickle self._noInvalidation = no_invalidation self._key_normalization = key_normalization def _getMemcacheProtocol(self): if self._memcacheProtocol is not None: return self._memcacheProtocol if config.Memcached.Pools.Default.ClientEnabled: self._memcacheProtocol = self.getCachePool() elif config.ProcessType == "Single" or self._noInvalidation or self.allowTestCache: # The memory cacher handles python types natively, but we need to treat non-str types as an error # if pickling is off, so we use two global memory cachers for each pickle state if Memcacher.memoryCacheInstance[self._pickle] is None: Memcacher.memoryCacheInstance[ self._pickle] = Memcacher.memoryCacher(self._pickle) self._memcacheProtocol = Memcacher.memoryCacheInstance[ self._pickle] else: # NB no need to pickle the null cacher as it handles python types natively self._memcacheProtocol = Memcacher.nullCacher() self._pickle = False return self._memcacheProtocol def _normalizeKey(self, key): if isinstance(key, unicode): key = key.encode("utf-8") assert isinstance(key, str), "Key must be a str." if self._key_normalization: hash = hashlib.md5(key).hexdigest() key = key[:Memcacher.TRUNCATED_KEY_LENGTH] return "%s-%s" % ( key.translate(Memcacher.keyNormalizeTranslateTable), hash, ) else: return key def add(self, key, value, expireTime=0): proto = self._getMemcacheProtocol() my_value = value if self._pickle: my_value = cPickle.dumps(value) self.log.debug("Adding Cache Token for %r" % (key, )) return proto.add('%s:%s' % (self._namespace, self._normalizeKey(key)), my_value, expireTime=expireTime) def set(self, key, value, expireTime=0): proto = self._getMemcacheProtocol() my_value = value if self._pickle: my_value = cPickle.dumps(value) self.log.debug("Setting Cache Token for %r" % (key, )) return proto.set('%s:%s' % (self._namespace, self._normalizeKey(key)), my_value, expireTime=expireTime) def checkAndSet(self, key, value, cas, flags=0, expireTime=0): proto = self._getMemcacheProtocol() my_value = value if self._pickle: my_value = cPickle.dumps(value) self.log.debug("Setting Cache Token for %r" % (key, )) return proto.checkAndSet('%s:%s' % (self._namespace, self._normalizeKey(key)), my_value, cas, expireTime=expireTime) def get(self, key, withIdentifier=False): def _gotit(result, withIdentifier): if withIdentifier: _ignore_flags, identifier, value = result else: _ignore_flags, value = result if self._pickle and value is not None: value = cPickle.loads(value) if withIdentifier: value = (identifier, value) return value self.log.debug("Getting Cache Token for %r" % (key, )) d = self._getMemcacheProtocol().get( '%s:%s' % (self._namespace, self._normalizeKey(key)), withIdentifier=withIdentifier) d.addCallback(_gotit, withIdentifier) return d def delete(self, key): self.log.debug("Deleting Cache Token for %r" % (key, )) return self._getMemcacheProtocol().delete( '%s:%s' % (self._namespace, self._normalizeKey(key))) def incr(self, key, delta=1): self.log.debug("Incrementing Cache Token for %r" % (key, )) return self._getMemcacheProtocol().incr( '%s:%s' % (self._namespace, self._normalizeKey(key)), delta) def decr(self, key, delta=1): self.log.debug("Decrementing Cache Token for %r" % (key, )) return self._getMemcacheProtocol().incr( '%s:%s' % (self._namespace, self._normalizeKey(key)), delta) def flushAll(self): self.log.debug("Flushing All Cache Tokens") return self._getMemcacheProtocol().flushAll() @classmethod def reset(cls): """ Reset the memory cachers """ cls.memoryCacheInstance = {True: None, False: None}
class CalendarUserProxyPrincipalResource(CalDAVComplianceMixIn, PermissionsMixIn, DAVResourceWithChildrenMixin, DAVPrincipalResource): """ Calendar user proxy principal resource. """ log = Logger() def __init__(self, parent, proxyType): """ @param parent: the parent of this resource. @param proxyType: a C{str} containing the name of the resource. """ if self.isCollection(): slash = "/" else: slash = "" url = joinURL(parent.principalURL(), proxyType) + slash super(CalendarUserProxyPrincipalResource, self).__init__() DAVResourceWithChildrenMixin.__init__(self) self.parent = parent self.proxyType = proxyType self._url = url # FIXME: if this is supposed to be public, it needs a better name: self.pcollection = self.parent.parent.parent # Principal UID is parent's GUID plus the proxy type; this we can easily # map back to a principal. self.uid = "%s#%s" % (self.parent.principalUID(), proxyType) self._alternate_urls = tuple( joinURL(url, proxyType) + slash for url in parent.alternateURIs() if url.startswith("/")) def __str__(self): return "%s [%s]" % (self.parent, self.proxyType) def _index(self): """ Return the SQL database for this group principal. @return: the L{ProxyDB} for the principal collection. """ return ProxyDBService def resourceType(self): if self.proxyType == "calendar-proxy-read": return davxml.ResourceType.calendarproxyread # @UndefinedVariable elif self.proxyType == "calendar-proxy-write": return davxml.ResourceType.calendarproxywrite # @UndefinedVariable elif self.proxyType == "calendar-proxy-read-for": return davxml.ResourceType.calendarproxyreadfor # @UndefinedVariable elif self.proxyType == "calendar-proxy-write-for": return davxml.ResourceType.calendarproxywritefor # @UndefinedVariable else: return super(CalendarUserProxyPrincipalResource, self).resourceType() def isProxyType(self, read_write): if (read_write and self.proxyType == "calendar-proxy-write" or not read_write and self.proxyType == "calendar-proxy-read"): return True else: return False def isCollection(self): return True def etag(self): return succeed(None) def deadProperties(self): if not hasattr(self, "_dead_properties"): self._dead_properties = NonePropertyStore(self) return self._dead_properties def writeProperty(self, property, request): assert isinstance(property, davxml.WebDAVElement) if property.qname() == (dav_namespace, "group-member-set"): return self.setGroupMemberSet(property, request) return super(CalendarUserProxyPrincipalResource, self).writeProperty(property, request) @inlineCallbacks def setGroupMemberSet(self, new_members, request): # FIXME: as defined right now it is not possible to specify a # calendar-user-proxy group as a member of any other group since the # directory service does not know how to lookup these special resource # UIDs. # # Really, c-u-p principals should be treated the same way as any other # principal, so they should be allowed as members of groups. # # This implementation now raises an exception for any principal it # cannot find. # Break out the list into a set of URIs. members = [str(h) for h in new_members.children] # Map the URIs to principals and a set of UIDs. principals = [] newUIDs = set() for uri in members: principal = yield self.pcollection._principalForURI(uri) # Invalid principals MUST result in an error. if principal is None or principal.principalURL() != uri: raise HTTPError( StatusResponse( responsecode.BAD_REQUEST, "Attempt to use a non-existent principal %s " "as a group member of %s." % ( uri, self.principalURL(), ))) principals.append(principal) newUIDs.add(principal.principalUID()) # Get the old set of UIDs # oldUIDs = (yield self._index().getMembers(self.uid)) oldPrincipals = yield self.groupMembers() oldUIDs = [p.principalUID() for p in oldPrincipals] # Change membership yield self.setGroupMemberSetPrincipals(principals) # Invalidate the primary principal's cache, and any principal's whose # membership status changed yield self.parent.cacheNotifier.changed() changedUIDs = newUIDs.symmetric_difference(oldUIDs) for uid in changedUIDs: principal = yield self.pcollection.principalForUID(uid) if principal: yield principal.cacheNotifier.changed() returnValue(True) @inlineCallbacks def setGroupMemberSetPrincipals(self, principals): # Find our pseudo-record record = yield self.parent.record.service.recordWithShortName( self._recordTypeFromProxyType(), self.parent.principalUID()) # Set the members memberRecords = [p.record for p in principals] yield record.setMembers(memberRecords) ## # HTTP ## def htmlElement(self): """ Customize HTML display of proxy groups. """ return ProxyPrincipalElement(self) ## # DAV ## def displayName(self): return self.proxyType ## # ACL ## def alternateURIs(self): # FIXME: Add API to IDirectoryRecord for getting a record URI? return self._alternate_urls def principalURL(self): return self._url def principalUID(self): return self.uid def principalCollections(self): return self.parent.principalCollections() @inlineCallbacks def _expandMemberPrincipals(self, uid=None, relatives=None, uids=None, infinity=False): if uid is None: uid = self.principalUID() if relatives is None: relatives = set() if uids is None: uids = set() if uid not in uids: from twistedcaldav.directory.principal import DirectoryPrincipalResource uids.add(uid) principal = yield self.pcollection.principalForUID(uid) if isinstance(principal, CalendarUserProxyPrincipalResource): members = yield self._directGroupMembers() for member in members: if member.principalUID() not in uids: relatives.add(member) if infinity: yield self._expandMemberPrincipals( member.principalUID(), relatives, uids, infinity=infinity) elif isinstance(principal, DirectoryPrincipalResource): if infinity: members = yield principal.expandedGroupMembers() else: members = yield principal.groupMembers() relatives.update(members) returnValue(relatives) def _recordTypeFromProxyType(self): return { "calendar-proxy-read": DelegateRecordType.readDelegateGroup, "calendar-proxy-write": DelegateRecordType.writeDelegateGroup, "calendar-proxy-read-for": DelegateRecordType.readDelegatorGroup, "calendar-proxy-write-for": DelegateRecordType.writeDelegatorGroup, }.get(self.proxyType) @inlineCallbacks def _directGroupMembers(self): """ Fault in the record representing the sub principal for this proxy type (either read-only or read-write), then fault in the direct members of that record. """ memberPrincipals = [] record = yield self.parent.record.service.recordWithShortName( self._recordTypeFromProxyType(), self.parent.principalUID()) if record is not None: memberRecords = yield record.members() for record in memberRecords: if record is not None: principal = yield self.pcollection.principalForRecord( record) if principal is not None: if (principal.record.loginAllowed or principal.record.recordType is BaseRecordType.group): memberPrincipals.append(principal) returnValue(memberPrincipals) def groupMembers(self): return self._expandMemberPrincipals() @inlineCallbacks def expandedGroupMembers(self): """ Return the complete, flattened set of principals belonging to this group. """ returnValue((yield self._expandMemberPrincipals(infinity=True))) def groupMemberships(self): # Unlikely to ever want to put a subprincipal into a group return succeed([]) @inlineCallbacks def containsPrincipal(self, principal): """ Uses proxyFor information to turn the "contains principal" question around; rather than expanding this principal's groups to see if the other principal is a member, ask the other principal if they are a proxy for this principal's parent resource, since this principal is a proxy principal. @param principal: The principal to check @type principal: L{DirectoryCalendarPrincipalResource} @return: True if principal is a proxy (of the correct type) of our parent @rtype: C{boolean} """ readWrite = self.isProxyType(True) # is read-write if principal and self.parent in (yield principal.proxyFor(readWrite)): returnValue(True) returnValue(False)
class ProxyDB(AbstractADBAPIDatabase): """ A database to maintain calendar user proxy group memberships. SCHEMA: Group Database: ROW: GROUPNAME, MEMBER """ log = Logger() schema_version = "4" schema_type = "CALENDARUSERPROXY" class ProxyDBMemcacher(Memcacher): def __init__(self, namespace): super(ProxyDB.ProxyDBMemcacher, self).__init__( namespace, key_normalization=config.Memcached.ProxyDBKeyNormalization) def setMembers(self, guid, members): return self.set("members:%s" % (str(guid), ), str(",".join(members))) def setMemberships(self, guid, memberships): return self.set("memberships:%s" % (str(guid), ), str(",".join(memberships))) def getMembers(self, guid): def _value(value): if value: return set(value.split(",")) elif value is None: return None else: return set() d = self.get("members:%s" % (str(guid), )) d.addCallback(_value) return d def getMemberships(self, guid): def _value(value): if value: return set(value.split(",")) elif value is None: return None else: return set() d = self.get("memberships:%s" % (str(guid), )) d.addCallback(_value) return d def deleteMember(self, guid): return self.delete("members:%s" % (str(guid), )) def deleteMembership(self, guid): return self.delete("memberships:%s" % (str(guid), )) def __init__(self, dbID, dbapiName, dbapiArgs, **kwargs): AbstractADBAPIDatabase.__init__(self, dbID, dbapiName, dbapiArgs, True, **kwargs) self._memcacher = ProxyDB.ProxyDBMemcacher("ProxyDB") @inlineCallbacks def setGroupMembers(self, principalUID, members): """ Add a group membership record. @param principalUID: the UID of the group principal to add. @param members: a list UIDs of principals that are members of this group. """ # Get current members before we change them current_members = yield self.getMembers(principalUID) if current_members is None: current_members = () current_members = set(current_members) # Find changes update_members = set(members) remove_members = current_members.difference(update_members) add_members = update_members.difference(current_members) yield self.changeGroupMembersInDatabase(principalUID, add_members, remove_members) # Update cache for member in itertools.chain( remove_members, add_members, ): yield self._memcacher.deleteMembership(member) yield self._memcacher.deleteMember(principalUID) @inlineCallbacks def setGroupMembersInDatabase(self, principalUID, members): """ A blocking call to add a group membership record in the database. @param principalUID: the UID of the group principal to add. @param members: a list UIDs of principals that are members of this group. """ # Remove what is there, then add it back. yield self._delete_from_db(principalUID) yield self._add_to_db(principalUID, members) @inlineCallbacks def changeGroupMembersInDatabase(self, principalUID, addMembers, removeMembers): """ A blocking call to add a group membership record in the database. @param principalUID: the UID of the group principal to add. @param addMembers: a list UIDs of principals to be added as members of this group. @param removeMembers: a list UIDs of principals to be removed as members of this group. """ # Remove what is there, then add it back. for member in removeMembers: yield self._delete_from_db_one(principalUID, member) for member in addMembers: yield self._add_to_db_one(principalUID, member) @inlineCallbacks def removeGroup(self, principalUID): """ Remove a group membership record. @param principalUID: the UID of the group principal to remove. """ # Need to get the members before we do the delete members = yield self.getMembers(principalUID) yield self._delete_from_db(principalUID) # Update cache if members: for member in members: yield self._memcacher.deleteMembership(member) yield self._memcacher.deleteMember(principalUID) def getMembers(self, principalUID): """ Return the list of group member UIDs for the specified principal. @return: a deferred returning a C{set} of members. """ def gotCachedMembers(members): if members is not None: return members # Cache miss; compute members and update cache def gotMembersFromDB(dbmembers): members = set([row[0].encode("utf-8") for row in dbmembers]) d = self._memcacher.setMembers(principalUID, members) d.addCallback(lambda _: members) return d d = self.query("select MEMBER from GROUPS where GROUPNAME = :1", (principalUID.decode("utf-8"), )) d.addCallback(gotMembersFromDB) return d d = self._memcacher.getMembers(principalUID) d.addCallback(gotCachedMembers) return d def getMemberships(self, principalUID): """ Return the list of group principal UIDs the specified principal is a member of. @return: a deferred returning a C{set} of memberships. """ def gotCachedMemberships(memberships): if memberships is not None: return memberships # Cache miss; compute memberships and update cache def gotMembershipsFromDB(dbmemberships): memberships = set( [row[0].encode("utf-8") for row in dbmemberships]) d = self._memcacher.setMemberships(principalUID, memberships) d.addCallback(lambda _: memberships) return d d = self.query("select GROUPNAME from GROUPS where MEMBER = :1", (principalUID.decode("utf-8"), )) d.addCallback(gotMembershipsFromDB) return d d = self._memcacher.getMemberships(principalUID) d.addCallback(gotCachedMemberships) return d @inlineCallbacks def _add_to_db(self, principalUID, members): """ Insert the specified entry into the database. @param principalUID: the UID of the group principal to add. @param members: a list of UIDs or principals that are members of this group. """ for member in members: yield self.execute( """ insert into GROUPS (GROUPNAME, MEMBER) values (:1, :2) """, ( principalUID.decode("utf-8"), member, )) def _add_to_db_one(self, principalUID, memberUID): """ Insert the specified entry into the database. @param principalUID: the UID of the group principal to add. @param memberUID: the UID of the principal that is being added as a member of this group. """ return self.execute( """ insert into GROUPS (GROUPNAME, MEMBER) values (:1, :2) """, ( principalUID.decode("utf-8"), memberUID.decode("utf-8"), )) def _delete_from_db(self, principalUID): """ Deletes the specified entry from the database. @param principalUID: the UID of the group principal to remove. """ return self.execute("delete from GROUPS where GROUPNAME = :1", (principalUID.decode("utf-8"), )) def _delete_from_db_one(self, principalUID, memberUID): """ Deletes the specified entry from the database. @param principalUID: the UID of the group principal to remove. @param memberUID: the UID of the principal that is being removed as a member of this group. """ return self.execute( "delete from GROUPS where GROUPNAME = :1 and MEMBER = :2", ( principalUID.decode("utf-8"), memberUID.decode("utf-8"), )) def _delete_from_db_member(self, principalUID): """ Deletes the specified member entry from the database. @param principalUID: the UID of the member principal to remove. """ return self.execute("delete from GROUPS where MEMBER = :1", (principalUID.decode("utf-8"), )) def _db_version(self): """ @return: the schema version assigned to this index. """ return ProxyDB.schema_version def _db_type(self): """ @return: the collection type assigned to this index. """ return ProxyDB.schema_type @inlineCallbacks def _db_init_data_tables(self): """ Initialise the underlying database tables. @param q: a database cursor to use. """ # # GROUPS table # yield self._create_table( "GROUPS", ( ("GROUPNAME", "text"), ("MEMBER", "text"), ), ifnotexists=True, ) yield self._create_index( "GROUPNAMES", "GROUPS", ("GROUPNAME", ), ifnotexists=True, ) yield self._create_index( "MEMBERS", "GROUPS", ("MEMBER", ), ifnotexists=True, ) @inlineCallbacks def open(self): """ Open the database, normalizing all UUIDs in the process if necessary. """ result = yield super(ProxyDB, self).open() yield self._maybeNormalizeUUIDs() returnValue(result) @inlineCallbacks def _maybeNormalizeUUIDs(self): """ Normalize the UUIDs in the proxy database so they correspond to the normalized UUIDs in the main calendar database. """ alreadyDone = yield self._db_value_for_sql( "select VALUE from CALDAV where KEY = 'UUIDS_NORMALIZED'") if alreadyDone is None: for (groupname, member) in ((yield self._db_all_values_for_sql( "select GROUPNAME, MEMBER from GROUPS"))): grouplist = groupname.split("#") grouplist[0] = normalizeUUID(grouplist[0]) newGroupName = "#".join(grouplist) newMemberName = normalizeUUID(member) if newGroupName != groupname or newMemberName != member: yield self._db_execute( """ update GROUPS set GROUPNAME = :1, MEMBER = :2 where GROUPNAME = :3 and MEMBER = :4 """, [newGroupName, newMemberName, groupname, member]) yield self._db_execute(""" insert or ignore into CALDAV (KEY, VALUE) values ('UUIDS_NORMALIZED', 'YES') """) @inlineCallbacks def _db_upgrade_data_tables(self, old_version): """ Upgrade the data from an older version of the DB. @param old_version: existing DB's version number @type old_version: str """ # Add index if old version is less than "4" if int(old_version) < 4: yield self._create_index( "GROUPNAMES", "GROUPS", ("GROUPNAME", ), ifnotexists=True, ) yield self._create_index( "MEMBERS", "GROUPS", ("MEMBER", ), ifnotexists=True, ) def _db_empty_data_tables(self): """ Empty the underlying database tables. @param q: a database cursor to use. """ # # GROUPS table # return self._db_execute("delete from GROUPS") @inlineCallbacks def clean(self): if not self.initialized: yield self.open() for group in [ row[0] for row in (yield self.query("select GROUPNAME from GROUPS")) ]: self.removeGroup(group) yield super(ProxyDB, self).clean() @inlineCallbacks def getAllMembers(self): """ Retrieve all members that have been directly delegated to """ returnValue([ row[0] for row in (yield self.query("select DISTINCT MEMBER from GROUPS")) ])
class LinkResource(CalDAVComplianceMixIn, WrapperResource): """ This is similar to a WrapperResource except that we locate our resource dynamically. We need to deal with the case of a missing underlying resource (broken link) as indicated by self._linkedResource being None. """ log = Logger() def __init__(self, parent, link_url): self.parent = parent self.linkURL = link_url self.loopDetect = set() super(LinkResource, self).__init__(self.parent.principalCollections()) @inlineCallbacks def linkedResource(self, request): if not hasattr(self, "_linkedResource"): if self.linkURL in self.loopDetect: raise HTTPError( StatusResponse( responsecode.LOOP_DETECTED, "Recursive link target: %s" % (self.linkURL, ))) else: self.loopDetect.add(self.linkURL) self._linkedResource = (yield request.locateResource(self.linkURL)) self.loopDetect.remove(self.linkURL) if self._linkedResource is None: raise HTTPError( StatusResponse(responsecode.NOT_FOUND, "Missing link target: %s" % (self.linkURL, ))) returnValue(self._linkedResource) def isCollection(self): return True if hasattr(self, "_linkedResource") else False def resourceType(self): return self._linkedResource.resourceType() if hasattr( self, "_linkedResource") else davxml.ResourceType.link def locateChild(self, request, segments): def _defer(result): if result is None: return (self, server.StopTraversal) else: return (result, segments) d = self.linkedResource(request) d.addCallback(_defer) return d @inlineCallbacks def renderHTTP(self, request): linked_to = (yield self.linkedResource(request)) if linked_to: returnValue(linked_to) else: returnValue( http.StatusResponse( responsecode.OK, "Link resource with missing target: %s" % (self.linkURL, ))) def getChild(self, name): return self._linkedResource.getChild(name) if hasattr( self, "_linkedResource") else None @inlineCallbacks def hasProperty(self, property, request): hosted = (yield self.linkedResource(request)) result = (yield hosted.hasProperty(property, request)) if hosted else False returnValue(result) @inlineCallbacks def readProperty(self, property, request): hosted = (yield self.linkedResource(request)) result = (yield hosted.readProperty(property, request)) if hosted else None returnValue(result) @inlineCallbacks def writeProperty(self, property, request): hosted = (yield self.linkedResource(request)) result = (yield hosted.writeProperty(property, request)) if hosted else None returnValue(result)
class AbstractPropertyStore(DictMixin, object): """ Base property store. """ log = Logger() implements(IPropertyStore) _defaultShadowableKeys = frozenset() _defaultProxyOverrideKeys = frozenset() _defaultGlobalKeys = frozenset(( PropertyName.fromElement(davxml.ACL), PropertyName.fromElement(davxml.ResourceID), PropertyName.fromElement(davxml.ResourceType), PropertyName.fromElement(davxml.GETContentType), PropertyName.fromElement(TwistedGETContentMD5), PropertyName.fromElement(TwistedQuotaRootProperty), )) def __init__(self, defaultUser, shareeUser=None, proxyUser=None): """ Instantiate the property store for a user. The default is the default user (owner) property to read in the case of global or shadowable properties. The sharee user is a user sharing the user to read for per-user properties. @param defaultUser: the default user uid @type defaultUser: C{str} @param shareeUser: the per user uid or None if the same as defaultUser @type shareeUser: C{str} @param proxyUser: the proxy uid or None if no proxy @type proxyUser: C{str} """ assert (defaultUser is not None or shareeUser is not None) self._defaultUser = shareeUser if defaultUser is None else defaultUser self._perUser = defaultUser if shareeUser is None else shareeUser self._proxyUser = self._perUser if proxyUser is None else proxyUser self._shadowableKeys = set( AbstractPropertyStore._defaultShadowableKeys) self._proxyOverrideKeys = set( AbstractPropertyStore._defaultProxyOverrideKeys) self._globalKeys = set(AbstractPropertyStore._defaultGlobalKeys) def __str__(self): return "<%s>" % (self.__class__.__name__) def _setDefaultUserUID(self, uid): self._defaultUser = uid def _setPerUserUID(self, uid): self._perUser = uid def _setProxyUID(self, uid): self._proxyUser = uid def setSpecialProperties(self, shadowableKeys, globalKeys, proxyOverrideKeys): self._shadowableKeys.update(shadowableKeys) self._proxyOverrideKeys.update(proxyOverrideKeys) self._globalKeys.update(globalKeys) # # Subclasses must override these # def _getitem_uid(self, key, uid): raise NotImplementedError() def _setitem_uid(self, key, value, uid): raise NotImplementedError() def _delitem_uid(self, key, uid): raise NotImplementedError() def _keys_uid(self, uid): raise NotImplementedError() def _removeResource(self): raise NotImplementedError() def flush(self): raise NotImplementedError() def abort(self): raise NotImplementedError() # # Required UserDict implementations # def __getitem__(self, key): # Return proxy value if it exists, else fall through to normal logic if self._proxyUser != self._perUser and self.isProxyOverrideProperty( key): try: return self._getitem_uid(key, self._proxyUser) except KeyError: pass # Handle per-user behavior if self.isShadowableProperty(key): try: result = self._getitem_uid(key, self._perUser) except KeyError: result = self._getitem_uid(key, self._defaultUser) return result elif self.isGlobalProperty(key): return self._getitem_uid(key, self._defaultUser) else: return self._getitem_uid(key, self._perUser) def __setitem__(self, key, value): # Handle per-user behavior if self.isGlobalProperty(key): return self._setitem_uid(key, value, self._defaultUser) # Handle proxy behavior elif self._proxyUser != self._perUser and self.isProxyOverrideProperty( key): return self._setitem_uid(key, value, self._proxyUser) # Remainder is per user else: return self._setitem_uid(key, value, self._perUser) def __delitem__(self, key): # Delete proxy value if it exists, else fall through to normal logic if self._proxyUser != self._perUser and self.isProxyOverrideProperty( key): try: self._delitem_uid(key, self._proxyUser) return except KeyError: pass # Handle per-user behavior if self.isShadowableProperty(key): try: self._delitem_uid(key, self._perUser) except KeyError: # It is OK for shadowable delete to fail pass elif self.isGlobalProperty(key): self._delitem_uid(key, self._defaultUser) else: self._delitem_uid(key, self._perUser) def keys(self): userkeys = list(self._keys_uid(self._perUser)) if self._defaultUser != self._perUser: defaultkeys = self._keys_uid(self._defaultUser) for key in defaultkeys: if self.isShadowableProperty(key) and key not in userkeys: userkeys.append(key) return tuple(userkeys) def update(self, other): # FIXME: direct tests. # FIXME: support positional signature (although since strings aren't # valid, it should just raise an error. for key in other: self[key] = other[key] def isShadowableProperty(self, key): return key in self._shadowableKeys def isProxyOverrideProperty(self, key): return key in self._proxyOverrideKeys def isGlobalProperty(self, key): return key in self._globalKeys def copyAllProperties(self, other): """ Copy all the properties from another store into this one. This needs to be done independently of the UID. Each underlying store will need to implement this. """ pass
class MailGatewayTokensDatabase(AbstractSQLDatabase): """ A database to maintain "plus-address" tokens for IMIP requests. SCHEMA: Token Database: ROW: TOKEN, ORGANIZER, ATTENDEE, ICALUID, DATESTAMP """ log = Logger() dbType = "MAILGATEWAYTOKENS" dbFilename = "mailgatewaytokens.sqlite" dbFormatVersion = "1" def __init__(self, path): if path != ":memory:": path = os.path.join(path, MailGatewayTokensDatabase.dbFilename) super(MailGatewayTokensDatabase, self).__init__(path, True) def createToken(self, organizer, attendee, icaluid, token=None): if token is None: token = str(uuid.uuid4()) self._db_execute( """ insert into TOKENS (TOKEN, ORGANIZER, ATTENDEE, ICALUID, DATESTAMP) values (:1, :2, :3, :4, :5) """, token, organizer, attendee, icaluid, datetime.date.today()) self._db_commit() return token def lookupByToken(self, token): results = list( self._db_execute( """ select ORGANIZER, ATTENDEE, ICALUID from TOKENS where TOKEN = :1 """, token)) if len(results) != 1: return None return results[0] def getToken(self, organizer, attendee, icaluid): token = self._db_value_for_sql( """ select TOKEN from TOKENS where ORGANIZER = :1 and ATTENDEE = :2 and ICALUID = :3 """, organizer, attendee, icaluid) if token is not None: # update the datestamp on the token to keep it from being purged self._db_execute( """ update TOKENS set DATESTAMP = :1 WHERE TOKEN = :2 """, datetime.date.today(), token) return str(token) else: return None def getAllTokens(self): results = list( self._db_execute(""" select TOKEN, ORGANIZER, ATTENDEE, ICALUID from TOKENS """)) return results def deleteToken(self, token): self._db_execute( """ delete from TOKENS where TOKEN = :1 """, token) self._db_commit() def purgeOldTokens(self, before): self._db_execute( """ delete from TOKENS where DATESTAMP < :1 """, before) self._db_commit() def lowercase(self): """ Lowercase mailto: addresses (and uppercase urn:uuid: addresses!) so they can be located via normalized names. """ rows = self._db_execute(""" select ORGANIZER, ATTENDEE from TOKENS """) for row in rows: organizer = row[0] attendee = row[1] if organizer.lower().startswith("mailto:"): self._db_execute( """ update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2 """, organizer.lower(), organizer) else: from txdav.base.datastore.util import normalizeUUIDOrNot self._db_execute( """ update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2 """, normalizeUUIDOrNot(organizer), organizer) # ATTENDEEs are always mailto: so unconditionally lower(). self._db_execute( """ update TOKENS set ATTENDEE = :1 WHERE ATTENDEE = :2 """, attendee.lower(), attendee) self._db_commit() def _db_version(self): """ @return: the schema version assigned to this index. """ return MailGatewayTokensDatabase.dbFormatVersion def _db_type(self): """ @return: the collection type assigned to this index. """ return MailGatewayTokensDatabase.dbType def _db_init_data_tables(self, q): """ Initialise the underlying database tables. @param q: a database cursor to use. """ # # TOKENS table # q.execute(""" create table TOKENS ( TOKEN text, ORGANIZER text, ATTENDEE text, ICALUID text, DATESTAMP date ) """) q.execute(""" create index TOKENSINDEX on TOKENS (TOKEN) """) def _db_upgrade_data_tables(self, q, old_version): """ Upgrade the data from an older version of the DB. @param q: a database cursor to use. @param old_version: existing DB's version number @type old_version: str """ pass
class MemcachedUIDReserver(CachePoolUserMixIn): log = Logger() def __init__(self, index, cachePool=None): self.index = index self._cachePool = cachePool def _key(self, uid): return 'reservation:%s' % (hashlib.md5( '%s:%s' % (uid, self.index.resource.fp.path)).hexdigest()) def reserveUID(self, uid): uid = uid.encode('utf-8') self.log.debug( "Reserving UID {uid} @ {path}", uid=uid, path=self.index.resource.fp.path, ) def _handleFalse(result): if result is False: raise ReservationError( "UID %s already reserved for calendar collection %s." % (uid, self.index.resource)) d = self.getCachePool().add(self._key(uid), 'reserved', expireTime=config.UIDReservationTimeOut) d.addCallback(_handleFalse) return d def unreserveUID(self, uid): uid = uid.encode('utf-8') self.log.debug( "Unreserving UID {uid} @ {path}", uid=uid, path=self.index.resource.fp.path, ) def _handleFalse(result): if result is False: raise ReservationError( "UID %s is not reserved for calendar collection %s." % (uid, self.index.resource)) d = self.getCachePool().delete(self._key(uid)) d.addCallback(_handleFalse) return d def isReservedUID(self, uid): uid = uid.encode('utf-8') self.log.debug( "Is reserved UID {uid} @ {path}", uid=uid, path=self.index.resource.fp.path, ) def _checkValue((flags, value)): if value is None: return False else: return True d = self.getCachePool().get(self._key(uid)) d.addCallback(_checkValue) return d
class AbstractCalendarIndex(AbstractSQLDatabase): """ Calendar collection index abstract base class that defines the apis for the index. This will be subclassed for the two types of index behaviour we need: one for regular calendar collections, one for schedule calendar collections. """ log = Logger() def __init__(self, resource): """ @param resource: the L{CalDAVResource} resource to index. C{resource} must be a calendar collection (ie. C{resource.isPseudoCalendarCollection()} returns C{True}.) """ self.resource = resource db_filename = self.resource.fp.child(db_basename).path super(AbstractCalendarIndex, self).__init__(db_filename, False) self.resource._txn.postCommit(self._db_close) self.resource._txn.postAbort(self._db_close) def create(self): """ Create the index and initialize it. """ self._db() def reserveUID(self, uid): """ Reserve a UID for this index's resource. @param uid: the UID to reserve @raise ReservationError: if C{uid} is already reserved """ raise NotImplementedError def unreserveUID(self, uid): """ Unreserve a UID for this index's resource. @param uid: the UID to reserve @raise ReservationError: if C{uid} is not reserved """ raise NotImplementedError def isReservedUID(self, uid): """ Check to see whether a UID is reserved. @param uid: the UID to check @return: True if C{uid} is reserved, False otherwise. """ raise NotImplementedError def isAllowedUID(self, uid, *names): """ Checks to see whether to allow an operation with adds the the specified UID is allowed to the index. Specifically, the operation may not violate the constraint that UIDs must be unique, and the UID must not be reserved. @param uid: the UID to check @param names: the names of resources being replaced or deleted by the operation; UIDs associated with these resources are not checked. @return: True if the UID is not in the index and is not reserved, False otherwise. """ raise NotImplementedError def resourceNamesForUID(self, uid): """ Looks up the names of the resources with the given UID. @param uid: the UID of the resources to look up. @return: a list of resource names """ names = self._db_values_for_sql( "select NAME from RESOURCE where UID = :1", uid) # # Check that each name exists as a child of self.resource. If not, the # resource record is stale. # resources = [] for name in names: name_utf8 = name.encode("utf-8") if name is not None and self.resource.getChild(name_utf8) is None: # Clean up log.error( "Stale resource record found for child {name} with UID {uid} in {rsrc!r}", name=name, uid=uid, resource=self.resource) self._delete_from_db(name, uid, False) self._db_commit() else: resources.append(name_utf8) return resources def resourceNameForUID(self, uid): """ Looks up the name of the resource with the given UID. @param uid: the UID of the resource to look up. @return: If the resource is found, its name; C{None} otherwise. """ result = None for name in self.resourceNamesForUID(uid): assert result is None, "More than one resource with UID %s in calendar collection %r" % ( uid, self) result = name return result def resourceUIDForName(self, name): """ Looks up the UID of the resource with the given name. @param name: the name of the resource to look up. @return: If the resource is found, the UID of the resource; C{None} otherwise. """ uid = self._db_value_for_sql( "select UID from RESOURCE where NAME = :1", name) return uid def componentTypeCounts(self): """ Count each type of component. """ return self._db_execute( "select TYPE, COUNT(TYPE) from RESOURCE group by TYPE") def addResource(self, name, calendar, fast=False, reCreate=False): """ Adding or updating an existing resource. To check for an update we attempt to get an existing UID for the resource name. If present, then the index entries for that UID are removed. After that the new index entries are added. @param name: the name of the resource to add. @param calendar: a L{Calendar} object representing the resource contents. @param fast: if C{True} do not do commit, otherwise do commit. """ oldUID = self.resourceUIDForName(name) if oldUID is not None: self._delete_from_db(name, oldUID, False) self._add_to_db(name, calendar, reCreate=reCreate) if not fast: self._db_commit() def deleteResource(self, name): """ Remove this resource from the index. @param name: the name of the resource to add. @param uid: the UID of the calendar component in the resource. """ uid = self.resourceUIDForName(name) if uid is not None: self._delete_from_db(name, uid) self._db_commit() def resourceExists(self, name): """ Determines whether the specified resource name exists in the index. @param name: the name of the resource to test @return: True if the resource exists, False if not """ uid = self._db_value_for_sql( "select UID from RESOURCE where NAME = :1", name) return uid is not None def resourcesExist(self, names): """ Determines whether the specified resource name exists in the index. @param names: a C{list} containing the names of the resources to test @return: a C{list} of all names that exist """ statement = "select NAME from RESOURCE where NAME in (" for ctr in (item[0] for item in enumerate(names)): if ctr != 0: statement += ", " statement += ":%s" % (ctr, ) statement += ")" results = self._db_values_for_sql(statement, *names) return results def testAndUpdateIndex(self, minDate): # Find out if the index is expanded far enough names = self.notExpandedBeyond(minDate) # Actually expand recurrence max for name in names: self.log.info( "Search falls outside range of index for {name} {date}", name=name, date=minDate) self.reExpandResource(name, minDate) def whatchanged(self, revision): results = [ (name.encode("utf-8"), deleted) for name, deleted in self._db_execute( "select NAME, DELETED from REVISIONS where REVISION > :1", revision) ] results.sort(key=lambda x: x[1]) changed = [] deleted = [] invalid = [] for name, wasdeleted in results: if name: if wasdeleted == 'Y': if revision: deleted.append(name) else: changed.append(name) else: raise SyncTokenValidException return (changed, deleted, invalid) def lastRevision(self): return self._db_value_for_sql("select REVISION from REVISION_SEQUENCE") def bumpRevision(self, fast=False): self._db_execute( """ update REVISION_SEQUENCE set REVISION = REVISION + 1 """, ) self._db_commit() return self._db_value_for_sql( """ select REVISION from REVISION_SEQUENCE """, ) def indexedSearch(self, filter, useruid="", fbtype=False): """ Finds resources matching the given qualifiers. @param filter: the L{Filter} for the calendar-query to execute. @return: an iterable of tuples for each resource matching the given C{qualifiers}. The tuples are C{(name, uid, type)}, where C{name} is the resource name, C{uid} is the resource UID, and C{type} is the resource iCalendar component type. """ # Make sure we have a proper Filter element and get the partial SQL # statement to use. if isinstance(filter, Filter): if fbtype: # Lookup the useruid - try the empty (default) one if needed dbuseruid = self._db_value_for_sql( "select PERUSERID from PERUSER where USERUID == :1", useruid, ) else: dbuseruid = "" qualifiers = sqlcalendarquery(filter, None, dbuseruid, fbtype) if qualifiers is not None: # Determine how far we need to extend the current expansion of # events. If we have an open-ended time-range we will expand one # year past the start. That should catch bounded recurrences - unbounded # will have been indexed with an "infinite" value always included. maxDate, isStartDate = filter.getmaxtimerange() if maxDate: maxDate = maxDate.duplicate() maxDate.setDateOnly(True) if isStartDate: maxDate += Duration(days=365) self.testAndUpdateIndex(maxDate) else: # We cannot handle this filter in an indexed search raise IndexedSearchException() else: qualifiers = None # Perform the search if qualifiers is None: rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE") else: if fbtype: # For a free-busy time-range query we return all instances rowiter = self._db_execute( "select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE, RESOURCE.ORGANIZER, TIMESPAN.FLOAT, TIMESPAN.START, TIMESPAN.END, TIMESPAN.FBTYPE, TIMESPAN.TRANSPARENT, TRANSPARENCY.TRANSPARENT" + qualifiers[0], *qualifiers[1]) else: rowiter = self._db_execute( "select DISTINCT RESOURCE.NAME, RESOURCE.UID, RESOURCE.TYPE" + qualifiers[0], *qualifiers[1]) # Check result for missing resources results = [] for row in rowiter: name = row[0] if self.resource.getChild(name.encode("utf-8")): if fbtype: row = list(row) if row[9]: row[8] = row[9] del row[9] results.append(row) else: log.error( "Calendar resource {name} is missing from {rsrc!r}. Removing from index.", name=name, rsrc=self.resource, ) self.deleteResource(name) return results def bruteForceSearch(self): """ List the whole index and tests for existence, updating the index @return: all resources in the index """ # List all resources rowiter = self._db_execute("select NAME, UID, TYPE from RESOURCE") # Check result for missing resources: results = [] for row in rowiter: name = row[0] if self.resource.getChild(name.encode("utf-8")): results.append(row) else: log.error( "Calendar resource {name} is missing from {rsrc!r}. Removing from index.", name=name, rsrc=self.resource, ) return results def _db_version(self): """ @return: the schema version assigned to this index. """ return schema_version def _add_to_db(self, name, calendar, cursor=None, expand_until=None, reCreate=False): """ Records the given calendar resource in the index with the given name. Resource names and UIDs must both be unique; only one resource name may be associated with any given UID and vice versa. NB This method does not commit the changes to the db - the caller MUST take care of that @param name: the name of the resource to add. @param calendar: a L{Calendar} object representing the resource contents. """ raise NotImplementedError def _delete_from_db(self, name, uid, dorevision=True): """ Deletes the specified entry from all dbs. @param name: the name of the resource to delete. @param uid: the uid of the resource to delete. """ raise NotImplementedError