def doWork(self):
# Get the minimum valid revision
minValidRevision = int(
(yield self.transaction.calendarserverValue("MIN-VALID-REVISION")))
# get max revision on table rows before dateLimit
dateLimit = self.dateCutoff()
maxRevOlderThanDate = 0
# TODO: Use one Select statement
for table in (
schema.CALENDAR_OBJECT_REVISIONS,
schema.NOTIFICATION_OBJECT_REVISIONS,
schema.ADDRESSBOOK_OBJECT_REVISIONS,
schema.ABO_MEMBERS,
):
revisionRows = yield Select(
[Max(table.REVISION)],
From=table,
Where=(table.MODIFIED < dateLimit),
).on(self.transaction)
if revisionRows:
tableMaxRevision = revisionRows[0][0]
if tableMaxRevision > maxRevOlderThanDate:
maxRevOlderThanDate = tableMaxRevision
if maxRevOlderThanDate > minValidRevision:
# save new min valid revision
yield self.transaction.updateCalendarserverValue(
"MIN-VALID-REVISION", maxRevOlderThanDate + 1)
# Schedule revision cleanup
yield RevisionCleanupWork.reschedule(self.transaction, seconds=0)
def _revisionsForResourceIDs(cls, resourceIDs):
rev = cls._revisionsSchema
return Select(
[rev.RESOURCE_ID, Max(rev.REVISION)],
From=rev,
Where=rev.RESOURCE_ID.In(Parameter(
"resourceIDs", len(resourceIDs))).And(
(rev.RESOURCE_NAME != None).Or(rev.DELETED == False)),
GroupBy=rev.RESOURCE_ID)
def _childSyncTokenQuery(cls):
"""
DAL query for retrieving the sync token of a L{CommonHomeChild} based on
its resource ID.
"""
rev = cls._revisionsSchema
return Select([Max(rev.REVISION)],
From=rev,
Where=rev.RESOURCE_ID == Parameter("resourceID"))
def determineNewest(uid, homeType):
"""
Construct a query to determine the modification time of the newest object
in a given home.
@param uid: the UID of the home to scan.
@type uid: C{str}
@param homeType: The type of home to scan; C{ECALENDARTYPE},
C{ENOTIFICATIONTYPE}, or C{EADDRESSBOOKTYPE}.
@type homeType: C{int}
@return: A select query that will return a single row containing a single
column which is the maximum value.
@rtype: L{Select}
"""
if homeType == ENOTIFICATIONTYPE:
return Select([Max(schema.NOTIFICATION.MODIFIED)],
From=schema.NOTIFICATION_HOME.join(
schema.NOTIFICATION,
on=schema.NOTIFICATION_HOME.RESOURCE_ID ==
schema.NOTIFICATION.NOTIFICATION_HOME_RESOURCE_ID),
Where=schema.NOTIFICATION_HOME.OWNER_UID == uid)
homeTypeName = {
ECALENDARTYPE: "CALENDAR",
EADDRESSBOOKTYPE: "ADDRESSBOOK"
}[homeType]
home = getattr(schema, homeTypeName + "_HOME")
bind = getattr(schema, homeTypeName + "_BIND")
child = getattr(schema, homeTypeName)
obj = getattr(schema, homeTypeName + "_OBJECT")
return Select([Max(obj.MODIFIED)],
From=home.join(
bind, on=bind.HOME_RESOURCE_ID == home.RESOURCE_ID).join(
child,
on=child.RESOURCE_ID == bind.RESOURCE_ID).join(
obj,
on=obj.PARENT_RESOURCE_ID == child.RESOURCE_ID),
Where=(bind.BIND_MODE == 0).And(home.OWNER_UID == uid))
class NotificationCollection(FancyEqMixin, _SharedSyncLogic):
log = Logger()
implements(INotificationCollection)
compareAttributes = (
"_ownerUID",
"_resourceID",
)
_revisionsSchema = schema.NOTIFICATION_OBJECT_REVISIONS
_homeSchema = schema.NOTIFICATION_HOME
_externalClass = None
@classmethod
def makeClass(cls, transaction, homeData):
"""
Build the actual home class taking into account the possibility that we might need to
switch in the external version of the class.
@param transaction: transaction
@type transaction: L{CommonStoreTransaction}
@param homeData: home table column data
@type homeData: C{list}
"""
status = homeData[cls.homeColumns().index(cls._homeSchema.STATUS)]
if status == _HOME_STATUS_EXTERNAL:
home = cls._externalClass(transaction, homeData)
else:
home = cls(transaction, homeData)
return home.initFromStore()
@classmethod
def homeColumns(cls):
"""
Return a list of column names to retrieve when doing an ownerUID->home lookup.
"""
# Common behavior is to have created and modified
return (
cls._homeSchema.RESOURCE_ID,
cls._homeSchema.OWNER_UID,
cls._homeSchema.STATUS,
)
@classmethod
def homeAttributes(cls):
"""
Return a list of attributes names to map L{homeColumns} to.
"""
# Common behavior is to have created and modified
return (
"_resourceID",
"_ownerUID",
"_status",
)
def __init__(self, txn, homeData):
self._txn = txn
for attr, value in zip(self.homeAttributes(), homeData):
setattr(self, attr, value)
self._txn = txn
self._dataVersion = None
self._notifications = {}
self._notificationNames = None
self._syncTokenRevision = None
# Make sure we have push notifications setup to push on this collection
# as well as the home it is in
self._notifiers = dict([(
factory_name,
factory.newNotifier(self),
) for factory_name, factory in txn._notifierFactories.items()])
@inlineCallbacks
def initFromStore(self):
"""
Initialize this object from the store.
"""
yield self._loadPropertyStore()
returnValue(self)
@property
def _home(self):
"""
L{NotificationCollection} serves as its own C{_home} for the purposes of
working with L{_SharedSyncLogic}.
"""
return self
@classmethod
def notificationsWithUID(cls, txn, uid, status=None, create=False):
return cls.notificationsWith(txn,
None,
uid,
status=status,
create=create)
@classmethod
def notificationsWithResourceID(cls, txn, rid):
return cls.notificationsWith(txn, rid, None)
@classmethod
@inlineCallbacks
def notificationsWith(cls, txn, rid, uid, status=None, create=False):
"""
@param uid: I'm going to assume uid is utf-8 encoded bytes
"""
if rid is not None:
query = cls._homeSchema.RESOURCE_ID == rid
elif uid is not None:
query = cls._homeSchema.OWNER_UID == uid
if status is not None:
query = query.And(cls._homeSchema.STATUS == status)
else:
statusSet = (
_HOME_STATUS_NORMAL,
_HOME_STATUS_EXTERNAL,
)
if txn._allowDisabled:
statusSet += (_HOME_STATUS_DISABLED, )
query = query.And(cls._homeSchema.STATUS.In(statusSet))
else:
raise AssertionError("One of rid or uid must be set")
results = yield Select(
cls.homeColumns(),
From=cls._homeSchema,
Where=query,
).on(txn)
if len(results) > 1:
# Pick the best one in order: normal, disabled and external
byStatus = dict([
(result[cls.homeColumns().index(cls._homeSchema.STATUS)],
result) for result in results
])
result = byStatus.get(_HOME_STATUS_NORMAL)
if result is None:
result = byStatus.get(_HOME_STATUS_DISABLED)
if result is None:
result = byStatus.get(_HOME_STATUS_EXTERNAL)
elif results:
result = results[0]
else:
result = None
if result:
# Return object that already exists in the store
homeObject = yield cls.makeClass(txn, result)
returnValue(homeObject)
else:
# Can only create when uid is specified
if not create or uid is None:
returnValue(None)
# Determine if the user is local or external
record = yield txn.directoryService().recordWithUID(
uid.decode("utf-8"))
if record is None:
raise DirectoryRecordNotFoundError(
"Cannot create home for UID since no directory record exists: {}"
.format(uid))
if status is None:
createStatus = _HOME_STATUS_NORMAL if record.thisServer(
) else _HOME_STATUS_EXTERNAL
elif status == _HOME_STATUS_MIGRATING:
if record.thisServer():
raise RecordNotAllowedError(
"Cannot migrate a user data for a user already hosted on this server"
)
createStatus = status
elif status in (
_HOME_STATUS_NORMAL,
_HOME_STATUS_EXTERNAL,
):
createStatus = status
else:
raise RecordNotAllowedError(
"Cannot create home with status {}: {}".format(
status, uid))
# Use savepoint so we can do a partial rollback if there is a race
# condition where this row has already been inserted
savepoint = SavepointAction("notificationsWithUID")
yield savepoint.acquire(txn)
try:
resourceid = (yield Insert(
{
cls._homeSchema.OWNER_UID: uid,
cls._homeSchema.STATUS: createStatus,
},
Return=cls._homeSchema.RESOURCE_ID).on(txn))[0][0]
except Exception:
# FIXME: Really want to trap the pg.DatabaseError but in a non-
# DB specific manner
yield savepoint.rollback(txn)
# Retry the query - row may exist now, if not re-raise
results = yield Select(
cls.homeColumns(),
From=cls._homeSchema,
Where=query,
).on(txn)
if results:
homeObject = yield cls.makeClass(txn, results[0])
returnValue(homeObject)
else:
raise
else:
yield savepoint.release(txn)
# Note that we must not cache the owner_uid->resource_id
# mapping in the query cacher when creating as we don't want that to appear
# until AFTER the commit
results = yield Select(
cls.homeColumns(),
From=cls._homeSchema,
Where=cls._homeSchema.RESOURCE_ID == resourceid,
).on(txn)
homeObject = yield cls.makeClass(txn, results[0])
if homeObject.normal():
yield homeObject._initSyncToken()
yield homeObject.notifyChanged()
returnValue(homeObject)
@inlineCallbacks
def _loadPropertyStore(self):
self._propertyStore = yield PropertyStore.load(
self._ownerUID,
self._ownerUID,
None,
self._txn,
self._resourceID,
notifyCallback=self.notifyChanged)
def __repr__(self):
return "<%s: %s>" % (self.__class__.__name__, self._resourceID)
def id(self):
"""
Retrieve the store identifier for this collection.
@return: store identifier.
@rtype: C{int}
"""
return self._resourceID
@classproperty
def _dataVersionQuery(cls):
nh = cls._homeSchema
return Select([nh.DATAVERSION],
From=nh,
Where=nh.RESOURCE_ID == Parameter("resourceID"))
@inlineCallbacks
def dataVersion(self):
if self._dataVersion is None:
self._dataVersion = (yield self._dataVersionQuery.on(
self._txn, resourceID=self._resourceID))[0][0]
returnValue(self._dataVersion)
def name(self):
return "notification"
def uid(self):
return self._ownerUID
def status(self):
return self._status
@inlineCallbacks
def setStatus(self, newStatus):
"""
Mark this home as being purged.
"""
# Only if different
if self._status != newStatus:
yield Update(
{
self._homeSchema.STATUS: newStatus
},
Where=(self._homeSchema.RESOURCE_ID == self._resourceID),
).on(self._txn)
self._status = newStatus
def normal(self):
"""
Is this an normal (internal) home.
@return: a L{bool}.
"""
return self._status == _HOME_STATUS_NORMAL
def external(self):
"""
Is this an external home.
@return: a L{bool}.
"""
return self._status == _HOME_STATUS_EXTERNAL
def owned(self):
return True
def ownerHome(self):
return self._home
def viewerHome(self):
return self._home
def notificationObjectRecords(self):
return NotificationObjectRecord.querysimple(
self._txn, notificationHomeResourceID=self.id())
@inlineCallbacks
def notificationObjects(self):
results = (yield NotificationObject.loadAllObjects(self))
for result in results:
self._notifications[result.uid()] = result
self._notificationNames = sorted([result.name() for result in results])
returnValue(results)
_notificationUIDsForHomeQuery = Select(
[schema.NOTIFICATION.NOTIFICATION_UID],
From=schema.NOTIFICATION,
Where=schema.NOTIFICATION.NOTIFICATION_HOME_RESOURCE_ID == Parameter(
"resourceID"))
@inlineCallbacks
def listNotificationObjects(self):
"""
List the names of all notification objects in this collection. Note that the name
is actually the UID value with ".xml" appended, as per L{NotificationObject.name}.
"""
if self._notificationNames is None:
rows = yield self._notificationUIDsForHomeQuery.on(
self._txn, resourceID=self._resourceID)
self._notificationNames = sorted([row[0] + ".xml" for row in rows])
returnValue(self._notificationNames)
# used by _SharedSyncLogic.resourceNamesSinceRevision()
def listObjectResources(self):
return self.listNotificationObjects()
def _nameToUID(self, name):
"""
Based on the file-backed implementation, the 'name' is just uid +
".xml".
"""
return name.rsplit(".", 1)[0]
def notificationObjectWithName(self, name):
return self.notificationObjectWithUID(self._nameToUID(name))
@memoizedKey("uid", "_notifications")
@inlineCallbacks
def notificationObjectWithUID(self, uid):
"""
Create an empty notification object first then have it initialize itself
from the store.
"""
no = NotificationObject(self, uid)
no = (yield no.initFromStore())
returnValue(no)
@inlineCallbacks
def writeNotificationObject(self, uid, notificationtype, notificationdata):
inserting = False
notificationObject = yield self.notificationObjectWithUID(uid)
if notificationObject is None:
notificationObject = NotificationObject(self, uid)
inserting = True
yield notificationObject.setData(uid,
notificationtype,
notificationdata,
inserting=inserting)
if inserting:
yield self._insertRevision(notificationObject.name())
if self._notificationNames is not None:
self._notificationNames.append(notificationObject.name())
else:
yield self._updateRevision(notificationObject.name())
yield self.notifyChanged()
returnValue(notificationObject)
def removeNotificationObjectWithName(self, name):
if self._notificationNames is not None:
self._notificationNames.remove(name)
return self.removeNotificationObjectWithUID(self._nameToUID(name))
_removeByUIDQuery = Delete(
From=schema.NOTIFICATION,
Where=(schema.NOTIFICATION.NOTIFICATION_UID == Parameter("uid")).And(
schema.NOTIFICATION.NOTIFICATION_HOME_RESOURCE_ID == Parameter(
"resourceID")))
@inlineCallbacks
def removeNotificationObjectWithUID(self, uid):
yield self._removeByUIDQuery.on(self._txn,
uid=uid,
resourceID=self._resourceID)
self._notifications.pop(uid, None)
yield self._deleteRevision("%s.xml" % (uid, ))
yield self.notifyChanged()
_initSyncTokenQuery = Insert(
{
_revisionsSchema.HOME_RESOURCE_ID: Parameter("resourceID"),
_revisionsSchema.RESOURCE_NAME: None,
_revisionsSchema.REVISION: schema.REVISION_SEQ,
_revisionsSchema.DELETED: False
},
Return=_revisionsSchema.REVISION)
@inlineCallbacks
def _initSyncToken(self):
self._syncTokenRevision = (yield self._initSyncTokenQuery.on(
self._txn, resourceID=self._resourceID))[0][0]
_syncTokenQuery = Select(
[Max(_revisionsSchema.REVISION)],
From=_revisionsSchema,
Where=_revisionsSchema.HOME_RESOURCE_ID == Parameter("resourceID"))
@inlineCallbacks
def syncToken(self):
if self._syncTokenRevision is None:
self._syncTokenRevision = yield self.syncTokenRevision()
returnValue("%s_%s" % (self._resourceID, self._syncTokenRevision))
@inlineCallbacks
def syncTokenRevision(self):
revision = (yield
self._syncTokenQuery.on(self._txn,
resourceID=self._resourceID))[0][0]
if revision is None:
revision = int(
(yield self._txn.calendarserverValue("MIN-VALID-REVISION")))
returnValue(revision)
def properties(self):
return self._propertyStore
def addNotifier(self, factory_name, notifier):
self._notifiers[factory_name] = notifier
def getNotifier(self, factory_name):
return self._notifiers.get(factory_name)
def notifierID(self):
return (
self._txn._homeClass[self._txn._primaryHomeType]._notifierPrefix,
"%s/notification" % (self.ownerHome().uid(), ),
)
def parentNotifierID(self):
return (
self._txn._homeClass[self._txn._primaryHomeType]._notifierPrefix,
"%s" % (self.ownerHome().uid(), ),
)
@inlineCallbacks
def notifyChanged(self, category=ChangeCategory.default):
"""
Send notifications, change sync token and bump last modified because
the resource has changed. We ensure we only do this once per object
per transaction.
"""
if self._txn.isNotifiedAlready(self):
returnValue(None)
self._txn.notificationAddedForObject(self)
# Send notifications
if self._notifiers:
# cache notifiers run in post commit
notifier = self._notifiers.get("cache", None)
if notifier:
self._txn.postCommit(notifier.notify)
# push notifiers add their work items immediately
notifier = self._notifiers.get("push", None)
if notifier:
yield notifier.notify(self._txn, priority=category.value)
returnValue(None)
@classproperty
def _completelyNewRevisionQuery(cls):
rev = cls._revisionsSchema
return Insert(
{
rev.HOME_RESOURCE_ID:
Parameter("homeID"),
# rev.RESOURCE_ID: Parameter("resourceID"),
rev.RESOURCE_NAME:
Parameter("name"),
rev.REVISION:
schema.REVISION_SEQ,
rev.DELETED:
False
},
Return=rev.REVISION)
def _maybeNotify(self):
"""
Emit a push notification after C{_changeRevision}.
"""
return self.notifyChanged()
@inlineCallbacks
def remove(self):
"""
Remove DB rows corresponding to this notification home.
"""
# Delete NOTIFICATION rows
no = schema.NOTIFICATION
kwds = {"ResourceID": self._resourceID}
yield Delete(
From=no,
Where=(
no.NOTIFICATION_HOME_RESOURCE_ID == Parameter("ResourceID")),
).on(self._txn, **kwds)
# Delete NOTIFICATION_HOME (will cascade to NOTIFICATION_OBJECT_REVISIONS)
nh = schema.NOTIFICATION_HOME
yield Delete(
From=nh,
Where=(nh.RESOURCE_ID == Parameter("ResourceID")),
).on(self._txn, **kwds)
purge = remove
def getResourceIDsToPurge(self, home_id, calendar_id, calendar_name):
"""
For the given calendar find which calendar objects are older than the cut-off and return the
resource-ids of those.
@param home_id: resource-id of calendar home
@type home_id: L{int}
@param calendar_id: resource-id of the calendar to check
@type calendar_id: L{int}
@param calendar_name: name of the calendar to check
@type calendar_name: L{str}
"""
log.debug(" Checking calendar: {id} '{name}'", id=calendar_id, name=calendar_name)
purge = set()
txn = self.store.newTransaction(label="Find matching resources")
co = schema.CALENDAR_OBJECT
tr = schema.TIME_RANGE
kwds = {"calendar_id": calendar_id}
rows = (yield Select(
[co.RESOURCE_ID, co.RECURRANCE_MAX, co.RECURRANCE_MIN, Max(tr.END_DATE)],
From=co.join(tr, on=(co.RESOURCE_ID == tr.CALENDAR_OBJECT_RESOURCE_ID)),
Where=(co.CALENDAR_RESOURCE_ID == Parameter("calendar_id")).And(
co.ICALENDAR_TYPE == "VEVENT"
),
GroupBy=(co.RESOURCE_ID, co.RECURRANCE_MAX, co.RECURRANCE_MIN,),
Having=(
(co.RECURRANCE_MAX == None).And(Max(tr.END_DATE) < pyCalendarToSQLTimestamp(self.cutoff))
).Or(
(co.RECURRANCE_MAX != None).And(co.RECURRANCE_MAX < pyCalendarToSQLTimestamp(self.cutoff))
),
).on(txn, **kwds))
log.debug(" Found {len} resources to check", len=len(rows))
for resource_id, recurrence_max, recurrence_min, max_end_date in rows:
recurrence_max = parseSQLDateToPyCalendar(recurrence_max) if recurrence_max else None
recurrence_min = parseSQLDateToPyCalendar(recurrence_min) if recurrence_min else None
max_end_date = parseSQLDateToPyCalendar(max_end_date) if max_end_date else None
# Find events where we know the max(end_date) represents a valid,
# untruncated expansion
if recurrence_min is None or recurrence_min < self.cutoff:
if recurrence_max is None:
# Here we know max_end_date is the fully expand final instance
if max_end_date < self.cutoff:
purge.add(self.PurgeEvent(home_id, calendar_id, resource_id,))
continue
elif recurrence_max > self.cutoff:
# Here we know that there are instances newer than the cut-off
# but they have not yet been indexed out that far
continue
# Manually detect the max_end_date from the actual calendar data
calendar = yield self.getCalendar(txn, resource_id)
if calendar is not None:
if self.checkLastInstance(calendar):
purge.add(self.PurgeEvent(home_id, calendar_id, resource_id,))
yield txn.commit()
log.debug(" Found {len} resources to purge", len=len(purge))
returnValue(purge)