def mergeHomes(sqlTxn, one, other, homeType):
"""
Merge two homes together. This determines which of C{one} or C{two} is
newer - that is, has been modified more recently - and pulls all the data
from the older into the newer home. Then, it changes the UID of the old
home to its UID, normalized and prefixed with "old.", and then re-names the
new home to its name, normalized.
Because the UIDs of both homes have changed, B{both one and two will be
invalid to all other callers from the start of the invocation of this
function}.
@param sqlTxn: the transaction to use
@type sqlTxn: A L{CommonTransaction}
@param one: A calendar home.
@type one: L{ICalendarHome}
@param two: Another, different calendar home.
@type two: L{ICalendarHome}
@param homeType: The type of home to scan; L{ECALENDARTYPE} or
L{EADDRESSBOOKTYPE}.
@type homeType: C{int}
@return: a L{Deferred} which fires with with the newer of C{one} or C{two},
into which the data from the other home has been merged, when the merge
is complete.
"""
from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
from txdav.carddav.datastore.util import migrateHome as migrateABHome
migrateHome = {
EADDRESSBOOKTYPE: migrateABHome,
ECALENDARTYPE: migrateCalendarHome,
ENOTIFICATIONTYPE: _dontBotherWithNotifications
}[homeType]
homeTable = {
EADDRESSBOOKTYPE: schema.ADDRESSBOOK_HOME,
ECALENDARTYPE: schema.CALENDAR_HOME,
ENOTIFICATIONTYPE: schema.NOTIFICATION_HOME
}[homeType]
both = []
both.append([one, (yield determineNewest(one.uid(), homeType).on(sqlTxn))])
both.append(
[other, (yield determineNewest(other.uid(), homeType).on(sqlTxn))])
both.sort(key=lambda x: x[1])
older = both[0][0]
newer = both[1][0]
yield migrateHome(older, newer, merge=True)
# Rename the old one to 'old.<correct-guid>'
newNormalized = normalizeUUIDOrNot(newer.uid())
oldNormalized = normalizeUUIDOrNot(older.uid())
yield _renameHome(sqlTxn, homeTable, older.uid(), "old." + oldNormalized)
# Rename the new one to '<correct-guid>'
if newer.uid() != newNormalized:
yield _renameHome(sqlTxn, homeTable, newer.uid(), newNormalized)
yield returnValue(newer)
def mergeHomes(sqlTxn, one, other, homeType):
"""
Merge two homes together. This determines which of C{one} or C{two} is
newer - that is, has been modified more recently - and pulls all the data
from the older into the newer home. Then, it changes the UID of the old
home to its UID, normalized and prefixed with "old.", and then re-names the
new home to its name, normalized.
Because the UIDs of both homes have changed, B{both one and two will be
invalid to all other callers from the start of the invocation of this
function}.
@param sqlTxn: the transaction to use
@type sqlTxn: A L{CommonTransaction}
@param one: A calendar home.
@type one: L{ICalendarHome}
@param two: Another, different calendar home.
@type two: L{ICalendarHome}
@param homeType: The type of home to scan; L{ECALENDARTYPE} or
L{EADDRESSBOOKTYPE}.
@type homeType: C{int}
@return: a L{Deferred} which fires with with the newer of C{one} or C{two},
into which the data from the other home has been merged, when the merge
is complete.
"""
from txdav.caldav.datastore.util import migrateHome as migrateCalendarHome
from txdav.carddav.datastore.util import migrateHome as migrateABHome
migrateHome = {EADDRESSBOOKTYPE: migrateABHome,
ECALENDARTYPE: migrateCalendarHome,
ENOTIFICATIONTYPE: _dontBotherWithNotifications}[homeType]
homeTable = {EADDRESSBOOKTYPE: schema.ADDRESSBOOK_HOME,
ECALENDARTYPE: schema.CALENDAR_HOME,
ENOTIFICATIONTYPE: schema.NOTIFICATION_HOME}[homeType]
both = []
both.append([one,
(yield determineNewest(one.uid(), homeType).on(sqlTxn))])
both.append([other,
(yield determineNewest(other.uid(), homeType).on(sqlTxn))])
both.sort(key=lambda x: x[1])
older = both[0][0]
newer = both[1][0]
yield migrateHome(older, newer, merge=True)
# Rename the old one to 'old.<correct-guid>'
newNormalized = normalizeUUIDOrNot(newer.uid())
oldNormalized = normalizeUUIDOrNot(older.uid())
yield _renameHome(sqlTxn, homeTable, older.uid(), "old." + oldNormalized)
# Rename the new one to '<correct-guid>'
if newer.uid() != newNormalized:
yield _renameHome(sqlTxn, homeTable, newer.uid(), newNormalized)
yield returnValue(newer)
def lowercase(self):
"""
Lowercase mailto: addresses (and uppercase urn:uuid: addresses!) so
they can be located via normalized names.
"""
rows = self._db_execute("""
select ORGANIZER, ATTENDEE from TOKENS
""")
for row in rows:
organizer = row[0]
attendee = row[1]
if organizer.lower().startswith("mailto:"):
self._db_execute(
"""
update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2
""", organizer.lower(), organizer)
else:
from txdav.base.datastore.util import normalizeUUIDOrNot
self._db_execute(
"""
update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2
""", normalizeUUIDOrNot(organizer), organizer)
# ATTENDEEs are always mailto: so unconditionally lower().
self._db_execute(
"""
update TOKENS set ATTENDEE = :1 WHERE ATTENDEE = :2
""", attendee.lower(), attendee)
self._db_commit()
def fixOneCalendarObject(component):
"""
Correct the properties which may contain a user's directory UUID within a
single calendar component, by normalizing the directory UUID.
@param component: any iCalendar component.
@type component: L{twistedcaldav.ical.Component}
@return: a 2-tuple of the number of fixes performed and the new
L{Component}
"""
fixes = 0
for calprop in component.properties():
if calprop.name() in (
"ATTENDEE", "ORGANIZER", PERUSER_UID
):
preval = calprop.value()
postval = normalizeUUIDOrNot(preval)
if preval != postval:
fixes += 1
calprop.setValue(postval)
for subc in component.subcomponents():
count, _ignore_fixsubc = fixOneCalendarObject(subc)
fixes += count
return fixes, component
def _normalizeColumnUUIDs(txn, column):
"""
Upper-case the UUIDs in the given SQL DAL column.
@param txn: The transaction.
@type txn: L{CommonStoreTransaction}
@param column: the column, which may contain UIDs, to normalize.
@type column: L{ColumnSyntax}
@return: A L{Deferred} that will fire when the UUID normalization of the
given column has completed.
"""
tableModel = column.model.table
# Get a primary key made of column syntax objects for querying and
# comparison later.
pkey = [ColumnSyntax(columnModel) for columnModel in tableModel.primaryKey]
for row in (yield Select([column] + pkey,
From=TableSyntax(tableModel)).on(txn)):
before = row[0]
pkeyparts = row[1:]
after = normalizeUUIDOrNot(before)
if after != before:
where = _AndNothing
# Build a where clause out of the primary key and the parts of the
# primary key that were found.
for pkeycol, pkeypart in zip(pkeyparts, pkey):
where = where.And(pkeycol == pkeypart)
yield Update({column: after}, Where=where).on(txn)
def migrateOneHome(self, fileTxn, homeType, fileHome):
"""
Migrate an individual calendar or addressbook home.
"""
migrateFunc, destFunc = homeTypeLookup.get(homeType)
uid = normalizeUUIDOrNot(fileHome.uid())
self.log.warn("Starting migration transaction %s UID %r" %
(homeType, uid))
sqlTxn = self.sqlStore.newTransaction()
homeGetter = destFunc(sqlTxn)
sqlHome = yield homeGetter(uid, create=False)
if sqlHome is not None and not self.merge:
self.log.warn(
"%s home %r already existed not migrating" % (
homeType, uid))
yield sqlTxn.abort()
returnValue(None)
try:
if sqlHome is None:
sqlHome = yield homeGetter(uid, create=True)
yield migrateFunc(fileHome, sqlHome, merge=self.merge)
except:
f = Failure()
yield sqlTxn.abort()
f.raiseException()
else:
yield sqlTxn.commit()
# Remove file home after migration. FIXME: instead, this should be a
# public remove...HomeWithUID() API for de-provisioning. (If we had
# this, this would simply be a store-to-store migrator rather than a
# filesystem-to-database upgrade.)
fileHome._path.remove()
def lowercase(self):
"""
Lowercase mailto: addresses (and uppercase urn:uuid: addresses!) so
they can be located via normalized names.
"""
rows = self._db_execute(
"""
select ORGANIZER, ATTENDEE from TOKENS
"""
)
for row in rows:
organizer = row[0]
attendee = row[1]
if organizer.lower().startswith("mailto:"):
self._db_execute(
"""
update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2
""", organizer.lower(), organizer
)
else:
from txdav.base.datastore.util import normalizeUUIDOrNot
self._db_execute(
"""
update TOKENS set ORGANIZER = :1 WHERE ORGANIZER = :2
""", normalizeUUIDOrNot(organizer), organizer
)
# ATTENDEEs are always mailto: so unconditionally lower().
self._db_execute(
"""
update TOKENS set ATTENDEE = :1 WHERE ATTENDEE = :2
""", attendee.lower(), attendee
)
self._db_commit()
def migrateOneHome(self, fileTxn, homeType, fileHome):
"""
Migrate an individual calendar or addressbook home.
"""
migrateFunc, destFunc = homeTypeLookup.get(homeType)
uid = normalizeUUIDOrNot(fileHome.uid())
self.log.warn("Starting migration transaction %s UID %r" %
(homeType, uid))
sqlTxn = self.sqlStore.newTransaction(
label="UpgradeToDatabaseStep.migrateOneHome")
homeGetter = destFunc(sqlTxn)
sqlHome = yield homeGetter(uid, create=False)
if sqlHome is not None and not self.merge:
self.log.warn("%s home %r already existed not migrating" %
(homeType, uid))
yield sqlTxn.abort()
returnValue(None)
try:
if sqlHome is None:
sqlHome = yield homeGetter(uid, create=True)
yield migrateFunc(fileHome, sqlHome, merge=self.merge)
except:
f = Failure()
yield sqlTxn.abort()
f.raiseException()
else:
yield sqlTxn.commit()
# Remove file home after migration. FIXME: instead, this should be a
# public remove...HomeWithUID() API for de-provisioning. (If we had
# this, this would simply be a store-to-store migrator rather than a
# filesystem-to-database upgrade.)
fileHome._path.remove()
def _normalizeColumnUUIDs(txn, column):
"""
Upper-case the UUIDs in the given SQL DAL column.
@param txn: The transaction.
@type txn: L{CommonStoreTransaction}
@param column: the column, which may contain UIDs, to normalize.
@type column: L{ColumnSyntax}
@return: A L{Deferred} that will fire when the UUID normalization of the
given column has completed.
"""
tableModel = column.model.table
# Get a primary key made of column syntax objects for querying and
# comparison later.
pkey = [ColumnSyntax(columnModel)
for columnModel in tableModel.primaryKey]
for row in (yield Select([column] + pkey,
From=TableSyntax(tableModel)).on(txn)):
before = row[0]
pkeyparts = row[1:]
after = normalizeUUIDOrNot(before)
if after != before:
where = _AndNothing
# Build a where clause out of the primary key and the parts of the
# primary key that were found.
for pkeycol, pkeypart in zip(pkeyparts, pkey):
where = where.And(pkeycol == pkeypart)
yield Update({column: after}, Where=where).on(txn)
def _needsNormalizationUpgrade(txn):
"""
Determine whether a given store requires a UUID normalization data upgrade.
@param txn: the transaction to use
@type txn: L{CommonStoreTransaction}
@return: a L{Deferred} that fires with C{True} or C{False} depending on
whether we need the normalization upgrade or not.
"""
for x in [schema.CALENDAR_HOME, schema.ADDRESSBOOK_HOME,
schema.NOTIFICATION_HOME]:
slct = Select([x.OWNER_UID], From=x,
Where=x.OWNER_UID != Upper(x.OWNER_UID))
rows = yield slct.on(txn)
if rows:
for [uid] in rows:
if normalizeUUIDOrNot(uid) != uid:
returnValue(True)
returnValue(False)
def migrateOneHome(self, fileTxn, homeType, fileHome):
"""
Migrate an individual calendar or addressbook home.
"""
migrateFunc, destFunc = homeTypeLookup.get(homeType)
uid = normalizeUUIDOrNot(fileHome.uid())
self.log.warn("Starting migration transaction {type} UID {uid}",
type=homeType,
uid=uid)
sqlTxn = self.sqlStore.newTransaction(
label="UpgradeToDatabaseStep.migrateOneHome")
homeGetter = destFunc(sqlTxn)
sqlHome = yield homeGetter(uid, create=False)
if sqlHome is not None and not self.merge:
self.log.warn("{ty[e} home {uid} already existed not migrating",
type=homeType,
uid=uid)
yield sqlTxn.abort()
returnValue(None)
try:
if sqlHome is None:
try:
sqlHome = yield homeGetter(uid, create=True)
except DirectoryRecordNotFoundError:
# The directory record does not exist; skip this home
self.log.warn(
"Skipping migration of {uid} because it's missing from the directory service",
uid=uid)
returnValue(None)
yield migrateFunc(fileHome, sqlHome, merge=self.merge)
except:
f = Failure()
yield sqlTxn.abort()
f.raiseException()
else:
yield sqlTxn.commit()
# Remove file home after migration. FIXME: instead, this should be a
# public remove...HomeWithUID() API for de-provisioning. (If we had
# this, this would simply be a store-to-store migrator rather than a
# filesystem-to-database upgrade.)
fileHome._path.remove()
def fixOneCalendarObject(component):
"""
Correct the properties which may contain a user's directory UUID within a
single calendar component, by normalizing the directory UUID.
@param component: any iCalendar component.
@type component: L{twistedcaldav.ical.Component}
@return: a 2-tuple of the number of fixes performed and the new
L{Component}
"""
fixes = 0
for calprop in component.properties():
if calprop.name() in ("ATTENDEE", "ORGANIZER", PERUSER_UID):
preval = calprop.value()
postval = normalizeUUIDOrNot(preval)
if preval != postval:
fixes += 1
calprop.setValue(postval)
for subc in component.subcomponents():
count, _ignore_fixsubc = fixOneCalendarObject(subc)
fixes += count
return fixes, component
def _needsNormalizationUpgrade(txn):
"""
Determine whether a given store requires a UUID normalization data upgrade.
@param txn: the transaction to use
@type txn: L{CommonStoreTransaction}
@return: a L{Deferred} that fires with C{True} or C{False} depending on
whether we need the normalization upgrade or not.
"""
for x in [
schema.CALENDAR_HOME, schema.ADDRESSBOOK_HOME,
schema.NOTIFICATION_HOME
]:
slct = Select([x.OWNER_UID],
From=x,
Where=x.OWNER_UID != Upper(x.OWNER_UID))
rows = yield slct.on(txn)
if rows:
for [uid] in rows:
if normalizeUUIDOrNot(uid) != uid:
returnValue(True)
returnValue(False)
