def setUp(self):
"""
Replace self.site.resource with an appropriately provisioned
AddressBookHomeFile, and replace self.docroot with a path pointing at that
file.
"""
super(AddressBookHomeTestCase, self).setUp()
fp = FilePath(self.mktemp())
fp.createDirectory()
self.createStockDirectoryService()
# Need a data store
_newStore = CommonDataStore(fp, None, True, False)
self.homeProvisioner = DirectoryAddressBookHomeProvisioningResource(
self.directoryService, "/addressbooks/",
_newStore
)
def _defer(user):
# Commit the transaction
self.site.resource._associatedTransaction.commit()
self.docroot = user._newStoreHome._path.path
return self._refreshRoot().addCallback(_defer)
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
def doDirectoryTest(self, addedNames, modify=lambda x: None,
expectedNames=None):
"""
Do a test of a L{DAVFile} pointed at a directory, verifying that files
existing with the given names will be faithfully 'played back' via HTML
rendering.
"""
if expectedNames is None:
expectedNames = addedNames
fp = FilePath(self.mktemp())
fp.createDirectory()
for sampleName in expectedNames:
fp.child(sampleName).touch()
df = DAVFile(fp)
modify(df)
responseText = (yield df.render(SimpleFakeRequest('/'))).stream.read()
responseXML = browserHTML2ETree(responseText)
names = set([element.text.encode("utf-8")
for element in responseXML.findall(".//a")])
self.assertEquals(set(expectedNames), names)
def doDirectoryTest(self, addedNames, modify=lambda x: None,
expectedNames=None):
"""
Do a test of a L{DAVFile} pointed at a directory, verifying that files
existing with the given names will be faithfully 'played back' via HTML
rendering.
"""
if expectedNames is None:
expectedNames = addedNames
fp = FilePath(self.mktemp())
fp.createDirectory()
for sampleName in expectedNames:
fp.child(sampleName).touch()
df = DAVFile(fp)
modify(df)
responseText = (yield df.render(SimpleFakeRequest('/'))).stream.read()
responseXML = browserHTML2ETree(responseText)
names = set([element.text.encode("utf-8")
for element in responseXML.findall(".//a")])
self.assertEquals(set(expectedNames), names)
def test_triggerGroupCacherUpdate(self):
"""
Verify triggerGroupCacherUpdate can read a pidfile and send a SIGHUP
"""
self.calledArgs = None
def killMethod(pid, sig):
self.calledArgs = (pid, sig)
class StubConfig(object):
def __init__(self, runRootPath):
self.RunRoot = runRootPath
runRootDir = FilePath(self.mktemp())
runRootDir.createDirectory()
pidFile = runRootDir.child("groupcacher.pid")
pidFile.setContent("1234")
testConfig = StubConfig(runRootDir.path)
triggerGroupCacherUpdate(testConfig, killMethod=killMethod)
self.assertEquals(self.calledArgs, (1234, signal.SIGHUP))
runRootDir.remove()
class PostgresService(MultiService):
def __init__(self, dataStoreDirectory, subServiceFactory,
schema, resetSchema=False, databaseName='subpostgres',
clusterName="cluster",
logFile="postgres.log", socketDir="/tmp",
listenAddresses=[], sharedBuffers=30,
maxConnections=20, options=[],
testMode=False,
uid=None, gid=None,
spawnedDBUser="******",
importFileName=None,
pgCtl="pg_ctl",
initDB="initdb",
reactor=None):
"""
Initialize a L{PostgresService} pointed at a data store directory.
@param dataStoreDirectory: the directory to
@type dataStoreDirectory: L{twext.python.filepath.CachingFilePath}
@param subServiceFactory: a 1-arg callable that will be called with a
1-arg callable which returns a DB-API cursor.
@type subServiceFactory: C{callable}
@param spawnedDBUser: the postgres role
@type spawnedDBUser: C{str}
@param importFileName: path to SQL file containing previous data to import
@type importFileName: C{str}
"""
# FIXME: By default there is very little (4MB) shared memory available,
# so at the moment I am lowering these postgres config options to allow
# multiple servers to run. We might want to look into raising
# kern.sysv.shmmax.
# See: http://www.postgresql.org/docs/8.4/static/kernel-resources.html
MultiService.__init__(self)
self.subServiceFactory = subServiceFactory
self.dataStoreDirectory = dataStoreDirectory
self.workingDir = self.dataStoreDirectory.child("working")
self.resetSchema = resetSchema
# In order to delay a shutdown until database initialization has
# completed, our stopService( ) examines the delayedShutdown flag.
# If True, we wait on the shutdownDeferred to fire before proceeding.
# The deferred gets fired once database init is complete.
self.delayedShutdown = False # set to True when in critical code
self.shutdownDeferred = None # the actual deferred
# Options from config
self.databaseName = databaseName
self.clusterName = clusterName
# Make logFile absolute in case the working directory of postgres is
# elsewhere:
self.logFile = os.path.abspath(logFile)
if listenAddresses:
self.socketDir = None
self.host, self.port = listenAddresses[0].split(":") if ":" in listenAddresses[0] else (listenAddresses[0], None,)
self.listenAddresses = [addr.split(":")[0] for addr in listenAddresses]
else:
if socketDir:
# Unix socket length path limit
self.socketDir = CachingFilePath("%s/ccs_postgres_%s/" %
(socketDir, md5(dataStoreDirectory.path).hexdigest()))
if len(self.socketDir.path) > 64:
socketDir = "/tmp"
self.socketDir = CachingFilePath("/tmp/ccs_postgres_%s/" %
(md5(dataStoreDirectory.path).hexdigest()))
self.host = self.socketDir.path
self.port = None
else:
self.socketDir = None
self.host = "localhost"
self.port = None
self.listenAddresses = []
self.sharedBuffers = sharedBuffers if not testMode else 16
self.maxConnections = maxConnections if not testMode else 4
self.options = options
self.uid = uid
self.gid = gid
self.spawnedDBUser = spawnedDBUser
self.importFileName = importFileName
self.schema = schema
self.monitor = None
self.openConnections = []
self._pgCtl = pgCtl
self._initdb = initDB
self._reactor = reactor
self._postgresPid = None
@property
def reactor(self):
if self._reactor is None:
from twisted.internet import reactor
self._reactor = reactor
return self._reactor
def pgCtl(self):
"""
Locate the path to pg_ctl.
"""
return which(self._pgCtl)[0]
def initdb(self):
return which(self._initdb)[0]
def activateDelayedShutdown(self):
"""
Call this when starting database initialization code to protect against
shutdown.
Sets the delayedShutdown flag to True so that if reactor shutdown
commences, the shutdown will be delayed until deactivateDelayedShutdown
is called.
"""
self.delayedShutdown = True
def deactivateDelayedShutdown(self):
"""
Call this when database initialization code has completed so that the
reactor can shutdown.
"""
self.delayedShutdown = False
if self.shutdownDeferred:
self.shutdownDeferred.callback(None)
def _connectorFor(self, databaseName=None):
if databaseName is None:
databaseName = self.databaseName
if self.spawnedDBUser:
dsn = "%s:dbname=%s:%s" % (self.host, databaseName, self.spawnedDBUser)
elif self.uid is not None:
dsn = "%s:dbname=%s:%s" % (self.host, databaseName,
pwd.getpwuid(self.uid).pw_name)
else:
dsn = "%s:dbname=%s" % (self.host, databaseName)
kwargs = {}
if self.port:
kwargs["host"] = "%s:%s" % (self.host, self.port,)
return DBAPIConnector(pgdb, postgresPreflight, dsn, **kwargs)
def produceConnection(self, label="<unlabeled>", databaseName=None):
"""
Produce a DB-API 2.0 connection pointed at this database.
"""
return self._connectorFor(databaseName).connect(label)
def ready(self, createDatabaseConn, createDatabaseCursor):
"""
Subprocess is ready. Time to initialize the subservice.
If the database has not been created and there is a dump file,
then the dump file is imported.
"""
if self.resetSchema:
try:
createDatabaseCursor.execute(
"drop database %s" % (self.databaseName)
)
except pgdb.DatabaseError:
pass
try:
createDatabaseCursor.execute(
"create database %s with encoding 'UTF8'" % (self.databaseName)
)
except:
# database already exists
executeSQL = False
else:
# database does not yet exist; if dump file exists, execute it, otherwise
# execute schema
executeSQL = True
sqlToExecute = self.schema
if self.importFileName:
importFilePath = CachingFilePath(self.importFileName)
if importFilePath.exists():
sqlToExecute = importFilePath.getContent()
createDatabaseCursor.close()
createDatabaseConn.close()
if executeSQL:
connection = self.produceConnection()
cursor = connection.cursor()
cursor.execute(sqlToExecute)
connection.commit()
connection.close()
if self.shutdownDeferred is None:
# Only continue startup if we've not begun shutdown
self.subServiceFactory(self.produceConnection, self).setServiceParent(self)
def pauseMonitor(self):
"""
Pause monitoring. This is a testing hook for when (if) we are
continuously monitoring output from the 'postgres' process.
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.stopReading()
# pipe.stopWriting()
pass
def unpauseMonitor(self):
"""
Unpause monitoring.
@see: L{pauseMonitor}
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.startReading()
# pipe.startWriting()
pass
def startDatabase(self):
"""
Start the database and initialize the subservice.
"""
def createConnection():
createDatabaseConn = self.produceConnection(
'schema creation', 'postgres'
)
createDatabaseCursor = createDatabaseConn.cursor()
createDatabaseCursor.execute("commit")
return createDatabaseConn, createDatabaseCursor
monitor = _PostgresMonitor(self)
pgCtl = self.pgCtl()
# check consistency of initdb and postgres?
options = []
options.append(
"-c listen_addresses='%s'" % (",".join(self.listenAddresses))
)
if self.socketDir:
options.append("-k '%s'" % (self.socketDir.path,))
if self.port:
options.append("-c port=%s" % (self.port,))
options.append("-c shared_buffers=%d" % (self.sharedBuffers,))
options.append("-c max_connections=%d" % (self.maxConnections,))
options.append("-c standard_conforming_strings=on")
options.extend(self.options)
log.warn("Requesting postgres start via {cmd}", cmd=pgCtl)
self.reactor.spawnProcess(
monitor, pgCtl,
[
pgCtl,
"start",
"-l", self.logFile,
"-w",
# XXX what are the quoting rules for '-o'? do I need to repr()
# the path here?
"-o",
" ".join(options),
],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
self.monitor = monitor
def gotStatus(result):
"""
Grab the postgres pid from the pgCtl status call in case we need
to kill it directly later on in hardStop(). Useful in conjunction
with the DataStoreMonitor so we can shut down if DataRoot has been
removed/renamed/unmounted.
"""
reResult = re.search("PID: (\d+)\D", result)
if reResult != None:
self._postgresPid = int(reResult.group(1))
self.ready(*createConnection())
self.deactivateDelayedShutdown()
def gotReady(result):
"""
We started postgres; we're responsible for stopping it later.
Call pgCtl status to get the pid.
"""
log.warn("{cmd} exited", cmd=pgCtl)
self.shouldStopDatabase = True
d = Deferred()
statusMonitor = CapturingProcessProtocol(d, None)
self.reactor.spawnProcess(
statusMonitor, pgCtl, [pgCtl, "status"],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return d.addCallback(gotStatus)
def couldNotStart(f):
"""
There was an error trying to start postgres. Try to connect
because it might already be running. In this case, we won't
be the one to stop it.
"""
d = Deferred()
statusMonitor = CapturingProcessProtocol(d, None)
self.reactor.spawnProcess(
statusMonitor, pgCtl, [pgCtl, "status"],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return d.addCallback(gotStatus).addErrback(giveUp)
def giveUp(f):
"""
We can't start postgres or connect to a running instance. Shut
down.
"""
log.failure("Can't start or connect to postgres", f)
self.deactivateDelayedShutdown()
self.reactor.stop()
self.monitor.completionDeferred.addCallback(
gotReady).addErrback(couldNotStart)
shouldStopDatabase = False
def startService(self):
MultiService.startService(self)
self.activateDelayedShutdown()
clusterDir = self.dataStoreDirectory.child(self.clusterName)
env = self.env = os.environ.copy()
env.update(PGDATA=clusterDir.path,
PGHOST=self.host,
PGUSER=self.spawnedDBUser)
initdb = self.initdb()
if self.socketDir:
if not self.socketDir.isdir():
self.socketDir.createDirectory()
if self.uid and self.gid:
os.chown(self.socketDir.path, self.uid, self.gid)
if self.dataStoreDirectory.isdir():
self.startDatabase()
else:
self.dataStoreDirectory.createDirectory()
if not self.workingDir.isdir():
self.workingDir.createDirectory()
if self.uid and self.gid:
os.chown(self.dataStoreDirectory.path, self.uid, self.gid)
os.chown(self.workingDir.path, self.uid, self.gid)
dbInited = Deferred()
self.reactor.spawnProcess(
CapturingProcessProtocol(dbInited, None),
initdb, [initdb, "-E", "UTF8", "-U", self.spawnedDBUser],
env=env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
def doCreate(result):
if result.find("FATAL:") != -1:
log.error(result)
raise RuntimeError("Unable to initialize postgres database: %s" % (result,))
self.startDatabase()
dbInited.addCallback(doCreate)
def stopService(self):
"""
Stop all child services, then stop the subprocess, if it's running.
"""
if self.delayedShutdown:
# We're still in the process of initializing the database, so
# delay shutdown until the shutdownDeferred fires.
d = self.shutdownDeferred = Deferred()
d.addCallback(lambda ignored: MultiService.stopService(self))
else:
d = MultiService.stopService(self)
def superStopped(result):
# If pg_ctl's startup wasn't successful, don't bother to stop the
# database. (This also happens in command-line tools.)
if self.shouldStopDatabase:
monitor = _PostgresMonitor()
pgCtl = self.pgCtl()
# FIXME: why is this 'logfile' and not self.logfile?
self.reactor.spawnProcess(monitor, pgCtl,
[pgCtl, '-l', 'logfile', 'stop'],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return monitor.completionDeferred
return d.addCallback(superStopped)
# def maybeStopSubprocess(result):
# if self.monitor is not None:
# self.monitor.transport.signalProcess("INT")
# return self.monitor.completionDeferred
# return result
# d.addCallback(maybeStopSubprocess)
# return d
def hardStop(self):
"""
Stop postgres quickly by sending it SIGQUIT
"""
if self._postgresPid is not None:
try:
os.kill(self._postgresPid, signal.SIGQUIT)
except OSError:
pass
class PostgresService(MultiService):
def __init__(
self,
dataStoreDirectory,
subServiceFactory,
schema,
resetSchema=False,
databaseName="subpostgres",
clusterName="cluster",
logFile="postgres.log",
logDirectory="",
socketDir="",
socketName="",
listenAddresses=[],
txnTimeoutSeconds=30,
sharedBuffers=30,
maxConnections=20,
options=[],
testMode=False,
uid=None,
gid=None,
spawnedDBUser="******",
pgCtl="pg_ctl",
initDB="initdb",
reactor=None,
):
"""
Initialize a L{PostgresService} pointed at a data store directory.
@param dataStoreDirectory: the directory to
@type dataStoreDirectory: L{twext.python.filepath.CachingFilePath}
@param subServiceFactory: a 1-arg callable that will be called with a
1-arg callable which returns a DB-API cursor.
@type subServiceFactory: C{callable}
@param spawnedDBUser: the postgres role
@type spawnedDBUser: C{str}
"""
# FIXME: By default there is very little (4MB) shared memory available,
# so at the moment I am lowering these postgres config options to allow
# multiple servers to run. We might want to look into raising
# kern.sysv.shmmax.
# See: http://www.postgresql.org/docs/8.4/static/kernel-resources.html
MultiService.__init__(self)
self.subServiceFactory = subServiceFactory
self.dataStoreDirectory = dataStoreDirectory
self.workingDir = self.dataStoreDirectory.child("working")
self.resetSchema = resetSchema
# In order to delay a shutdown until database initialization has
# completed, our stopService( ) examines the delayedShutdown flag.
# If True, we wait on the shutdownDeferred to fire before proceeding.
# The deferred gets fired once database init is complete.
self.delayedShutdown = False # set to True when in critical code
self.shutdownDeferred = None # the actual deferred
# Options from config
self.databaseName = databaseName
self.clusterName = clusterName
# Make logFile absolute in case the working directory of postgres is
# elsewhere:
self.logFile = os.path.abspath(logFile)
if logDirectory:
self.logDirectory = os.path.abspath(logDirectory)
else:
self.logDirectory = ""
# Always use our own configured socket dir in case the built-in
# postgres tries to use a directory we don't have permissions for
if not socketDir:
# Socket directory was not specified, so come up with one
# in /tmp and based on a hash of the data store directory
digest = md5(dataStoreDirectory.path).hexdigest()
socketDir = "/tmp/ccs_postgres_" + digest
self.socketDir = CachingFilePath(socketDir)
self.socketName = socketName
if listenAddresses:
if ":" in listenAddresses[0]:
self.host, self.port = listenAddresses[0].split(":")
else:
self.host, self.port = (listenAddresses[0], None)
self.listenAddresses = [
addr.split(":")[0] for addr in listenAddresses
]
else:
self.host = self.socketDir.path
self.port = None
self.listenAddresses = []
self.txnTimeoutSeconds = txnTimeoutSeconds
self.testMode = testMode
self.sharedBuffers = max(sharedBuffers if not testMode else 16, 16)
self.maxConnections = maxConnections if not testMode else 8
self.options = options
self.uid = uid
self.gid = gid
self.spawnedDBUser = spawnedDBUser
self.schema = schema
self.monitor = None
self.openConnections = []
def locateCommand(name, cmd):
for found in which(cmd):
return found
raise InternalDataStoreError(
"Unable to locate {} command: {}".format(name, cmd))
self._pgCtl = locateCommand("pg_ctl", pgCtl)
# Make note of the inode for the pg_ctl script; if it changes or is
# missing when it comes time to stop postgres, instead send SIGTERM
# to stop our postgres (since we can't do a graceful shutdown)
try:
self._pgCtlInode = os.stat(self._pgCtl).st_ino
except:
self._pgCtlInode = 0
self._initdb = locateCommand("initdb", initDB)
self._reactor = reactor
self._postgresPid = None
@property
def reactor(self):
if self._reactor is None:
from twisted.internet import reactor
self._reactor = reactor
return self._reactor
def activateDelayedShutdown(self):
"""
Call this when starting database initialization code to
protect against shutdown.
Sets the delayedShutdown flag to True so that if reactor shutdown
commences, the shutdown will be delayed until deactivateDelayedShutdown
is called.
"""
self.delayedShutdown = True
def deactivateDelayedShutdown(self):
"""
Call this when database initialization code has completed so that the
reactor can shutdown.
"""
self.delayedShutdown = False
if self.shutdownDeferred:
self.shutdownDeferred.callback(None)
def _connectorFor(self, databaseName=None):
if databaseName is None:
databaseName = self.databaseName
kwargs = {
"database": databaseName,
}
if self.host.startswith("/"):
kwargs["endpoint"] = "unix:{}".format(self.host)
else:
kwargs["endpoint"] = "tcp:{}".format(self.host)
if self.port:
kwargs["endpoint"] = "{}:{}".format(kwargs["endpoint"],
self.port)
if self.spawnedDBUser:
kwargs["user"] = self.spawnedDBUser
elif self.uid is not None:
kwargs["user"] = pwd.getpwuid(self.uid).pw_name
kwargs["txnTimeoutSeconds"] = self.txnTimeoutSeconds
return DBAPIConnector.connectorFor("postgres", **kwargs)
def produceConnection(self, label="<unlabeled>", databaseName=None):
"""
Produce a DB-API 2.0 connection pointed at this database.
"""
return self._connectorFor(databaseName).connect(label)
def ready(self, createDatabaseConn, createDatabaseCursor):
"""
Subprocess is ready. Time to initialize the subservice.
If the database has not been created and there is a dump file,
then the dump file is imported.
"""
if self.resetSchema:
try:
createDatabaseCursor.execute("drop database {}".format(
self.databaseName))
except postgres.DatabaseError:
pass
try:
createDatabaseCursor.execute(
"create database {} with encoding 'UTF8'".format(
self.databaseName))
except:
# database already exists
sqlToExecute = None
else:
# database does not yet exist; if dump file exists, execute it,
# otherwise execute schema
sqlToExecute = self.schema
createDatabaseCursor.close()
createDatabaseConn.close()
if sqlToExecute is not None:
connection = self.produceConnection()
cursor = connection.cursor()
for statement in splitSQLString(sqlToExecute):
cursor.execute(statement)
connection.commit()
connection.close()
if self.shutdownDeferred is None:
# Only continue startup if we've not begun shutdown
self.subServiceFactory(self.produceConnection,
self).setServiceParent(self)
def pauseMonitor(self):
"""
Pause monitoring. This is a testing hook for when (if) we are
continuously monitoring output from the 'postgres' process.
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.stopReading()
# pipe.stopWriting()
pass
def unpauseMonitor(self):
"""
Unpause monitoring.
@see: L{pauseMonitor}
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.startReading()
# pipe.startWriting()
pass
def startDatabase(self):
"""
Start the database and initialize the subservice.
"""
def createConnection():
try:
createDatabaseConn = self.produceConnection(
"schema creation", "postgres")
except postgres.DatabaseError as e:
log.error(
"Unable to connect to database for schema creation:"
" {error}",
error=e)
raise
createDatabaseCursor = createDatabaseConn.cursor()
if postgres.__name__ == "pg8000":
createDatabaseConn.realConnection.autocommit = True
elif postgres.__name__ == "pgdb":
createDatabaseCursor.execute("commit")
else:
raise InternalDataStoreError(
"Unknown Postgres DBM module: {}".format(postgres))
return createDatabaseConn, createDatabaseCursor
monitor = PostgresMonitor(self)
# check consistency of initdb and postgres?
options = []
options.append("-c listen_addresses={}".format(
shell_quote(",".join(self.listenAddresses))))
if self.socketDir:
options.append("-c unix_socket_directories={}".format(
shell_quote(self.socketDir.path)))
if self.port:
options.append("-c port={}".format(shell_quote(self.port)))
options.append("-c shared_buffers={:d}".format(
self.sharedBuffers) # int: don't quote
)
options.append("-c max_connections={:d}".format(
self.maxConnections) # int: don't quote
)
options.append("-c standard_conforming_strings=on")
options.append("-c unix_socket_permissions=0770")
options.extend(self.options)
if self.logDirectory: # tell postgres to rotate logs
options.append("-c log_directory={}".format(
shell_quote(self.logDirectory)))
options.append("-c log_truncate_on_rotation=on")
options.append("-c log_filename=postgresql_%w.log")
options.append("-c log_rotation_age=1440")
options.append("-c logging_collector=on")
options.append("-c log_line_prefix=%t")
if self.testMode:
options.append("-c log_statement=all")
args = [
self._pgCtl,
"start",
"--log={}".format(self.logFile),
"--timeout=86400", # Plenty of time for a long cluster upgrade
"-w", # Wait for startup to complete
"-o",
" ".join(options), # Options passed to postgres
]
log.info("Requesting postgres start via: {args}", args=args)
self.reactor.spawnProcess(
monitor,
self._pgCtl,
args,
env=self.env,
path=self.workingDir.path,
uid=self.uid,
gid=self.gid,
)
self.monitor = monitor
def gotStatus(result):
"""
Grab the postgres pid from the pgCtl status call in case we need
to kill it directly later on in hardStop(). Useful in conjunction
with the DataStoreMonitor so we can shut down if DataRoot has been
removed/renamed/unmounted.
"""
reResult = re.search("PID: (\d+)\D", result)
if reResult is not None:
self._postgresPid = int(reResult.group(1))
self.ready(*createConnection())
self.deactivateDelayedShutdown()
def gotReady(result):
"""
We started postgres; we're responsible for stopping it later.
Call pgCtl status to get the pid.
"""
log.info("{cmd} exited", cmd=self._pgCtl)
self.shouldStopDatabase = True
d = Deferred()
statusMonitor = CapturingProcessProtocol(d, None)
self.reactor.spawnProcess(
statusMonitor,
self._pgCtl,
[self._pgCtl, "status"],
env=self.env,
path=self.workingDir.path,
uid=self.uid,
gid=self.gid,
)
return d.addCallback(gotStatus)
def couldNotStart(f):
"""
There was an error trying to start postgres. Try to connect
because it might already be running. In this case, we won't
be the one to stop it.
"""
d = Deferred()
statusMonitor = CapturingProcessProtocol(d, None)
self.reactor.spawnProcess(
statusMonitor,
self._pgCtl,
[self._pgCtl, "status"],
env=self.env,
path=self.workingDir.path,
uid=self.uid,
gid=self.gid,
)
return d.addCallback(gotStatus).addErrback(giveUp)
def giveUp(f):
"""
We can't start postgres or connect to a running instance. Shut
down.
"""
log.critical("Can't start or connect to postgres: {failure.value}",
failure=f)
self.deactivateDelayedShutdown()
self.reactor.stop()
self.monitor.completionDeferred.addCallback(gotReady).addErrback(
couldNotStart)
shouldStopDatabase = False
def startService(self):
MultiService.startService(self)
self.activateDelayedShutdown()
clusterDir = self.dataStoreDirectory.child(self.clusterName)
env = self.env = os.environ.copy()
env.update(PGDATA=clusterDir.path,
PGHOST=self.host,
PGUSER=self.spawnedDBUser)
if self.socketDir:
if not self.socketDir.isdir():
log.info("Creating {dir}",
dir=self.socketDir.path.decode("utf-8"))
self.socketDir.createDirectory()
if self.uid and self.gid:
os.chown(self.socketDir.path, self.uid, self.gid)
os.chmod(self.socketDir.path, 0770)
if not self.dataStoreDirectory.isdir():
log.info("Creating {dir}",
dir=self.dataStoreDirectory.path.decode("utf-8"))
self.dataStoreDirectory.createDirectory()
if not self.workingDir.isdir():
log.info("Creating {dir}",
dir=self.workingDir.path.decode("utf-8"))
self.workingDir.createDirectory()
if self.uid and self.gid:
os.chown(self.dataStoreDirectory.path, self.uid, self.gid)
os.chown(self.workingDir.path, self.uid, self.gid)
if not clusterDir.isdir():
# No cluster directory, run initdb
log.info("Running initdb for {dir}",
dir=clusterDir.path.decode("utf-8"))
dbInited = Deferred()
self.reactor.spawnProcess(
CapturingProcessProtocol(dbInited, None),
self._initdb,
[self._initdb, "-E", "UTF8", "-U", self.spawnedDBUser],
env=env,
path=self.workingDir.path,
uid=self.uid,
gid=self.gid,
)
def doCreate(result):
if result.find("FATAL:") != -1:
log.error(result)
raise InternalDataStoreError(
"Unable to initialize postgres database: {}".format(
result))
self.startDatabase()
dbInited.addCallback(doCreate)
else:
log.info("Cluster already exists at {dir}",
dir=clusterDir.path.decode("utf-8"))
self.startDatabase()
def stopService(self):
"""
Stop all child services, then stop the subprocess, if it's running.
"""
if self.delayedShutdown:
# We're still in the process of initializing the database, so
# delay shutdown until the shutdownDeferred fires.
d = self.shutdownDeferred = Deferred()
d.addCallback(lambda ignored: MultiService.stopService(self))
else:
d = MultiService.stopService(self)
def superStopped(result):
# If pg_ctl's startup wasn't successful, don't bother to stop the
# database. (This also happens in command-line tools.)
if self.shouldStopDatabase:
# Compare pg_ctl inode with one we saw at the start; if different
# (or missing), fall back to SIGTERM
try:
newInode = os.stat(self._pgCtl).st_ino
except OSError:
# Missing
newInode = -1
if self._pgCtlInode != newInode:
# send SIGTERM to postgres
log.info("Postgres control script mismatch")
if self._postgresPid:
log.info("Sending SIGTERM to Postgres")
try:
os.kill(self._postgresPid, signal.SIGTERM)
except OSError:
pass
return succeed(None)
else:
# use pg_ctl stop
monitor = PostgresMonitor()
args = [
self._pgCtl,
"stop",
"--log={}".format(self.logFile),
]
log.info("Requesting postgres stop via: {args}", args=args)
self.reactor.spawnProcess(
monitor,
self._pgCtl,
args,
env=self.env,
path=self.workingDir.path,
uid=self.uid,
gid=self.gid,
)
return monitor.completionDeferred
return d.addCallback(superStopped)
def hardStop(self):
"""
Stop postgres quickly by sending it SIGQUIT
"""
if self._postgresPid is not None:
try:
os.kill(self._postgresPid, signal.SIGQUIT)
except OSError:
pass
def createDataStore(self):
# FIXME: AddressBookHomeTestCase needs the same treatment.
fp = FilePath(self.mktemp())
fp.createDirectory()
return CommonDataStore(fp, None, True, False)
class HomeMigrationTests(CommonCommonTests, TestCase):
"""
Tests for L{UpgradeToDatabaseStep}.
"""
av1 = Component.fromString("""BEGIN:VCALENDAR
VERSION:2.0
CALSCALE:GREGORIAN
PRODID:-//calendarserver.org//Zonal//EN
BEGIN:VAVAILABILITY
ORGANIZER:mailto:[email protected]
UID:[email protected]
DTSTAMP:20061005T133225Z
DTEND:20140101T000000Z
BEGIN:AVAILABLE
UID:[email protected]
DTSTAMP:20061005T133225Z
SUMMARY:Monday to Friday from 9:00 to 17:00
DTSTART:20130101T090000Z
DTEND:20130101T170000Z
RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
END:AVAILABLE
END:VAVAILABILITY
END:VCALENDAR
""")
@inlineCallbacks
def setUp(self):
"""
Set up two stores to migrate between.
"""
yield super(HomeMigrationTests, self).setUp()
yield self.buildStoreAndDirectory(extraUids=(
u"home1",
u"home2",
u"home3",
u"home_defaults",
u"home_no_splits",
u"home_splits",
u"home_splits_shared",
))
self.sqlStore = self.store
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, self.directory,
True, True)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child("__uids__").child("ho").child(
"me").child("home1").child(".some-extra-data").setContent(
"some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child("__uids__").child(
"ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
# Add some properties we want to check get migrated over
txn = self.fileStore.newTransaction()
home = yield txn.calendarHomeWithUID("home_defaults")
cal = yield home.calendarWithName("calendar_1")
props = cal.properties()
props[PropertyName.fromElement(
caldavxml.SupportedCalendarComponentSet
)] = caldavxml.SupportedCalendarComponentSet(
caldavxml.CalendarComponent(name="VEVENT"),
caldavxml.CalendarComponent(name="VTODO"),
)
props[PropertyName.fromElement(
element.ResourceType)] = element.ResourceType(
element.Collection(),
caldavxml.Calendar(),
)
props[PropertyName.fromElement(
customxml.GETCTag)] = customxml.GETCTag.fromString("foobar")
inbox = yield home.calendarWithName("inbox")
props = inbox.properties()
props[PropertyName.fromElement(
customxml.CalendarAvailability
)] = customxml.CalendarAvailability.fromString(str(self.av1))
props[PropertyName.fromElement(
caldavxml.ScheduleDefaultCalendarURL
)] = caldavxml.ScheduleDefaultCalendarURL(
element.HRef.fromString(
"/calendars/__uids__/home_defaults/calendar_1"), )
yield txn.commit()
def mergeRequirements(self, a, b):
"""
Merge two requirements dictionaries together, modifying C{a} and
returning it.
@param a: Some requirements, in the format of
L{CommonTests.requirements}.
@type a: C{dict}
@param b: Some additional requirements, to be merged into C{a}.
@type b: C{dict}
@return: C{a}
@rtype: C{dict}
"""
for homeUID in b:
homereq = a.setdefault(homeUID, {})
homeExtras = b[homeUID]
for calendarUID in homeExtras:
calreq = homereq.setdefault(calendarUID, {})
calendarExtras = homeExtras[calendarUID]
calreq.update(calendarExtras)
return a
@withSpecialValue(
"extraRequirements", {
"home1": {
"calendar_1": {
"bogus.ics":
(getModule("twistedcaldav").filePath.sibling("zoneinfo").
child("EST.ics").getContent(), CommonTests.metadata1)
}
}
})
@inlineCallbacks
def test_unknownTypeNotMigrated(self):
"""
The only types of calendar objects that should get migrated are VEVENTs
and VTODOs. Other component types, such as free-standing VTIMEZONEs,
don't have a UID and can't be stored properly in the database, so they
should not be migrated.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
self.assertIdentical(
None, (yield
(yield
(yield
(yield txn.calendarHomeWithUID("home1")).calendarWithName(
"calendar_1"))).calendarObjectWithName("bogus.ics")))
@inlineCallbacks
def test_upgradeCalendarHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
# Create a fake directory in the same place as a home, but with a non-existent uid
fake_dir = self.filesPath.child("calendars").child("__uids__").child(
"ho").child("me").child("foobar")
fake_dir.makedirs()
# Create a fake file in the same place as a home,with a name that matches the hash uid prefix
fake_file = self.filesPath.child("calendars").child("__uids__").child(
"ho").child("me").child("home_file")
fake_file.setContent("")
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(None,
(yield txn.calendarHomeWithUID(uid)))
# Successfully migrated calendar homes are deleted
self.assertFalse(
self.filesPath.child("calendars").child("__uids__").child(
"ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
for name, metadata, md5 in (
("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
):
object = (yield calendar.calendarObjectWithName(name))
self.assertEquals(object.getMetadata(), metadata)
self.assertEquals(object.md5(), md5)
@withSpecialValue("extraRequirements", {"nonexistent": {"calendar_1": {}}})
@inlineCallbacks
def test_upgradeCalendarHomesMissingDirectoryRecord(self):
"""
Test an upgrade where a directory record is missing for a home;
the original home directory will remain on disk.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(None,
(yield txn.calendarHomeWithUID(uid)))
self.assertIdentical(None,
(yield txn.calendarHomeWithUID(u"nonexistent")))
# Skipped calendar homes are not deleted
self.assertTrue(
self.filesPath.child("calendars").child("__uids__").child(
"no").child("ne").child("nonexistent").exists())
@inlineCallbacks
def test_upgradeExistingHome(self):
"""
L{UpgradeToDatabaseService.startService} will skip migrating existing
homes.
"""
startTxn = self.sqlStore.newTransaction("populate empty sample")
yield startTxn.calendarHomeWithUID("home1", create=True)
yield startTxn.commit()
yield self.upgrader.stepWithResult(None)
vrfyTxn = self.sqlStore.newTransaction("verify sample still empty")
self.addCleanup(vrfyTxn.commit)
home = yield vrfyTxn.calendarHomeWithUID("home1")
# The default calendar is still there.
self.assertNotIdentical(None,
(yield home.calendarWithName("calendar")))
# The migrated calendar isn't.
self.assertIdentical(None, (yield home.calendarWithName("calendar_1")))
@inlineCallbacks
def test_upgradeAttachments(self):
"""
L{UpgradeToDatabaseService.startService} upgrades calendar attachments
as well.
"""
# Need to tweak config and settings to setup dropbox to work
self.patch(config, "EnableDropBox", True)
self.patch(config, "EnableManagedAttachments", False)
self.sqlStore.enableManagedAttachments = False
txn = self.sqlStore.newTransaction()
cs = schema.CALENDARSERVER
yield Delete(From=cs, Where=cs.NAME == "MANAGED-ATTACHMENTS").on(txn)
yield txn.commit()
txn = self.fileStore.newTransaction()
committed = []
def maybeCommit():
if not committed:
committed.append(True)
return txn.commit()
self.addCleanup(maybeCommit)
@inlineCallbacks
def getSampleObj():
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
object = (yield calendar.calendarObjectWithName("1.ics"))
returnValue(object)
inObject = yield getSampleObj()
someAttachmentName = "some-attachment"
someAttachmentType = MimeType.fromString("application/x-custom-type")
attachment = yield inObject.createAttachmentWithName(
someAttachmentName, )
transport = attachment.store(someAttachmentType)
someAttachmentData = "Here is some data for your attachment, enjoy."
transport.write(someAttachmentData)
yield transport.loseConnection()
yield maybeCommit()
yield self.upgrader.stepWithResult(None)
committed = []
txn = self.sqlStore.newTransaction()
outObject = yield getSampleObj()
outAttachment = yield outObject.attachmentWithName(someAttachmentName)
allDone = Deferred()
class SimpleProto(Protocol):
data = ''
def dataReceived(self, data):
self.data += data
def connectionLost(self, reason):
allDone.callback(self.data)
self.assertEquals(outAttachment.contentType(), someAttachmentType)
outAttachment.retrieve(SimpleProto())
allData = yield allDone
self.assertEquals(allData, someAttachmentData)
@inlineCallbacks
def test_upgradeAddressBookHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in ABCommonTests.requirements:
if ABCommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.addressbookHomeWithUID(uid)))
# Successfully migrated addressbook homes are deleted
self.assertFalse(
self.filesPath.child("addressbooks").child("__uids__").child(
"ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.addressbookHomeWithUID("home1"))
adbk = (yield home.addressbookWithName("addressbook"))
for name, md5 in (
("1.vcf", ABCommonTests.md5Values[0]),
("2.vcf", ABCommonTests.md5Values[1]),
("3.vcf", ABCommonTests.md5Values[2]),
):
object = (yield adbk.addressbookObjectWithName(name))
self.assertEquals(object.md5(), md5)
@inlineCallbacks
def test_upgradeProperties(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home_defaults"))
cal = (yield home.calendarWithName("calendar_1"))
inbox = (yield home.calendarWithName("inbox"))
# Supported components
self.assertEqual(cal.getSupportedComponents(), "VEVENT")
self.assertTrue(cal.properties().get(
PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet))
is None)
# Resource type removed
self.assertTrue(cal.properties().get(
PropertyName.fromElement(element.ResourceType)) is None)
# Ctag removed
self.assertTrue(cal.properties().get(
PropertyName.fromElement(customxml.GETCTag)) is None)
# Availability
self.assertEquals(str(home.getAvailability()), str(self.av1))
self.assertTrue(inbox.properties().get(
PropertyName.fromElement(customxml.CalendarAvailability)) is None)
# Default calendar
self.assertTrue(home.isDefaultCalendar(cal))
self.assertTrue(inbox.properties().get(
PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)) is
None)
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
class PostgresService(MultiService):
def __init__(
self, dataStoreDirectory, subServiceFactory,
schema, resetSchema=False, databaseName="subpostgres",
clusterName="cluster",
logFile="postgres.log",
logDirectory="",
socketDir="",
listenAddresses=[], sharedBuffers=30,
maxConnections=20, options=[],
testMode=False,
uid=None, gid=None,
spawnedDBUser="******",
importFileName=None,
pgCtl="pg_ctl",
initDB="initdb",
reactor=None,
):
"""
Initialize a L{PostgresService} pointed at a data store directory.
@param dataStoreDirectory: the directory to
@type dataStoreDirectory: L{twext.python.filepath.CachingFilePath}
@param subServiceFactory: a 1-arg callable that will be called with a
1-arg callable which returns a DB-API cursor.
@type subServiceFactory: C{callable}
@param spawnedDBUser: the postgres role
@type spawnedDBUser: C{str}
@param importFileName: path to SQL file containing previous data to
import
@type importFileName: C{str}
"""
# FIXME: By default there is very little (4MB) shared memory available,
# so at the moment I am lowering these postgres config options to allow
# multiple servers to run. We might want to look into raising
# kern.sysv.shmmax.
# See: http://www.postgresql.org/docs/8.4/static/kernel-resources.html
MultiService.__init__(self)
self.subServiceFactory = subServiceFactory
self.dataStoreDirectory = dataStoreDirectory
self.workingDir = self.dataStoreDirectory.child("working")
self.resetSchema = resetSchema
# In order to delay a shutdown until database initialization has
# completed, our stopService( ) examines the delayedShutdown flag.
# If True, we wait on the shutdownDeferred to fire before proceeding.
# The deferred gets fired once database init is complete.
self.delayedShutdown = False # set to True when in critical code
self.shutdownDeferred = None # the actual deferred
# Options from config
self.databaseName = databaseName
self.clusterName = clusterName
# Make logFile absolute in case the working directory of postgres is
# elsewhere:
self.logFile = os.path.abspath(logFile)
if logDirectory:
self.logDirectory = os.path.abspath(logDirectory)
else:
self.logDirectory = ""
# Always use our own configured socket dir in case the built-in
# postgres tries to use a directory we don't have permissions for
if not socketDir:
# Socket directory was not specified, so come up with one
# in /tmp and based on a hash of the data store directory
digest = md5(dataStoreDirectory.path).hexdigest()
socketDir = "/tmp/ccs_postgres_" + digest
self.socketDir = CachingFilePath(socketDir)
if listenAddresses:
if ":" in listenAddresses[0]:
self.host, self.port = listenAddresses[0].split(":")
else:
self.host, self.port = (listenAddresses[0], None)
self.listenAddresses = [
addr.split(":")[0] for addr in listenAddresses
]
else:
self.host = self.socketDir.path
self.port = None
self.listenAddresses = []
self.sharedBuffers = sharedBuffers if not testMode else 16
self.maxConnections = maxConnections if not testMode else 8
self.options = options
self.uid = uid
self.gid = gid
self.spawnedDBUser = spawnedDBUser
self.importFileName = importFileName
self.schema = schema
self.monitor = None
self.openConnections = []
self._pgCtl = pgCtl
self._initdb = initDB
self._reactor = reactor
self._postgresPid = None
@property
def reactor(self):
if self._reactor is None:
from twisted.internet import reactor
self._reactor = reactor
return self._reactor
def pgCtl(self):
"""
Locate the path to pg_ctl.
"""
return which(self._pgCtl)[0]
def initdb(self):
return which(self._initdb)[0]
def activateDelayedShutdown(self):
"""
Call this when starting database initialization code to protect against
shutdown.
Sets the delayedShutdown flag to True so that if reactor shutdown
commences, the shutdown will be delayed until deactivateDelayedShutdown
is called.
"""
self.delayedShutdown = True
def deactivateDelayedShutdown(self):
"""
Call this when database initialization code has completed so that the
reactor can shutdown.
"""
self.delayedShutdown = False
if self.shutdownDeferred:
self.shutdownDeferred.callback(None)
def _connectorFor(self, databaseName=None):
if databaseName is None:
databaseName = self.databaseName
if self.spawnedDBUser:
dsn = "{}:dbname={}:{}".format(
self.host, databaseName, self.spawnedDBUser
)
elif self.uid is not None:
dsn = "{}:dbname={}:{}".format(
self.host, databaseName, pwd.getpwuid(self.uid).pw_name
)
else:
dsn = "{}:dbname={}".format(self.host, databaseName)
kwargs = {}
if self.port:
kwargs["host"] = "{}:{}".format(self.host, self.port)
return DBAPIConnector(pgdb, postgresPreflight, dsn, **kwargs)
def produceConnection(self, label="<unlabeled>", databaseName=None):
"""
Produce a DB-API 2.0 connection pointed at this database.
"""
return self._connectorFor(databaseName).connect(label)
def ready(self, createDatabaseConn, createDatabaseCursor):
"""
Subprocess is ready. Time to initialize the subservice.
If the database has not been created and there is a dump file,
then the dump file is imported.
"""
if self.resetSchema:
try:
createDatabaseCursor.execute(
"drop database {}".format(self.databaseName)
)
except pgdb.DatabaseError:
pass
try:
createDatabaseCursor.execute(
"create database {} with encoding 'UTF8'"
.format(self.databaseName)
)
except:
# database already exists
executeSQL = False
else:
# database does not yet exist; if dump file exists, execute it,
# otherwise execute schema
executeSQL = True
sqlToExecute = self.schema
if self.importFileName:
importFilePath = CachingFilePath(self.importFileName)
if importFilePath.exists():
sqlToExecute = importFilePath.getContent()
createDatabaseCursor.close()
createDatabaseConn.close()
if executeSQL:
connection = self.produceConnection()
cursor = connection.cursor()
cursor.execute(sqlToExecute)
connection.commit()
connection.close()
if self.shutdownDeferred is None:
# Only continue startup if we've not begun shutdown
self.subServiceFactory(
self.produceConnection, self
).setServiceParent(self)
def pauseMonitor(self):
"""
Pause monitoring. This is a testing hook for when (if) we are
continuously monitoring output from the 'postgres' process.
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.stopReading()
# pipe.stopWriting()
pass
def unpauseMonitor(self):
"""
Unpause monitoring.
@see: L{pauseMonitor}
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.startReading()
# pipe.startWriting()
pass
def startDatabase(self):
"""
Start the database and initialize the subservice.
"""
def createConnection():
createDatabaseConn = self.produceConnection(
"schema creation", "postgres"
)
createDatabaseCursor = createDatabaseConn.cursor()
createDatabaseCursor.execute("commit")
return createDatabaseConn, createDatabaseCursor
monitor = _PostgresMonitor(self)
pgCtl = self.pgCtl()
# check consistency of initdb and postgres?
options = []
options.append(
"-c listen_addresses={}"
.format(shell_quote(",".join(self.listenAddresses)))
)
if self.socketDir:
options.append(
"-k {}"
.format(shell_quote(self.socketDir.path))
)
if self.port:
options.append(
"-c port={}".format(shell_quote(self.port))
)
options.append(
"-c shared_buffers={:d}"
.format(self.sharedBuffers) # int: don't quote
)
options.append(
"-c max_connections={:d}"
.format(self.maxConnections) # int: don't quote
)
options.append("-c standard_conforming_strings=on")
options.append("-c unix_socket_permissions=0770")
options.extend(self.options)
if self.logDirectory: # tell postgres to rotate logs
options.append(
"-c log_directory={}".format(shell_quote(self.logDirectory))
)
options.append("-c log_truncate_on_rotation=on")
options.append("-c log_filename=postgresql_%w.log")
options.append("-c log_rotation_age=1440")
options.append("-c logging_collector=on")
log.warn(
"Requesting postgres start via {cmd} {opts}",
cmd=pgCtl, opts=options
)
self.reactor.spawnProcess(
monitor, pgCtl,
[
pgCtl,
"start",
"-l", self.logFile,
"-t 86400", # Give plenty of time for a long cluster upgrade
"-w",
# XXX what are the quoting rules for '-o'? do I need to repr()
# the path here?
"-o",
" ".join(options),
],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
self.monitor = monitor
def gotStatus(result):
"""
Grab the postgres pid from the pgCtl status call in case we need
to kill it directly later on in hardStop(). Useful in conjunction
with the DataStoreMonitor so we can shut down if DataRoot has been
removed/renamed/unmounted.
"""
reResult = re.search("PID: (\d+)\D", result)
if reResult is not None:
self._postgresPid = int(reResult.group(1))
self.ready(*createConnection())
self.deactivateDelayedShutdown()
def gotReady(result):
"""
We started postgres; we're responsible for stopping it later.
Call pgCtl status to get the pid.
"""
log.warn("{cmd} exited", cmd=pgCtl)
self.shouldStopDatabase = True
d = Deferred()
statusMonitor = CapturingProcessProtocol(d, None)
self.reactor.spawnProcess(
statusMonitor, pgCtl, [pgCtl, "status"],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return d.addCallback(gotStatus)
def couldNotStart(f):
"""
There was an error trying to start postgres. Try to connect
because it might already be running. In this case, we won't
be the one to stop it.
"""
d = Deferred()
statusMonitor = CapturingProcessProtocol(d, None)
self.reactor.spawnProcess(
statusMonitor, pgCtl, [pgCtl, "status"],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return d.addCallback(gotStatus).addErrback(giveUp)
def giveUp(f):
"""
We can't start postgres or connect to a running instance. Shut
down.
"""
log.failure("Can't start or connect to postgres", f)
self.deactivateDelayedShutdown()
self.reactor.stop()
self.monitor.completionDeferred.addCallback(
gotReady).addErrback(couldNotStart)
shouldStopDatabase = False
def startService(self):
MultiService.startService(self)
self.activateDelayedShutdown()
clusterDir = self.dataStoreDirectory.child(self.clusterName)
env = self.env = os.environ.copy()
env.update(PGDATA=clusterDir.path,
PGHOST=self.host,
PGUSER=self.spawnedDBUser)
initdb = self.initdb()
if self.socketDir:
if not self.socketDir.isdir():
log.warn("Creating {dir}", dir=self.socketDir.path)
self.socketDir.createDirectory()
if self.uid and self.gid:
os.chown(self.socketDir.path, self.uid, self.gid)
os.chmod(self.socketDir.path, 0770)
if not self.dataStoreDirectory.isdir():
log.warn("Creating {dir}", dir=self.dataStoreDirectory.path)
self.dataStoreDirectory.createDirectory()
if not self.workingDir.isdir():
log.warn("Creating {dir}", dir=self.workingDir.path)
self.workingDir.createDirectory()
if self.uid and self.gid:
os.chown(self.dataStoreDirectory.path, self.uid, self.gid)
os.chown(self.workingDir.path, self.uid, self.gid)
if not clusterDir.isdir():
# No cluster directory, run initdb
log.warn("Running initdb for {dir}", dir=clusterDir.path)
dbInited = Deferred()
self.reactor.spawnProcess(
CapturingProcessProtocol(dbInited, None),
initdb, [initdb, "-E", "UTF8", "-U", self.spawnedDBUser],
env=env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
def doCreate(result):
if result.find("FATAL:") != -1:
log.error(result)
raise RuntimeError(
"Unable to initialize postgres database: {}"
.format(result)
)
self.startDatabase()
dbInited.addCallback(doCreate)
else:
log.warn("Cluster already exists at {dir}", dir=clusterDir.path)
self.startDatabase()
def stopService(self):
"""
Stop all child services, then stop the subprocess, if it's running.
"""
if self.delayedShutdown:
# We're still in the process of initializing the database, so
# delay shutdown until the shutdownDeferred fires.
d = self.shutdownDeferred = Deferred()
d.addCallback(lambda ignored: MultiService.stopService(self))
else:
d = MultiService.stopService(self)
def superStopped(result):
# If pg_ctl's startup wasn't successful, don't bother to stop the
# database. (This also happens in command-line tools.)
if self.shouldStopDatabase:
monitor = _PostgresMonitor()
pgCtl = self.pgCtl()
# FIXME: why is this 'logfile' and not self.logfile?
self.reactor.spawnProcess(
monitor, pgCtl,
[pgCtl, "-l", "logfile", "stop"],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return monitor.completionDeferred
return d.addCallback(superStopped)
# def maybeStopSubprocess(result):
# if self.monitor is not None:
# self.monitor.transport.signalProcess("INT")
# return self.monitor.completionDeferred
# return result
# d.addCallback(maybeStopSubprocess)
# return d
def hardStop(self):
"""
Stop postgres quickly by sending it SIGQUIT
"""
if self._postgresPid is not None:
try:
os.kill(self._postgresPid, signal.SIGQUIT)
except OSError:
pass
class PostgresService(MultiService):
def __init__(
self, dataStoreDirectory, subServiceFactory,
schema, resetSchema=False, databaseName="subpostgres",
clusterName="cluster",
logFile="postgres.log",
logDirectory="",
socketDir="",
socketName="",
listenAddresses=[], sharedBuffers=30,
maxConnections=20, options=[],
testMode=False,
uid=None, gid=None,
spawnedDBUser="******",
pgCtl="pg_ctl",
initDB="initdb",
reactor=None,
):
"""
Initialize a L{PostgresService} pointed at a data store directory.
@param dataStoreDirectory: the directory to
@type dataStoreDirectory: L{twext.python.filepath.CachingFilePath}
@param subServiceFactory: a 1-arg callable that will be called with a
1-arg callable which returns a DB-API cursor.
@type subServiceFactory: C{callable}
@param spawnedDBUser: the postgres role
@type spawnedDBUser: C{str}
"""
# FIXME: By default there is very little (4MB) shared memory available,
# so at the moment I am lowering these postgres config options to allow
# multiple servers to run. We might want to look into raising
# kern.sysv.shmmax.
# See: http://www.postgresql.org/docs/8.4/static/kernel-resources.html
MultiService.__init__(self)
self.subServiceFactory = subServiceFactory
self.dataStoreDirectory = dataStoreDirectory
self.workingDir = self.dataStoreDirectory.child("working")
self.resetSchema = resetSchema
# In order to delay a shutdown until database initialization has
# completed, our stopService( ) examines the delayedShutdown flag.
# If True, we wait on the shutdownDeferred to fire before proceeding.
# The deferred gets fired once database init is complete.
self.delayedShutdown = False # set to True when in critical code
self.shutdownDeferred = None # the actual deferred
# Options from config
self.databaseName = databaseName
self.clusterName = clusterName
# Make logFile absolute in case the working directory of postgres is
# elsewhere:
self.logFile = os.path.abspath(logFile)
if logDirectory:
self.logDirectory = os.path.abspath(logDirectory)
else:
self.logDirectory = ""
# Always use our own configured socket dir in case the built-in
# postgres tries to use a directory we don't have permissions for
if not socketDir:
# Socket directory was not specified, so come up with one
# in /tmp and based on a hash of the data store directory
digest = md5(dataStoreDirectory.path).hexdigest()
socketDir = "/tmp/ccs_postgres_" + digest
self.socketDir = CachingFilePath(socketDir)
self.socketName = socketName
if listenAddresses:
if ":" in listenAddresses[0]:
self.host, self.port = listenAddresses[0].split(":")
else:
self.host, self.port = (listenAddresses[0], None)
self.listenAddresses = [
addr.split(":")[0] for addr in listenAddresses
]
else:
self.host = self.socketDir.path
self.port = None
self.listenAddresses = []
self.testMode = testMode
self.sharedBuffers = sharedBuffers if not testMode else 16
self.maxConnections = maxConnections if not testMode else 8
self.options = options
self.uid = uid
self.gid = gid
self.spawnedDBUser = spawnedDBUser
self.schema = schema
self.monitor = None
self.openConnections = []
def locateCommand(name, cmd):
for found in which(cmd):
return found
raise InternalDataStoreError(
"Unable to locate {} command: {}".format(name, cmd)
)
self._pgCtl = locateCommand("pg_ctl", pgCtl)
self._initdb = locateCommand("initdb", initDB)
self._reactor = reactor
self._postgresPid = None
@property
def reactor(self):
if self._reactor is None:
from twisted.internet import reactor
self._reactor = reactor
return self._reactor
def activateDelayedShutdown(self):
"""
Call this when starting database initialization code to
protect against shutdown.
Sets the delayedShutdown flag to True so that if reactor shutdown
commences, the shutdown will be delayed until deactivateDelayedShutdown
is called.
"""
self.delayedShutdown = True
def deactivateDelayedShutdown(self):
"""
Call this when database initialization code has completed so that the
reactor can shutdown.
"""
self.delayedShutdown = False
if self.shutdownDeferred:
self.shutdownDeferred.callback(None)
def _connectorFor(self, databaseName=None):
if databaseName is None:
databaseName = self.databaseName
kwargs = {
"database": databaseName,
}
if self.host.startswith("/"):
kwargs["endpoint"] = "unix:{}".format(self.host)
else:
kwargs["endpoint"] = "tcp:{}".format(self.host)
if self.port:
kwargs["endpoint"] = "{}:{}".format(kwargs["endpoint"], self.port)
if self.spawnedDBUser:
kwargs["user"] = self.spawnedDBUser
elif self.uid is not None:
kwargs["user"] = pwd.getpwuid(self.uid).pw_name
return DBAPIConnector.connectorFor("postgres", **kwargs)
def produceConnection(self, label="<unlabeled>", databaseName=None):
"""
Produce a DB-API 2.0 connection pointed at this database.
"""
connection = self._connectorFor(databaseName).connect(label)
if postgres.__name__ == "pg8000":
# Patch pg8000 behavior to match what we need wrt text processing
def my_text_out(v):
return v.encode("utf-8") if isinstance(v, unicode) else str(v)
connection.realConnection.py_types[str] = (705, postgres.core.FC_TEXT, my_text_out)
connection.realConnection.py_types[postgres.six.text_type] = (705, postgres.core.FC_TEXT, my_text_out)
def my_text_recv(data, offset, length):
return str(data[offset: offset + length])
connection.realConnection.default_factory = lambda: (postgres.core.FC_TEXT, my_text_recv)
connection.realConnection.pg_types[19] = (postgres.core.FC_BINARY, my_text_recv)
connection.realConnection.pg_types[25] = (postgres.core.FC_BINARY, my_text_recv)
connection.realConnection.pg_types[705] = (postgres.core.FC_BINARY, my_text_recv)
connection.realConnection.pg_types[829] = (postgres.core.FC_TEXT, my_text_recv)
connection.realConnection.pg_types[1042] = (postgres.core.FC_BINARY, my_text_recv)
connection.realConnection.pg_types[1043] = (postgres.core.FC_BINARY, my_text_recv)
connection.realConnection.pg_types[2275] = (postgres.core.FC_BINARY, my_text_recv)
return connection
def ready(self, createDatabaseConn, createDatabaseCursor):
"""
Subprocess is ready. Time to initialize the subservice.
If the database has not been created and there is a dump file,
then the dump file is imported.
"""
if self.resetSchema:
try:
createDatabaseCursor.execute(
"drop database {}".format(self.databaseName)
)
except postgres.DatabaseError:
pass
try:
createDatabaseCursor.execute(
"create database {} with encoding 'UTF8'"
.format(self.databaseName)
)
except:
# database already exists
sqlToExecute = None
else:
# database does not yet exist; if dump file exists, execute it,
# otherwise execute schema
sqlToExecute = self.schema
createDatabaseCursor.close()
createDatabaseConn.close()
if sqlToExecute is not None:
connection = self.produceConnection()
cursor = connection.cursor()
for statement in splitSQLString(sqlToExecute):
cursor.execute(statement)
connection.commit()
connection.close()
if self.shutdownDeferred is None:
# Only continue startup if we've not begun shutdown
self.subServiceFactory(
self.produceConnection, self
).setServiceParent(self)
def pauseMonitor(self):
"""
Pause monitoring. This is a testing hook for when (if) we are
continuously monitoring output from the 'postgres' process.
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.stopReading()
# pipe.stopWriting()
pass
def unpauseMonitor(self):
"""
Unpause monitoring.
@see: L{pauseMonitor}
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.startReading()
# pipe.startWriting()
pass
def startDatabase(self):
"""
Start the database and initialize the subservice.
"""
def createConnection():
try:
createDatabaseConn = self.produceConnection(
"schema creation", "postgres"
)
except postgres.DatabaseError as e:
log.error(
"Unable to connect to database for schema creation:"
" {error}",
error=e
)
raise
createDatabaseCursor = createDatabaseConn.cursor()
if postgres.__name__ == "pg8000":
createDatabaseConn.realConnection.autocommit = True
elif postgres.__name__ == "pgdb":
createDatabaseCursor.execute("commit")
else:
raise InternalDataStoreError(
"Unknown Postgres DBM module: {}".format(postgres)
)
return createDatabaseConn, createDatabaseCursor
monitor = PostgresMonitor(self)
# check consistency of initdb and postgres?
options = []
options.append(
"-c listen_addresses={}"
.format(shell_quote(",".join(self.listenAddresses)))
)
if self.socketDir:
options.append(
"-c unix_socket_directories={}"
.format(shell_quote(self.socketDir.path))
)
if self.port:
options.append(
"-c port={}".format(shell_quote(self.port))
)
options.append(
"-c shared_buffers={:d}"
.format(self.sharedBuffers) # int: don't quote
)
options.append(
"-c max_connections={:d}"
.format(self.maxConnections) # int: don't quote
)
options.append("-c standard_conforming_strings=on")
options.append("-c unix_socket_permissions=0770")
options.extend(self.options)
if self.logDirectory: # tell postgres to rotate logs
options.append(
"-c log_directory={}".format(shell_quote(self.logDirectory))
)
options.append("-c log_truncate_on_rotation=on")
options.append("-c log_filename=postgresql_%w.log")
options.append("-c log_rotation_age=1440")
options.append("-c logging_collector=on")
options.append("-c log_line_prefix=%t")
if self.testMode:
options.append("-c log_statement=all")
args = [
self._pgCtl, "start",
"--log={}".format(self.logFile),
"--timeout=86400", # Plenty of time for a long cluster upgrade
"-w", # Wait for startup to complete
"-o", " ".join(options), # Options passed to postgres
]
log.info("Requesting postgres start via: {args}", args=args)
self.reactor.spawnProcess(
monitor, self._pgCtl, args,
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
self.monitor = monitor
def gotStatus(result):
"""
Grab the postgres pid from the pgCtl status call in case we need
to kill it directly later on in hardStop(). Useful in conjunction
with the DataStoreMonitor so we can shut down if DataRoot has been
removed/renamed/unmounted.
"""
reResult = re.search("PID: (\d+)\D", result)
if reResult is not None:
self._postgresPid = int(reResult.group(1))
self.ready(*createConnection())
self.deactivateDelayedShutdown()
def gotReady(result):
"""
We started postgres; we're responsible for stopping it later.
Call pgCtl status to get the pid.
"""
log.info("{cmd} exited", cmd=self._pgCtl)
self.shouldStopDatabase = True
d = Deferred()
statusMonitor = CapturingProcessProtocol(d, None)
self.reactor.spawnProcess(
statusMonitor, self._pgCtl, [self._pgCtl, "status"],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return d.addCallback(gotStatus)
def couldNotStart(f):
"""
There was an error trying to start postgres. Try to connect
because it might already be running. In this case, we won't
be the one to stop it.
"""
d = Deferred()
statusMonitor = CapturingProcessProtocol(d, None)
self.reactor.spawnProcess(
statusMonitor, self._pgCtl, [self._pgCtl, "status"],
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return d.addCallback(gotStatus).addErrback(giveUp)
def giveUp(f):
"""
We can't start postgres or connect to a running instance. Shut
down.
"""
log.critical(
"Can't start or connect to postgres: {failure.value}",
failure=f
)
self.deactivateDelayedShutdown()
self.reactor.stop()
self.monitor.completionDeferred.addCallback(
gotReady).addErrback(couldNotStart)
shouldStopDatabase = False
def startService(self):
MultiService.startService(self)
self.activateDelayedShutdown()
clusterDir = self.dataStoreDirectory.child(self.clusterName)
env = self.env = os.environ.copy()
env.update(PGDATA=clusterDir.path,
PGHOST=self.host,
PGUSER=self.spawnedDBUser)
if self.socketDir:
if not self.socketDir.isdir():
log.info("Creating {dir}", dir=self.socketDir.path.decode("utf-8"))
self.socketDir.createDirectory()
if self.uid and self.gid:
os.chown(self.socketDir.path, self.uid, self.gid)
os.chmod(self.socketDir.path, 0770)
if not self.dataStoreDirectory.isdir():
log.info("Creating {dir}", dir=self.dataStoreDirectory.path.decode("utf-8"))
self.dataStoreDirectory.createDirectory()
if not self.workingDir.isdir():
log.info("Creating {dir}", dir=self.workingDir.path.decode("utf-8"))
self.workingDir.createDirectory()
if self.uid and self.gid:
os.chown(self.dataStoreDirectory.path, self.uid, self.gid)
os.chown(self.workingDir.path, self.uid, self.gid)
if not clusterDir.isdir():
# No cluster directory, run initdb
log.info("Running initdb for {dir}", dir=clusterDir.path.decode("utf-8"))
dbInited = Deferred()
self.reactor.spawnProcess(
CapturingProcessProtocol(dbInited, None),
self._initdb,
[self._initdb, "-E", "UTF8", "-U", self.spawnedDBUser],
env=env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
def doCreate(result):
if result.find("FATAL:") != -1:
log.error(result)
raise InternalDataStoreError(
"Unable to initialize postgres database: {}"
.format(result)
)
self.startDatabase()
dbInited.addCallback(doCreate)
else:
log.info("Cluster already exists at {dir}", dir=clusterDir.path.decode("utf-8"))
self.startDatabase()
def stopService(self):
"""
Stop all child services, then stop the subprocess, if it's running.
"""
if self.delayedShutdown:
# We're still in the process of initializing the database, so
# delay shutdown until the shutdownDeferred fires.
d = self.shutdownDeferred = Deferred()
d.addCallback(lambda ignored: MultiService.stopService(self))
else:
d = MultiService.stopService(self)
def superStopped(result):
# If pg_ctl's startup wasn't successful, don't bother to stop the
# database. (This also happens in command-line tools.)
if self.shouldStopDatabase:
monitor = PostgresMonitor()
args = [
self._pgCtl, "stop",
"--log={}".format(self.logFile),
]
log.info("Requesting postgres stop via: {args}", args=args)
self.reactor.spawnProcess(
monitor, self._pgCtl,
args,
env=self.env, path=self.workingDir.path,
uid=self.uid, gid=self.gid,
)
return monitor.completionDeferred
return d.addCallback(superStopped)
# def maybeStopSubprocess(result):
# if self.monitor is not None:
# self.monitor.transport.signalProcess("INT")
# return self.monitor.completionDeferred
# return result
# d.addCallback(maybeStopSubprocess)
# return d
def hardStop(self):
"""
Stop postgres quickly by sending it SIGQUIT
"""
if self._postgresPid is not None:
try:
os.kill(self._postgresPid, signal.SIGQUIT)
except OSError:
pass
class PostgresService(MultiService):
def __init__(self, dataStoreDirectory, subServiceFactory,
schema, databaseName='subpostgres', resetSchema=False,
logFile="postgres.log", testMode=False,
uid=None, gid=None):
"""
Initialize a L{PostgresService} pointed at a data store directory.
@param dataStoreDirectory: the directory to
@type dataStoreDirectory: L{twext.python.filepath.CachingFilePath}
@param subServiceFactory: a 1-arg callable that will be called with a
1-arg callable which returns a DB-API cursor.
@type subServiceFactory: C{callable}
"""
MultiService.__init__(self)
self.subServiceFactory = subServiceFactory
self.dataStoreDirectory = dataStoreDirectory
self.resetSchema = resetSchema
if os.getuid() == 0:
socketRoot = "/var/run"
else:
socketRoot = "/tmp"
self.socketDir = CachingFilePath("%s/ccs_postgres_%s/" %
(socketRoot, md5(dataStoreDirectory.path).hexdigest()))
self.databaseName = databaseName
self.logFile = logFile
self.uid = uid
self.gid = gid
self.schema = schema
self.monitor = None
self.openConnections = []
# FIXME: By default there is very little (4MB) shared memory available,
# so at the moment I am lowering these postgres config options to allow
# multiple servers to run. We might want to look into raising
# kern.sysv.shmmax.
# See: http://www.postgresql.org/docs/8.4/static/kernel-resources.html
if testMode:
self.sharedBuffers = 16
self.maxConnections = 2
else:
self.sharedBuffers = 30
self.maxConnections = 20
def produceConnection(self, label="<unlabeled>", databaseName=None):
"""
Produce a DB-API 2.0 connection pointed at this database.
"""
if databaseName is None:
databaseName = self.databaseName
if self.uid is not None:
dsn = "%s:dbname=%s:%s" % (self.socketDir.path, databaseName,
pwd.getpwuid(self.uid).pw_name)
else:
dsn = "%s:dbname=%s" % (self.socketDir.path, databaseName)
connection = pgdb.connect(dsn)
w = DiagnosticConnectionWrapper(connection, label)
c = w.cursor()
# Turn on standard conforming strings. This option is _required_ if
# you want to get correct behavior out of parameter-passing with the
# pgdb module. If it is not set then the server is potentially
# vulnerable to certain types of SQL injection.
c.execute("set standard_conforming_strings=on")
# Abort any second that takes more than 30 seconds (30000ms) to
# execute. This is necessary as a temporary workaround since it's
# hypothetically possible that different database operations could
# block each other, while executing SQL in the same process (in the
# same thread, since SQL executes in the main thread now). It's
# preferable to see some exceptions while we're in this state than to
# have the entire worker process hang.
c.execute("set statement_timeout=30000")
w.commit()
c.close()
return w
def ready(self):
"""
Subprocess is ready. Time to initialize the subservice.
"""
createDatabaseConn = self.produceConnection(
'schema creation', 'postgres'
)
createDatabaseCursor = createDatabaseConn.cursor()
createDatabaseCursor.execute("commit")
if self.resetSchema:
try:
createDatabaseCursor.execute(
"drop database %s" % (self.databaseName)
)
except pgdb.DatabaseError:
pass
try:
createDatabaseCursor.execute(
"create database %s" % (self.databaseName)
)
except:
execSchema = False
else:
execSchema = True
createDatabaseCursor.close()
createDatabaseConn.close()
if execSchema:
connection = self.produceConnection()
cursor = connection.cursor()
cursor.execute(self.schema)
connection.commit()
connection.close()
connection = self.produceConnection()
cursor = connection.cursor()
self.subServiceFactory(self.produceConnection).setServiceParent(self)
def pauseMonitor(self):
"""
Pause monitoring. This is a testing hook for when (if) we are
continuously monitoring output from the 'postgres' process.
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.stopReading()
# pipe.stopWriting()
def unpauseMonitor(self):
"""
Unpause monitoring.
@see: L{pauseMonitor}
"""
# for pipe in self.monitor.transport.pipes.values():
# pipe.startReading()
# pipe.startWriting()
def startDatabase(self):
"""
Start the database and initialize the subservice.
"""
monitor = _PostgresMonitor(self)
pg_ctl = which("pg_ctl")[0]
# check consistency of initdb and postgres?
reactor.spawnProcess(
monitor, pg_ctl,
[
pg_ctl,
"start",
"-l", self.logFile,
"-w",
# XXX what are the quoting rules for '-o'? do I need to repr()
# the path here?
"-o", "-c listen_addresses='' -k '%s' -c standard_conforming_strings=on -c shared_buffers=%d -c max_connections=%d"
% (self.socketDir.path, self.sharedBuffers, self.maxConnections),
],
self.env,
uid=self.uid, gid=self.gid,
)
self.monitor = monitor
def gotReady(result):
self.ready()
def reportit(f):
log.err(f)
self.monitor.completionDeferred.addCallback(
gotReady).addErrback(reportit)
def startService(self):
MultiService.startService(self)
clusterDir = self.dataStoreDirectory.child("cluster")
workingDir = self.dataStoreDirectory.child("working")
env = self.env = os.environ.copy()
env.update(PGDATA=clusterDir.path,
PGHOST=self.socketDir.path)
initdb = which("initdb")[0]
if not self.socketDir.isdir():
self.socketDir.createDirectory()
if self.uid and self.gid:
os.chown(self.socketDir.path, self.uid, self.gid)
if self.dataStoreDirectory.isdir():
self.startDatabase()
else:
self.dataStoreDirectory.createDirectory()
workingDir.createDirectory()
if self.uid and self.gid:
os.chown(self.dataStoreDirectory.path, self.uid, self.gid)
os.chown(workingDir.path, self.uid, self.gid)
dbInited = Deferred()
reactor.spawnProcess(
CapturingProcessProtocol(dbInited, None),
initdb, [initdb], env, workingDir.path,
uid=self.uid, gid=self.gid,
)
def doCreate(result):
self.startDatabase()
dbInited.addCallback(doCreate)
def stopService(self):
"""
Stop all child services, then stop the subprocess, if it's running.
"""
d = MultiService.stopService(self)
def superStopped(result):
# Probably want to stop and wait for startup if that hasn't
# completed yet...
monitor = _PostgresMonitor()
pg_ctl = which("pg_ctl")[0]
reactor.spawnProcess(monitor, pg_ctl,
[pg_ctl, '-l', 'logfile', 'stop'],
self.env,
uid=self.uid, gid=self.gid,
)
return monitor.completionDeferred
return d.addCallback(superStopped)
class HomeMigrationTests(CommonCommonTests, TestCase):
"""
Tests for L{UpgradeToDatabaseStep}.
"""
av1 = Component.fromString("""BEGIN:VCALENDAR
VERSION:2.0
CALSCALE:GREGORIAN
PRODID:-//calendarserver.org//Zonal//EN
BEGIN:VAVAILABILITY
ORGANIZER:mailto:[email protected]
UID:[email protected]
DTSTAMP:20061005T133225Z
DTEND:20140101T000000Z
BEGIN:AVAILABLE
UID:[email protected]
DTSTAMP:20061005T133225Z
SUMMARY:Monday to Friday from 9:00 to 17:00
DTSTART:20130101T090000Z
DTEND:20130101T170000Z
RRULE:FREQ=WEEKLY;BYDAY=MO,TU,WE,TH,FR
END:AVAILABLE
END:VAVAILABILITY
END:VCALENDAR
""")
@inlineCallbacks
def setUp(self):
"""
Set up two stores to migrate between.
"""
yield super(HomeMigrationTests, self).setUp()
yield self.buildStoreAndDirectory(
extraUids=(
u"home1",
u"home2",
u"home3",
u"home_defaults",
u"home_no_splits",
u"home_splits",
u"home_splits_shared",
)
)
self.sqlStore = self.store
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, self.directory, True, True
)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
# Add some properties we want to check get migrated over
txn = self.fileStore.newTransaction()
home = yield txn.calendarHomeWithUID("home_defaults")
cal = yield home.calendarWithName("calendar_1")
props = cal.properties()
props[PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)] = caldavxml.SupportedCalendarComponentSet(
caldavxml.CalendarComponent(name="VEVENT"),
caldavxml.CalendarComponent(name="VTODO"),
)
props[PropertyName.fromElement(element.ResourceType)] = element.ResourceType(
element.Collection(),
caldavxml.Calendar(),
)
props[PropertyName.fromElement(customxml.GETCTag)] = customxml.GETCTag.fromString("foobar")
inbox = yield home.calendarWithName("inbox")
props = inbox.properties()
props[PropertyName.fromElement(customxml.CalendarAvailability)] = customxml.CalendarAvailability.fromString(str(self.av1))
props[PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)] = caldavxml.ScheduleDefaultCalendarURL(
element.HRef.fromString("/calendars/__uids__/home_defaults/calendar_1"),
)
yield txn.commit()
def mergeRequirements(self, a, b):
"""
Merge two requirements dictionaries together, modifying C{a} and
returning it.
@param a: Some requirements, in the format of
L{CommonTests.requirements}.
@type a: C{dict}
@param b: Some additional requirements, to be merged into C{a}.
@type b: C{dict}
@return: C{a}
@rtype: C{dict}
"""
for homeUID in b:
homereq = a.setdefault(homeUID, {})
homeExtras = b[homeUID]
for calendarUID in homeExtras:
calreq = homereq.setdefault(calendarUID, {})
calendarExtras = homeExtras[calendarUID]
calreq.update(calendarExtras)
return a
@withSpecialValue(
"extraRequirements",
{
"home1": {
"calendar_1": {
"bogus.ics": (
getModule("twistedcaldav").filePath.sibling("zoneinfo")
.child("EST.ics").getContent(),
CommonTests.metadata1
)
}
}
}
)
@inlineCallbacks
def test_unknownTypeNotMigrated(self):
"""
The only types of calendar objects that should get migrated are VEVENTs
and VTODOs. Other component types, such as free-standing VTIMEZONEs,
don't have a UID and can't be stored properly in the database, so they
should not be migrated.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
self.assertIdentical(
None,
(yield (yield (yield (
yield txn.calendarHomeWithUID("home1")
).calendarWithName("calendar_1"))
).calendarObjectWithName("bogus.ics"))
)
@inlineCallbacks
def test_upgradeCalendarHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
# Create a fake directory in the same place as a home, but with a non-existent uid
fake_dir = self.filesPath.child("calendars").child("__uids__").child("ho").child("me").child("foobar")
fake_dir.makedirs()
# Create a fake file in the same place as a home,with a name that matches the hash uid prefix
fake_file = self.filesPath.child("calendars").child("__uids__").child("ho").child("me").child("home_file")
fake_file.setContent("")
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.calendarHomeWithUID(uid))
)
# Successfully migrated calendar homes are deleted
self.assertFalse(self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
for name, metadata, md5 in (
("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
):
object = (yield calendar.calendarObjectWithName(name))
self.assertEquals(object.getMetadata(), metadata)
self.assertEquals(object.md5(), md5)
@withSpecialValue(
"extraRequirements",
{
"nonexistent": {
"calendar_1": {
}
}
}
)
@inlineCallbacks
def test_upgradeCalendarHomesMissingDirectoryRecord(self):
"""
Test an upgrade where a directory record is missing for a home;
the original home directory will remain on disk.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.calendarHomeWithUID(uid))
)
self.assertIdentical(None, (yield txn.calendarHomeWithUID(u"nonexistent")))
# Skipped calendar homes are not deleted
self.assertTrue(self.filesPath.child("calendars").child(
"__uids__").child("no").child("ne").child("nonexistent").exists())
@inlineCallbacks
def test_upgradeExistingHome(self):
"""
L{UpgradeToDatabaseService.startService} will skip migrating existing
homes.
"""
startTxn = self.sqlStore.newTransaction("populate empty sample")
yield startTxn.calendarHomeWithUID("home1", create=True)
yield startTxn.commit()
yield self.upgrader.stepWithResult(None)
vrfyTxn = self.sqlStore.newTransaction("verify sample still empty")
self.addCleanup(vrfyTxn.commit)
home = yield vrfyTxn.calendarHomeWithUID("home1")
# The default calendar is still there.
self.assertNotIdentical(None, (yield home.calendarWithName("calendar")))
# The migrated calendar isn't.
self.assertIdentical(None, (yield home.calendarWithName("calendar_1")))
@inlineCallbacks
def test_upgradeAttachments(self):
"""
L{UpgradeToDatabaseService.startService} upgrades calendar attachments
as well.
"""
# Need to tweak config and settings to setup dropbox to work
self.patch(config, "EnableDropBox", True)
self.patch(config, "EnableManagedAttachments", False)
self.sqlStore.enableManagedAttachments = False
txn = self.sqlStore.newTransaction()
cs = schema.CALENDARSERVER
yield Delete(
From=cs,
Where=cs.NAME == "MANAGED-ATTACHMENTS"
).on(txn)
yield txn.commit()
txn = self.fileStore.newTransaction()
committed = []
def maybeCommit():
if not committed:
committed.append(True)
return txn.commit()
self.addCleanup(maybeCommit)
@inlineCallbacks
def getSampleObj():
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
object = (yield calendar.calendarObjectWithName("1.ics"))
returnValue(object)
inObject = yield getSampleObj()
someAttachmentName = "some-attachment"
someAttachmentType = MimeType.fromString("application/x-custom-type")
attachment = yield inObject.createAttachmentWithName(
someAttachmentName,
)
transport = attachment.store(someAttachmentType)
someAttachmentData = "Here is some data for your attachment, enjoy."
transport.write(someAttachmentData)
yield transport.loseConnection()
yield maybeCommit()
yield self.upgrader.stepWithResult(None)
committed = []
txn = self.sqlStore.newTransaction()
outObject = yield getSampleObj()
outAttachment = yield outObject.attachmentWithName(someAttachmentName)
allDone = Deferred()
class SimpleProto(Protocol):
data = ''
def dataReceived(self, data):
self.data += data
def connectionLost(self, reason):
allDone.callback(self.data)
self.assertEquals(outAttachment.contentType(), someAttachmentType)
outAttachment.retrieve(SimpleProto())
allData = yield allDone
self.assertEquals(allData, someAttachmentData)
@inlineCallbacks
def test_upgradeAddressBookHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in ABCommonTests.requirements:
if ABCommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.addressbookHomeWithUID(uid))
)
# Successfully migrated addressbook homes are deleted
self.assertFalse(self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.addressbookHomeWithUID("home1"))
adbk = (yield home.addressbookWithName("addressbook"))
for name, md5 in (
("1.vcf", ABCommonTests.md5Values[0]),
("2.vcf", ABCommonTests.md5Values[1]),
("3.vcf", ABCommonTests.md5Values[2]),
):
object = (yield adbk.addressbookObjectWithName(name))
self.assertEquals(object.md5(), md5)
@inlineCallbacks
def test_upgradeProperties(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home_defaults"))
cal = (yield home.calendarWithName("calendar_1"))
inbox = (yield home.calendarWithName("inbox"))
# Supported components
self.assertEqual(cal.getSupportedComponents(), "VEVENT")
self.assertTrue(cal.properties().get(PropertyName.fromElement(caldavxml.SupportedCalendarComponentSet)) is None)
# Resource type removed
self.assertTrue(cal.properties().get(PropertyName.fromElement(element.ResourceType)) is None)
# Ctag removed
self.assertTrue(cal.properties().get(PropertyName.fromElement(customxml.GETCTag)) is None)
# Availability
self.assertEquals(str(home.getAvailability()), str(self.av1))
self.assertTrue(inbox.properties().get(PropertyName.fromElement(customxml.CalendarAvailability)) is None)
# Default calendar
self.assertTrue(home.isDefaultCalendar(cal))
self.assertTrue(inbox.properties().get(PropertyName.fromElement(caldavxml.ScheduleDefaultCalendarURL)) is None)
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
class HomeMigrationTests(TestCase):
"""
Tests for L{UpgradeToDatabaseStep}.
"""
@inlineCallbacks
def setUp(self):
"""
Set up two stores to migrate between.
"""
# Add some files to the file store.
self.filesPath = CachingFilePath(self.mktemp())
self.filesPath.createDirectory()
fileStore = self.fileStore = CommonDataStore(
self.filesPath, {"push": StubNotifierFactory()}, TestStoreDirectoryService(), True, True
)
self.sqlStore = yield theStoreBuilder.buildStore(
self, StubNotifierFactory()
)
self.upgrader = UpgradeToDatabaseStep(self.fileStore, self.sqlStore)
requirements = CommonTests.requirements
extras = deriveValue(self, "extraRequirements", lambda t: {})
requirements = self.mergeRequirements(requirements, extras)
yield populateCalendarsFrom(requirements, fileStore)
md5s = CommonTests.md5s
yield resetCalendarMD5s(md5s, fileStore)
self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
requirements = ABCommonTests.requirements
yield populateAddressBooksFrom(requirements, fileStore)
md5s = ABCommonTests.md5s
yield resetAddressBookMD5s(md5s, fileStore)
self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").child(
".some-extra-data").setContent("some extra data")
def mergeRequirements(self, a, b):
"""
Merge two requirements dictionaries together, modifying C{a} and
returning it.
@param a: Some requirements, in the format of
L{CommonTests.requirements}.
@type a: C{dict}
@param b: Some additional requirements, to be merged into C{a}.
@type b: C{dict}
@return: C{a}
@rtype: C{dict}
"""
for homeUID in b:
homereq = a.setdefault(homeUID, {})
homeExtras = b[homeUID]
for calendarUID in homeExtras:
calreq = homereq.setdefault(calendarUID, {})
calendarExtras = homeExtras[calendarUID]
calreq.update(calendarExtras)
return a
@withSpecialValue(
"extraRequirements",
{
"home1": {
"calendar_1": {
"bogus.ics": (
getModule("twistedcaldav").filePath.sibling("zoneinfo")
.child("EST.ics").getContent(),
CommonTests.metadata1
)
}
}
}
)
@inlineCallbacks
def test_unknownTypeNotMigrated(self):
"""
The only types of calendar objects that should get migrated are VEVENTs
and VTODOs. Other component types, such as free-standing VTIMEZONEs,
don't have a UID and can't be stored properly in the database, so they
should not be migrated.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
self.assertIdentical(
None,
(yield (yield (yield
(yield txn.calendarHomeWithUID("home1"))
.calendarWithName("calendar_1")))
.calendarObjectWithName("bogus.ics"))
)
@inlineCallbacks
def test_upgradeCalendarHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in CommonTests.requirements:
if CommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.calendarHomeWithUID(uid))
)
# Successfully migrated calendar homes are deleted
self.assertFalse(self.filesPath.child("calendars").child(
"__uids__").child("ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
for name, metadata, md5 in (
("1.ics", CommonTests.metadata1, CommonTests.md5Values[0]),
("2.ics", CommonTests.metadata2, CommonTests.md5Values[1]),
("3.ics", CommonTests.metadata3, CommonTests.md5Values[2]),
):
object = (yield calendar.calendarObjectWithName(name))
self.assertEquals(object.getMetadata(), metadata)
self.assertEquals(object.md5(), md5)
@inlineCallbacks
def test_upgradeExistingHome(self):
"""
L{UpgradeToDatabaseService.startService} will skip migrating existing
homes.
"""
startTxn = self.sqlStore.newTransaction("populate empty sample")
yield startTxn.calendarHomeWithUID("home1", create=True)
yield startTxn.commit()
yield self.upgrader.stepWithResult(None)
vrfyTxn = self.sqlStore.newTransaction("verify sample still empty")
self.addCleanup(vrfyTxn.commit)
home = yield vrfyTxn.calendarHomeWithUID("home1")
# The default calendar is still there.
self.assertNotIdentical(None, (yield home.calendarWithName("calendar")))
# The migrated calendar isn't.
self.assertIdentical(None, (yield home.calendarWithName("calendar_1")))
@inlineCallbacks
def test_upgradeAttachments(self):
"""
L{UpgradeToDatabaseService.startService} upgrades calendar attachments
as well.
"""
# Need to tweak config and settings to setup dropbox to work
self.patch(config, "EnableDropBox", True)
self.patch(config, "EnableManagedAttachments", False)
self.sqlStore.enableManagedAttachments = False
txn = self.sqlStore.newTransaction()
cs = schema.CALENDARSERVER
yield Delete(
From=cs,
Where=cs.NAME == "MANAGED-ATTACHMENTS"
).on(txn)
yield txn.commit()
txn = self.fileStore.newTransaction()
committed = []
def maybeCommit():
if not committed:
committed.append(True)
return txn.commit()
self.addCleanup(maybeCommit)
@inlineCallbacks
def getSampleObj():
home = (yield txn.calendarHomeWithUID("home1"))
calendar = (yield home.calendarWithName("calendar_1"))
object = (yield calendar.calendarObjectWithName("1.ics"))
returnValue(object)
inObject = yield getSampleObj()
someAttachmentName = "some-attachment"
someAttachmentType = MimeType.fromString("application/x-custom-type")
attachment = yield inObject.createAttachmentWithName(
someAttachmentName,
)
transport = attachment.store(someAttachmentType)
someAttachmentData = "Here is some data for your attachment, enjoy."
transport.write(someAttachmentData)
yield transport.loseConnection()
yield maybeCommit()
yield self.upgrader.stepWithResult(None)
committed = []
txn = self.sqlStore.newTransaction()
outObject = yield getSampleObj()
outAttachment = yield outObject.attachmentWithName(someAttachmentName)
allDone = Deferred()
class SimpleProto(Protocol):
data = ''
def dataReceived(self, data):
self.data += data
def connectionLost(self, reason):
allDone.callback(self.data)
self.assertEquals(outAttachment.contentType(), someAttachmentType)
outAttachment.retrieve(SimpleProto())
allData = yield allDone
self.assertEquals(allData, someAttachmentData)
@inlineCallbacks
def test_upgradeAddressBookHomes(self):
"""
L{UpgradeToDatabaseService.startService} will do the upgrade, then
start its dependent service by adding it to its service hierarchy.
"""
yield self.upgrader.stepWithResult(None)
txn = self.sqlStore.newTransaction()
self.addCleanup(txn.commit)
for uid in ABCommonTests.requirements:
if ABCommonTests.requirements[uid] is not None:
self.assertNotIdentical(
None, (yield txn.addressbookHomeWithUID(uid))
)
# Successfully migrated addressbook homes are deleted
self.assertFalse(self.filesPath.child("addressbooks").child(
"__uids__").child("ho").child("me").child("home1").exists())
# Want metadata preserved
home = (yield txn.addressbookHomeWithUID("home1"))
adbk = (yield home.addressbookWithName("addressbook"))
for name, md5 in (
("1.vcf", ABCommonTests.md5Values[0]),
("2.vcf", ABCommonTests.md5Values[1]),
("3.vcf", ABCommonTests.md5Values[2]),
):
object = (yield adbk.addressbookObjectWithName(name))
self.assertEquals(object.md5(), md5)
def test_fileStoreFromPath(self):
"""
Verify that fileStoreFromPath() will return a CommonDataStore if
the given path contains either "calendars" or "addressbooks"
sub-directories. Otherwise it returns None
"""
# No child directories
docRootPath = CachingFilePath(self.mktemp())
docRootPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertEquals(step, None)
# "calendars" child directory exists
childPath = docRootPath.child("calendars")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()
# "addressbooks" child directory exists
childPath = docRootPath.child("addressbooks")
childPath.createDirectory()
step = UpgradeToDatabaseStep.fileStoreFromPath(docRootPath)
self.assertTrue(isinstance(step, CommonDataStore))
childPath.remove()