def setup(self, check_version=True, verbose=True):
db_url = self.configured_url = self.master.config.db['db_url']
log.msg("Setting up database with URL %r"
% util.stripUrlPassword(db_url))
# set up the engine and pool
self._engine = enginestrategy.create_engine(db_url,
basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine, verbose=verbose)
# make sure the db is up to date, unless specifically asked not to
if check_version:
d = self.model.is_current()
@d.addCallback
def check_current(res):
if not res:
for l in upgrade_message.format(basedir=self.master.basedir).split('\n'):
log.msg(l)
raise exceptions.DatabaseNotReadyError()
else:
d = defer.succeed(None)
return d
def __init__(self, db_url, basedir):
self.basedir = basedir
"basedir for this master - used for upgrades"
self._engine = enginestrategy.create_engine(db_url, basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine)
"thread pool (L{buildbot.db.pool.DBThreadPool}) for this db"
self._oldpool = TempAdbapiPool(self._engine)
self._query_times = collections.deque()
self._change_cache = util.LRUCache() # TODO: remove
self._sourcestamp_cache = util.LRUCache()
self._active_operations = set() # protected by synchronized=
self._pending_notifications = []
self._subscribers = bbcollections.defaultdict(set)
self._pending_operation_count = 0
self._started = False
# set up components
self.model = model.Model(self)
"L{buildbot.db.model.Model} instance"
self.changes = changes.ChangesConnectorComponent(self)
"L{buildbot.db.changes.ChangesConnectorComponent} instance"
def __init__(self, master, db_url, basedir):
service.MultiService.__init__(self)
self.master = master
self.basedir = basedir
self._engine = enginestrategy.create_engine(db_url,
basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine)
# set up components
self.model = model.Model(self)
self.changes = changes.ChangesConnectorComponent(self)
self.schedulers = schedulers.SchedulersConnectorComponent(self)
self.sourcestamps = sourcestamps.SourceStampsConnectorComponent(self)
self.buildsets = buildsets.BuildsetsConnectorComponent(self)
self.buildrequests = buildrequests.BuildRequestsConnectorComponent(
self)
self.state = state.StateConnectorComponent(self)
self.builds = builds.BuildsConnectorComponent(self)
self.cleanup_timer = internet.TimerService(self.CLEANUP_PERIOD,
self.doCleanup)
self.cleanup_timer.setServiceParent(self)
self.changeHorizon = None # default value; set by master
def setup(self, check_version=True, verbose=True):
db_url = self.configured_url = self.master.config.db['db_url']
log.msg("Setting up database with URL %r" % (db_url, ))
# set up the engine and pool
self._engine = enginestrategy.create_engine(db_url,
basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine, verbose=verbose)
# make sure the db is up to date, unless specifically asked not to
if check_version:
d = self.model.is_current()
def check_current(res):
if not res:
for l in upgrade_message.split('\n'):
log.msg(l)
raise DatabaseNotReadyError()
d.addCallback(check_current)
else:
d = defer.succeed(None)
return d
def setup(self, check_version=True, verbose=True):
db_url = self.configured_url = self.master.config.db['db_url']
log.msg("Setting up database with URL %r" %
util.stripUrlPassword(db_url))
# set up the engine and pool
self._engine = enginestrategy.create_engine(db_url,
basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine,
reactor=self.master.reactor,
verbose=verbose)
# make sure the db is up to date, unless specifically asked not to
if check_version:
if db_url == 'sqlite://':
# Using in-memory database. Since it is reset after each process
# restart, `buildbot upgrade-master` cannot be used (data is not
# persistent). Upgrade model here to allow startup to continue.
self.model.upgrade()
current = yield self.model.is_current()
if not current:
for l in upgrade_message.format(
basedir=self.master.basedir).split('\n'):
log.msg(l)
raise exceptions.DatabaseNotReadyError()
def setUpRealDatabase(self, basedir="basedir", want_pool=True):
"""
Set up a database. Ordinarily sets up an engine and a pool and takes
care of cleaning out any existing tables in the database. If
C{want_pool} is false, then no pool will be created, and the database
will not be cleaned.
@param basedir: (optional) basedir for the engine
@param want_pool: false to not create C{self.db_pool}
@returns: Deferred
"""
self.__want_pool = want_pool
memory = "sqlite://"
self.db_url = os.environ.get(
"BUILDBOT_TEST_DB_URL",
### XXX TEMPORARY until sqlalchemification is complete
"sqlite:///%s" % (os.path.abspath("test.db")),
)
self.__using_memory_db = self.db_url == memory
self.db_engine = enginestrategy.create_engine(self.db_url, basedir=basedir)
# if the caller does not want a pool, we're done.
if not want_pool:
return defer.succeed(None)
self.db_pool = pool.DBThreadPool(self.db_engine)
log.msg("cleaning database %s" % self.db_url)
return self.db_pool.do_with_engine(self.__thd_clean_database)
def setUpRealDatabase(self, table_names=[], basedir='basedir',
want_pool=True):
"""
Set up a database. Ordinarily sets up an engine and a pool and takes
care of cleaning out any existing tables in the database. If
C{want_pool} is false, then no pool will be created, and the database
will not be cleaned.
@param table_names: list of names of tables to instantiate
@param basedir: (optional) basedir for the engine
@param want_pool: (optional) false to not create C{self.db_pool}
@returns: Deferred
"""
self.__want_pool = want_pool
memory = 'sqlite://'
self.db_url = os.environ.get('BUILDBOT_TEST_DB_URL', memory)
self.__using_memory_db = (self.db_url == memory)
self.db_engine = enginestrategy.create_engine(self.db_url,
basedir=basedir)
# if the caller does not want a pool, we're done.
if not want_pool:
return defer.succeed(None)
self.db_pool = pool.DBThreadPool(self.db_engine)
log.msg("cleaning database %s" % self.db_url)
d = self.db_pool.do(self.__thd_clean_database)
d.addCallback(lambda _ :
self.db_pool.do(self.__thd_create_tables, table_names))
return d
def make_fake_pool(_):
engine = enginestrategy.create_engine(self.db_url, basedir=os.path.abspath("basedir"))
# mock out the pool, and set up the model
self.db = mock.Mock()
self.db.pool.do_with_engine = lambda thd: defer.maybeDeferred(thd, engine)
self.db.model = model.Model(self.db)
self.db.start()
def make_fake_pool(_):
engine = enginestrategy.create_engine(self.db_url,
basedir=os.path.abspath('basedir'))
# mock out the pool, and set up the model
self.db = mock.Mock()
self.db.pool.do_with_engine = lambda thd: defer.maybeDeferred(thd, engine)
self.db.model = model.Model(self.db)
self.db.start()
def setUp(self):
self.setUpRealDatabase()
engine = enginestrategy.create_engine(self.db_url,
basedir=os.path.abspath('basedir'))
# mock out the pool, and set up the model
self.db = mock.Mock()
self.db.pool.do_with_engine = lambda thd : defer.maybeDeferred(thd,engine)
self.db.model = model.Model(self.db)
self.db.start()
def setUp(self):
yield self.setUpRealDatabase()
engine = enginestrategy.create_engine(
self.db_url, basedir=os.path.abspath('basedir'))
# mock out the pool, and set up the model
self.db = mock.Mock()
self.db.pool.do_with_engine = lambda thd: defer.maybeDeferred(
thd, engine)
self.db.model = model.Model(self.db)
self.db.start()
def __init__(self, master, db_url, basedir):
service.MultiService.__init__(self)
self.master = master
self.basedir = basedir
"basedir for this master - used for upgrades"
self._engine = enginestrategy.create_engine(db_url,
basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine)
"thread pool (L{buildbot.db.pool.DBThreadPool}) for this db"
self._oldpool = TempAdbapiPool(self._engine)
self._sourcestamp_cache = util.LRUCache() # TODO: remove
self._active_operations = set(
) # protected by synchronized= TODO: remove
self._pending_notifications = [] # TODO: remove
self._subscribers = bbcollections.defaultdict(set)
self._started = False
# set up components
self.model = model.Model(self)
"L{buildbot.db.model.Model} instance"
self.changes = changes.ChangesConnectorComponent(self)
"L{buildbot.db.changes.ChangesConnectorComponent} instance"
self.schedulers = schedulers.SchedulersConnectorComponent(self)
"L{buildbot.db.schedulers.ChangesConnectorComponent} instance"
self.sourcestamps = sourcestamps.SourceStampsConnectorComponent(self)
"L{buildbot.db.sourcestamps.SourceStampsConnectorComponent} instance"
self.buildsets = buildsets.BuildsetsConnectorComponent(self)
"L{buildbot.db.sourcestamps.BuildsetsConnectorComponent} instance"
self.buildrequests = buildrequests.BuildRequestsConnectorComponent(
self)
"L{buildbot.db.sourcestamps.BuildRequestsConnectorComponent} instance"
self.state = state.StateConnectorComponent(self)
"L{buildbot.db.state.StateConnectorComponent} instance"
self.cleanup_timer = internet.TimerService(self.CLEANUP_PERIOD,
self.doCleanup)
self.cleanup_timer.setServiceParent(self)
self.changeHorizon = None # default value; set by master
def setup(self, check_version=True, verbose=True):
db_url = self.configured_url = self.master.config.db["db_url"]
log.msg("Setting up database with URL %r" % util.stripUrlPassword(db_url))
# set up the engine and pool
self._engine = enginestrategy.create_engine(db_url, basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine, verbose=verbose)
# make sure the db is up to date, unless specifically asked not to
if check_version:
current = yield self.model.is_current()
if not current:
for l in upgrade_message.format(basedir=self.master.basedir).split("\n"):
log.msg(l)
raise exceptions.DatabaseNotReadyError()
def __init__(self, master, db_url, basedir):
service.MultiService.__init__(self)
self.master = master
self.basedir = basedir
"basedir for this master - used for upgrades"
self._engine = enginestrategy.create_engine(db_url, basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine)
"thread pool (L{buildbot.db.pool.DBThreadPool}) for this db"
self._oldpool = TempAdbapiPool(self._engine)
self._sourcestamp_cache = util.LRUCache() # TODO: remove
self._active_operations = set() # protected by synchronized= TODO: remove
self._pending_notifications = [] # TODO: remove
self._subscribers = bbcollections.defaultdict(set)
self._started = False
# set up components
self.model = model.Model(self)
"L{buildbot.db.model.Model} instance"
self.changes = changes.ChangesConnectorComponent(self)
"L{buildbot.db.changes.ChangesConnectorComponent} instance"
self.schedulers = schedulers.SchedulersConnectorComponent(self)
"L{buildbot.db.schedulers.ChangesConnectorComponent} instance"
self.sourcestamps = sourcestamps.SourceStampsConnectorComponent(self)
"L{buildbot.db.sourcestamps.SourceStampsConnectorComponent} instance"
self.buildsets = buildsets.BuildsetsConnectorComponent(self)
"L{buildbot.db.sourcestamps.BuildsetsConnectorComponent} instance"
self.buildrequests = buildrequests.BuildRequestsConnectorComponent(self)
"L{buildbot.db.sourcestamps.BuildRequestsConnectorComponent} instance"
self.state = state.StateConnectorComponent(self)
"L{buildbot.db.state.StateConnectorComponent} instance"
self.cleanup_timer = internet.TimerService(self.CLEANUP_PERIOD, self.doCleanup)
self.cleanup_timer.setServiceParent(self)
self.changeHorizon = None # default value; set by master
def setup(self, check_version=True, verbose=True):
db_url = self.configured_url = self.master.config.db['db_url']
log.msg("Setting up database with URL %r"
% util.stripUrlPassword(db_url))
# set up the engine and pool
self._engine = enginestrategy.create_engine(db_url,
basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine, verbose=verbose)
# make sure the db is up to date, unless specifically asked not to
if check_version:
current = yield self.model.is_current()
if not current:
for l in upgrade_message.format(basedir=self.master.basedir).split('\n'):
log.msg(l)
raise exceptions.DatabaseNotReadyError()
def setUpRealDatabase(self,
table_names=None,
basedir='basedir',
want_pool=True,
sqlite_memory=True):
"""
Set up a database. Ordinarily sets up an engine and a pool and takes
care of cleaning out any existing tables in the database. If
C{want_pool} is false, then no pool will be created, and the database
will not be cleaned.
@param table_names: list of names of tables to instantiate
@param basedir: (optional) basedir for the engine
@param want_pool: (optional) false to not create C{self.db_pool}
@param sqlite_memory: (optional) False to avoid using an in-memory db
@returns: Deferred
"""
if table_names is None:
table_names = []
self.__want_pool = want_pool
default_sqlite = 'sqlite://'
self.db_url = os.environ.get('BUILDBOT_TEST_DB_URL', default_sqlite)
if not sqlite_memory and self.db_url == default_sqlite:
self.db_url = "sqlite:///tmp.sqlite"
if not os.path.exists(basedir):
os.makedirs(basedir)
self.basedir = basedir
self.db_engine = enginestrategy.create_engine(self.db_url,
basedir=basedir)
# if the caller does not want a pool, we're done.
if not want_pool:
return defer.succeed(None)
self.db_pool = pool.DBThreadPool(self.db_engine, reactor=reactor)
log.msg("cleaning database %s" % self.db_url)
d = self.db_pool.do(self.__thd_clean_database)
d.addCallback(
lambda _: self.db_pool.do(self.__thd_create_tables, table_names))
return d
def setUpRealDatabase(self, table_names=None, basedir='basedir',
want_pool=True, sqlite_memory=True):
"""
Set up a database. Ordinarily sets up an engine and a pool and takes
care of cleaning out any existing tables in the database. If
C{want_pool} is false, then no pool will be created, and the database
will not be cleaned.
@param table_names: list of names of tables to instantiate
@param basedir: (optional) basedir for the engine
@param want_pool: (optional) false to not create C{self.db_pool}
@param sqlite_memory: (optional) False to avoid using an in-memory db
@returns: Deferred
"""
if table_names is None:
table_names = []
self.__want_pool = want_pool
default_sqlite = 'sqlite://'
self.db_url = os.environ.get('BUILDBOT_TEST_DB_URL', default_sqlite)
if not sqlite_memory and self.db_url == default_sqlite:
self.db_url = "sqlite:///tmp.sqlite"
if not os.path.exists(basedir):
os.makedirs(basedir)
self.basedir = basedir
self.db_engine = enginestrategy.create_engine(self.db_url,
basedir=basedir)
# if the caller does not want a pool, we're done.
if not want_pool:
return defer.succeed(None)
self.db_pool = pool.DBThreadPool(self.db_engine, reactor=reactor)
log.msg("cleaning database %s" % self.db_url)
d = self.db_pool.do(self.__thd_clean_database)
d.addCallback(lambda _:
self.db_pool.do(self.__thd_create_tables, table_names))
return d
def getDb(self):
try:
db_engine = enginestrategy.create_engine(self.db_url,
basedir=self.basedir)
except Exception:
# db_url is probably trash. Just ignore, config.py db part will create proper message
return None
db = FakeDBConnector()
db.master = FakeMaster()
db.pool = FakePool()
db.pool.engine = db_engine
db.master.caches = FakeCacheManager()
db.model = model.Model(db)
db.state = state.StateConnectorComponent(db)
try:
self.objectid = db.state.thdGetObjectId(db_engine, self.name, "DbConfig")['id']
except (ProgrammingError, OperationalError):
# ProgrammingError: mysql&pg, OperationalError: sqlite
# assume db is not initialized
db.pool.engine.close()
return None
return db
def __init__(self, master, db_url, basedir):
service.MultiService.__init__(self)
self.master = master
self.basedir = basedir
self._engine = enginestrategy.create_engine(db_url, basedir=self.basedir)
self.pool = pool.DBThreadPool(self._engine)
# set up components
self.model = model.Model(self)
self.changes = changes.ChangesConnectorComponent(self)
self.schedulers = schedulers.SchedulersConnectorComponent(self)
self.sourcestamps = sourcestamps.SourceStampsConnectorComponent(self)
self.buildsets = buildsets.BuildsetsConnectorComponent(self)
self.buildrequests = buildrequests.BuildRequestsConnectorComponent(self)
self.state = state.StateConnectorComponent(self)
self.builds = builds.BuildsConnectorComponent(self)
self.cleanup_timer = internet.TimerService(self.CLEANUP_PERIOD, self.doCleanup)
self.cleanup_timer.setServiceParent(self)
self.changeHorizon = None # default value; set by master
def setUpRealDatabase(self,
table_names=[],
basedir='basedir',
want_pool=True):
"""
Set up a database. Ordinarily sets up an engine and a pool and takes
care of cleaning out any existing tables in the database. If
C{want_pool} is false, then no pool will be created, and the database
will not be cleaned.
@param table_names: list of names of tables to instantiate
@param basedir: (optional) basedir for the engine
@param want_pool: (optional) false to not create C{self.db_pool}
@returns: Deferred
"""
self.__want_pool = want_pool
memory = 'sqlite://'
self.db_url = os.environ.get(
'BUILDBOT_TEST_DB_URL',
### XXX TEMPORARY until sqlalchemification is complete
'sqlite:///%s' % (os.path.abspath('test.db')))
self.__using_memory_db = (self.db_url == memory)
self.db_engine = enginestrategy.create_engine(self.db_url,
basedir=basedir)
# if the caller does not want a pool, we're done.
if not want_pool:
return defer.succeed(None)
self.db_pool = pool.DBThreadPool(self.db_engine)
log.msg("cleaning database %s" % self.db_url)
d = self.db_pool.do(self.__thd_clean_database)
d.addCallback(
lambda _: self.db_pool.do(self.__thd_create_tables, table_names))
return d
def getDb(self):
try:
db_engine = enginestrategy.create_engine(self.db_url,
basedir=self.basedir)
except Exception:
# db_url is probably trash. Just ignore, config.py db part will create proper message
return None
db = FakeDBConnector()
db.master = FakeMaster()
db.pool = FakePool()
db.pool.engine = db_engine
db.master.caches = FakeCacheManager()
db.model = model.Model(db)
db.state = state.StateConnectorComponent(db)
try:
self.objectid = db.state.thdGetObjectId(db_engine, self.name,
"DbConfig")['id']
except (ProgrammingError, OperationalError):
# ProgrammingError: mysql&pg, OperationalError: sqlite
# assume db is not initialized
db.pool.engine.close()
return None
return db
def setup(self, check_version=True, verbose=True):
db_url = self.configured_url = self.master.config.db['db_url']
log.msg("Setting up database with URL %r"
% util.stripUrlPassword(db_url))
# set up the engine and pool
self._engine = enginestrategy.create_engine(db_url,
basedir=self.basedir)
self.pool = pool.DBThreadPool(
self._engine, reactor=self.master.reactor, verbose=verbose)
# make sure the db is up to date, unless specifically asked not to
if check_version:
if db_url == 'sqlite://':
# Using in-memory database. Since it is reset after each process
# restart, `buildbot upgrade-master` cannot be used (data is not
# persistent). Upgrade model here to allow startup to continue.
self.model.upgrade()
current = yield self.model.is_current()
if not current:
for l in upgrade_message.format(basedir=self.master.basedir).split('\n'):
log.msg(l)
raise exceptions.DatabaseNotReadyError()
def test_create_engine(self):
engine = enginestrategy.create_engine('sqlite://', basedir="/base")
self.assertEqual(engine.scalar("SELECT 13 + 14"), 27)
def test_create_engine(self):
engine = enginestrategy.create_engine('sqlite://', basedir="/base")
self.assertEqual(engine.scalar("SELECT 13 + 14"), 27)
def setUpConnectorComponent(self, db_url, basedir='basedir'):
"""Set up C{self.db}, using the given db_url and basedir."""
self.db = mock.Mock()
self.db._engine = enginestrategy.create_engine(db_url, basedir=basedir)
self.db.pool = pool.DBThreadPool(self.db._engine)
self.db.model = model.Model(self.db)
