def test_cleanup_bad_config2(self): self.createMasterCfg(extraconfig="++++ # syntaxerror") res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) self.assertEqual(res, 1) self.assertInStdout("error while parsing config") # config logs an error via log.err, we must eat it or trial will complain self.flushLoggedErrors()
def test_cleanup(self): # test may use mysql or pg if configured in env if "BUILDBOT_TEST_DB_URL" not in os.environ: patch_environ( self, "BUILDBOT_TEST_DB_URL", "sqlite:///" + os.path.join(self.origcwd, "basedir", "state.sqlite")) # we reuse RealDatabaseMixin to setup the db yield self.setUpRealDatabase(table_names=[ 'logs', 'logchunks', 'steps', 'builds', 'builders', 'masters', 'buildrequests', 'buildsets', 'workers' ]) master = fakemaster.make_master() master.config.db['db_url'] = self.db_url self.db = DBConnector(self.basedir) self.db.setServiceParent(master) self.db.pool = self.db_pool # we reuse the fake db background data from db.logs unit tests yield self.insertTestData(test_db_logs.Tests.backgroundData) # insert a log with lots of redundancy LOGDATA = "xx\n" * 2000 logid = yield self.db.logs.addLog(102, "x", "x", "s") yield self.db.logs.appendLog(logid, LOGDATA) # test all methods lengths = {} for mode in self.db.logs.COMPRESSION_MODE.keys(): if mode == "lz4" and not hasLz4: # ok.. lz4 is not installed, dont fail lengths["lz4"] = 40 continue # create a master.cfg with different compression method self.createMasterCfg("c['logCompressionMethod'] = '%s'" % (mode, )) res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) self.assertEqual(res, 0) # make sure the compression don't change the data we can retrieve # via api res = yield self.db.logs.getLogLines(logid, 0, 2000) self.assertEqual(res, LOGDATA) # retrieve the actual data size in db using raw sqlalchemy def thd(conn): tbl = self.db.model.logchunks q = sa.select([tbl.c.content]) q = q.where(tbl.c.logid == logid) return sum([len(row.content) for row in conn.execute(q)]) lengths[mode] = yield self.db.pool.do(thd) self.assertDictAlmostEqual(lengths, { 'raw': 5999, 'bz2': 44, 'lz4': 40, 'gz': 31 })
def test_cleanup(self): # test may use mysql or pg if configured in env if "BUILDBOT_TEST_DB_URL" not in os.environ: patch_environ(self, "BUILDBOT_TEST_DB_URL", "sqlite:///" + os.path.join(self.origcwd, "basedir", "state.sqlite")) # we reuse RealDatabaseMixin to setup the db yield self.setUpRealDatabase(table_names=['logs', 'logchunks', 'steps', 'builds', 'builders', 'masters', 'buildrequests', 'buildsets', 'workers']) master = fakemaster.make_master() master.config.db['db_url'] = self.db_url self.db = DBConnector(self.basedir) self.db.setServiceParent(master) self.db.pool = self.db_pool # we reuse the fake db background data from db.logs unit tests yield self.insertTestData(test_db_logs.Tests.backgroundData) # insert a log with lots of redundancy LOGDATA = "xx\n" * 2000 logid = yield self.db.logs.addLog(102, "x", "x", "s") yield self.db.logs.appendLog(logid, LOGDATA) # test all methods lengths = {} for mode in self.db.logs.COMPRESSION_MODE.keys(): if mode == "lz4" and not hasLz4: # ok.. lz4 is not installed, dont fail lengths["lz4"] = 40 continue # create a master.cfg with different compression method self.createMasterCfg("c['logCompressionMethod'] = '%s'" % (mode,)) res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) self.assertEqual(res, 0) # make sure the compression don't change the data we can retrieve # via api res = yield self.db.logs.getLogLines(logid, 0, 2000) self.assertEqual(res, LOGDATA) # retrieve the actual data size in db using raw sqlalchemy def thd(conn): tbl = self.db.model.logchunks q = sa.select([tbl.c.content]) q = q.where(tbl.c.logid == logid) return sum([len(row.content) for row in conn.execute(q)]) lengths[mode] = yield self.db.pool.do(thd) self.assertDictAlmostEqual( lengths, {'raw': 5999, 'bz2': 44, 'lz4': 40, 'gz': 31})
def test_cleanup_bad_config2(self): # test may use mysql or pg if configured in env if "BUILDBOT_TEST_DB_URL" not in os.environ: patch_environ(self, "BUILDBOT_TEST_DB_URL", "sqlite:///" + os.path.join(self.origcwd, "basedir", "state.sqlite")) self.createMasterCfg(extraconfig="++++ # syntaxerror") res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) self.assertEqual(res, 1) self.assertInStdout("encountered a SyntaxError while parsing config file:") # config logs an error via log.err, we must eat it or trial will complain self.flushLoggedErrors()
def test_cleanup_bad_config2(self): # test may use mysql or pg if configured in env if "BUILDBOT_TEST_DB_URL" not in os.environ: patch_environ(self, "BUILDBOT_TEST_DB_URL", "sqlite:///" + os.path.join(self.origcwd, "basedir", "state.sqlite")) self.createMasterCfg(extraconfig="++++ # syntaxerror") res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) self.assertEqual(res, 1) self.assertInStdout( "encountered a SyntaxError while parsing config file:") # config logs an error via log.err, we must eat it or trial will # complain self.flushLoggedErrors()
def test_cleanup(self): # we reuse the fake db background data from db.logs unit tests yield self.insertTestData(test_logs.Tests.backgroundData) # insert a log with lots of redundancy LOGDATA = "xx\n" * 2000 logid = yield self.master.db.logs.addLog(102, "x", "x", "s") yield self.master.db.logs.appendLog(logid, LOGDATA) # test all methods lengths = {} for mode in self.master.db.logs.COMPRESSION_MODE: if mode == "lz4" and not hasLz4: # ok.. lz4 is not installed, don't fail lengths["lz4"] = 40 continue # create a master.cfg with different compression method self.createMasterCfg( "c['logCompressionMethod'] = '{}'".format(mode)) res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) self.assertEqual(res, 0) # make sure the compression don't change the data we can retrieve # via api res = yield self.master.db.logs.getLogLines(logid, 0, 2000) self.assertEqual(res, LOGDATA) # retrieve the actual data size in db using raw sqlalchemy def thd(conn): tbl = self.master.db.model.logchunks q = sa.select([sa.func.sum(sa.func.length(tbl.c.content))]) q = q.where(tbl.c.logid == logid) return conn.execute(q).fetchone()[0] lengths[mode] = yield self.master.db.pool.do(thd) self.assertDictAlmostEqual(lengths, { 'raw': 5999, 'bz2': 44, 'lz4': 40, 'gz': 31 })
def test_cleanup_bad_config(self): res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='basedir')) self.assertEqual(res, 1) self.assertInStdout("master.cfg' does not exist")
def test_cleanup_not_basedir(self): res = yield cleanupdb._cleanupDatabase(mkconfig(basedir='doesntexist')) self.assertEqual(res, 1) self.assertInStdout('invalid buildmaster directory')