def test_getLogs(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201,
stepid=101,
name='stdio',
slug='stdio',
complete=0,
num_lines=200,
type='s'),
fakedb.Log(id=202,
stepid=101,
name='dbg.log',
slug='dbg_log',
complete=1,
num_lines=300,
type='t'),
fakedb.Log(id=203,
stepid=102,
name='stdio',
slug='stdio',
complete=0,
num_lines=200,
type='s'),
])
logdicts = yield self.db.logs.getLogs(101)
for logdict in logdicts:
validation.verifyDbDict(self, 'logdict', logdict)
self.assertEqual(sorted([ld['id'] for ld in logdicts]), [201, 202])
def test_getLogBySlug(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201, stepid=101, name='stdio', slug='stdio',
complete=0, num_lines=200, type='s'),
fakedb.Log(id=202, stepid=101, name='dbg.log', slug='dbg_log',
complete=1, num_lines=200, type='s'),
])
logdict = yield self.db.logs.getLogBySlug(101, 'dbg_log')
validation.verifyDbDict(self, 'logdict', logdict)
self.assertEqual(logdict['id'], 202)
def test_getLogBySlug_missing(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201, stepid=101, name='stdio', slug='stdio',
complete=0, num_lines=200, type='s'),
])
logdict = yield self.db.logs.getLogBySlug(102, 'stdio')
self.assertEqual(logdict, None)
def test_getLogLines_empty(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201, stepid=101, name='stdio', slug='stdio',
complete=0, num_lines=200, type='s'),
])
self.assertEqual((yield self.db.logs.getLogLines(201, 9, 99)), '')
self.assertEqual((yield self.db.logs.getLogLines(999, 9, 99)), '')
def do_addLogLines_huge_log(self, NUM_CHUNKS=3000, chunk=('xy' * 70 + '\n') * 3):
if chunk.endswith("\n"):
chunk = chunk[:-1]
linesperchunk = chunk.count("\n") + 1
test_data = [
fakedb.LogChunk(logid=201, first_line=i * linesperchunk,
last_line=i * linesperchunk + linesperchunk - 1, compressed=0,
content=chunk)
for i in range(NUM_CHUNKS)
]
yield self.insertTestData(
self.backgroundData + [
fakedb.Log(id=201, stepid=101, name='stdio', slug='stdio',
complete=0, num_lines=NUM_CHUNKS * 3, type='s')] +
test_data)
wholeLog = yield self.db.logs.getLogLines(201, 0, NUM_CHUNKS * 3)
for i in range(10):
yield self.db.logs.compressLog(201)
wholeLog2 = yield self.db.logs.getLogLines(201, 0, NUM_CHUNKS * 3)
self.assertEqual(wholeLog, wholeLog2)
self.assertEqual(wholeLog, wholeLog2)
def countChunk(conn):
tbl = self.db.model.logchunks
q = sa.select([sa.func.count(tbl.c.content)])
q = q.where(tbl.c.logid == 201)
return conn.execute(q).fetchone()[0]
chunks = yield self.db.pool.do(countChunk)
# make sure MAX_CHUNK_LINES is taken in account
self.assertGreaterEqual(
chunks, NUM_CHUNKS * linesperchunk / logs.LogsConnectorComponent.MAX_CHUNK_LINES)
def test_masterDeactivated(self):
self.master.db.insertTestData([
fakedb.Master(id=14, name='other', active=0,
last_active=0),
# set up a running build with some steps
fakedb.Builder(id=77, name='b1'),
fakedb.Worker(id=13, name='wrk'),
fakedb.Buildset(id=8822),
fakedb.BuildRequest(id=82, builderid=77, buildsetid=8822),
fakedb.BuildRequestClaim(brid=82, masterid=14,
claimed_at=SOMETIME),
fakedb.Build(id=13, builderid=77, masterid=14, workerid=13,
buildrequestid=82, number=3, results=None),
fakedb.Step(id=200, buildid=13),
fakedb.Log(id=2000, stepid=200, num_lines=2),
fakedb.LogChunk(logid=2000, first_line=1, last_line=2,
content='ab\ncd')
])
# mock out the _masterDeactivated methods this will call
for rtype in 'builder', 'scheduler', 'changesource':
rtype_obj = getattr(self.master.data.rtypes, rtype)
m = mock.Mock(name=f'{rtype}._masterDeactivated',
spec=rtype_obj._masterDeactivated)
m.side_effect = lambda masterid: defer.succeed(None)
rtype_obj._masterDeactivated = m
# and the update methods..
for meth in 'finishBuild', 'finishStep', 'finishLog':
m = mock.create_autospec(getattr(self.master.data.updates, meth))
m.side_effect = lambda *args, **kwargs: defer.succeed(None)
setattr(self.master.data.updates, meth, m)
yield self.rtype._masterDeactivated(14, 'other')
self.master.data.rtypes.builder._masterDeactivated. \
assert_called_with(masterid=14)
self.master.data.rtypes.scheduler._masterDeactivated. \
assert_called_with(masterid=14)
self.master.data.rtypes.changesource._masterDeactivated. \
assert_called_with(masterid=14)
# see that we finished off that build and its steps and logs
updates = self.master.data.updates
updates.finishLog.assert_called_with(logid=2000)
updates.finishStep.assert_called_with(
stepid=200, results=RETRY, hidden=False)
updates.finishBuild.assert_called_with(buildid=13, results=RETRY)
self.assertEqual(self.master.mq.productions, [
(('masters', '14', 'stopped'),
dict(masterid=14, name='other', active=False)),
])
def setupDb(self):
self.db = self.master.db
self.db.insertTestData([
fakedb.Master(id=92),
fakedb.Worker(id=13, name='wrk'),
fakedb.Buildset(id=98, results=SUCCESS, reason="testReason1"),
fakedb.Builder(id=80, name='Builder1'),
fakedb.BuildRequest(id=9, buildsetid=97, builderid=80),
fakedb.BuildRequest(id=10, buildsetid=97, builderid=80),
fakedb.BuildRequest(id=11, buildsetid=98, builderid=80),
fakedb.BuildRequest(id=12, buildsetid=98, builderid=80),
fakedb.Build(id=18, number=0, builderid=80, buildrequestid=9, workerid=13,
masterid=92, results=FAILURE),
fakedb.Build(id=19, number=1, builderid=80, buildrequestid=10, workerid=13,
masterid=92, results=RETRY),
fakedb.Build(id=20, number=2, builderid=80, buildrequestid=11, workerid=13,
masterid=92, results=SUCCESS),
fakedb.Build(id=21, number=3, builderid=80, buildrequestid=12, workerid=13,
masterid=92, results=SUCCESS),
fakedb.BuildsetSourceStamp(buildsetid=98, sourcestampid=234),
fakedb.SourceStamp(id=234),
fakedb.Change(changeid=13, branch='trunk', revision='9283', author='me@foo',
repository='svn://...', codebase='cbsvn',
project='world-domination', sourcestampid=234),
fakedb.Patch(id=99, patch_base64='aGVsbG8sIHdvcmxk',
patch_author='him@foo', patch_comment='foo', subdir='/foo',
patchlevel=3),
fakedb.SourceStamp(id=235, patchid=99),
])
for _id in (20, 21):
self.db.insertTestData([
fakedb.BuildProperty(
buildid=_id, name="workername", value="wrk"),
fakedb.BuildProperty(
buildid=_id, name="reason", value="because"),
fakedb.BuildProperty(
buildid=_id, name="owner", value="him"),
fakedb.Step(id=100 + _id, buildid=_id, name="step1"),
fakedb.Step(id=200 + _id, buildid=_id, name="step2"),
fakedb.Log(id=60 + _id, stepid=100 + _id, name='stdio', slug='stdio', type='s',
num_lines=2),
fakedb.LogChunk(logid=60 + _id, first_line=0, last_line=1, compressed=0,
content=self.LOGCONTENT),
])
@defer.inlineCallbacks
def getChangesForBuild(buildid):
assert buildid == 20
ch = yield self.master.db.changes.getChange(13)
return [ch]
self.master.db.changes.getChangesForBuild = getChangesForBuild
def setupBuildResults(self, results, wantPreviousBuild=False):
# this testsuite always goes through setupBuildResults so that
# the data is sure to be the real data schema known coming from data
# api
self.db = self.master.db
self.db.insertTestData([
fakedb.Master(id=92),
fakedb.Worker(id=13, name='wrk'),
fakedb.Buildset(id=98, results=results, reason="testReason1"),
fakedb.Builder(id=80, name='Builder1'),
fakedb.BuildRequest(id=11, buildsetid=98, builderid=80),
fakedb.Build(id=20, number=0, builderid=80, buildrequestid=11, workerid=13,
masterid=92, results=results),
fakedb.Step(id=50, buildid=20, number=5, name='make'),
fakedb.BuildsetSourceStamp(buildsetid=98, sourcestampid=234),
fakedb.SourceStamp(id=234, patchid=99),
fakedb.Change(changeid=13, branch='trunk', revision='9283', author='me@foo',
repository='svn://...', codebase='cbsvn',
project='world-domination', sourcestampid=234),
fakedb.Log(id=60, stepid=50, name='stdio', slug='stdio', type='s',
num_lines=7),
fakedb.LogChunk(logid=60, first_line=0, last_line=1, compressed=0,
content='Unicode log with non-ascii (\u00E5\u00E4\u00F6).'),
fakedb.Patch(id=99, patch_base64='aGVsbG8sIHdvcmxk',
patch_author='him@foo', patch_comment='foo', subdir='/foo',
patchlevel=3),
])
for _id in (20,):
self.db.insertTestData([
fakedb.BuildProperty(
buildid=_id, name="workername", value="wrk"),
fakedb.BuildProperty(
buildid=_id, name="reason", value="because"),
fakedb.BuildProperty(
buildid=_id, name="scheduler", value="checkin"),
fakedb.BuildProperty(
buildid=_id, name="branch", value="master"),
])
res = yield utils.getDetailsForBuildset(self.master, 98, wantProperties=True,
wantPreviousBuild=wantPreviousBuild)
builds = res['builds']
buildset = res['buildset']
@defer.inlineCallbacks
def getChangesForBuild(buildid):
assert buildid == 20
ch = yield self.master.db.changes.getChange(13)
return [ch]
self.master.db.changes.getChangesForBuild = getChangesForBuild
return (buildset, builds)
def setUp(self):
self.setUpEndpoint()
self.db.insertTestData([
fakedb.Builder(id=77, name='builder77'),
fakedb.Master(id=88),
fakedb.Worker(id=13, name='wrk'),
fakedb.Buildset(id=8822),
fakedb.BuildRequest(id=82, buildsetid=8822),
fakedb.Build(id=13,
builderid=77,
masterid=88,
workerid=13,
buildrequestid=82,
number=3),
fakedb.Step(id=50, buildid=13, number=5, name='make'),
fakedb.Log(id=60, stepid=50, name='stdio', slug='stdio', type='s'),
fakedb.Log(id=61,
stepid=50,
name='errors',
slug='errors',
type='t'),
])
def setUp(self):
self.setUpEndpoint()
self.db.insertTestData([
fakedb.Builder(id=77),
fakedb.Master(id=88),
fakedb.Worker(id=13, name='wrk'),
fakedb.Buildset(id=8822),
fakedb.BuildRequest(id=82, buildsetid=8822),
fakedb.Build(id=13,
builderid=77,
masterid=88,
workerid=13,
buildrequestid=82,
number=3),
fakedb.Step(id=50, buildid=13, number=9, name='make'),
fakedb.Log(id=60, stepid=50, name='stdio', type='s'),
fakedb.Log(id=61, stepid=50, name='errors', type='t'),
fakedb.Step(id=51, buildid=13, number=10, name='make_install'),
fakedb.Log(id=70, stepid=51, name='stdio', type='s'),
fakedb.Log(id=71, stepid=51, name='results_html', type='h'),
fakedb.Step(id=52, buildid=13, number=11, name='nothing'),
])
def setUp(self):
self.setUpEndpoint()
self.db.insertTestData([
fakedb.Builder(id=77),
fakedb.Worker(id=13, name='wrk'),
fakedb.Master(id=88),
fakedb.Buildset(id=8822),
fakedb.BuildRequest(id=82, buildsetid=8822),
fakedb.Build(id=13, builderid=77, masterid=88, workerid=13,
buildrequestid=82, number=3),
fakedb.Step(id=50, buildid=13, number=9, name='make'),
fakedb.Log(id=60, stepid=50, name='stdio', slug='stdio', type='s',
num_lines=7),
fakedb.LogChunk(logid=60, first_line=0, last_line=1, compressed=0,
content=textwrap.dedent("""\
line zero
line 1""")),
fakedb.LogChunk(logid=60, first_line=2, last_line=4, compressed=0,
content=textwrap.dedent("""\
line TWO
line 3
line 2**2""")),
fakedb.LogChunk(logid=60, first_line=5, last_line=5, compressed=0,
content="another line"),
fakedb.LogChunk(logid=60, first_line=6, last_line=6, compressed=0,
content="yet another line"),
fakedb.Log(id=61, stepid=50, name='errors', slug='errors',
type='t', num_lines=100),
] + [
fakedb.LogChunk(logid=61, first_line=i, last_line=i, compressed=0,
content=f"{i:08d}")
for i in range(100)
] + [
fakedb.Log(id=62, stepid=50, name='notes', slug='notes', type='t',
num_lines=0),
# logid 62 is empty
])
def test_compressLog_empty_log(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201, stepid=101, name='stdio', slug='stdio',
complete=1, num_lines=0, type='s'),
])
yield self.db.logs.compressLog(201)
logdict = yield self.db.logs.getLog(201)
self.assertEqual(logdict, {
'stepid': 101,
'num_lines': 0,
'name': 'stdio',
'id': 201,
'type': 's',
'slug': 'stdio',
'complete': True})
def test_getLog(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201, stepid=101, name='stdio', slug='stdio',
complete=0, num_lines=200, type='s'),
])
logdict = yield self.db.logs.getLog(201)
validation.verifyDbDict(self, 'logdict', logdict)
self.assertEqual(logdict, {
'id': 201,
'stepid': 101,
'name': 'stdio',
'slug': 'stdio',
'complete': False,
'num_lines': 200,
'type': 's',
})
class Tests(interfaces.InterfaceTests):
TIMESTAMP_STEP101 = 100000
TIMESTAMP_STEP102 = 200000
backgroundData = [
fakedb.Worker(id=47, name='linux'),
fakedb.Buildset(id=20),
fakedb.Builder(id=88, name='b1'),
fakedb.BuildRequest(id=41, buildsetid=20, builderid=88),
fakedb.Master(id=88),
fakedb.Build(id=30,
buildrequestid=41,
number=7,
masterid=88,
builderid=88,
workerid=47),
fakedb.Step(id=101,
buildid=30,
number=1,
name='one',
started_at=TIMESTAMP_STEP101),
fakedb.Step(id=102,
buildid=30,
number=2,
name='two',
started_at=TIMESTAMP_STEP102),
]
testLogLines = [
fakedb.Log(id=201,
stepid=101,
name='stdio',
slug='stdio',
complete=0,
num_lines=7,
type='s'),
fakedb.LogChunk(logid=201,
first_line=0,
last_line=1,
compressed=0,
content=textwrap.dedent("""\
line zero
line 1""" + "x" * 200)),
fakedb.LogChunk(logid=201,
first_line=2,
last_line=4,
compressed=0,
content=textwrap.dedent("""\
line TWO
line 2**2""")),
fakedb.LogChunk(logid=201,
first_line=5,
last_line=5,
compressed=0,
content="another line"),
fakedb.LogChunk(logid=201,
first_line=6,
last_line=6,
compressed=0,
content="yet another line"),
]
bug3101Content = base64.b64decode("""
PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT0
9PT09PT09PT09PT09PT09PT09PT09PT09PT09PQpbU0tJUFBFRF0Kbm90IGEgd2luMz
IgcGxhdGZvcm0KCmJ1aWxkc2xhdmUudGVzdC51bml0LnRlc3RfcnVucHJvY2Vzcy5UZ
XN0UnVuUHJvY2Vzcy50ZXN0UGlwZVN0cmluZwotLS0tLS0tLS0tLS0tLS0tLS0tLS0t
LS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0
tLS0tLS0tClJhbiAyNjcgdGVzdHMgaW4gNS4zNzhzCgpQQVNTRUQgKHNraXBzPTEsIH
N1Y2Nlc3Nlcz0yNjYpCnByb2dyYW0gZmluaXNoZWQgd2l0aCBleGl0IGNvZGUgMAplb
GFwc2VkVGltZT04LjI0NTcwMg==""")
bug3101Rows = [
fakedb.Log(id=1470,
stepid=101,
name='problems',
slug='problems',
complete=1,
num_lines=11,
type='t'),
fakedb.LogChunk(logid=1470,
first_line=0,
last_line=10,
compressed=0,
content=bug3101Content),
]
@defer.inlineCallbacks
def checkTestLogLines(self):
expLines = [
'line zero', 'line 1' + "x" * 200, 'line TWO', '', 'line 2**2',
'another line', 'yet another line'
]
for first_line in range(0, 7):
for last_line in range(first_line, 7):
got_lines = yield self.db.logs.getLogLines(
201, first_line, last_line)
self.assertEqual(
got_lines,
"\n".join(expLines[first_line:last_line + 1] + [""]))
# check overflow
self.assertEqual((yield self.db.logs.getLogLines(201, 5, 20)),
"\n".join(expLines[5:7] + [""]))
# signature tests
def test_signature_getLog(self):
@self.assertArgSpecMatches(self.db.logs.getLog)
def getLog(self, logid):
pass
def test_signature_getLogBySlug(self):
@self.assertArgSpecMatches(self.db.logs.getLogBySlug)
def getLogBySlug(self, stepid, slug):
pass
def test_signature_getLogs(self):
@self.assertArgSpecMatches(self.db.logs.getLogs)
def getLogs(self, stepid=None):
pass
def test_signature_getLogLines(self):
@self.assertArgSpecMatches(self.db.logs.getLogLines)
def getLogLines(self, logid, first_line, last_line):
pass
def test_signature_addLog(self):
@self.assertArgSpecMatches(self.db.logs.addLog)
def addLog(self, stepid, name, slug, type):
pass
def test_signature_appendLog(self):
@self.assertArgSpecMatches(self.db.logs.appendLog)
def appendLog(self, logid, content):
pass
def test_signature_finishLog(self):
@self.assertArgSpecMatches(self.db.logs.finishLog)
def finishLog(self, logid):
pass
def test_signature_compressLog(self):
@self.assertArgSpecMatches(self.db.logs.compressLog)
def compressLog(self, logid, force=False):
pass
def test_signature_deleteOldLogChunks(self):
@self.assertArgSpecMatches(self.db.logs.deleteOldLogChunks)
def deleteOldLogChunks(self, older_than_timestamp):
pass
# method tests
@defer.inlineCallbacks
def test_getLog(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201,
stepid=101,
name='stdio',
slug='stdio',
complete=0,
num_lines=200,
type='s'),
])
logdict = yield self.db.logs.getLog(201)
validation.verifyDbDict(self, 'logdict', logdict)
self.assertEqual(
logdict, {
'id': 201,
'stepid': 101,
'name': 'stdio',
'slug': 'stdio',
'complete': False,
'num_lines': 200,
'type': 's',
})
@defer.inlineCallbacks
def test_getLog_missing(self):
logdict = yield self.db.logs.getLog(201)
self.assertEqual(logdict, None)
@defer.inlineCallbacks
def test_getLogBySlug(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201,
stepid=101,
name='stdio',
slug='stdio',
complete=0,
num_lines=200,
type='s'),
fakedb.Log(id=202,
stepid=101,
name='dbg.log',
slug='dbg_log',
complete=1,
num_lines=200,
type='s'),
])
logdict = yield self.db.logs.getLogBySlug(101, 'dbg_log')
validation.verifyDbDict(self, 'logdict', logdict)
self.assertEqual(logdict['id'], 202)
@defer.inlineCallbacks
def test_getLogBySlug_missing(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201,
stepid=101,
name='stdio',
slug='stdio',
complete=0,
num_lines=200,
type='s'),
])
logdict = yield self.db.logs.getLogBySlug(102, 'stdio')
self.assertEqual(logdict, None)
@defer.inlineCallbacks
def test_getLogs(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201,
stepid=101,
name='stdio',
slug='stdio',
complete=0,
num_lines=200,
type='s'),
fakedb.Log(id=202,
stepid=101,
name='dbg.log',
slug='dbg_log',
complete=1,
num_lines=300,
type='t'),
fakedb.Log(id=203,
stepid=102,
name='stdio',
slug='stdio',
complete=0,
num_lines=200,
type='s'),
])
logdicts = yield self.db.logs.getLogs(101)
for logdict in logdicts:
validation.verifyDbDict(self, 'logdict', logdict)
self.assertEqual(sorted([ld['id'] for ld in logdicts]), [201, 202])
@defer.inlineCallbacks
def test_getLogLines(self):
yield self.insertTestData(self.backgroundData + self.testLogLines)
yield self.checkTestLogLines()
# check line number reversal
self.assertEqual((yield self.db.logs.getLogLines(201, 6, 3)), '')
@defer.inlineCallbacks
def test_getLogLines_empty(self):
yield self.insertTestData(self.backgroundData + [
fakedb.Log(id=201,
stepid=101,
name='stdio',
slug='stdio',
complete=0,
num_lines=200,
type='s'),
])
self.assertEqual((yield self.db.logs.getLogLines(201, 9, 99)), '')
self.assertEqual((yield self.db.logs.getLogLines(999, 9, 99)), '')
@defer.inlineCallbacks
def test_getLogLines_bug3101(self):
# regression test for #3101
content = self.bug3101Content
yield self.insertTestData(self.backgroundData + self.bug3101Rows)
# overall content is the same, with '\n' padding at the end
expected = bytes2unicode(self.bug3101Content + b'\n')
self.assertEqual((yield self.db.logs.getLogLines(1470, 0, 99)),
expected)
# try to fetch just one line
expected = bytes2unicode(content.split(b'\n')[0] + b'\n')
self.assertEqual((yield self.db.logs.getLogLines(1470, 0, 0)),
expected)
@defer.inlineCallbacks
def test_addLog_getLog(self):
yield self.insertTestData(self.backgroundData)
logid = yield self.db.logs.addLog(stepid=101,
name='config.log',
slug='config_log',
type='t')
logdict = yield self.db.logs.getLog(logid)
validation.verifyDbDict(self, 'logdict', logdict)
self.assertEqual(
logdict, {
'id': logid,
'stepid': 101,
'name': 'config.log',
'slug': 'config_log',
'complete': False,
'num_lines': 0,
'type': 't',
})
@defer.inlineCallbacks
def test_appendLog_getLogLines(self):
yield self.insertTestData(self.backgroundData + self.testLogLines)
logid = yield self.db.logs.addLog(stepid=102,
name='another',
slug='another',
type='s')
self.assertEqual((yield self.db.logs.appendLog(logid, 'xyz\n')),
(0, 0))
self.assertEqual((yield self.db.logs.appendLog(201, 'abc\ndef\n')),
(7, 8))
self.assertEqual((yield self.db.logs.appendLog(logid, 'XYZ\n')),
(1, 1))
self.assertEqual((yield self.db.logs.getLogLines(201, 6, 7)),
"yet another line\nabc\n")
self.assertEqual((yield self.db.logs.getLogLines(201, 7, 8)),
"abc\ndef\n")
self.assertEqual((yield self.db.logs.getLogLines(201, 8, 8)), "def\n")
self.assertEqual((yield self.db.logs.getLogLines(logid, 0, 1)),
"xyz\nXYZ\n")
self.assertEqual(
(yield self.db.logs.getLog(logid)), {
'complete': False,
'id': logid,
'name': 'another',
'slug': 'another',
'num_lines': 2,
'stepid': 102,
'type': 's',
})
@defer.inlineCallbacks
def test_compressLog(self):
yield self.insertTestData(self.backgroundData + self.testLogLines)
yield self.db.logs.compressLog(201)
# test log lines should still be readable just the same
yield self.checkTestLogLines()
@defer.inlineCallbacks
def test_addLogLines_big_chunk(self):
yield self.insertTestData(self.backgroundData + self.testLogLines)
self.assertEqual(
(yield self.db.logs.appendLog(201, 'abc\n' * 20000)), # 80k
(7, 20006))
lines = yield self.db.logs.getLogLines(201, 7, 50000)
self.assertEqual(len(lines), 80000)
self.assertEqual(lines, ('abc\n' * 20000))
@defer.inlineCallbacks
def test_addLogLines_big_chunk_big_lines(self):
yield self.insertTestData(self.backgroundData + self.testLogLines)
line = 'x' * 33000 + '\n'
self.assertEqual((yield self.db.logs.appendLog(201, line * 3)),
(7, 9)) # three long lines, all truncated
lines = yield self.db.logs.getLogLines(201, 7, 100)
self.assertEqual(len(lines), 99003)
self.assertEqual(lines, (line * 3))
def insert_initial_data(self):
self.master.db.insertTestData([
fakedb.Master(id=1),
fakedb.Worker(id=1, name='example-worker'),
fakedb.Scheduler(id=1, name='custom', enabled=1),
fakedb.Scheduler(id=2, name='all', enabled=2),
fakedb.Scheduler(id=3, name='force', enabled=3),
fakedb.SchedulerMaster(schedulerid=1, masterid=1),
fakedb.SchedulerMaster(schedulerid=2, masterid=1),
fakedb.SchedulerMaster(schedulerid=3, masterid=1),
fakedb.Builder(id=1, name='runtests1'),
fakedb.Builder(id=2, name='runtests2'),
fakedb.Builder(id=3, name='runtests3'),
fakedb.BuilderMaster(id=1, builderid=1, masterid=1),
fakedb.BuilderMaster(id=2, builderid=2, masterid=1),
fakedb.BuilderMaster(id=3, builderid=3, masterid=1),
fakedb.Tag(id=1, name='tag1'),
fakedb.Tag(id=2, name='tag12'),
fakedb.Tag(id=3, name='tag23'),
fakedb.BuildersTags(id=1, builderid=1, tagid=1),
fakedb.BuildersTags(id=2, builderid=1, tagid=2),
fakedb.BuildersTags(id=3, builderid=2, tagid=2),
fakedb.BuildersTags(id=4, builderid=2, tagid=3),
fakedb.BuildersTags(id=5, builderid=3, tagid=3),
fakedb.Buildset(id=1,
results=SUCCESS,
reason="Force reason 1",
submitted_at=100000,
complete_at=100110,
complete=1),
fakedb.Buildset(id=2,
results=SUCCESS,
reason="Force reason 2",
submitted_at=100200,
complete_at=100330,
complete=1),
fakedb.Buildset(id=3,
results=SUCCESS,
reason="Force reason 3",
submitted_at=100400,
complete_at=100550,
complete=1),
fakedb.BuildsetProperty(buildsetid=1,
property_name='scheduler',
property_value='["custom", "Scheduler"]'),
fakedb.BuildsetProperty(buildsetid=2,
property_name='scheduler',
property_value='["all", "Scheduler"]'),
fakedb.BuildsetProperty(buildsetid=3,
property_name='scheduler',
property_value='["force", "Scheduler"]'),
fakedb.BuildsetProperty(
buildsetid=3,
property_name='owner',
property_value='["*****@*****.**", "Force Build Form"]'),
fakedb.SourceStamp(id=1, branch='master', revision='1234abcd'),
fakedb.Change(changeid=1,
branch='master',
revision='1234abcd',
sourcestampid=1),
fakedb.ChangeProperty(
changeid=1,
property_name="owner",
property_value='["*****@*****.**", "change"]'),
fakedb.ChangeProperty(changeid=1,
property_name="other_prop",
property_value='["value", "change"]'),
fakedb.BuildsetSourceStamp(id=1, buildsetid=1, sourcestampid=1),
fakedb.BuildsetSourceStamp(id=2, buildsetid=2, sourcestampid=1),
fakedb.BuildsetSourceStamp(id=3, buildsetid=3, sourcestampid=1),
fakedb.BuildRequest(id=1,
buildsetid=1,
builderid=1,
results=SUCCESS,
submitted_at=100001,
complete_at=100109,
complete=1),
fakedb.BuildRequest(id=2,
buildsetid=2,
builderid=1,
results=SUCCESS,
submitted_at=100201,
complete_at=100329,
complete=1),
fakedb.BuildRequest(id=3,
buildsetid=3,
builderid=2,
results=SUCCESS,
submitted_at=100401,
complete_at=100549,
complete=1),
fakedb.Build(id=1,
number=1,
buildrequestid=1,
builderid=1,
workerid=1,
masterid=1001,
started_at=100002,
complete_at=100108,
state_string='build successful',
results=SUCCESS),
fakedb.Build(id=2,
number=2,
buildrequestid=2,
builderid=1,
workerid=1,
masterid=1001,
started_at=100202,
complete_at=100328,
state_string='build successful',
results=SUCCESS),
fakedb.Build(id=3,
number=1,
buildrequestid=3,
builderid=2,
workerid=1,
masterid=1001,
started_at=100402,
complete_at=100548,
state_string='build successful',
results=SUCCESS),
fakedb.BuildProperty(buildid=3,
name='reason',
value='"force build"',
source="Force Build Form"),
fakedb.BuildProperty(buildid=3,
name='owner',
value='"*****@*****.**"',
source="Force Build Form"),
fakedb.BuildProperty(buildid=3,
name='scheduler',
value='"force"',
source="Scheduler"),
fakedb.BuildProperty(buildid=3,
name='buildername',
value='"runtests3"',
source="Builder"),
fakedb.BuildProperty(buildid=3,
name='workername',
value='"example-worker"',
source="Worker"),
fakedb.Step(id=1,
number=1,
name='step1',
buildid=1,
started_at=100010,
complete_at=100019,
state_string='step1 done'),
fakedb.Step(id=2,
number=2,
name='step2',
buildid=1,
started_at=100020,
complete_at=100029,
state_string='step2 done'),
fakedb.Step(id=3,
number=3,
name='step3',
buildid=1,
started_at=100030,
complete_at=100039,
state_string='step3 done'),
fakedb.Step(id=11,
number=1,
name='step1',
buildid=2,
started_at=100210,
complete_at=100219,
state_string='step1 done'),
fakedb.Step(id=12,
number=2,
name='step2',
buildid=2,
started_at=100220,
complete_at=100229,
state_string='step2 done'),
fakedb.Step(id=13,
number=3,
name='step3',
buildid=2,
started_at=100230,
complete_at=100239,
state_string='step3 done'),
fakedb.Step(id=21,
number=1,
name='step1',
buildid=3,
started_at=100410,
complete_at=100419,
state_string='step1 done'),
fakedb.Step(id=22,
number=2,
name='step2',
buildid=3,
started_at=100420,
complete_at=100429,
state_string='step2 done'),
fakedb.Step(id=23,
number=3,
name='step3',
buildid=3,
started_at=100430,
complete_at=100439,
state_string='step3 done'),
fakedb.Log(id=1,
name='stdio',
slug='stdio',
stepid=1,
complete=1,
num_lines=10),
fakedb.Log(id=2,
name='stdio',
slug='stdio',
stepid=2,
complete=1,
num_lines=20),
fakedb.Log(id=3,
name='stdio',
slug='stdio',
stepid=3,
complete=1,
num_lines=30),
fakedb.Log(id=11,
name='stdio',
slug='stdio',
stepid=11,
complete=1,
num_lines=30),
fakedb.Log(id=12,
name='stdio',
slug='stdio',
stepid=12,
complete=1,
num_lines=40),
fakedb.Log(id=13,
name='stdio',
slug='stdio',
stepid=13,
complete=1,
num_lines=50),
fakedb.Log(id=21,
name='stdio',
slug='stdio',
stepid=21,
complete=1,
num_lines=50),
fakedb.Log(id=22,
name='stdio',
slug='stdio',
stepid=22,
complete=1,
num_lines=60),
fakedb.Log(id=23,
name='stdio',
slug='stdio',
stepid=23,
complete=1,
num_lines=70),
fakedb.LogChunk(logid=1,
first_line=0,
last_line=2,
content='o line1\no line2\n'),
fakedb.LogChunk(logid=1,
first_line=2,
last_line=3,
content='o line3\n'),
fakedb.LogChunk(logid=2,
first_line=0,
last_line=4,
content='o line1\no line2\no line3\no line4\n'),
])
def insertTestData(self,
buildResults,
finalResult,
insertSS=True,
parentPlan=False,
insert_patch=False):
self.db = self.master.db
self.db.insertTestData([
fakedb.Master(id=92),
fakedb.Worker(id=13, name='wrk'),
fakedb.Builder(id=79, name='Builder0'),
fakedb.Builder(id=80, name='Builder1'),
fakedb.Buildset(id=98,
results=finalResult,
reason="testReason1",
parent_buildid=19 if parentPlan else None),
fakedb.Change(changeid=13,
branch=self.TEST_BRANCH,
revision='9283',
author='me@foo',
repository=self.TEST_REPO,
codebase=self.TEST_CODEBASE,
project='world-domination',
sourcestampid=234),
])
if parentPlan:
self.db.insertTestData([
fakedb.Worker(id=12, name='wrk_parent'),
fakedb.Builder(id=78, name='Builder_parent'),
fakedb.Buildset(id=97,
results=finalResult,
reason="testReason0"),
fakedb.BuildRequest(id=10, buildsetid=98, builderid=78),
fakedb.Build(id=19,
number=1,
builderid=78,
buildrequestid=10,
workerid=12,
masterid=92,
results=finalResult,
state_string="buildText"),
])
if insertSS:
patchid = 99 if insert_patch else None
self.db.insertTestData([
fakedb.BuildsetSourceStamp(buildsetid=98, sourcestampid=234),
fakedb.SourceStamp(id=234,
branch=self.TEST_BRANCH,
project=self.TEST_PROJECT,
revision=self.TEST_REVISION,
repository=self.TEST_REPO,
codebase=self.TEST_CODEBASE,
patchid=patchid),
fakedb.Patch(id=99,
patch_base64='aGVsbG8sIHdvcmxk',
patch_author='him@foo',
patch_comment='foo',
subdir='/foo',
patchlevel=3),
])
for i, results in enumerate(buildResults):
started_at = 10000001
complete_at = None if results is None else 10000005
self.db.insertTestData([
fakedb.BuildRequest(id=11 + i, buildsetid=98,
builderid=79 + i),
fakedb.Build(id=20 + i,
number=i,
builderid=79 + i,
buildrequestid=11 + i,
workerid=13,
masterid=92,
results=results,
state_string="buildText",
started_at=started_at,
complete_at=complete_at),
fakedb.Step(id=50 + i, buildid=20 + i, number=5, name='make'),
fakedb.Log(id=60 + i,
stepid=50 + i,
name='stdio',
slug='stdio',
type='s',
num_lines=7),
fakedb.LogChunk(
logid=60 + i,
first_line=0,
last_line=1,
compressed=0,
content='Unicode log with non-ascii (\u00E5\u00E4\u00F6).'
),
fakedb.BuildProperty(buildid=20 + i,
name="workername",
value="wrk"),
fakedb.BuildProperty(buildid=20 + i,
name="reason",
value="because"),
fakedb.BuildProperty(buildid=20 + i,
name="buildername",
value="Builder0"),
fakedb.BuildProperty(buildid=20 + i,
name="buildnumber",
value="{}".format(i)),
fakedb.BuildProperty(buildid=20 + i,
name="scheduler",
value="checkin"),
])
for k, v in self.TEST_PROPS.items():
self.db.insertTestData(
[fakedb.BuildProperty(buildid=20 + i, name=k, value=v)])
@defer.inlineCallbacks
def getChangesForBuild(buildid):
assert buildid == 20
ch = yield self.master.db.changes.getChange(13)
return [ch]
self.master.db.changes.getChangesForBuild = getChangesForBuild
def setup_db(self,
results1,
results2,
with_steps=False,
extra_build_properties=None):
if extra_build_properties is None:
extra_build_properties = {}
self.db = self.master.db
self.db.insertTestData([
fakedb.Master(id=92),
fakedb.Worker(id=13, name='wrkr'),
fakedb.Buildset(id=98, results=results1, reason="testReason1"),
fakedb.Buildset(id=99, results=results2, reason="testReason2"),
fakedb.Builder(id=80, name='Builder1'),
fakedb.BuildRequest(id=11, buildsetid=98, builderid=80),
fakedb.BuildRequest(id=12, buildsetid=99, builderid=80),
fakedb.Build(id=20,
number=0,
builderid=80,
buildrequestid=11,
workerid=13,
masterid=92,
results=results1),
fakedb.Build(id=21,
number=1,
builderid=80,
buildrequestid=12,
workerid=13,
masterid=92,
results=results2),
])
for build_id in (20, 21):
self.db.insertTestData([
fakedb.BuildProperty(buildid=build_id,
name="workername",
value="wrkr"),
fakedb.BuildProperty(buildid=build_id,
name="reason",
value="because"),
])
for name, value in extra_build_properties.items():
self.db.insertTestData([
fakedb.BuildProperty(buildid=build_id,
name=name,
value=value),
])
if with_steps:
self.db.insertTestData([
fakedb.Step(id=151,
buildid=21,
number=1,
results=SUCCESS,
name='first step'),
fakedb.Step(id=152,
buildid=21,
number=2,
results=results2,
name='second step'),
fakedb.Step(id=153,
buildid=21,
number=3,
results=SUCCESS,
name='third step'),
fakedb.Log(id=251,
stepid=152,
name='stdio',
slug='stdio',
type='s',
num_lines=7),
fakedb.Log(id=252,
stepid=152,
name='stderr',
slug='stderr',
type='s',
num_lines=7),
fakedb.Log(id=253,
stepid=153,
name='stdio',
slug='stdio',
type='s',
num_lines=7),
])
def insertTestData(self, buildResults, finalResult, insertSS=True):
self.db = self.master.db
self.db.insertTestData([
fakedb.Master(id=92),
fakedb.Worker(id=13, name='wrk'),
fakedb.Builder(id=79, name='Builder0'),
fakedb.Builder(id=80, name='Builder1'),
fakedb.Buildset(id=98, results=finalResult, reason="testReason1"),
fakedb.Change(changeid=13,
branch='master',
revision='9283',
author='me@foo',
repository=self.TEST_REPO,
codebase=self.TEST_CODEBASE,
project='world-domination',
sourcestampid=234),
])
if insertSS:
self.db.insertTestData([
fakedb.BuildsetSourceStamp(buildsetid=98, sourcestampid=234),
fakedb.SourceStamp(id=234,
project=self.TEST_PROJECT,
revision=self.TEST_REVISION,
repository=self.TEST_REPO,
codebase=self.TEST_CODEBASE)
])
for i, results in enumerate(buildResults):
self.db.insertTestData([
fakedb.BuildRequest(id=11 + i, buildsetid=98,
builderid=79 + i),
fakedb.Build(id=20 + i,
number=i,
builderid=79 + i,
buildrequestid=11 + i,
workerid=13,
masterid=92,
results=results,
state_string="buildText"),
fakedb.Step(id=50 + i, buildid=20 + i, number=5, name='make'),
fakedb.Log(id=60 + i,
stepid=50 + i,
name='stdio',
slug='stdio',
type='s',
num_lines=7),
fakedb.LogChunk(
logid=60 + i,
first_line=0,
last_line=1,
compressed=0,
content='Unicode log with non-ascii (\u00E5\u00E4\u00F6).'
),
fakedb.BuildProperty(buildid=20 + i,
name="workername",
value="wrk"),
fakedb.BuildProperty(buildid=20 + i,
name="reason",
value="because"),
fakedb.BuildProperty(buildid=20 + i,
name="buildername",
value="Builder0"),
])
for k, v in self.TEST_PROPS.items():
self.db.insertTestData(
[fakedb.BuildProperty(buildid=20 + i, name=k, value=v)])
