def add_log_table(env, db):
"""Add a table for storing the builds logs."""
from bitten.model import BuildLog, BuildStep
cursor = db.cursor()
connector, _ = DatabaseManager(env)._get_connector()
for table in BuildLog._schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("SELECT build,name,log FROM bitten_step "
"WHERE log IS NOT NULL")
for build, step, log in cursor:
build_log = BuildLog(env, build, step)
build_log.messages = [(BuildLog.INFO, msg) for msg in log.splitlines()]
build_log.insert(db)
cursor.execute("CREATE TEMPORARY TABLE old_step AS SELECT * FROM bitten_step")
cursor.execute("DROP TABLE bitten_step")
for table in BuildStep._schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("INSERT INTO bitten_step (build,name,description,status,"
"started,stopped) SELECT build,name,description,status,"
"started,stopped FROM old_step")
def add_log_table(env, db):
"""Add a table for storing the builds logs."""
from bitten.model import BuildLog, BuildStep
cursor = db.cursor()
connector, _ = DatabaseManager(env)._get_connector()
for table in BuildLog._schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("SELECT build,name,log FROM bitten_step "
"WHERE log IS NOT NULL")
for build, step, log in cursor:
build_log = BuildLog(env, build, step)
build_log.messages = [(BuildLog.INFO, msg) for msg in log.splitlines()]
build_log.insert(db)
cursor.execute(
"CREATE TEMPORARY TABLE old_step AS SELECT * FROM bitten_step")
cursor.execute("DROP TABLE bitten_step")
for table in BuildStep._schema:
for stmt in connector.to_sql(table):
cursor.execute(stmt)
cursor.execute("INSERT INTO bitten_step (build,name,description,status,"
"started,stopped) SELECT build,name,description,status,"
"started,stopped FROM old_step")
def test_delete(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"INSERT INTO bitten_log (build,step,generator,filename) "
"VALUES (%s,%s,%s,%s)", (1, 'test', 'distutils', '1.log'))
id = db.get_last_id(cursor, 'bitten_log')
logs_dir = self.env.config.get("bitten", "logsdir", "log/bitten")
if os.path.isabs(logs_dir):
raise ValueError(
"Should not have absolute logs directory for temporary test")
logs_dir = os.path.join(self.env.path, logs_dir)
full_file = os.path.join(logs_dir, "1.log")
open(full_file, "wb").writelines(["running tests\n", "tests failed\n"])
log = BuildLog.fetch(self.env, id=id, db=db)
self.assertEqual(True, log.exists)
log.delete()
self.assertEqual(False, log.exists)
cursor.execute("SELECT * FROM bitten_log WHERE id=%s", (id, ))
self.assertEqual(True, not cursor.fetchall())
file_exists = os.path.exists(full_file)
if os.path.exists(full_file):
os.remove(full_file)
assert not file_exists
def get_all_log_messages_for_step(self, step):
messages = []
for log in BuildLog.select(self.env,
build=self.build.id,
step=step.name):
messages.extend(log.messages)
return messages
def test_select(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("INSERT INTO bitten_log (build,step,generator,filename) "
"VALUES (%s,%s,%s,%s)", (1, 'test', 'distutils', '1.log'))
id = db.get_last_id(cursor, 'bitten_log')
logs_dir = self.env.config.get("bitten", "logsdir", "log/bitten")
if os.path.isabs(logs_dir):
raise ValueError("Should not have absolute logs directory for temporary test")
logs_dir = os.path.join(self.env.path, logs_dir)
full_file = os.path.join(logs_dir, "1.log")
open(full_file, "wb").writelines(["running tests\n", "tests failed\n", u"test unicode\xbb\n".encode("UTF-8")])
logs = BuildLog.select(self.env, build=1, step='test', db=db)
log = logs.next()
self.assertEqual(True, log.exists)
self.assertEqual(id, log.id)
self.assertEqual(1, log.build)
self.assertEqual('test', log.step)
self.assertEqual('distutils', log.generator)
self.assertEqual((BuildLog.UNKNOWN, 'running tests'), log.messages[0])
self.assertEqual((BuildLog.UNKNOWN, 'tests failed'), log.messages[1])
self.assertEqual((BuildLog.UNKNOWN, u'test unicode\xbb'), log.messages[2])
self.assertRaises(StopIteration, logs.next)
if os.path.exists(full_file):
os.remove(full_file)
def test_process_build_step_success_with_log(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env, 'test', path='somepath', active=True,
recipe=recipe).insert()
build = Build(self.env, 'test', '123', 1, slave='hal', rev_time=42,
started=42, status=Build.IN_PROGRESS)
build.slave_info[Build.IP_ADDRESS] = '127.0.0.1';
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.cmlenz.net/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST', base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'), abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1', args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write)
module = BuildMaster(self.env)
assert module.match_request(req)
try:
module.process_request(req)
self.fail('Expected RequestDone')
except RequestDone:
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
logs = list(BuildLog.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(logs))
self.assertEqual('http://bitten.cmlenz.net/tools/python#unittest',
logs[0].generator)
self.assertEqual(2, len(logs[0].messages))
self.assertEqual((u'info', u'Doing stuff'), logs[0].messages[0])
self.assertEqual((u'error', u'Ouch that hurt'), logs[0].messages[1])
def _render_log(self, req, build, formatters, step):
items = []
for log in BuildLog.select(self.env, build=build.id, step=step.name):
for level, message in log.messages:
for format in formatters:
message = format(step, log.generator, level, message)
items.append({'level': level, 'message': message})
return items
def _render_log(self, req, build, formatters, step):
items = []
for log in BuildLog.select(self.env, build=build.id, step=step.name):
for level, message in log.messages:
for format in formatters:
message = format(step, log.generator, level, message)
items.append({'level': level, 'message': message})
return items
def test_new(self):
log = BuildLog(self.env)
self.assertEqual(False, log.exists)
self.assertEqual(None, log.id)
self.assertEqual(None, log.build)
self.assertEqual(None, log.step)
self.assertEqual('', log.generator)
self.assertEqual([], log.messages)
def get_faillog(self, build):
faillog = ''
for step in self.get_failed_steps(build):
build_logs = BuildLog.select(self.env,
build=build.id,
step=step.name)
for log in build_logs:
faillog += '\n'.join(['%5s: %s' % (level, msg) \
for level, msg in log.messages])
return faillog
def test_insert(self):
log = BuildLog(self.env,
build=1,
step='test',
generator='distutils',
filename='1.log.gz')
full_file = log.get_log_file('1.log.gz')
if os.path.exists(full_file):
os.remove(full_file)
log.messages = [(BuildLog.INFO, 'running tests'),
(BuildLog.ERROR, 'tests failed')]
log.insert()
self.assertNotEqual(None, log.id)
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute(
"SELECT build,step,generator,filename FROM bitten_log "
"WHERE id=%s", (log.id, ))
self.assertEqual((1, 'test', 'distutils', '1.log.gz'),
cursor.fetchone())
lines = gzip.open(full_file, "rb").readlines()
self.assertEqual('running tests\n', lines[0])
self.assertEqual('tests failed\n', lines[1])
if os.path.exists(full_file):
os.remove(full_file)
def test_insert_and_delete_files(self):
# create - files should be created automatically
build_log = BuildLog(self.env, build=1, step='test', generator='make')
build_log.messages = [(BuildLog.INFO, 'running')]
build_log.insert()
# fetch it fresh - check object and files
build_log = BuildLog.fetch(self.env, id=build_log.id)
self.assertEquals(build_log.filename, "%s.log.gz" % build_log.id)
log_file = build_log.get_log_file(build_log.filename)
levels_file = build_log.get_level_file(build_log.filename)
self.failUnless(os.path.exists(log_file), 'log_file does not exist')
self.failUnless(os.path.exists(levels_file),
'levels_file does not exist')
self.assertEquals(build_log.messages, [(BuildLog.INFO, 'running')])
# delete - object and file should be gone
build_log.delete()
self.assertEquals(None, BuildLog.fetch(self.env, id=build_log.id))
self.failIf(os.path.exists(log_file), 'log_file exists after delete()')
self.failIf(os.path.exists(levels_file),
'levels_file exists after delete()')
def test_insert_and_delete_files(self):
# create - files should be created automatically
build_log = BuildLog(self.env, build=1, step='test', generator='make')
build_log.messages = [(BuildLog.INFO, 'running')]
build_log.insert()
# fetch it fresh - check object and files
build_log = BuildLog.fetch(self.env, id=build_log.id)
self.assertEquals(build_log.filename, "%s.log" % build_log.id)
log_file = build_log.get_log_file(build_log.filename)
levels_file = log_file+'.levels'
self.failUnless(os.path.exists(log_file), 'log_file does not exist')
self.failUnless(os.path.exists(levels_file),
'levels_file does not exist')
self.assertEquals(build_log.messages, [(BuildLog.INFO, 'running')])
# delete - object and file should be gone
build_log.delete()
self.assertEquals(None, BuildLog.fetch(self.env, id=build_log.id))
self.failIf(os.path.exists(log_file), 'log_file exists after delete()')
self.failIf(os.path.exists(levels_file),
'levels_file exists after delete()')
def test_insert(self):
log = BuildLog(self.env, build=1, step='test', generator='distutils', filename='1.log')
full_file = log.get_log_file('1.log')
if os.path.exists(full_file):
os.remove(full_file)
log.messages = [
(BuildLog.INFO, 'running tests'),
(BuildLog.ERROR, 'tests failed')
]
log.insert()
self.assertNotEqual(None, log.id)
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT build,step,generator,filename FROM bitten_log "
"WHERE id=%s", (log.id,))
self.assertEqual((1, 'test', 'distutils', '1.log'), cursor.fetchone())
lines = open(full_file, "rb").readlines()
self.assertEqual('running tests\n', lines[0])
self.assertEqual('tests failed\n', lines[1])
if os.path.exists(full_file):
os.remove(full_file)
def test_insert_empty(self):
log = BuildLog(self.env, build=1, step='test', generator='distutils', filename="1.log")
full_file = log.get_log_file('1.log')
if os.path.exists(full_file):
os.remove(full_file)
log.messages = []
log.insert()
self.assertNotEqual(None, log.id)
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("SELECT build,step,generator,filename FROM bitten_log "
"WHERE id=%s", (log.id,))
self.assertEqual((1, 'test', 'distutils', '1.log'), cursor.fetchone())
file_exists = os.path.exists(full_file)
if file_exists:
os.remove(full_file)
assert not file_exists
def test_delete(self):
db = self.env.get_db_cnx()
cursor = db.cursor()
cursor.execute("INSERT INTO bitten_log (build,step,generator,filename) "
"VALUES (%s,%s,%s,%s)", (1, 'test', 'distutils', '1.log'))
id = db.get_last_id(cursor, 'bitten_log')
logs_dir = self.env.config.get("bitten", "logsdir", "log/bitten")
if os.path.isabs(logs_dir):
raise ValueError("Should not have absolute logs directory for temporary test")
logs_dir = os.path.join(self.env.path, logs_dir)
full_file = os.path.join(logs_dir, "1.log")
open(full_file, "wb").writelines(["running tests\n", "tests failed\n"])
log = BuildLog.fetch(self.env, id=id, db=db)
self.assertEqual(True, log.exists)
log.delete()
self.assertEqual(False, log.exists)
cursor.execute("SELECT * FROM bitten_log WHERE id=%s", (id,))
self.assertEqual(True, not cursor.fetchall())
file_exists = os.path.exists(full_file)
if os.path.exists(full_file):
os.remove(full_file)
assert not file_exists
def get_all_log_messages_for_step(self, step):
messages = []
for log in BuildLog.select(self.env, build=self.build.id,
step=step.name):
messages.extend(log.messages)
return messages
step.status = BuildStep.FAILURE
if current_step.onerror == 'fail':
last_step = True
else:
step.status = BuildStep.SUCCESS
step.errors += [error.gettext() for error in elem.children('error')]
# TODO: step.update(db=db)
step.delete(db=db)
step.insert(db=db)
# Collect log messages from the request body
for idx, log_elem in enumerate(elem.children('log')):
build_log = BuildLog(self.env,
build=build.id,
step=stepname,
generator=log_elem.attr.get('generator'),
orderno=idx)
for message_elem in log_elem.children('message'):
build_log.messages.append(
(message_elem.attr['level'], message_elem.gettext()))
build_log.insert(db=db)
# Collect report data from the request body
for report_elem in elem.children('report'):
report = Report(self.env,
build=build.id,
step=stepname,
category=report_elem.attr.get('category'),
generator=report_elem.attr.get('generator'))
for item_elem in report_elem.children():
def test_delete_new(self):
log = BuildLog(self.env, build=1, step='test', generator='foo')
self.assertRaises(AssertionError, log.delete)
def test_insert_no_build_or_step(self):
log = BuildLog(self.env, step='test')
self.assertRaises(AssertionError, log.insert) # No build
step = BuildStep(self.env, build=1)
self.assertRaises(AssertionError, log.insert) # No step
def test_process_build_step_success_with_log(self):
recipe = """<build>
<step id="foo">
</step>
</build>"""
BuildConfig(self.env,
'test',
path='somepath',
active=True,
recipe=recipe).insert()
build = Build(self.env,
'test',
'123',
1,
slave='hal',
rev_time=42,
started=42,
status=Build.IN_PROGRESS)
build.slave_info[Build.TOKEN] = '123'
build.insert()
inbody = StringIO("""<result step="foo" status="success"
time="2007-04-01T15:30:00.0000"
duration="3.45">
<log generator="http://bitten.edgewall.org/tools/python#unittest">
<message level="info">Doing stuff</message>
<message level="error">Ouch that hurt</message>
</log>
</result>""")
outheaders = {}
outbody = StringIO()
req = Mock(method='POST',
base_path='',
path_info='/builds/%d/steps/' % build.id,
href=Href('/trac'),
abs_href=Href('http://example.org/trac'),
remote_addr='127.0.0.1',
args={},
perm=PermissionCache(self.env, 'hal'),
read=inbody.read,
send_response=lambda x: outheaders.setdefault('Status', x),
send_header=lambda x, y: outheaders.setdefault(x, y),
write=outbody.write,
incookie=Cookie('trac_auth=123'))
module = BuildMaster(self.env)
module._start_new_step(build, 'foo').insert()
assert module.match_request(req)
self.assertRaises(RequestDone, module.process_request, req)
self.assertEqual(201, outheaders['Status'])
self.assertEqual('20', outheaders['Content-Length'])
self.assertEqual('text/plain', outheaders['Content-Type'])
self.assertEqual('Build step processed', outbody.getvalue())
build = Build.fetch(self.env, build.id)
self.assertEqual(Build.SUCCESS, build.status)
assert build.stopped
assert build.stopped > build.started
steps = list(BuildStep.select(self.env, build.id))
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].name)
self.assertEqual(BuildStep.SUCCESS, steps[0].status)
logs = list(BuildLog.select(self.env, build=build.id, step='foo'))
self.assertEqual(1, len(logs))
self.assertEqual('http://bitten.edgewall.org/tools/python#unittest',
logs[0].generator)
self.assertEqual(2, len(logs[0].messages))
self.assertEqual((u'info', u'Doing stuff'), logs[0].messages[0])
self.assertEqual((u'error', u'Ouch that hurt'), logs[0].messages[1])
self.log.warning('Build %s step %s failed', build.id, stepname)
step.status = BuildStep.FAILURE
if current_step.onerror == 'fail':
last_step = True
else:
step.status = BuildStep.SUCCESS
step.errors += [error.gettext() for error in elem.children('error')]
# TODO: step.update(db=db)
step.delete(db=db)
step.insert(db=db)
# Collect log messages from the request body
for idx, log_elem in enumerate(elem.children('log')):
build_log = BuildLog(self.env, build=build.id, step=stepname,
generator=log_elem.attr.get('generator'),
orderno=idx)
for message_elem in log_elem.children('message'):
build_log.messages.append((message_elem.attr['level'],
message_elem.gettext()))
build_log.insert(db=db)
# Collect report data from the request body
for report_elem in elem.children('report'):
report = Report(self.env, build=build.id, step=stepname,
category=report_elem.attr.get('category'),
generator=report_elem.attr.get('generator'))
for item_elem in report_elem.children():
item = {'type': item_elem.name}
item.update(item_elem.attr)
for child_elem in item_elem.children():
