"""Trac request handler implementation for the build master."""

implements(IRequestHandler)

# Configuration options

adjust_timestamps = BoolOption('bitten', 'adjust_timestamps', False, doc=

"""Whether the timestamps of builds should be adjusted to be close

to the timestamps of the corresponding changesets.""")

build_all = BoolOption('bitten', 'build_all', False, doc=

"""Whether to request builds of older revisions even if a younger

revision has already been built.""")

stabilize_wait = IntOption('bitten', 'stabilize_wait', 0, doc=

"""The time in seconds to wait for the repository to stabilize before

queuing up a new build. This allows time for developers to check in

a group of related changes back to back without spawning multiple

builds.""")

slave_timeout = IntOption('bitten', 'slave_timeout', 3600, doc=

"""The time in seconds after which a build is cancelled if the slave

does not report progress.""")

logs_dir = Option('bitten', 'logs_dir', "log/bitten", doc=

"""The directory on the server in which client log files will be stored.""")

quick_status = BoolOption('bitten', 'quick_status', False, doc=

"""Whether to show the current build status within the Trac main

navigation bar. '''Note:''' The feature requires expensive database and

repository checks for every page request, and should not be enabled

if the project has a large repository or uses a non-Subversion

repository such as Mercurial or Git.""")

def __init__(self):

self.env.systeminfo.append(('Bitten',

__import__('bitten', ['__version__']).__version__))

# IRequestHandler methods

def match_request(self, req):

match = re.match(r'/builds(?:/(\d+)(?:/(\w+)/([^/]+)?)?)?$',

req.path_info)

if match:

if match.group(1):

req.args['id'] = match.group(1)

req.args['collection'] = match.group(2)

req.args['member'] = match.group(3)

return True

def process_request(self, req):

req.perm.assert_permission('BUILD_EXEC')

if 'trac_auth' in req.incookie:

slave_token = req.incookie['trac_auth'].value

else:

slave_token = req.session.sid

if 'id' not in req.args:

if req.method != 'POST':

self._send_response(req,

body='Only POST allowed for build creation.')

return self._process_build_creation(req, slave_token)

build = Build.fetch(self.env, req.args['id'])

if not build:

self._send_error(req, HTTP_NOT_FOUND,

'No such build (%s)' % req.args['id'])

build_token = build.slave_info.get('token', '')

if build_token != slave_token:

self._send_error(req, HTTP_CONFLICT,

'Token mismatch (wrong slave): slave=%s, build=%s' \

% (slave_token, build_token))

config = BuildConfig.fetch(self.env, build.config)

if not req.args['collection']:

if req.method == 'DELETE':

return self._process_build_cancellation(req, config, build)

else:

return self._process_build_initiation(req, config, build)

if req.method != 'POST':

self._send_error(req, HTTP_METHOD_NOT_ALLOWED,

'Method %s not allowed' % req.method)

if req.args['collection'] == 'steps':

return self._process_build_step(req, config, build)

elif req.args['collection'] == 'attach':

return self._process_attachment(req, config, build)

elif req.args['collection'] == 'keepalive':

return self._process_keepalive(req, config, build)

else:

self._send_error(req, HTTP_NOT_FOUND,

"No such collection '%s'" % req.args['collection'])

# Internal methods

def _send_response(self, req, code=200, body='', headers=None):

""" Formats and sends the response, raising ``RequestDone``. """

if isinstance(body, unicode):

body = body.encode('utf-8')

req.send_response(code)

headers = headers or {}

headers.setdefault('Content-Length', len(body))

for header in headers:

req.send_header(header, headers[header])

req.write(body)

raise RequestDone

def _send_error(self, req, code=500, message=''):

""" Formats and sends the error, raising ``RequestDone``. """

headers = {'Content-Type': 'text/plain',

'Content-Length': str(len(message))}

self._send_response(req, code, body=message, headers=headers)

def _process_build_creation(self, req, slave_token):

queue = BuildQueue(self.env, build_all=self.build_all,

stabilize_wait=self.stabilize_wait,

timeout=self.slave_timeout)

try:

queue.populate()

except AssertionError, e:

self.log.error(e.message, exc_info=True)

self._send_error(req, HTTP_BAD_REQUEST, e.message)

try:

elem = xmlio.parse(req.read())

except xmlio.ParseError, e:

self.log.error('Error parsing build initialization request: %s', e,

exc_info=True)

self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')

slave_version = int(elem.attr.get('version', 1))

# FIXME: Remove version compatibility code.

# The initial difference between protocol version 3 and 4 is that

# the master allows keepalive requests-- the master must be

# at least 4 before slaves supporting version 4 are allowed. When

# the first force master/slave upgrade requirement comes in

# (or we bump the) version number again, remove this code.

if slave_version == 3 and PROTOCOL_VERSION == 4:

self.log.info('Allowing slave version %d to process build for '

'compatibility. Upgrade slave to support build '

'keepalives.', slave_version)

elif slave_version != PROTOCOL_VERSION:

self._send_error(req, HTTP_BAD_REQUEST,

"Master-Slave version mismatch: master=%d, slave=%d" % \

(PROTOCOL_VERSION, slave_version))

slavename = elem.attr['name']

properties = {'name': slavename, Build.IP_ADDRESS: req.remote_addr,

Build.TOKEN: slave_token}

self.log.info('Build slave %r connected from %s with token %s',

slavename, req.remote_addr, slave_token)

for child in elem.children():

if child.name == 'platform':

properties[Build.MACHINE] = child.gettext()

properties[Build.PROCESSOR] = child.attr.get('processor')

elif child.name == 'os':

properties[Build.OS_NAME] = child.gettext()

properties[Build.OS_FAMILY] = child.attr.get('family')

properties[Build.OS_VERSION] = child.attr.get('version')

elif child.name == 'package':

for name, value in child.attr.items():

if name == 'name':

continue

properties[child.attr['name'] + '.' + name] = value

self.log.debug('Build slave configuration: %r', properties)

build = queue.get_build_for_slave(slavename, properties)

if not build:

self._send_response(req, 204, '', {})

self._send_response(req, 201, 'Build pending', headers={

'Content-Type': 'text/plain',

'Location': req.abs_href.builds(build.id)})

def _process_build_cancellation(self, req, config, build):

self.log.info('Build slave %r cancelled build %d', build.slave,

build.id)

build.status = Build.PENDING

build.slave = None

build.slave_info = {}

build.started = 0

db = self.env.get_db_cnx()

for step in list(BuildStep.select(self.env, build=build.id, db=db)):

step.delete(db=db)

build.update(db=db)

Attachment.delete_all(self.env, 'build', build.resource.id, db)

db.commit()

for listener in BuildSystem(self.env).listeners:

listener.build_aborted(build)

self._send_response(req, 204, '', {})

def _process_build_initiation(self, req, config, build):

self.log.info('Build slave %r initiated build %d', build.slave,

build.id)

build.started = int(time.time())

build.last_activity = build.started

build.update()

for listener in BuildSystem(self.env).listeners:

listener.build_started(build)

repos_name, repos, repos_path = get_repos(

self.env, config.path, req.authname)

xml = xmlio.parse(config.recipe)

xml.attr['path'] = config.path

xml.attr['revision'] = build.rev

xml.attr['config'] = config.name

xml.attr['build'] = str(build.id)

target_platform = TargetPlatform.fetch(self.env, build.platform)

xml.attr['platform'] = target_platform.name

xml.attr['name'] = build.slave

xml.attr['form_token'] = req.form_token # For posting attachments

xml.attr['reponame'] = repos_name != '(default)' and repos_name or ''

xml.attr['repopath'] = repos_path.strip('/')

body = str(xml)

self.log.info('Build slave %r initiated build %d', build.slave,

build.id)

# create the first step, mark it as in-progress.

recipe = Recipe(xmlio.parse(config.recipe))

stepname = recipe.__iter__().next().id

step = self._start_new_step(build, stepname)

step.insert()

self._send_response(req, 200, body, headers={

'Content-Type': 'application/x-bitten+xml',

'Content-Length': str(len(body)),

'Content-Disposition':

'attachment; filename=recipe_%s_r%s.xml' %

(config.name, build.rev)})

def _process_build_step(self, req, config, build):

try:

elem = xmlio.parse(req.read())

except xmlio.ParseError, e:

self.log.error('Error parsing build step result: %s', e,

exc_info=True)

self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')

stepname = elem.attr['step']

# we should have created this step previously; if it hasn't,

# the master and slave are processing steps out of order.

step = BuildStep.fetch(self.env, build=build.id, name=stepname)

if not step:

self._send_error(req, HTTP_CONFLICT, 'Build step has not been created.')

recipe = Recipe(xmlio.parse(config.recipe))

index = None

current_step = None

for num, recipe_step in enumerate(recipe):

if recipe_step.id == stepname:

index = num

current_step = recipe_step

if index is None:

self._send_error(req, HTTP_FORBIDDEN,

'No such build step' % stepname)

last_step = index == num

self.log.debug('Slave %s (build %d) completed step %d (%s) with '

'status %s', build.slave, build.id, index, stepname,

elem.attr['status'])

db = self.env.get_db_cnx()

step.stopped = int(time.time())

if elem.attr['status'] == 'failure':

self.log.warning('Build %s step %s failed', build.id, stepname)

step.status = BuildStep.FAILURE

if current_step.onerror == 'fail':

last_step = True

else:

step.status = BuildStep.SUCCESS

step.errors += [error.gettext() for error in elem.children('error')]

# TODO: step.update(db=db)

step.delete(db=db)

step.insert(db=db)

# Collect log messages from the request body

for idx, log_elem in enumerate(elem.children('log')):

build_log = BuildLog(self.env, build=build.id, step=stepname,

generator=log_elem.attr.get('generator'),

orderno=idx)

for message_elem in log_elem.children('message'):

build_log.messages.append((message_elem.attr['level'],

message_elem.gettext()))

build_log.insert(db=db)

# Collect report data from the request body

for report_elem in elem.children('report'):

report = Report(self.env, build=build.id, step=stepname,

category=report_elem.attr.get('category'),

generator=report_elem.attr.get('generator'))

for item_elem in report_elem.children():

item = {'type': item_elem.name}

item.update(item_elem.attr)

for child_elem in item_elem.children():

item[child_elem.name] = child_elem.gettext()

report.items.append(item)

report.insert(db=db)

# If this was the last step in the recipe we mark the build as

# completed otherwise just update last_activity

if last_step:

self.log.info('Slave %s completed build %d ("%s" as of [%s])',

build.slave, build.id, build.config, build.rev)

build.stopped = step.stopped

build.last_activity = build.stopped

# Determine overall outcome of the build by checking the outcome

# of the individual steps against the "onerror" specification of

# each step in the recipe

for num, recipe_step in enumerate(recipe):

step = BuildStep.fetch(self.env, build.id, recipe_step.id)

if step.status == BuildStep.FAILURE:

if recipe_step.onerror == 'fail' or \

recipe_step.onerror == 'continue':

build.status = Build.FAILURE

break

else:

build.status = Build.SUCCESS

build.update(db=db)

else:

build.last_activity = step.stopped

build.update(db=db)

# start the next step.

for num, recipe_step in enumerate(recipe):

if num == index + 1:

next_step = recipe_step

if next_step is None:

self._send_error(req, HTTP_FORBIDDEN,

'Unable to find step after ' % stepname)

step = self._start_new_step(build, next_step.id)

step.insert(db=db)

db.commit()

if last_step:

for listener in BuildSystem(self.env).listeners:

listener.build_completed(build)

body = 'Build step processed'

self._send_response(req, 201, body, {

'Content-Type': 'text/plain',

'Content-Length': str(len(body)),

'Location': req.abs_href.builds(

build.id, 'steps', stepname)})

def _process_attachment(self, req, config, build):

resource_id = req.args['member'] == 'config' \

and build.config or build.resource.id

upload = req.args['file']

if not upload.file:

send_error(req, message="Attachment not received.")

self.log.debug('Received attachment %s for attaching to build:%s',

upload.filename, resource_id)

# Determine size of file

upload.file.seek(0, 2) # to the end

size = upload.file.tell()

upload.file.seek(0) # beginning again

# Delete attachment if it already exists

try:

old_attach = Attachment(self.env, 'build',

parent_id=resource_id, filename=upload.filename)

old_attach.delete()

except ResourceNotFound:

pass

# Save new attachment

attachment = Attachment(self.env, 'build', parent_id=resource_id)

attachment.description = req.args.get('description', '')

attachment.author = req.authname

attachment.insert(upload.filename, upload.file, size)

self._send_response(req, 201, 'Attachment created', headers={

'Content-Type': 'text/plain',

'Content-Length': str(len('Attachment created'))})

def _process_keepalive(self, req, config, build):

build.last_activity = int(time.time())

build.update()

self.log.info('Slave %s build %d keepalive ("%s" as of [%s])',

build.slave, build.id, build.config, build.rev)

body = 'Keepalive processed'

self._send_response(req, 200, body, {

'Content-Type': 'text/plain',

'Content-Length': str(len(body))})

def _start_new_step(self, build, stepname):

"""Creates the in-memory representation for a newly started

step, ready to be persisted to the database.

"""

step = BuildStep(self.env, build=build.id, name=stepname)

step.status = BuildStep.IN_PROGRESS

step.started = int(time.time())

step.stopped = 0