def test_validate_successful(self):
xml = xmlio.parse('<build>'
'<step id="foo"><somecmd></somecmd></step>'
'<step id="bar"><othercmd></othercmd></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
recipe.validate()
def test_validate_successful(self):
xml = xmlio.parse('<build>'
'<step id="foo"><somecmd></somecmd></step>'
'<step id="bar"><othercmd></othercmd></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
recipe.validate()
def _process_build_initiation(self, req, config, build):
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
build.started = int(time.time())
build.last_activity = build.started
build.update()
for listener in BuildSystem(self.env).listeners:
listener.build_started(build)
repos_name, repos, repos_path = get_repos(self.env, config.path,
req.authname)
xml = xmlio.parse(config.recipe)
xml.attr['path'] = config.path
xml.attr['revision'] = build.rev
xml.attr['config'] = config.name
xml.attr['build'] = str(build.id)
target_platform = TargetPlatform.fetch(self.env, build.platform)
xml.attr['platform'] = target_platform.name
xml.attr['name'] = build.slave
xml.attr['form_token'] = req.form_token # For posting attachments
xml.attr['reponame'] = repos_name != '(default)' and repos_name or ''
xml.attr['repopath'] = repos_path.strip('/')
body = str(xml)
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
# create the first step, mark it as in-progress.
recipe = Recipe(xmlio.parse(config.recipe))
stepname = recipe.__iter__().next().id
step = self._start_new_step(build, stepname)
step.insert()
self._send_response(req,
200,
body,
headers={
'Content-Type':
'application/x-bitten+xml',
'Content-Length':
str(len(body)),
'Content-Disposition':
'attachment; filename=recipe_%s_r%s.xml' %
(config.name, build.rev)
})
def _execute_build(self, build_url, fileobj):
build_id = build_url and int(build_url.split('/')[-1]) or 0
xml = xmlio.parse(fileobj)
basedir = ''
try:
recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir),
self.config)
basedir = recipe.ctxt.basedir
log.debug('Running build in directory %s' % basedir)
if not os.path.exists(basedir):
os.mkdir(basedir)
for step in recipe:
log.info('Executing build step %r', step.id)
if not self._execute_step(build_url, recipe, step):
log.warning('Stopping build due to failure')
break
else:
log.info('Build completed')
if self.dry_run:
self._cancel_build(build_url)
finally:
if not self.keep_files and os.path.isdir(basedir):
log.debug('Removing build directory %s' % basedir)
_rmtree(basedir)
if self.single_build:
log.info('Exiting after single build completed.')
raise ExitSlave(EX_OK)
def _execute_build(self, build_url, fileobj):
build_id = build_url and int(build_url.split('/')[-1]) or 0
xml = xmlio.parse(fileobj)
basedir = ''
try:
if not self.local:
keepalive_thread = KeepAliveThread(self.opener, build_url,
self.single_build,
self.keepalive_interval)
keepalive_thread.start()
recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir),
self.config)
basedir = recipe.ctxt.basedir
log.debug('Running build in directory %s' % basedir)
if not os.path.exists(basedir):
os.mkdir(basedir)
for step in recipe:
try:
log.info('Executing build step %r, onerror = %s', step.id,
step.onerror)
if not self._execute_step(build_url, recipe, step):
log.warning('Stopping build due to failure')
break
except Exception, e:
log.error(
'Exception raised processing step %s. Reraising %s',
step.id, e)
raise
else:
def test_validate_step_with_duplicate_id(self):
xml = xmlio.parse('<build>'
'<step id="test"><somecmd></somecmd></step>'
'<step id="test"><othercmd></othercmd></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def test_empty_step(self):
xml = xmlio.parse('<build>'
' <step id="foo" description="Bar"></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
steps = list(recipe)
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].id)
self.assertEqual('Bar', steps[0].description)
self.assertEqual('fail', steps[0].onerror)
def test_onerror_override(self):
xml = xmlio.parse('<build onerror="ignore">'
' <step id="foo" description="Bar" onerror="continue"></step>'
'</build>')
recipe = Recipe(xml, basedir=self.basedir)
steps = list(recipe)
self.assertEqual(1, len(steps))
self.assertEqual('foo', steps[0].id)
self.assertEqual('Bar', steps[0].description)
self.assertEqual('continue', steps[0].onerror)
def _process_build_initiation(self, req, config, build):
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
build.started = int(time.time())
build.last_activity = build.started
build.update()
for listener in BuildSystem(self.env).listeners:
listener.build_started(build)
xml = xmlio.parse(config.recipe)
xml.attr['path'] = config.path
xml.attr['revision'] = build.rev
xml.attr['config'] = config.name
xml.attr['build'] = str(build.id)
target_platform = TargetPlatform.fetch(self.env, build.platform)
xml.attr['platform'] = target_platform.name
xml.attr['name'] = build.slave
xml.attr['form_token'] = req.form_token # For posting attachments
body = str(xml)
self.log.info('Build slave %r initiated build %d', build.slave,
build.id)
# create the first step, mark it as in-progress.
recipe = Recipe(xmlio.parse(config.recipe))
stepname = recipe.__iter__().next().id
step = self._start_new_step(build, stepname)
step.insert()
self._send_response(req, 200, body, headers={
'Content-Type': 'application/x-bitten+xml',
'Content-Length': str(len(body)),
'Content-Disposition':
'attachment; filename=recipe_%s_r%s.xml' %
(config.name, build.rev)})
class BuildConfigurationsAdminPageProvider(Component):
"""Web administration panel for configuring the build master."""
implements(IAdminPanelProvider)
# IAdminPanelProvider methods
def get_admin_panels(self, req):
if req.perm.has_permission('BUILD_MODIFY'):
yield ('bitten', 'Builds', 'configs', 'Configurations')
def render_admin_panel(self, req, cat, page, path_info):
data = {}
# Analyze url
try:
config_name, platform_id = path_info.split('/', 1)
except:
config_name = path_info
platform_id = None
if config_name: # Existing build config
warnings = []
if platform_id or (
# Editing or creating one of the config's target platforms
req.method == 'POST' and 'new' in req.args):
if platform_id: # Editing target platform
platform_id = int(platform_id)
platform = TargetPlatform.fetch(self.env, platform_id)
if req.method == 'POST':
if 'cancel' in req.args or \
self._update_platform(req, platform):
req.redirect(
req.abs_href.admin(cat, page, config_name))
else: # creating target platform
platform = self._create_platform(req, config_name)
req.redirect(
req.abs_href.admin(cat, page, config_name,
platform.id))
# Set up template variables
data['platform'] = {
'id':
platform.id,
'name':
platform.name,
'exists':
platform.exists,
'rules': [{
'property': propname,
'pattern': pattern
} for propname, pattern in platform.rules] or [('', '')]
}
else: # Editing existing build config itself
config = BuildConfig.fetch(self.env, config_name)
platforms = list(
TargetPlatform.select(self.env, config=config.name))
if req.method == 'POST':
if 'remove' in req.args: # Remove selected platforms
self._remove_platforms(req)
add_notice(req, "Target Platform(s) Removed.")
req.redirect(req.abs_href.admin(
cat, page, config.name))
elif 'save' in req.args: # Save this build config
warnings = self._update_config(req, config)
if not warnings:
add_notice(req, "Configuration Saved.")
req.redirect(req.abs_href.admin(
cat, page, config.name))
for warning in warnings:
add_warning(req, warning)
# FIXME: Deprecation notice for old namespace.
# Remove notice code when migration to new namespace is complete
if 'http://bitten.cmlenz.net/tools/' in config.recipe:
add_notice(
req, "Recipe uses a deprecated namespace. "
"Replace 'http://bitten.cmlenz.net/tools/' with "
"'http://bitten.edgewall.org/tools/'.")
# Add a notice if configuration is not active
if not warnings and not config.active and config.recipe:
add_notice(
req, "Configuration is not active. Activate "
"from main 'Configurations' listing to enable it.")
# Prepare template variables
data['config'] = {
'name':
config.name,
'label':
config.label or config.name,
'active':
config.active,
'branch':
config.branch,
'description':
config.description,
'recipe':
config.recipe,
'platforms': [{
'name':
platform.name,
'id':
platform.id,
'href':
req.href.admin('bitten', 'configs', config.name,
platform.id),
'rules': [{
'property': propname,
'pattern': pattern
} for propname, pattern in platform.rules]
} for platform in platforms]
}
else: # At the top level build config list
if req.method == 'POST':
if 'add' in req.args: # Add build config
config = self._create_config(req)
req.redirect(req.abs_href.admin(cat, page, config.name))
elif 'remove' in req.args: # Remove selected build configs
self._remove_configs(req)
elif 'apply' in req.args: # Update active state of configs
self._activate_configs(req)
req.redirect(req.abs_href.admin(cat, page))
# Prepare template variables
configs = []
for config in BuildConfig.select(self.env, include_inactive=True):
configs.append({
'name':
config.name,
'label':
config.label or config.name,
'active':
config.active,
'branch':
config.branch,
'href':
req.href.admin('bitten', 'configs', config.name),
'recipe':
config.recipe and True or False
})
data['configs'] = configs
add_stylesheet(req, 'bitten/admin.css')
add_script(req, 'common/js/suggest.js')
return 'bitten_admin_configs.html', data
# Internal methods
def _activate_configs(self, req):
req.perm.assert_permission('BUILD_MODIFY')
active = req.args.get('active') or []
active = isinstance(active, list) and active or [active]
db = self.env.get_db_cnx()
for config in list(
BuildConfig.select(self.env, db=db, include_inactive=True)):
config.active = config.name in active
config.update(db=db)
db.commit()
def _create_config(self, req):
req.perm.assert_permission('BUILD_CREATE')
config = BuildConfig(self.env)
warnings = self._update_config(req, config)
if warnings:
if len(warnings) == 1:
raise TracError(warnings[0], 'Add Configuration')
else:
raise TracError('Errors: %s' % ' '.join(warnings),
'Add Configuration')
return config
def _remove_configs(self, req):
req.perm.assert_permission('BUILD_DELETE')
sel = req.args.get('sel')
if not sel:
raise TracError('No configuration selected')
sel = isinstance(sel, list) and sel or [sel]
db = self.env.get_db_cnx()
for name in sel:
config = BuildConfig.fetch(self.env, name, db=db)
if not config:
raise TracError('Configuration %r not found' % name)
config.delete(db=db)
db.commit()
def _update_config(self, req, config):
warnings = []
req.perm.assert_permission('BUILD_MODIFY')
name = req.args.get('name')
if not name:
warnings.append('Missing required field "name".')
if name and not re.match(r'^[\w.-]+$', name):
warnings.append('The field "name" may only contain letters, '
'digits, periods, or dashes.')
branch = req.args.get('branch', '')
repos = self.env.get_repository(req.authname)
try:
if not (branch in [b[0] for b in repos.git.get_branches()
]) and branch != 'master':
raise TracError('Invalid Repository Branch "%s".' % branch)
except (AssertionError, TracError), e:
warnings.append('Invalid Repository Branch "%s".' % branch)
recipe_xml = req.args.get('recipe', '')
if recipe_xml:
try:
Recipe(xmlio.parse(recipe_xml)).validate()
except xmlio.ParseError, e:
warnings.append('Failure parsing recipe: %s.' % unicode(e))
except InvalidRecipeError, e:
warnings.append('Invalid Recipe: %s.' % unicode(e))
def test_validate_step_with_empty_id(self):
xml = xmlio.parse('<build><step id=""><cmd/></step></build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
elem = xmlio.parse(req.read())
except xmlio.ParseError, e:
self.log.error('Error parsing build step result: %s',
e,
exc_info=True)
self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
stepname = elem.attr['step']
# we should have created this step previously; if it hasn't,
# the master and slave are processing steps out of order.
step = BuildStep.fetch(self.env, build=build.id, name=stepname)
if not step:
self._send_error(req, HTTP_CONFLICT,
'Build step has not been created.')
recipe = Recipe(xmlio.parse(config.recipe))
index = None
current_step = None
for num, recipe_step in enumerate(recipe):
if recipe_step.id == stepname:
index = num
current_step = recipe_step
if index is None:
self._send_error(req, HTTP_FORBIDDEN,
'No such build step' % stepname)
last_step = index == num
self.log.debug(
'Slave %s (build %d) completed step %d (%s) with '
'status %s', build.slave, build.id, index, stepname,
elem.attr['status'])
try:
node = repos.get_node(repos_path, max_rev)
assert node.isdir, '%s is not a directory' % node.path
except (AssertionError, TracError), e:
warnings.append('Invalid Repository Path "%s".' % path)
if min_rev:
try:
repos.get_node(repos_path, min_rev)
except TracError, e:
warnings.append('Invalid Oldest Revision: %s.' % unicode(e))
recipe_xml = req.args.get('recipe', '')
if recipe_xml:
try:
Recipe(xmlio.parse(recipe_xml)).validate()
except xmlio.ParseError, e:
warnings.append('Failure parsing recipe: %s.' % unicode(e))
except InvalidRecipeError, e:
warnings.append('Invalid Recipe: %s.' % unicode(e))
config.name = name
config.path = __multirepos__ and path or repos.normalize_path(path)
config.recipe = recipe_xml
config.min_rev = min_rev
config.max_rev = max_rev
config.label = req.args.get('label', config.name)
config.description = req.args.get('description', '')
if warnings: # abort
return warnings
def test_validate_bad_root(self):
xml = xmlio.parse('<foo></foo>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def test_validate_child_not_step(self):
xml = xmlio.parse('<build><foo/></build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def test_empty_recipe(self):
xml = xmlio.parse('<build/>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertEqual(self.basedir, recipe.ctxt.basedir)
steps = list(recipe)
self.assertEqual(0, len(steps))
def test_validate_step_without_commands(self):
xml = xmlio.parse('<build><step id="test"/></build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)
def test_validate_step_with_command_children(self):
xml = xmlio.parse('<build><step id="test">'
'<somecmd><child1/><child2/></somecmd>'
'</step></build>')
recipe = Recipe(xml, basedir=self.basedir)
self.assertRaises(InvalidRecipeError, recipe.validate)