Esempio n. 1
0
    def _process_build_initiation(self, req, config, build):
        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)
        build.started = int(time.time())
        build.last_activity = build.started
        build.update()

        for listener in BuildSystem(self.env).listeners:
            listener.build_started(build)

        repos_name, repos, repos_path = get_repos(self.env, config.path,
                                                  req.authname)
        xml = xmlio.parse(config.recipe)
        xml.attr['path'] = config.path
        xml.attr['revision'] = build.rev
        xml.attr['config'] = config.name
        xml.attr['build'] = str(build.id)
        target_platform = TargetPlatform.fetch(self.env, build.platform)
        xml.attr['platform'] = target_platform.name
        xml.attr['name'] = build.slave
        xml.attr['form_token'] = req.form_token  # For posting attachments
        xml.attr['reponame'] = repos_name != '(default)' and repos_name or ''
        xml.attr['repopath'] = repos_path.strip('/')
        body = str(xml)

        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)

        # create the first step, mark it as in-progress.

        recipe = Recipe(xmlio.parse(config.recipe))
        stepname = recipe.__iter__().next().id

        step = self._start_new_step(build, stepname)
        step.insert()

        self._send_response(req,
                            200,
                            body,
                            headers={
                                'Content-Type':
                                'application/x-bitten+xml',
                                'Content-Length':
                                str(len(body)),
                                'Content-Disposition':
                                'attachment; filename=recipe_%s_r%s.xml' %
                                (config.name, build.rev)
                            })
Esempio n. 2
0
 def test_validate_successful(self):
     xml = xmlio.parse('<build>'
                       '<step id="foo"><somecmd></somecmd></step>'
                       '<step id="bar"><othercmd></othercmd></step>'
                       '</build>')
     recipe = Recipe(xml, basedir=self.basedir)
     recipe.validate()
Esempio n. 3
0
 def test_ParsedElement_encoding(self):
     u = u'<root foo="øüé€"/>'
     s = '<root foo="\xc3\xb8\xc3\xbc\xc3\xa9\xe2\x82\xac"/>'
     self.assertEquals(u, s.decode('utf-8'))
     # unicode input
     x = xmlio.parse(u)
     out_u = str(x)
     self.assertEquals(out_u, s)
     self.assertEquals(out_u.decode('utf-8'), u)
     # utf-8 input
     x = xmlio.parse(s)
     out_s = str(x)
     self.assertEquals(out_s, s)
     self.assertEquals(out_s.decode('utf-8'), u)
     # identical results
     self.assertEquals(out_u, out_s)
Esempio n. 4
0
    def _execute_build(self, build_url, fileobj):
        build_id = build_url and int(build_url.split('/')[-1]) or 0
        xml = xmlio.parse(fileobj)
        basedir = ''
        try:
            recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir), 
                            self.config)
            basedir = recipe.ctxt.basedir
            log.debug('Running build in directory %s' % basedir)
            if not os.path.exists(basedir):
                os.mkdir(basedir)

            for step in recipe:
                log.info('Executing build step %r', step.id)
                if not self._execute_step(build_url, recipe, step):
                    log.warning('Stopping build due to failure')
                    break
            else:
                log.info('Build completed')
            if self.dry_run:
                self._cancel_build(build_url)
        finally:
            if not self.keep_files and os.path.isdir(basedir):
                log.debug('Removing build directory %s' % basedir)
                _rmtree(basedir)
            if self.single_build:
                log.info('Exiting after single build completed.')
                raise ExitSlave(EX_OK)
Esempio n. 5
0
 def test_validate_successful(self):
     xml = xmlio.parse('<build>'
                       '<step id="foo"><somecmd></somecmd></step>'
                       '<step id="bar"><othercmd></othercmd></step>'
                       '</build>')
     recipe = Recipe(xml, basedir=self.basedir)
     recipe.validate()
Esempio n. 6
0
    def _execute_build(self, build_url, fileobj):
        build_id = build_url and int(build_url.split('/')[-1]) or 0
        xml = xmlio.parse(fileobj)
        try:
            recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir), 
                            self.config)
            basedir = recipe.ctxt.basedir
            log.debug('Running build in directory %s' % basedir)
            if not os.path.exists(basedir):
                os.mkdir(basedir)

            for step in recipe:
                log.info('Executing build step %r', step.id)
                if not self._execute_step(build_url, recipe, step):
                    log.warning('Stopping build due to failure')
                    break
            else:
                log.info('Build completed')
            if self.dry_run:
                self._cancel_build(build_url)
        finally:
            if not self.keep_files:
                log.debug('Removing build directory %s' % basedir)
                _rmtree(basedir)
            if self.single_build:
                log.info('Exiting after single build completed.')
                raise ExitSlave(EX_OK)
Esempio n. 7
0
 def test_ParsedElement_encoding(self):
     u = u'<root foo="øüé€"/>'
     s = '<root foo="\xc3\xb8\xc3\xbc\xc3\xa9\xe2\x82\xac"/>'
     self.assertEquals(u, s.decode('utf-8'))
     # unicode input
     x = xmlio.parse(u)
     out_u = str(x)
     self.assertEquals(out_u, s)
     self.assertEquals(out_u.decode('utf-8'), u)
     # utf-8 input
     x = xmlio.parse(s)
     out_s = str(x)
     self.assertEquals(out_s, s)
     self.assertEquals(out_s.decode('utf-8'), u)
     # identical results
     self.assertEquals(out_u, out_s)
Esempio n. 8
0
def coverage(ctxt, file_=None):
    assert file_, 'Missing required attribute "file"'
    try:
        fileobj = file(ctxt.resolve(file_), 'r')
        try:
            results = xmlio.Fragment()
            for child in xmlio.parse(fileobj).children():
                cover = xmlio.Element('coverage')
                for name, value in child.attr.items():
                    if name == 'file':
                        value = os.path.realpath(value)
                        if value.startswith(ctxt.basedir):
                            value = value[len(ctxt.basedir) + 1:]
                            value = value.replace(os.sep, '/')
                        else:
                            continue
                    cover.attr[name] = value
                    for grandchild in child.children():
                        cover.append(xmlio.Element(grandchild.name)[
                            grandchild.gettext()
                        ])
                results.append(cover)
            ctxt.report('nosecoverage', results)
        finally:
            fileobj.close()
    except IOError, e:
        log.warning('Error opening coverage results file (%s)', e)
Esempio n. 9
0
    def _execute_build(self, build_url, fileobj):
        build_id = build_url and int(build_url.split('/')[-1]) or 0
        xml = xmlio.parse(fileobj)
        basedir = ''
        try:
            if not self.local:
                keepalive_thread = KeepAliveThread(self.opener, build_url,
                                                   self.single_build,
                                                   self.keepalive_interval)
                keepalive_thread.start()
            recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir),
                            self.config)
            basedir = recipe.ctxt.basedir
            log.debug('Running build in directory %s' % basedir)
            if not os.path.exists(basedir):
                os.mkdir(basedir)

            for step in recipe:
                try:
                    log.info('Executing build step %r, onerror = %s', step.id,
                             step.onerror)
                    if not self._execute_step(build_url, recipe, step):
                        log.warning('Stopping build due to failure')
                        break
                except Exception, e:
                    log.error(
                        'Exception raised processing step %s. Reraising %s',
                        step.id, e)
                    raise
            else:
Esempio n. 10
0
 def _process_build_step(self, req, config, build):
     try:
         elem = xmlio.parse(req.read())
     except xmlio.ParseError, e:
         self.log.error('Error parsing build step result: %s', e,
                        exc_info=True)
         raise HTTPBadRequest('XML parser error')
Esempio n. 11
0
 def test_validate_step_with_duplicate_id(self):
     xml = xmlio.parse('<build>'
                       '<step id="test"><somecmd></somecmd></step>'
                       '<step id="test"><othercmd></othercmd></step>'
                       '</build>')
     recipe = Recipe(xml, basedir=self.basedir)
     self.assertRaises(InvalidRecipeError, recipe.validate)
Esempio n. 12
0
 def _process_build_step(self, req, config, build):
     try:
         elem = xmlio.parse(req.read())
     except xmlio.ParseError, e:
         self.log.error('Error parsing build step result: %s', e,
                        exc_info=True)
         self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
Esempio n. 13
0
    def _process_build_initiation(self, req, config, build):
        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)
        build.started = int(time.time())
        build.update()

        for listener in BuildSystem(self.env).listeners:
            listener.build_started(build)

        xml = xmlio.parse(config.recipe)
        xml.attr['path'] = config.path
        xml.attr['revision'] = build.rev
        xml.attr['config'] = config.name
        xml.attr['build'] = str(build.id)
        target_platform = TargetPlatform.fetch(self.env, build.platform)
        xml.attr['platform'] = target_platform.name
        body = str(xml)

        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)

        req.send_response(200)
        req.send_header('Content-Type', 'application/x-bitten+xml')
        req.send_header('Content-Length', str(len(body)))
        req.send_header('Content-Disposition',
                        'attachment; filename=recipe_%s_r%s.xml' %
                        (config.name, build.rev))
        req.write(body)
        raise RequestDone
Esempio n. 14
0
 def report_file(self, category=None, file_=None):
     """Read report data from a file and record it.
     
     :param category: the name of the category of the report
     :param file\_: the path to the file containing the report data, relative
                    to the base directory
     """
     filename = self.resolve(file_)
     try:
         fileobj = file(filename, 'r')
         try:
             xml_elem = xmlio.Fragment()
             for child in xmlio.parse(fileobj).children():
                 child_elem = xmlio.Element(child.name, **dict([
                     (name, value) for name, value in child.attr.items()
                     if value is not None
                 ]))
                 xml_elem.append(child_elem[
                     [xmlio.Element(grandchild.name)[grandchild.gettext()]
                     for grandchild in child.children()]
                 ])
             self.output.append((Recipe.REPORT, category, None, xml_elem))
         finally:
             fileobj.close()
     except xmlio.ParseError, e:
         self.error('Failed to parse %s report at %s: %s'
                    % (category, filename, e))
Esempio n. 15
0
    def _process_build_initiation(self, req, config, build):
        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)
        build.started = int(time.time())
        build.update()

        for listener in BuildSystem(self.env).listeners:
            listener.build_started(build)

        xml = xmlio.parse(config.recipe)
        xml.attr['branch'] = config.branch
        xml.attr['revision'] = build.rev
        xml.attr['config'] = config.name
        xml.attr['build'] = str(build.id)
        target_platform = TargetPlatform.fetch(self.env, build.platform)
        xml.attr['platform'] = target_platform.name
        xml.attr['name'] = build.slave
        body = str(xml)

        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)

        self._send_response(req,
                            200,
                            body,
                            headers={
                                'Content-Type':
                                'application/x-bitten+xml',
                                'Content-Length':
                                str(len(body)),
                                'Content-Disposition':
                                'attachment; filename=recipe_%s_r%s.xml' %
                                (config.name, build.rev)
                            })
Esempio n. 16
0
def coverage(ctxt, file_=None):
    """Extract data from a Phing code coverage report."""
    assert file_, 'Missing required attribute "file"'
    try:
        summary_file = file(ctxt.resolve(file_), 'r')
        try:
            coverage = xmlio.Fragment()
            for package in xmlio.parse(summary_file).children('package'):
                for cls in package.children('class'):
                    statements = float(cls.attr['statementcount'])
                    covered = float(cls.attr['statementscovered'])
                    if statements:
                        percentage = covered / statements * 100
                    else:
                        percentage = 100
                    class_coverage = xmlio.Element('coverage',
                        name=cls.attr['name'],
                        lines=int(statements),
                        percentage=percentage
                    )
                    source = list(cls.children())[0]
                    if 'sourcefile' in source.attr:
                        sourcefile = os.path.realpath(source.attr['sourcefile'])
                        if sourcefile.startswith(ctxt.basedir):
                            sourcefile = sourcefile[len(ctxt.basedir) + 1:]
                        sourcefile = sourcefile.replace(os.sep, '/')
                        class_coverage.attr['file'] = sourcefile
                    coverage.append(class_coverage)
        finally:
            summary_file.close()
        ctxt.report('coverage', coverage)
    except IOError, e:
        ctxt.log('Error opening coverage summary file (%s)' % e)
Esempio n. 17
0
    def _process_build_initiation(self, req, config, build):
        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)
        build.started = int(time.time())
        build.update()

        for listener in BuildSystem(self.env).listeners:
            listener.build_started(build)

        xml = xmlio.parse(config.recipe)
        xml.attr['branch'] = config.branch
        xml.attr['revision'] = build.rev
        xml.attr['config'] = config.name
        xml.attr['build'] = str(build.id)
        target_platform = TargetPlatform.fetch(self.env, build.platform)
        xml.attr['platform'] = target_platform.name
        xml.attr['name'] = build.slave
        body = str(xml)

        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)

        self._send_response(req, 200, body, headers={
                    'Content-Type': 'application/x-bitten+xml',
                    'Content-Length': str(len(body)),
                    'Content-Disposition':
                        'attachment; filename=recipe_%s_r%s.xml' %
                        (config.name, build.rev)})
Esempio n. 18
0
 def report_file(self, category=None, file_=None):
     """Read report data from a file and record it.
     
     :param category: the name of the category of the report
     :param file\_: the path to the file containing the report data, relative
                    to the base directory
     """
     filename = self.resolve(file_)
     try:
         fileobj = file(filename, 'r')
         try:
             xml_elem = xmlio.Fragment()
             for child in xmlio.parse(fileobj).children():
                 child_elem = xmlio.Element(
                     child.name,
                     **dict([(name, value)
                             for name, value in child.attr.items()
                             if value is not None]))
                 xml_elem.append(child_elem[[
                     xmlio.Element(grandchild.name)[grandchild.gettext()]
                     for grandchild in child.children()
                 ]])
             self.output.append((Recipe.REPORT, category, None, xml_elem))
         finally:
             fileobj.close()
     except xmlio.ParseError, e:
         self.error('Failed to parse %s report at %s: %s' %
                    (category, filename, e))
Esempio n. 19
0
 def test_validate_step_with_duplicate_id(self):
     xml = xmlio.parse('<build>'
                       '<step id="test"><somecmd></somecmd></step>'
                       '<step id="test"><othercmd></othercmd></step>'
                       '</build>')
     recipe = Recipe(xml, basedir=self.basedir)
     self.assertRaises(InvalidRecipeError, recipe.validate)
Esempio n. 20
0
 def _process_build_step(self, req, config, build):
     try:
         elem = xmlio.parse(req.read())
     except xmlio.ParseError, e:
         self.log.error('Error parsing build step result: %s',
                        e,
                        exc_info=True)
         self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
Esempio n. 21
0
def junit(ctxt, file_=None, srcdir=None):
    """Extract test results from a JUnit XML report.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path to the JUnit XML test results; may contain globbing
                  wildcards for matching multiple results files
    :param srcdir: name of the directory containing the test sources, used to
                   link test results to the corresponding source files
    """
    assert file_, 'Missing required attribute "file"'
    try:
        total, failed = 0, 0
        results = xmlio.Fragment()
        for path in glob(ctxt.resolve(file_)):
            fileobj = file(path, 'r')
            try:
                for testcase in xmlio.parse(fileobj).children('testcase'):
                    test = xmlio.Element('test')
                    test.attr['fixture'] = testcase.attr['classname']
                    test.attr['name'] = testcase.attr['name']
                    if 'time' in testcase.attr:
                        test.attr['duration'] = testcase.attr['time']
                    if srcdir is not None:
                        cls = testcase.attr['classname'].split('.')
                        test.attr['file'] = posixpath.join(srcdir, *cls) + \
                                            '.java'

                    result = list(testcase.children())
                    if result:
                        test.attr['status'] = result[0].name
                        # Sometimes the traceback isn't prefixed with the
                        # exception type and message, so add it in if needed
                        tracebackprefix = "%s: %s" % (
                            result[0].attr['type'], result[0].attr['message'])
                        if result[0].gettext().startswith(tracebackprefix):
                            test.append(
                                xmlio.Element('traceback')[
                                    result[0].gettext()])
                        else:
                            test.append(
                                xmlio.Element('traceback')["\n".join(
                                    (tracebackprefix, result[0].gettext()))])
                        failed += 1
                    else:
                        test.attr['status'] = 'success'

                    results.append(test)
                    total += 1
            finally:
                fileobj.close()
        if failed:
            ctxt.error('%d of %d test%s failed' %
                       (failed, total, total != 1 and 's' or ''))
        ctxt.report('test', results)
    except IOError, e:
        log.warning('Error opening JUnit results file (%s)', e)
Esempio n. 22
0
 def test_parse(self):
     """Tests that simple test data is parsed correctly"""
     s = """<build xmlns:c="http://bitten.edgewall.org/tools/c">
              <step id="build" description="Configure and build">
                <c:configure />
              </step>\
            </build>"""
     x = xmlio.parse(s)
     assert x.name == "build"
Esempio n. 23
0
 def test_parse(self):
     """Tests that simple test data is parsed correctly"""
     s = """<build xmlns:c="http://bitten.edgewall.org/tools/c">
              <step id="build" description="Configure and build">
                <c:configure />
              </step>\
            </build>"""
     x = xmlio.parse(s)
     assert x.name == "build"
Esempio n. 24
0
 def test_empty_step(self):
     xml = xmlio.parse('<build>'
                       ' <step id="foo" description="Bar"></step>'
                       '</build>')
     recipe = Recipe(xml, basedir=self.basedir)
     steps = list(recipe)
     self.assertEqual(1, len(steps))
     self.assertEqual('foo', steps[0].id)
     self.assertEqual('Bar', steps[0].description)
     self.assertEqual('fail', steps[0].onerror)
Esempio n. 25
0
 def test_empty_step(self):
     xml = xmlio.parse('<build>'
                       ' <step id="foo" description="Bar"></step>'
                       '</build>')
     recipe = Recipe(xml, basedir=self.basedir)
     steps = list(recipe)
     self.assertEqual(1, len(steps))
     self.assertEqual('foo', steps[0].id)
     self.assertEqual('Bar', steps[0].description)
     self.assertEqual('fail', steps[0].onerror)
Esempio n. 26
0
 def test_onerror_override(self):
     xml = xmlio.parse('<build onerror="ignore">'
                       ' <step id="foo" description="Bar" onerror="continue"></step>'
                       '</build>')
     recipe = Recipe(xml, basedir=self.basedir)
     steps = list(recipe)
     self.assertEqual(1, len(steps))
     self.assertEqual('foo', steps[0].id)
     self.assertEqual('Bar', steps[0].description)
     self.assertEqual('continue', steps[0].onerror)
Esempio n. 27
0
def junit(ctxt, file_=None, srcdir=None):
    """Extract test results from a JUnit XML report.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path to the JUnit XML test results; may contain globbing
                  wildcards for matching multiple results files
    :param srcdir: name of the directory containing the test sources, used to
                   link test results to the corresponding source files
    """
    assert file_, 'Missing required attribute "file"'
    try:
        total, failed = 0, 0
        results = xmlio.Fragment()
        for path in glob(ctxt.resolve(file_)):
            fileobj = file(path, 'r')
            try:
                for testcase in xmlio.parse(fileobj).children('testcase'):
                    test = xmlio.Element('test')
                    test.attr['fixture'] = testcase.attr['classname']
                    test.attr['name'] = testcase.attr['name']
                    if 'time' in testcase.attr:
                        test.attr['duration'] = testcase.attr['time']
                    if srcdir is not None:
                        cls = testcase.attr['classname'].split('.')
                        test.attr['file'] = posixpath.join(srcdir, *cls) + \
                                            '.java'

                    result = list(testcase.children())
                    if result:
                        test.attr['status'] = result[0].name
                        # Sometimes the traceback isn't prefixed with the
                        # exception type and message, so add it in if needed
                        tracebackprefix = "%s: %s" % (result[0].attr['type'],
                                                      result[0].attr['message'])
                        if result[0].gettext().startswith(tracebackprefix):
                            test.append(xmlio.Element('traceback')[
                                        result[0].gettext()])
                        else:
                            test.append(xmlio.Element('traceback')[
                                        "\n".join((tracebackprefix, result[0].gettext()))])
                        failed += 1
                    else:
                        test.attr['status'] = 'success'

                    results.append(test)
                    total += 1
            finally:
                fileobj.close()
        if failed:
            ctxt.error('%d of %d test%s failed' % (failed, total,
                       total != 1 and 's' or ''))
        ctxt.report('test', results)
    except IOError, e:
        log.warning('Error opening JUnit results file (%s)', e)
Esempio n. 28
0
    def _process_build_creation(self, req):
        queue = BuildQueue(self.env, build_all=self.build_all, 
                           stabilize_wait=self.stabilize_wait,
                           timeout=self.slave_timeout)
        queue.populate()

        try:
            elem = xmlio.parse(req.read())
        except xmlio.ParseError, e:
            self.log.error('Error parsing build initialization request: %s', e,
                           exc_info=True)
            raise HTTPBadRequest('XML parser error')
Esempio n. 29
0
    def _process_build_initiation(self, req, config, build):
        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)
        build.started = int(time.time())
        build.last_activity = build.started
        build.update()

        for listener in BuildSystem(self.env).listeners:
            listener.build_started(build)

        xml = xmlio.parse(config.recipe)
        xml.attr['path'] = config.path
        xml.attr['revision'] = build.rev
        xml.attr['config'] = config.name
        xml.attr['build'] = str(build.id)
        target_platform = TargetPlatform.fetch(self.env, build.platform)
        xml.attr['platform'] = target_platform.name
        xml.attr['name'] = build.slave
        xml.attr['form_token'] = req.form_token # For posting attachments
        body = str(xml)

        self.log.info('Build slave %r initiated build %d', build.slave,
                      build.id)

        # create the first step, mark it as in-progress.

        recipe = Recipe(xmlio.parse(config.recipe))
        stepname = recipe.__iter__().next().id

        step = self._start_new_step(build, stepname)
        step.insert()

        self._send_response(req, 200, body, headers={
                    'Content-Type': 'application/x-bitten+xml',
                    'Content-Length': str(len(body)),
                    'Content-Disposition':
                        'attachment; filename=recipe_%s_r%s.xml' %
                        (config.name, build.rev)})
Esempio n. 30
0
def phpunit(ctxt, file_=None):
    """Extract test results from a PHPUnit XML report."""
    assert file_, 'Missing required attribute "file"'

    def _process_testsuite(testsuite, results, parent_file=''):
        for testcase in testsuite.children():
            if testcase.name == 'testsuite':
                _process_testsuite(testcase, results,
                        parent_file=testcase.attr.get('file', parent_file))
                continue
            test = xmlio.Element('test')
            test.attr['fixture'] = testsuite.attr['name']
            test.attr['name'] = testcase.attr['name']
            test.attr['duration'] = testcase.attr['time']
            result = list(testcase.children())
            if result:
                test.append(xmlio.Element('traceback')[
                    result[0].gettext()
                ])
                test.attr['status'] = result[0].name
            else:
                test.attr['status'] = 'success'
            if 'file' in testsuite.attr or parent_file:
                testfile = os.path.realpath(
                                    testsuite.attr.get('file', parent_file))
                if testfile.startswith(ctxt.basedir):
                    testfile = testfile[len(ctxt.basedir) + 1:]
                testfile = testfile.replace(os.sep, '/')
                test.attr['file'] = testfile
            results.append(test)

    try:
        total, failed = 0, 0
        results = xmlio.Fragment()
        fileobj = file(ctxt.resolve(file_), 'r')
        try:
            for testsuite in xmlio.parse(fileobj).children('testsuite'):
                total += int(testsuite.attr['tests'])
                failed += int(testsuite.attr['failures']) + \
                            int(testsuite.attr['errors'])

                _process_testsuite(testsuite, results)
        finally:
            fileobj.close()
        if failed:
            ctxt.error('%d of %d test%s failed' % (failed, total,
                        total != 1 and 's' or ''))
        ctxt.report('test', results)
    except IOError, e:
        ctxt.log('Error opening PHPUnit results file (%s)' % e)
Esempio n. 31
0
    def _process_build_creation(self, req, slave_token):
        queue = BuildQueue(self.env,
                           build_all=self.build_all,
                           stabilize_wait=self.stabilize_wait,
                           timeout=self.slave_timeout)
        queue.populate()

        try:
            elem = xmlio.parse(req.read())
        except xmlio.ParseError, e:
            self.log.error('Error parsing build initialization request: %s',
                           e,
                           exc_info=True)
            self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
Esempio n. 32
0
def phpunit(ctxt, file_=None):
    """Extract test results from a PHPUnit XML report."""
    assert file_, 'Missing required attribute "file"'

    def _process_testsuite(testsuite, results, parent_file=''):
        for testcase in testsuite.children():
            if testcase.name == 'testsuite':
                _process_testsuite(testcase,
                                   results,
                                   parent_file=testcase.attr.get(
                                       'file', parent_file))
                continue
            test = xmlio.Element('test')
            test.attr['fixture'] = testsuite.attr['name']
            test.attr['name'] = testcase.attr['name']
            test.attr['duration'] = testcase.attr['time']
            result = list(testcase.children())
            if result:
                test.append(xmlio.Element('traceback')[result[0].gettext()])
                test.attr['status'] = result[0].name
            else:
                test.attr['status'] = 'success'
            if 'file' in testsuite.attr or parent_file:
                testfile = os.path.realpath(
                    testsuite.attr.get('file', parent_file))
                if testfile.startswith(ctxt.basedir):
                    testfile = testfile[len(ctxt.basedir) + 1:]
                testfile = testfile.replace(os.sep, '/')
                test.attr['file'] = testfile
            results.append(test)

    try:
        total, failed = 0, 0
        results = xmlio.Fragment()
        fileobj = file(ctxt.resolve(file_), 'r')
        try:
            for testsuite in xmlio.parse(fileobj).children('testsuite'):
                total += int(testsuite.attr['tests'])
                failed += int(testsuite.attr['failures']) + \
                            int(testsuite.attr['errors'])

                _process_testsuite(testsuite, results)
        finally:
            fileobj.close()
        if failed:
            ctxt.error('%d of %d test%s failed' %
                       (failed, total, total != 1 and 's' or ''))
        ctxt.report('test', results)
    except IOError, e:
        ctxt.log('Error opening PHPUnit results file (%s)' % e)
Esempio n. 33
0
def cobertura(ctxt, file_=None):
    """Extract test coverage information from a Cobertura XML report.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path to the Cobertura XML output
    """
    assert file_, 'Missing required attribute "file"'

    coverage = xmlio.Fragment()
    doc = xmlio.parse(open(ctxt.resolve(file_)))
    srcdir = [
        s.gettext().strip() for ss in doc.children('sources')
        for s in ss.children('source')
    ][0]

    classes = [
        cls for pkgs in doc.children('packages')
        for pkg in pkgs.children('package') for clss in pkg.children('classes')
        for cls in clss.children('class')
    ]

    counters = {}
    class_names = {}

    for cls in classes:
        filename = cls.attr['filename'].replace(os.sep, '/')
        name = cls.attr['name']
        if not '$' in name:  # ignore internal classes
            class_names[filename] = name
        counter = counters.get(filename)
        if counter is None:
            counter = counters[filename] = _LineCounter()
        lines = [
            l for ls in cls.children('lines') for l in ls.children('line')
        ]
        for line in lines:
            counter[line.attr['number']] = line.attr['hits']

    for filename, name in class_names.iteritems():
        counter = counters[filename]
        module = xmlio.Element('coverage',
                               name=name,
                               file=posixpath.join(srcdir, filename),
                               lines=counter.num_lines,
                               percentage=counter.percentage)
        module.append(xmlio.Element('line_hits')[counter.line_hits])
        coverage.append(module)
    ctxt.report('coverage', coverage)
Esempio n. 34
0
def cobertura(ctxt, file_=None):
    """Extract test coverage information from a Cobertura XML report.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path to the Cobertura XML output
    """
    assert file_, 'Missing required attribute "file"'

    coverage = xmlio.Fragment()
    doc = xmlio.parse(open(ctxt.resolve(file_)))
    srcdir = [s.gettext().strip() for ss in doc.children('sources')
                                  for s in ss.children('source')][0]

    classes = [cls for pkgs in doc.children('packages')
                   for pkg in pkgs.children('package')
                   for clss in pkg.children('classes')
                   for cls in clss.children('class')]

    counters = {}
    class_names = {}

    for cls in classes:
        filename = cls.attr['filename'].replace(os.sep, '/')
        name = cls.attr['name']
        if not '$' in name: # ignore internal classes
            class_names[filename] = name
        counter = counters.get(filename)
        if counter is None:
            counter = counters[filename] = _LineCounter()
        lines = [l for ls in cls.children('lines')
                   for l in ls.children('line')]
        for line in lines:
            counter[line.attr['number']] = line.attr['hits']

    for filename, name in class_names.iteritems():
        counter = counters[filename]
        module = xmlio.Element('coverage', name=name,
                               file=posixpath.join(srcdir, filename),
                               lines=counter.num_lines,
                               percentage=counter.percentage)
        module.append(xmlio.Element('line_hits')[counter.line_hits])
        coverage.append(module)
    ctxt.report('coverage', coverage)
Esempio n. 35
0
def unittest(ctxt, file_=None):
    """Extract data from a unittest results file in XML format.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: name of the file containing the test results
    """
    assert file_, 'Missing required attribute "file"'

    try:
        fileobj = file(ctxt.resolve(file_), 'r')
        try:
            total, failed = 0, 0
            results = xmlio.Fragment()
            for child in xmlio.parse(fileobj).children():
                test = xmlio.Element('test')
                for name, value in child.attr.items():
                    if name == 'file':
                        value = os.path.realpath(value)
                        if value.startswith(ctxt.basedir):
                            value = value[len(ctxt.basedir) + 1:]
                            value = value.replace(os.sep, '/')
                        else:
                            continue
                    test.attr[name] = value
                    if name == 'status' and value in ('error', 'failure'):
                        failed += 1
                for grandchild in child.children():
                    test.append(xmlio.Element(grandchild.name)[
                        grandchild.gettext()
                    ])
                results.append(test)
                total += 1
            if failed:
                ctxt.error('%d of %d test%s failed' % (failed, total,
                           total != 1 and 's' or ''))
            ctxt.report('test', results)
        finally:
            fileobj.close()
    except IOError, e:
        log.warning('Error opening unittest results file (%s)', e)
Esempio n. 36
0
    def _execute_build(self, build_url, fileobj):
        build_id = build_url and int(build_url.split("/")[-1]) or 0
        xml = xmlio.parse(fileobj)
        basedir = ""
        try:
            if not self.local:
                keepalive_thread = KeepAliveThread(self.opener, build_url, self.single_build, self.keepalive_interval)
                keepalive_thread.start()
            recipe = Recipe(xml, os.path.join(self.work_dir, self.build_dir), self.config)
            basedir = recipe.ctxt.basedir
            log.debug("Running build in directory %s" % basedir)
            if not os.path.exists(basedir):
                os.mkdir(basedir)

            for step in recipe:
                try:
                    log.info("Executing build step %r, onerror = %s", step.id, step.onerror)
                    if not self._execute_step(build_url, recipe, step):
                        log.warning("Stopping build due to failure")
                        break
                except Exception, e:
                    log.error("Exception raised processing step %s. Reraising %s", step.id, e)
                    raise
            else:
Esempio n. 37
0
def ant(ctxt, file_=None, target=None, keep_going=False, args=None):
    """Run an Ant build.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: name of the Ant build file
    :param target: name of the target that should be executed (optional)
    :param keep_going: whether Ant should keep going when errors are encountered
    :param args: additional arguments to pass to Ant
    """
    executable = 'ant'
    ant_home = ctxt.config.get_dirpath('ant.home')
    if ant_home:
        executable = os.path.join(ant_home, 'bin', 'ant')

    java_home = ctxt.config.get_dirpath('java.home')
    if java_home:
        os.environ['JAVA_HOME'] = java_home

    logfile = tempfile.NamedTemporaryFile(prefix='ant_log', suffix='.xml')
    logfile.close()
    if args:
        args = shlex.split(args)
    else:
        args = []
    args += [
        '-noinput', '-listener', 'org.apache.tools.ant.XmlLogger',
        '-Dant.XmlLogger.stylesheet.uri', '""', '-DXmlLogger.file',
        logfile.name
    ]
    if file_:
        args += ['-buildfile', ctxt.resolve(file_)]
    if keep_going:
        args.append('-keep-going')
    if target:
        args.append(target)

    shell = False
    if os.name == 'nt':
        # Need to execute ant.bat through a shell on Windows
        shell = True

    cmdline = CommandLine(executable, args, cwd=ctxt.basedir, shell=shell)
    for out, err in cmdline.execute():
        if out is not None:
            log.info(out)
        if err is not None:
            log.error(err)

    error_logged = False
    log_elem = xmlio.Fragment()
    try:
        xml_log = xmlio.parse(file(logfile.name, 'r'))

        def collect_log_messages(node):
            for child in node.children():
                if child.name == 'message':
                    if child.attr['priority'] == 'debug':
                        continue
                    log_elem.append(
                        xmlio.Element('message', level=child.attr['priority'])
                        [child.gettext().replace(ctxt.basedir + os.sep,
                                                 '').replace(ctxt.basedir,
                                                             '')])
                else:
                    collect_log_messages(child)

        collect_log_messages(xml_log)

        if 'error' in xml_log.attr:
            ctxt.error(xml_log.attr['error'])
            error_logged = True

    except xmlio.ParseError, e:
        log.warning('Error parsing Ant XML log file (%s)', e)
Esempio n. 38
0
def cppunit(ctxt, file_=None, srcdir=None):
    """Collect CppUnit XML data.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path of the file containing the CppUnit results; may contain
                  globbing wildcards to match multiple files
    :param srcdir: name of the directory containing the source files, used to
                   link the test results to the corresponding files
    """
    assert file_, 'Missing required attribute "file"'

    try:
        fileobj = file(ctxt.resolve(file_), 'r')
        try:
            total, failed = 0, 0
            results = xmlio.Fragment()
            for group in xmlio.parse(fileobj):
                if group.name not in ('FailedTests', 'SuccessfulTests'):
                    continue
                for child in group.children():
                    test = xmlio.Element('test')
                    name = child.children('Name').next().gettext()
                    if '::' in name:
                        parts = name.split('::')
                        test.attr['fixture'] = '::'.join(parts[:-1])
                        name = parts[-1]
                    test.attr['name'] = name

                    for location in child.children('Location'):
                        for file_elem in location.children('File'):
                            filepath = file_elem.gettext()
                            if srcdir is not None:
                                filepath = posixpath.join(srcdir, filepath)
                            test.attr['file'] = filepath
                            break
                        for line_elem in location.children('Line'):
                            test.attr['line'] = line_elem.gettext()
                            break
                        break

                    if child.name == 'FailedTest':
                        for message in child.children('Message'):
                            test.append(xmlio.Element('traceback')[
                                message.gettext()
                            ])
                        test.attr['status'] = 'failure'
                        failed += 1
                    else:
                        test.attr['status'] = 'success'

                    results.append(test)
                    total += 1

            if failed:
                ctxt.error('%d of %d test%s failed' % (failed, total,
                           total != 1 and 's' or ''))

            ctxt.report('test', results)

        finally:
            fileobj.close()

    except IOError, e:
        log.warning('Error opening CppUnit results file (%s)', e)
Esempio n. 39
0
def junit(ctxt, file_=None, srcdir=None):
    """Extract test results from a JUnit XML report.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path to the JUnit XML test results; may contain globbing
                  wildcards for matching multiple results files
    :param srcdir: name of the directory containing the test sources, used to
                   link test results to the corresponding source files
    """
    assert file_, 'Missing required attribute "file"'
    try:
        total, failed = 0, 0
        results = xmlio.Fragment()
        for path in glob(ctxt.resolve(file_)):
            fileobj = file(path, 'r')
            try:
                output = xmlio.parse(fileobj)
            finally:
                fileobj.close()
            if output.name == 'testsuites':
                # top level wrapper for testsuites
                testcases = []
                for t_suite in output.children('testsuite'):
                    testcases.extend(
                        [t_case for t_case in t_suite.children('testcase')])
            else:
                testcases = [t_case for t_case in output.children('testcase')]
            for testcase in testcases:
                test = xmlio.Element('test')
                test.attr['fixture'] = testcase.attr['classname']
                test.attr['name'] = testcase.attr['name']
                if 'time' in testcase.attr:
                    test.attr['duration'] = testcase.attr['time']
                if srcdir is not None:
                    cls = testcase.attr['classname'].split('.')
                    test.attr['file'] = posixpath.join(srcdir, *cls) + \
                                        '.java'

                result = list(testcase.children())
                if result:
                    junit_status = result[0].name
                    test.append(
                        xmlio.Element('traceback')[_fix_traceback(result)])
                    if junit_status == 'skipped':
                        test.attr['status'] = 'ignore'
                    elif junit_status == 'error':
                        test.attr['status'] = 'error'
                        failed += 1
                    else:
                        test.attr['status'] = 'failure'
                        failed += 1
                else:
                    test.attr['status'] = 'success'

                results.append(test)
                total += 1
        if failed:
            ctxt.error('%d of %d test%s failed' %
                       (failed, total, total != 1 and 's' or ''))
        ctxt.report('test', results)
    except IOError, e:
        log.warning('Error opening JUnit results file (%s)', e)
Esempio n. 40
0
        try:
            node = repos.get_node(repos_path, max_rev)
            assert node.isdir, '%s is not a directory' % node.path
        except (AssertionError, TracError), e:
            warnings.append('Invalid Repository Path "%s".' % path)

        if min_rev:
            try:
                repos.get_node(repos_path, min_rev)
            except TracError, e:
                warnings.append('Invalid Oldest Revision: %s.' % unicode(e))

        recipe_xml = req.args.get('recipe', '')
        if recipe_xml:
            try:
                Recipe(xmlio.parse(recipe_xml)).validate()
            except xmlio.ParseError, e:
                warnings.append('Failure parsing recipe: %s.' % unicode(e))
            except InvalidRecipeError, e:
                warnings.append('Invalid Recipe: %s.' % unicode(e))

        config.name = name
        config.path = __multirepos__ and path or repos.normalize_path(path)
        config.recipe = recipe_xml
        config.min_rev = min_rev
        config.max_rev = max_rev
        config.label = req.args.get('label', config.name)
        config.description = req.args.get('description', '')

        if warnings:  # abort
            return warnings
Esempio n. 41
0
            elem = xmlio.parse(req.read())
        except xmlio.ParseError, e:
            self.log.error('Error parsing build step result: %s',
                           e,
                           exc_info=True)
            self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
        stepname = elem.attr['step']

        # we should have created this step previously; if it hasn't,
        # the master and slave are processing steps out of order.
        step = BuildStep.fetch(self.env, build=build.id, name=stepname)
        if not step:
            self._send_error(req, HTTP_CONFLICT,
                             'Build step has not been created.')

        recipe = Recipe(xmlio.parse(config.recipe))
        index = None
        current_step = None
        for num, recipe_step in enumerate(recipe):
            if recipe_step.id == stepname:
                index = num
                current_step = recipe_step
        if index is None:
            self._send_error(req, HTTP_FORBIDDEN,
                             'No such build step' % stepname)
        last_step = index == num

        self.log.debug(
            'Slave %s (build %d) completed step %d (%s) with '
            'status %s', build.slave, build.id, index, stepname,
            elem.attr['status'])
Esempio n. 42
0
def xmldb_to_db(env, db):
    """Migrate report data from Berkeley DB XML to SQL database.

    Depending on the number of reports stored, this might take rather long.
    After the upgrade is done, the bitten.dbxml file (and any BDB XML log files)
    may be deleted. BDB XML is no longer used by Bitten.
    """
    from bitten.util import xmlio
    try:
        from bsddb3 import db as bdb
        import dbxml
    except ImportError:
        return

    dbfile = os.path.join(env.path, 'db', 'bitten.dbxml')
    if not os.path.isfile(dbfile):
        return

    dbenv = bdb.DBEnv()
    dbenv.open(os.path.dirname(dbfile),
               bdb.DB_CREATE | bdb.DB_INIT_LOCK | bdb.DB_INIT_LOG |
               bdb.DB_INIT_MPOOL | bdb.DB_INIT_TXN, 0)

    mgr = dbxml.XmlManager(dbenv, 0)
    xtn = mgr.createTransaction()
    container = mgr.openContainer(dbfile, dbxml.DBXML_TRANSACTIONAL)

    def get_pylint_items(xml):
        for problems_elem in xml.children('problems'):
            for problem_elem in problems_elem.children('problem'):
                item = {'type': 'problem'}
                item.update(problem_elem.attr)
                yield item

    def get_trace_items(xml):
        for cov_elem in xml.children('coverage'):
            item = {'type': 'coverage', 'name': cov_elem.attr['module'],
                    'file': cov_elem.attr['file'],
                    'percentage': cov_elem.attr['percentage']}
            lines = 0
            line_hits = []
            for line_elem in cov_elem.children('line'):
                lines += 1
                line_hits.append(line_elem.attr['hits'])
            item['lines'] = lines
            item['line_hits'] = ' '.join(line_hits)
            yield item

    def get_unittest_items(xml):
        for test_elem in xml.children('test'):
            item = {'type': 'test'}
            item.update(test_elem.attr)
            for child_elem in test_elem.children():
                item[child_elem.name] = child_elem.gettext()
            yield item

    qc = mgr.createQueryContext()
    for value in mgr.query(xtn, 'collection("%s")/report' % dbfile, qc, 0):
        doc = value.asDocument()
        metaval = dbxml.XmlValue()
        build, step = None, None
        if doc.getMetaData('', 'build', metaval):
            build = metaval.asNumber()
        if doc.getMetaData('', 'step', metaval):
            step = metaval.asString()

        report_types = {'pylint':   ('lint', get_pylint_items),
                        'trace':    ('coverage', get_trace_items),
                        'unittest': ('test', get_unittest_items)}
        xml = xmlio.parse(value.asString())
        report_type = xml.attr['type']
        category, get_items = report_types[report_type]
        sys.stderr.write('.')
        sys.stderr.flush()

        items = list(get_items(xml))

        cursor = db.cursor()
        cursor.execute("SELECT bitten_report.id FROM bitten_report "
                       "WHERE build=%s AND step=%s AND category=%s",
                       (build, step, category))
        rows = cursor.fetchall()
        if rows:
            # Duplicate report, skip
            continue

        cursor.execute("INSERT INTO bitten_report "
                       "(build,step,category,generator) VALUES (%s,%s,%s,%s)",
                       (build, step, category, report_type))
        id = db.get_last_id(cursor, 'bitten_report')

        for idx, item in enumerate(items):
            cursor.executemany("INSERT INTO bitten_report_item "
                               "(report,item,name,value) VALUES (%s,%s,%s,%s)",
                               [(id, idx, key, value) for key, value
                                in item.items()])

    sys.stderr.write('\n')
    sys.stderr.flush()

    xtn.abort()
    container.close()
    dbenv.close(0)
Esempio n. 43
0
def coverage(ctxt, file_=None):
    """Extract data from Phing or PHPUnit code coverage report."""
    assert file_, 'Missing required attribute "file"'

    def _process_phing_coverage(ctxt, element, coverage):
        for cls in element.children('class'):
            statements = float(cls.attr['statementcount'])
            covered = float(cls.attr['statementscovered'])
            if statements:
                percentage = covered / statements * 100
            else:
                percentage = 100
            class_coverage = xmlio.Element('coverage',
                                           name=cls.attr['name'],
                                           lines=int(statements),
                                           percentage=percentage)
            source = list(cls.children())[0]
            if 'sourcefile' in source.attr:
                sourcefile = os.path.realpath(source.attr['sourcefile'])
                if sourcefile.startswith(ctxt.basedir):
                    sourcefile = sourcefile[len(ctxt.basedir) + 1:]
                sourcefile = sourcefile.replace(os.sep, '/')
                class_coverage.attr['file'] = sourcefile
            coverage.append(class_coverage)

    def _process_phpunit_coverage(ctxt, element, coverage):
        for cls in element._node.getElementsByTagName('class'):
            sourcefile = cls.parentNode.getAttribute('name')
            if not os.path.isabs(sourcefile):
                sourcefile = os.path.join(ctxt.basedir, sourcefile)
            if sourcefile.startswith(ctxt.basedir):
                loc, ncloc = 0, 0.0
                for line in cls.parentNode.getElementsByTagName('line'):
                    if str(line.getAttribute('type')) == 'stmt':
                        loc += 1
                        if int(line.getAttribute('count')) == 0:
                            ncloc += 1
                if loc > 0:
                    percentage = 100 - (ncloc / loc * 100)
                else:
                    percentage = 100

                if sourcefile.startswith(ctxt.basedir):
                    sourcefile = sourcefile[len(ctxt.basedir) + 1:]
                class_coverage = xmlio.Element('coverage',
                                               name=cls.getAttribute('name'),
                                               lines=int(loc),
                                               percentage=int(percentage),
                                               file=sourcefile.replace(
                                                   os.sep, '/'))
                coverage.append(class_coverage)

    try:
        summary_file = file(ctxt.resolve(file_), 'r')
        summary = xmlio.parse(summary_file)
        coverage = xmlio.Fragment()
        try:
            for element in summary.children():
                if element.name == 'package':
                    _process_phing_coverage(ctxt, element, coverage)
                elif element.name == 'project':
                    _process_phpunit_coverage(ctxt, element, coverage)
        finally:
            summary_file.close()
        ctxt.report('coverage', coverage)
    except IOError, e:
        ctxt.log('Error opening coverage summary file (%s)' % e)
Esempio n. 44
0
 def test_validate_step_with_command_children(self):
     xml = xmlio.parse('<build><step id="test">'
                       '<somecmd><child1/><child2/></somecmd>'
                       '</step></build>')
     recipe = Recipe(xml, basedir=self.basedir)
     self.assertRaises(InvalidRecipeError, recipe.validate)
Esempio n. 45
0
 def test_validate_step_without_commands(self):
     xml = xmlio.parse('<build><step id="test"/></build>')
     recipe = Recipe(xml, basedir=self.basedir)
     self.assertRaises(InvalidRecipeError, recipe.validate)
Esempio n. 46
0
 def test_validate_step_with_empty_id(self):
     xml = xmlio.parse('<build><step id=""><cmd/></step></build>')
     recipe = Recipe(xml, basedir=self.basedir)
     self.assertRaises(InvalidRecipeError, recipe.validate)
Esempio n. 47
0
 def test_validate_child_not_step(self):
     xml = xmlio.parse('<build><foo/></build>')
     recipe = Recipe(xml, basedir=self.basedir)
     self.assertRaises(InvalidRecipeError, recipe.validate)
Esempio n. 48
0
 def test_validate_bad_root(self):
     xml = xmlio.parse('<foo></foo>')
     recipe = Recipe(xml, basedir=self.basedir)
     self.assertRaises(InvalidRecipeError, recipe.validate)
Esempio n. 49
0
def cppunit(ctxt, file_=None, srcdir=None):
    """Collect CppUnit XML data.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path of the file containing the CppUnit results; may contain
                  globbing wildcards to match multiple files
    :param srcdir: name of the directory containing the source files, used to
                   link the test results to the corresponding files
    """
    assert file_, 'Missing required attribute "file"'

    try:
        fileobj = file(ctxt.resolve(file_), "r")
        try:
            total, failed = 0, 0
            results = xmlio.Fragment()
            for group in xmlio.parse(fileobj):
                if group.name not in ("FailedTests", "SuccessfulTests"):
                    continue
                for child in group.children():
                    test = xmlio.Element("test")
                    name = child.children("Name").next().gettext()
                    if "::" in name:
                        parts = name.split("::")
                        test.attr["fixture"] = "::".join(parts[:-1])
                        name = parts[-1]
                    test.attr["name"] = name

                    for location in child.children("Location"):
                        for file_elem in location.children("File"):
                            filepath = file_elem.gettext()
                            if srcdir is not None:
                                filepath = posixpath.join(srcdir, filepath)
                            test.attr["file"] = filepath
                            break
                        for line_elem in location.children("Line"):
                            test.attr["line"] = line_elem.gettext()
                            break
                        break

                    if child.name == "FailedTest":
                        for message in child.children("Message"):
                            test.append(xmlio.Element("traceback")[message.gettext()])
                        test.attr["status"] = "failure"
                        failed += 1
                    else:
                        test.attr["status"] = "success"

                    results.append(test)
                    total += 1

            if failed:
                ctxt.error("%d of %d test%s failed" % (failed, total, total != 1 and "s" or ""))

            ctxt.report("test", results)

        finally:
            fileobj.close()

    except IOError, e:
        log.warning("Error opening CppUnit results file (%s)", e)
Esempio n. 50
0
def cunit(ctxt, file_=None, srcdir=None):
    """Collect CUnit XML data.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path of the file containing the CUnit results; may contain
                  globbing wildcards to match multiple files
    :param srcdir: name of the directory containing the source files, used to
                   link the test results to the corresponding files
    """
    assert file_, 'Missing required attribute "file"'

    try:
        fileobj = file(ctxt.resolve(file_), "r")
        try:
            total, failed = 0, 0
            results = xmlio.Fragment()
            log_elem = xmlio.Fragment()

            def info(msg):
                log.info(msg)
                log_elem.append(xmlio.Element("message", level="info")[msg])

            def warning(msg):
                log.warning(msg)
                log_elem.append(xmlio.Element("message", level="warning")[msg])

            def error(msg):
                log.error(msg)
                log_elem.append(xmlio.Element("message", level="error")[msg])

            for node in xmlio.parse(fileobj):
                if node.name != "CUNIT_RESULT_LISTING":
                    continue
                for suiteRun in node.children("CUNIT_RUN_SUITE"):
                    for suite in suiteRun.children():
                        if suite.name not in ("CUNIT_RUN_SUITE_SUCCESS", "CUNIT_RUN_SUITE_FAILURE"):
                            warning("Unknown node: %s" % suite.name)
                            continue
                        suiteName = suite.children("SUITE_NAME").next().gettext()
                        info("%s [%s]" % ("*" * (57 - len(suiteName)), suiteName))
                        for record in suite.children("CUNIT_RUN_TEST_RECORD"):
                            for result in record.children():
                                if result.name not in ("CUNIT_RUN_TEST_SUCCESS", "CUNIT_RUN_TEST_FAILURE"):
                                    continue
                                testName = result.children("TEST_NAME").next().gettext()
                                info("Running %s..." % testName)
                                test = xmlio.Element("test")
                                test.attr["fixture"] = suiteName
                                test.attr["name"] = testName
                                if result.name == "CUNIT_RUN_TEST_FAILURE":
                                    error(
                                        "%s(%d): %s"
                                        % (
                                            result.children("FILE_NAME").next().gettext(),
                                            int(result.children("LINE_NUMBER").next().gettext()),
                                            result.children("CONDITION").next().gettext(),
                                        )
                                    )
                                    test.attr["status"] = "failure"
                                    failed += 1
                                else:
                                    test.attr["status"] = "success"

                                results.append(test)
                                total += 1

            if failed:
                ctxt.error("%d of %d test%s failed" % (failed, total, total != 1 and "s" or ""))

            ctxt.report("test", results)
            ctxt.log(log_elem)

        finally:
            fileobj.close()

    except IOError, e:
        log.warning("Error opening CUnit results file (%s)", e)
Esempio n. 51
0
 def _gather(self, method, url, body=None, headers=None):
     self.results.append(xmlio.parse(body))
     return DummyResponse(201)
Esempio n. 52
0
        max_rev = req.args.get('max_rev') or None
        try:
            node = repos.get_node(path, max_rev)
            assert node.isdir, '%s is not a directory' % node.path
        except (AssertionError, TracError), e:
            raise TracError(unicode(e), 'Invalid Repository Path')
        if req.args.get('min_rev'):
            try:
                repos.get_node(path, req.args.get('min_rev'))
            except TracError, e:
                raise TracError(unicode(e), 'Invalid Oldest Revision')

        recipe_xml = req.args.get('recipe', '')
        if recipe_xml:
            try:
                Recipe(xmlio.parse(recipe_xml)).validate()
            except xmlio.ParseError, e:
                raise TracError('Failure parsing recipe: %s' % e,
                                'Invalid Recipe')
            except InvalidRecipeError, e:
                raise TracError(unicode(e), 'Invalid Recipe')

        config.name = name
        config.path = repos.normalize_path(path)
        config.recipe = recipe_xml
        config.min_rev = req.args.get('min_rev')
        config.max_rev = req.args.get('max_rev')
        config.label = req.args.get('label', config.name)
        config.description = req.args.get('description', '')

        if config.exists:
Esempio n. 53
0
    def test_transform(self):
        src_file = file(self.ctxt.resolve('src.xml'), 'w')
        try:
            src_file.write("""<doc>
<title>Document Title</title>
<section>
<title>Section Title</title>
<para>This is a test.</para>
<note>This is a note.</note>
</section>
</doc>
""")
        finally:
            src_file.close()

        style_file = file(self.ctxt.resolve('style.xsl'), 'w')
        try:
            style_file.write("""<xsl:stylesheet version="1.0"
    xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
    xmlns="http://www.w3.org/TR/xhtml1/strict">
 <xsl:template match="doc">
  <html>
   <head>
    <title><xsl:value-of select="title"/></title>
   </head>
   <body>
    <xsl:apply-templates/>
   </body>
  </html>
 </xsl:template>
 <xsl:template match="doc/title">
  <h1><xsl:apply-templates/></h1>
 </xsl:template>
 <xsl:template match="section/title">
  <h2><xsl:apply-templates/></h2>
 </xsl:template>
 <xsl:template match="para">
  <p><xsl:apply-templates/></p>
 </xsl:template>
 <xsl:template match="note">
  <p class="note"><b>NOTE: </b><xsl:apply-templates/></p>
 </xsl:template>
</xsl:stylesheet>
""")
        finally:
            style_file.close()

        xmltools.transform(self.ctxt, src='src.xml', dest='dest.xml',
                           stylesheet='style.xsl')

        dest_file = file(self.ctxt.resolve('dest.xml'))
        try:
            dest = xmlio.parse(dest_file)
        finally:
            dest_file.close()

        self.assertEqual('html', dest.name)
        self.assertEqual('http://www.w3.org/TR/xhtml1/strict', dest.namespace)
        children = list(dest.children())
        self.assertEqual(2, len(children))
        self.assertEqual('head', children[0].name)
        head_children = list(children[0].children())
        self.assertEqual(1, len(head_children))
        self.assertEqual('title', head_children[0].name)
        self.assertEqual('Document Title', head_children[0].gettext())
        self.assertEqual('body', children[1].name)
        body_children = list(children[1].children())
        self.assertEqual(4, len(body_children))
        self.assertEqual('h1', body_children[0].name)
        self.assertEqual('Document Title', body_children[0].gettext())
        self.assertEqual('h2', body_children[1].name)
        self.assertEqual('Section Title', body_children[1].gettext())
        self.assertEqual('p', body_children[2].name)
        self.assertEqual('This is a test.', body_children[2].gettext())
        self.assertEqual('p', body_children[3].name)
        self.assertEqual('note', body_children[3].attr['class'])
        self.assertEqual('This is a note.', body_children[3].gettext())
Esempio n. 54
0
class BuildConfigurationsAdminPageProvider(Component):
    """Web administration panel for configuring the build master."""

    implements(IAdminPanelProvider)

    # IAdminPanelProvider methods

    def get_admin_panels(self, req):
        if req.perm.has_permission('BUILD_MODIFY'):
            yield ('bitten', 'Builds', 'configs', 'Configurations')

    def render_admin_panel(self, req, cat, page, path_info):
        data = {}

        # Analyze url
        try:
            config_name, platform_id = path_info.split('/', 1)
        except:
            config_name = path_info
            platform_id = None

        if config_name:  # Existing build config
            warnings = []
            if platform_id or (
                    # Editing or creating one of the config's target platforms
                    req.method == 'POST' and 'new' in req.args):

                if platform_id:  # Editing target platform
                    platform_id = int(platform_id)
                    platform = TargetPlatform.fetch(self.env, platform_id)

                    if req.method == 'POST':
                        if 'cancel' in req.args or \
                                self._update_platform(req, platform):
                            req.redirect(
                                req.abs_href.admin(cat, page, config_name))
                else:  # creating target platform
                    platform = self._create_platform(req, config_name)
                    req.redirect(
                        req.abs_href.admin(cat, page, config_name,
                                           platform.id))

                # Set up template variables
                data['platform'] = {
                    'id':
                    platform.id,
                    'name':
                    platform.name,
                    'exists':
                    platform.exists,
                    'rules': [{
                        'property': propname,
                        'pattern': pattern
                    } for propname, pattern in platform.rules] or [('', '')]
                }

            else:  # Editing existing build config itself
                config = BuildConfig.fetch(self.env, config_name)
                platforms = list(
                    TargetPlatform.select(self.env, config=config.name))

                if req.method == 'POST':
                    if 'remove' in req.args:  # Remove selected platforms
                        self._remove_platforms(req)
                        add_notice(req, "Target Platform(s) Removed.")
                        req.redirect(req.abs_href.admin(
                            cat, page, config.name))

                    elif 'save' in req.args:  # Save this build config
                        warnings = self._update_config(req, config)

                    if not warnings:
                        add_notice(req, "Configuration Saved.")
                        req.redirect(req.abs_href.admin(
                            cat, page, config.name))

                    for warning in warnings:
                        add_warning(req, warning)

                # FIXME: Deprecation notice for old namespace.
                # Remove notice code when migration to new namespace is complete
                if 'http://bitten.cmlenz.net/tools/' in config.recipe:
                    add_notice(
                        req, "Recipe uses a deprecated namespace. "
                        "Replace 'http://bitten.cmlenz.net/tools/' with "
                        "'http://bitten.edgewall.org/tools/'.")

                # Add a notice if configuration is not active
                if not warnings and not config.active and config.recipe:
                    add_notice(
                        req, "Configuration is not active. Activate "
                        "from main 'Configurations' listing to enable it.")

                # Prepare template variables
                data['config'] = {
                    'name':
                    config.name,
                    'label':
                    config.label or config.name,
                    'active':
                    config.active,
                    'branch':
                    config.branch,
                    'description':
                    config.description,
                    'recipe':
                    config.recipe,
                    'platforms': [{
                        'name':
                        platform.name,
                        'id':
                        platform.id,
                        'href':
                        req.href.admin('bitten', 'configs', config.name,
                                       platform.id),
                        'rules': [{
                            'property': propname,
                            'pattern': pattern
                        } for propname, pattern in platform.rules]
                    } for platform in platforms]
                }

        else:  # At the top level build config list
            if req.method == 'POST':
                if 'add' in req.args:  # Add build config
                    config = self._create_config(req)
                    req.redirect(req.abs_href.admin(cat, page, config.name))

                elif 'remove' in req.args:  # Remove selected build configs
                    self._remove_configs(req)

                elif 'apply' in req.args:  # Update active state of configs
                    self._activate_configs(req)
                req.redirect(req.abs_href.admin(cat, page))

            # Prepare template variables
            configs = []
            for config in BuildConfig.select(self.env, include_inactive=True):
                configs.append({
                    'name':
                    config.name,
                    'label':
                    config.label or config.name,
                    'active':
                    config.active,
                    'branch':
                    config.branch,
                    'href':
                    req.href.admin('bitten', 'configs', config.name),
                    'recipe':
                    config.recipe and True or False
                })
            data['configs'] = configs

        add_stylesheet(req, 'bitten/admin.css')
        add_script(req, 'common/js/suggest.js')
        return 'bitten_admin_configs.html', data

    # Internal methods

    def _activate_configs(self, req):
        req.perm.assert_permission('BUILD_MODIFY')

        active = req.args.get('active') or []
        active = isinstance(active, list) and active or [active]

        db = self.env.get_db_cnx()
        for config in list(
                BuildConfig.select(self.env, db=db, include_inactive=True)):
            config.active = config.name in active
            config.update(db=db)
        db.commit()

    def _create_config(self, req):
        req.perm.assert_permission('BUILD_CREATE')

        config = BuildConfig(self.env)
        warnings = self._update_config(req, config)
        if warnings:
            if len(warnings) == 1:
                raise TracError(warnings[0], 'Add Configuration')
            else:
                raise TracError('Errors: %s' % ' '.join(warnings),
                                'Add Configuration')
        return config

    def _remove_configs(self, req):
        req.perm.assert_permission('BUILD_DELETE')

        sel = req.args.get('sel')
        if not sel:
            raise TracError('No configuration selected')
        sel = isinstance(sel, list) and sel or [sel]

        db = self.env.get_db_cnx()
        for name in sel:
            config = BuildConfig.fetch(self.env, name, db=db)
            if not config:
                raise TracError('Configuration %r not found' % name)
            config.delete(db=db)
        db.commit()

    def _update_config(self, req, config):
        warnings = []
        req.perm.assert_permission('BUILD_MODIFY')

        name = req.args.get('name')
        if not name:
            warnings.append('Missing required field "name".')
        if name and not re.match(r'^[\w.-]+$', name):
            warnings.append('The field "name" may only contain letters, '
                            'digits, periods, or dashes.')

        branch = req.args.get('branch', '')
        repos = self.env.get_repository(req.authname)
        try:
            if not (branch in [b[0] for b in repos.git.get_branches()
                               ]) and branch != 'master':
                raise TracError('Invalid Repository Branch "%s".' % branch)
        except (AssertionError, TracError), e:
            warnings.append('Invalid Repository Branch "%s".' % branch)

        recipe_xml = req.args.get('recipe', '')
        if recipe_xml:
            try:
                Recipe(xmlio.parse(recipe_xml)).validate()
            except xmlio.ParseError, e:
                warnings.append('Failure parsing recipe: %s.' % unicode(e))
            except InvalidRecipeError, e:
                warnings.append('Invalid Recipe: %s.' % unicode(e))
Esempio n. 55
0
class BuildMaster(Component):
    """Trac request handler implementation for the build master."""

    implements(IRequestHandler)

    # Configuration options

    adjust_timestamps = BoolOption(
        'bitten',
        'adjust_timestamps',
        False,
        doc="""Whether the timestamps of builds should be adjusted to be close
        to the timestamps of the corresponding changesets.""")

    build_all = BoolOption(
        'bitten',
        'build_all',
        False,
        doc="""Whether to request builds of older revisions even if a younger
        revision has already been built.""")

    stabilize_wait = IntOption(
        'bitten',
        'stabilize_wait',
        0,
        doc="""The time in seconds to wait for the repository to stabilize before
        queuing up a new build.  This allows time for developers to check in
        a group of related changes back to back without spawning multiple
        builds.""")

    slave_timeout = IntOption(
        'bitten',
        'slave_timeout',
        3600,
        doc="""The time in seconds after which a build is cancelled if the slave
        does not report progress.""")

    logs_dir = Option(
        'bitten',
        'logs_dir',
        "log/bitten",
        doc=
        """The directory on the server in which client log files will be stored."""
    )

    quick_status = BoolOption(
        'bitten',
        'quick_status',
        False,
        doc="""Whether to show the current build status within the Trac main
            navigation bar. '''Note:''' The feature requires expensive database and
            repository checks for every page request, and should not be enabled
            if the project has a large repository or uses a non-Subversion
            repository such as Mercurial or Git.""")

    def __init__(self):
        self.env.systeminfo.append(
            ('Bitten', __import__('bitten', ['__version__']).__version__))

    # IRequestHandler methods

    def match_request(self, req):
        match = re.match(r'/builds(?:/(\d+)(?:/(\w+)/([^/]+)?)?)?$',
                         req.path_info)
        if match:
            if match.group(1):
                req.args['id'] = match.group(1)
                req.args['collection'] = match.group(2)
                req.args['member'] = match.group(3)
            return True

    def process_request(self, req):
        req.perm.assert_permission('BUILD_EXEC')

        if 'trac_auth' in req.incookie:
            slave_token = req.incookie['trac_auth'].value
        else:
            slave_token = req.session.sid

        if 'id' not in req.args:
            if req.method != 'POST':
                self._send_response(
                    req, body='Only POST allowed for build creation.')
            return self._process_build_creation(req, slave_token)

        build = Build.fetch(self.env, req.args['id'])
        if not build:
            self._send_error(req, HTTP_NOT_FOUND,
                             'No such build (%s)' % req.args['id'])

        build_token = build.slave_info.get('token', '')
        if build_token != slave_token:
            self._send_error(req, HTTP_CONFLICT,
                          'Token mismatch (wrong slave): slave=%s, build=%s' \
                               % (slave_token, build_token))

        config = BuildConfig.fetch(self.env, build.config)

        if not req.args['collection']:
            if req.method == 'DELETE':
                return self._process_build_cancellation(req, config, build)
            else:
                return self._process_build_initiation(req, config, build)

        if req.method != 'POST':
            self._send_error(req, HTTP_METHOD_NOT_ALLOWED,
                             'Method %s not allowed' % req.method)

        if req.args['collection'] == 'steps':
            return self._process_build_step(req, config, build)
        elif req.args['collection'] == 'attach':
            return self._process_attachment(req, config, build)
        elif req.args['collection'] == 'keepalive':
            return self._process_keepalive(req, config, build)
        else:
            self._send_error(
                req, HTTP_NOT_FOUND,
                "No such collection '%s'" % req.args['collection'])

    # Internal methods

    def _send_response(self, req, code=200, body='', headers=None):
        """ Formats and sends the response, raising ``RequestDone``. """
        if isinstance(body, unicode):
            body = body.encode('utf-8')
        req.send_response(code)
        headers = headers or {}
        headers.setdefault('Content-Length', len(body))
        for header in headers:
            req.send_header(header, headers[header])
        req.write(body)
        raise RequestDone

    def _send_error(self, req, code=500, message=''):
        """ Formats and sends the error, raising ``RequestDone``. """
        headers = {
            'Content-Type': 'text/plain',
            'Content-Length': str(len(message))
        }
        self._send_response(req, code, body=message, headers=headers)

    def _process_build_creation(self, req, slave_token):
        queue = BuildQueue(self.env,
                           build_all=self.build_all,
                           stabilize_wait=self.stabilize_wait,
                           timeout=self.slave_timeout)
        try:
            queue.populate()
        except AssertionError, e:
            self.log.error(e.message, exc_info=True)
            self._send_error(req, HTTP_BAD_REQUEST, e.message)

        try:
            elem = xmlio.parse(req.read())
        except xmlio.ParseError, e:
            self.log.error('Error parsing build initialization request: %s',
                           e,
                           exc_info=True)
            self._send_error(req, HTTP_BAD_REQUEST, 'XML parser error')
Esempio n. 56
0
def _get_cases(fileobj):
    for testsuite in xmlio.parse(fileobj).children('test-suite'):
        for suite in _parse_suite(testsuite):
            yield suite
Esempio n. 57
0
def cunit (ctxt, file_=None, srcdir=None):
    """Collect CUnit XML data.
    
    :param ctxt: the build context
    :type ctxt: `Context`
    :param file\_: path of the file containing the CUnit results; may contain
                  globbing wildcards to match multiple files
    :param srcdir: name of the directory containing the source files, used to
                   link the test results to the corresponding files
    """
    assert file_, 'Missing required attribute "file"'

    try:
        fileobj = file(ctxt.resolve(file_), 'r')
        try:
            total, failed = 0, 0
            results = xmlio.Fragment()
            log_elem = xmlio.Fragment()
            def info (msg):
                log.info (msg)
                log_elem.append (xmlio.Element ('message', level='info')[msg])
            def warning (msg):
                log.warning (msg)
                log_elem.append (xmlio.Element ('message', level='warning')[msg])
            def error (msg):
                log.error (msg)
                log_elem.append (xmlio.Element ('message', level='error')[msg])
            for node in xmlio.parse(fileobj):
                if node.name != 'CUNIT_RESULT_LISTING':
                    continue
                for suiteRun in node.children ('CUNIT_RUN_SUITE'):
                    for suite in suiteRun.children():
                        if suite.name not in ('CUNIT_RUN_SUITE_SUCCESS', 'CUNIT_RUN_SUITE_FAILURE'):
                            warning ("Unknown node: %s" % suite.name)
                            continue
                        suiteName = suite.children ('SUITE_NAME').next().gettext()
                        info ("%s [%s]" % ("*" * (57 - len (suiteName)), suiteName))
                        for record in suite.children ('CUNIT_RUN_TEST_RECORD'):
                            for result in record.children():
                                if result.name not in ('CUNIT_RUN_TEST_SUCCESS', 'CUNIT_RUN_TEST_FAILURE'):
                                    continue
                                testName = result.children ('TEST_NAME').next().gettext()
                                info ("Running %s..." % testName);
                                test = xmlio.Element('test')
                                test.attr['fixture'] = suiteName
                                test.attr['name'] = testName
                                if result.name == 'CUNIT_RUN_TEST_FAILURE':
                                    error ("%s(%d): %s"
                                               % (result.children ('FILE_NAME').next().gettext(),
                                                  int (result.children ('LINE_NUMBER').next().gettext()),
                                                  result.children ('CONDITION').next().gettext()))
                                    test.attr['status'] = 'failure'
                                    failed += 1
                                else:
                                    test.attr['status'] = 'success'

                                results.append(test)
                                total += 1

            if failed:
                ctxt.error('%d of %d test%s failed' % (failed, total,
                           total != 1 and 's' or ''))

            ctxt.report('test', results)
            ctxt.log (log_elem)

        finally:
            fileobj.close()

    except IOError, e:
        log.warning('Error opening CUnit results file (%s)', e)