Beispiel #1
0
 def _step_AdditionalProperties(self):
     self.addStep(
         SetProperty(name="modifying configures",
                     doStepIf=lambda step:
                     (step.getProperty("slavename") in self.wineslaves),
                     property="gcc_config_args",
                     command=[
                         "echo",
                         Property("gcc_config_args", default=""),
                         "--build=i686-w64-mingw32"
                     ]))
     self.addStep(
         SetProperty(name="modifying configures",
                     doStepIf=lambda step:
                     (step.getProperty("slavename") in self.wineslaves),
                     property="binutils_config_args",
                     command=[
                         "echo",
                         Property("binutils_config_args", default=""),
                         "--build=i686-w64-mingw32"
                     ]))
     self.addStep(
         SetProperty(name="modifying configures",
                     doStepIf=lambda step:
                     (step.getProperty("slavename") in self.wineslaves),
                     property="mingw_config_args",
                     command=[
                         "echo",
                         Property("mingw_config_args", default=""),
                         "--build=i686-w64-mingw32"
                     ]))
Beispiel #2
0
def getAnnotatedBuildFactory(script,
                             clean=False,
                             depends_on_projects=None,
                             env=None,
                             timeout=1200):
    """
    Returns a new build factory that uses AnnotatedCommand, which
    allows the build to be run by version-controlled scripts that do
    not require a buildmaster restart to update.
    """

    f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
                         llvm_srcdir='llvm.src')

    if clean:
        f.addStep(SetProperty(property='clean', command='echo 1'))

    # We normally use the clean property to indicate that we want a
    # clean build, but AnnotatedCommand uses the clobber property
    # instead. Therefore, set clobber if clean is set to a truthy
    # value.  This will cause AnnotatedCommand to set
    # BUILDBOT_CLOBBER=1 in the environment, which is how we
    # communicate to the script that we need a clean build.
    f.addStep(
        SetProperty(
            property='clobber',
            command='echo 1',
            doStepIf=lambda step: step.build.getProperty('clean', False)))

    merged_env = {
        'TERM': 'dumb'  # Be cautious and disable color output from all tools.
    }
    if env is not None:
        # Overwrite pre-set items with the given ones, so user can set
        # anything.
        merged_env.update(env)

    scripts_dir = "annotated"
    f.addStep(
        SVN(name='update-annotate-scripts',
            mode='update',
            svnurl='http://llvm.org/svn/llvm-project/zorg/trunk/'
            'zorg/buildbot/builders/annotated',
            workdir=scripts_dir,
            alwaysUseLatest=True))

    # Explicitly use '/' as separator, because it works on *nix and Windows.
    script_path = "../%s/%s" % (scripts_dir, script)
    f.addStep(
        AnnotatedCommand(name="annotate",
                         description="annotate",
                         timeout=timeout,
                         haltOnFailure=True,
                         command=WithProperties(
                             "python %(script)s --jobs=%(jobs:-)s",
                             script=lambda _: script_path),
                         env=merged_env))
    return f
    def __init__(self, python, package, **kw):
        SetProperty.__init__(
            self,
            name="check-%s-version" % package,
            description="checking %s version" % package,
            command=[python, '-c', self.src % dict(package=package)],
            extract_fn=self._extractVersion, **kw)

        self.package = package
Beispiel #4
0
    def __init__(self, python, package, **kw):
        SetProperty.__init__(
            self,
            name="check-%s-version" % package,
            description="checking %s version" % package,
            command=[python, '-c', self.src % dict(package=package)],
            extract_fn=self._extractVersion, **kw)

        self.package = package
    def __init__(self, **kwargs):

        factory.BuildFactory.__init__(self, **kwargs)

        self.addStep(
            SetProperty(
                property="optionTargetOs",
                doStepIf=lambda step: step.build.getProperty("target-os"),
                command=["echo", "--os"]))
        self.addStep(
            SetProperty(property="optionPath",
                        doStepIf=lambda step: step.build.getProperty("path"),
                        command=["echo", "--path"]))
        self.addStep(
            SetProperty(
                property="optionDatestamp",
                doStepIf=lambda step: step.build.getProperty("datestamp"),
                command=["echo", "--datestamp"]))

        # this assumes the following properties have been set:
        # masterdir - the path to the master configuration (has tarballs + scripts + passwords)
        # filename  - the name of the file to upload (relative to masterdir)
        # destname  - how to name the file on the server
        self.addStep(
            ShellCommand(name="sfupload",
                         description=["uploading"],
                         descriptionDone=["upload"],
                         doStepIf=lambda step:
                         (step.build.getProperty("is_nightly")),
                         workdir=Property("masterdir"),
                         command=[
                             "scripts/upload.py",
                             Property("filename"),
                             Property("destname"),
                             Property("optionTargetOs", default=""),
                             Property("target-os", default=""),
                             Property("optionPath", default=""),
                             Property("path", default=""),
                             Property("optionDatestamp", default=""),
                             Property("datestamp", default="")
                         ],
                         haltOnFailure=True))

        self.addStep(
            ShellCommand(name="uploadComplete",
                         description=["removing", "temporary", "file"],
                         descriptionDone=["temporary", "file", "removed"],
                         doStepIf=lambda step:
                         (step.build.getProperty("is_nightly")),
                         workdir=Property("masterdir"),
                         command=["rm", "-fv",
                                  Property("filename")]))
Beispiel #6
0
 def __init__(self,
              condprop=None,
              condinvert=False,
              condvalue=None,
              conddefault=None,
              **kwargs):
     SetProperty.__init__(self, **kwargs)
     self.addFactoryArguments(condprop=condprop,
                              condinvert=condinvert,
                              condvalue=condvalue,
                              conddefault=conddefault)
     self.condProp = condprop
     self.condInvert = condinvert
     self.condValue = condvalue
     self.condDefault = conddefault
def setProperty(f, new_property, new_value):
    f.addStep(SetProperty(name = 'set.' + new_property,
                          command=['echo', new_value],
                          property=new_property,
                          description=['set property', new_property],
                          workdir='.'))
    return f
def getUserDir(f):
    f.addStep(SetProperty(command=['sh', '-c', 'cd ~;pwd'],
                          haltOnFailure=True,
                          property='user_dir',
                          description=['set property', 'user_dir'],
                          workdir='.'))
    return f
def getBuildDir(f):
    f.addStep(SetProperty(name='get.build.dir',
                          command=['pwd'],
                          property='builddir',
                          description='set build dir',
                          workdir='.'))
    return f
 def update_repo(self, repo, branch='default', clobber=False):
     workdir = repo.split("/")[-1]
     repourl = 'http://%s/%s' % (self.hgHost, repo)
     if clobber:
         self.addStep(ShellCommand(
             name='%s_clobber' % workdir,
             command=['rm', '-rf', workdir],
             workdir='.',
         ))
     self.addStep(ShellCommand(
         name='%s_update' % workdir,
         command=['bash', '-c',
                  'if test -d %(workdir)s; then hg -R %(workdir)s pull; \
                  else hg clone --noupdate %(repourl)s %(workdir)s; fi && \
                  hg -R %(workdir)s up -C --rev %(branch)s' % locals()],
         timeout=3*60,
         descriptionDone="%s source" % workdir,
         workdir='.',
     ))
     self.addStep(SetProperty(
         name='set_%s_revision' % workdir,
         command=['hg', 'identify', '-i'],
         property='%s_revision' % workdir,
         workdir=workdir,
     ))
Beispiel #11
0
 def _step_AdditionalProperties(self):
     self.addStep(
         SetProperty(
             property="host-extra",
             command=[
                 "bash", "-c",
                 """cygcheck -c -d cygwin | perl -ne 'm/^cygwin\s+(\S+)/ and print "-$1"'"""
             ]))
Beispiel #12
0
def getTestConfig(f):
    def getRemoteCfg(rc, stdout, stderr):
        return json.loads(stdout)
    f.addStep(SetProperty(name="get test config",
                          command="cat test_cfg.json",
                          extract_fn=getRemoteCfg,
                          description="get remote target",
                          workdir="scripts"))
    return f
Beispiel #13
0
def getSanitizerBuildFactory(clean=False,
                             depends_on_projects=None,
                             env=None,
                             timeout=1200,
                             is_legacy_mode=False):

    # Prepare environmental variables. Set here all env we want everywhere.
    merged_env = {
        "TERM": "dumb",  # Make sure Clang doesn't use color escape sequences.
    }
    # Clobber bot if we need a clean build.
    if clean:
        merged_env["BUILDBOT_CLOBBER"] = "1"
    # Overwrite pre-set items with the given ones, so user can set anything.
    if env is not None:
        merged_env.update(env)

    if depends_on_projects is None:
        depends_on_projects = [
            "llvm", "clang", "compiler-rt", "libcxx", "libcxxabi", "libunwind",
            "lld"
        ]

    # Explicitly use '/' as separator, because it works on *nix and Windows.
    sanitizer_script_dir = "sanitizer_buildbot/sanitizers"
    sanitizer_script = "../%s/zorg/buildbot/builders/sanitizers/%s" % (
        sanitizer_script_dir, "buildbot_selector.py")

    f = LLVMBuildFactory(is_legacy_mode=is_legacy_mode,
                         clean=clean,
                         depends_on_projects=depends_on_projects,
                         llvm_srcdir=sanitizer_script_dir)

    # Determine the build directory.
    f.addStep(
        SetProperty(name="get_builddir",
                    command=["pwd"],
                    property="builddir",
                    description="set build dir",
                    workdir=".",
                    env=merged_env))

    # Get sanitizer buildbot scripts.
    f.addGetSourcecodeForProject(name='update-annotate-scripts',
                                 project='zorg',
                                 src_dir=sanitizer_script_dir,
                                 alwaysUseLatest=True)

    # Run annotated command for sanitizer.
    f.addStep(
        AnnotatedCommand(name="annotate",
                         description="annotate",
                         timeout=timeout,
                         haltOnFailure=True,
                         command="python " + sanitizer_script,
                         env=merged_env))
    return f
Beispiel #14
0
def addGCSUploadSteps(f, package_name, install_prefix, gcs_directory, env,
                      gcs_url_property=None, use_pixz_compression=False,
                      xz_compression_factor=6):
    """
    Add steps to upload to the Google Cloud Storage bucket.

    f - The BuildFactory to modify.
    package_name - The name of this package for the descriptions (e.g.
                   'stage 1')
    install_prefix - The directory the build has been installed to.
    gcs_directory - The subdirectory of the bucket root to upload to. This
                    should match the builder name.
    env - The environment to use. Set BOTO_CONFIG to use a configuration file
          in a non-standard location, and BUCKET to use a different GCS bucket.
    gcs_url_property - Property to assign the GCS url to.
    """

    gcs_url_fmt = ('gs://%(gcs_bucket)s/%(gcs_directory)s/'
                   'clang-r%(got_revision)s-t%(now)s-b%(buildnumber)s.tar.xz')
    time_fmt = '%Y-%m-%d_%H-%M-%S'
    output_file_name = '../install.tar.xz'

    gcs_url = \
        WithProperties(
            gcs_url_fmt,
            gcs_bucket=lambda _: env.get('BUCKET', 'llvm-build-artifacts'),
            gcs_directory=lambda _: gcs_directory,
            now=lambda _: datetime.utcnow().strftime(time_fmt))

    if gcs_url_property:
        f.addStep(SetProperty(
                      name="record GCS url for " + package_name,
                      command=['echo', gcs_url],
                      property=gcs_url_property))

    if use_pixz_compression:
        # tweak the xz compression level to generate packages faster
        tar_command = ['tar', '-Ipixz', '-cvf', output_file_name, '.']
    else:
        xz_command = 'xz -{0}'.format(xz_compression_factor)
        tar_command = ['tar', '-I', xz_command, '-cvf', output_file_name, '.']

    f.addStep(ShellCommand(name='package ' + package_name,
                           command=tar_command,
                           description='packaging ' + package_name + '...',
                           workdir=install_prefix,
                           env=env))

    f.addStep(ShellCommand(
                  name='upload ' + package_name + ' to storage bucket',
                  command=['gsutil', 'cp', '../install.tar.xz', gcs_url],
                  description=('uploading ' + package_name +
                               'to storage bucket ...'),
                  workdir=install_prefix,
                  env=env))
Beispiel #15
0
    def start(self):
        if self.condProp is not None:
            value = self.condDefault
            matched = False
            if self.build.getProperties().has_key(self.condProp):
                value = self.build.getProperty(self.condProp)
            elif self.condDefault is None:
                # no value and no default, assume we want to execute
                matched = True

            if not matched:
                if self.condValue is None:
                    matched = bool(value)
                else:
                    matched = (value == self.condValue)
                if self.condInvert:
                    matched = not matched

            if not matched:
                return SKIPPED
        SetProperty.start(self)
Beispiel #16
0
    def __init__(self, repo, *testnames):
        BuildProcedure.__init__(self, 'elisp')
        self.addSteps(
            Git(repourl='git://github.com/%s.git' % repo),
            SetPropertiesFromEnv(variables=['EMACS']),
            SetProperty(
                command=[
                    Emacs(), '--batch', '--eval',
                    '(princ (make-temp-file "home" t ".bbot"))'
                ],
                extract_fn=lambda rc, stdout, stderr: dict(FakeHome=stdout)))

        for t in testnames or ['test/test']:
            self.addStep(EmacsTest(load=t + '.el'))
    def testGoodStep(self):
        f = BuildFactory()
        f.addStep(SetProperty(command=["echo", "value"], property="propname"))

        ss = SourceStamp()

        req = FakeBuildRequest("Testing", ss, None)

        b = f.newBuild([req])
        b.build_status = FakeBuildStatus()
        b.slavebuilder = FakeSlaveBuilder()

        # This shouldn't raise an exception
        b.setupBuild(None)
 def __init__(self, hgHost, **kwargs):
     self.parent_class = BuildFactory
     self.parent_class.__init__(self, **kwargs)
     self.hgHost = hgHost
     self.addStep(SetProperty(
         name='set_topdir',
         command=['pwd'],
         property='topdir',
         workdir='.',
     ))
     self.addStep(ShellCommand(
         name='rm_pyc',
         command=['find', '.', '-name', '*.pyc', '-exec', 'rm', '-fv', '{}',
                  ';'],
         workdir=".",
     ))
Beispiel #19
0
    def testCommand(self):
        f = BuildFactory()
        f.addStep(SetProperty(command=["echo", "value"], property="propname"))
        f.addStep(
            ShellCommand(
                command=["echo", WithProperties("%(propname)s")]))

        ss = SourceStamp()

        req = FakeBuildRequest("Testing", {ss.repository: ss}, None)

        b = f.newBuild([req])
        b.build_status = FakeBuildStatus()
        b.slavebuilder = FakeSlaveBuilder()

        # This shouldn't raise an exception
        b.setupBuild(None)
Beispiel #20
0
def xmippBundleFactory(groupId):
    xmippTestSteps = util.BuildFactory()
    xmippTestSteps.workdir = settings.SDEVEL_XMIPP_HOME

    env = {
        "SCIPION_HOME": util.Property("SCIPION_HOME"),
        "SCIPION_LOCAL_CONFIG": util.Property("SCIPION_LOCAL_CONFIG"),
        "LD_LIBRARY_PATH": LD_LIBRARY_PATH,
        "EM_ROOT": settings.EM_ROOT
    }

    xmippTestSteps.addStep(
        SetProperty(command=["bash", "-c", "source build/xmipp.bashrc; env"],
                    extract_fn=glob2list,
                    env=env))

    xmippTestSteps.addStep(
        GenerateStagesCommand(
            command=["./xmipp", "test", "--show"],
            name="Generate test stages for Xmipp programs",
            description="Generating test stages for Xmipp programs",
            descriptionDone="Generate test stages for Xmipp programs",
            haltOnFailure=False,
            pattern='./xmipp test (.*)',
            rootName=settings.XMIPP_CMD,
            timeout=settings.timeOutExecute,
            blacklist=settings.SCIPION_TESTS_BLACKLIST,
            env=util.Property('env')))

    xmippTestSteps.addStep(
        GenerateStagesCommand(
            command=["./xmipp", "test", "--show"],
            name="Generate test stages for Xmipp functions",
            description="Generating test stages for Xmipp functions",
            descriptionDone="Generate test stages for Xmipp functions",
            haltOnFailure=False,
            timeout=settings.timeOutExecute,
            pattern='xmipp_test_(.*)',
            rootName=settings.XMIPP_CMD,
            blacklist=settings.SCIPION_TESTS_BLACKLIST,
            env=util.Property('env')))

    return xmippTestSteps
Beispiel #21
0
def getLLDBCMakeBuildFactory(
        clean=False,
        jobs="%(jobs)s",

        # Source directory containing a built python
        python_source_dir=None,

        # Default values for VS devenv and build configuration
        vs=None,
        config='Release',
        target_arch='x86',
        extra_cmake_args=None,
        test=False,
        testTimeout=2400,
        install=False):

    ############# PREPARING

    # Global configurations
    build_dir = 'build'

    f = LLVMBuildFactory(is_legacy_mode=False,
                         depends_on_projects=["llvm", "clang", "lldb", "lld"],
                         obj_dir=build_dir)

    # Determine Slave Environment and Set MSVC environment.
    if vs:
        f.addStep(
            SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
                        extract_fn=extractSlaveEnvironment))

    f.addGetSourcecodeSteps()

    build_cmd = ['ninja']
    install_cmd = ['ninja', 'install']
    test_cmd = ['ninja', 'check-lldb']

    if jobs:
        build_cmd.append(WithProperties("-j%s" % jobs))
        install_cmd.append(WithProperties("-j%s" % jobs))
        test_cmd.append(WithProperties("-j%s" % jobs))

    ############# CLEANING
    cleanBuildRequested = lambda step: clean or step.build.getProperty(
        "clean", default=step.build.getProperty("clean_obj"))
    f.addStep(
        RemoveDirectory(name='clean ' + build_dir,
                        dir=build_dir,
                        haltOnFailure=False,
                        flunkOnFailure=False,
                        doStepIf=cleanBuildRequested))

    rel_src_dir = LLVMBuildFactory.pathRelativeToBuild(f.llvm_srcdir,
                                                       f.obj_dir)
    cmake_options = [
        "-G",
        "Ninja",
        "-DCMAKE_BUILD_TYPE=" + config,
        "-DCMAKE_INSTALL_PREFIX=../install",
        "-DLLVM_ENABLE_PROJECTS=%s" % ";".join(f.depends_on_projects),
    ]
    if python_source_dir:
        cmake_options.append("-DPYTHON_HOME=" + python_source_dir)
    if extra_cmake_args:
        cmake_options += extra_cmake_args

    f.addStep(
        CmakeCommand(name="cmake-configure",
                     description=["cmake configure"],
                     haltOnFailure=True,
                     options=cmake_options,
                     path=rel_src_dir,
                     env=Property('slave_env'),
                     workdir=build_dir))

    f.addStep(
        WarningCountingShellCommand(name='build',
                                    command=build_cmd,
                                    haltOnFailure=True,
                                    description='ninja build',
                                    workdir=build_dir,
                                    env=Property('slave_env')))

    ignoreInstallFail = bool(install != 'ignoreFail')
    f.addStep(
        ShellCommand(name='install',
                     command=install_cmd,
                     flunkOnFailure=ignoreInstallFail,
                     description='ninja install',
                     workdir=build_dir,
                     doStepIf=bool(install),
                     env=Property('slave_env')))

    ignoreTestFail = bool(test != 'ignoreFail')
    f.addStep(
        ShellCommand(name='test',
                     command=test_cmd,
                     flunkOnFailure=ignoreTestFail,
                     timeout=testTimeout,
                     description='ninja test',
                     workdir=build_dir,
                     doStepIf=bool(test),
                     env=Property('slave_env')))

    return f
Beispiel #22
0
def getLLDBUbuntuCMakeBuildFactory(build_compiler,
                                   build_type,
                                   local_test_archs=None,
                                   local_test_compilers=None,
                                   remote_configs=None,
                                   jobs='%(jobs)s',
                                   env=None):
    """Generate factory steps for ubuntu cmake builder

       Arguments:
       build_compiler       -- string of compile name, example 'clang',
                               the compiler will be used to build binaries for host platform
       build_type           -- 'Debug' or 'Release',
                               used to define build type for host platform as well as remote platform if any
       local_test_archs     -- list of architectures, example ['i386','x86_64'],
                               defines architectures to run local tests against, if None, local tests won't be executed
       local_test_compiler  -- list of compilers, example ['clang','gcc4.8.2'],
                               definds compilers to run local tests with, if None, local tests won't be executed
       remote_configs       -- list of RemoteConfig objects, example [RemoteConfig(...)], if None, remote tests won't be executed
       jobs                 -- number of threads for compilation step, example 40
                               default value is jobs number defined during slave creation
       env                  -- environment variables passed to shell commands

    """
    if env is None:
        env = {}

    llvm_srcdir = "llvm"
    llvm_builddir = "build"
    bindir = '%(builddir)s/' + llvm_builddir + '/bin'

    f = buildbot.process.factory.BuildFactory()

    # Determine the build directory.
    f.addStep(
        SetProperty(name="get_builddir",
                    command=["pwd"],
                    property="builddir",
                    description="set build dir",
                    workdir="."))
    # Determine the binary directory of *-tblgen.
    f.addStep(
        SetProperty(name="get tblgen dir",
                    command=["echo", WithProperties(bindir)],
                    property="tblgen_bindir",
                    description="set tblgen dir",
                    workdir="."))
    # Get source code
    f = getLLDBSource(f, llvm_srcdir)

    # Clean Build Folder
    f.addStep(
        ShellCommand(name="clean",
                     command="rm -rf *",
                     description="clear build folder",
                     env=env,
                     workdir='%s' % llvm_builddir))

    f = getLLDBCmakeAndCompileSteps(f, build_compiler, build_type, [], bindir,
                                    'linux', 'x86_64', env)

    # TODO: it will be good to check that architectures listed in test_archs are compatible with host architecture
    # For now, the caller of this function should make sure that each target architecture is supported by builder machine

    # Add local test steps
    f = getLLDBTestSteps(f, bindir, local_test_archs, local_test_compilers)
    # Remote test steps
    if remote_configs is not None:
        for config in remote_configs:
            f = getLLDBRemoteTestSteps(f, bindir, build_type, config, env)
    # archive test traces
    f = archiveLLDBTestTraces(f, "build/lldb-test-traces-*")
    return f
Beispiel #23
0
def getLLDBRemoteTestSteps(f, bindir, build_type, remote_config, env):
    if None in (remote_config.test_archs, remote_config.test_compilers):
        return f
    # only supports linux and android as remote target at this time
    if remote_config.platform not in ('linux', 'android'):
        return f
    llvm_srcdir = "llvm"
    llvm_builddir = "build"
    stepDesc = remote_config.platform + "-" + remote_config.host_arch
    # get hostname
    slave_hostname = None
    f.addStep(
        SetProperty(name="get hostname",
                    command=["hostname"],
                    property="slave_hostname",
                    description="set slave hostname",
                    workdir="."))

    # get configuration of remote target
    # config file should be placed under builddir on builder machine
    # file name: remote_cfg.json
    # content: json format with keys [remote_platform]-[remote_arch]
    # the value for each key defines "remote_host", "remote_port", "remote_dir", "toolchain", "deviceId"
    # example: {"android-i386": {"remote_host":"localhost",
    #                            "remote_port":"5430",
    #                            "remote_dir":"/data/local/tmp/lldb",
    #                            "toolchain_build":"/home/lldb_build/Toolchains/i386-android-toolchain-21",
    #                            "toolchain_test":"/home/lldb_build/Toolchains/i386-android-toolchain-16",
    #                            "deviceid":"XXXXXXX"},

    def getRemoteCfg(rc, stdout, stderr):
        return json.loads(stdout)[stepDesc]

    f.addStep(
        SetProperty(name="get remote target " + stepDesc,
                    command="cat remote_cfg.json",
                    extract_fn=getRemoteCfg,
                    description="get remote target",
                    workdir="."))
    # rsync
    if remote_config.platform is 'linux':
        shellcmd = ['ssh', WithProperties('%(remote_host)s')]
        hostname = '%(slave_hostname)s'
        launchcmd = shellcmd + ['screen', '-d', '-m']
        terminatecmd = shellcmd + ['pkill', 'lldb-server']
        cleandircmd = WithProperties(
            'ssh %(remote_host)s rm -r %(remote_dir)s/*')
        f.addStep(
            ShellCommand(name="rsync lldb-server",
                         command=[
                             'rsync', '-havL', 'bin/lldb-server',
                             WithProperties('%(remote_host)s:%(remote_dir)s')
                         ],
                         description="rsync lldb-server " + stepDesc,
                         haltOnFailure=True,
                         env=env,
                         workdir='%s' % llvm_builddir))
        f.addStep(
            ShellCommand(name="rsync python2.7",
                         command=[
                             'rsync', '-havL', 'lib/python2.7',
                             WithProperties('%(remote_host)s:%(remote_dir)s')
                         ],
                         description="rsync python2.7 " + stepDesc,
                         haltOnFailure=True,
                         env=env,
                         workdir='%s' % llvm_builddir))
    elif remote_config.platform is 'android':
        shellcmd = ['adb', '-s', WithProperties('%(deviceid)s'), 'shell']
        hostname = '127.0.0.1'
        launchcmd = ['screen', '-d', '-m'] + shellcmd + [
            WithProperties("TMPDIR=%(remote_dir)s/tmp")
        ]
        terminatecmd = 'ps | grep lldb-server | awk \'{print $2}\' | xargs'
        terminatecmd = WithProperties('adb -s %(deviceid)s shell ' +
                                      terminatecmd +
                                      ' adb -s %(deviceid)s shell kill')
        cleandircmd = WithProperties(
            'adb -s %(deviceid)s shell rm -rf %(remote_dir)s/*')
        # compile lldb-server for target platform
        f = getLLDBCmakeAndCompileSteps(f, 'gcc', build_type, ['lldb-server'],
                                        bindir, remote_config.platform,
                                        remote_config.host_arch, env)

        f.addStep(
            ShellCommand(name="adb push lldb-server " + stepDesc,
                         command=[
                             'adb', '-s',
                             WithProperties('%(deviceid)s'), 'push',
                             remote_config.platform + '-' +
                             remote_config.host_arch + '/bin/lldb-server',
                             WithProperties('%(remote_dir)s/')
                         ],
                         description="lldb-server",
                         env=env,
                         haltOnFailure=True,
                         workdir='%s' % llvm_builddir))
        f.addStep(
            ShellCommand(name="Build fingerprint " + stepDesc,
                         command=[
                             'adb', '-s',
                             WithProperties('%(deviceid)s'), 'shell',
                             'getprop', 'ro.build.fingerprint'
                         ],
                         description="get build fingerprint",
                         env=env,
                         haltOnFailure=False,
                         workdir='%s' % llvm_builddir))
    # launch lldb-server
    f.addStep(
        ShellCommand(
            name="launch lldb-server " + stepDesc,
            command=launchcmd + [
                WithProperties('%(remote_dir)s/lldb-server'), 'platform',
                '--listen',
                WithProperties(hostname + ':%(remote_port)s'), '--server'
            ],
            description="launch lldb-server on remote host",
            env=env,
            haltOnFailure=True,
            workdir='%s' % llvm_builddir))
    # test steps
    f = getLLDBTestSteps(f, bindir, remote_config.test_archs,
                         remote_config.test_compilers, remote_config.platform,
                         '%(remote_host)s', '%(remote_port)s',
                         '%(remote_dir)s', env)
    # terminate lldb-server on remote host
    f.addStep(
        ShellCommand(name="terminate lldb-server " + stepDesc,
                     command=terminatecmd,
                     description="terminate lldb-server",
                     env=env,
                     workdir='%s' % llvm_builddir))
    # clean remote test directory
    f.addStep(
        ShellCommand(name="clean remote dir " + stepDesc,
                     command=cleandircmd,
                     description="clean remote dir",
                     env=env))
    return f
Beispiel #24
0
def getLLDBBuildFactory(triple,
                        useTwoStage=False,
                        make='make',
                        jobs='%(jobs)s',
                        extra_configure_args=[],
                        env={},
                        *args,
                        **kwargs):

    llvm_srcdir = "llvm.src"
    llvm_objdir = "llvm.obj"

    f = buildbot.process.factory.BuildFactory()

    # Determine the build directory.
    f.addStep(
        SetProperty(name="get_builddir",
                    command=["pwd"],
                    property="builddir",
                    description="set build dir",
                    workdir="."))

    # Find out what version of llvm and clang are needed to build this version
    # of lldb. Right now we will assume they use the same version.
    # XXX - could this be done directly on the master instead of the slave?
    f.addStep(
        SetProperty(
            command=
            'svn cat http://llvm.org/svn/llvm-project/lldb/trunk/scripts/build-llvm.pl | grep ^our.*llvm_revision | cut -d \\" -f 2',
            property='llvmrev'))

    # The SVN build step provides no mechanism to check out a specific revision
    # based on a property, so just run the commands directly here.

    svn_co = ['svn', 'checkout', '--force']
    svn_co += ['--revision', WithProperties('%(llvmrev)s')]

    # build llvm svn checkout command
    svn_co_llvm = svn_co + \
     [WithProperties('http://llvm.org/svn/llvm-project/llvm/trunk@%(llvmrev)s'),
                     llvm_srcdir]
    # build clang svn checkout command
    svn_co_clang = svn_co + \
     [WithProperties('http://llvm.org/svn/llvm-project/cfe/trunk@%(llvmrev)s'),
                     '%s/tools/clang' % llvm_srcdir]

    f.addStep(
        ShellCommand(name='svn-llvm',
                     command=svn_co_llvm,
                     haltOnFailure=True,
                     workdir='.'))
    f.addStep(
        ShellCommand(name='svn-clang',
                     command=svn_co_clang,
                     haltOnFailure=True,
                     workdir='.'))

    f.addStep(
        SVN(name='svn-lldb',
            mode='update',
            baseURL='http://llvm.org/svn/llvm-project/lldb/',
            defaultBranch='trunk',
            always_purge=True,
            workdir='%s/tools/lldb' % llvm_srcdir))

    # Run configure
    config_args = [
        WithProperties("%%(builddir)s/%s/configure" % llvm_srcdir),
        "--disable-bindings",
        "--without-llvmgcc",
        "--without-llvmgxx",
    ]
    if triple:
        config_args += ['--build=%s' % triple]
    config_args += extra_configure_args

    f.addStep(
        Configure(name='configure',
                  command=config_args,
                  env=env,
                  workdir=llvm_objdir))

    f.addStep(
        WarningCountingShellCommand(
            name="compile",
            command=['nice', '-n', '10', make,
                     WithProperties("-j%s" % jobs)],
            env=env,
            haltOnFailure=True,
            workdir=llvm_objdir))

    # Test.
    f.addStep(
        LitTestCommand(name="test lldb",
                       command=['nice', '-n', '10', make],
                       description="test lldb",
                       env=env,
                       workdir='%s/tools/lldb/test' % llvm_objdir))

    return f
def getCmakeWithNinjaWithMSVCBuildFactory(
        depends_on_projects=None,
        llvm_srcdir=None,
        obj_dir=None,
        checks=None,
        install_dir=None,
        clean=False,
        extra_configure_args=None,
        # VS tools environment variable if using MSVC. For example,
        # %VS140COMNTOOLS% selects the 2015 toolchain.
        vs=None,
        target_arch=None,
        env=None,
        **kwargs):

    assert not env, "Can't have custom builder env vars with MSVC build"

    # Make a local copy of the configure args, as we are going to modify that.
    if extra_configure_args:
        cmake_args = extra_configure_args[:]
    else:
        cmake_args = list()

    if checks is None:
        checks = ['check-all']

    # Set up VS environment, if appropriate.
    if not vs:
        # We build by Visual Studio 2015, unless otherwise is requested.
        vs = r"""%VS140COMNTOOLS%"""

    f = getLLVMBuildFactoryAndSVNSteps(
        depends_on_projects=depends_on_projects,
        llvm_srcdir=llvm_srcdir,
        obj_dir=obj_dir,
        install_dir=install_dir,
        **kwargs)  # Pass through all the extra arguments.

    f.addStep(
        SetProperty(command=builders_util.getVisualStudioEnvironment(
            vs, target_arch),
                    extract_fn=builders_util.extractSlaveEnvironment))
    env = Property('slave_env')

    # Some options are required for this build no matter what.
    CmakeCommand.applyRequiredOptions(cmake_args, [
        ('-G', 'Ninja'),
    ])

    cleanBuildRequested = lambda step: step.build.getProperty(
        "clean", default=step.build.getProperty("clean_obj")) or clean

    addCmakeSteps(f,
                  cleanBuildRequested=cleanBuildRequested,
                  obj_dir=f.obj_dir,
                  install_dir=f.install_dir,
                  extra_configure_args=cmake_args,
                  env=env,
                  **kwargs)

    addNinjaSteps(f,
                  obj_dir=obj_dir,
                  checks=checks,
                  install_dir=f.install_dir,
                  env=env,
                  **kwargs)

    return f
Beispiel #26
0
def getLLDBCMakeBuildFactory(
        clean=False,
        cmake='cmake',
        jobs="%(jobs)s",

        # Source directory containing a built python
        python_source_dir=None,

        # Default values for VS devenv and build configuration
        vs=None,
        config='Release',
        target_arch='x86',
        extra_cmake_args=None,
        test=False,
        install=False):

    ############# PREPARING
    f = buildbot.process.factory.BuildFactory()

    # Determine Slave Environment and Set MSVC environment.
    if vs:
        f.addStep(
            SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
                        extract_fn=extractSlaveEnvironment))

    f = getLLDBSource(f, 'llvm')

    build_cmd = ['ninja']
    install_cmd = ['ninja', 'install']
    test_cmd = ['ninja', 'check-lldb']

    if jobs:
        build_cmd.append(WithProperties("-j%s" % jobs))
        install_cmd.append(WithProperties("-j%s" % jobs))
        test_cmd.append(WithProperties("-j%s" % jobs))

    # Global configurations
    build_dir = 'build'

    # get full path to build directory
    f.addStep(
        SetProperty(name="get_builddir",
                    command=["pwd"],
                    property="builddir",
                    description="set build dir",
                    workdir=build_dir))

    ############# CLEANING
    cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
    f.addStep(
        RemoveDirectory(name='clean ' + build_dir,
                        dir=build_dir,
                        haltOnFailure=False,
                        flunkOnFailure=False,
                        doStepIf=cleanBuildRequested))

    cmake_cmd = [
        "cmake", "-G", "Ninja", "../llvm", "-DCMAKE_BUILD_TYPE=" + config,
        "-DCMAKE_INSTALL_PREFIX=../install"
    ]
    if python_source_dir:
        cmake_cmd.append("-DPYTHON_HOME=" + python_source_dir)
    if extra_cmake_args:
        cmake_cmd += extra_cmake_args
    # Note: ShellCommand does not pass the params with special symbols right.
    # The " ".join is a workaround for this bug.
    f.addStep(
        ShellCommand(name="cmake-configure",
                     description=["cmake configure"],
                     command=WithProperties(" ".join(cmake_cmd)),
                     haltOnFailure=True,
                     warnOnWarnings=True,
                     workdir=build_dir,
                     env=Property('slave_env')))

    f.addStep(
        WarningCountingShellCommand(name='build',
                                    command=build_cmd,
                                    haltOnFailure=True,
                                    description='ninja build',
                                    workdir=build_dir,
                                    env=Property('slave_env')))

    ignoreInstallFail = bool(install != 'ignoreFail')
    f.addStep(
        ShellCommand(name='install',
                     command=install_cmd,
                     flunkOnFailure=ignoreInstallFail,
                     description='ninja install',
                     workdir=build_dir,
                     doStepIf=bool(install),
                     env=Property('slave_env')))

    ignoreTestFail = bool(test != 'ignoreFail')
    f.addStep(
        ShellCommand(name='test',
                     command=test_cmd,
                     flunkOnFailure=ignoreTestFail,
                     timeout=2400,
                     description='ninja test',
                     workdir=build_dir,
                     doStepIf=bool(test),
                     env=Property('slave_env')))

    return f
 def getText(self, cmd, results):
     if 'version' in self.property_changes:
         return [ "%s version %s" % (self.package, self.property_changes['version']) ]
     else:
         SetProperty.getText(self, cmd, results)
Beispiel #28
0
def _getClangCMakeBuildFactory(
        clean=True,
        test=True,
        cmake='cmake',
        jobs=None,

        # VS tools environment variable if using MSVC. For example,
        # %VS120COMNTOOLS% selects the 2013 toolchain.
        vs=None,
        vs_target_arch='x86',

        # Multi-stage compilation
        useTwoStage=False,
        testStage1=True,
        stage1_config='Release',
        stage2_config='Release',

        # Test-suite
        runTestSuite=False,
        nt_flags=None,
        testsuite_flags=None,
        submitURL=None,
        testerName=None,

        # Environmental variables for all steps.
        env=None,
        extra_cmake_args=None,

        # Extra repositories
        checkout_clang_tools_extra=True,
        checkout_compiler_rt=True,
        checkout_lld=True,
        checkout_libcxx=False,
        checkout_test_suite=False,
        checkout_flang=False,

        # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
        stage1_upload_directory=None,

        # Use a lower compression level to generate the build-cache package faster
        # default is 6 according to documentation
        xz_compression_factor=6,
        use_pixz_compression=False,

        # Triggers
        trigger_after_stage1=None):

    ############# PREPARING
    if nt_flags is None:
        nt_flags = []
    if testsuite_flags is None:
        testsuite_flags = []
    if env is None:
        env = {}
    if extra_cmake_args is None:
        extra_cmake_args = []
    if trigger_after_stage1 is None:
        trigger_after_stage1 = []

    clean_build_requested = lambda step: \
        step.build.getProperty( \
            "clean", \
            default=step.build.getProperty("clean_obj") \
        ) or clean

    # We *must* checkout at least Clang+LLVM
    depends_on_projects = ['llvm', 'clang']
    if checkout_clang_tools_extra:
        depends_on_projects.append('clang-tools-extra')
    if checkout_compiler_rt:
        depends_on_projects.append('compiler-rt')
    if checkout_lld:
        depends_on_projects.append('lld')
    if checkout_libcxx:
        depends_on_projects.append('libcxx')
        depends_on_projects.append('libcxxabi')
        depends_on_projects.append('libunwind')
    if checkout_flang:
        depends_on_projects.append('flang')
        depends_on_projects.append('mlir')

    f = LLVMBuildFactory(depends_on_projects=depends_on_projects,
                         llvm_srcdir='llvm')

    # Checkout the latest code for LNT
    # and the test-suite separately. Le's do this first,
    # so we wouldn't poison got_revision property.
    if runTestSuite or checkout_test_suite:
        f.addGetSourcecodeForProject(project='lnt',
                                     src_dir='test/lnt',
                                     alwaysUseLatest=True)
        f.addGetSourcecodeForProject(project='test-suite',
                                     src_dir='test/test-suite',
                                     alwaysUseLatest=True)

    # Then get the LLVM source code revision this particular build is for.
    f.addGetSourcecodeSteps()

    # If jobs not defined, Ninja will choose a suitable value
    jobs_cmd = []
    lit_args = "'-v"
    if jobs is not None:
        jobs_cmd = ["-j" + str(jobs)]
        lit_args += " -j" + str(jobs) + "'"
    else:
        lit_args += "'"
    ninja_cmd = ['ninja'] + jobs_cmd
    ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
    ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd

    # Global configurations
    stage1_build = 'stage1'
    stage1_install = 'stage1.install'
    stage2_build = 'stage2'
    stage2_install = 'stage2.install'

    # Set up VS environment, if appropriate.
    if vs and vs != "manual":
        f.addStep(
            SetProperty(command=builders_util.getVisualStudioEnvironment(
                vs, vs_target_arch),
                        extract_fn=builders_util.extractSlaveEnvironment))
        assert not env, "Can't have custom builder env vars with VS"
        env = Property('slave_env')

    ############# CLEANING
    f.addStep(
        ShellCommand(name='clean stage 1',
                     command=['rm', '-rf', stage1_build],
                     warnOnFailure=True,
                     haltOnFailure=False,
                     flunkOnFailure=False,
                     description='cleaning stage 1',
                     descriptionDone='clean',
                     workdir='.',
                     doStepIf=clean_build_requested))

    ############# STAGE 1
    CmakeCommand.applyRequiredOptions(extra_cmake_args, [
        ('-DLLVM_ENABLE_PROJECTS=', ";".join(f.depends_on_projects)),
    ])
    rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir, stage1_build)

    f.addStep(
        ShellCommand(
            name='cmake stage 1',
            command=[
                cmake, "-G", "Ninja", rel_src_dir,
                "-DCMAKE_BUILD_TYPE=" + stage1_config,
                "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
                "-DCMAKE_INSTALL_PREFIX=../" + stage1_install
            ] + extra_cmake_args,
            haltOnFailure=True,
            description='cmake stage 1',
            workdir=stage1_build,
            doStepIf=FileDoesNotExist("build.ninja"),
            env=env))

    f.addStep(
        WarningCountingShellCommand(name='build stage 1',
                                    command=ninja_cmd,
                                    haltOnFailure=True,
                                    description='ninja all',
                                    workdir=stage1_build,
                                    env=env))

    if test and testStage1:
        haltOnStage1Check = not useTwoStage and not runTestSuite
        f.addStep(
            LitTestCommand(name='ninja check 1',
                           command=ninja_check_cmd,
                           haltOnFailure=haltOnStage1Check,
                           description=["checking stage 1"],
                           descriptionDone=["stage 1 checked"],
                           workdir=stage1_build,
                           env=env))

    if useTwoStage or runTestSuite or stage1_upload_directory:
        f.addStep(
            ShellCommand(name='clean stage 1 install',
                         command=['rm', '-rf', stage1_install],
                         warnOnFailure=True,
                         haltOnFailure=False,
                         flunkOnFailure=False,
                         description='cleaning stage 1 install',
                         descriptionDone='clean',
                         workdir='.'))
        f.addStep(
            ShellCommand(name='install stage 1',
                         command=ninja_install_cmd,
                         description='ninja install',
                         workdir=stage1_build,
                         env=env))

    if stage1_upload_directory:
        addGCSUploadSteps(f,
                          'stage 1',
                          stage1_install,
                          stage1_upload_directory,
                          env,
                          gcs_url_property='stage1_package_gcs_url',
                          use_pixz_compression=use_pixz_compression,
                          xz_compression_factor=xz_compression_factor)

    # Compute the cmake define flag to set the C and C++ compiler to clang. Use
    # clang-cl if we used MSVC for stage1.
    if not vs:
        cc = 'clang'
        cxx = 'clang++'
    else:
        cc = 'clang-cl.exe'
        cxx = 'clang-cl.exe'

    ############# STAGE 2
    if useTwoStage:
        # We always cleanly build the stage 2. If the compiler has been
        # changed on the stage 1, we cannot trust any of the intermediate file
        # from the old compiler. And if the stage 1 compiler is the same, we
        # should not build in the first place.
        f.addStep(
            ShellCommand(name='clean stage 2',
                         command=['rm', '-rf', stage2_build],
                         warnOnFailure=True,
                         description='cleaning stage 2',
                         descriptionDone='clean',
                         workdir='.'))

        # Set the compiler using the CC and CXX environment variables to work around
        # backslash string escaping bugs somewhere between buildbot and cmake. The
        # env.exe helper is required to run the tests, so hopefully it's already on
        # PATH.
        rel_src_dir = LLVMBuildFactory.pathRelativeTo(f.llvm_srcdir,
                                                      stage2_build)
        cmake_cmd2 = [
            cmake, "-G", "Ninja", rel_src_dir,
            WithProperties("-DCMAKE_C_COMPILER=%(workdir)s/" + stage1_install +
                           "/bin/" + cc),
            WithProperties("-DCMAKE_CXX_COMPILER=%(workdir)s/" +
                           stage1_install + "/bin/" + cxx),
            "-DCMAKE_BUILD_TYPE=" + stage2_config,
            "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
            "-DCMAKE_INSTALL_PREFIX=../" + stage2_install
        ] + extra_cmake_args

        f.addStep(
            ShellCommand(name='cmake stage 2',
                         command=cmake_cmd2,
                         haltOnFailure=True,
                         description='cmake stage 2',
                         workdir=stage2_build,
                         env=env))

        f.addStep(
            WarningCountingShellCommand(name='build stage 2',
                                        command=ninja_cmd,
                                        haltOnFailure=True,
                                        description='ninja all',
                                        workdir=stage2_build,
                                        env=env))

        if test:
            f.addStep(
                LitTestCommand(name='ninja check 2',
                               command=ninja_check_cmd,
                               haltOnFailure=not runTestSuite,
                               description=["checking stage 2"],
                               descriptionDone=["stage 2 checked"],
                               workdir=stage2_build,
                               env=env))

    ############# TEST SUITE
    ## Test-Suite (stage 2 if built, stage 1 otherwise)
    if runTestSuite:
        compiler_path = stage1_install
        if useTwoStage:
            compiler_path = stage2_install
            f.addStep(
                ShellCommand(name='clean stage 2 install',
                             command=['rm', '-rf', stage2_install],
                             warnOnFailure=True,
                             description='cleaning stage 2 install',
                             descriptionDone='clean',
                             workdir='.'))
            f.addStep(
                ShellCommand(name='install stage 2',
                             command=ninja_install_cmd,
                             description='ninja install 2',
                             workdir=stage2_build,
                             env=env))

        # Get generated python, lnt
        python = WithProperties('%(workdir)s/test/sandbox/bin/python')
        lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
        lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')

        # Paths
        sandbox = WithProperties('%(workdir)s/test/sandbox')
        test_suite_dir = WithProperties('%(workdir)s/test/test-suite')

        # Get latest built Clang (stage1 or stage2)
        cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
        cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)

        # LNT Command line (don't pass -jN. Users need to pass both --threads
        # and --build-threads in nt_flags/test_suite_flags to get the same effect)
        use_runtest_testsuite = len(nt_flags) == 0
        if not use_runtest_testsuite:
            test_suite_cmd = [
                python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
                sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx',
                cxx
            ]
            # Append any option provided by the user
            test_suite_cmd.extend(nt_flags)
        else:
            lit = WithProperties('%(workdir)s/' + stage1_build +
                                 '/bin/llvm-lit')
            test_suite_cmd = [
                python, lnt, 'runtest', 'test-suite', '--no-timestamp',
                '--sandbox', sandbox, '--test-suite', test_suite_dir, '--cc',
                cc, '--cxx', cxx, '--use-lit', lit
            ]
            # Append any option provided by the user
            test_suite_cmd.extend(testsuite_flags)

        # Only submit if a URL has been specified
        if submitURL is not None:
            if not isinstance(submitURL, list):
                submitURL = [submitURL]
            for url in submitURL:
                test_suite_cmd.extend(['--submit', url])
            # lnt runtest test-suite doesn't understand --no-machdep-info:
            if testerName and not use_runtest_testsuite:
                test_suite_cmd.extend(['--no-machdep-info', testerName])
        # CC and CXX are needed as env for build-tools
        test_suite_env = copy.deepcopy(env)
        test_suite_env['CC'] = cc
        test_suite_env['CXX'] = cxx

        # Steps to prepare, build and run LNT
        f.addStep(
            ShellCommand(name='clean sandbox',
                         command=['rm', '-rf', 'sandbox'],
                         haltOnFailure=True,
                         description='removing sandbox directory',
                         workdir='test',
                         env=env))
        f.addStep(
            ShellCommand(name='recreate sandbox',
                         command=['virtualenv', 'sandbox'],
                         haltOnFailure=True,
                         description='recreating sandbox',
                         workdir='test',
                         env=env))
        f.addStep(
            ShellCommand(name='setup lit',
                         command=[python, lnt_setup, 'develop'],
                         haltOnFailure=True,
                         description='setting up LNT in sandbox',
                         workdir='test/sandbox',
                         env=env))
        f.addStep(
            LitTestCommand(name='test-suite',
                           command=test_suite_cmd,
                           haltOnFailure=True,
                           description=['running the test suite'],
                           workdir='test/sandbox',
                           logfiles={
                               'configure.log': 'build/configure.log',
                               'build-tools.log': 'build/build-tools.log',
                               'test.log': 'build/test.log',
                               'report.json': 'build/report.json'
                           },
                           env=test_suite_env))

    return f
Beispiel #29
0
def getSanitizerWindowsBuildFactory(
        clean=False,
        cmake='cmake',

        # Default values for VS devenv and build configuration
        vs=r"""%VS120COMNTOOLS%""",
        config='Release',
        target_arch='x86',
        extra_cmake_args=[]):

    ############# PREPARING
    f = buildbot.process.factory.BuildFactory()

    # Kill any stale symbolizer processes for the last run. If there are any
    # stale processes, the build will fail during linking. This can happen to
    # any process, but it is most likely to happen to llvm-symbolizer if its
    # pipe isn't closed.
    taskkill_cmd = 'taskkill /f /im llvm-symbolizer.exe || exit /b 0'
    f.addStep(
        ShellCommand(name='taskkill',
                     description='kill stale processes',
                     command=['cmd', '/c', taskkill_cmd],
                     haltOnFailure=False))

    # Determine Slave Environment and Set MSVC environment.
    f.addStep(
        SetProperty(command=getVisualStudioEnvironment(vs, target_arch),
                    extract_fn=extractSlaveEnvironment))

    f = getSource(f, 'llvm')

    # Global configurations
    build_dir = 'build'

    ############# CLEANING
    cleanBuildRequested = lambda step: step.build.getProperty("clean") or clean
    f.addStep(
        RemoveDirectory(name='clean ' + build_dir,
                        dir=build_dir,
                        haltOnFailure=False,
                        flunkOnFailure=False,
                        doStepIf=cleanBuildRequested))

    f.addStep(
        ShellCommand(
            name='cmake',
            command=[
                cmake, "-G", "Ninja", "../llvm",
                "-DCMAKE_BUILD_TYPE=" + config, "-DLLVM_ENABLE_ASSERTIONS=ON"
            ] + extra_cmake_args,
            haltOnFailure=True,
            workdir=build_dir,
            env=Property('slave_env')))

    # Build compiler-rt first to speed up detection of Windows-specific
    # compiler-time errors in the sanitizer runtimes.
    f.addStep(
        NinjaCommand(name='build compiler-rt',
                     targets=['compiler-rt'],
                     haltOnFailure=True,
                     description='ninja compiler-rt',
                     workdir=build_dir,
                     env=Property('slave_env')))

    # Build Clang and LLD next so that most compilation errors occur in a build
    # step.
    f.addStep(
        NinjaCommand(name='build clang lld',
                     targets=['clang', 'lld'],
                     haltOnFailure=True,
                     description='ninja clang lld',
                     workdir=build_dir,
                     env=Property('slave_env')))

    # Only run sanitizer tests.
    # Don't build targets that are not required in order to speed up the cycle.
    for target in [
            'check-asan', 'check-asan-dynamic', 'check-sanitizer', 'check-cfi'
    ]:
        f.addStep(
            NinjaCommand(name='run %s' % target,
                         targets=[target],
                         haltOnFailure=False,
                         description='ninja %s' % target,
                         workdir=build_dir,
                         env=Property('slave_env')))

    return f
Beispiel #30
0
def getLLDBxcodebuildFactory(use_cc=None,
                             build_type='Debug',
                             remote_configs=None,
                             env=None):
    if env is None:
        env = {}
    f = buildbot.process.factory.BuildFactory()
    f.addStep(
        SetProperty(name='get_builddir',
                    command=['pwd'],
                    property='builddir',
                    description='set build dir',
                    workdir='.'))
    lldb_srcdir = 'lldb'
    OBJROOT = '%(builddir)s/' + lldb_srcdir + '/build'
    f.addStep(
        SetProperty(name='get_bindir',
                    command=[
                        'echo',
                        WithProperties('%(builddir)s/' + lldb_srcdir +
                                       '/build/' + build_type)
                    ],
                    property='lldb_bindir',
                    description='set bin dir',
                    workdir='.'))
    # cleaning out the build directory is vital for codesigning.
    f.addStep(
        ShellCommand(name='clean.lldb-buid',
                     command=['rm', '-rf',
                              WithProperties(OBJROOT)],
                     haltOnFailure=True,
                     workdir=WithProperties('%(builddir)s')))
    f.addStep(
        ShellCommand(name='clean.llvm-buid',
                     command=['rm', '-rf',
                              '%s/llvm-build' % lldb_srcdir],
                     haltOnFailure=True,
                     workdir=WithProperties('%(builddir)s')))
    f.addStep(
        ShellCommand(name='clean.test trace',
                     command='rm -rf build/*',
                     haltOnFailure=False,
                     flunkOnFailure=False,
                     workdir='.'))
    # Remove symbolic link to lldb, otherwise xcodebuild will have circular dependency
    f.addStep(
        ShellCommand(name='remove symbolic link lldb',
                     command=['rm', lldb_srcdir + '/llvm/tools/lldb'],
                     haltOnFailure=False,
                     flunkOnFailure=False,
                     workdir=WithProperties('%(builddir)s')))
    f.addStep(
        SVN(name='svn-lldb',
            mode='update',
            baseURL='http://llvm.org/svn/llvm-project/lldb/',
            defaultBranch='trunk',
            workdir=lldb_srcdir))
    f.addStep(
        SVN(name='svn-llvm',
            mode='update',
            baseURL='http://llvm.org/svn/llvm-project/llvm/',
            defaultBranch='trunk',
            workdir='%s/llvm' % lldb_srcdir))
    f.addStep(
        SVN(name='svn-clang',
            mode='update',
            baseURL='http://llvm.org/svn/llvm-project/cfe/',
            defaultBranch='trunk',
            workdir='%s/llvm/tools/clang' % lldb_srcdir))
    # setup keychain for codesign
    # In order for the codesigning to work inside of buildbot, security must be
    # called to unlock the keychain, which requires a password.
    # I've set up a special keychain for this purpose, so as to not compromise
    # the login password of the buildslave.
    # This means I have to set the special keychain as the default and unlock it
    # prior to building the sources.
    f.addStep(
        ShellCommand(name='check.keychain',
                     command=['security', 'default-keychain'],
                     haltOnFailure=True,
                     workdir=WithProperties('%(builddir)s')))
    f.addStep(
        ShellCommand(
            name='find.certificate',
            command=['security', 'find-certificate', '-c', 'lldb_codesign'],
            haltOnFailure=True,
            workdir=WithProperties('%(builddir)s')))
    # Building the sources
    #
    buildcmd = ' '.join([
        'xcrun', 'xcodebuild', '-target', 'desktop', '-configuration',
        build_type, 'SYMROOT=' + OBJROOT, 'OBJROOT=' + OBJROOT
    ])
    f.addStep(
        ShellCommand(name='lldb-build',
                     command=WithProperties(buildcmd + " || " + buildcmd),
                     haltOnFailure=True,
                     workdir=lldb_srcdir))

    # Testing
    #
    if not use_cc:
        use_cc = '/Applications/Xcode.app/Contents/Developer/Toolchains/'
        use_cc += 'XcodeDefault.xctoolchain/usr/bin/clang'
        f.addStep(
            SetProperty(name='set.cc',
                        command=['xcrun', '-find', 'clang'],
                        property='use_cc',
                        description='set cc',
                        workdir=lldb_srcdir))
    else:
        f.addStep(
            SetProperty(name='set.cc',
                        command=['echo', use_cc],
                        property='use_cc',
                        description='set cc',
                        workdir=lldb_srcdir))
    DOTEST_OPTS = ' '.join([
        '--executable', '%(lldb_bindir)s/lldb', '--filecheck',
        '%(lldb_bindir)s/FileCheck', '--framework',
        '%(lldb_bindir)s/LLDB.framework', '-A', 'x86_64', '-C', 'clang', '-s',
        '../../build/lldb-test-traces'
    ])
    f.addStep(
        LitTestCommand(
            name='lldb-test',
            command=['./dosep.py', '--options',
                     WithProperties(DOTEST_OPTS)],
            haltOnFailure=False,
            workdir='%s/test' % lldb_srcdir,
            env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')}))
    # Remote test steps
    if remote_configs is not None:
        # Source structure to use cmake command
        f.addStep(
            SetProperty(
                name='get tblgen bindir',
                command=[
                    'echo',
                    WithProperties(
                        '%(builddir)s/' + lldb_srcdir +
                        '/llvm-build/Release+Asserts/x86_64/Release+Asserts/bin'
                    )
                ],
                property='tblgen_bindir',
                description='set tblgen binaries dir',
                workdir='.'))
        f = getSymbLinkSteps(f, lldb_srcdir)
        for config in remote_configs:
            f = getLLDBRemoteTestSteps(
                f,
                '%(lldb_bindir)s',
                build_type,
                config,
                env={'DYLD_FRAMEWORK_PATH': WithProperties('%(lldb_bindir)s')})


# Compress and upload test log
    f = archiveLLDBTestTraces(f, "build/lldb-test-traces*")

    # Results go in a directory coded named according to the date and time of the test run, e.g.:
    #
    # 2012-10-16-11_26_48/Failure-x86_64-_Applications_Xcode.app_Contents_Developer_Toolchains_XcodeDefault.xctoolchain_usr_bin_clang-TestLogging.LogTestCase.test_with_dsym.log
    #
    # Possible results are ExpectedFailure, Failure, SkippedTest, UnexpectedSuccess, and Error.    return f
    return f
Beispiel #31
0
def getLLDBScriptCommandsFactory(
    downloadBinary=True,
    buildAndroid=False,
    runTest=True,
    scriptExt='.sh',
    extra_cmake_args=None,
    depends_on_projects=None,
):
    if scriptExt is '.bat':
        pathSep = '.\\'
    else:
        pathSep = './'

    if extra_cmake_args is None:
        extra_cmake_args = []

    if depends_on_projects is None:
        f = buildbot.process.factory.BuildFactory()
    else:
        f = LLVMBuildFactory(depends_on_projects=depends_on_projects)

    # Update scripts
    getShellCommandStep(f,
                        name='update scripts',
                        command=['updateScripts' + scriptExt])

    # Acquire lock
    if downloadBinary:
        getShellCommandStep(
            f,
            name='acquire lock',
            command=[pathSep + 'acquireLock' + scriptExt, 'totBuild'],
            description='get')

    # Checkout source code
    getShellCommandStep(f,
                        name='checkout source code',
                        command=[
                            pathSep + 'checkoutSource' + scriptExt,
                            WithProperties('%(revision)s')
                        ])

    # Set source revision
    f.addStep(
        SetProperty(name="set revision",
                    command=[pathSep + 'getRevision' + scriptExt],
                    property="got_revision",
                    workdir="scripts"))

    # Configure
    getShellCommandStep(f,
                        name='cmake local',
                        command=[pathSep + 'cmake' + scriptExt] +
                        extra_cmake_args)

    # Build
    getShellCommandStep(f,
                        name='ninja build local',
                        command=[pathSep + 'buildLocal' + scriptExt])
    if buildAndroid:
        getShellCommandStep(f,
                            name='build android',
                            command=[pathSep + 'buildAndroid' + scriptExt])

    # Get lldb-server binaries
    if downloadBinary:
        getShellCommandStep(f,
                            name='get lldb-server binaries',
                            command=[
                                pathSep + 'downloadBinaries' + scriptExt,
                                WithProperties('%(got_revision)s')
                            ])

    # Test
    if runTest:
        f.addStep(
            LitTestCommand(name="run unit tests",
                           command=[pathSep + 'testUnit' + scriptExt],
                           description=["testing"],
                           descriptionDone=["unit test"],
                           workdir='scripts'))
        getTestSteps(f, scriptExt, pathSep)
        # upload test traces
        getShellCommandStep(f,
                            name='upload test traces',
                            command=[
                                pathSep + 'uploadTestTrace' + scriptExt,
                                WithProperties('%(buildnumber)s'),
                                WithProperties('%(buildername)s')
                            ],
                            flunkOnFailure=False)

    # Upload lldb-server binaries and trigger android builders
    if buildAndroid:
        getShellCommandStep(f,
                            name='upload lldb-server binaries',
                            command=[pathSep + 'uploadBinaries' + scriptExt])
        f.addStep(
            trigger.Trigger(schedulerNames=['lldb_android_scheduler'],
                            updateSourceStamp=False,
                            waitForFinish=False))
    # Release lock
    if downloadBinary:
        getShellCommandStep(
            f,
            name='release lock',
            command=[pathSep + 'releaseLock' + scriptExt, 'totBuild'],
            description='release',
            alwaysRun=True)
    return f
Beispiel #32
0
def _getClangCMakeBuildFactory(
        clean=True,
        test=True,
        cmake='cmake',
        jobs=None,

        # VS tools environment variable if using MSVC. For example,
        # %VS120COMNTOOLS% selects the 2013 toolchain.
        vs=None,
        vs_target_arch='x86',

        # Multi-stage compilation
        useTwoStage=False,
        testStage1=True,
        stage1_config='Release',
        stage2_config='Release',

        # Test-suite
        runTestSuite=False,
        nt_flags=[],
        submitURL=None,
        testerName=None,

        # Environmental variables for all steps.
        env={},
        extra_cmake_args=[],

        # Extra repositories
        checkout_clang_tools_extra=True,
        checkout_compiler_rt=True,

        # Upload artifacts to Google Cloud Storage (for the llvmbisect tool)
        stage1_upload_directory=None,

        # Triggers
        trigger_after_stage1=[]):

    ############# PREPARING
    f = buildbot.process.factory.BuildFactory()

    addSVNUpdateSteps(f,
                      checkout_clang_tools_extra=checkout_clang_tools_extra,
                      checkout_compiler_rt=checkout_compiler_rt,
                      checkout_test_suite=runTestSuite)

    # If jobs not defined, Ninja will choose a suitable value
    jobs_cmd = []
    lit_args = "'-v"
    if jobs is not None:
        jobs_cmd = ["-j" + str(jobs)]
        lit_args += " -j" + str(jobs) + "'"
    else:
        lit_args += "'"
    ninja_cmd = ['ninja'] + jobs_cmd
    ninja_install_cmd = ['ninja', 'install'] + jobs_cmd
    ninja_check_cmd = ['ninja', 'check-all'] + jobs_cmd
    check_build_cmd = [
        "sh", "-c", "test -e build.ninja && echo OK || echo Missing"
    ]
    if vs:
        check_build_cmd = [
            "cmd", "/c",
            "if exist build.ninja (echo OK) " + " else (echo Missing & exit 1)"
        ]

    # Global configurations
    stage1_build = 'stage1'
    stage1_install = 'stage1.install'
    stage2_build = 'stage2'
    stage2_install = 'stage2.install'

    # Set up VS environment, if appropriate.
    if vs:
        f.addStep(
            SetProperty(command=builders_util.getVisualStudioEnvironment(
                vs, vs_target_arch),
                        extract_fn=builders_util.extractSlaveEnvironment))
        assert not env, "Can't have custom builder env vars with VS"
        env = Property('slave_env')

    ############# CLEANING
    if clean:
        f.addStep(
            ShellCommand(name='clean stage 1',
                         command=['rm', '-rf', stage1_build],
                         warnOnFailure=True,
                         description='cleaning stage 1',
                         descriptionDone='clean',
                         workdir='.',
                         env=env))
    else:
        f.addStep(
            SetProperty(name="check ninja files 1",
                        workdir=stage1_build,
                        command=check_build_cmd,
                        flunkOnFailure=False,
                        property="exists_ninja_1"))

    ############# STAGE 1
    f.addStep(
        ShellCommand(
            name='cmake stage 1',
            command=[
                cmake, "-G", "Ninja", "../llvm",
                "-DCMAKE_BUILD_TYPE=" + stage1_config,
                "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
                "-DCMAKE_INSTALL_PREFIX=../" + stage1_install
            ] + extra_cmake_args,
            haltOnFailure=True,
            description='cmake stage 1',
            workdir=stage1_build,
            doStepIf=lambda step: step.build.getProperty("exists_ninja_1"
                                                         ) != "OK",
            env=env))

    f.addStep(
        WarningCountingShellCommand(name='build stage 1',
                                    command=ninja_cmd,
                                    haltOnFailure=True,
                                    description='ninja all',
                                    workdir=stage1_build,
                                    env=env))

    if test and testStage1:
        haltOnStage1Check = not useTwoStage and not runTestSuite
        f.addStep(
            lit_test_command.LitTestCommand(
                name='ninja check 1',
                command=ninja_check_cmd,
                haltOnFailure=haltOnStage1Check,
                description=["checking stage 1"],
                descriptionDone=["stage 1 checked"],
                workdir=stage1_build,
                env=env))

    if useTwoStage or runTestSuite or stage1_upload_directory:
        f.addStep(
            ShellCommand(name='install stage 1',
                         command=ninja_install_cmd,
                         description='ninja install',
                         workdir=stage1_build,
                         env=env))

    if stage1_upload_directory:
        addGCSUploadSteps(f,
                          'stage 1',
                          stage1_install,
                          stage1_upload_directory,
                          env,
                          gcs_url_property='stage1_package_gcs_url')

    # Compute the cmake define flag to set the C and C++ compiler to clang. Use
    # clang-cl if we used MSVC for stage1.
    if not vs:
        cc = 'clang'
        cxx = 'clang++'
    else:
        cc = 'clang-cl.exe'
        cxx = 'clang-cl.exe'

    ############# STAGE 2
    if useTwoStage:
        # We always cleanly build the stage 2. If the compiler has been
        # changed on the stage 1, we cannot trust any of the intermediate file
        # from the old compiler. And if the stage 1 compiler is the same, we
        # should not build in the first place.
        f.addStep(
            ShellCommand(name='clean stage 2',
                         command=['rm', '-rf', stage2_build],
                         warnOnFailure=True,
                         description='cleaning stage 2',
                         descriptionDone='clean',
                         workdir='.',
                         env=env))

        # Set the compiler using the CC and CXX environment variables to work around
        # backslash string escaping bugs somewhere between buildbot and cmake. The
        # env.exe helper is required to run the tests, so hopefully it's already on
        # PATH.
        cmake_cmd2 = [
            'env',
            WithProperties('CC=%(workdir)s/' + stage1_install + '/bin/' + cc),
            WithProperties('CXX=%(workdir)s/' + stage1_install + '/bin/' +
                           cxx), cmake, "-G", "Ninja", "../llvm",
            "-DCMAKE_BUILD_TYPE=" + stage2_config,
            "-DLLVM_ENABLE_ASSERTIONS=True", "-DLLVM_LIT_ARGS=" + lit_args,
            "-DCMAKE_INSTALL_PREFIX=../" + stage2_install
        ] + extra_cmake_args

        f.addStep(
            ShellCommand(name='cmake stage 2',
                         command=cmake_cmd2,
                         haltOnFailure=True,
                         description='cmake stage 2',
                         workdir=stage2_build,
                         doStepIf=lambda step: step.build.getProperty(
                             "exists_ninja_2") != "OK",
                         env=env))

        f.addStep(
            WarningCountingShellCommand(name='build stage 2',
                                        command=ninja_cmd,
                                        haltOnFailure=True,
                                        description='ninja all',
                                        workdir=stage2_build,
                                        env=env))

        if test:
            f.addStep(
                lit_test_command.LitTestCommand(
                    name='ninja check 2',
                    command=ninja_check_cmd,
                    haltOnFailure=not runTestSuite,
                    description=["checking stage 2"],
                    descriptionDone=["stage 2 checked"],
                    workdir=stage2_build,
                    env=env))

    ############# TEST SUITE
    ## Test-Suite (stage 2 if built, stage 1 otherwise)
    if runTestSuite:
        compiler_path = stage1_install
        if useTwoStage:
            compiler_path = stage2_install
            f.addStep(
                ShellCommand(name='install stage 2',
                             command=ninja_install_cmd,
                             description='ninja install 2',
                             workdir=stage2_build,
                             env=env))

        # Get generated python, lnt
        python = WithProperties('%(workdir)s/test/sandbox/bin/python')
        lnt = WithProperties('%(workdir)s/test/sandbox/bin/lnt')
        lnt_setup = WithProperties('%(workdir)s/test/lnt/setup.py')

        # Paths
        sandbox = WithProperties('%(workdir)s/test/sandbox')
        test_suite_dir = WithProperties('%(workdir)s/test/test-suite')

        # Get latest built Clang (stage1 or stage2)
        cc = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cc)
        cxx = WithProperties('%(workdir)s/' + compiler_path + '/bin/' + cxx)

        # LNT Command line (don't pass -jN. Users need to pass both --threads
        # and --build-threads in nt_flags to get the same effect)
        test_suite_cmd = [
            python, lnt, 'runtest', 'nt', '--no-timestamp', '--sandbox',
            sandbox, '--test-suite', test_suite_dir, '--cc', cc, '--cxx', cxx
        ]
        # Append any option provided by the user
        test_suite_cmd.extend(nt_flags)
        # Only submit if a URL has been specified
        if submitURL is not None:
            if not isinstance(submitURL, list):
                submitURL = [submitURL]
            for url in submitURL:
                test_suite_cmd.extend(['--submit', url])
            if testerName:
                test_suite_cmd.extend(['--no-machdep-info', testerName])
        # CC and CXX are needed as env for build-tools
        test_suite_env = copy.deepcopy(env)
        test_suite_env['CC'] = cc
        test_suite_env['CXX'] = cxx

        # Steps to prepare, build and run LNT
        f.addStep(
            ShellCommand(name='clean sandbox',
                         command=['rm', '-rf', 'sandbox'],
                         haltOnFailure=True,
                         description='removing sandbox directory',
                         workdir='test',
                         env=env))
        f.addStep(
            ShellCommand(name='recreate sandbox',
                         command=['virtualenv', 'sandbox'],
                         haltOnFailure=True,
                         description='recreating sandbox',
                         workdir='test',
                         env=env))
        f.addStep(
            ShellCommand(name='setup lit',
                         command=[python, lnt_setup, 'develop'],
                         haltOnFailure=True,
                         description='setting up LNT in sandbox',
                         workdir='test/sandbox',
                         env=env))
        f.addStep(
            commands.LitTestCommand.LitTestCommand(
                name='test-suite',
                command=test_suite_cmd,
                haltOnFailure=True,
                description=['running the test suite'],
                workdir='test/sandbox',
                logfiles={
                    'configure.log': 'build/configure.log',
                    'build-tools.log': 'build/build-tools.log',
                    'test.log': 'build/test.log',
                    'report.json': 'build/report.json'
                },
                env=test_suite_env))

    return f