def __init__(self, steps=None, depends_on_projects=None, **kwargs): # Cannot use "super" here as BuildFactory is an old style class. BuildFactory.__init__(self, steps) if depends_on_projects is None: self.depends_on_projects = frozenset(['llvm']) else: self.depends_on_projects = frozenset(depends_on_projects) # By default LLVMBuildFactory works in the legacy mode. self.is_legacy_mode = kwargs.pop('is_legacy_mode', False) # Directories. self.llvm_srcdir = kwargs.pop('llvm_srcdir', None) self.obj_dir = kwargs.pop('obj_dir', None) self.install_dir = kwargs.pop('install_dir', None) # Preserve the rest of the given extra attributes if any, so we could # expand the factory later. for k, v in kwargs.items(): setattr(self, k, v) if self.is_legacy_mode: self.llvm_srcdir = self.llvm_srcdir or "llvm" self.obj_dir = self.obj_dir or "build" else: self.monorepo_dir = self.llvm_srcdir or "llvm-project" self.llvm_srcdir = \ "%(monorepo_dir)s/llvm" % {'monorepo_dir' : self.monorepo_dir} self.obj_dir = \ self.obj_dir or "build" # Repourl_prefix could be specified per builder. Otherwise we use github. self.repourl_prefix = kwargs.pop('repourl_prefix', 'https://github.com/llvm/')
def __init__(self, branch, python, projects, *a, **kw): BuildFactory.__init__(self, *a, **kw) self.addStep(Mercurial, repourl="http://hg.python.org/cpython", defaultBranch=branch, branchType='inrepo', mode="copy") self.addStep(ShellCommand, name="configure-python", description=["configuring", "python"], descriptionDone=["configure", "python"], command="./configure --prefix=$PWD/install") self.addStep(ShellCommand, name="install-python", description=["installing", "python"], descriptionDone=["install", "python"], command=["make", "install"]) pythonc = "install/bin/" + python self.addStep(ShellCommand, name="link-binary", description=["linking", "binary"], descriptionDone=["link", "binary"], command=["ln", "-nsf", "build/" + pythonc, "python"], workdir=".") self.buildModules(pythonc, projects)
def __init__( self, python, source, uncleanWarnings, trialTests=None, trialMode=None, virtualenv=False, virtualenv_module="virtualenv", platform="unix", forceGarbageCollection=False, ): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python, "-Wall"] assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self.python = python self.virtualenv = virtualenv self.uncleanWarnings = uncleanWarnings self.forceGarbageCollection = forceGarbageCollection self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests if virtualenv: # Hopefully temporary workaround for --clear not working: # https://github.com/pypa/virtualenv/issues/929 self.addStep( shell.ShellCommand, command=self.python + ["-c", "import shutil, sys;" "shutil.rmtree(sys.argv[1], ignore_errors=True)", self._virtualEnvPath], ) self.addStep(shell.ShellCommand, command=self.python + ["-m", virtualenv_module, self._virtualEnvPath]) else: # Report the versions, since we're using the system ones. If it's a # virtualenv, it's up to the venv factory to report the versions # itself. self._reportVersions(python=self.python)
def __init__(self, sources, arch): BuildFactory.__init__(self, sources) # Download the helpers for helper in ("pkgdepends", "pkgprovides", "pkgversion", "ccm-setup", "changelog", "gitrev"): self.addStep(steps.FileDownload(name="helper " + helper, mastersrc="helpers/pkgbuild/" + helper, slavedest="../helpers/" + helper, mode=0755)) # Create a directory to hold the packages that have been built self.addStep(steps.RemoveDirectory(name="rm-repository", dir="repository")) # Create a directory to hold the packages that have been built self.addStep(steps.MakeDirectory(name="mkdir-repository", dir="repository")) # Create or update the chroot self.addStep(PrepareCcm(arch=arch)) # Copy the channel configuration from slave to master self.addStep(steps.FileUpload("channels.yml", "tmp/channels.yml", name="config-upload")) self.addStep(steps.FileUpload("buildinfo.yml", "tmp/buildinfo.yml", name="buildinfo-upload")) # Scan repository and find packages to build self.addStep(RepositoryScan(arch=arch)) # Create a changelog for the repo self.addStep(Changelog(arch=arch)) self.addStep(steps.FileDownload(mastersrc="tmp/buildinfo.yml", slavedest="buildinfo.yml", name="buildinfo-download")) # Publish the repository self.addStep(steps.MasterShellCommand(command="rm -rf /srv/http/repos/papyros/" + arch)) self.addStep(steps.DirectoryUpload('../repository', '/srv/http/repos/papyros/' + arch)) self.addStep(steps.MasterShellCommand(command="chmod a+rX -R /srv/http/repos/papyros")) # Push back changes to version control (only push for x86_64 so we don't have duplicates) if arch == "x86_64": self.addStep(PushSourceChanges())
def __init__( self, python, source, uncleanWarnings, trialTests=None, trialMode=None, virtualenv=False, ): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python] self.python = python self.uncleanWarnings = uncleanWarnings self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests if virtualenv: # Each time we create a new virtualenv as latest pip can build # wheels on the fly and install them from user's cache. self.addStep( shell.ShellCommand, command=[ 'virtualenv', '--clear', '-p', self.python[0], self._virtualEnvPath, ], ) self.addStep( ReportPythonModuleVersions, python=self.python, moduleInfo=[ ("Python", "sys", "sys.version"), ("OpenSSL", "OpenSSL", "OpenSSL.__version__"), ("PyCrypto", "Crypto", "Crypto.__version__"), ("gmpy", "gmpy", "gmpy.version()"), ("SOAPpy", "SOAPpy", "SOAPpy.__version__"), ("ctypes", "ctypes", "ctypes.__version__"), ("gtk", "gtk", "gtk.gtk_version"), ("pygtk", "gtk", "gtk.pygtk_version"), ("pywin32", "win32api", "win32api.GetFileVersionInfo(win32api.__file__, chr(92))['FileVersionLS'] >> 16"), ("pyasn1", "pyasn1", "pyasn1.__version__"), ("cffi", "cffi", "cffi.__version__"), ], pkg_resources=[ ("subunit", "subunit"), ("zope.interface", "zope.interface"), ])
def __init__(self, repo, steps=None): self.basicSteps = [ # Checkout s( SocialtextSVN , mode='update' , baseURL=repo , defaultBranch='/trunk' ), # Fix tarball (harmless on nonMacs) nlwShellStep( 'fix help tarball' , 'dev-bin/fix-help-tarball-for-mac' ), # Stop test servers; clear cached fixtures. nlwShellStep( 'cleanup' , 'if [ -d t/tmp ]; then dev-bin/nlwctl -t stop; rm -r t/tmp*; fi' ), # Configure. nlwShellStep( 'configure' , './configure --dev=1 --apache-proxy=1 [email protected]' )] BuildFactory.__init__(self, self.basicSteps + steps)
def __init__(self, git_url, cmd, buildout_content): BuildFactory.__init__(self) self.addStep(PgSafeStopSb()) self.addStep(CleanUp()) self.addStep(CreateSandbox()) self.addStep(Clone(git_url, 'pkg')) self.addStep(BootstrapSb()) # Change the working directory of the next build steps wd = os.path.join(self.workdir, "pkg") self.addStep(CreateFile(self.buildout_file, buildout_content, workdir=wd)) self.addStep(Buildout(cmd_args=['-U', '-c', self.buildout_file])) self.addStep(PgStartSb()) self.addStep(ShellCommand( name="custom_cmd", haltOnFailure=False, flunkOnFailure=True, workdir=wd, command=cmd )) self.addStep(PgStopSb())
def __init__(self, branch, python, projects, *a, **kw): BuildFactory.__init__(self, *a, **kw) self.addStep( Mercurial, repourl="http://hg.python.org/cpython", defaultBranch=branch, branchType='inrepo', mode="copy") self.addStep( ShellCommand, name="configure-python", description=["configuring", "python"], descriptionDone=["configure", "python"], command="./configure --prefix=$PWD/install") self.addStep( ShellCommand, name="install-python", description=["installing", "python"], descriptionDone=["install", "python"], command=["make", "install"]) pythonc = "install/bin/" + python self.addStep( ShellCommand, name="link-binary", description=["linking", "binary"], descriptionDone=["link", "binary"], command=["ln", "-nsf", "build/" + pythonc, "python"], workdir=".") self.buildModules(pythonc, projects)
def __init__(self, test, timeout, branch=None, talos_config_file='sample.config', results_server=None, reboot=True, base_dir='/builds', reboot_cmd=['sudo', 'reboot-user'], nochrome=False, cvsroot=":pserver:[email protected]:/cvsroot", hg_host='http://hg.mozilla.org', tools_repo_path='build/tools', talos_tarball=None, pageloader_tarball=None, cleanup_glob='tools talos fennec* *.tar.bz2 *.zip', tp4_source='/tools/tp4', browser_wait=7, **kwargs): BuildFactory.__init__(self, **kwargs) self.test = test self.timeout = timeout self.branch = branch self.talos_config_file = talos_config_file self.results_server = results_server self.reboot = reboot self.base_dir = base_dir self.reboot_cmd = reboot_cmd self.nochrome = '' if nochrome is None else '--noChrome' self.cvsroot = cvsroot #We are using a static ip because of dns failures self.hg_host = hg_host self.tools_repo_path = tools_repo_path self.talos_tarball = talos_tarball self.pageloader_tarball = pageloader_tarball self.cleanup_glob = cleanup_glob self.tp4_source = tp4_source self.browser_wait = browser_wait self.addStartupSteps() self.addCleanupSteps() self.addSetupSteps() self.addObtainBuildSteps() self.addRunSteps() self.addFinalSteps()
def __init__(self, repourl, submodules=False, branch='master', codebase='', imageset=None): BuildFactory.__init__(self) self.addStep(steps.SetProperty(property='datestamp', value=datestamp)) self.addStep(steps.Git(repourl=repourl, submodules=submodules, branch=branch, codebase=codebase, name='git-checkout-{}'.format(branch), mode=('full' if submodules else 'incremental'), method='clobber', doStepIf=lambda step: not is_pull_request(step.build.getProperties()), hideStepIf=lambda results, step: results == bbres.SKIPPED)) self.addStep(steps.GitHub(repourl=repourl, submodules=submodules, branch=branch, codebase=codebase, name='git-checkout-pullrequest-ref', mode=('full' if submodules else 'incremental'), method='clobber', doStepIf=lambda step: is_pull_request(step.build.getProperties()), hideStepIf=lambda results, step: results == bbres.SKIPPED)) env_vars = ENV_VARS.copy() # First, remove duplicates from PATH, # then strip out the virtualenv bin directory if we're in a virtualenv. setup_cmd = 'PATH=`echo -n "$PATH" | awk -v RS=: -v ORS=: \'!arr[$0]++\'`;' + \ 'if [ -n "$VIRTUAL_ENV" ]; then ' + \ 'PATH=`echo "$PATH" | sed -re "s,(^|:)$VIRTUAL_ENV/bin(:|$),\\2,g;s,^:,,"`; ' + \ 'fi; . %(prop:setup_script)s; printenv' # Setup steps self.addStep(steps.RemoveDirectory('build/build', name='cleanup', description=['Removing', 'old', 'build', 'directory'], descriptionDone=['Removed', 'old', 'build', 'directory'])) self.addStep(steps.SetPropertyFromCommand(command=['bash', '-c', util.Interpolate(setup_cmd)], extract_fn=extract_env_vars, name='EnvironmentSetup', description=['Running', 'setup', 'script'], descriptionDone=['Ran', 'setup', 'script'])) self.addStep(steps.StringDownload(s=make_autoconf, workerdest='auto.conf', workdir=util.Interpolate("%(prop:BUILDDIR)s/conf"), name='make-auto.conf', description=['Creating', 'auto.conf'], descriptionDone=['Created', 'auto.conf'])) for i, img in enumerate(imageset.imagespecs, start=1): tgtenv = env_vars.copy() tgtenv.update(img.env) bbcmd = "bitbake" if img.keep_going: bbcmd += " -k" cmd = util.Interpolate(bbcmd + "%(kw:bitbake_options)s " + ' '.join(img.args), bitbake_options=bitbake_options) self.addStep(steps.ShellCommand(command=['bash', '-c', cmd], timeout=None, env=tgtenv, workdir=util.Property('BUILDDIR'), name='build_%s_%s' % (imageset.name, img.name), description=['Building', imageset.name, img.name], descriptionDone=['Built', imageset.name, img.name])) self.addStep(steps.ShellCommand(command=store_artifacts_cmd, workdir=util.Property('BUILDDIR'), name='StoreArtifacts', timeout=None, description=['Storing', 'artifacts'], descriptionDone=['Stored', 'artifacts']))
def __init__(self, python, source, uncleanWarnings, trialTests=None, trialMode=None, virtualenv=False, virtualenv_module='virtualenv', platform='unix', forceGarbageCollection=False): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python, "-Wall"] assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self.python = python self.virtualenv = virtualenv self.uncleanWarnings = uncleanWarnings self.forceGarbageCollection = forceGarbageCollection self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests if virtualenv: # Each time we create a new virtualenv as latest pip can build # wheels on the fly and install them from user's cache. self.addStep( shell.ShellCommand, command = self.python + [ "-m", virtualenv_module, '--clear', self._virtualEnvPath, ], ) else: # Report the versions, since we're using the system ones. If it's a # virtualenv, it's up to the venv factory to report the versions # itself. self._reportVersions()
def __init__(self, steps=None, depends_on_projects=None, **kwargs): # Cannot use "super" here as BuildFactory is an old style class. BuildFactory.__init__(self, steps) if depends_on_projects is None: self.depends_on_projects = ['llvm'] else: self.depends_on_projects = depends_on_projects # Preserve all the given extra attributes if any, so we could # expand the factory later. for k,v in kwargs.items(): setattr(self, k, v)
def __init__(self, python, source, uncleanWarnings, trialTests=None, trialMode=None): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python] self.python = python self.uncleanWarnings = uncleanWarnings self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests self.addStep( ReportPythonModuleVersions, python=self.python, moduleInfo=[ ("Python", "sys", "sys.version"), ("OpenSSL", "OpenSSL", "OpenSSL.__version__"), ("PyCrypto", "Crypto", "Crypto.__version__"), ("gmpy", "gmpy", "gmpy.version()"), ("SOAPpy", "SOAPpy", "SOAPpy.__version__"), ("ctypes", "ctypes", "ctypes.__version__"), ("gtk", "gtk", "gtk.gtk_version"), ("pygtk", "gtk", "gtk.pygtk_version"), ("pywin32", "win32api", "win32api.GetFileVersionInfo(win32api.__file__, chr(92))['FileVersionLS'] >> 16" ), ("pyasn1", "pyasn1", "pyasn1.majorVersionId"), ("cffi", "cffi", "cffi.__version"), ], pkg_resources=[ ("subunit", "subunit"), ("zope.interface", "zope.interface"), ])
def __init__(self, branch_name, triggers=None): BuildFactory.__init__(self) self.addStep(CleanUp()) self.addStep(CreateSandbox()) self.addStep(GitClone(component='sact.installator', state='main')) self.addStep(GetCryptographicKeys()) self.addStep(BootstrapSb()) self.addStep(Buildout()) self.addStep(CreatePackage(branch_name)) self.addStep(SavePackage(branch_name, 'ftp.securactive.lan', '/rd/nova/packages')) if triggers is not None: self.addStep(Trigger(schedulerNames=triggers))
def __init__(self, sources, arch, ostree_lock): BuildFactory.__init__(self, sources) # Download the helpers for helper in ('pacstrap-create', 'ostreeinit', 'ostreesetup', 'post-install', 'post-commit'): self.addStep(steps.FileDownload(name="helper " + helper, mastersrc="helpers/ostree/" + helper, slavedest="../helpers/" + helper, mode=0755)) # Copy the channel configuration from slave to master self.addStep(steps.FileUpload("channels.yml", "tmp/channels.yml", name="config-upload")) self.addStep(ScanChannels(arch, ostree_lock=ostree_lock))
def __init__(self, name, image, package_path, ftp_user="******", ftp_password="******", triggers=None): BuildFactory.__init__(self) self.addStep(install.DestroyVirtualInstance(name)) self.addStep(install.CreateVirtualInstance(name, image)) self.addStep(install.FindLatestPackage(package_path)) self.addStep(install.RetrievePackage()) self.addStep(install.UploadPackage(ftp_user, ftp_password)) self.addStep(install.CheckStatusProcesses()) self.addStep(install.CheckInstalledVersion()) if triggers is not None: self.addStep(Trigger(schedulerNames=triggers))
def __init__(self, state, component): BuildFactory.__init__(self) self.component = component self.state = state self.addStep(CleanUp()) self.addStep(CreateSandbox()) self.addStep(InstallSetuptoolsSASb()) self.addStep(GitClone(component=self.component, state=self.state)) self.addStep(BootstrapSb()) self.addStep(Buildout(cmd_args=["install", "doc"])) self.addStep(LaunchAutogen()) self.addStep(BuildDoc()) self.addStep(UploadDoc(component=self.component, state=self.state))
def __init__(self, sources, arch): BuildFactory.__init__(self, sources) # Download the helpers for helper in ('ostree-install', 'ostree-pack'): self.addStep(steps.FileDownload(name="helper " + helper, mastersrc="helpers/diskimage/" + helper, slavedest="../helpers/" + helper, mode=0755)) # Copy the channel configuration from slave to master self.addStep(steps.FileUpload("channels.yml", "tmp/channels.yml", name="config-upload")) self.addStep(ScanChannels(arch))
def __init__(self, state, component, upload_doc=True, upload_egg=True, use_local_pg=False, do_test=True): BuildFactory.__init__(self) self.component = component self.state = state self.addStep(CleanUp()) self.addStep(CreateSandbox()) self.addStep(InstallSetuptoolsSASb()) self.addStep(GitClone(component=self.component, state=self.state)) self.addStep(BootstrapSb()) # Change the working directory of the next build steps wd = os.path.join(self.workdir, "pkg") buildout_filename = "buildout.dev-bb.cfg" # This is intepreted by the shell, be sure to protect special characters # like "$"! buildout_content = dedent(""" [buildout] eggs-directory = /srv/local-cache/eggs extends = buildout.cfg """) self.addStep(CreateFile(buildout_filename, buildout_content, workdir=wd)) self.addStep(Buildout(cmd_args=["-c", buildout_filename])) if use_local_pg: self.addStep(PgStartSb()) if do_test: self.addStep(ShowEggInTest()) self.addStep(Test()) if use_local_pg: self.addStep(PgStopSb()) # self.addStep(BuildEgg()) self.addStep(LaunchAutogen()) self.addStep(BuildDoc()) if upload_egg: self.addStep(RegisterInternalPyPI()) self.addStep(UploadEggInternalPyPI()) if upload_doc: self.addStep(UploadDoc(component=self.component,state=self.state))
def __init__(self, source, bootstrap=["autoreconf", "-vi"], configure=["../configure"], configureEnv={}, configureFlags=[], compile=["make", "all"], test=["make", "check"], install=["make", "install"], distcheck=["make", "distcheck"], clean=True): """Perform the following steps: source, bootstrap, configure, compile, test, install, distcheck any step can be disabled by setting the appropriate argument to `None' except source and boostrap, all steps are done in `build_dir' distcheck is made using configureFlags, stripping `--prefix' out set clean to False to disable deletion of `build_dir' before configure""" BuildFactory.__init__(self, [source]) if bootstrap: # some people also call it autogen self.addStep(Bootstrap, command=bootstrap) if clean: self.addStep(Cleaning, command=["rm", "-rf", GNUAutoconf.build_dir]) if configure: self.addStep(Configure, command=configure + configureFlags, env=configureEnv, workdir=GNUAutoconf.build_dir) if compile: self.addStep(Compile, command=compile, workdir=GNUAutoconf.build_dir) if test: self.addStep(Test, command=test, workdir=GNUAutoconf.build_dir) if install: self.addStep(Install, command=install, workdir=GNUAutoconf.build_dir) if distcheck: distcheck_configure_flags = [ flag for flag in configureFlags if not flag.startswith("--prefix") ] self.addStep(Distcheck, command=distcheck + [ "DISTCHECK_CONFIGURE_FLAGS=" + " ".join(distcheck_configure_flags) ], workdir=GNUAutoconf.build_dir)
def __init__(self, source, toxEnv, reactors=["default"], allowSystemPackages=False, platform="unix", python="python", env=None, ): BuildFactory.__init__(self, source) # Use the test-case-name property or fallback to 'twisted'. self._tests = [WithProperties("%(test-case-name:~twisted)s")] assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self._toxEnv = toxEnv self._env = env self._reactors = reactors self._allowSystemPackages = allowSystemPackages # Workaround for virtualenv --clear issue. # https://github.com/pypa/virtualenv/issues/929 self.addStep( shell.ShellCommand, description="clearing virtualenv".split(" "), command = [python, "-c", "import shutil; shutil.rmtree('" + self._virtualEnvPath + "', True)"], ) self.addStep( shell.ShellCommand, description="making virtualenv".split(" "), command = [python, "-m", "virtualenv", "--clear", self._virtualEnvPath] ) self.addVirtualEnvStep( shell.ShellCommand, description="installing tox".split(" "), command=["python", "-m", "pip", "install", "tox", "virtualenv"] ) self._addRunSteps()
def __init__(self, config, project, profile): BuildFactory.__init__(self, []) env = {} for setup in profile.setups: desc = "Preparing %s" % setup prepare_dict = dict(name=desc, description=desc, descriptionDone=desc) self.addStep(SetupStep(setup, config=config, env=env, **prepare_dict)) self.addStep(create_checkout_step(project)) profile_commands = project.inplace.profile_commands(profile) for pc in profile_commands: shell_dict = dict(name=pc.name, description=pc.name, descriptionDone=pc.name) if len(pc.commands) == 1: self.addStep(ShellCommand(command=pc.commands[0], env=env, **shell_dict)) else: self.addStep(ShellSequence(pc.commands, env=env, **shell_dict))
def __init__(self, state, component, cmd_args): BuildFactory.__init__(self) self.component = component self.state = state self.cmd_args = cmd_args self.addStep(PgSafeStopSb()) self.addStep(CleanUp()) self.addStep(CreateSandbox()) self.addStep(GitClone(component=self.component, state=self.state)) self.addStep(BootstrapSb()) self.addStep(Buildout(cmd_args=self.cmd_args)) self.addStep(ShowEggInTest()) self.addStep(PgStartSb()) self.addStep(LocalPgTest()) self.addStep(PgStopSb())
def __init__(self, steps=None, depends_on_projects=None, **kwargs): # Cannot use "super" here as BuildFactory is an old style class. BuildFactory.__init__(self, steps) if depends_on_projects is None: self.depends_on_projects = frozenset(['llvm']) else: self.depends_on_projects = frozenset(depends_on_projects) # Preserve all the given extra attributes if any, so we could # expand the factory later. for k,v in kwargs.items(): setattr(self, k, v) # Default source code directory. if kwargs.get('llvm_srcdir', None) is None: self.llvm_srcdir = "llvm"
def __init__(self, steps=None, depends_on_projects=None, **kwargs): # Cannot use "super" here as BuildFactory is an old style class. BuildFactory.__init__(self, steps) if depends_on_projects is None: self.depends_on_projects = frozenset(['llvm']) else: self.depends_on_projects = frozenset(depends_on_projects) # Preserve all the given extra attributes if any, so we could # expand the factory later. for k,v in kwargs.items(): setattr(self, k, v) # Default source code directory. if kwargs.get('llvm_srcdir', None) is None: self.llvm_srcdir = "llvm"
def __init__(self, python, source, uncleanWarnings, trialTests=None, trialMode=None): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python] self.python = python self.uncleanWarnings = uncleanWarnings self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests self.addStep( ReportPythonModuleVersions, python=self.python, moduleInfo=[ ("Python", "sys", "sys.version"), ("OpenSSL", "OpenSSL", "OpenSSL.__version__"), ("PyCrypto", "Crypto", "Crypto.__version__"), ("gmpy", "gmpy", "gmpy.version()"), ("SOAPpy", "SOAPpy", "SOAPpy.__version__"), ("ctypes", "ctypes", "ctypes.__version__"), ("gtk", "gtk", "gtk.gtk_version"), ("pygtk", "gtk", "gtk.pygtk_version"), ("pywin32", "win32api", "win32api.GetFileVersionInfo(win32api.__file__, chr(92))['FileVersionLS'] >> 16"), ("pyasn1", "pyasn1", "pyasn1.majorVersionId"), ("cffi", "cffi", "cffi.__version"), ], pkg_resources=[ ("subunit", "subunit"), ("zope.interface", "zope.interface"), ])
def __init__(self, translationArguments, targetArguments, projects, *a, **kw): BuildFactory.__init__(self, *a, **kw) self.addStep(Mercurial, repourl="https://bitbucket.org/pypy/pypy") self.addStep(Translate, translationArgs=translationArguments, targetArgs=targetArguments) self.addStep(ShellCommand, name="link-binary", description=["linking", "binary"], descriptionDone=["link", "binary"], command=["ln", "-nsf", "build/pypy/goal/pypy-c", "."], workdir=".") # Don't try building these yet. PyPy doesn't quite work well # enough. pypyc = "pypy/goal/pypy-c" self.buildModules(pypyc, projects)
def __init__(self, git_base_url, git_base_dir, entry_component, mmf_name): """Create a build to test a MMF :param git_base_url: The URL to clone Git projects from, for example git://git.rd.securactive.lan/ :param: git_base_dir: The directory which contains projects on the server side. We will want to look directly in this directory to query the status of the projects there :param entry_component: The component which will be build (ex: 'sact.nova') :param mmf_name: The MMF we want to build """ BuildFactory.__init__(self) self.git_base_url = git_base_url self.git_base_dir = git_base_dir self.entry_component = entry_component self.mmf_name = mmf_name
def __init__(self, translationArguments, targetArguments, projects, *a, **kw): BuildFactory.__init__(self, *a, **kw) self.addStep( Mercurial, repourl="https://bitbucket.org/pypy/pypy") self.addStep( Translate, translationArgs=translationArguments, targetArgs=targetArguments) self.addStep( ShellCommand, name="link-binary", description=["linking", "binary"], descriptionDone=["link", "binary"], command=["ln", "-nsf", "build/pypy/goal/pypy-c", "."], workdir=".") # Don't try building these yet. PyPy doesn't quite work well # enough. pypyc = "pypy/goal/pypy-c" self.buildModules(pypyc, projects)
def __init__( self, source, toxEnv, reactors=["default"], allowSystemPackages=False, platform="unix", python="python" ): BuildFactory.__init__(self, source) tests = [WithProperties("%(test-case-name:~)s")] tests = [] assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self.addStep( shell.ShellCommand, description="clearing virtualenv".split(" "), command=[python, "-c", "import shutil; shutil.rmtree('" + self._virtualEnvPath + "', True)"], ) self.addStep( shell.ShellCommand, description="making virtualenv".split(" "), command=[python, "-m", "virtualenv", self._virtualEnvPath], ) self.addVirtualEnvStep( shell.ShellCommand, description="installing tox".split(" "), command=["python", "-m", "pip", "install", "tox", "virtualenv"], ) for reactor in reactors: self.addVirtualEnvStep( TrialTox, tests=tests, allowSystemPackages=allowSystemPackages, reactor=reactor, toxEnv=toxEnv )
def __init__(self, source): BuildFactory.__init__(self, [source])
def __init__ ( self, baseURL, useBuildType=None, arguments=[]): """Factory for CMake out-of-source builds with extra buildbot code. Uses the following builder properties (strings) when generating a build: - generator: CMake generator - arguments: build-specific CMake arguments @cvar baseURL: url of the svn repository @cvar useBuildType: for multi-configuration generators (see CMAKE_BUILD_TYPE) @cvar arguments: extra CMake arguments """ assert baseURL is not None assert type(arguments) == type([]) from buildbot.steps import source, slave, shell from buildbot.process.properties import WithProperties, Property BuildFactory.__init__(self) if useBuildType is None: buildTypeArgument=[] else: buildTypeArgument=["--config", useBuildType] # update svn self.addStep(source.SVN( workdir=self.sourcedir, mode='update', baseURL=baseURL, defaultBranch='trunk', retry=(30,2) )) # recreate build dir self.addStep(slave.RemoveDirectory( dir=self.workdir, haltOnFailure=False, flunkOnFailure=False)) self.addStep(slave.MakeDirectory( dir=self.workdir)) # configure self.addStep(shell.Configure( command=["cmake", "../source", WithProperties("-G%s", 'generator'), arguments, Property('extra_arguments', default=[])], logEnviron=False)) # compile - the install target builds and then copies files to the # production directory (default is subdirectory 'install') and removes # full paths from library dependencies so it works when you copy the # production files elsewhere self.addStep(shell.Compile( command=["cmake", "--build", ".", "--target", "install"] + buildTypeArgument, logEnviron=False)) # package - the package target creates an installer/package/archive # that contains the production files; the target is only available if # CPack and a package generator are available self.addStep(shell.SetProperty( haltOnFailure = True, flunkOnFailure = True, extract_fn=lambda rc, stdout, stderr: {"WITH_CPACK": stdout.find("WITH_CPACK:BOOL=ON") != -1}, command=["cmake", "-N", "-LA"], logEnviron=False)) self.addStep(shell.ShellCommand( name = "Package", description = ["packaging"], descriptionDone = ["package"], haltOnFailure = False, flunkOnFailure = False, doStepIf=lambda step: step.build.getProperty("WITH_CPACK"), command=["cmake", "--build", ".", "--target", "package"] + buildTypeArgument, logEnviron=False))
def __init__(self, python, source, uncleanWarnings, trialTests=None, trialMode=None, virtualenv=False, virtualenv_module='virtualenv', platform='unix', forceGarbageCollection=False): if not isinstance(source, list): source = [source] else: source = list(source) # If permissions get messed up on a slave, this can fix it. # But it breaks on old slaves so it's not enabled all the time # (and it can't fix old slaves, obviously). # self._fixPermissions(source) BuildFactory.__init__(self, source) if type(python) is str: python = [python, "-Wall"] assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self.python = python self.virtualenv = virtualenv self.uncleanWarnings = uncleanWarnings self.forceGarbageCollection = forceGarbageCollection self.trialMode = trialMode if trialTests is None: trialTests = [WithProperties("%(test-case-name:~twisted)s")] self.trialTests = trialTests if virtualenv: # Hopefully temporary workaround for --clear not working: # https://github.com/pypa/virtualenv/issues/929 self.addStep( shell.ShellCommand, command = self.python + [ "-c", "import shutil, sys;" "shutil.rmtree(sys.argv[1], ignore_errors=True)", self._virtualEnvPath, ] ) self.addStep( shell.ShellCommand, command = self.python + [ "-m", virtualenv_module, self._virtualEnvPath, ] ) else: # Report the versions, since we're using the system ones. If it's a # virtualenv, it's up to the venv factory to report the versions # itself. self._reportVersions(python=self.python)
def __init__(self, rmConfig, **kwargs): BuildFactory.__init__(self, **kwargs) self.addStep( SetProperty( name='set_topdir', command=['pwd'], property='topdir', workdir='.', )) self.addStep( ShellCommand( command=['rm', '-rvf', 'tools', 'buildbot-configs'], workdir='.', )) self.addStep( ShellCommand( command=[ 'hg', 'clone', make_hg_url(rmConfig['HG_HOST'], 'build/tools'), 'tools' ], workdir='.', haltOnFailure=True, )) self.addStep( ShellCommand( command=[ rmConfig['PYTHON'], 'scripts/preproduction/repo_setup.py', '-c', 'scripts/preproduction/repo_setup_config.py' ], workdir='tools', haltOnFailure=True, )) self.addStep( SetProperty( property='previousSetupMakefile', command='ls %s/Makefile 2>/dev/null || exit 0' % rmConfig['releaseMasterDir'], flunkOnFailure=False, haltOnFailure=False, warnOnFailure=True, )) def previousSetupExists(step): return \ step.build.getProperties().has_key('previousSetupMakefile') \ and len(step.build.getProperty('previousSetupMakefile')) > 0 self.addStep( ShellCommand( command=[ rmConfig['PYTHON'], 'tools/buildfarm/maintenance/buildbot-wrangler.py', 'stop', '%s/master' % rmConfig['releaseMasterDir'] ], workdir=rmConfig['releaseMasterDir'], flunkOnFailure=False, doStepIf=previousSetupExists, )) self.addStep( ShellCommand( command=['rm', '-rvf', rmConfig['releaseMasterDir']], workdir='.', haltOnFailure=True, )) self.addStep( ShellCommand( command=['rm', '-rvf', 'buildbot-configs'], workdir='.', haltOnFailure=True, )) self.addStep( ShellCommand( command=[ 'hg', 'clone', make_hg_url(rmConfig['HG_HOST'], '%s/buildbot-configs' % rmConfig['HG_DIR']), 'buildbot-configs' ], workdir='.', haltOnFailure=True, )) self.addStep( ShellCommand( command=[ 'make', '-f', 'Makefile.setup', 'PYTHON=%s' % rmConfig['PYTHON'], 'VIRTUALENV=%s' % rmConfig['VIRTUALENV'], 'HG_DIR=%s' % rmConfig['HG_DIR'], 'MASTER_NAME=pp-release', 'MASTERS_JSON=/home/cltbld/conf/master_config.json', 'BASEDIR=%s' % rmConfig['releaseMasterDir'], 'BUILDBOTCUSTOM_BRANCH=%s' % rmConfig['BUILDBOTCUSTOM_BRANCH'], 'BUILDBOTCONFIGS_BRANCH=%s' % rmConfig['BUILDBOTCONFIGS_BRANCH'], 'virtualenv', 'deps', 'install-buildbot', 'master', 'master-makefile' ], workdir='buildbot-configs', env={ 'PIP_DOWNLOAD_CACHE': WithProperties('%(topdir)s/cache'), 'CC': None, 'CXX': None }, haltOnFailure=True, )) self.addStep( ShellCommand( command=['make', 'checkconfig'], workdir=rmConfig['releaseMasterDir'], haltOnFailure=True, )) self.addStep( ShellCommand( command=['touch', 'twistd.log'], workdir='%s/master' % rmConfig['releaseMasterDir'], )) self.addStep( ShellCommand( command=[ 'bash', '-c', 'if [ -e ~/conf/passwords.py ]; then cp -fv ~/conf/passwords.py ./; fi' ], workdir='%s/master' % rmConfig['releaseMasterDir'], )) self.addStep( ShellCommand( command=[ rmConfig['PYTHON'], 'tools/buildfarm/maintenance/buildbot-wrangler.py', 'start', '%s/master' % rmConfig['releaseMasterDir'] ], workdir=rmConfig['releaseMasterDir'], )) for release_config in rmConfig['releaseConfigs']: self.addStep( SetProperty( property='release_tag', command=[ rmConfig['PYTHON'], '-c', 'execfile("%s"); \ print releaseConfig["baseTag"] + "_RELEASE"' % release_config ], workdir='%s/buildbot-configs' % rmConfig['releaseMasterDir'], )) self.addStep( SetProperty( property='release_branch', command=[ rmConfig['PYTHON'], '-c', 'execfile("%s"); \ print releaseConfig["sourceRepositories"]["mozilla"]["path"]' % release_config ], workdir='%s/buildbot-configs' % rmConfig['releaseMasterDir'], )) self.addStep( ShellCommand( command=[ 'buildbot', 'sendchange', '--username', 'preproduction', '--master', rmConfig['releaseMasterHostPort'], '--branch', WithProperties('%(release_branch)s'), '-p', 'products:firefox', '-p', WithProperties('script_repo_revision:%(release_tag)s'), 'release_build' ], workdir='.', ))
def __init__(self, pyVersion, useTrial=True): BuildFactory.__init__(self, [pyOpenSSLSource]) self.uploadBase = 'build_products/' self.useTrial = useTrial self.learnVersion(pyVersion)
def __init__(self, env, objdir, platform, branch, sourceRepo, configRepo, configSubDir, profiledBuild, stageServer=None, stageUsername=None, stageGroup=None, stageSshKey=None, stageBasePath=None, ausBaseUploadDir=None, updatePlatform=None, downloadBaseURL=None, ausUser=None, ausHost=None, nightly=False, leakTest=False, codesighs=True, uploadPackages=True, dependToDated=True, **kwargs): BuildFactory.__init__(self, **kwargs) self.env = env self.objdir = objdir self.platform = platform self.branch = branch self.sourceRepo = sourceRepo self.configRepo = configRepo self.configSubDir = configSubDir self.profiledBuild = profiledBuild self.stageServer = stageServer self.stageUsername = stageUsername self.stageGroup = stageGroup self.stageSshKey = stageSshKey self.stageBasePath = stageBasePath self.ausBaseUploadDir = ausBaseUploadDir self.updatePlatform = updatePlatform self.downloadBaseURL = downloadBaseURL self.ausUser = ausUser self.ausHost = ausHost self.nightly = nightly self.leakTest = leakTest self.codesighs = codesighs self.uploadPackages = uploadPackages self.dependToDated = dependToDated if self.uploadPackages: assert stageServer and stageUsername and stageSshKey assert stageBasePath if self.nightly: assert ausBaseUploadDir and updatePlatform and downloadBaseURL assert ausUser and ausHost # platform actually contains more than just the platform... # we need that to figure out which mozconfig to use # but for other purposes we only need to know linux/win32/macosx # platform can be things like: linux, win32-debug, macosx-release, etc. self.mozconfig = 'configs/%s/%s/mozconfig' % (self.configSubDir, self.platform) # we don't need the extra cruft in 'platform' anymore self.platform = platform.split('-')[0].replace('64', '') assert self.platform in ('linux', 'win32', 'macosx') self.logUploadDir = 'tinderbox-builds/%s-%s/' % (self.branch, self.platform) # this is a tad ugly because we need to python interpolation # as well as WithProperties # here's an example of what it translates to: # /opt/aus2/build/0/Firefox/mozilla2/WINNT_x86-msvc/2008010103/en-US self.ausFullUploadDir = '%s/%s/%%(buildid)s/en-US' % \ (self.ausBaseUploadDir, self.updatePlatform) # now, generate the steps # regular dep builds (no clobber, no leaktest): # addBuildSteps() # addUploadSteps() # addCodesighsSteps() # leak test builds (no clobber, leaktest): # addBuildSteps() # addLeakTestSteps() # nightly builds (clobber) # addBuildSteps() # addUploadSteps() # addUpdateSteps() # addSymbolSteps() # for everything: # addCleanupSteps() self.addBuildSteps() if self.leakTest: self.addLeakTestSteps() if self.uploadPackages: self.addUploadSteps() if self.codesighs: self.addCodesighsSteps() if self.nightly: self.addUpdateSteps() self.addSymbolsSteps() self.addCleanupSteps()
def __init__(self, *args, **kwargs): BuildFactory.__init__(self, *args, **kwargs)
def __init__(self, OS, envName, buildBranch, configOptions, buildSearchString, buildDir, buildPath, talosCmd, customManifest='', cvsRoot=":pserver:[email protected]:/cvsroot"): BuildFactory.__init__(self) if OS in ( 'linux', 'linuxbranch', ): cleanCmd = self.linuxClean elif OS in ('win', ): cleanCmd = self.winClean else: cleanCmd = self.macClean self.addStep(ShellCommand, workdir=".", description="Cleanup", command=cleanCmd, env=MozillaEnvironments[envName]) self.addStep(ShellCommand, command=[ "cvs", "-d", cvsRoot, "co", "-d", "talos", "mozilla/testing/performance/talos" ], workdir=".", description="checking out talos", haltOnFailure=True, flunkOnFailure=True, env=MozillaEnvironments[envName]) self.addStep(FileDownload, mastersrc="scripts/generate-tpcomponent.py", slavedest="generate-tpcomponent.py", workdir="talos/page_load_test") if customManifest <> '': self.addStep(FileDownload, mastersrc=customManifest, slavedest="manifest.txt", workdir="talos/page_load_test") self.addStep(ShellCommand, command=["python", "generate-tpcomponent.py"], workdir="talos/page_load_test", description="setting up pageloader", haltOnFailure=True, flunkOnFailure=True, env=MozillaEnvironments[envName]) self.addStep(MozillaTryServerWgetLatest, workdir=".", branch=buildBranch, url=buildDir, filenameSearchString=buildSearchString, env=MozillaEnvironments[envName]) #install the browser, differs based upon platform if OS == 'linux': self.addStep(MozillaInstallTarBz2, workdir=".", branch=buildBranch, haltOnFailure=True, env=MozillaEnvironments[envName]) elif OS == 'linuxbranch': #special case for old linux builds self.addStep(MozillaInstallTarGz, workdir=".", branch=buildBranch, haltOnFailure=True, env=MozillaEnvironments[envName]) elif OS == 'win': self.addStep(MozillaInstallZip, workdir=".", branch=buildBranch, haltOnFailure=True, env=MozillaEnvironments[envName]), self.addStep(ShellCommand, workdir="firefox/", flunkOnFailure=False, warnOnFailure=False, description="chmod files (see msys bug)", command=["chmod", "-v", "-R", "a+x", "."], env=MozillaEnvironments[envName]) elif OS == 'tiger': self.addStep(FileDownload, mastersrc="scripts/installdmg.sh", slavedest="installdmg.sh", workdir=".") self.addStep(MozillaInstallDmg, workdir=".", branch=buildBranch, haltOnFailure=True, env=MozillaEnvironments[envName]) else: #leopard self.addStep(FileDownload, mastersrc="scripts/installdmg.ex", slavedest="installdmg.ex", workdir=".") self.addStep(MozillaInstallDmgEx, workdir=".", branch=buildBranch, haltOnFailure=True, env=MozillaEnvironments[envName]) self.addStep(MozillaUpdateConfig, workdir="talos/", branch=buildBranch, haltOnFailure=True, executablePath=buildPath, addOptions=configOptions, env=MozillaEnvironments[envName]) self.addStep(MozillaRunPerfTests, warnOnWarnings=True, workdir="talos/", branch=buildBranch, timeout=21600, haltOnFailure=True, command=talosCmd, env=MozillaEnvironments[envName])
def __init__( self, source, toxEnv, buildID, reactors=["default"], allowSystemPackages=False, platform="unix", python="python", env={}, ): tests = [WithProperties("%(test-case-name:~)s")] tests = [] BuildFactory.__init__(self, source) assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self.addStep( shell.ShellCommand, description="clearing virtualenv".split(" "), command=[python, "-c", "import shutil; shutil.rmtree('" + self._virtualEnvPath + "', True)"], ) self.addStep( shell.ShellCommand, description="making virtualenv".split(" "), command=[python, "-m", "virtualenv", self._virtualEnvPath], ) self.addVirtualEnvStep( shell.ShellCommand, description="installing tox".split(" "), command=["python", "-m", "pip", "install", "tox", "virtualenv", "coverage", "codecov"], ) self.addVirtualEnvStep( shell.ShellCommand, description="clearing coverage".split(" "), command=["coverage", "erase"] ) for reactor in reactors: self.addVirtualEnvStep( TrialTox, allowSystemPackages=allowSystemPackages, reactor=reactor, tests=tests, toxEnv=toxEnv, commandNumber=2, _env=env, ) self.addVirtualEnvStep( shell.ShellCommand, description="run coverage combine".split(" "), command=["python", "-m", "coverage", "combine"], ) self.addVirtualEnvStep( shell.ShellCommand, description="run coverage xml".split(" "), command=["python", "-m", "coverage", "xml", "-o", "coverage.xml", "-i"], ) self.addVirtualEnvStep( shell.ShellCommand, warnOnFailure=True, description="upload to codecov".split(" "), command=[ "codecov", "--token={}".format(private.codecov_twisted_token), "--build={}".format(buildID), "--file=coverage.xml", WithProperties("--commit=%(got_revision)s"), ], )
def __init__(self, *args, **kwargs): BuildFactory.__init__(self, *args, **kwargs)
def __init__(self, source, toxEnv, buildID, reactors=["default"], allowSystemPackages=False, platform="unix", python="python", env={}): tests = [WithProperties("%(test-case-name:~)s")] tests = [] BuildFactory.__init__(self, source) assert platform in ["unix", "windows"] self._platform = platform if platform == "unix": self._path = posixpath elif platform == "windows": self._path = ntpath self.addStep( shell.ShellCommand, description="clearing virtualenv".split(" "), command = [python, "-c", "import shutil; shutil.rmtree('" + self._virtualEnvPath + "', True)"], ) self.addStep( shell.ShellCommand, description="making virtualenv".split(" "), command = [python, "-m", "virtualenv", "--clear", self._virtualEnvPath] ) self.addVirtualEnvStep( shell.ShellCommand, description="installing tox".split(" "), command=["python", "-m", "pip", "install", "tox", "virtualenv", "coverage", "codecov"] ) self.addVirtualEnvStep( shell.ShellCommand, description="clearing coverage".split(" "), command=["coverage", "erase"] ) for reactor in reactors: self.addVirtualEnvStep(TrialTox, allowSystemPackages=allowSystemPackages, reactor=reactor, tests=tests, toxEnv=toxEnv, commandNumber=2, _env=env, ) self.addVirtualEnvStep( shell.ShellCommand, description = "run coverage combine".split(" "), command=["python", "-m", "coverage", 'combine'] ) self.addVirtualEnvStep( shell.ShellCommand, description = "run coverage xml".split(" "), command=["python", "-m", "coverage", 'xml', '-o', 'coverage.xml', '-i']) self.addVirtualEnvStep( shell.ShellCommand, warnOnFailure=True, description="upload to codecov".split(" "), command=["codecov", "--token={}".format(private.codecov_twisted_token), "--build={}".format(buildID), "--file=coverage.xml", WithProperties("--commit=%(got_revision)s") ], )
def __init__(self, source): BuildFactory.__init__(self, [source])
def __init__(self, pyVersion, useTrial=True): BuildFactory.__init__(self, [pyOpenSSLSource]) self.uploadBase = 'build_products/' self.useTrial = useTrial self.learnVersion(pyVersion)