def addSetupSteps(self): if self.talos_tarball is None: self.addStep(simpleCVS(cvsroot=self.cvsroot, module='talos', path='mozilla/testing/performance/talos', workdir=self.base_dir)) else: self.addTarballTalosSteps() if self.pageloader_tarball is None: self.addHGPageloaderSteps() else: self.addTarballPageloaderSteps() self.addStep(ShellCommand( command=['sudo', 'chmod', '-R', '+rx', '.'], workdir=self.base_dir, description=['fix', 'permissions'], name='fix_perms', )) if 'tp4' in self.test: self.addStep(ShellCommand( #maybe do rsync -a self.tp4_source.rstrip('/') # self.base_dir/talos/page_load_test+'/' #so we know we are always working with a pristine tp4 set command=['ln', '-s', self.tp4_source, '.'], workdir='%s/talos/page_load_test' % self.base_dir, description=['create', 'tp4', 'symlink'], name='tp4_symlink', haltOnFailure=True, ))
def __init__(self, slavedir, baseFilename, scpString, slavesrc=None, slavesrcdir=None, targetSubDir=None, **kwargs): """ @type slavedir: string @param slavedir: The directory that contains the file that will be transferred (on the BuildSlave) @type baseFilename: string @param baseFilename: The filename (without the identifier) of the file that will be transferred @type scpString: string @param scpString: The scp user@host:/dir string to upload the file to. For example, [email protected]:/var/www. This user should have passwordless access to the host. """ ShellCommand.__init__(self, **kwargs) self.addFactoryArguments(slavedir=slavedir, baseFilename=baseFilename, scpString=scpString) self.slavedir = slavedir self.baseFilename = baseFilename self.scpString = scpString self.super_class = ShellCommand self.slavesrc = slavesrc self.slavesrcdir = slavesrcdir self.targetSubDir = targetSubDir
def __init__(self, cvsroot, workdir=".", **kwargs): ShellCommand.__init__(self, workdir=workdir, **kwargs) self.addFactoryArguments(cvsroot=cvsroot) self.cvsroot = cvsroot self.workdir = workdir # command may be overridden in start() self.command = ["cvs", "-d", cvsroot, "co", "mozilla/client.mk"] self.super_class = ShellCommand
def __init__(self, isOptional=False, workdir="build", **kwargs): """ 'workdir' is assumed to be 'build' and should be passed if it is anything else. 'isOptional' is assumed to be False; if the patch is optional, pass True. """ ShellCommand.__init__(self, workdir=workdir, **kwargs) self.addFactoryArguments(isOptional=isOptional, workdir=workdir) self.optional = isOptional self.super_class = ShellCommand
def addTarballTalosSteps(self): self.addStep(ShellCommand( command=['wget', self.talos_tarball, '-O', 'talos.tar.bz2'], workdir=self.base_dir, description=['get', 'talos'], name='get_talos', haltOnFailure=True, )) self.addStep(ShellCommand( command=['tar', 'jxvf', 'talos.tar.bz2'], workdir=self.base_dir, description=['unpack', 'talos'], name='unpack_talos', haltOnFailure=True, ))
def addRunSteps(self): test_command = ['python', 'maemkit-chunked.py'] log_name = 'log_%s.txt' % self.test_type test_command.append('--testtype=%s' % self.test_type) if self.clients: assert issubclass(type(self.clients), tuple), "self.clients must be tuple" assert len(self.clients) is 2, "self.clients too long" client, total = self.clients test_command.append('--client-number=%d' % client) test_command.append('--total-clients=%d' % total) self.addStep(ShellCommand( command=test_command, workdir="%s/maemkit" % self.base_dir, description=['run', self.test], name="run_%s" % self.test, haltOnFailure=True, timeout=self.timeout, )) self.addStep(MobileParseTestLog( name=self.test_type, command=['cat', log_name], knownFailCount=self.known_fail_count, workdir="/builds/fennec", description=['parse', self.test_type, 'log'], timeout=120, flunkOnFailure=False, haltOnFailure=False, ))
def addTarballMaemkitSteps(self): self.addStep(ShellCommand( command=['wget', self.maemkit_tarball, '-O', 'maemkit.tar.bz2'], workdir=self.base_dir, description=['get', 'maemkit'], name='get_maemkit', haltOnFailure=True, )) self.addStep(ShellCommand( command=['tar', 'jxvf', 'maemkit.tar.bz2'], workdir='%s' % self.base_dir, description=['unpack', 'maemkit'], name='unpack_maemkit', haltOnFailure=True, ))
def addTarballPageloaderSteps(self): self.addStep(ShellCommand( command=['wget', self.pageloader_tarball, '-O', 'pageloader.tar.bz2'], workdir=self.base_dir, #PerfConfigurator.py will copy the pageloader extension description=['get', 'pageloader'], name='get_pageloader', haltOnFailure=True, )) self.addStep(ShellCommand( command=['tar', 'jxvf', '%s/pageloader.tar.bz2' % self.base_dir], workdir='%s/fennec' % self.base_dir, description=['unpack', 'pageloader'], name='unpack_pageloader', haltOnFailure=True, ))
def simpleCVS(cvsroot, module, path, workdir): #TODO: Add ability to pull up to a certain date (-D) return ShellCommand( command=['cvs', '-d', cvsroot, 'co', '-d', module, path], description=[module, 'checkout'], name="%s_checkout" % module, workdir=workdir, haltOnFailure=True, )
def simpleHG(host, repo, workdir, target=None): if target is None: target = repo.rstrip('/').split('/')[-1] steps = [] steps.append(ShellCommand( command=['rm', '-rf', target], workdir=workdir, description=['delete', target, 'repo'], name='rm_%s_repo' % target, )) steps.append(ShellCommand( command=['hg', '-v', 'clone', '%s/%s' % (host, repo), target], workdir=workdir, description=[target, 'clone'], name='clone_%s' % target, haltOnFailure=True, )) return steps
def addSetupSteps(self): self.addStep(ShellCommand( command=['mkdir', '-p', '%s/maemkit_logs' % self.base_dir], name='mkdir_maemkit_logs', description=['make', 'maemkit', 'log', 'dir'], )) if self.maemkit_tarball is not None: self.addTarballMaemkitSteps() else: self.addSteps(simpleHG(self.hg_host, self.maemkit_repo_path, self.base_dir, 'maemkit'))
def addCleanupSteps(self): self.addStep(ShellCommand( command=['sh', '-c', "rm talos/tp4 ; rm -rf %s" % self.cleanup_glob], # If you don't delete the tp4 symlink before cleanup # bad things happen! workdir=self.base_dir, name='cleanup', description='cleanup', haltOnFailure=True, ))
def addRunSteps(self): perfconf_cmd=['python', "PerfConfigurator.py", '-v', '-e', '%s/fennec/fennec' % self.base_dir, '-t', WithProperties("%(slavename)s"), '--branch', self.branch, '--branchName', self.branch, '--activeTests', self.test, '--sampleConfig', self.talos_config_file, '--browserWait', str(self.browser_wait), '--resultsServer', self.results_server, self.nochrome, '--resultsLink', '/server/collect.cgi', '--output', 'local.config'] runtest_cmd=["python", "run_tests.py", "--noisy", "--debug", "local.config"] self.addStep(ShellCommand( command=perfconf_cmd, workdir="%s/talos" % self.base_dir, description=['perfconfig', self.test], haltOnFailure=True, name='perfconfig_%s' % self.test, )) self.addStep(ShellCommand( command=["cat", "local.config"], workdir="%s/talos" % self.base_dir, description=["config", self.test], name='cat_config_%s' % self.test, haltOnFailure=True, )) self.addStep(MozillaRunPerfTests( command=runtest_cmd, workdir = "%s/talos" % self.base_dir, description=['run', self.test], timeout=self.timeout + 2*60, #make sure that can hit the test #timeout before we kill the command warnOnWarnings=True, haltOnFailure=False, name='run_test_%s' % self.test, ))
def addStartupSteps(self): self.addStep(ShellCommand( command=['echo', WithProperties("TinderboxPrint: %(slavename)s")], description="hostname", name='hostname' )) self.addStep(ShellCommand( command=['sh', '-c', 'echo TinderboxPrint: $(uname -m)'], description='arch', name='arch', )) self.addStep(ShellCommand( command=['free'], workdir=self.base_dir, description=['memory', 'check'], name='memory_check', )) self.addStep(ShellCommand( command=['df', '-h'], workdir=self.base_dir, description=['check', 'disk'], name='disk_check', ))
def addObtainBuildSteps(self): MobileTalosFactory.addObtainBuildSteps(self) self.addStep(SetProperty( command=['echo', WithProperties("%(fileURL)s")], extract_fn=download_dir, name='download_dir', description=['download', 'directory', 'property'], )) self.addStep(ShellCommand( command=['wget', WithProperties("%(download_dir)s/fennec-%(version)s.en-US.linux-gnueabi-arm.tests.zip"), '-O', 'fennec-tests.zip'], workdir=self.base_dir, haltOnFailure=True, name="get_tests", description=['get', 'unit', 'tests'], )) self.addStep(ShellCommand( command=['unzip', '%s/fennec-tests.zip' % self.base_dir], workdir="%s/fennec" % self.base_dir, name='unpack_test', description=['unpack', 'tests'], haltOnFailure=True, ))
def addObtainBuildSteps(self): self.addDownloadBuildStep() self.addUnpackBuildSteps() #If people *hate* lambdas, i could define simple functions mozilla_fn = lambda x,y,z: parse_build_info(' '.join([y,z]), 'mozilla') mobile_fn = lambda x,y,z: parse_build_info(' '.join([y,z]), 'mobile') self.addStep(SetProperty( command=['cat', 'platform.ini'], workdir='%s/fennec' % self.base_dir, extract_fn=mozilla_fn, description=['get', 'mobile', 'revision'], name='mozilla_rev', )) self.addStep(SetProperty( command=['cat', 'application.ini'], workdir='%s/fennec' % self.base_dir, extract_fn=mobile_fn, description=['get', 'mobile', 'revision'], name='mobile_rev', )) self.addStep(SetProperty( command=['echo', WithProperties('%(mozilla_changeset)s:%(mobile_changeset)s')], property='got_revision', name='set_got_revision', description=['set_got_revision'] )) self.addStep(ShellCommand( command=['echo', 'TinderboxPrint:', WithProperties('<a href=%(mozilla_repository)s/rev/%(mozilla_changeset)s' + 'title="Built from Mozilla revision %(mozilla_changeset)s">' + 'moz:%(mozilla_changeset)s</a> <br />' + '<a href=%(mobile_repository)s/rev/%(mobile_changeset)s' + 'title="Built from Mobile revision %(mobile_changeset)s">' + 'mobile:%(mobile_changeset)s</a>')], description=['list', 'revisions'], name='rev_info', ))
def __init__(self, timeout=2 * 3600, maxTime=4 * 3600, **kwargs): self.my_shellcommand = ShellCommand ShellCommand.__init__(self, timeout=timeout, maxTime=maxTime, **kwargs)
def __init__(self, scpString, targetSubDir=None, **kwargs): ShellCommand.__init__(self, **kwargs) self.addFactoryArguments(scpString=scpString) (self.sshHost, self.sshDir) = scpString.split(":") self.super_class = ShellCommand self.targetSubDir = targetSubDir
def __init__(self, objdir, username, milestone, platform, remoteHost, remoteBasePath, group=None, chmodMode=755, sshKey=None, releaseToDated=True, releaseToLatest=True, releaseToTinderboxBuilds=True, tinderboxBuildsDir=None, releaseToCandidates=False, remoteCandidatesPath=None, dependToDated=True, uploadCompleteMar=True, uploadLangPacks=False, packageGlob=None, **kwargs): """ @type objdir: string @param objdir: The obj directory used for the build. This is needed to find the packages in the source tree. @type username: string @param username: The username used to login with on the remote host. The buildslaves should have passwordless logins to this account. @type milestone: string @param milestone: The milestone of the build (eg, trunk) @type platform: string @param platform: The platform we are uploading for. One of 'win32', 'linux', 'linux64', 'macosx' or 'macosx64'. @type remoteHost: string @param remoteHost: The server to upload the builds to. @type remoteBasePath: string @param remoteBasePath: The directory on the server used as a base path for these builds. eg: /home/ftp/pub/firefox @type group: string @param group: If group is set, any files uploaded will be chgrp'ed to it. Default: None @type chmodMode: int @param chmodMode: The mode used when fixing permissions on remoteHost. Default: 755 @type sshKey: string @param sshKey: If defined, the filename of the ssh key to use. It should be relative to ${HOME}/.ssh/. Default: None @type releaseToDated: bool @param releaseToDated: If True, builds will be pushed to 'remoteBasePath'/nightly/yyyy/mm/yyyy-mm-dd-hh-milestone. This directory will also be symlinked in 'remoteBasePath'/nightly/. Generally, this should be True for nightlies. Default: True @type releaseToLatest: bool @param releaseToLatest: If True, builds will be pushed to 'remoteBasePath'/nightly/latest-milestone. If releaseToDated=True, builds will be copied from 'remoteBasePath'/nightly/yyyy/mm/yyyy-mm-dd-hh-milestone. Otherwise, builds will be uploaded from the slave. Generally, this should be True for nightlies. Default: True @type releaseToTinderboxBuilds: bool @param releaseToTinderboxBuilds: If True, builds will be pushed to 'remoteBasePath'/tinderbox-builds/$hostname. This should generally be set to True for all builds. Default: True @type tinderboxBuildsDir: string @param tinderboxBuildsDir: This option only has effect when releaseToTinderboxBuilds is True. If this option is None (default), builds will be uploaded to: tinderbox-builds/builderName If otherwise set builds will be uploaded to tinderbox-builds/tinderboxBuildsDir. @type releaseToCandidates: bool @param releaseToCandidates: If True, builds will be pushed to remoteCandidatesDir. This should only be set for releases. @type remoteCandidatesDir: string @param remoteCandidatesDir: This option only has effect, and is required, when releaseToCandidates is True. @type dependToDated: This option only has effect when releaseToTinderboxBuilds is True. When dependToDated is True builds will be placed in a subdirectory named for the build start time (in unix time) when being pushed to the tinderbox-builds dir. For example: tinderbox-builds/builder/1203094573. The option defaults to True. @type uploadCompleteMar: bool @param uploadCompleteMar: When True, the MozillaStageUpload will upload the complete mar file found in dist/update to the datedDir/latestDir. This option only applies when releaseToDated or releaseToLatest is True. Default: True @type uploadLangPacks: bool @param uploadLangPacks: When True, the MozillaStageUpload will upload language pack XPIs to the datedDir/latestDir. This option only applies when releaseToDated or releaseToLatest is True. Default: False @type packageGlob: string @param packageGlob: The shell wildcard pattern that expresses the build files we will be uploading. Default: each platform gets a sensible default in objdir/dist/*.{ext} with ext tailored for that platform (i.e. .zip, .dmg, .tar.gz) """ ShellCommand.__init__(self, **kwargs) self.addFactoryArguments(objdir=objdir, username=username, milestone=milestone, platform=platform, remoteHost=remoteHost, remoteBasePath=remoteBasePath, packageGlob=packageGlob, group=group, chmodMode=chmodMode, sshKey=sshKey, releaseToDated=releaseToDated, releaseToLatest=releaseToLatest, releaseToTinderboxBuilds=releaseToTinderboxBuilds, tinderboxBuildsDir=tinderboxBuildsDir, releaseToCandidates=releaseToCandidates, remoteCandidatesPath=remoteCandidatesPath, dependToDated=dependToDated, uploadCompleteMar=uploadCompleteMar, uploadLangPacks=uploadLangPacks) assert platform in getSupportedPlatforms() if releaseToCandidates: assert remoteCandidatesPath self.objdir = objdir self.username = username self.milestone = milestone self.platform = platform self.remoteHost = remoteHost self.remoteBasePath = remoteBasePath self.packageGlob = packageGlob self.group = group self.chmodMode = chmodMode self.sshKey = sshKey self.releaseToDated = releaseToDated self.releaseToLatest = releaseToLatest self.releaseToCandidates = releaseToCandidates self.remoteCandidatesPath = remoteCandidatesPath self.releaseToTinderboxBuilds = releaseToTinderboxBuilds self.tinderboxBuildsDir = tinderboxBuildsDir self.dependToDated = dependToDated self.uploadCompleteMar = uploadCompleteMar self.uploadLangPacks = uploadLangPacks self.description = ["uploading package(s) to", remoteHost] self.descriptionDone = ["upload package(s) to", remoteHost]
def start(self): datedDir = self.getLongDatedPath() latestDir = self.getLatestPath() tinderboxBuildsDir = self.getTinderboxBuildsPath() candidatesDir = self.getCandidatesPath() commands = [] if self.releaseToDated: # 1) Create the directory on the staging server. # 2) Upload the package(s). # 3) Fix the permissions on the package(s). # 4) Maybe adjust the group on the package(s). # 5) Symlink the longer dated directory to the shorter one. cmd = "" cmd += self.createDirCommand(datedDir) + " && " + \ self.uploadCommand(datedDir) if self.uploadCompleteMar: cmd += " && " + self.uploadCompleteMarCommand(datedDir) if self.uploadLangPacks: cmd += " && " + self.uploadLangPacksCommand(datedDir) cmd += " && " + self.chmodCommand(datedDir) if self.group: cmd += " && " + self.chgrpCommand(datedDir) cmd += " && " + self.symlinkDateDirCommand(datedDir) commands.append(cmd) if self.releaseToLatest: # 1) Create the directory on the staging server. # 2) If there was a dated release, rsync those files to the # latest-(milestone) directory. # 3) If not, upload the package(s). # 4) Fix the permissions on the package(s). # 5) Maybe adjust the group on the package(s). cmd = "" cmd += self.createDirCommand(latestDir) + " && " if self.releaseToDated: cmd += self.syncCommand(datedDir, latestDir) + " && " else: cmd += self.uploadCommand(latestDir) + " && " if self.uploadCompleteMar: cmd += self.uploadCompleteMarCommand(latestDir) + " && " if self.uploadLangPacks: cmd += self.uploadLangPacksCommand(latestDir) + " && " cmd += self.chmodCommand(latestDir) if self.group: cmd += " && " + self.chgrpCommand(latestDir) commands.append(cmd) if self.releaseToTinderboxBuilds: # 1) Create the directory on the staging server. # 2) Upload the package(s). # 3) Fix the permissions on the package(s). # 4) Maybe adjust the group on the package(s). cmd = "" cmd += self.createDirCommand(tinderboxBuildsDir) + " && " + \ self.uploadCommand(tinderboxBuildsDir) + " && " + \ self.chmodCommand(tinderboxBuildsDir) if self.group: cmd += " && " + self.chgrpCommand(tinderboxBuildsDir) commands.append(cmd) if self.releaseToCandidates: # 1) Create the directory on the staging server. # 2) Upload the package(s). # 3) Fix the permissions on the package(s). # 4) Maybe adjust the group on the package(s). cmd = "" cmd += self.createDirCommand(candidatesDir) + " && " + \ self.uploadCommand(candidatesDir) + " && " + \ self.chmodCommand(candidatesDir) if self.group: cmd += " && " + self.chgrpCommand(candidatesDir) commands.append(cmd) finalCommand = ' && '.join(commands) self.setCommand(finalCommand) ShellCommand.start(self)
def __init__(self, **kwargs): self.my_shellcommand = ShellCommand ShellCommand.__init__(self, **kwargs)
def __init__(self, objdir, username, milestone, platform, remoteHost, remoteBasePath, group=None, chmodMode=755, sshKey=None, releaseToDated=True, releaseToLatest=True, releaseToTinderboxBuilds=True, tinderboxBuildsDir=None, releaseToCandidates=False, remoteCandidatesPath=None, dependToDated=True, uploadCompleteMar=True, uploadLangPacks=False, packageGlob=None, **kwargs): """ @type objdir: string @param objdir: The obj directory used for the build. This is needed to find the packages in the source tree. @type username: string @param username: The username used to login with on the remote host. The buildslaves should have passwordless logins to this account. @type milestone: string @param milestone: The milestone of the build (eg, trunk) @type platform: string @param platform: The platform we are uploading for. One of 'win32', 'linux', 'linux64', 'macosx' or 'macosx64'. @type remoteHost: string @param remoteHost: The server to upload the builds to. @type remoteBasePath: string @param remoteBasePath: The directory on the server used as a base path for these builds. eg: /home/ftp/pub/firefox @type group: string @param group: If group is set, any files uploaded will be chgrp'ed to it. Default: None @type chmodMode: int @param chmodMode: The mode used when fixing permissions on remoteHost. Default: 755 @type sshKey: string @param sshKey: If defined, the filename of the ssh key to use. It should be relative to ${HOME}/.ssh/. Default: None @type releaseToDated: bool @param releaseToDated: If True, builds will be pushed to 'remoteBasePath'/nightly/yyyy/mm/yyyy-mm-dd-hh-milestone. This directory will also be symlinked in 'remoteBasePath'/nightly/. Generally, this should be True for nightlies. Default: True @type releaseToLatest: bool @param releaseToLatest: If True, builds will be pushed to 'remoteBasePath'/nightly/latest-milestone. If releaseToDated=True, builds will be copied from 'remoteBasePath'/nightly/yyyy/mm/yyyy-mm-dd-hh-milestone. Otherwise, builds will be uploaded from the slave. Generally, this should be True for nightlies. Default: True @type releaseToTinderboxBuilds: bool @param releaseToTinderboxBuilds: If True, builds will be pushed to 'remoteBasePath'/tinderbox-builds/$hostname. This should generally be set to True for all builds. Default: True @type tinderboxBuildsDir: string @param tinderboxBuildsDir: This option only has effect when releaseToTinderboxBuilds is True. If this option is None (default), builds will be uploaded to: tinderbox-builds/builderName If otherwise set builds will be uploaded to tinderbox-builds/tinderboxBuildsDir. @type releaseToCandidates: bool @param releaseToCandidates: If True, builds will be pushed to remoteCandidatesDir. This should only be set for releases. @type remoteCandidatesDir: string @param remoteCandidatesDir: This option only has effect, and is required, when releaseToCandidates is True. @type dependToDated: This option only has effect when releaseToTinderboxBuilds is True. When dependToDated is True builds will be placed in a subdirectory named for the build start time (in unix time) when being pushed to the tinderbox-builds dir. For example: tinderbox-builds/builder/1203094573. The option defaults to True. @type uploadCompleteMar: bool @param uploadCompleteMar: When True, the MozillaStageUpload will upload the complete mar file found in dist/update to the datedDir/latestDir. This option only applies when releaseToDated or releaseToLatest is True. Default: True @type uploadLangPacks: bool @param uploadLangPacks: When True, the MozillaStageUpload will upload language pack XPIs to the datedDir/latestDir. This option only applies when releaseToDated or releaseToLatest is True. Default: False @type packageGlob: string @param packageGlob: The shell wildcard pattern that expresses the build files we will be uploading. Default: each platform gets a sensible default in objdir/dist/*.{ext} with ext tailored for that platform (i.e. .zip, .dmg, .tar.gz) """ ShellCommand.__init__(self, **kwargs) self.addFactoryArguments( objdir=objdir, username=username, milestone=milestone, platform=platform, remoteHost=remoteHost, remoteBasePath=remoteBasePath, packageGlob=packageGlob, group=group, chmodMode=chmodMode, sshKey=sshKey, releaseToDated=releaseToDated, releaseToLatest=releaseToLatest, releaseToTinderboxBuilds=releaseToTinderboxBuilds, tinderboxBuildsDir=tinderboxBuildsDir, releaseToCandidates=releaseToCandidates, remoteCandidatesPath=remoteCandidatesPath, dependToDated=dependToDated, uploadCompleteMar=uploadCompleteMar, uploadLangPacks=uploadLangPacks) assert platform in getSupportedPlatforms() if releaseToCandidates: assert remoteCandidatesPath self.objdir = objdir self.username = username self.milestone = milestone self.platform = platform self.remoteHost = remoteHost self.remoteBasePath = remoteBasePath self.packageGlob = packageGlob self.group = group self.chmodMode = chmodMode self.sshKey = sshKey self.releaseToDated = releaseToDated self.releaseToLatest = releaseToLatest self.releaseToCandidates = releaseToCandidates self.remoteCandidatesPath = remoteCandidatesPath self.releaseToTinderboxBuilds = releaseToTinderboxBuilds self.tinderboxBuildsDir = tinderboxBuildsDir self.dependToDated = dependToDated self.uploadCompleteMar = uploadCompleteMar self.uploadLangPacks = uploadLangPacks self.description = ["uploading package(s) to", remoteHost] self.descriptionDone = ["upload package(s) to", remoteHost]