def testFailure(self): self.setupStep( transfer.MultipleFileUpload(workersrcs=["srcfile", "srcdir"], masterdest=self.destdir)) self.expectCommands( Expect('stat', dict(file="srcfile", workdir='wkdir')) + Expect.update('stat', [stat.S_IFREG, 99, 99]) + 0, Expect('uploadFile', dict( workersrc="srcfile", workdir='wkdir', blocksize=16384, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + 1) self.expectOutcome( result=FAILURE, state_string="uploading 2 files (failure)") d = self.runStep() return d
def testBasic(self): self.setupStep( transfer.FileUpload(slavesrc='srcfile', masterdest=self.destfile)) self.expectCommands( Expect( 'uploadFile', dict(slavesrc="srcfile", workdir='wkdir', blocksize=16384, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(transfer._FileWriter))) + Expect.behavior(uploadString("Hello world!")) + 0) self.expectOutcome(result=SUCCESS, status_text=["uploading", "srcfile"]) d = self.runStep() return d
def testBasic(self): self.setupStep(transfer.JSONPropertiesDownload("props.json")) self.step.build.setProperty('key1', 'value1', 'test') read = [] self.expectCommands( Expect('downloadFile', dict( workerdest="props.json", workdir='wkdir', blocksize=16384, maxsize=None, mode=None, reader=ExpectRemoteRef(remotetransfer.StringFileReader)) ) + Expect.behavior(downloadString(read.append)) + 0) self.expectOutcome( result=SUCCESS, state_string="downloading to props.json") yield self.runStep() # we decode as key order is dependent of python version self.assertEqual(json.loads((b''.join(read)).decode()), { "properties": {"key1": "value1"}, "sourcestamps": []})
def testWorker2_16(self): self.setupStep(transfer.FileUpload(workersrc='srcfile', masterdest=self.destfile), worker_version={'*': '2.16'}) self.expectCommands( Expect( 'uploadFile', dict(slavesrc="srcfile", workdir='wkdir', blocksize=262144, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + Expect.behavior(uploadString("Hello world!")) + 0) self.expectOutcome(result=SUCCESS, state_string="uploading srcfile") d = self.runStep() return d
def test_url_text(self): self.setupStep(transfer.DirectoryUpload(workersrc="srcdir", masterdest=self.destdir, url="http://server/dir", urlText='url text')) self.step.addURL = Mock() self.expectCommands( Expect('uploadDirectory', {'workersrc': 'srcdir', 'workdir': 'wkdir', 'blocksize': 16384, 'compress': None, 'maxsize': None, 'writer': ExpectRemoteRef(remotetransfer.DirectoryWriter)}) + Expect.behavior(uploadTarFile('fake.tar', test="Hello world!")) + 0) self.expectOutcome(result=SUCCESS, state_string="uploading srcdir") yield self.runStep() self.step.addURL.assert_called_once_with("url text", "http://server/dir")
def testDirectory(self): self.setupStep( transfer.MultipleFileUpload(workersrcs=["srcdir"], masterdest=self.destdir)) self.expectCommands( Expect('stat', dict(file="srcdir", workdir='wkdir')) + Expect.update('stat', [stat.S_IFDIR, 99, 99]) + 0, Expect('uploadDirectory', dict( slavesrc="srcdir", workdir='wkdir', blocksize=16384, compress=None, maxsize=None, writer=ExpectRemoteRef(remotetransfer.DirectoryWriter))) + Expect.behavior(uploadTarFile('fake.tar', test="Hello world!")) + 0) self.expectOutcome(result=SUCCESS, state_string="uploading 1 file") d = self.runStep() return d
def test_getFileContentFromWorker2_16(self): @defer.inlineCallbacks def testFunc(x): res = yield x.getFileContentFromWorker("file.txt") self.assertEqual(res, "Hello world!") self.setupStep( CompositeUser(testFunc), worker_version={'*': '2.16'}) self.expectCommands( Expect('uploadFile', dict( slavesrc="file.txt", workdir='wkdir', blocksize=32 * 1024, maxsize=None, writer=ExpectRemoteRef(remotetransfer.StringFileWriter))) + Expect.behavior(uploadString("Hello world!")) + 0 ) self.expectOutcome(result=SUCCESS) return self.runStep()
def testFile(self): self.setupStep( transfer.MultipleFileUpload(workersrcs=["srcfile"], masterdest=self.destdir)) self.expectCommands( Expect('stat', dict(file="srcfile", workdir='wkdir')) + Expect.update('stat', [stat.S_IFREG, 99, 99]) + 0, Expect('uploadFile', dict( slavesrc="srcfile", workdir='wkdir', blocksize=16384, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + Expect.behavior(uploadString("Hello world!")) + 0) self.expectOutcome(result=SUCCESS, state_string="uploading 1 file") d = self.runStep() return d
def testFailure(self): self.setupStep( transfer.DirectoryUpload(workersrc="srcdir", masterdest=self.destdir)) self.expectCommands( Expect( 'uploadDirectory', dict(workersrc="srcdir", workdir='wkdir', blocksize=16384, compress=None, maxsize=None, writer=ExpectRemoteRef(remotetransfer.DirectoryWriter))) + 1) self.expectOutcome(result=FAILURE, state_string="uploading srcdir (failure)") yield self.runStep()
def test_success(self): self.setupStep(UploadBuiltProduct()) self.setProperty('fullPlatform', 'mac-sierra') self.setProperty('configuration', 'release') self.setProperty('architecture', 'x86_64') self.setProperty('patch_id', '1234') self.expectHidden(False) self.expectRemoteCommands( Expect('uploadFile', dict( workersrc='WebKitBuild/release.zip', workdir='wkdir', blocksize=1024 * 256, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter), )) + Expect.behavior(uploadFileWithContentsOfString('Dummy zip file content.')) + 0, ) self.expectUploadedFile('public_html/archives/mac-sierra-x86_64-release/1234.zip') self.expectOutcome(result=SUCCESS, state_string='uploading release.zip') return self.runStep()
async def test_result_log_from_file(self): content = fixture.read_text() self.setupStep( MyStepWithResult(workdir='build', result_file='result.json') ) self.expectCommands( Expect('uploadFile', dict( workersrc='result.json', workdir='build', blocksize=32 * 1024, maxsize=None, writer=ExpectRemoteRef(remotetransfer.StringFileWriter)) ) + Expect.behavior(upload_string(content)) + 0 ) self.expectLogfile('result', content) self.expectOutcome(result=SUCCESS) return await self.runStep()
def testWorker2_16(self): self.setupStep(transfer.DirectoryUpload(workersrc="srcdir", masterdest=self.destdir), worker_version={'*': '2.16'}) self.expectCommands( Expect( 'uploadDirectory', dict(slavesrc="srcdir", workdir='wkdir', blocksize=16384, compress=None, maxsize=None, writer=ExpectRemoteRef(remotetransfer.DirectoryWriter))) + Expect.behavior(uploadTarFile('fake.tar', test="Hello world!")) + 0) self.expectOutcome(result=SUCCESS, state_string="uploading srcdir") d = self.runStep() return d
def testBasic(self): self.setupStep( transfer.DirectoryUpload(slavesrc="srcdir", masterdest=self.destdir)) self.expectCommands( Expect( 'uploadDirectory', dict(slavesrc="srcdir", workdir='wkdir', blocksize=16384, compress=None, maxsize=None, writer=ExpectRemoteRef(transfer._DirectoryWriter))) + Expect.behavior(uploadTarFile('fake.tar', test="Hello world!")) + 0) self.expectOutcome(result=SUCCESS, status_text=["uploading", "srcdir"]) d = self.runStep() return d
def testException(self): self.setupStep( transfer.FileUpload(workersrc='srcfile', masterdest=self.destfile)) behavior = UploadError(uploadString("Hello world!")) self.expectCommands( Expect('uploadFile', dict( workersrc="srcfile", workdir='wkdir', blocksize=262144, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + Expect.behavior(behavior)) self.expectOutcome( result=EXCEPTION, state_string="uploading srcfile (exception)") yield self.runStep() self.assertEqual(behavior.writer.cancel.called, True) self.assertEqual( len(self.flushLoggedErrors(RuntimeError)), 1)
def test_downloadFileContentToWorker(self): @defer.inlineCallbacks def testFunc(x): res = yield x.downloadFileContentToWorker("/path/dest1", "file text") self.assertEqual(res, None) exp_args = { 'maxsize': None, 'workdir': 'wkdir', 'mode': None, 'reader': ExpectRemoteRef(remotetransfer.FileReader), 'blocksize': 32768, 'workerdest': '/path/dest1' } self.setupStep(CompositeUser(testFunc)) self.expectCommands(Expect('downloadFile', exp_args)) self.expectOutcome(result=SUCCESS) return self.runStep()
def do_test_suppressions(self, step, supps_file='', stdout='', exp_warning_count=0, exp_warning_log='', exp_exception=False): self.setupStep(step) # Invoke the expected callbacks for the suppression file upload. Note # that this assumes all of the remote_* are synchronous, but can be # easily adapted to suit if that changes (using inlineCallbacks) def upload_behavior(command): writer = command.args['writer'] writer.remote_write(supps_file) writer.remote_close() command.rc = 0 self.expectCommands( # step will first get the remote suppressions file Expect('uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='supps', workdir='wkdir', writer=ExpectRemoteRef(slave.StringFileWriter))) + Expect.behavior(upload_behavior), # and then run the command ExpectShell(workdir='wkdir', usePTY='slave-config', command=["make"]) + ExpectShell.log('stdio', stdout=stdout) + 0 ) if exp_exception: self.expectOutcome(result=EXCEPTION, state_string="'make' (exception)") else: if exp_warning_count != 0: self.expectOutcome(result=WARNINGS, state_string="'make' (warnings)") self.expectLogfile("warnings (%d)" % exp_warning_count, exp_warning_log) else: self.expectOutcome(result=SUCCESS, state_string="'make'") self.expectProperty("warnings-count", exp_warning_count) return self.runStep()
def test_interrupt(self): self.setupStep( transfer.FileUpload(workersrc='srcfile', masterdest=self.destfile)) self.expectCommands( Expect('uploadFile', { 'workersrc': 'srcfile', 'workdir': 'wkdir', 'blocksize': 262144, 'maxsize': None, 'keepstamp': False, 'writer': ExpectRemoteRef(remotetransfer.FileWriter) }, interrupted=True) + 0) self.interrupt_nth_remote_command(0) self.expectOutcome(result=CANCELLED, state_string="uploading srcfile (cancelled)") self.expectLogfile('interrupt', 'interrupt reason') yield self.runStep()
def testException(self): self.setupStep( transfer.DirectoryUpload(workersrc='srcdir', masterdest=self.destdir)) behavior = UploadError(uploadTarFile('fake.tar', test="Hello world!")) self.expectCommands( Expect('uploadDirectory', dict( workersrc="srcdir", workdir='wkdir', blocksize=16384, compress=None, maxsize=None, writer=ExpectRemoteRef(remotetransfer.DirectoryWriter))) + Expect.behavior(behavior)) self.expectOutcome( result=EXCEPTION, state_string="uploading srcdir (exception)") yield self.runStep() self.assertEqual(behavior.writer.cancel.called, True) self.assertEqual( len(self.flushLoggedErrors(RuntimeError)), 1)
def test_success(self): self.setupStep(UploadTestResults()) self.setProperty('configuration', 'release') self.setProperty('architecture', 'x86_64') self.setProperty('patch_id', '1234') self.setProperty('buildername', 'macOS-Sierra-Release-WK2-Tests-EWS') self.setProperty('buildnumber', '12') self.expectHidden(False) self.expectRemoteCommands( Expect('uploadFile', dict( workersrc='layout-test-results.zip', workdir='wkdir', blocksize=1024 * 256, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter), )) + Expect.behavior(uploadFileWithContentsOfString('Dummy zip file content.')) + 0, ) self.expectUploadedFile('public_html/results/macOS-Sierra-Release-WK2-Tests-EWS/r1234-12.zip') self.expectOutcome(result=SUCCESS, state_string='uploading layout-test-results.zip') return self.runStep()
def testDescriptionDone(self): self.setupStep( transfer.FileUpload(workersrc=__file__, masterdest=self.destfile, url="http://server/file", descriptionDone="Test File Uploaded")) self.step.addURL = Mock() self.expectCommands( Expect('uploadFile', dict( workersrc=__file__, workdir='wkdir', blocksize=262144, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + Expect.behavior(uploadString("Hello world!")) + 0) self.expectOutcome( result=SUCCESS, state_string="Test File Uploaded") d = self.runStep() return d
def test_url_text(self): self.setupStep(transfer.MultipleFileUpload(workersrcs=["srcfile"], masterdest=self.destdir, url="http://server/dir", urlText='url text')) self.step.addURL = Mock() self.expectCommands( Expect('stat', {'file': "srcfile", 'workdir': 'wkdir'}) + Expect.update('stat', [stat.S_IFREG, 99, 99]) + 0, Expect('uploadFile', {'workersrc': "srcfile", 'workdir': 'wkdir', 'blocksize': 16384, 'maxsize': None, 'keepstamp': False, 'writer': ExpectRemoteRef(remotetransfer.FileWriter)}) + Expect.behavior(uploadString("Hello world!")) + 0) self.expectOutcome(result=SUCCESS, state_string="uploading 1 file") yield self.runStep() self.step.addURL.assert_called_once_with("url text", "http://server/dir")
def testURLText(self): self.setupStep( transfer.FileUpload(workersrc=__file__, masterdest=self.destfile, url="http://server/file", urlText="testfile")) self.step.addURL = Mock() self.expectCommands( Expect('uploadFile', dict( workersrc=__file__, workdir='wkdir', blocksize=262144, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + Expect.behavior(uploadString("Hello world!")) + 0) self.expectOutcome( result=SUCCESS, state_string="uploading %s" % os.path.basename(__file__)) yield self.runStep() self.step.addURL.assert_called_once_with( "testfile", "http://server/file")
def setupStep(self, yaml): super(TestReadConfFromYamlExecution, self).setupStep(yaml_parser.ReadConfFromYaml('main.yml')) # mock addStepsAfterCurrentStep self.step.build.addStepsAfterCurrentStep = lambda *args, **kw: None self.expectCommands( Expect( 'uploadFile', dict(workersrc='main.yml', workdir='wkdir', blocksize=262144, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + 0) if isinstance(yaml, RawYaml): yaml.filedump(self.step.masterdest) else: with open(self.step.masterdest, 'w') as fp: fp.write(yaml)
def test_mode_incremental_retry(self): self.setupStep( cvs.CVS( cvsroot=":pserver:[email protected]:/cvsroot", cvsmodule="mozilla/browser/", mode='incremental', login=True, retry=(0, 1))) self.expectCommands( ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, Expect('stat', dict(file='wkdir/.buildbot-patched', logEnviron=True)) + 1, Expect( 'uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='wkdir/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + 1, Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0, ExpectShell( workdir='', command=[ 'cvs', '-d', ':pserver:[email protected]:/cvsroot', '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/' ]) + 1, Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0, ExpectShell( workdir='', command=[ 'cvs', '-d', ':pserver:[email protected]:/cvsroot', '-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/' ]) + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) self.expectProperty('got_revision', '2012-09-09 12:00:39 +0000', 'CVS') return self.runStep()
def test_mode_full_copy_wrong_repo(self): self.setupStep( cvs.CVS( cvsroot=":pserver:[email protected]:/cvsroot", cvsmodule="mozilla/browser/", mode='full', method='copy', login=True)) self.expectCommands( ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0, Expect('stat', dict(file='wkdir/.buildbot-patched', logEnviron=True)) + 1, Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0, Expect( 'uploadFile', dict(blocksize=32768, maxsize=None, slavesrc='Root', workdir='source/CVS', writer=ExpectRemoteRef(shell.StringFileWriter))) + Expect.behavior(uploadString('the-end-of-the-universe')) + 0, Expect('rmdir', dict(dir='source', logEnviron=True)) + 0, ExpectShell( workdir='', command=[ 'cvs', '-d', ':pserver:[email protected]:/cvsroot', '-z3', 'checkout', '-d', 'source', 'mozilla/browser/' ]) + 0, Expect('cpdir', { 'fromdir': 'source', 'todir': 'build', 'logEnviron': True }) + 0, ) self.expectOutcome(result=SUCCESS, status_text=["update"]) self.expectProperty('got_revision', '2012-09-09 12:00:39 +0000', 'CVS') return self.runStep()
def testBasic(self): self.setupStep(transfer.StringDownload("Hello World", "hello.txt")) self.step.worker = Mock() self.step.remote = Mock() # A place to store what gets read read = [] self.expectCommands( Expect('downloadFile', dict( workerdest="hello.txt", workdir='wkdir', blocksize=16384, maxsize=None, mode=None, reader=ExpectRemoteRef(remotetransfer.StringFileReader))) + Expect.behavior(downloadString(read.append)) + 0) self.expectOutcome( result=SUCCESS, state_string="downloading to hello.txt") yield self.runStep() def checkCalls(res): self.assertEqual(b''.join(read), b"Hello World")
def testGlob(self): self.setupStep( transfer.MultipleFileUpload( workersrcs=["src*"], masterdest=self.destdir, glob=True)) self.expectCommands( Expect('glob', dict(path=os.path.join('wkdir', 'src*'), logEnviron=False)) + Expect.update('files', ["srcfile"]) + 0, Expect('stat', dict(file="srcfile", workdir='wkdir')) + Expect.update('stat', [stat.S_IFREG, 99, 99]) + 0, Expect('uploadFile', dict( workersrc="srcfile", workdir='wkdir', blocksize=16384, maxsize=None, keepstamp=False, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + Expect.behavior(uploadString("Hello world!")) + 0, ) self.expectOutcome( result=SUCCESS, state_string="uploading 1 file") d = self.runStep() return d
def test_mode_full_clobber_revision_worker_2_16(self): self.setupStep( darcs.Darcs(repourl='http://localhost/darcs', mode='full', method='clobber'), dict(revision='abcdef01'), worker_version={'*': '2.16'}) self.expectCommands( ExpectShell(workdir='wkdir', command=['darcs', '--version']) + 0, Expect('stat', dict(file='wkdir/.buildbot-patched', logEnviron=True)) + 1, Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0, Expect('downloadFile', dict(blocksize=32768, maxsize=None, reader=ExpectRemoteRef( remotetransfer.StringFileReader), slavedest='.darcs-context', workdir='wkdir', mode=None)) + 0, ExpectShell(workdir='.', command=['darcs', 'get', '--verbose', '--lazy', '--repo-name', 'wkdir', '--context', '.darcs-context', 'http://localhost/darcs']) + 0, ExpectShell(workdir='wkdir', command=['darcs', 'changes', '--max-count=1']) + ExpectShell.log('stdio', stdout='Tue Aug 20 09:18:41 IST 2013 [email protected]') + 0, ) self.expectOutcome(result=SUCCESS) self.expectProperty( 'got_revision', 'Tue Aug 20 09:18:41 IST 2013 [email protected]', 'Darcs') return self.runStep()
def testTimestamp(self): self.setupStep( transfer.FileUpload(workersrc=__file__, masterdest=self.destfile, keepstamp=True)) timestamp = (os.path.getatime(__file__), os.path.getmtime(__file__)) self.expectCommands( Expect( 'uploadFile', dict(workersrc=__file__, workdir='wkdir', blocksize=16384, maxsize=None, keepstamp=True, writer=ExpectRemoteRef(remotetransfer.FileWriter))) + Expect.behavior(uploadString('test', timestamp=timestamp)) + 0) self.expectOutcome(result=SUCCESS, state_string="uploading %s" % os.path.basename(__file__)) d = self.runStep() @d.addCallback def checkTimestamp(_): desttimestamp = (os.path.getatime(self.destfile), os.path.getmtime(self.destfile)) srctimestamp = map(int, timestamp) desttimestamp = map(int, desttimestamp) self.assertEqual(srctimestamp[0], desttimestamp[0]) self.assertEqual(srctimestamp[1], desttimestamp[1]) return d
def testBasic(self): self.setupStep( transfer.DirectoryUpload(slavesrc="srcdir", masterdest=self.destdir)) def upload_behavior(command): from cStringIO import StringIO f = StringIO() archive = tarfile.TarFile(fileobj=f, name='fake.tar', mode='w') archive.addfile(tarfile.TarInfo("test"), StringIO("Hello World!")) writer = command.args['writer'] writer.remote_write(f.getvalue()) writer.remote_unpack() self.expectCommands( Expect('uploadDirectory', dict( slavesrc="srcdir", workdir='wkdir', blocksize=16384, compress=None, maxsize=None, writer=ExpectRemoteRef(transfer._DirectoryWriter))) + Expect.behavior(upload_behavior) + 0) self.expectOutcome(result=SUCCESS, status_text=["uploading", "srcdir"]) d = self.runStep() return d