def testFailure(self):
self.setupStep(
transfer.MultipleFileUpload(workersrcs=["srcfile", "srcdir"], masterdest=self.destdir))
self.expectCommands(
Expect('stat', dict(file="srcfile",
workdir='wkdir'))
+ Expect.update('stat', [stat.S_IFREG, 99, 99])
+ 0,
Expect('uploadFile', dict(
workersrc="srcfile", workdir='wkdir',
blocksize=16384, maxsize=None, keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter)))
+ 1)
self.expectOutcome(
result=FAILURE, state_string="uploading 2 files (failure)")
d = self.runStep()
return d
def testBasic(self):
self.setupStep(
transfer.FileUpload(slavesrc='srcfile', masterdest=self.destfile))
self.expectCommands(
Expect(
'uploadFile',
dict(slavesrc="srcfile",
workdir='wkdir',
blocksize=16384,
maxsize=None,
keepstamp=False,
writer=ExpectRemoteRef(transfer._FileWriter))) +
Expect.behavior(uploadString("Hello world!")) + 0)
self.expectOutcome(result=SUCCESS,
status_text=["uploading", "srcfile"])
d = self.runStep()
return d
def testBasic(self):
self.setupStep(transfer.JSONPropertiesDownload("props.json"))
self.step.build.setProperty('key1', 'value1', 'test')
read = []
self.expectCommands(
Expect('downloadFile', dict(
workerdest="props.json", workdir='wkdir',
blocksize=16384, maxsize=None, mode=None,
reader=ExpectRemoteRef(remotetransfer.StringFileReader))
)
+ Expect.behavior(downloadString(read.append))
+ 0)
self.expectOutcome(
result=SUCCESS, state_string="downloading to props.json")
yield self.runStep()
# we decode as key order is dependent of python version
self.assertEqual(json.loads((b''.join(read)).decode()), {
"properties": {"key1": "value1"}, "sourcestamps": []})
def testWorker2_16(self):
self.setupStep(transfer.FileUpload(workersrc='srcfile',
masterdest=self.destfile),
worker_version={'*': '2.16'})
self.expectCommands(
Expect(
'uploadFile',
dict(slavesrc="srcfile",
workdir='wkdir',
blocksize=262144,
maxsize=None,
keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter))) +
Expect.behavior(uploadString("Hello world!")) + 0)
self.expectOutcome(result=SUCCESS, state_string="uploading srcfile")
d = self.runStep()
return d
def test_url_text(self):
self.setupStep(transfer.DirectoryUpload(workersrc="srcdir", masterdest=self.destdir,
url="http://server/dir", urlText='url text'))
self.step.addURL = Mock()
self.expectCommands(
Expect('uploadDirectory', {'workersrc': 'srcdir', 'workdir': 'wkdir',
'blocksize': 16384, 'compress': None, 'maxsize': None,
'writer': ExpectRemoteRef(remotetransfer.DirectoryWriter)})
+ Expect.behavior(uploadTarFile('fake.tar', test="Hello world!"))
+ 0)
self.expectOutcome(result=SUCCESS,
state_string="uploading srcdir")
yield self.runStep()
self.step.addURL.assert_called_once_with("url text", "http://server/dir")
def testDirectory(self):
self.setupStep(
transfer.MultipleFileUpload(workersrcs=["srcdir"], masterdest=self.destdir))
self.expectCommands(
Expect('stat', dict(file="srcdir",
workdir='wkdir'))
+ Expect.update('stat', [stat.S_IFDIR, 99, 99])
+ 0,
Expect('uploadDirectory', dict(
slavesrc="srcdir", workdir='wkdir',
blocksize=16384, compress=None, maxsize=None,
writer=ExpectRemoteRef(remotetransfer.DirectoryWriter)))
+ Expect.behavior(uploadTarFile('fake.tar', test="Hello world!"))
+ 0)
self.expectOutcome(result=SUCCESS, state_string="uploading 1 file")
d = self.runStep()
return d
def test_getFileContentFromWorker2_16(self):
@defer.inlineCallbacks
def testFunc(x):
res = yield x.getFileContentFromWorker("file.txt")
self.assertEqual(res, "Hello world!")
self.setupStep(
CompositeUser(testFunc),
worker_version={'*': '2.16'})
self.expectCommands(
Expect('uploadFile', dict(
slavesrc="file.txt", workdir='wkdir',
blocksize=32 * 1024, maxsize=None,
writer=ExpectRemoteRef(remotetransfer.StringFileWriter))) +
Expect.behavior(uploadString("Hello world!")) +
0
)
self.expectOutcome(result=SUCCESS)
return self.runStep()
def testFile(self):
self.setupStep(
transfer.MultipleFileUpload(workersrcs=["srcfile"], masterdest=self.destdir))
self.expectCommands(
Expect('stat', dict(file="srcfile",
workdir='wkdir'))
+ Expect.update('stat', [stat.S_IFREG, 99, 99])
+ 0,
Expect('uploadFile', dict(
slavesrc="srcfile", workdir='wkdir',
blocksize=16384, maxsize=None, keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter)))
+ Expect.behavior(uploadString("Hello world!"))
+ 0)
self.expectOutcome(result=SUCCESS, state_string="uploading 1 file")
d = self.runStep()
return d
def testFailure(self):
self.setupStep(
transfer.DirectoryUpload(workersrc="srcdir",
masterdest=self.destdir))
self.expectCommands(
Expect(
'uploadDirectory',
dict(workersrc="srcdir",
workdir='wkdir',
blocksize=16384,
compress=None,
maxsize=None,
writer=ExpectRemoteRef(remotetransfer.DirectoryWriter))) +
1)
self.expectOutcome(result=FAILURE,
state_string="uploading srcdir (failure)")
yield self.runStep()
def test_success(self):
self.setupStep(UploadBuiltProduct())
self.setProperty('fullPlatform', 'mac-sierra')
self.setProperty('configuration', 'release')
self.setProperty('architecture', 'x86_64')
self.setProperty('patch_id', '1234')
self.expectHidden(False)
self.expectRemoteCommands(
Expect('uploadFile', dict(
workersrc='WebKitBuild/release.zip', workdir='wkdir',
blocksize=1024 * 256, maxsize=None, keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter),
))
+ Expect.behavior(uploadFileWithContentsOfString('Dummy zip file content.'))
+ 0,
)
self.expectUploadedFile('public_html/archives/mac-sierra-x86_64-release/1234.zip')
self.expectOutcome(result=SUCCESS, state_string='uploading release.zip')
return self.runStep()
async def test_result_log_from_file(self):
content = fixture.read_text()
self.setupStep(
MyStepWithResult(workdir='build', result_file='result.json')
)
self.expectCommands(
Expect('uploadFile', dict(
workersrc='result.json',
workdir='build',
blocksize=32 * 1024,
maxsize=None,
writer=ExpectRemoteRef(remotetransfer.StringFileWriter))
) +
Expect.behavior(upload_string(content)) +
0
)
self.expectLogfile('result', content)
self.expectOutcome(result=SUCCESS)
return await self.runStep()
def testWorker2_16(self):
self.setupStep(transfer.DirectoryUpload(workersrc="srcdir",
masterdest=self.destdir),
worker_version={'*': '2.16'})
self.expectCommands(
Expect(
'uploadDirectory',
dict(slavesrc="srcdir",
workdir='wkdir',
blocksize=16384,
compress=None,
maxsize=None,
writer=ExpectRemoteRef(remotetransfer.DirectoryWriter))) +
Expect.behavior(uploadTarFile('fake.tar', test="Hello world!")) +
0)
self.expectOutcome(result=SUCCESS, state_string="uploading srcdir")
d = self.runStep()
return d
def testBasic(self):
self.setupStep(
transfer.DirectoryUpload(slavesrc="srcdir",
masterdest=self.destdir))
self.expectCommands(
Expect(
'uploadDirectory',
dict(slavesrc="srcdir",
workdir='wkdir',
blocksize=16384,
compress=None,
maxsize=None,
writer=ExpectRemoteRef(transfer._DirectoryWriter))) +
Expect.behavior(uploadTarFile('fake.tar', test="Hello world!")) +
0)
self.expectOutcome(result=SUCCESS, status_text=["uploading", "srcdir"])
d = self.runStep()
return d
def testException(self):
self.setupStep(
transfer.FileUpload(workersrc='srcfile', masterdest=self.destfile))
behavior = UploadError(uploadString("Hello world!"))
self.expectCommands(
Expect('uploadFile', dict(
workersrc="srcfile", workdir='wkdir',
blocksize=262144, maxsize=None, keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter)))
+ Expect.behavior(behavior))
self.expectOutcome(
result=EXCEPTION, state_string="uploading srcfile (exception)")
yield self.runStep()
self.assertEqual(behavior.writer.cancel.called, True)
self.assertEqual(
len(self.flushLoggedErrors(RuntimeError)), 1)
def test_downloadFileContentToWorker(self):
@defer.inlineCallbacks
def testFunc(x):
res = yield x.downloadFileContentToWorker("/path/dest1",
"file text")
self.assertEqual(res, None)
exp_args = {
'maxsize': None,
'workdir': 'wkdir',
'mode': None,
'reader': ExpectRemoteRef(remotetransfer.FileReader),
'blocksize': 32768,
'workerdest': '/path/dest1'
}
self.setupStep(CompositeUser(testFunc))
self.expectCommands(Expect('downloadFile', exp_args))
self.expectOutcome(result=SUCCESS)
return self.runStep()
def do_test_suppressions(self, step, supps_file='', stdout='',
exp_warning_count=0, exp_warning_log='',
exp_exception=False):
self.setupStep(step)
# Invoke the expected callbacks for the suppression file upload. Note
# that this assumes all of the remote_* are synchronous, but can be
# easily adapted to suit if that changes (using inlineCallbacks)
def upload_behavior(command):
writer = command.args['writer']
writer.remote_write(supps_file)
writer.remote_close()
command.rc = 0
self.expectCommands(
# step will first get the remote suppressions file
Expect('uploadFile', dict(blocksize=32768, maxsize=None,
slavesrc='supps', workdir='wkdir',
writer=ExpectRemoteRef(slave.StringFileWriter)))
+ Expect.behavior(upload_behavior),
# and then run the command
ExpectShell(workdir='wkdir', usePTY='slave-config',
command=["make"])
+ ExpectShell.log('stdio', stdout=stdout)
+ 0
)
if exp_exception:
self.expectOutcome(result=EXCEPTION,
state_string="'make' (exception)")
else:
if exp_warning_count != 0:
self.expectOutcome(result=WARNINGS,
state_string="'make' (warnings)")
self.expectLogfile("warnings (%d)" % exp_warning_count,
exp_warning_log)
else:
self.expectOutcome(result=SUCCESS,
state_string="'make'")
self.expectProperty("warnings-count", exp_warning_count)
return self.runStep()
def test_interrupt(self):
self.setupStep(
transfer.FileUpload(workersrc='srcfile', masterdest=self.destfile))
self.expectCommands(
Expect('uploadFile', {
'workersrc': 'srcfile',
'workdir': 'wkdir',
'blocksize': 262144,
'maxsize': None,
'keepstamp': False,
'writer': ExpectRemoteRef(remotetransfer.FileWriter)
},
interrupted=True) + 0)
self.interrupt_nth_remote_command(0)
self.expectOutcome(result=CANCELLED,
state_string="uploading srcfile (cancelled)")
self.expectLogfile('interrupt', 'interrupt reason')
yield self.runStep()
def testException(self):
self.setupStep(
transfer.DirectoryUpload(workersrc='srcdir', masterdest=self.destdir))
behavior = UploadError(uploadTarFile('fake.tar', test="Hello world!"))
self.expectCommands(
Expect('uploadDirectory', dict(
workersrc="srcdir", workdir='wkdir',
blocksize=16384, compress=None, maxsize=None,
writer=ExpectRemoteRef(remotetransfer.DirectoryWriter)))
+ Expect.behavior(behavior))
self.expectOutcome(
result=EXCEPTION,
state_string="uploading srcdir (exception)")
yield self.runStep()
self.assertEqual(behavior.writer.cancel.called, True)
self.assertEqual(
len(self.flushLoggedErrors(RuntimeError)), 1)
def test_success(self):
self.setupStep(UploadTestResults())
self.setProperty('configuration', 'release')
self.setProperty('architecture', 'x86_64')
self.setProperty('patch_id', '1234')
self.setProperty('buildername', 'macOS-Sierra-Release-WK2-Tests-EWS')
self.setProperty('buildnumber', '12')
self.expectHidden(False)
self.expectRemoteCommands(
Expect('uploadFile', dict(
workersrc='layout-test-results.zip', workdir='wkdir',
blocksize=1024 * 256, maxsize=None, keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter),
))
+ Expect.behavior(uploadFileWithContentsOfString('Dummy zip file content.'))
+ 0,
)
self.expectUploadedFile('public_html/results/macOS-Sierra-Release-WK2-Tests-EWS/r1234-12.zip')
self.expectOutcome(result=SUCCESS, state_string='uploading layout-test-results.zip')
return self.runStep()
def testDescriptionDone(self):
self.setupStep(
transfer.FileUpload(workersrc=__file__, masterdest=self.destfile,
url="http://server/file", descriptionDone="Test File Uploaded"))
self.step.addURL = Mock()
self.expectCommands(
Expect('uploadFile', dict(
workersrc=__file__, workdir='wkdir',
blocksize=262144, maxsize=None, keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter)))
+ Expect.behavior(uploadString("Hello world!"))
+ 0)
self.expectOutcome(
result=SUCCESS,
state_string="Test File Uploaded")
d = self.runStep()
return d
def test_url_text(self):
self.setupStep(transfer.MultipleFileUpload(workersrcs=["srcfile"], masterdest=self.destdir,
url="http://server/dir", urlText='url text'))
self.step.addURL = Mock()
self.expectCommands(
Expect('stat', {'file': "srcfile", 'workdir': 'wkdir'})
+ Expect.update('stat', [stat.S_IFREG, 99, 99])
+ 0,
Expect('uploadFile', {'workersrc': "srcfile", 'workdir': 'wkdir',
'blocksize': 16384, 'maxsize': None, 'keepstamp': False,
'writer': ExpectRemoteRef(remotetransfer.FileWriter)})
+ Expect.behavior(uploadString("Hello world!"))
+ 0)
self.expectOutcome(result=SUCCESS,
state_string="uploading 1 file")
yield self.runStep()
self.step.addURL.assert_called_once_with("url text", "http://server/dir")
def testURLText(self):
self.setupStep(
transfer.FileUpload(workersrc=__file__, masterdest=self.destfile, url="http://server/file", urlText="testfile"))
self.step.addURL = Mock()
self.expectCommands(
Expect('uploadFile', dict(
workersrc=__file__, workdir='wkdir',
blocksize=262144, maxsize=None, keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter)))
+ Expect.behavior(uploadString("Hello world!"))
+ 0)
self.expectOutcome(
result=SUCCESS,
state_string="uploading %s" % os.path.basename(__file__))
yield self.runStep()
self.step.addURL.assert_called_once_with(
"testfile", "http://server/file")
def setupStep(self, yaml):
super(TestReadConfFromYamlExecution,
self).setupStep(yaml_parser.ReadConfFromYaml('main.yml'))
# mock addStepsAfterCurrentStep
self.step.build.addStepsAfterCurrentStep = lambda *args, **kw: None
self.expectCommands(
Expect(
'uploadFile',
dict(workersrc='main.yml',
workdir='wkdir',
blocksize=262144,
maxsize=None,
keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter))) + 0)
if isinstance(yaml, RawYaml):
yaml.filedump(self.step.masterdest)
else:
with open(self.step.masterdest, 'w') as fp:
fp.write(yaml)
def test_mode_incremental_retry(self):
self.setupStep(
cvs.CVS(
cvsroot=":pserver:[email protected]:/cvsroot",
cvsmodule="mozilla/browser/",
mode='incremental',
login=True,
retry=(0, 1)))
self.expectCommands(
ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0,
Expect('stat', dict(file='wkdir/.buildbot-patched',
logEnviron=True)) + 1,
Expect(
'uploadFile',
dict(blocksize=32768,
maxsize=None,
slavesrc='Root',
workdir='wkdir/CVS',
writer=ExpectRemoteRef(shell.StringFileWriter))) + 1,
Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0,
ExpectShell(
workdir='',
command=[
'cvs', '-d',
':pserver:[email protected]:/cvsroot',
'-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/'
]) + 1,
Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0,
ExpectShell(
workdir='',
command=[
'cvs', '-d',
':pserver:[email protected]:/cvsroot',
'-z3', 'checkout', '-d', 'wkdir', 'mozilla/browser/'
]) + 0,
)
self.expectOutcome(result=SUCCESS, status_text=["update"])
self.expectProperty('got_revision', '2012-09-09 12:00:39 +0000', 'CVS')
return self.runStep()
def test_mode_full_copy_wrong_repo(self):
self.setupStep(
cvs.CVS(
cvsroot=":pserver:[email protected]:/cvsroot",
cvsmodule="mozilla/browser/",
mode='full',
method='copy',
login=True))
self.expectCommands(
ExpectShell(workdir='wkdir', command=['cvs', '--version']) + 0,
Expect('stat', dict(file='wkdir/.buildbot-patched',
logEnviron=True)) + 1,
Expect('rmdir', dict(dir='wkdir', logEnviron=True)) + 0,
Expect(
'uploadFile',
dict(blocksize=32768,
maxsize=None,
slavesrc='Root',
workdir='source/CVS',
writer=ExpectRemoteRef(shell.StringFileWriter))) +
Expect.behavior(uploadString('the-end-of-the-universe')) + 0,
Expect('rmdir', dict(dir='source', logEnviron=True)) + 0,
ExpectShell(
workdir='',
command=[
'cvs', '-d',
':pserver:[email protected]:/cvsroot',
'-z3', 'checkout', '-d', 'source', 'mozilla/browser/'
]) + 0,
Expect('cpdir', {
'fromdir': 'source',
'todir': 'build',
'logEnviron': True
}) + 0,
)
self.expectOutcome(result=SUCCESS, status_text=["update"])
self.expectProperty('got_revision', '2012-09-09 12:00:39 +0000', 'CVS')
return self.runStep()
def testBasic(self):
self.setupStep(transfer.StringDownload("Hello World", "hello.txt"))
self.step.worker = Mock()
self.step.remote = Mock()
# A place to store what gets read
read = []
self.expectCommands(
Expect('downloadFile', dict(
workerdest="hello.txt", workdir='wkdir',
blocksize=16384, maxsize=None, mode=None,
reader=ExpectRemoteRef(remotetransfer.StringFileReader)))
+ Expect.behavior(downloadString(read.append))
+ 0)
self.expectOutcome(
result=SUCCESS, state_string="downloading to hello.txt")
yield self.runStep()
def checkCalls(res):
self.assertEqual(b''.join(read), b"Hello World")
def testGlob(self):
self.setupStep(
transfer.MultipleFileUpload(
workersrcs=["src*"], masterdest=self.destdir, glob=True))
self.expectCommands(
Expect('glob', dict(path=os.path.join('wkdir', 'src*'), logEnviron=False))
+ Expect.update('files', ["srcfile"])
+ 0,
Expect('stat', dict(file="srcfile",
workdir='wkdir'))
+ Expect.update('stat', [stat.S_IFREG, 99, 99])
+ 0,
Expect('uploadFile', dict(
workersrc="srcfile", workdir='wkdir',
blocksize=16384, maxsize=None, keepstamp=False,
writer=ExpectRemoteRef(remotetransfer.FileWriter)))
+ Expect.behavior(uploadString("Hello world!"))
+ 0,
)
self.expectOutcome(
result=SUCCESS, state_string="uploading 1 file")
d = self.runStep()
return d
def test_mode_full_clobber_revision_worker_2_16(self):
self.setupStep(
darcs.Darcs(repourl='http://localhost/darcs',
mode='full', method='clobber'),
dict(revision='abcdef01'),
worker_version={'*': '2.16'})
self.expectCommands(
ExpectShell(workdir='wkdir',
command=['darcs', '--version']) +
0,
Expect('stat', dict(file='wkdir/.buildbot-patched',
logEnviron=True)) +
1,
Expect('rmdir', dict(dir='wkdir',
logEnviron=True)) +
0,
Expect('downloadFile', dict(blocksize=32768, maxsize=None,
reader=ExpectRemoteRef(
remotetransfer.StringFileReader),
slavedest='.darcs-context', workdir='wkdir',
mode=None)) +
0,
ExpectShell(workdir='.',
command=['darcs', 'get', '--verbose', '--lazy',
'--repo-name', 'wkdir', '--context',
'.darcs-context', 'http://localhost/darcs']) +
0,
ExpectShell(workdir='wkdir',
command=['darcs', 'changes', '--max-count=1']) +
ExpectShell.log('stdio',
stdout='Tue Aug 20 09:18:41 IST 2013 [email protected]') +
0,
)
self.expectOutcome(result=SUCCESS)
self.expectProperty(
'got_revision', 'Tue Aug 20 09:18:41 IST 2013 [email protected]', 'Darcs')
return self.runStep()
def testTimestamp(self):
self.setupStep(
transfer.FileUpload(workersrc=__file__,
masterdest=self.destfile,
keepstamp=True))
timestamp = (os.path.getatime(__file__), os.path.getmtime(__file__))
self.expectCommands(
Expect(
'uploadFile',
dict(workersrc=__file__,
workdir='wkdir',
blocksize=16384,
maxsize=None,
keepstamp=True,
writer=ExpectRemoteRef(remotetransfer.FileWriter))) +
Expect.behavior(uploadString('test', timestamp=timestamp)) + 0)
self.expectOutcome(result=SUCCESS,
state_string="uploading %s" %
os.path.basename(__file__))
d = self.runStep()
@d.addCallback
def checkTimestamp(_):
desttimestamp = (os.path.getatime(self.destfile),
os.path.getmtime(self.destfile))
srctimestamp = map(int, timestamp)
desttimestamp = map(int, desttimestamp)
self.assertEqual(srctimestamp[0], desttimestamp[0])
self.assertEqual(srctimestamp[1], desttimestamp[1])
return d
def testBasic(self):
self.setupStep(
transfer.DirectoryUpload(slavesrc="srcdir", masterdest=self.destdir))
def upload_behavior(command):
from cStringIO import StringIO
f = StringIO()
archive = tarfile.TarFile(fileobj=f, name='fake.tar', mode='w')
archive.addfile(tarfile.TarInfo("test"), StringIO("Hello World!"))
writer = command.args['writer']
writer.remote_write(f.getvalue())
writer.remote_unpack()
self.expectCommands(
Expect('uploadDirectory', dict(
slavesrc="srcdir", workdir='wkdir',
blocksize=16384, compress=None, maxsize=None,
writer=ExpectRemoteRef(transfer._DirectoryWriter)))
+ Expect.behavior(upload_behavior)
+ 0)
self.expectOutcome(result=SUCCESS, status_text=["uploading", "srcdir"])
d = self.runStep()
return d
