class CrossbowReport(CrossbowBuilder): steps = Extend([ SetPropertyFromCommand( 'crossbow_job_id', extract_fn=lambda stdout, stderr: stdout.strip(), command=Crossbow(args=[ '--github-token', util.Secret('ursabot/github_token'), 'latest-prefix', crossbow_prefix ]), workdir='arrow/dev/tasks'), Crossbow(name='Generate and send nightly report', args=util.FlattenList([ '--github-token', util.Secret('ursabot/github_token'), 'report', '--send', '--poll', '--poll-max-minutes', 120, '--poll-interval-minutes', 15, '--sender-name', 'Crossbow', '--sender-email', '*****@*****.**', '--recipient-email', '*****@*****.**', '--smtp-user', util.Secret('crossbow/smtp_user'), '--smtp-password', util.Secret('crossbow/smtp_password'), util.Property('crossbow_job_id') ]), workdir='arrow/dev/tasks'), Crossbow(name="Update Crossbow's Github page", args=util.FlattenList([ '--github-token', util.Secret('ursabot/github_token'), 'github-page', 'generate', '-n', 20, '--github-push-token', util.Secret('kszucs/github_status_token') ]), workdir='arrow/dev/tasks') ])
class CrossbowTrigger(DockerBuilder): tags = ['crossbow'] steps = [ GitHub(name='Clone Arrow', repourl=util.Property('repository'), workdir='arrow', mode='full'), GitHub( name='Clone Crossbow', # TODO(kszucs): read it from the comment and set as a property repourl='https://github.com/ursa-labs/crossbow', workdir='crossbow', branch='master', mode='full', # quite misleasing option, but it prevents checking out the branch # set in the sourcestamp by the pull request, which refers to arrow alwaysUseLatest=True), Crossbow(args=util.FlattenList([ '--github-token', util.Secret('ursabot/github_token'), 'submit', '--output', 'job.yml', '--job-prefix', 'ursabot', '--arrow-remote', util.Property('repository'), util.Property('crossbow_args', []) ]), workdir='arrow/dev/tasks', result_file='job.yml') ] images = images.filter(name='crossbow', tag='worker')
class CppBenchmark(DockerBuilder): """Run C++ benchmarks via the Archery CLI tool This builder is parametrized with builtbot properties which are set by the github hook, for more see commands.py """ tags = ['arrow', 'cpp', 'benchmark'] properties = dict( CMAKE_INSTALL_PREFIX='/usr/local', CMAKE_INSTALL_LIBDIR='lib' ) steps = [ checkout_arrow, Pip(['install', '-e', '.'], workdir='dev/archery'), Archery( args=util.FlattenList([ 'benchmark', 'diff', '--output=diff.json', util.Property('benchmark_options', []), 'WORKSPACE', util.Property('benchmark_baseline', 'master') ]), result_file='diff.json' ) ] image_filter = Filter( name='cpp-benchmark', tag='worker', variant=None, # plain linux images, not conda platform=Filter( arch='amd64', # until ARROW-5382: SSE on ARM NEON gets resolved distro='ubuntu' ) )
class CppBenchmark(DockerBuilder): tags = ['arrow', 'cpp', 'benchmark'] properties = { 'CMAKE_INSTALL_PREFIX': '/usr/local', 'CMAKE_INSTALL_LIBDIR': 'lib' } steps = [ checkout_arrow, Pip(['install', '-e', '.'], workdir='dev/archery'), Archery( args=util.FlattenList([ 'benchmark', 'diff', '--output=diff.json', util.Property('benchmark_options', []), 'WORKSPACE', util.Property('benchmark_baseline', 'master') ]), result_file='diff.json' ) ] images = images.filter( name='cpp-benchmark', os=startswith('ubuntu'), arch='amd64', # until ARROW-5382: SSE on ARM NEON gets resolved variant=None, # plain linux images, not conda tag='worker' )
def test_constructor(self): # this checks that an exception is raised for invalid arguments msg = 'No command was provided' with pytest.raises(ValueError, match=msg): ShellCommand() cmd = ShellCommand(command='something') assert cmd.command == util.FlattenList(['something']) cmd = ShellCommand(command='something', args=['arg1', 'arg2']) assert cmd.command == util.FlattenList(['something', ['arg1', 'arg2']]) cmd = ShellCommand(command=['echo', '1']) assert cmd.command == util.FlattenList([('echo', '1')]) cmd = MyDockerCommand(args=['--help']) assert cmd.command == util.FlattenList([['my-docker-binary'], ['--help']])
def __init__(self, args=tuple(), command=tuple(), **kwargs): command, args = command or self.command, args or self.args if not IRenderable.providedBy(command) and not command: raise ValueError('No command was provided') kwargs['command'] = util.FlattenList([command, args]) kwargs = self.setupShellMixin(kwargs) super().__init__(**kwargs)
def __init__(self, args=tuple(), command=tuple(), as_shell=False, quote=True, **kwargs): command, args = command or self.command, args or self.args if not IRenderable.providedBy(command) and not command: raise ValueError('No command was provided') cmd = util.FlattenList([command, args]) if as_shell: # runs the command as is without quoting any arguments of it cmd = util.Transform(' '.join, cmd) kwargs['command'] = cmd kwargs = self.setupShellMixin(kwargs) super().__init__(**kwargs)
class CrossbowSubmit(CrossbowBuilder): """Submit crossbow jobs This builder is driven via buildbot properties, the `crossbow_args` property is either set by the github hook which parses the github comments like `@ursabot package -g conda` (ror more see commands.py) or by explicitly passing by NightlySchedulers. """ steps = Extend([ Crossbow(args=util.FlattenList([ '--output-file', 'result.yaml', '--github-token', util.Secret('kszucs/github_status_token'), 'submit', '--arrow-remote', arrow_repository, '--job-prefix', crossbow_prefix, util.Property('crossbow_args', []) ]), workdir='arrow/dev/tasks', result_file='result.yaml') ])
class CrossbowTrigger(DockerBuilder): tags = ['crossbow'] env = { 'GIT_COMMITTER_NAME': 'ursabot', 'GIT_COMMITTER_EMAIL': '*****@*****.**' } steps = [ GitHub( name='Clone Arrow', repourl=util.Property('repository'), workdir='arrow', mode='full' ), GitHub( name='Clone Crossbow', repourl=util.Property('crossbow_repo'), workdir='crossbow', branch='master', mode='full', # quite misleasing option, but it prevents checking out the branch # set in the sourcestamp by the pull request, which refers to arrow alwaysUseLatest=True ), Crossbow( args=util.FlattenList([ '--github-token', util.Secret('ursabot/github_token'), 'submit', '--output', 'job.yml', '--job-prefix', 'ursabot', '--arrow-remote', util.Property('repository'), util.Property('crossbow_args', []) ]), workdir='arrow/dev/tasks', result_file='job.yml' ) ] images = images.filter( name='crossbow', tag='worker' )