class CrossbowReport(CrossbowBuilder):
steps = Extend([
SetPropertyFromCommand(
'crossbow_job_id',
extract_fn=lambda stdout, stderr: stdout.strip(),
command=Crossbow(args=[
'--github-token',
util.Secret('ursabot/github_token'), 'latest-prefix',
crossbow_prefix
]),
workdir='arrow/dev/tasks'),
Crossbow(name='Generate and send nightly report',
args=util.FlattenList([
'--github-token',
util.Secret('ursabot/github_token'), 'report', '--send',
'--poll', '--poll-max-minutes', 120,
'--poll-interval-minutes', 15, '--sender-name',
'Crossbow', '--sender-email', '*****@*****.**',
'--recipient-email', '*****@*****.**',
'--smtp-user',
util.Secret('crossbow/smtp_user'), '--smtp-password',
util.Secret('crossbow/smtp_password'),
util.Property('crossbow_job_id')
]),
workdir='arrow/dev/tasks'),
Crossbow(name="Update Crossbow's Github page",
args=util.FlattenList([
'--github-token',
util.Secret('ursabot/github_token'), 'github-page',
'generate', '-n', 20, '--github-push-token',
util.Secret('kszucs/github_status_token')
]),
workdir='arrow/dev/tasks')
])
class CrossbowTrigger(DockerBuilder):
tags = ['crossbow']
steps = [
GitHub(name='Clone Arrow',
repourl=util.Property('repository'),
workdir='arrow',
mode='full'),
GitHub(
name='Clone Crossbow',
# TODO(kszucs): read it from the comment and set as a property
repourl='https://github.com/ursa-labs/crossbow',
workdir='crossbow',
branch='master',
mode='full',
# quite misleasing option, but it prevents checking out the branch
# set in the sourcestamp by the pull request, which refers to arrow
alwaysUseLatest=True),
Crossbow(args=util.FlattenList([
'--github-token',
util.Secret('ursabot/github_token'), 'submit', '--output',
'job.yml', '--job-prefix', 'ursabot', '--arrow-remote',
util.Property('repository'),
util.Property('crossbow_args', [])
]),
workdir='arrow/dev/tasks',
result_file='job.yml')
]
images = images.filter(name='crossbow', tag='worker')
class CppBenchmark(DockerBuilder):
"""Run C++ benchmarks via the Archery CLI tool
This builder is parametrized with builtbot properties which are set by
the github hook, for more see commands.py
"""
tags = ['arrow', 'cpp', 'benchmark']
properties = dict(
CMAKE_INSTALL_PREFIX='/usr/local',
CMAKE_INSTALL_LIBDIR='lib'
)
steps = [
checkout_arrow,
Pip(['install', '-e', '.'], workdir='dev/archery'),
Archery(
args=util.FlattenList([
'benchmark',
'diff',
'--output=diff.json',
util.Property('benchmark_options', []),
'WORKSPACE',
util.Property('benchmark_baseline', 'master')
]),
result_file='diff.json'
)
]
image_filter = Filter(
name='cpp-benchmark',
tag='worker',
variant=None, # plain linux images, not conda
platform=Filter(
arch='amd64', # until ARROW-5382: SSE on ARM NEON gets resolved
distro='ubuntu'
)
)
class CppBenchmark(DockerBuilder):
tags = ['arrow', 'cpp', 'benchmark']
properties = {
'CMAKE_INSTALL_PREFIX': '/usr/local',
'CMAKE_INSTALL_LIBDIR': 'lib'
}
steps = [
checkout_arrow,
Pip(['install', '-e', '.'], workdir='dev/archery'),
Archery(
args=util.FlattenList([
'benchmark',
'diff',
'--output=diff.json',
util.Property('benchmark_options', []),
'WORKSPACE',
util.Property('benchmark_baseline', 'master')
]),
result_file='diff.json'
)
]
images = images.filter(
name='cpp-benchmark',
os=startswith('ubuntu'),
arch='amd64', # until ARROW-5382: SSE on ARM NEON gets resolved
variant=None, # plain linux images, not conda
tag='worker'
)
def test_constructor(self):
# this checks that an exception is raised for invalid arguments
msg = 'No command was provided'
with pytest.raises(ValueError, match=msg):
ShellCommand()
cmd = ShellCommand(command='something')
assert cmd.command == util.FlattenList(['something'])
cmd = ShellCommand(command='something', args=['arg1', 'arg2'])
assert cmd.command == util.FlattenList(['something', ['arg1', 'arg2']])
cmd = ShellCommand(command=['echo', '1'])
assert cmd.command == util.FlattenList([('echo', '1')])
cmd = MyDockerCommand(args=['--help'])
assert cmd.command == util.FlattenList([['my-docker-binary'],
['--help']])
def __init__(self, args=tuple(), command=tuple(), **kwargs):
command, args = command or self.command, args or self.args
if not IRenderable.providedBy(command) and not command:
raise ValueError('No command was provided')
kwargs['command'] = util.FlattenList([command, args])
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
def __init__(self,
args=tuple(),
command=tuple(),
as_shell=False,
quote=True,
**kwargs):
command, args = command or self.command, args or self.args
if not IRenderable.providedBy(command) and not command:
raise ValueError('No command was provided')
cmd = util.FlattenList([command, args])
if as_shell:
# runs the command as is without quoting any arguments of it
cmd = util.Transform(' '.join, cmd)
kwargs['command'] = cmd
kwargs = self.setupShellMixin(kwargs)
super().__init__(**kwargs)
class CrossbowSubmit(CrossbowBuilder):
"""Submit crossbow jobs
This builder is driven via buildbot properties, the `crossbow_args`
property is either set by the github hook which parses the github comments
like `@ursabot package -g conda` (ror more see commands.py) or by
explicitly passing by NightlySchedulers.
"""
steps = Extend([
Crossbow(args=util.FlattenList([
'--output-file', 'result.yaml', '--github-token',
util.Secret('kszucs/github_status_token'), 'submit',
'--arrow-remote', arrow_repository, '--job-prefix',
crossbow_prefix,
util.Property('crossbow_args', [])
]),
workdir='arrow/dev/tasks',
result_file='result.yaml')
])
class CrossbowTrigger(DockerBuilder):
tags = ['crossbow']
env = {
'GIT_COMMITTER_NAME': 'ursabot',
'GIT_COMMITTER_EMAIL': '*****@*****.**'
}
steps = [
GitHub(
name='Clone Arrow',
repourl=util.Property('repository'),
workdir='arrow',
mode='full'
),
GitHub(
name='Clone Crossbow',
repourl=util.Property('crossbow_repo'),
workdir='crossbow',
branch='master',
mode='full',
# quite misleasing option, but it prevents checking out the branch
# set in the sourcestamp by the pull request, which refers to arrow
alwaysUseLatest=True
),
Crossbow(
args=util.FlattenList([
'--github-token', util.Secret('ursabot/github_token'),
'submit',
'--output', 'job.yml',
'--job-prefix', 'ursabot',
'--arrow-remote', util.Property('repository'),
util.Property('crossbow_args', [])
]),
workdir='arrow/dev/tasks',
result_file='job.yml'
)
]
images = images.filter(
name='crossbow',
tag='worker'
)