class RustTest(DockerBuilder):
tags = ['arrow', 'rust']
env = dict(
ARROW_TEST_DATA=arrow_test_data_path,
PARQUET_TEST_DATA=parquet_test_data_path
)
steps = [
checkout_arrow,
Cargo(
args=['build'],
workdir='rust',
name='Rust Build'
),
Cargo(
args=['test'],
workdir='rust',
name='Rust Test'
)
]
image_filter = Filter(
name=Matching('rust*'),
tag='worker',
platform=Filter(
arch='amd64'
)
)
class CppTest(DockerBuilder):
tags = ['arrow', 'cpp', 'gandiva', 'parquet', 'plasma']
volumes = [
util.Interpolate('%(prop:builddir)s:/root/.ccache:rw')
]
properties = dict(
ARROW_GANDIVA='ON',
ARROW_PARQUET='ON',
ARROW_PLASMA='ON',
CMAKE_INSTALL_PREFIX='/usr/local',
CMAKE_INSTALL_LIBDIR='lib'
)
env = {
'PARQUET_TEST_DATA': parquet_test_data_path # for parquet
}
steps = [
checkout_arrow,
cpp_mkdir,
cpp_cmake,
cpp_compile,
cpp_install,
cpp_test
]
image_filter = Filter(
name='cpp',
tag='worker',
variant=None,
platform=Filter(
arch=AnyOf('amd64', 'arm64v8'),
distro='ubuntu'
)
)
class CppBenchmark(DockerBuilder):
"""Run C++ benchmarks via the Archery CLI tool
This builder is parametrized with builtbot properties which are set by
the github hook, for more see commands.py
"""
tags = ['arrow', 'cpp', 'benchmark']
properties = dict(
CMAKE_INSTALL_PREFIX='/usr/local',
CMAKE_INSTALL_LIBDIR='lib'
)
steps = [
checkout_arrow,
Pip(['install', '-e', '.'], workdir='dev/archery'),
Archery(
args=util.FlattenList([
'benchmark',
'diff',
'--output=diff.json',
util.Property('benchmark_options', []),
'WORKSPACE',
util.Property('benchmark_baseline', 'master')
]),
result_file='diff.json'
)
]
image_filter = Filter(
name='cpp-benchmark',
tag='worker',
variant=None, # plain linux images, not conda
platform=Filter(
arch='amd64', # until ARROW-5382: SSE on ARM NEON gets resolved
distro='ubuntu'
)
)
class GoTest(DockerBuilder):
tags = ['arrow', 'go']
env = {
'GO111MODULE': 'on',
}
steps = [
checkout_arrow,
Go(
args=['get', '-v', '-t', './...'],
workdir='go/arrow',
name='Go Build',
),
Go(
args=['test', './...'],
workdir='go/arrow',
name='Go Test',
)
]
image_filter = Filter(
name=Matching('go*'),
tag='worker',
platform=Filter(
arch='amd64'
)
)
def test_filter():
def filter_list(f, items):
return list(filter(f, items))
Item = namedtuple('Item', ('name', 'id'))
items = [
Item(name='tset', id=1),
Item(name='test', id=2),
Item(name='else', id=3),
Item(name='test', id=4),
Item(name='test', id=4)
]
f = Filter(id=1)
assert filter_list(f, items) == [Item(name='tset', id=1)]
f = Filter(name='test', id=2)
assert filter_list(f, items) == [Item(name='test', id=2)]
f = Filter(name=Glob('t*'))
assert filter_list(f, items) == [
Item(name='tset', id=1),
Item(name='test', id=2),
Item(name='test', id=4),
Item(name='test', id=4)
]
def test_image_collection(collection):
assert isinstance(collection, ImageCollection)
assert len(collection) == 11
amd64_images = collection.filter(platform=Filter(arch='amd64'))
amd64_images = [i.name for i in amd64_images]
assert sorted(amd64_images) == ['a', 'c', 'd', 'e', 'j', 'k']
centos_images = collection.filter(platform=Filter(distro='centos'))
centos_images = [i.name for i in centos_images]
assert sorted(centos_images) == ['b', 'f', 'g', 'h', 'i']
class PythonDockerTest(PythonTest, DockerBuilder):
hostconfig = dict(
shm_size='2G', # required for plasma
)
image_filter = Filter(
name=Matching('python*'),
tag='worker',
variant=None, # plain linux images, not conda
platform=Filter(
arch=AnyOf('amd64', 'arm64v8'),
distro='ubuntu'
)
)
class CrossbowBuilder(DockerBuilder):
"""Builder to trigger various crossbow tasks
The crossbow tool is hosted within arrow, so we need to clone both arrow
and the crossbow repository which serves as a queue for 3rdparty CI
services like Travis or CircleCI. Then using crossbow's command line
interface it triggers builds by adding new branches to the crossbow
repository.
"""
tags = ['crossbow']
env = dict(GIT_COMMITTER_NAME='ursabot',
GIT_COMMITTER_EMAIL='*****@*****.**')
steps = [
GitHub(name='Clone Arrow',
repourl=arrow_repository,
workdir='arrow',
mode='full'),
GitHub(
name='Clone Crossbow',
repourl=crossbow_repository,
workdir='crossbow',
branch='master',
mode='full',
# quite misleasing option, but it prevents checking out the branch
# set in the sourcestamp by the pull request, which refers to arrow
alwaysUseLatest=True)
]
image_filter = Filter(name='crossbow', tag='worker')
class RTest(CppTest):
tags = Extend(['r'])
steps = Extend([
# runs the C++ tests too
r_deps,
r_build,
r_check
])
image_filter = Filter(
name='r',
tag='worker',
variant=None, # plain linux images, not conda
platform=Filter(
arch='amd64'
)
)
class JavaTest(DockerBuilder):
tags = ['arrow', 'java']
steps = [
checkout_arrow,
Maven(
args=['-B', 'test'],
workdir='java',
name='Maven Test',
)
]
image_filter = Filter(
name=Matching('java*'),
tag='worker',
platform=Filter(
arch='amd64'
)
)
class CppCudaTest(CppTest):
tags = Extend(['cuda'])
hostconfig = {
'runtime': 'nvidia'
}
properties = Merge(
ARROW_CUDA='ON'
)
worker_filter = Filter(
tags=Has('cuda')
)
image_filter = Filter(
name='cpp',
tag='worker',
variant='cuda',
platform=Filter(
arch='amd64'
)
)
class CGLibTest(CppTest):
tags = Extend(['c-glib'])
steps = Extend([
# runs the C++ tests too
c_glib_meson,
c_glib_compile,
c_glib_install,
c_glib_install_test_dependencies,
c_glib_test,
])
image_filter = Filter(
name='c-glib',
tag='worker',
variant=None,
platform=Filter(
arch=AnyOf('amd64', 'arm64v8'),
distro='ubuntu'
)
)
class JSTest(DockerBuilder):
tags = ['arrow', 'js']
volumes = [
util.Interpolate('%(prop:builddir)s:/root/.npm:rw')
]
steps = [
checkout_arrow,
Npm(['install', '-g', 'npm@latest'], workdir='js', name='Update NPM'),
Npm(['install'], workdir='js', name='Install Dependencies'),
Npm(['run', 'lint'], workdir='js', name='Lint'),
Npm(['run', 'build'], workdir='js', name='Build'),
Npm(['run', 'test'], workdir='js', name='Test')
]
image_filter = Filter(
name=Matching('js*'),
tag='worker',
platform=Filter(
arch='amd64'
)
)
class PythonCudaTest(PythonTest):
tags = Extend(['cuda'])
hostconfig = dict(
shm_size='2G', # required for plasma
runtime='nvidia', # required for cuda
)
properties = Merge(
ARROW_CUDA='ON', # also sets PYARROW_WITH_CUDA
)
worker_filter = Filter(
tags=Has('cuda')
)
image_filter = Filter(
name=Matching('python*'),
tag='worker',
variant='cuda',
platform=Filter(
arch='amd64'
)
)
class RCondaTest(CppCondaTest):
tags = Extend(['r'])
steps = Extend([
r_deps,
r_build,
r_check
])
image_filter = Filter(
name='r',
variant='conda',
tag='worker'
)
class PythonTest(CppTest):
tags = Extend(['python'])
hostconfig = dict(
shm_size='2G', # required for plasma
)
properties = Merge(
ARROW_PYTHON='ON'
)
steps = Extend([
python_install,
python_test
])
image_filter = Filter(
name=Matching('python*'),
tag='worker',
variant=None, # plain linux images, not conda
platform=Filter(
arch=AnyOf('amd64', 'arm64v8'),
distro='ubuntu'
)
)
class CppCondaTest(DockerBuilder):
tags = ['arrow', 'cpp', 'flight', 'gandiva', 'parquet', 'plasma']
volumes = [
util.Interpolate('%(prop:builddir)s:/root/.ccache:rw')
]
properties = dict(
ARROW_S3='ON',
ARROW_FLIGHT='ON',
ARROW_PLASMA='ON',
ARROW_PARQUET='ON',
ARROW_GANDIVA='ON',
CMAKE_INSTALL_LIBDIR='lib'
)
env = dict(
ARROW_TEST_DATA=arrow_test_data_path, # for flight
PARQUET_TEST_DATA=parquet_test_data_path # for parquet
)
steps = [
SetPropertiesFromEnv(dict(
CXX='CXX',
CMAKE_AR='AR',
CMAKE_RANLIB='RANLIB',
CMAKE_INSTALL_PREFIX='CONDA_PREFIX',
ARROW_BUILD_TOOLCHAIN='CONDA_PREFIX'
)),
# pass system includes paths to clang
SetPropertyFromCommand(
'ARROW_GANDIVA_PC_CXX_FLAGS',
extract_fn=as_system_includes,
command=[util.Property('CXX', 'c++')],
args=['-E', '-Wp,-v', '-xc++', '-'],
collect_stdout=False,
collect_stderr=True,
workdir='.'
),
checkout_arrow,
cpp_mkdir,
cpp_cmake,
cpp_compile,
cpp_install,
cpp_test
]
image_filter = Filter(
name='cpp',
variant='conda',
tag='worker'
)
class PythonCondaTest(CppCondaTest):
tags = Extend(['python'])
hostconfig = dict(
shm_size='2G', # required for plasma
)
properties = Merge(
ARROW_PYTHON='ON'
)
steps = Extend([
python_install,
python_test
])
image_filter = Filter(
name=Matching('python*'),
variant='conda',
tag='worker'
)
class Good2(Builder):
name = 'test'
steps = []
worker_filter = Filter(name=Matching('worker_*'))
