class RustTest(DockerBuilder): tags = ['arrow', 'rust'] env = dict( ARROW_TEST_DATA=arrow_test_data_path, PARQUET_TEST_DATA=parquet_test_data_path ) steps = [ checkout_arrow, Cargo( args=['build'], workdir='rust', name='Rust Build' ), Cargo( args=['test'], workdir='rust', name='Rust Test' ) ] image_filter = Filter( name=Matching('rust*'), tag='worker', platform=Filter( arch='amd64' ) )
class CppTest(DockerBuilder): tags = ['arrow', 'cpp', 'gandiva', 'parquet', 'plasma'] volumes = [ util.Interpolate('%(prop:builddir)s:/root/.ccache:rw') ] properties = dict( ARROW_GANDIVA='ON', ARROW_PARQUET='ON', ARROW_PLASMA='ON', CMAKE_INSTALL_PREFIX='/usr/local', CMAKE_INSTALL_LIBDIR='lib' ) env = { 'PARQUET_TEST_DATA': parquet_test_data_path # for parquet } steps = [ checkout_arrow, cpp_mkdir, cpp_cmake, cpp_compile, cpp_install, cpp_test ] image_filter = Filter( name='cpp', tag='worker', variant=None, platform=Filter( arch=AnyOf('amd64', 'arm64v8'), distro='ubuntu' ) )
class CppBenchmark(DockerBuilder): """Run C++ benchmarks via the Archery CLI tool This builder is parametrized with builtbot properties which are set by the github hook, for more see commands.py """ tags = ['arrow', 'cpp', 'benchmark'] properties = dict( CMAKE_INSTALL_PREFIX='/usr/local', CMAKE_INSTALL_LIBDIR='lib' ) steps = [ checkout_arrow, Pip(['install', '-e', '.'], workdir='dev/archery'), Archery( args=util.FlattenList([ 'benchmark', 'diff', '--output=diff.json', util.Property('benchmark_options', []), 'WORKSPACE', util.Property('benchmark_baseline', 'master') ]), result_file='diff.json' ) ] image_filter = Filter( name='cpp-benchmark', tag='worker', variant=None, # plain linux images, not conda platform=Filter( arch='amd64', # until ARROW-5382: SSE on ARM NEON gets resolved distro='ubuntu' ) )
class GoTest(DockerBuilder): tags = ['arrow', 'go'] env = { 'GO111MODULE': 'on', } steps = [ checkout_arrow, Go( args=['get', '-v', '-t', './...'], workdir='go/arrow', name='Go Build', ), Go( args=['test', './...'], workdir='go/arrow', name='Go Test', ) ] image_filter = Filter( name=Matching('go*'), tag='worker', platform=Filter( arch='amd64' ) )
def test_filter(): def filter_list(f, items): return list(filter(f, items)) Item = namedtuple('Item', ('name', 'id')) items = [ Item(name='tset', id=1), Item(name='test', id=2), Item(name='else', id=3), Item(name='test', id=4), Item(name='test', id=4) ] f = Filter(id=1) assert filter_list(f, items) == [Item(name='tset', id=1)] f = Filter(name='test', id=2) assert filter_list(f, items) == [Item(name='test', id=2)] f = Filter(name=Glob('t*')) assert filter_list(f, items) == [ Item(name='tset', id=1), Item(name='test', id=2), Item(name='test', id=4), Item(name='test', id=4) ]
def test_image_collection(collection): assert isinstance(collection, ImageCollection) assert len(collection) == 11 amd64_images = collection.filter(platform=Filter(arch='amd64')) amd64_images = [i.name for i in amd64_images] assert sorted(amd64_images) == ['a', 'c', 'd', 'e', 'j', 'k'] centos_images = collection.filter(platform=Filter(distro='centos')) centos_images = [i.name for i in centos_images] assert sorted(centos_images) == ['b', 'f', 'g', 'h', 'i']
class PythonDockerTest(PythonTest, DockerBuilder): hostconfig = dict( shm_size='2G', # required for plasma ) image_filter = Filter( name=Matching('python*'), tag='worker', variant=None, # plain linux images, not conda platform=Filter( arch=AnyOf('amd64', 'arm64v8'), distro='ubuntu' ) )
class CrossbowBuilder(DockerBuilder): """Builder to trigger various crossbow tasks The crossbow tool is hosted within arrow, so we need to clone both arrow and the crossbow repository which serves as a queue for 3rdparty CI services like Travis or CircleCI. Then using crossbow's command line interface it triggers builds by adding new branches to the crossbow repository. """ tags = ['crossbow'] env = dict(GIT_COMMITTER_NAME='ursabot', GIT_COMMITTER_EMAIL='*****@*****.**') steps = [ GitHub(name='Clone Arrow', repourl=arrow_repository, workdir='arrow', mode='full'), GitHub( name='Clone Crossbow', repourl=crossbow_repository, workdir='crossbow', branch='master', mode='full', # quite misleasing option, but it prevents checking out the branch # set in the sourcestamp by the pull request, which refers to arrow alwaysUseLatest=True) ] image_filter = Filter(name='crossbow', tag='worker')
class RTest(CppTest): tags = Extend(['r']) steps = Extend([ # runs the C++ tests too r_deps, r_build, r_check ]) image_filter = Filter( name='r', tag='worker', variant=None, # plain linux images, not conda platform=Filter( arch='amd64' ) )
class JavaTest(DockerBuilder): tags = ['arrow', 'java'] steps = [ checkout_arrow, Maven( args=['-B', 'test'], workdir='java', name='Maven Test', ) ] image_filter = Filter( name=Matching('java*'), tag='worker', platform=Filter( arch='amd64' ) )
class CppCudaTest(CppTest): tags = Extend(['cuda']) hostconfig = { 'runtime': 'nvidia' } properties = Merge( ARROW_CUDA='ON' ) worker_filter = Filter( tags=Has('cuda') ) image_filter = Filter( name='cpp', tag='worker', variant='cuda', platform=Filter( arch='amd64' ) )
class CGLibTest(CppTest): tags = Extend(['c-glib']) steps = Extend([ # runs the C++ tests too c_glib_meson, c_glib_compile, c_glib_install, c_glib_install_test_dependencies, c_glib_test, ]) image_filter = Filter( name='c-glib', tag='worker', variant=None, platform=Filter( arch=AnyOf('amd64', 'arm64v8'), distro='ubuntu' ) )
class JSTest(DockerBuilder): tags = ['arrow', 'js'] volumes = [ util.Interpolate('%(prop:builddir)s:/root/.npm:rw') ] steps = [ checkout_arrow, Npm(['install', '-g', 'npm@latest'], workdir='js', name='Update NPM'), Npm(['install'], workdir='js', name='Install Dependencies'), Npm(['run', 'lint'], workdir='js', name='Lint'), Npm(['run', 'build'], workdir='js', name='Build'), Npm(['run', 'test'], workdir='js', name='Test') ] image_filter = Filter( name=Matching('js*'), tag='worker', platform=Filter( arch='amd64' ) )
class PythonCudaTest(PythonTest): tags = Extend(['cuda']) hostconfig = dict( shm_size='2G', # required for plasma runtime='nvidia', # required for cuda ) properties = Merge( ARROW_CUDA='ON', # also sets PYARROW_WITH_CUDA ) worker_filter = Filter( tags=Has('cuda') ) image_filter = Filter( name=Matching('python*'), tag='worker', variant='cuda', platform=Filter( arch='amd64' ) )
class RCondaTest(CppCondaTest): tags = Extend(['r']) steps = Extend([ r_deps, r_build, r_check ]) image_filter = Filter( name='r', variant='conda', tag='worker' )
class PythonTest(CppTest): tags = Extend(['python']) hostconfig = dict( shm_size='2G', # required for plasma ) properties = Merge( ARROW_PYTHON='ON' ) steps = Extend([ python_install, python_test ]) image_filter = Filter( name=Matching('python*'), tag='worker', variant=None, # plain linux images, not conda platform=Filter( arch=AnyOf('amd64', 'arm64v8'), distro='ubuntu' ) )
class CppCondaTest(DockerBuilder): tags = ['arrow', 'cpp', 'flight', 'gandiva', 'parquet', 'plasma'] volumes = [ util.Interpolate('%(prop:builddir)s:/root/.ccache:rw') ] properties = dict( ARROW_S3='ON', ARROW_FLIGHT='ON', ARROW_PLASMA='ON', ARROW_PARQUET='ON', ARROW_GANDIVA='ON', CMAKE_INSTALL_LIBDIR='lib' ) env = dict( ARROW_TEST_DATA=arrow_test_data_path, # for flight PARQUET_TEST_DATA=parquet_test_data_path # for parquet ) steps = [ SetPropertiesFromEnv(dict( CXX='CXX', CMAKE_AR='AR', CMAKE_RANLIB='RANLIB', CMAKE_INSTALL_PREFIX='CONDA_PREFIX', ARROW_BUILD_TOOLCHAIN='CONDA_PREFIX' )), # pass system includes paths to clang SetPropertyFromCommand( 'ARROW_GANDIVA_PC_CXX_FLAGS', extract_fn=as_system_includes, command=[util.Property('CXX', 'c++')], args=['-E', '-Wp,-v', '-xc++', '-'], collect_stdout=False, collect_stderr=True, workdir='.' ), checkout_arrow, cpp_mkdir, cpp_cmake, cpp_compile, cpp_install, cpp_test ] image_filter = Filter( name='cpp', variant='conda', tag='worker' )
class PythonCondaTest(CppCondaTest): tags = Extend(['python']) hostconfig = dict( shm_size='2G', # required for plasma ) properties = Merge( ARROW_PYTHON='ON' ) steps = Extend([ python_install, python_test ]) image_filter = Filter( name=Matching('python*'), variant='conda', tag='worker' )
class Good2(Builder): name = 'test' steps = [] worker_filter = Filter(name=Matching('worker_*'))