def get_shuffle_params(params, index):
random.seed(0)
keys = params.keys()
iters = [iter_shuffle(params[key]) for key in keys]
vals = next(itertools.islice(six.moves.zip(*iters), index, None))
ret = dict(zip(keys, vals))
# avoid SEGV
if ret['numpy'] == '1.9' and ret.get('h5py'):
ret['numpy'] = '1.10'
py_ver = docker.get_python_version(ret['base'])
# Avoid unsupported NumPy/SciPy version for the Python version.
if py_ver[:2] == (3, 5):
# Python 3.5 is first supported in NumPy 1.11.
if ret['numpy'] in ['1.9', '1.10']:
ret['numpy'] = '1.11'
elif py_ver[:2] == (3, 6):
# Python 3.6 is first supported in NumPy 1.12.
if ret['numpy'] in ['1.9', '1.10', '1.11']:
ret['numpy'] = '1.12'
# Python 3.6 is first supported in SciPy 0.19.
if ret.get('scipy', None) in ['0.18']:
ret['scipy'] = '0.19'
elif py_ver[:2] == (3, 7):
# Python 3.7 is first supported in NumPy 1.14.4.
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12', '1.13']:
ret['numpy'] = '1.14'
# Python 3.7 is first supported in SciPy 1.0.
if ret.get('scipy', None) in ['0.18', '0.19']:
ret['scipy'] = '1.0'
# Avoid iDeep in unsupported Python versions
if not _is_ideep_supported(py_ver):
ret['ideep'] = None
# TODO(kmaehashi) Currently iDeep can only be tested on Ubuntu.
if 'ubuntu' not in ret['base']:
ret['ideep'] = None
# iDeep requires NumPy 1.13.0 or later.
if ret.get('ideep'):
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12']:
ret['numpy'] = '1.13'
cuda, cudnn, nccl = ret['cuda_cudnn_nccl']
if ('centos6' in ret['base']
or 'ubuntu16' in ret['base'] and cuda < 'cuda8'):
# nccl is not supported on these environment
ret['cuda_cudnn_nccl'] = (cuda, cudnn, 'none')
if 'centos6' in ret['base'] and ret.get('protobuf') == 'cpp-3':
ret['protobuf'] = '3'
return ret
def _is_shuffle_params_valid(ret):
base = ret['base']
# avoid SEGV
if ret['numpy'] == '1.9' and ret.get('h5py'):
return False, 'NumPy 1.9 incompatible with h5py'
py_ver = docker.get_python_version(base)
# Avoid unsupported library versions for the Python version.
if py_ver[:2] == (2, 7):
pass
elif py_ver[:2] == (3, 4):
pass
elif py_ver[:2] == (3, 5):
# Python 3.5 is first supported in NumPy 1.11.
if ret['numpy'] in ['1.9', '1.10']:
return False, 'NumPy version does not support Python 3.5'
elif py_ver[:2] == (3, 6):
# Python 3.6 is first supported in NumPy 1.12.
if ret['numpy'] in ['1.9', '1.10', '1.11']:
return False, 'NumPy version does not support Python 3.6'
# Python 3.6 is first supported in SciPy 0.19.
if ret.get('scipy', None) in ['0.18']:
return False, 'SciPy version does not support Python 3.7'
# Python 3.6 is first supported in h5py 2.6.
if ret.get('h5py', None) in ['2.5']:
return False, 'h5py version does not support Python 3.6'
# Python 3.6 is first supported in pillow 4.
if ret.get('pillow', None) in ['3.4']:
return False, 'pillow version does not support Python 3.6'
elif py_ver[:2] == (3, 7):
# Python 3.7 is first supported in NumPy 1.14.4.
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12', '1.13']:
return False, 'NumPy version does not support Python 3.7'
# Python 3.7 is first supported in SciPy 1.0.
if ret.get('scipy', None) in ['0.18', '0.19']:
return False, 'SciPy version does not support Python 3.7'
# Python 3.7 is first supported in h5py 2.8.
if ret.get('h5py', None) in ['2.5', '2.6', '2.7']:
return False, 'h5py version does not support Python 3.7'
# Python 3.7 is first supported in pillow 5.2.
if ret.get('pillow', None) in ['3.4', '4.0', '4.1']:
return False, 'pillow version does not support Python 3.7'
elif py_ver[:2] == (3, 8):
# Python 3.8 is first supported in NumPy 1.17.3.
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12', '1.13', '1.14', '1.15', '1.16']:
return False, 'NumPy version does not support Python 3.8'
# Python 3.8 is first supported in SciPy 1.3.2.
if ret.get('scipy', None) in ['0.18', '0.19', '1.0']:
return False, 'SciPy version does not support Python 3.8'
# Python 3.8 is first supported in h5py 2.10.
if ret.get('h5py', None) in ['2.5', '2.6', '2.7', '2.8', '2.9']:
return False, 'h5py version does not support Python 3.8'
# Python 3.8 is first supported in pillow 6.2.1.
if ret.get('pillow', None) in ['3.4', '4.0', '4.1']:
return False, 'pillow version does not support Python 3.8'
else:
# Unknown Python version
assert False
# iDeep requirements:
# - Ubuntu 16.04 or CentOS 7.4 or OS X
# - NumPy 1.13.0+ with Python 2.7/3.5/3.6
# - NumPy 1.16.0+ with Python 3.7+
if ret.get('ideep'):
if (('centos6' in base) or
not _is_ideep_supported(py_ver)):
return False, 'iDeep not supported on {}'.format(base)
elif py_ver[:2] >= (3, 7):
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12', '1.13', '1.14', '1.15']:
return False, 'iDeep not supported on this Python/NumPy combination'
else:
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12']:
return False, 'iDeep not supported on this NumPy version'
# SciPy 0.19 installation from source (--no-binary) fails with new NumPy 1.16+.
# Theano 1.0.3 or earlier does not support NumPy 1.16+.
if ret['numpy'] not in ['1.9', '1.10', '1.11', '1.12', '1.13', '1.14', '1.15']:
if ret.get('scipy', None) in ['0.18', '0.19']:
return False, 'SciPy version does not support this NumPy version'
if ret.get('theano') in ['0.8', '0.9']:
return False, 'Theano version does not support this NumPy version'
if ret.get('scipy', None) == '1.6':
# Python 3.7+ and NumPy 1.16.5+
if py_ver[:2] < (3, 7):
return False, 'SciPy version does not support this Python version'
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12', '1.13', '1.14', '1.15']:
return False, 'SciPy version does not support this NumPy version'
if 'centos6' in base and ret.get('protobuf') == 'cpp-3':
return False, 'protobuf cpp-3 not supported on centos6'
cuda, cudnn, nccl, cutensor = ret['cuda_libs']
if 'centos6' in base and nccl != 'none':
# https://docs.nvidia.com/deeplearning/sdk/nccl-install-guide/index.html#rhel_centos
return False, 'NCCL is not supported in centos6'
if 'centos6' in base and cutensor != 'none':
return False, 'cuTENSOR is not supported in centos'
if (cuda == 'cuda80' and
not any(base.startswith(x) for x in ['ubuntu16', 'centos6', 'centos7'])):
# https://docs.nvidia.com/cuda/archive/8.0/cuda-installation-guide-linux/index.html
return False, 'CUDA 8.0 is not supported on {}'.format(base)
elif (cuda in ['cuda90', 'cuda91', 'cuda92'] and
not any(base.startswith(x) for x in ['ubuntu16', 'centos6', 'centos7'])):
# https://docs.nvidia.com/cuda/archive/9.0/cuda-installation-guide-linux/index.html
# https://docs.nvidia.com/cuda/archive/9.1/cuda-installation-guide-linux/index.html
# https://docs.nvidia.com/cuda/archive/9.2/cuda-installation-guide-linux/index.html
return False, 'CUDA 9.x is not supported on {}'.format(base)
elif (cuda == 'cuda100' and
not any(base.startswith(x) for x in ['ubuntu16', 'ubuntu18', 'centos6', 'centos7'])):
# https://docs.nvidia.com/cuda/archive/10.0/cuda-installation-guide-linux/index.html
return False, 'CUDA 10.0 is not supported on {}'.format(base)
elif (cuda == 'cuda101' and
not any(base.startswith(x) for x in ['ubuntu16', 'ubuntu18', 'centos7'])):
# https://docs.nvidia.com/cuda/archive/10.1/cuda-installation-guide-linux/index.html
# CUDA 10.1+ on CentOS 6 requires different CUDA installer.
# For simplicity and considering the fact that it will EOL in 2020 (so
# CentOS 6 will not be used for new deployments), we exclude it from
# the test.
return False, 'CUDA 10.1 is not supported on {}'.format(base)
return True, None
argconfig.setup_argument_parser(parser)
args = parser.parse_args()
if version.is_master_branch('cupy'):
params['base'] = docker.base_choices_master_cupy
else:
params['base'] = docker.base_choices_stable_cupy
if args.clone_chainer:
version.clone_chainer()
conf = shuffle.make_shuffle_conf(params, args.id)
# pip has dropped Python 3.4 support since 19.2.
# TODO(niboshi): More generic and elegant approach to handle special requirements.
pip_require = 'pip<19.2' if docker.get_python_version(
conf['base'])[:2] == (3, 4) else 'pip'
conf['requires'] = ['setuptools<42', pip_require, 'cython==0.29.13'
] + conf['requires']
volume = []
env = {'CUDNN': conf['cudnn']}
argconfig.parse_args(args, env, conf, volume)
argconfig.setup_coverage(args, env)
if args.interactive:
docker.run_interactive(conf,
no_cache=args.no_cache,
volume=volume,
env=env,
def get_shuffle_params(params, index):
random.seed(0)
keys = sorted(params.keys())
iters = [iter_shuffle(params[key]) for key in keys]
vals = tuple(next(itertools.islice(iter, index, None)) for iter in iters)
ret = dict(zip(keys, vals))
# avoid SEGV
if ret['numpy'] == '1.9' and ret.get('h5py'):
ret['numpy'] = '1.10'
py_ver = docker.get_python_version(ret['base'])
# Avoid unsupported NumPy/SciPy version for the Python version.
if py_ver[:2] == (3, 5):
# Python 3.5 is first supported in NumPy 1.11.
if ret['numpy'] in ['1.9', '1.10']:
ret['numpy'] = '1.11'
elif py_ver[:2] == (3, 6):
# Python 3.6 is first supported in NumPy 1.12.
if ret['numpy'] in ['1.9', '1.10', '1.11']:
ret['numpy'] = '1.12'
# Python 3.6 is first supported in SciPy 0.19.
if ret.get('scipy', None) in ['0.18']:
ret['scipy'] = '0.19'
elif py_ver[:2] == (3, 7):
# Python 3.7 is first supported in NumPy 1.14.4.
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12', '1.13']:
ret['numpy'] = '1.14'
# Python 3.7 is first supported in SciPy 1.0.
if ret.get('scipy', None) in ['0.18', '0.19']:
ret['scipy'] = '1.0'
# iDeep requirements:
# - Ubuntu 16.04 or CentOS 7.4 or OS X
# - NumPy 1.13.0+ with Python 2.7/3.5/3.6
# - NumPy 1.16.0+ with Python 3.7+
if ret.get('ideep'):
base = ret['base']
if (('centos6' in base or 'ubuntu14' in base) or
not _is_ideep_supported(py_ver)):
ret['ideep'] = None
elif py_ver[:2] >= (3, 7):
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12', '1.13', '1.14', '1.15']:
ret['numpy'] = '1.16'
else:
if ret['numpy'] in ['1.9', '1.10', '1.11', '1.12']:
ret['numpy'] = '1.13'
# SciPy 0.19 installation from source (--no-binary) fails with new NumPy 1.16+.
# Theano 1.0.3 or earlier does not support NumPy 1.16+.
if ret['numpy'] not in ['1.9', '1.10', '1.11', '1.12', '1.13', '1.14', '1.15']:
if ret.get('scipy', None) in ['0.18', '0.19']:
ret['scipy'] = '1.0'
if ret.get('theano') in ['0.8', '0.9']:
ret['theano'] = '1.0'
cuda, cudnn, nccl = ret['cuda_cudnn_nccl']
if ('centos6' in ret['base'] or
'ubuntu16' in ret['base'] and cuda < 'cuda8'):
# nccl is not supported on these environment
ret['cuda_cudnn_nccl'] = (cuda, cudnn, 'none')
if 'centos6' in ret['base'] and ret.get('protobuf') == 'cpp-3':
ret['protobuf'] = '3'
return ret
