def ensure_mongo():
try:
if not exists('{}tmp/mongo_ensured'.format(
os.environ['ssn_dlab_path'])):
sudo(
'echo -e "[mongodb-org-3.2]\nname=MongoDB Repository'
'\nbaseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/3.2/x86_64/'
'\ngpgcheck=1'
'\nenabled=1'
'\ngpgkey=https://www.mongodb.org/static/pgp/server-3.2.asc" '
'> /etc/yum.repos.d/mongodb.repo')
manage_pkg('-y install', 'remote', 'mongodb-org')
sudo('semanage port -a -t mongod_port_t -p tcp 27017')
sudo('chkconfig mongod on')
sudo(
'echo "d /var/run/mongodb 0755 mongod mongod" > /lib/tmpfiles.d/mongodb.conf'
)
sudo('sudo systemd-tmpfiles --create mongodb.conf')
sudo('systemctl start mongod.service')
sudo('touch {}tmp/mongo_ensured'.format(
os.environ['ssn_dlab_path']))
except Exception as err:
traceback.print_exc()
print('Failed to install MongoDB: ', str(err))
sys.exit(1)
def ensure_additional_python_libs(os_user):
if not exists('/home/' + os_user +
'/.ensure_dir/additional_python_libs_ensured'):
try:
manage_pkg('clean', 'remote', 'all')
manage_pkg('-y install', 'remote',
'zlib-devel libjpeg-turbo-devel --nogpgcheck')
if os.environ['application'] in ('jupyter', 'zeppelin'):
sudo(
'pip2 install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'
.format(os.environ['notebook_numpy_version']))
sudo(
'python3.5 -m pip install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'
.format(os.environ['notebook_numpy_version']))
if os.environ['application'] in ('tensor', 'deeplearning'):
sudo(
'python2.7 -m pip install opencv-python h5py --no-cache-dir'
)
sudo(
'python3.5 -m pip install opencv-python h5py --no-cache-dir'
)
sudo('touch /home/' + os_user +
'/.ensure_dir/additional_python_libs_ensured')
except:
sys.exit(1)
def configure_http_proxy_server(config):
try:
if not exists('/tmp/http_proxy_ensured'):
manage_pkg('-y install', 'remote', 'squid')
template_file = config['template_file']
proxy_subnet = config['exploratory_subnet']
put(template_file, '/tmp/squid.conf')
sudo('\cp /tmp/squid.conf /etc/squid/squid.conf')
sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet))
sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name']))
sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host']))
sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn']))
sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user']))
sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password']))
sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib/squid/basic_ldap_auth'))
replace_string = ''
for cidr in config['vpc_cidrs']:
replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr)
sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
replace_string = ''
for cidr in config['allowed_ip_cidr']:
replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr)
sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string))
sudo('service squid reload')
sudo('sysv-rc-conf squid on')
sudo('touch /tmp/http_proxy_ensured')
except Exception as err:
print("Failed to install and configure squid: " + str(err))
sys.exit(1)
def ensure_python3_libraries(os_user):
if not exists('/home/' + os_user +
'/.ensure_dir/python3_libraries_ensured'):
try:
manage_pkg('-y install', 'remote',
'https://centos7.iuscommunity.org/ius-release.rpm')
manage_pkg('-y install', 'remote',
'python35u python35u-pip python35u-devel')
sudo(
'python3.5 -m pip install -U pip=={} setuptools --no-cache-dir'
.format(os.environ['conf_pip_version']))
sudo('python3.5 -m pip install boto3 --no-cache-dir')
sudo(
'python3.5 -m pip install fabvenv fabric-virtualenv future --no-cache-dir'
)
try:
sudo('python3.5 -m pip install tornado=={0} ipython==7.9.0 ipykernel=={1} --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
except:
sudo('python3.5 -m pip install tornado=={0} ipython==5.0.0 ipykernel=={1} --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
sudo('touch /home/' + os_user +
'/.ensure_dir/python3_libraries_ensured')
except:
sys.exit(1)
def install_build_dep():
try:
if not exists('{}tmp/build_dep_ensured'.format(
os.environ['ssn_dlab_path'])):
maven_version = '3.5.4'
manage_pkg(
'-y install', 'remote',
'java-1.8.0-openjdk java-1.8.0-openjdk-devel git wget unzip')
with cd('/opt/'):
sudo(
'wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'
.format(maven_version.split('.')[0], maven_version))
sudo('unzip apache-maven-{}-bin.zip'.format(maven_version))
sudo('mv apache-maven-{} maven'.format(maven_version))
sudo(
'bash -c "curl --silent --location https://rpm.nodesource.com/setup_12.x | bash -"'
)
manage_pkg('-y install', 'remote', 'nodejs')
sudo('npm config set unsafe-perm=true')
sudo('touch {}tmp/build_dep_ensured'.format(
os.environ['ssn_dlab_path']))
except Exception as err:
traceback.print_exc()
print('Failed to install build dependencies for UI: ', str(err))
sys.exit(1)
def ensure_python2_libraries(os_user):
if not exists('/home/' + os_user +
'/.ensure_dir/python2_libraries_ensured'):
try:
sudo('pip2 install pyparsing==2.0.3')
manage_pkg('-y install', 'remote',
'python-setuptools python-wheel')
manage_pkg(
'-y install', 'remote',
'python-virtualenv openssl-devel python-devel openssl-libs libxslt-devel --nogpgcheck'
)
try:
sudo('python2 -m pip install backports.shutil_get_terminal_size tornado=={0} ipython ipykernel=={1} --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
except:
sudo('python2 -m pip install backports.shutil_get_terminal_size tornado=={0} ipython==5.0.0 ipykernel=={1} --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
sudo(
'echo y | python2 -m pip uninstall backports.shutil_get_terminal_size'
)
sudo(
'python2 -m pip install backports.shutil_get_terminal_size --no-cache-dir'
)
sudo('pip2 install -UI pip=={} setuptools --no-cache-dir'.format(
os.environ['conf_pip_version']))
sudo('pip2 install boto3 backoff --no-cache-dir')
sudo(
'pip2 install fabvenv fabric-virtualenv future --no-cache-dir')
downgrade_python_version()
sudo('touch /home/' + os_user +
'/.ensure_dir/python2_libraries_ensured')
except:
sys.exit(1)
def ensure_jre_jdk(os_user):
if not exists('/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'):
try:
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk')
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk-devel')
sudo('touch /home/' + os_user + '/.ensure_dir/jre_jdk_ensured')
except:
sys.exit(1)
def ensure_sbt(os_user):
if not exists('/home/{}/.ensure_dir/sbt_ensured'.format(os_user)):
try:
sudo(
'curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo'
)
manage_pkg('-y install', 'remote', 'sbt')
sudo('touch /home/{}/.ensure_dir/sbt_ensured'.format(os_user))
except:
sys.exit(1)
def ensure_supervisor():
try:
if not exists(os.environ['ssn_dlab_path'] + 'tmp/superv_ensured'):
manage_pkg('-y install', 'remote', 'supervisor')
sudo('update-rc.d supervisor defaults')
sudo('update-rc.d supervisor enable')
sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/superv_ensured')
except Exception as err:
traceback.print_exc()
print('Failed to install Supervisor: ', str(err))
sys.exit(1)
def ensure_nginx(dlab_path):
try:
if not exists('{}tmp/nginx_ensured'.format(dlab_path)):
manage_pkg('-y install', 'remote', 'nginx')
sudo('systemctl restart nginx.service')
sudo('chkconfig nginx on')
sudo('touch {}tmp/nginx_ensured'.format(dlab_path))
except Exception as err:
traceback.print_exc()
print('Failed to ensure Nginx: ', str(err))
sys.exit(1)
def ensure_jenkins(dlab_path):
try:
if not exists(dlab_path + 'tmp/jenkins_ensured'):
sudo('wget -q -O - https://pkg.jenkins.io/debian/jenkins-ci.org.key | apt-key add -')
sudo('echo deb http://pkg.jenkins.io/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list')
manage_pkg('-y update', 'remote', '')
manage_pkg('-y install', 'remote', 'jenkins')
sudo('touch ' + dlab_path + 'tmp/jenkins_ensured')
except Exception as err:
traceback.print_exc()
print('Failed to ensure Jenkins: ', str(err))
sys.exit(1)
def ensure_nginx(dlab_path):
try:
if not exists(dlab_path + 'tmp/nginx_ensured'):
manage_pkg('-y install', 'remote', 'nginx')
sudo('service nginx restart')
sudo('update-rc.d nginx defaults')
sudo('update-rc.d nginx enable')
sudo('touch ' + dlab_path + 'tmp/nginx_ensured')
except Exception as err:
traceback.print_exc()
print('Failed to ensure Nginx: ', str(err))
sys.exit(1)
def ensure_mongo():
try:
if not exists(os.environ['ssn_dlab_path'] + 'tmp/mongo_ensured'):
sudo('apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927')
sudo('ver=`lsb_release -cs`; echo "deb http://repo.mongodb.org/apt/ubuntu $ver/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list; apt-get update')
manage_pkg('-y --allow-unauthenticated install', 'remote', 'mongodb-org')
sudo('systemctl enable mongod.service')
sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/mongo_ensured')
except Exception as err:
traceback.print_exc()
print('Failed to install MongoDB: ', str(err))
sys.exit(1)
def install_torch(os_user):
if not exists('/home/{}/.ensure_dir/torch_ensured'.format(os_user)):
run('git clone https://github.com/torch/distro.git ~/torch --recursive'
)
with cd('/home/{}/torch/'.format(os_user)):
manage_pkg(
'-y install --nogpgcheck', 'remote',
'cmake curl readline-devel ncurses-devel gcc-c++ gcc-gfortran git gnuplot unzip libjpeg-turbo-devel libpng-devel ImageMagick GraphicsMagick-devel fftw-devel sox-devel sox zeromq3-devel qt-devel qtwebkit-devel sox-plugins-freeworld qt-devel'
)
run('./install.sh -b')
run('source /home/{}/.bashrc'.format(os_user))
sudo('touch /home/{}/.ensure_dir/torch_ensured'.format(os_user))
def enable_proxy(proxy_host, proxy_port):
try:
proxy_string = "http://%s:%s" % (proxy_host, proxy_port)
sudo('sed -i "/^export http_proxy/d" /etc/profile')
sudo('sed -i "/^export https_proxy/d" /etc/profile')
sudo('echo export http_proxy=' + proxy_string + ' >> /etc/profile')
sudo('echo export https_proxy=' + proxy_string + ' >> /etc/profile')
if exists('/etc/yum.conf'):
sudo('sed -i "/^proxy=/d" /etc/yum.conf')
sudo("echo 'proxy={}' >> /etc/yum.conf".format(proxy_string))
manage_pkg('clean all', 'remote', '')
except:
sys.exit(1)
def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
if not exists('/home/' + os_user + '/.ensure_dir/rstudio_ensured'):
try:
manage_pkg(
'-y install --nogpgcheck', 'remote',
'https://download2.rstudio.org/server/centos6/x86_64/rstudio-server-rhel-{}-x86_64.rpm'
.format(rstudio_version))
sudo('mkdir -p /mnt/var')
sudo('chown {0}:{0} /mnt/var'.format(os_user))
sudo(
"sed -i '/Type=forking/a \Environment=USER=dlab-user' /etc/systemd/system/rstudio-server.service"
)
sudo(
"sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service"
)
sudo(
"sed -i '/ExecStart/s|$|\"|g' /etc/systemd/system/rstudio-server.service"
)
sudo("systemctl daemon-reload")
sudo('touch /home/{}/.Renviron'.format(os_user))
sudo('chown {0}:{0} /home/{0}/.Renviron'.format(os_user))
sudo('''echo 'SPARK_HOME="{0}"' >> /home/{1}/.Renviron'''.format(
local_spark_path, os_user))
sudo('touch /home/{}/.Rprofile'.format(os_user))
sudo('chown {0}:{0} /home/{0}/.Rprofile'.format(os_user))
sudo(
'''echo 'library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))' >> /home/{}/.Rprofile'''
.format(os_user))
http_proxy = run('echo $http_proxy')
https_proxy = run('echo $https_proxy')
sudo(
'''echo 'Sys.setenv(http_proxy = \"{}\")' >> /home/{}/.Rprofile'''
.format(http_proxy, os_user))
sudo(
'''echo 'Sys.setenv(https_proxy = \"{}\")' >> /home/{}/.Rprofile'''
.format(https_proxy, os_user))
sudo('rstudio-server start')
sudo('echo "{0}:{1}" | chpasswd'.format(os_user, rstudio_pass))
sudo("sed -i '/exit 0/d' /etc/rc.local")
sudo(
'''bash -c "echo \'sed -i 's/^#SPARK_HOME/SPARK_HOME/' /home/{}/.Renviron\' >> /etc/rc.local"'''
.format(os_user))
sudo("bash -c 'echo exit 0 >> /etc/rc.local'")
sudo('touch /home/{}/.ensure_dir/rstudio_ensured'.format(os_user))
except:
sys.exit(1)
else:
try:
sudo('echo "{0}:{1}" | chpasswd'.format(os_user, rstudio_pass))
except:
sys.exit(1)
def ensure_supervisor():
try:
if not exists('{}tmp/superv_ensured'.format(
os.environ['ssn_dlab_path'])):
manage_pkg('-y install', 'remote', 'supervisor')
#sudo('pip install supervisor')
sudo('chkconfig supervisord on')
sudo('systemctl start supervisord')
sudo('touch {}tmp/superv_ensured'.format(
os.environ['ssn_dlab_path']))
except Exception as err:
traceback.print_exc()
print('Failed to install supervisor: ', str(err))
sys.exit(1)
def get_available_os_pkgs():
try:
manage_pkg('update-minimal --security -y --skip-broken', 'remote', '')
downgrade_python_version()
yum_raw = sudo(
'python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print {pkg.name:pkg.vr for pkg in pl.available}"'
)
yum_re = re.sub\
(r'\w*\s\w*\D\s\w*.\w*.\s\w*.\w*.\w.\w*.\w*.\w*', '', yum_raw)
yum_list = yum_re.replace("'", "\"")
os_pkgs = json.loads(yum_list)
return os_pkgs
except Exception as err:
append_result("Failed to get available os packages.", str(err))
sys.exit(1)
def ensure_docker_daemon(dlab_path, os_user, region):
try:
if not exists(dlab_path + 'tmp/docker_daemon_ensured'):
docker_version = os.environ['ssn_docker_version']
sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -')
sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
stable"')
manage_pkg('update', 'remote', '')
sudo('apt-cache policy docker-ce')
manage_pkg('-y install', 'remote', 'docker-ce={}~ce~3-0~ubuntu'.format(docker_version))
sudo('usermod -a -G docker ' + os_user)
sudo('update-rc.d docker defaults')
sudo('update-rc.d docker enable')
sudo('touch ' + dlab_path + 'tmp/docker_daemon_ensured')
return True
except:
return False
def ensure_python3_specific_version(python3_version, os_user):
if not exists('/home/' + os_user +
'/.ensure_dir/python3_specific_version_ensured'):
try:
manage_pkg('-y install', 'remote',
'yum-utils python34 openssl-devel')
manage_pkg('-y groupinstall', 'remote', 'development --nogpgcheck')
if len(python3_version) < 4:
python3_version = python3_version + ".0"
sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz'.
format(python3_version))
sudo(
'tar xzf Python-{0}.tgz; cd Python-{0}; ./configure --prefix=/usr/local; make altinstall'
.format(python3_version))
sudo('touch /home/' + os_user +
'/.ensure_dir/python3_specific_version_ensured')
except:
sys.exit(1)
def ensure_jenkins(dlab_path):
try:
if not exists('{}tmp/jenkins_ensured'.format(dlab_path)):
sudo(
'wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo'
)
try:
sudo(
'rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key'
)
except:
pass
manage_pkg('-y install', 'remote', 'jenkins')
manage_pkg('-y install', 'remote', 'policycoreutils-python')
sudo('touch {}tmp/jenkins_ensured'.format(dlab_path))
except Exception as err:
traceback.print_exc()
print('Failed to ensure Jenkins: ', str(err))
sys.exit(1)
def install_opencv(os_user):
if not exists('/home/{}/.ensure_dir/opencv_ensured'.format(os_user)):
manage_pkg('-y install', 'remote',
'cmake python34 python34-devel python34-pip gcc gcc-c++')
sudo('pip2 install numpy=={} --no-cache-dir'.format(
os.environ['notebook_numpy_version']))
sudo('pip3.4 install numpy=={} --no-cache-dir'.format(
os.environ['notebook_numpy_version']))
sudo('pip3.5 install numpy=={} --no-cache-dir'.format(
os.environ['notebook_numpy_version']))
run('git clone https://github.com/opencv/opencv.git')
with cd('/home/{}/opencv/'.format(os_user)):
run('git checkout 3.2.0')
run('mkdir release')
with cd('/home/{}/opencv/release/'.format(os_user)):
run('cmake -DINSTALL_TESTS=OFF -D CUDA_GENERATION=Auto -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=$(python2 -c "import sys; print(sys.prefix)") -D PYTHON_EXECUTABLE=$(which python2) ..'
)
run('make -j$(nproc)')
sudo('make install')
sudo('touch /home/' + os_user + '/.ensure_dir/opencv_ensured')
def ensure_r(os_user, r_libs, region, r_mirror):
if not exists('/home/{}/.ensure_dir/r_ensured'.format(os_user)):
try:
if region == 'cn-north-1':
r_repository = r_mirror
else:
r_repository = 'https://cloud.r-project.org'
manage_pkg('-y install', 'remote', 'cmake')
manage_pkg('-y install', 'remote', 'libcur*')
sudo(
'echo -e "[base]\nname=CentOS-7-Base\nbaseurl=http://buildlogs.centos.org/centos/7/os/x86_64-20140704-1/\ngpgcheck=1\ngpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7\npriority=1\nexclude=php mysql" >> /etc/yum.repos.d/CentOS-base.repo'
)
manage_pkg('-y install', 'remote',
'R R-core R-core-devel R-devel --nogpgcheck')
sudo('R CMD javareconf')
sudo(
'cd /root; git clone https://github.com/zeromq/zeromq4-x.git; cd zeromq4-x/; mkdir build; cd build; cmake ..; make install; ldconfig'
)
for i in r_libs:
sudo('R -e "install.packages(\'{}\',repos=\'{}\')"'.format(
i, r_repository))
sudo(
'R -e "library(\'devtools\');install.packages(repos=\'{}\',c(\'rzmq\',\'repr\',\'digest\',\'stringr\',\'RJSONIO\',\'functional\',\'plyr\'))"'
.format(r_repository))
sudo(
'R -e "library(\'devtools\');install_github(\'IRkernel/repr\');install_github(\'IRkernel/IRdisplay\');install_github(\'IRkernel/IRkernel\');"'
)
sudo(
'R -e "library(\'devtools\');install_version(\'keras\', version = \'{}\', repos = \'{}\');"'
.format(os.environ['notebook_keras_version'], r_repository))
sudo('R -e "install.packages(\'RJDBC\',repos=\'{}\',dep=TRUE)"'.
format(r_repository))
sudo('touch /home/{}/.ensure_dir/r_ensured'.format(os_user))
except:
sys.exit(1)
def ensure_docker_daemon(dlab_path, os_user, region):
try:
if not exists('{}tmp/docker_daemon_ensured'.format(dlab_path)):
docker_version = os.environ['ssn_docker_version']
if region == 'cn-north-1':
mirror = 'mirror.lzu.edu.cn'
else:
mirror = 'mirror.centos.org'
with cd('/etc/yum.repos.d/'):
sudo('echo "[centosrepo]" > centos.repo')
sudo('echo "name=Centos 7 Repository" >> centos.repo')
sudo(
'echo "baseurl=http://{}/centos/7/extras/x86_64/" >> centos.repo'
.format(mirror))
sudo('echo "enabled=1" >> centos.repo')
sudo('echo "gpgcheck=1" >> centos.repo')
sudo(
'echo "gpgkey=http://{}/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7" >> centos.repo'
.format(mirror))
sudo(
'yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo'
)
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install', 'remote', 'container-selinux')
manage_pkg('-y install', 'remote',
'docker-ce-{}.ce'.format(docker_version))
sudo('usermod -aG docker {}'.format(os_user))
sudo('systemctl enable docker.service')
sudo('systemctl start docker')
sudo('touch {}tmp/docker_daemon_ensured'.format(dlab_path))
return True
except:
return False
def install_caffe2(os_user, caffe2_version, cmake_version):
if not exists('/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)):
env.shell = "/bin/bash -l -c -i"
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg(
'-y install --nogpgcheck', 'remote',
'automake cmake3 gcc gcc-c++ kernel-devel leveldb-devel lmdb-devel libtool protobuf-devel graphviz'
)
sudo(
'pip2 install flask graphviz hypothesis jupyter matplotlib==2.0.2 numpy=={} protobuf pydot python-nvd3 pyyaml '
'requests scikit-image scipy setuptools tornado future --no-cache-dir'
.format(os.environ['notebook_numpy_version']))
sudo(
'pip3.5 install flask graphviz hypothesis jupyter matplotlib==2.0.2 numpy=={} protobuf pydot python-nvd3 pyyaml '
'requests scikit-image scipy setuptools tornado future --no-cache-dir'
.format(os.environ['notebook_numpy_version']))
sudo('cp /opt/cudnn/include/* /opt/cuda-8.0/include/')
sudo('cp /opt/cudnn/lib64/* /opt/cuda-8.0/lib64/')
sudo(
'wget https://cmake.org/files/v{2}/cmake-{1}.tar.gz -O /home/{0}/cmake-{1}.tar.gz'
.format(
os_user, cmake_version,
cmake_version.split('.')[0] + "." +
cmake_version.split('.')[1]))
sudo('tar -zxvf cmake-{}.tar.gz'.format(cmake_version))
with cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)):
sudo('./bootstrap --prefix=/usr/local && make && make install')
sudo('ln -s /usr/local/bin/cmake /bin/cmake{}'.format(cmake_version))
sudo('git clone https://github.com/pytorch/pytorch.git')
with cd('/home/{}/pytorch/'.format(os_user)):
sudo('git submodule update --init')
with settings(warn_only=True):
sudo('git checkout v{}'.format(caffe2_version))
sudo('git submodule update --recursive')
sudo(
'mkdir build && cd build && cmake{} .. && make "-j$(nproc)" install'
.format(cmake_version))
sudo('touch /home/' + os_user + '/.ensure_dir/caffe2_ensured')
def install_os_pkg(requisites):
status = list()
error_parser = "Could not|No matching|Error:|failed|Requires:|Errno"
try:
print(
"Updating repositories and installing requested tools: {}".format(
requisites))
manage_pkg('update-minimal --security -y --skip-broken', 'remote', '')
sudo('export LC_ALL=C')
for os_pkg in requisites:
manage_pkg(
'-y install', 'remote',
'{0} --nogpgcheck 2>&1 | if ! grep -w -E "({1})" > /tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'
.format(os_pkg, error_parser))
err = sudo('cat /tmp/os_install_{}.log'.format(os_pkg)).replace(
'"', "'")
try:
res = sudo(
'python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\'][0]"'
.format(os_pkg))
version = res.split('\r\n')[1].replace("'", "\"")
status.append({
"group": "os_pkg",
"name": os_pkg,
"version": version,
"status": "installed"
})
except:
status.append({
"group": "os_pkg",
"name": os_pkg,
"status": "failed",
"error_message": err
})
return status
except:
return "Fail to install OS packages"
def install_cntk(os_user, cntk_version):
if not exists('/home/{}/.ensure_dir/cntk_ensured'.format(os_user)):
sudo('echo "exclude=*.i386 *.i686" >> /etc/yum.conf')
manage_pkg('clean', 'remote', 'all')
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install --nogpgcheck', 'remote',
'openmpi openmpi-devel')
sudo(
'pip2 install https://cntk.ai/PythonWheel/GPU/cntk-{}-cp27-cp27mu-linux_x86_64.whl --no-cache-dir'
.format(cntk_version))
sudo(
'pip3.5 install https://cntk.ai/PythonWheel/GPU/cntk-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'
.format(cntk_version))
sudo('touch /home/{}/.ensure_dir/cntk_ensured'.format(os_user))
def install_tensor(os_user, cuda_version, cuda_file_name, cudnn_version,
cudnn_file_name, tensorflow_version, templates_dir,
nvidia_version):
if not exists('/home/{}/.ensure_dir/tensor_ensured'.format(os_user)):
try:
# install nvidia drivers
sudo(
'echo "blacklist nouveau" >> /etc/modprobe.d/blacklist-nouveau.conf'
)
sudo(
'echo "options nouveau modeset=0" >> /etc/modprobe.d/blacklist-nouveau.conf'
)
sudo('dracut --force')
with settings(warn_only=True):
reboot(wait=150)
manage_pkg(
'-y install', 'remote',
'libglvnd-opengl libglvnd-devel dkms gcc kernel-devel-$(uname -r) kernel-headers-$(uname -r)'
)
sudo(
'wget http://us.download.nvidia.com/XFree86/Linux-x86_64/{0}/NVIDIA-Linux-x86_64-{0}.run -O /home/{1}/NVIDIA-Linux-x86_64-{0}.run'
.format(nvidia_version, os_user))
sudo('/bin/bash /home/{0}/NVIDIA-Linux-x86_64-{1}.run -s --dkms'.
format(os_user, nvidia_version))
sudo('rm -f /home/{0}/NVIDIA-Linux-x86_64-{1}.run'.format(
os_user, nvidia_version))
# install cuda
sudo(
'python3.5 -m pip install --upgrade pip=={0} wheel numpy=={1} --no-cache-dir'
.format(os.environ['conf_pip_version'],
os.environ['notebook_numpy_version']))
sudo(
'wget -P /opt https://developer.nvidia.com/compute/cuda/{0}/prod/local_installers/{1}'
.format(cuda_version, cuda_file_name))
sudo('sh /opt/{} --silent --toolkit'.format(cuda_file_name))
sudo('mv /usr/local/cuda-{} /opt/'.format(cuda_version))
sudo(
'ln -s /opt/cuda-{0} /usr/local/cuda-{0}'.format(cuda_version))
sudo('rm -f /opt/{}'.format(cuda_file_name))
# install cuDNN
run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'
.format(cudnn_version, cudnn_file_name))
run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
sudo('mkdir -p /opt/cudnn/include')
sudo('mkdir -p /opt/cudnn/lib64')
sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include')
sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64')
sudo(
'chmod a+r /opt/cudnn/include/cudnn.h /opt/cudnn/lib64/libcudnn*'
)
run('echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc'
)
# install TensorFlow and run TensorBoard
sudo(
'wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl'
.format(tensorflow_version))
sudo(
'wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl'
.format(tensorflow_version))
sudo(
'python2.7 -m pip install --upgrade tensorflow_gpu-{}-cp27-none-linux_x86_64.whl --no-cache-dir'
.format(tensorflow_version))
sudo(
'python3.5 -m pip install --upgrade tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'
.format(tensorflow_version))
sudo('rm -rf /home/{}/tensorflow_gpu-*'.format(os_user))
sudo(
'mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard'
.format(os_user))
put('{}tensorboard.service'.format(templates_dir),
'/tmp/tensorboard.service')
sudo("sed -i 's|OS_USR|{}|' /tmp/tensorboard.service".format(
os_user))
sudo("chmod 644 /tmp/tensorboard.service")
sudo('\cp /tmp/tensorboard.service /etc/systemd/system/')
sudo("systemctl daemon-reload")
sudo("systemctl enable tensorboard")
sudo("systemctl start tensorboard")
sudo('touch /home/{}/.ensure_dir/tensor_ensured'.format(os_user))
except:
sys.exit(1)
def install_nodejs(os_user):
if not exists('/home/{}/.ensure_dir/nodejs_ensured'.format(os_user)):
sudo('curl -sL https://rpm.nodesource.com/setup_6.x | sudo -E bash -')
manage_pkg('-y install', 'remote', 'nodejs')
sudo('touch /home/{}/.ensure_dir/nodejs_ensured'.format(os_user))
def remove_os_pkg(pkgs):
try:
manage_pkg('remove -y', 'remote', '{}'.format(' '.join(pkgs)))
except:
sys.exit(1)
