def ensure_mongo(): try: if not exists('{}tmp/mongo_ensured'.format( os.environ['ssn_dlab_path'])): sudo( 'echo -e "[mongodb-org-3.2]\nname=MongoDB Repository' '\nbaseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/3.2/x86_64/' '\ngpgcheck=1' '\nenabled=1' '\ngpgkey=https://www.mongodb.org/static/pgp/server-3.2.asc" ' '> /etc/yum.repos.d/mongodb.repo') manage_pkg('-y install', 'remote', 'mongodb-org') sudo('semanage port -a -t mongod_port_t -p tcp 27017') sudo('chkconfig mongod on') sudo( 'echo "d /var/run/mongodb 0755 mongod mongod" > /lib/tmpfiles.d/mongodb.conf' ) sudo('sudo systemd-tmpfiles --create mongodb.conf') sudo('systemctl start mongod.service') sudo('touch {}tmp/mongo_ensured'.format( os.environ['ssn_dlab_path'])) except Exception as err: traceback.print_exc() print('Failed to install MongoDB: ', str(err)) sys.exit(1)
def ensure_additional_python_libs(os_user): if not exists('/home/' + os_user + '/.ensure_dir/additional_python_libs_ensured'): try: manage_pkg('clean', 'remote', 'all') manage_pkg('-y install', 'remote', 'zlib-devel libjpeg-turbo-devel --nogpgcheck') if os.environ['application'] in ('jupyter', 'zeppelin'): sudo( 'pip2 install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir' .format(os.environ['notebook_numpy_version'])) sudo( 'python3.5 -m pip install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir' .format(os.environ['notebook_numpy_version'])) if os.environ['application'] in ('tensor', 'deeplearning'): sudo( 'python2.7 -m pip install opencv-python h5py --no-cache-dir' ) sudo( 'python3.5 -m pip install opencv-python h5py --no-cache-dir' ) sudo('touch /home/' + os_user + '/.ensure_dir/additional_python_libs_ensured') except: sys.exit(1)
def configure_http_proxy_server(config): try: if not exists('/tmp/http_proxy_ensured'): manage_pkg('-y install', 'remote', 'squid') template_file = config['template_file'] proxy_subnet = config['exploratory_subnet'] put(template_file, '/tmp/squid.conf') sudo('\cp /tmp/squid.conf /etc/squid/squid.conf') sudo('sed -i "s|PROXY_SUBNET|{}|g" /etc/squid/squid.conf'.format(proxy_subnet)) sudo('sed -i "s|EDGE_USER_NAME|{}|g" /etc/squid/squid.conf'.format(config['project_name'])) sudo('sed -i "s|LDAP_HOST|{}|g" /etc/squid/squid.conf'.format(config['ldap_host'])) sudo('sed -i "s|LDAP_DN|{}|g" /etc/squid/squid.conf'.format(config['ldap_dn'])) sudo('sed -i "s|LDAP_SERVICE_USERNAME|{}|g" /etc/squid/squid.conf'.format(config['ldap_user'])) sudo('sed -i "s|LDAP_SERVICE_PASSWORD|{}|g" /etc/squid/squid.conf'.format(config['ldap_password'])) sudo('sed -i "s|LDAP_AUTH_PATH|{}|g" /etc/squid/squid.conf'.format('/usr/lib/squid/basic_ldap_auth')) replace_string = '' for cidr in config['vpc_cidrs']: replace_string += 'acl AWS_VPC_CIDR dst {}\\n'.format(cidr) sudo('sed -i "s|VPC_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string)) replace_string = '' for cidr in config['allowed_ip_cidr']: replace_string += 'acl AllowedCIDRS src {}\\n'.format(cidr) sudo('sed -i "s|ALLOWED_CIDRS|{}|g" /etc/squid/squid.conf'.format(replace_string)) sudo('service squid reload') sudo('sysv-rc-conf squid on') sudo('touch /tmp/http_proxy_ensured') except Exception as err: print("Failed to install and configure squid: " + str(err)) sys.exit(1)
def ensure_python3_libraries(os_user): if not exists('/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'): try: manage_pkg('-y install', 'remote', 'https://centos7.iuscommunity.org/ius-release.rpm') manage_pkg('-y install', 'remote', 'python35u python35u-pip python35u-devel') sudo( 'python3.5 -m pip install -U pip=={} setuptools --no-cache-dir' .format(os.environ['conf_pip_version'])) sudo('python3.5 -m pip install boto3 --no-cache-dir') sudo( 'python3.5 -m pip install fabvenv fabric-virtualenv future --no-cache-dir' ) try: sudo('python3.5 -m pip install tornado=={0} ipython==7.9.0 ipykernel=={1} --no-cache-dir' \ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'])) except: sudo('python3.5 -m pip install tornado=={0} ipython==5.0.0 ipykernel=={1} --no-cache-dir' \ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'])) sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured') except: sys.exit(1)
def install_build_dep(): try: if not exists('{}tmp/build_dep_ensured'.format( os.environ['ssn_dlab_path'])): maven_version = '3.5.4' manage_pkg( '-y install', 'remote', 'java-1.8.0-openjdk java-1.8.0-openjdk-devel git wget unzip') with cd('/opt/'): sudo( 'wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip' .format(maven_version.split('.')[0], maven_version)) sudo('unzip apache-maven-{}-bin.zip'.format(maven_version)) sudo('mv apache-maven-{} maven'.format(maven_version)) sudo( 'bash -c "curl --silent --location https://rpm.nodesource.com/setup_12.x | bash -"' ) manage_pkg('-y install', 'remote', 'nodejs') sudo('npm config set unsafe-perm=true') sudo('touch {}tmp/build_dep_ensured'.format( os.environ['ssn_dlab_path'])) except Exception as err: traceback.print_exc() print('Failed to install build dependencies for UI: ', str(err)) sys.exit(1)
def ensure_python2_libraries(os_user): if not exists('/home/' + os_user + '/.ensure_dir/python2_libraries_ensured'): try: sudo('pip2 install pyparsing==2.0.3') manage_pkg('-y install', 'remote', 'python-setuptools python-wheel') manage_pkg( '-y install', 'remote', 'python-virtualenv openssl-devel python-devel openssl-libs libxslt-devel --nogpgcheck' ) try: sudo('python2 -m pip install backports.shutil_get_terminal_size tornado=={0} ipython ipykernel=={1} --no-cache-dir' \ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'])) except: sudo('python2 -m pip install backports.shutil_get_terminal_size tornado=={0} ipython==5.0.0 ipykernel=={1} --no-cache-dir' \ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'])) sudo( 'echo y | python2 -m pip uninstall backports.shutil_get_terminal_size' ) sudo( 'python2 -m pip install backports.shutil_get_terminal_size --no-cache-dir' ) sudo('pip2 install -UI pip=={} setuptools --no-cache-dir'.format( os.environ['conf_pip_version'])) sudo('pip2 install boto3 backoff --no-cache-dir') sudo( 'pip2 install fabvenv fabric-virtualenv future --no-cache-dir') downgrade_python_version() sudo('touch /home/' + os_user + '/.ensure_dir/python2_libraries_ensured') except: sys.exit(1)
def ensure_jre_jdk(os_user): if not exists('/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'): try: manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk') manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk-devel') sudo('touch /home/' + os_user + '/.ensure_dir/jre_jdk_ensured') except: sys.exit(1)
def ensure_sbt(os_user): if not exists('/home/{}/.ensure_dir/sbt_ensured'.format(os_user)): try: sudo( 'curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo' ) manage_pkg('-y install', 'remote', 'sbt') sudo('touch /home/{}/.ensure_dir/sbt_ensured'.format(os_user)) except: sys.exit(1)
def ensure_supervisor(): try: if not exists(os.environ['ssn_dlab_path'] + 'tmp/superv_ensured'): manage_pkg('-y install', 'remote', 'supervisor') sudo('update-rc.d supervisor defaults') sudo('update-rc.d supervisor enable') sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/superv_ensured') except Exception as err: traceback.print_exc() print('Failed to install Supervisor: ', str(err)) sys.exit(1)
def ensure_nginx(dlab_path): try: if not exists('{}tmp/nginx_ensured'.format(dlab_path)): manage_pkg('-y install', 'remote', 'nginx') sudo('systemctl restart nginx.service') sudo('chkconfig nginx on') sudo('touch {}tmp/nginx_ensured'.format(dlab_path)) except Exception as err: traceback.print_exc() print('Failed to ensure Nginx: ', str(err)) sys.exit(1)
def ensure_jenkins(dlab_path): try: if not exists(dlab_path + 'tmp/jenkins_ensured'): sudo('wget -q -O - https://pkg.jenkins.io/debian/jenkins-ci.org.key | apt-key add -') sudo('echo deb http://pkg.jenkins.io/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list') manage_pkg('-y update', 'remote', '') manage_pkg('-y install', 'remote', 'jenkins') sudo('touch ' + dlab_path + 'tmp/jenkins_ensured') except Exception as err: traceback.print_exc() print('Failed to ensure Jenkins: ', str(err)) sys.exit(1)
def ensure_nginx(dlab_path): try: if not exists(dlab_path + 'tmp/nginx_ensured'): manage_pkg('-y install', 'remote', 'nginx') sudo('service nginx restart') sudo('update-rc.d nginx defaults') sudo('update-rc.d nginx enable') sudo('touch ' + dlab_path + 'tmp/nginx_ensured') except Exception as err: traceback.print_exc() print('Failed to ensure Nginx: ', str(err)) sys.exit(1)
def ensure_mongo(): try: if not exists(os.environ['ssn_dlab_path'] + 'tmp/mongo_ensured'): sudo('apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv EA312927') sudo('ver=`lsb_release -cs`; echo "deb http://repo.mongodb.org/apt/ubuntu $ver/mongodb-org/3.2 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-3.2.list; apt-get update') manage_pkg('-y --allow-unauthenticated install', 'remote', 'mongodb-org') sudo('systemctl enable mongod.service') sudo('touch ' + os.environ['ssn_dlab_path'] + 'tmp/mongo_ensured') except Exception as err: traceback.print_exc() print('Failed to install MongoDB: ', str(err)) sys.exit(1)
def install_torch(os_user): if not exists('/home/{}/.ensure_dir/torch_ensured'.format(os_user)): run('git clone https://github.com/torch/distro.git ~/torch --recursive' ) with cd('/home/{}/torch/'.format(os_user)): manage_pkg( '-y install --nogpgcheck', 'remote', 'cmake curl readline-devel ncurses-devel gcc-c++ gcc-gfortran git gnuplot unzip libjpeg-turbo-devel libpng-devel ImageMagick GraphicsMagick-devel fftw-devel sox-devel sox zeromq3-devel qt-devel qtwebkit-devel sox-plugins-freeworld qt-devel' ) run('./install.sh -b') run('source /home/{}/.bashrc'.format(os_user)) sudo('touch /home/{}/.ensure_dir/torch_ensured'.format(os_user))
def enable_proxy(proxy_host, proxy_port): try: proxy_string = "http://%s:%s" % (proxy_host, proxy_port) sudo('sed -i "/^export http_proxy/d" /etc/profile') sudo('sed -i "/^export https_proxy/d" /etc/profile') sudo('echo export http_proxy=' + proxy_string + ' >> /etc/profile') sudo('echo export https_proxy=' + proxy_string + ' >> /etc/profile') if exists('/etc/yum.conf'): sudo('sed -i "/^proxy=/d" /etc/yum.conf') sudo("echo 'proxy={}' >> /etc/yum.conf".format(proxy_string)) manage_pkg('clean all', 'remote', '') except: sys.exit(1)
def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version): if not exists('/home/' + os_user + '/.ensure_dir/rstudio_ensured'): try: manage_pkg( '-y install --nogpgcheck', 'remote', 'https://download2.rstudio.org/server/centos6/x86_64/rstudio-server-rhel-{}-x86_64.rpm' .format(rstudio_version)) sudo('mkdir -p /mnt/var') sudo('chown {0}:{0} /mnt/var'.format(os_user)) sudo( "sed -i '/Type=forking/a \Environment=USER=dlab-user' /etc/systemd/system/rstudio-server.service" ) sudo( "sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /etc/systemd/system/rstudio-server.service" ) sudo( "sed -i '/ExecStart/s|$|\"|g' /etc/systemd/system/rstudio-server.service" ) sudo("systemctl daemon-reload") sudo('touch /home/{}/.Renviron'.format(os_user)) sudo('chown {0}:{0} /home/{0}/.Renviron'.format(os_user)) sudo('''echo 'SPARK_HOME="{0}"' >> /home/{1}/.Renviron'''.format( local_spark_path, os_user)) sudo('touch /home/{}/.Rprofile'.format(os_user)) sudo('chown {0}:{0} /home/{0}/.Rprofile'.format(os_user)) sudo( '''echo 'library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))' >> /home/{}/.Rprofile''' .format(os_user)) http_proxy = run('echo $http_proxy') https_proxy = run('echo $https_proxy') sudo( '''echo 'Sys.setenv(http_proxy = \"{}\")' >> /home/{}/.Rprofile''' .format(http_proxy, os_user)) sudo( '''echo 'Sys.setenv(https_proxy = \"{}\")' >> /home/{}/.Rprofile''' .format(https_proxy, os_user)) sudo('rstudio-server start') sudo('echo "{0}:{1}" | chpasswd'.format(os_user, rstudio_pass)) sudo("sed -i '/exit 0/d' /etc/rc.local") sudo( '''bash -c "echo \'sed -i 's/^#SPARK_HOME/SPARK_HOME/' /home/{}/.Renviron\' >> /etc/rc.local"''' .format(os_user)) sudo("bash -c 'echo exit 0 >> /etc/rc.local'") sudo('touch /home/{}/.ensure_dir/rstudio_ensured'.format(os_user)) except: sys.exit(1) else: try: sudo('echo "{0}:{1}" | chpasswd'.format(os_user, rstudio_pass)) except: sys.exit(1)
def ensure_supervisor(): try: if not exists('{}tmp/superv_ensured'.format( os.environ['ssn_dlab_path'])): manage_pkg('-y install', 'remote', 'supervisor') #sudo('pip install supervisor') sudo('chkconfig supervisord on') sudo('systemctl start supervisord') sudo('touch {}tmp/superv_ensured'.format( os.environ['ssn_dlab_path'])) except Exception as err: traceback.print_exc() print('Failed to install supervisor: ', str(err)) sys.exit(1)
def get_available_os_pkgs(): try: manage_pkg('update-minimal --security -y --skip-broken', 'remote', '') downgrade_python_version() yum_raw = sudo( 'python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print {pkg.name:pkg.vr for pkg in pl.available}"' ) yum_re = re.sub\ (r'\w*\s\w*\D\s\w*.\w*.\s\w*.\w*.\w.\w*.\w*.\w*', '', yum_raw) yum_list = yum_re.replace("'", "\"") os_pkgs = json.loads(yum_list) return os_pkgs except Exception as err: append_result("Failed to get available os packages.", str(err)) sys.exit(1)
def ensure_docker_daemon(dlab_path, os_user, region): try: if not exists(dlab_path + 'tmp/docker_daemon_ensured'): docker_version = os.environ['ssn_docker_version'] sudo('curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -') sudo('add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \ stable"') manage_pkg('update', 'remote', '') sudo('apt-cache policy docker-ce') manage_pkg('-y install', 'remote', 'docker-ce={}~ce~3-0~ubuntu'.format(docker_version)) sudo('usermod -a -G docker ' + os_user) sudo('update-rc.d docker defaults') sudo('update-rc.d docker enable') sudo('touch ' + dlab_path + 'tmp/docker_daemon_ensured') return True except: return False
def ensure_python3_specific_version(python3_version, os_user): if not exists('/home/' + os_user + '/.ensure_dir/python3_specific_version_ensured'): try: manage_pkg('-y install', 'remote', 'yum-utils python34 openssl-devel') manage_pkg('-y groupinstall', 'remote', 'development --nogpgcheck') if len(python3_version) < 4: python3_version = python3_version + ".0" sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz'. format(python3_version)) sudo( 'tar xzf Python-{0}.tgz; cd Python-{0}; ./configure --prefix=/usr/local; make altinstall' .format(python3_version)) sudo('touch /home/' + os_user + '/.ensure_dir/python3_specific_version_ensured') except: sys.exit(1)
def ensure_jenkins(dlab_path): try: if not exists('{}tmp/jenkins_ensured'.format(dlab_path)): sudo( 'wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo' ) try: sudo( 'rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key' ) except: pass manage_pkg('-y install', 'remote', 'jenkins') manage_pkg('-y install', 'remote', 'policycoreutils-python') sudo('touch {}tmp/jenkins_ensured'.format(dlab_path)) except Exception as err: traceback.print_exc() print('Failed to ensure Jenkins: ', str(err)) sys.exit(1)
def install_opencv(os_user): if not exists('/home/{}/.ensure_dir/opencv_ensured'.format(os_user)): manage_pkg('-y install', 'remote', 'cmake python34 python34-devel python34-pip gcc gcc-c++') sudo('pip2 install numpy=={} --no-cache-dir'.format( os.environ['notebook_numpy_version'])) sudo('pip3.4 install numpy=={} --no-cache-dir'.format( os.environ['notebook_numpy_version'])) sudo('pip3.5 install numpy=={} --no-cache-dir'.format( os.environ['notebook_numpy_version'])) run('git clone https://github.com/opencv/opencv.git') with cd('/home/{}/opencv/'.format(os_user)): run('git checkout 3.2.0') run('mkdir release') with cd('/home/{}/opencv/release/'.format(os_user)): run('cmake -DINSTALL_TESTS=OFF -D CUDA_GENERATION=Auto -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=$(python2 -c "import sys; print(sys.prefix)") -D PYTHON_EXECUTABLE=$(which python2) ..' ) run('make -j$(nproc)') sudo('make install') sudo('touch /home/' + os_user + '/.ensure_dir/opencv_ensured')
def ensure_r(os_user, r_libs, region, r_mirror): if not exists('/home/{}/.ensure_dir/r_ensured'.format(os_user)): try: if region == 'cn-north-1': r_repository = r_mirror else: r_repository = 'https://cloud.r-project.org' manage_pkg('-y install', 'remote', 'cmake') manage_pkg('-y install', 'remote', 'libcur*') sudo( 'echo -e "[base]\nname=CentOS-7-Base\nbaseurl=http://buildlogs.centos.org/centos/7/os/x86_64-20140704-1/\ngpgcheck=1\ngpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7\npriority=1\nexclude=php mysql" >> /etc/yum.repos.d/CentOS-base.repo' ) manage_pkg('-y install', 'remote', 'R R-core R-core-devel R-devel --nogpgcheck') sudo('R CMD javareconf') sudo( 'cd /root; git clone https://github.com/zeromq/zeromq4-x.git; cd zeromq4-x/; mkdir build; cd build; cmake ..; make install; ldconfig' ) for i in r_libs: sudo('R -e "install.packages(\'{}\',repos=\'{}\')"'.format( i, r_repository)) sudo( 'R -e "library(\'devtools\');install.packages(repos=\'{}\',c(\'rzmq\',\'repr\',\'digest\',\'stringr\',\'RJSONIO\',\'functional\',\'plyr\'))"' .format(r_repository)) sudo( 'R -e "library(\'devtools\');install_github(\'IRkernel/repr\');install_github(\'IRkernel/IRdisplay\');install_github(\'IRkernel/IRkernel\');"' ) sudo( 'R -e "library(\'devtools\');install_version(\'keras\', version = \'{}\', repos = \'{}\');"' .format(os.environ['notebook_keras_version'], r_repository)) sudo('R -e "install.packages(\'RJDBC\',repos=\'{}\',dep=TRUE)"'. format(r_repository)) sudo('touch /home/{}/.ensure_dir/r_ensured'.format(os_user)) except: sys.exit(1)
def ensure_docker_daemon(dlab_path, os_user, region): try: if not exists('{}tmp/docker_daemon_ensured'.format(dlab_path)): docker_version = os.environ['ssn_docker_version'] if region == 'cn-north-1': mirror = 'mirror.lzu.edu.cn' else: mirror = 'mirror.centos.org' with cd('/etc/yum.repos.d/'): sudo('echo "[centosrepo]" > centos.repo') sudo('echo "name=Centos 7 Repository" >> centos.repo') sudo( 'echo "baseurl=http://{}/centos/7/extras/x86_64/" >> centos.repo' .format(mirror)) sudo('echo "enabled=1" >> centos.repo') sudo('echo "gpgcheck=1" >> centos.repo') sudo( 'echo "gpgkey=http://{}/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7" >> centos.repo' .format(mirror)) sudo( 'yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo' ) manage_pkg('update-minimal --security -y', 'remote', '') manage_pkg('-y install', 'remote', 'container-selinux') manage_pkg('-y install', 'remote', 'docker-ce-{}.ce'.format(docker_version)) sudo('usermod -aG docker {}'.format(os_user)) sudo('systemctl enable docker.service') sudo('systemctl start docker') sudo('touch {}tmp/docker_daemon_ensured'.format(dlab_path)) return True except: return False
def install_caffe2(os_user, caffe2_version, cmake_version): if not exists('/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)): env.shell = "/bin/bash -l -c -i" manage_pkg('update-minimal --security -y', 'remote', '') manage_pkg( '-y install --nogpgcheck', 'remote', 'automake cmake3 gcc gcc-c++ kernel-devel leveldb-devel lmdb-devel libtool protobuf-devel graphviz' ) sudo( 'pip2 install flask graphviz hypothesis jupyter matplotlib==2.0.2 numpy=={} protobuf pydot python-nvd3 pyyaml ' 'requests scikit-image scipy setuptools tornado future --no-cache-dir' .format(os.environ['notebook_numpy_version'])) sudo( 'pip3.5 install flask graphviz hypothesis jupyter matplotlib==2.0.2 numpy=={} protobuf pydot python-nvd3 pyyaml ' 'requests scikit-image scipy setuptools tornado future --no-cache-dir' .format(os.environ['notebook_numpy_version'])) sudo('cp /opt/cudnn/include/* /opt/cuda-8.0/include/') sudo('cp /opt/cudnn/lib64/* /opt/cuda-8.0/lib64/') sudo( 'wget https://cmake.org/files/v{2}/cmake-{1}.tar.gz -O /home/{0}/cmake-{1}.tar.gz' .format( os_user, cmake_version, cmake_version.split('.')[0] + "." + cmake_version.split('.')[1])) sudo('tar -zxvf cmake-{}.tar.gz'.format(cmake_version)) with cd('/home/{}/cmake-{}/'.format(os_user, cmake_version)): sudo('./bootstrap --prefix=/usr/local && make && make install') sudo('ln -s /usr/local/bin/cmake /bin/cmake{}'.format(cmake_version)) sudo('git clone https://github.com/pytorch/pytorch.git') with cd('/home/{}/pytorch/'.format(os_user)): sudo('git submodule update --init') with settings(warn_only=True): sudo('git checkout v{}'.format(caffe2_version)) sudo('git submodule update --recursive') sudo( 'mkdir build && cd build && cmake{} .. && make "-j$(nproc)" install' .format(cmake_version)) sudo('touch /home/' + os_user + '/.ensure_dir/caffe2_ensured')
def install_os_pkg(requisites): status = list() error_parser = "Could not|No matching|Error:|failed|Requires:|Errno" try: print( "Updating repositories and installing requested tools: {}".format( requisites)) manage_pkg('update-minimal --security -y --skip-broken', 'remote', '') sudo('export LC_ALL=C') for os_pkg in requisites: manage_pkg( '-y install', 'remote', '{0} --nogpgcheck 2>&1 | if ! grep -w -E "({1})" > /tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi' .format(os_pkg, error_parser)) err = sudo('cat /tmp/os_install_{}.log'.format(os_pkg)).replace( '"', "'") try: res = sudo( 'python -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\'][0]"' .format(os_pkg)) version = res.split('\r\n')[1].replace("'", "\"") status.append({ "group": "os_pkg", "name": os_pkg, "version": version, "status": "installed" }) except: status.append({ "group": "os_pkg", "name": os_pkg, "status": "failed", "error_message": err }) return status except: return "Fail to install OS packages"
def install_cntk(os_user, cntk_version): if not exists('/home/{}/.ensure_dir/cntk_ensured'.format(os_user)): sudo('echo "exclude=*.i386 *.i686" >> /etc/yum.conf') manage_pkg('clean', 'remote', 'all') manage_pkg('update-minimal --security -y', 'remote', '') manage_pkg('-y install --nogpgcheck', 'remote', 'openmpi openmpi-devel') sudo( 'pip2 install https://cntk.ai/PythonWheel/GPU/cntk-{}-cp27-cp27mu-linux_x86_64.whl --no-cache-dir' .format(cntk_version)) sudo( 'pip3.5 install https://cntk.ai/PythonWheel/GPU/cntk-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir' .format(cntk_version)) sudo('touch /home/{}/.ensure_dir/cntk_ensured'.format(os_user))
def install_tensor(os_user, cuda_version, cuda_file_name, cudnn_version, cudnn_file_name, tensorflow_version, templates_dir, nvidia_version): if not exists('/home/{}/.ensure_dir/tensor_ensured'.format(os_user)): try: # install nvidia drivers sudo( 'echo "blacklist nouveau" >> /etc/modprobe.d/blacklist-nouveau.conf' ) sudo( 'echo "options nouveau modeset=0" >> /etc/modprobe.d/blacklist-nouveau.conf' ) sudo('dracut --force') with settings(warn_only=True): reboot(wait=150) manage_pkg( '-y install', 'remote', 'libglvnd-opengl libglvnd-devel dkms gcc kernel-devel-$(uname -r) kernel-headers-$(uname -r)' ) sudo( 'wget http://us.download.nvidia.com/XFree86/Linux-x86_64/{0}/NVIDIA-Linux-x86_64-{0}.run -O /home/{1}/NVIDIA-Linux-x86_64-{0}.run' .format(nvidia_version, os_user)) sudo('/bin/bash /home/{0}/NVIDIA-Linux-x86_64-{1}.run -s --dkms'. format(os_user, nvidia_version)) sudo('rm -f /home/{0}/NVIDIA-Linux-x86_64-{1}.run'.format( os_user, nvidia_version)) # install cuda sudo( 'python3.5 -m pip install --upgrade pip=={0} wheel numpy=={1} --no-cache-dir' .format(os.environ['conf_pip_version'], os.environ['notebook_numpy_version'])) sudo( 'wget -P /opt https://developer.nvidia.com/compute/cuda/{0}/prod/local_installers/{1}' .format(cuda_version, cuda_file_name)) sudo('sh /opt/{} --silent --toolkit'.format(cuda_file_name)) sudo('mv /usr/local/cuda-{} /opt/'.format(cuda_version)) sudo( 'ln -s /opt/cuda-{0} /usr/local/cuda-{0}'.format(cuda_version)) sudo('rm -f /opt/{}'.format(cuda_file_name)) # install cuDNN run('wget http://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}' .format(cudnn_version, cudnn_file_name)) run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name)) sudo('mkdir -p /opt/cudnn/include') sudo('mkdir -p /opt/cudnn/lib64') sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include') sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64') sudo( 'chmod a+r /opt/cudnn/include/cudnn.h /opt/cudnn/lib64/libcudnn*' ) run('echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc' ) # install TensorFlow and run TensorBoard sudo( 'wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl' .format(tensorflow_version)) sudo( 'wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl' .format(tensorflow_version)) sudo( 'python2.7 -m pip install --upgrade tensorflow_gpu-{}-cp27-none-linux_x86_64.whl --no-cache-dir' .format(tensorflow_version)) sudo( 'python3.5 -m pip install --upgrade tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir' .format(tensorflow_version)) sudo('rm -rf /home/{}/tensorflow_gpu-*'.format(os_user)) sudo( 'mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard' .format(os_user)) put('{}tensorboard.service'.format(templates_dir), '/tmp/tensorboard.service') sudo("sed -i 's|OS_USR|{}|' /tmp/tensorboard.service".format( os_user)) sudo("chmod 644 /tmp/tensorboard.service") sudo('\cp /tmp/tensorboard.service /etc/systemd/system/') sudo("systemctl daemon-reload") sudo("systemctl enable tensorboard") sudo("systemctl start tensorboard") sudo('touch /home/{}/.ensure_dir/tensor_ensured'.format(os_user)) except: sys.exit(1)
def install_nodejs(os_user): if not exists('/home/{}/.ensure_dir/nodejs_ensured'.format(os_user)): sudo('curl -sL https://rpm.nodesource.com/setup_6.x | sudo -E bash -') manage_pkg('-y install', 'remote', 'nodejs') sudo('touch /home/{}/.ensure_dir/nodejs_ensured'.format(os_user))
def remove_os_pkg(pkgs): try: manage_pkg('remove -y', 'remote', '{}'.format(' '.join(pkgs))) except: sys.exit(1)