def ensure_docker_daemon(datalab_path, os_user, region): try: if not exists(datalab.fab.conn, datalab_path + 'tmp/docker_daemon_ensured'): docker_version = os.environ['ssn_docker_version'] datalab.fab.conn.sudo( 'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -' ) datalab.fab.conn.sudo( 'add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \ stable"') manage_pkg('update', 'remote', '') datalab.fab.conn.sudo('apt-cache policy docker-ce') manage_pkg( '-y install', 'remote', 'docker-ce=5:{}~3-0~ubuntu-focal'.format(docker_version)) datalab.fab.conn.sudo('usermod -a -G docker ' + os_user) datalab.fab.conn.sudo('update-rc.d docker defaults') datalab.fab.conn.sudo('update-rc.d docker enable') datalab.fab.conn.sudo('touch ' + datalab_path + 'tmp/docker_daemon_ensured') return True except: traceback.print_exc() print('Failed to ensure_docker_daemon') return False
def install_build_dep(): try: if not exists( conn, '{}tmp/build_dep_ensured'.format( os.environ['ssn_datalab_path'])): maven_version = '3.5.4' manage_pkg( '-y install', 'remote', 'java-1.8.0-openjdk java-1.8.0-openjdk-devel git wget unzip') conn.sudo( 'cd /opt/ && wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip' .format(maven_version.split('.')[0], maven_version)) conn.sudo( '''bash -c 'cd /opt/ &&unzip apache-maven-{}-bin.zip' '''. format(maven_version)) conn.sudo( '''bash -c 'cd /opt/ &&mv apache-maven-{} maven' '''.format( maven_version)) conn.sudo( 'bash -c "curl --silent --location https://rpm.nodesource.com/setup_12.x | bash -"' ) manage_pkg('-y install', 'remote', 'nodejs') conn.sudo('npm config set unsafe-perm=true') conn.sudo('touch {}tmp/build_dep_ensured'.format( os.environ['ssn_datalab_path'])) except Exception as err: traceback.print_exc() print('Failed to install build dependencies for UI: ', str(err)) sys.exit(1)
def ensure_mongo(): try: if not exists( conn, '{}tmp/mongo_ensured'.format( os.environ['ssn_datalab_path'])): conn.sudo( 'echo -e "[mongodb-org-3.2]\nname=MongoDB Repository' '\nbaseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/3.2/x86_64/' '\ngpgcheck=1' '\nenabled=1' '\ngpgkey=https://www.mongodb.org/static/pgp/server-3.2.asc" ' '> /etc/yum.repos.d/mongodb.repo') manage_pkg('-y install', 'remote', 'mongodb-org') conn.sudo('semanage port -a -t mongod_port_t -p tcp 27017') conn.sudo('chkconfig mongod on') conn.sudo( 'echo "d /var/run/mongodb 0755 mongod mongod" > /lib/tmpfiles.d/mongodb.conf' ) conn.sudo('sudo systemd-tmpfiles --create mongodb.conf') conn.sudo('systemctl start mongod.service') conn.sudo('touch {}tmp/mongo_ensured'.format( os.environ['ssn_datalab_path'])) except Exception as err: traceback.print_exc() print('Failed to install MongoDB: ', str(err)) sys.exit(1)
def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version): if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/rstudio_ensured'): try: manage_pkg('-y install --nogpgcheck', 'remote', 'https://download2.rstudio.org/server/centos6/x86_64/rstudio-server-rhel-{}-x86_64.rpm'.format(rstudio_version)) datalab.fab.conn.sudo('mkdir -p /mnt/var') datalab.fab.conn.sudo('chown {0}:{0} /mnt/var'.format(os_user)) datalab.fab.conn.sudo("sed -i '/Type=forking/a \Environment=USER=datalab-user' /lib/systemd/system/rstudio-server.service") datalab.fab.conn.sudo( "sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /lib/systemd/system/rstudio-server.service") datalab.fab.conn.sudo("sed -i '/ExecStart/s|$|\"|g' /lib/systemd/system/rstudio-server.service") datalab.fab.conn.sudo("systemctl daemon-reload") datalab.fab.conn.sudo('touch /home/{}/.Renviron'.format(os_user)) datalab.fab.conn.sudo('chown {0}:{0} /home/{0}/.Renviron'.format(os_user)) datalab.fab.conn.sudo('''echo 'SPARK_HOME="{0}"' >> /home/{1}/.Renviron'''.format(local_spark_path, os_user)) datalab.fab.conn.sudo('touch /home/{}/.Rprofile'.format(os_user)) datalab.fab.conn.sudo('chown {0}:{0} /home/{0}/.Rprofile'.format(os_user)) datalab.fab.conn.sudo('''echo 'library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))' >> /home/{}/.Rprofile'''.format(os_user)) http_proxy = datalab.fab.conn.run('''bash -l -c 'echo $http_proxy' ''').stdout.replace('\n','') https_proxy = datalab.fab.conn.run('''bash -l -c 'echo $https_proxy' ''').stdout.replace('\n','') datalab.fab.conn.sudo('''echo 'Sys.setenv(http_proxy = \"{}\")' >> /home/{}/.Rprofile'''.format(http_proxy, os_user)) datalab.fab.conn.sudo('''echo 'Sys.setenv(https_proxy = \"{}\")' >> /home/{}/.Rprofile'''.format(https_proxy, os_user)) datalab.fab.conn.sudo('rstudio-server start') datalab.fab.conn.sudo('''bash -c 'echo "{0}:{1}" | chpasswd' '''.format(os_user, rstudio_pass)) datalab.fab.conn.sudo("sed -i '/exit 0/d' /etc/rc.local") datalab.fab.conn.sudo('''bash -c "echo \'sed -i 's/^#SPARK_HOME/SPARK_HOME/' /home/{}/.Renviron\' >> /etc/rc.local"'''.format(os_user)) datalab.fab.conn.sudo("bash -c 'echo exit 0 >> /etc/rc.local'") datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/rstudio_ensured'.format(os_user)) except: sys.exit(1) else: try: datalab.fab.conn.sudo('''bash -c 'echo "{0}:{1}" | chpasswd' '''.format(os_user, rstudio_pass)) except: sys.exit(1)
def ensure_jre_jdk(os_user): if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'): try: manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk') manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk-devel') datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/jre_jdk_ensured') except: sys.exit(1)
def ensure_sbt(os_user): if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/sbt_ensured'.format(os_user)): try: datalab.fab.conn.sudo('curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo') manage_pkg('-y install', 'remote', 'sbt') datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/sbt_ensured'.format(os_user)) except: sys.exit(1)
def ensure_nginx(datalab_path): try: if not exists(conn, '{}tmp/nginx_ensured'.format(datalab_path)): manage_pkg('-y install', 'remote', 'nginx') conn.sudo('systemctl restart nginx.service') conn.sudo('chkconfig nginx on') conn.sudo('touch {}tmp/nginx_ensured'.format(datalab_path)) except Exception as err: traceback.print_exc() print('Failed to ensure Nginx: ', str(err)) sys.exit(1)
def ensure_additional_python_libs(os_user): if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/additional_python_libs_ensured'): try: manage_pkg('clean', 'remote', 'all') manage_pkg('-y install', 'remote', 'zlib-devel libjpeg-turbo-devel --nogpgcheck') if os.environ['application'] in ('jupyter', 'zeppelin'): datalab.fab.conn.sudo('python3.5 -m pip install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'.format(os.environ['notebook_numpy_version'])) if os.environ['application'] in ('tensor', 'deeplearning'): datalab.fab.conn.sudo('python3.8 -m pip install opencv-python h5py --no-cache-dir') datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/additional_python_libs_ensured') except: sys.exit(1)
def ensure_python3_specific_version(python3_version, os_user): if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/python3_specific_version_ensured'): try: manage_pkg('-y install', 'remote', 'yum-utils python34 openssl-devel') manage_pkg('-y groupinstall', 'remote', 'development --nogpgcheck') if len(python3_version) < 4: python3_version = python3_version + ".0" datalab.fab.conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz'.format(python3_version)) datalab.fab.conn.sudo('tar xzf Python-{0}.tgz; cd Python-{0}; ./configure --prefix=/usr/local; make altinstall'.format(python3_version)) datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_specific_version_ensured') except: sys.exit(1)
def install_opencv(os_user): if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/opencv_ensured'.format(os_user)): manage_pkg('-y install', 'remote', 'cmake python34 python34-devel python34-pip gcc gcc-c++') datalab.fab.conn.sudo('pip3.4 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version'])) datalab.fab.conn.sudo('pip3.5 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version'])) datalab.fab.conn.run('git clone https://github.com/opencv/opencv.git') datalab.fab.conn.run('cd /home/{}/opencv/ && git checkout 3.2.0'.format(os_user)) datalab.fab.conn.run('cd /home/{}/opencv/ && mkdir release'.format(os_user)) datalab.fab.conn.run('cd /home/{}/opencv/release/ && cmake -DINSTALL_TESTS=OFF -D CUDA_GENERATION=Auto -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=$(python2 -c "import sys; print(sys.prefix)") -D PYTHON_EXECUTABLE=$(which python2) ..') datalab.fab.conn.run('cd /home/{}/opencv/release/ && make -j$(nproc)') datalab.fab.conn.sudo('''bash -c 'cd /home/{}/opencv/release/ && make install' ''') datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/opencv_ensured')
def ensure_nginx(datalab_path): try: if not exists(datalab.fab.conn, datalab_path + 'tmp/nginx_ensured'): manage_pkg('-y install', 'remote', 'nginx') datalab.fab.conn.sudo('service nginx restart') datalab.fab.conn.sudo('update-rc.d nginx defaults') datalab.fab.conn.sudo('update-rc.d nginx enable') datalab.fab.conn.sudo('touch ' + datalab_path + 'tmp/nginx_ensured') except Exception as err: traceback.print_exc() print('Failed to ensure Nginx: ', str(err)) sys.exit(1)
def ensure_supervisor(): try: if not exists(datalab.fab.conn, os.environ['ssn_datalab_path'] + 'tmp/superv_ensured'): manage_pkg('-y install', 'remote', 'supervisor') datalab.fab.conn.sudo('update-rc.d supervisor defaults') datalab.fab.conn.sudo('update-rc.d supervisor enable') datalab.fab.conn.sudo('touch ' + os.environ['ssn_datalab_path'] + 'tmp/superv_ensured') except Exception as err: traceback.print_exc() print('Failed to install Supervisor: ', str(err)) sys.exit(1)
def enable_proxy(proxy_host, proxy_port): try: proxy_string = "http://%s:%s" % (proxy_host, proxy_port) proxy_https_string = "https://%s:%s" % (proxy_host, proxy_port) datalab.fab.conn.sudo('sed -i "/^export http_proxy/d" /etc/profile') datalab.fab.conn.sudo('sed -i "/^export https_proxy/d" /etc/profile') datalab.fab.conn.sudo('bash -c "echo export http_proxy=' + proxy_string + ' >> /etc/profile"') datalab.fab.conn.sudo('bash -c "echo export https_proxy=' + proxy_string + ' >> /etc/profile"') if exists(datalab.fab.conn, '/etc/yum.conf'): datalab.fab.conn.sudo('sed -i "/^proxy=/d" /etc/yum.conf') datalab.fab.conn.sudo('''bash -c "echo 'proxy={}' >> /etc/yum.conf" '''.format(proxy_string)) manage_pkg('clean all', 'remote', '') except: sys.exit(1)
def get_available_os_pkgs(): try: os_pkgs = dict() manage_pkg('update-minimal --security -y --skip-broken', 'remote', '') #downgrade_python_version() yum_names = datalab.fab.conn.sudo("yum list available | grep -v \"Loaded plugins:\|Available Packages\" | xargs -n3 | column -t | awk '{print $1}'").stdout.split('\n') for pkg in yum_names: if "." in pkg: os_pkgs[pkg.split('.')[0]] = 'N/A' elif pkg != '': os_pkgs[pkg] = 'N/A' return os_pkgs except Exception as err: append_result("Failed to get available os packages.", str(err)) sys.exit(1)
def ensure_supervisor(): try: if not exists( conn, '{}tmp/superv_ensured'.format( os.environ['ssn_datalab_path'])): manage_pkg('-y install', 'remote', 'supervisor') # conn.sudo('pip install supervisor') conn.sudo('chkconfig supervisord on') conn.sudo('systemctl start supervisord') conn.sudo('touch {}tmp/superv_ensured'.format( os.environ['ssn_datalab_path'])) except Exception as err: traceback.print_exc() print('Failed to install supervisor: ', str(err)) sys.exit(1)
def install_tensor(os_user, cuda_version, cuda_file_name, cudnn_version, cudnn_file_name, tensorflow_version, templates_dir, nvidia_version): if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/tensor_ensured'.format(os_user)): try: # install nvidia drivers datalab.fab.conn.sudo('''bash -c 'echo "blacklist nouveau" >> /etc/modprobe.d/blacklist-nouveau.conf' ''') datalab.fab.conn.sudo('''bash -c 'echo "options nouveau modeset=0" >> /etc/modprobe.d/blacklist-nouveau.conf' ''') datalab.fab.conn.sudo('dracut --force') datalab.fab.conn.sudo('reboot', warn=True) time.sleep(150) manage_pkg('-y install', 'remote', 'libglvnd-opengl libglvnd-devel dkms gcc kernel-devel-$(uname -r) kernel-headers-$(uname -r)') datalab.fab.conn.sudo('wget http://us.download.nvidia.com/XFree86/Linux-x86_64/{0}/NVIDIA-Linux-x86_64-{0}.run -O /home/{1}/NVIDIA-Linux-x86_64-{0}.run'.format(nvidia_version, os_user)) datalab.fab.conn.sudo('/bin/bash /home/{0}/NVIDIA-Linux-x86_64-{1}.run -s --dkms'.format(os_user, nvidia_version)) datalab.fab.conn.sudo('rm -f /home/{0}/NVIDIA-Linux-x86_64-{1}.run'.format(os_user, nvidia_version)) # install cuda datalab.fab.conn.sudo('python3.5 -m pip install --upgrade pip=={0} wheel numpy=={1} --no-cache-dir'. format(os.environ['conf_pip_version'], os.environ['notebook_numpy_version'])) datalab.fab.conn.sudo('wget -P /opt https://developer.nvidia.com/compute/cuda/{0}/prod/local_installers/{1}'.format(cuda_version, cuda_file_name)) datalab.fab.conn.sudo('sh /opt/{} --silent --toolkit'.format(cuda_file_name)) datalab.fab.conn.sudo('mv /usr/local/cuda-{} /opt/'.format(cuda_version[:-2])) datalab.fab.conn.sudo('ln -s /opt/cuda-{0} /usr/local/cuda-{0}'.format(cuda_version[:-2])) datalab.fab.conn.sudo('rm -f /opt/{}'.format(cuda_file_name)) # install cuDNN datalab.fab.conn.run('wget https://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(cudnn_version, cudnn_file_name)) datalab.fab.conn.run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name)) datalab.fab.conn.sudo('mkdir -p /opt/cudnn/include') datalab.fab.conn.sudo('mkdir -p /opt/cudnn/lib64') datalab.fab.conn.sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include') datalab.fab.conn.sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64') datalab.fab.conn.sudo('chmod a+r /opt/cudnn/include/cudnn.h /opt/cudnn/lib64/libcudnn*') datalab.fab.conn.run('''bash -l -c 'echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc' ''') # install TensorFlow and run TensorBoard datalab.fab.conn.sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl'.format(tensorflow_version)) datalab.fab.conn.sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl'.format(tensorflow_version)) datalab.fab.conn.sudo('python3.8 -m pip install --upgrade tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version)) datalab.fab.conn.sudo('rm -rf /home/{}/tensorflow_gpu-*'.format(os_user)) datalab.fab.conn.sudo('mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard'.format(os_user)) datalab.fab.conn.put('{}tensorboard.service'.format(templates_dir), '/tmp/tensorboard.service') datalab.fab.conn.sudo("sed -i 's|OS_USR|{}|' /tmp/tensorboard.service".format(os_user)) datalab.fab.conn.sudo("chmod 644 /tmp/tensorboard.service") datalab.fab.conn.sudo('\cp /tmp/tensorboard.service /etc/systemd/system/') datalab.fab.conn.sudo("systemctl daemon-reload") datalab.fab.conn.sudo("systemctl enable tensorboard") datalab.fab.conn.sudo("systemctl start tensorboard") datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/tensor_ensured'.format(os_user)) except: sys.exit(1)
def ensure_jenkins(datalab_path): try: if not exists(datalab.fab.conn, datalab_path + 'tmp/jenkins_ensured'): datalab.fab.conn.sudo( 'wget -q -O - https://pkg.jenkins.io/debian/jenkins-ci.org.key | sudo apt-key add -' ) datalab.fab.conn.sudo( 'echo deb http://pkg.jenkins.io/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list' ) manage_pkg('-y update', 'remote', '') manage_pkg('-y install', 'remote', 'jenkins') datalab.fab.conn.sudo('touch ' + datalab_path + 'tmp/jenkins_ensured') except Exception as err: traceback.print_exc() print('Failed to ensure Jenkins: ', str(err)) sys.exit(1)
def ensure_python3_libraries(os_user): if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'): try: manage_pkg('-y install', 'remote', 'https://centos7.iuscommunity.org/ius-release.rpm') manage_pkg('-y install', 'remote', 'python35u python35u-pip python35u-devel') datalab.fab.conn.sudo('python3.5 -m pip install -U pip=={} setuptools --no-cache-dir'.format(os.environ['conf_pip_version'])) datalab.fab.conn.sudo('python3.5 -m pip install boto3 --no-cache-dir') datalab.fab.conn.sudo('python3.5 -m pip install fabvenv fabric-virtualenv future patchwork --no-cache-dir') try: datalab.fab.conn.sudo('python3.5 -m pip install tornado=={0} ipython==7.9.0 ipykernel=={1} --no-cache-dir' \ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'])) except: datalab.fab.conn.sudo('python3.5 -m pip install tornado=={0} ipython==5.0.0 ipykernel=={1} --no-cache-dir' \ .format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version'])) datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured') except: sys.exit(1)
def ensure_jenkins(datalab_path): try: if not exists(conn, '{}tmp/jenkins_ensured'.format(datalab_path)): conn.sudo( 'wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo' ) try: conn.sudo( 'rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key' ) except: pass manage_pkg('-y install', 'remote', 'jenkins') manage_pkg('-y install', 'remote', 'policycoreutils-python') conn.sudo('touch {}tmp/jenkins_ensured'.format(datalab_path)) except Exception as err: traceback.print_exc() print('Failed to ensure Jenkins: ', str(err)) sys.exit(1)
def install_caffe2(os_user, caffe2_version, cmake_version): if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)): env.shell = "/bin/bash -l -c -i" manage_pkg('update-minimal --security -y', 'remote', '') manage_pkg('-y install --nogpgcheck', 'remote', 'automake cmake3 gcc gcc-c++ kernel-devel leveldb-devel lmdb-devel libtool protobuf-devel graphviz') datalab.fab.conn.sudo('pip3.5 install flask graphviz hypothesis jupyter matplotlib=={} numpy=={} protobuf pydot python-nvd3 pyyaml ' 'requests scikit-image scipy setuptools tornado future --no-cache-dir'.format(os.environ['notebook_matplotlib_version'], os.environ['notebook_numpy_version'])) datalab.fab.conn.sudo('cp /opt/cudnn/include/* /opt/cuda-8.0/include/') datalab.fab.conn.sudo('cp /opt/cudnn/lib64/* /opt/cuda-8.0/lib64/') datalab.fab.conn.sudo('wget https://cmake.org/files/v{2}/cmake-{1}.tar.gz -O /home/{0}/cmake-{1}.tar.gz'.format( os_user, cmake_version, cmake_version.split('.')[0] + "." + cmake_version.split('.')[1])) datalab.fab.conn.sudo('tar -zxvf cmake-{}.tar.gz'.format(cmake_version)) datalab.fab.conn.sudo('''bash -c 'cd /home/{}/cmake-{}/ && ./bootstrap --prefix=/usr/local && make && make install' '''.format(os_user, cmake_version)) datalab.fab.conn.sudo('ln -s /usr/local/bin/cmake /bin/cmake{}'.format(cmake_version)) datalab.fab.conn.sudo('git clone https://github.com/pytorch/pytorch.git') datalab.fab.conn.sudo('''bash -c 'cd /home/{}/pytorch/ && git submodule update --init' '''.format(os_user)) datalab.fab.conn.sudo('''bash -c 'cd /home/{}/pytorch/ && git checkout v{}' '''.format(os_user, caffe2_version), warn=True) datalab.fab.conn.sudo('''bash -c 'cd /home/{}/pytorch/ && git submodule update --recursive' '''.format(os_user), warn=True) datalab.fab.conn.sudo('''bash -c 'cd /home/{}/pytorch/ && mkdir build && cd build && cmake{} .. && make "-j$(nproc)" install' '''.format(os_user, cmake_version)) datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/caffe2_ensured')
def ensure_mongo(): try: if not exists(datalab.fab.conn, os.environ['ssn_datalab_path'] + 'tmp/mongo_ensured'): datalab.fab.conn.sudo( 'wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add -' ) os_version = datalab.fab.conn.sudo( 'lsb_release -cs').stdout.replace('\n', '') datalab.fab.conn.sudo( 'echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu {}/mongodb-org/4.4 ' 'multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list' .format(os_version)) manage_pkg('update', 'remote', '') manage_pkg('-y install', 'remote', 'mongodb-org') datalab.fab.conn.sudo('systemctl enable mongod.service') datalab.fab.conn.sudo('touch ' + os.environ['ssn_datalab_path'] + 'tmp/mongo_ensured') except Exception as err: traceback.print_exc() print('Failed to install MongoDB: ', str(err)) sys.exit(1)
def install_cntk(os_user, cntk_version): if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/cntk_ensured'.format(os_user)): datalab.fab.conn.sudo('echo "exclude=*.i386 *.i686" >> /etc/yum.conf') manage_pkg('clean', 'remote', 'all') manage_pkg('update-minimal --security -y', 'remote', '') manage_pkg('-y install --nogpgcheck', 'remote', 'openmpi openmpi-devel') datalab.fab.conn.sudo('pip3.5 install https://cntk.ai/PythonWheel/GPU/cntk-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(cntk_version)) datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/cntk_ensured'.format(os_user))
def ensure_r(os_user, r_libs): if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/r_ensured'.format(os_user)): try: r_repository = 'https://cloud.r-project.org' manage_pkg('-y install', 'remote', 'cmake') manage_pkg('-y install', 'remote', 'libcur*') datalab.fab.conn.sudo('echo -e "[base]\nname=CentOS-7-Base\nbaseurl=http://buildlogs.centos.org/centos/7/os/x86_64-20140704-1/\ngpgcheck=1\ngpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7\npriority=1\nexclude=php mysql" >> /etc/yum.repos.d/CentOS-base.repo') manage_pkg('-y install', 'remote', 'R R-core R-core-devel R-devel --nogpgcheck') datalab.fab.conn.sudo('R CMD javareconf') datalab.fab.conn.sudo('''bash -c 'cd /root; git clone https://github.com/zeromq/zeromq4-x.git; cd zeromq4-x/; mkdir build; cd build; cmake ..; make install; ldconfig' ''') for i in r_libs: datalab.fab.conn.sudo('R -e "install.packages(\'{}\',repos=\'{}\')"'.format(i, r_repository)) datalab.fab.conn.sudo('R -e "library(\'devtools\');install.packages(repos=\'{}\',c(\'rzmq\',\'repr\',\'digest\',\'stringr\',\'RJSONIO\',\'functional\',\'plyr\'))"'.format(r_repository)) datalab.fab.conn.sudo('R -e "library(\'devtools\');install_github(\'IRkernel/repr\');install_github(\'IRkernel/IRdisplay\');install_github(\'IRkernel/IRkernel\');"') datalab.fab.conn.sudo('R -e "library(\'devtools\');install_version(\'keras\', version = \'{}\', repos = \'{}\');"'.format(os.environ['notebook_keras_version'],r_repository)) datalab.fab.conn.sudo('R -e "install.packages(\'RJDBC\',repos=\'{}\',dep=TRUE)"'.format(r_repository)) datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/r_ensured'.format(os_user)) except: sys.exit(1)
def ensure_docker_daemon(datalab_path, os_user, region): try: if not exists(conn, '{}tmp/docker_daemon_ensured'.format(datalab_path)): docker_version = os.environ['ssn_docker_version'] if region == 'cn-north-1': mirror = 'mirror.lzu.edu.cn' else: mirror = 'mirror.centos.org' conn.sudo( '''bash -c 'cd /etc/yum.repos.d/ && echo "[centosrepo]" > centos.repo' ''' ) conn.sudo( '''bash -c 'cd /etc/yum.repos.d/ && echo "name=Centos 7 Repository" >> centos.repo' ''' ) conn.sudo( '''bash -c 'cd /etc/yum.repos.d/ && echo "baseurl=http://{}/centos/7/extras/x86_64/" >> centos.repo' ''' .format(mirror)) conn.sudo( '''bash -c 'cd /etc/yum.repos.d/ && echo "enabled=1" >> centos.repo' ''' ) conn.sudo( '''bash -c 'cd /etc/yum.repos.d/ && echo "gpgcheck=1" >> centos.repo' ''' ) conn.sudo( '''bash -c 'cd /etc/yum.repos.d/ && echo "gpgkey=http://{}/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7" >> centos.repo' ''' .format(mirror)) conn.sudo( 'yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo' ) manage_pkg('update-minimal --security -y', 'remote', '') manage_pkg('-y install', 'remote', 'container-selinux') manage_pkg('-y install', 'remote', 'docker-ce-{}.ce'.format(docker_version)) conn.sudo('usermod -aG docker {}'.format(os_user)) conn.sudo('systemctl enable docker.service') conn.sudo('systemctl start docker') conn.sudo('touch {}tmp/docker_daemon_ensured'.format(datalab_path)) return True except: return False
def remove_os_pkg(pkgs): try: manage_pkg('remove -y', 'remote', '{}'.format(' '.join(pkgs))) except: sys.exit(1)
def install_os_pkg(requisites): status = list() error_parser = "Could not|No matching|Error:|failed|Requires:|Errno" new_pkgs_parser = "Dependency Installed:" try: print("Updating repositories and installing requested tools: {}".format(requisites)) manage_pkg('update-minimal --security -y --skip-broken', 'remote', '') #datalab.fab.conn.sudo('export LC_ALL=C') for os_pkg in requisites: name, vers = os_pkg if vers != '' and vers !='N/A': version = vers os_pkg = "{}-{}".format(name, vers) else: version = 'N/A' os_pkg = name manage_pkg('-y install', 'remote', '{0} --nogpgcheck 2>&1 | tee /tmp/os_install_{2}.tmp; if ! grep -w -E "({1})" ' '/tmp/os_install_{2}.tmp > /tmp/os_install_{2}.log; then echo "no_error" > /tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name)) install_output = datalab.fab.conn.sudo('cat /tmp/os_install_{}.tmp'.format(name)).stdout err = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout.replace('"', "'") datalab.fab.conn.sudo('cat /tmp/os_install_{0}.tmp | if ! grep -w -E -A 30 "({1})" /tmp/os_install_{0}.tmp > ' '/tmp/os_install_{0}.log; then echo "no_dependencies" > /tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser)) dep = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout if 'no_dependencies' in dep: dep = [] else: dep = dep[len(new_pkgs_parser): dep.find("Complete!") - 1].replace(' ', '').strip().split('\r\n') for n, i in enumerate(dep): i = i.split('.')[0] datalab.fab.conn.sudo('yum info {0} 2>&1 | if ! grep Version > /tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(i)) dep[n] =sudo('cat /tmp/os_install_{}.log'.format(i)).replace('Version : ', '{} v.'.format(i)) dep = [i for i in dep if i] versions = [] datalab.fab.conn.sudo( 'yum list installed | if ! grep "{0}\." > /tmp/os_install_{0}.list; then echo "not_installed" > /tmp/os_install_{0}.list;fi'.format( name)) res = datalab.fab.conn.sudo('cat /tmp/os_install_{}.list '.format(name) + '| awk \'{print $1":"$2}\'').stdout.replace('\n', '') #res = datalab.fab.conn.sudo('python3 -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\']"'.format(name)).stdout.split('\r\n')[1] if "no_error" not in err: status_msg = 'installation_error' elif "not_installed" not in res: version = res.split(":")[1] status_msg = "installed" if 'No package {} available'.format(os_pkg) in install_output: versions = datalab.fab.conn.sudo('yum --showduplicates list ' + name + ' | expand | grep ' + name + ' | awk \'{print $2}\'').stdout.replace('\r\n', '') if versions and versions != 'Error: No matching Packages to list': versions = versions.split(' ') status_msg = 'invalid_version' for n, i in enumerate(versions): if ':' in i: versions[n] = i.split(':')[1].split('-')[0] else: versions[n] = i.split('-')[0] else: versions = [] status_msg = 'invalid_name' status.append({"group": "os_pkg", "name": name, "version": version, "status": status_msg, "error_message": err, "add_pkgs": dep, "available_versions": versions}) datalab.fab.conn.sudo('rm /tmp/*{}*'.format(name)) return status except Exception as err: for os_pkg in requisites: name, vers = os_pkg status.append( {"group": "os_pkg", "name": name, "version": vers, "status": 'installation_error', "error_message": err}) print("Failed to install OS packages: {}".format(requisites)) return status
def configure_dataengine_service(instance, emr_conf): emr_conf['instance_ip'] = instance.get('PrivateIpAddress') try: logging.info('[CREATING DATALAB SSH USER ON DATAENGINE SERVICE]') params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format \ (emr_conf['instance_ip'], emr_conf['key_path'], emr_conf['initial_user'], emr_conf['os_user'], emr_conf['sudo_group']) try: subprocess.run("~/scripts/{}.py {}".format('create_ssh_user', params), shell=True, check=True) except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed to create DataLab ssh user.", str(err)) datalab.actions_lib.terminate_emr(emr_conf['cluster_id']) sys.exit(1) # configuring proxy on Data Engine service try: logging.info('[CONFIGURE PROXY ON DATAENGINE SERVICE]') additional_config = { "proxy_host": emr_conf['edge_instance_hostname'], "proxy_port": "3128" } params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \ .format(emr_conf['instance_ip'], emr_conf['cluster_name'], emr_conf['key_path'], json.dumps(additional_config), emr_conf['os_user']) try: subprocess.run("~/scripts/{}.py {}".format( 'common_configure_proxy', params), shell=True, check=True) except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed to configure proxy.", str(err)) datalab.actions_lib.terminate_emr(emr_conf['cluster_id']) sys.exit(1) try: logging.info('[CONFIGURE DATAENGINE SERVICE]') try: datalab.fab.configure_data_engine_service_pip( emr_conf['instance_ip'], emr_conf['os_user'], emr_conf['key_path'], True) global conn conn = datalab.fab.init_datalab_connection(emr_conf['instance_ip'], emr_conf['os_user'], emr_conf['key_path']) conn.sudo( '''bash -c 'echo "[main]" > /etc/yum/pluginconf.d/priorities.conf ; echo "enabled = 0" >> /etc/yum/pluginconf.d/priorities.conf' ''' ) manage_pkg('-y install', 'remote', 'R-devel') conn.close() except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed to configure dataengine service.", str(err)) datalab.actions_lib.terminate_emr(emr_conf['cluster_id']) sys.exit(1) try: logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]') cluster_master_instances = emr_conf['cluster_master_instances'] slaves = [] for idx, instance in enumerate(emr_conf['cluster_core_instances']): slave = { 'name': 'datanode{}'.format(idx + 1), 'ip': instance.get('PrivateIpAddress'), 'dns': instance.get('PrivateDnsName') } slaves.append(slave) additional_info = { "computational_name": emr_conf['computational_name'], "master_ip": cluster_master_instances[0].get('PrivateIpAddress'), "master_dns": cluster_master_instances[0].get('PrivateDnsName'), "slaves": slaves, "tensor": False } params = "--edge_hostname {} " \ "--keyfile {} " \ "--os_user {} " \ "--type {} " \ "--exploratory_name {} " \ "--additional_info '{}'"\ .format(emr_conf['edge_instance_hostname'], emr_conf['key_path'], emr_conf['os_user'], 'dataengine-service', emr_conf['exploratory_name'], json.dumps(additional_info)) try: subprocess.run("~/scripts/{}.py {}".format( 'common_configure_reverse_proxy', params), shell=True, check=True) except: datalab.fab.append_result("Failed edge reverse proxy template") raise Exception except Exception as err: datalab.fab.append_result("Failed edge reverse proxy template", str(err)) datalab.actions_lib.terminate_emr(emr_conf['cluster_id']) sys.exit(1) try: logging.info('[INSTALLING USERs KEY]') additional_config = { "user_keyname": emr_conf['user_keyname'], "user_keydir": os.environ['conf_key_dir'] } params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format( emr_conf['instance_ip'], emr_conf['key_path'], json.dumps(additional_config), emr_conf['os_user']) try: subprocess.run("~/scripts/{}.py {}".format('install_user_key', params), shell=True, check=True) except: traceback.print_exc() raise Exception except Exception as err: datalab.fab.append_result("Failed installing users key", str(err)) datalab.actions_lib.terminate_emr(emr_conf['cluster_id']) sys.exit(1)
def install_nodejs(os_user): if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/nodejs_ensured'.format(os_user)): datalab.fab.conn.sudo('curl -sL https://rpm.nodesource.com/setup_6.x | sudo -E bash -') manage_pkg('-y install', 'remote', 'nodejs') datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/nodejs_ensured'.format(os_user))
def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url, keycloak_realm_name, keycloak_client_id, keycloak_client_secret, user, hostname, step_cert_sans): try: if not os.path.exists('/tmp/nginx_installed'): manage_pkg('-y install', 'remote', 'wget') conn.sudo( 'wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm' ) try: conn.sudo('rpm -ivh epel-release-latest-7.noarch.rpm') except: print('Looks like EPEL is already installed.') manage_pkg( '-y install', 'remote', 'gcc gcc-c++ make zlib-devel pcre-devel openssl-devel git openldap-devel' ) if os.environ['conf_stepcerts_enabled'] == 'true': conn.sudo('mkdir -p /home/{0}/keys'.format(user)) conn.sudo( '''bash -c 'echo "{0}" | base64 --decode > /etc/ssl/certs/root_ca.crt' ''' .format(os.environ['conf_stepcerts_root_ca'])) fingerprint = conn.sudo( 'step certificate fingerprint /etc/ssl/certs/root_ca.crt' ).stdout.replace('\n', '') conn.sudo('step ca bootstrap --fingerprint {0} --ca-url "{1}"'. format(fingerprint, os.environ['conf_stepcerts_ca_url'])) conn.sudo( 'echo "{0}" > /home/{1}/keys/provisioner_password'.format( os.environ['conf_stepcerts_kid_password'], user)) sans = "--san localhost --san 127.0.0.1 {0}".format( step_cert_sans) cn = edge_ip conn.sudo( 'step ca token {3} --kid {0} --ca-url "{1}" --root /etc/ssl/certs/root_ca.crt ' '--password-file /home/{2}/keys/provisioner_password {4} --output-file /tmp/step_token' .format(os.environ['conf_stepcerts_kid'], os.environ['conf_stepcerts_ca_url'], user, cn, sans)) token = conn.sudo('cat /tmp/step_token').stdout.replace( '\n', '') conn.sudo( 'step ca certificate "{0}" /etc/ssl/certs/datalab.crt /etc/ssl/certs/datalab.key ' '--token "{1}" --kty=RSA --size 2048 --provisioner {2} '. format(cn, token, os.environ['conf_stepcerts_kid'])) conn.sudo('touch /var/log/renew_certificates.log') conn.put('/root/templates/manage_step_certs.sh', '/tmp/manage_step_certs.sh') conn.sudo( 'cp /tmp/manage_step_certs.sh /usr/local/bin/manage_step_certs.sh' ) conn.sudo('chmod +x /usr/local/bin/manage_step_certs.sh') conn.sudo( 'sed -i "s|STEP_ROOT_CERT_PATH|/etc/ssl/certs/root_ca.crt|g" ' '/usr/local/bin/manage_step_certs.sh') conn.sudo( 'sed -i "s|STEP_CERT_PATH|/etc/ssl/certs/datalab.crt|g" /usr/local/bin/manage_step_certs.sh' ) conn.sudo( 'sed -i "s|STEP_KEY_PATH|/etc/ssl/certs/datalab.key|g" /usr/local/bin/manage_step_certs.sh' ) conn.sudo( 'sed -i "s|STEP_CA_URL|{0}|g" /usr/local/bin/manage_step_certs.sh' .format(os.environ['conf_stepcerts_ca_url'])) conn.sudo( 'sed -i "s|RESOURCE_TYPE|edge|g" /usr/local/bin/manage_step_certs.sh' ) conn.sudo( 'sed -i "s|SANS|{0}|g" /usr/local/bin/manage_step_certs.sh' .format(sans)) conn.sudo( 'sed -i "s|CN|{0}|g" /usr/local/bin/manage_step_certs.sh'. format(cn)) conn.sudo( 'sed -i "s|KID|{0}|g" /usr/local/bin/manage_step_certs.sh'. format(os.environ['conf_stepcerts_kid'])) conn.sudo( 'sed -i "s|STEP_PROVISIONER_PASSWORD_PATH|/home/{0}/keys/provisioner_password|g" ' '/usr/local/bin/manage_step_certs.sh'.format(user)) conn.sudo( 'bash -c \'echo "0 * * * * root /usr/local/bin/manage_step_certs.sh >> ' '/var/log/renew_certificates.log 2>&1" >> /etc/crontab \'') conn.put('/root/templates/step-cert-manager.service', '/tmp/step-cert-manager.service') conn.sudo( '''bash -c 'cd -f /tmp/step-cert-manager.service /etc/systemd/system/step-cert-manager.service' ''' ) conn.sudo('systemctl daemon-reload') conn.sudo('systemctl enable step-cert-manager.service') else: if os.environ['conf_letsencrypt_enabled'] == 'true': print( 'Lets Encrypt certificates are not supported for redhat in DataLab. Using self signed certificates' ) conn.sudo( 'openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/datalab.key \ -out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN={}"' .format(hostname)) conn.sudo('mkdir -p /tmp/lua') conn.sudo('mkdir -p /tmp/src') conn.sudo( '''bash -c 'cd /tmp/src/ && wget http://nginx.org/download/nginx-{}.tar.gz' ''' .format(nginx_version)) conn.sudo( '''bash -c 'cd /tmp/src/ && tar -xzf nginx-{}.tar.gz' '''. format(nginx_version)) conn.sudo( '''bash -c 'cd /tmp/src/ && wget https://github.com/openresty/lua-nginx-module/archive/v0.10.15.tar.gz' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/ && tar -xzf v0.10.15.tar.gz' ''') conn.sudo( '''bash -c 'cd /tmp/src/ && wget https://github.com/simplresty/ngx_devel_kit/archive/v0.3.1.tar.gz' ''' ) conn.sudo('''bash -c 'cd /tmp/src/ && tar -xzf v0.3.1.tar.gz' ''') conn.sudo( '''bash -c 'cd /tmp/src/ && wget http://luajit.org/download/LuaJIT-2.0.5.tar.gz' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/ && tar -xzf LuaJIT-2.0.5.tar.gz' ''') conn.sudo( '''bash -c 'cd /tmp/src/ && wget http://keplerproject.github.io/luarocks/releases/luarocks-2.2.2.tar.gz' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/ && tar -xzf luarocks-2.2.2.tar.gz' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/ && ln -sf nginx-{} nginx' '''.format( nginx_version)) conn.sudo('''bash -c 'cd /tmp/src/LuaJIT-2.0.5/ && make' ''') conn.sudo( '''bash -c 'cd /tmp/src/LuaJIT-2.0.5/ && make install' ''') conn.sudo( 'export LUAJIT_LIB=/usr/local/lib/ LUAJIT_INC=/usr/local/include/luajit-2.0' ) conn.sudo( '''bash -l -c 'cd /tmp/src/nginx/ && ./configure --user=nginx --group=nginx --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx \ --conf-path=/etc/nginx/nginx.conf --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx \ --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log \ --with-http_gzip_static_module --with-http_stub_status_module --with-http_ssl_module --with-pcre \ --with-http_realip_module --with-file-aio --with-ipv6 --with-http_v2_module --with-ld-opt="-Wl,-rpath,$LUAJIT_LIB" \ --without-http_scgi_module --without-http_uwsgi_module --without-http_fastcgi_module --with-http_sub_module \ --add-dynamic-module=/tmp/src/ngx_devel_kit-0.3.1 --add-dynamic-module=/tmp/src/lua-nginx-module-0.10.15' ''' ) conn.sudo('''bash -c 'cd /tmp/src/nginx/ && make' ''') conn.sudo('''bash -c 'cd /tmp/src/nginx/ && make install' ''') conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && ./configure --with-lua-include=/usr/local/include/luajit-2.0' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && make build' ''') conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && make install' ''') conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-jwt' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-session' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-http' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-openidc' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install luacrypto' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-cjson' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-core' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install random' ''' ) conn.sudo( '''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-string' ''' ) conn.sudo('useradd -r nginx') conn.sudo('rm -f /etc/nginx/nginx.conf') conn.sudo('mkdir -p /opt/datalab/templates') conn.local( 'cd /root/templates; tar -zcvf /tmp/templates.tar.gz *') conn.put('/tmp/templates.tar.gz', '/tmp/templates.tar.gz') conn.sudo( 'tar -zxvf /tmp/templates.tar.gz -C /opt/datalab/templates/') conn.sudo( 'sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/conf.d/proxy.conf' .format(edge_ip)) conn.sudo( 'sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/datalab/templates/conf.d/proxy.conf' .format(keycloak_auth_server_url)) conn.sudo( 'sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/conf.d/proxy.conf' .format(keycloak_realm_name)) conn.sudo( 'sed -i \'s/KEYCLOAK_CLIENT_ID/{}/g\' /opt/datalab/templates/conf.d/proxy.conf' .format(keycloak_client_id)) conn.sudo( 'sed -i \'s/KEYCLOAK_CLIENT_SECRET/{}/g\' /opt/datalab/templates/conf.d/proxy.conf' .format(keycloak_client_secret)) conn.sudo('cp /opt/datalab/templates/nginx.conf /etc/nginx/') conn.sudo('mkdir /etc/nginx/conf.d') conn.sudo( 'cp /opt/datalab/templates/conf.d/proxy.conf /etc/nginx/conf.d/' ) conn.sudo('mkdir /etc/nginx/locations') conn.sudo( 'cp /opt/datalab/templates/nginx_redhat /etc/init.d/nginx') conn.sudo('chmod +x /etc/init.d/nginx') conn.sudo('chkconfig --add nginx') conn.sudo('chkconfig --level 345 nginx on') conn.sudo('setsebool -P httpd_can_network_connect 1') conn.sudo('service nginx start') conn.sudo('touch /tmp/nginx_installed') except Exception as err: print("Failed install nginx with ldap: " + str(err)) sys.exit(1)