def ensure_docker_daemon(datalab_path, os_user, region):
try:
if not exists(datalab.fab.conn,
datalab_path + 'tmp/docker_daemon_ensured'):
docker_version = os.environ['ssn_docker_version']
datalab.fab.conn.sudo(
'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -'
)
datalab.fab.conn.sudo(
'add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) \
stable"')
manage_pkg('update', 'remote', '')
datalab.fab.conn.sudo('apt-cache policy docker-ce')
manage_pkg(
'-y install', 'remote',
'docker-ce=5:{}~3-0~ubuntu-focal'.format(docker_version))
datalab.fab.conn.sudo('usermod -a -G docker ' + os_user)
datalab.fab.conn.sudo('update-rc.d docker defaults')
datalab.fab.conn.sudo('update-rc.d docker enable')
datalab.fab.conn.sudo('touch ' + datalab_path +
'tmp/docker_daemon_ensured')
return True
except:
traceback.print_exc()
print('Failed to ensure_docker_daemon')
return False
def install_build_dep():
try:
if not exists(
conn, '{}tmp/build_dep_ensured'.format(
os.environ['ssn_datalab_path'])):
maven_version = '3.5.4'
manage_pkg(
'-y install', 'remote',
'java-1.8.0-openjdk java-1.8.0-openjdk-devel git wget unzip')
conn.sudo(
'cd /opt/ && wget http://mirrors.sonic.net/apache/maven/maven-{0}/{1}/binaries/apache-maven-{1}-bin.zip'
.format(maven_version.split('.')[0], maven_version))
conn.sudo(
'''bash -c 'cd /opt/ &&unzip apache-maven-{}-bin.zip' '''.
format(maven_version))
conn.sudo(
'''bash -c 'cd /opt/ &&mv apache-maven-{} maven' '''.format(
maven_version))
conn.sudo(
'bash -c "curl --silent --location https://rpm.nodesource.com/setup_12.x | bash -"'
)
manage_pkg('-y install', 'remote', 'nodejs')
conn.sudo('npm config set unsafe-perm=true')
conn.sudo('touch {}tmp/build_dep_ensured'.format(
os.environ['ssn_datalab_path']))
except Exception as err:
traceback.print_exc()
print('Failed to install build dependencies for UI: ', str(err))
sys.exit(1)
def ensure_mongo():
try:
if not exists(
conn, '{}tmp/mongo_ensured'.format(
os.environ['ssn_datalab_path'])):
conn.sudo(
'echo -e "[mongodb-org-3.2]\nname=MongoDB Repository'
'\nbaseurl=https://repo.mongodb.org/yum/redhat/7/mongodb-org/3.2/x86_64/'
'\ngpgcheck=1'
'\nenabled=1'
'\ngpgkey=https://www.mongodb.org/static/pgp/server-3.2.asc" '
'> /etc/yum.repos.d/mongodb.repo')
manage_pkg('-y install', 'remote', 'mongodb-org')
conn.sudo('semanage port -a -t mongod_port_t -p tcp 27017')
conn.sudo('chkconfig mongod on')
conn.sudo(
'echo "d /var/run/mongodb 0755 mongod mongod" > /lib/tmpfiles.d/mongodb.conf'
)
conn.sudo('sudo systemd-tmpfiles --create mongodb.conf')
conn.sudo('systemctl start mongod.service')
conn.sudo('touch {}tmp/mongo_ensured'.format(
os.environ['ssn_datalab_path']))
except Exception as err:
traceback.print_exc()
print('Failed to install MongoDB: ', str(err))
sys.exit(1)
def install_rstudio(os_user, local_spark_path, rstudio_pass, rstudio_version):
if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/rstudio_ensured'):
try:
manage_pkg('-y install --nogpgcheck', 'remote', 'https://download2.rstudio.org/server/centos6/x86_64/rstudio-server-rhel-{}-x86_64.rpm'.format(rstudio_version))
datalab.fab.conn.sudo('mkdir -p /mnt/var')
datalab.fab.conn.sudo('chown {0}:{0} /mnt/var'.format(os_user))
datalab.fab.conn.sudo("sed -i '/Type=forking/a \Environment=USER=datalab-user' /lib/systemd/system/rstudio-server.service")
datalab.fab.conn.sudo(
"sed -i '/ExecStart/s|=/usr/lib/rstudio-server/bin/rserver|=/bin/bash -c \"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64; /usr/lib/rstudio-server/bin/rserver --auth-none 1|g' /lib/systemd/system/rstudio-server.service")
datalab.fab.conn.sudo("sed -i '/ExecStart/s|$|\"|g' /lib/systemd/system/rstudio-server.service")
datalab.fab.conn.sudo("systemctl daemon-reload")
datalab.fab.conn.sudo('touch /home/{}/.Renviron'.format(os_user))
datalab.fab.conn.sudo('chown {0}:{0} /home/{0}/.Renviron'.format(os_user))
datalab.fab.conn.sudo('''echo 'SPARK_HOME="{0}"' >> /home/{1}/.Renviron'''.format(local_spark_path, os_user))
datalab.fab.conn.sudo('touch /home/{}/.Rprofile'.format(os_user))
datalab.fab.conn.sudo('chown {0}:{0} /home/{0}/.Rprofile'.format(os_user))
datalab.fab.conn.sudo('''echo 'library(SparkR, lib.loc = c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib")))' >> /home/{}/.Rprofile'''.format(os_user))
http_proxy = datalab.fab.conn.run('''bash -l -c 'echo $http_proxy' ''').stdout.replace('\n','')
https_proxy = datalab.fab.conn.run('''bash -l -c 'echo $https_proxy' ''').stdout.replace('\n','')
datalab.fab.conn.sudo('''echo 'Sys.setenv(http_proxy = \"{}\")' >> /home/{}/.Rprofile'''.format(http_proxy, os_user))
datalab.fab.conn.sudo('''echo 'Sys.setenv(https_proxy = \"{}\")' >> /home/{}/.Rprofile'''.format(https_proxy, os_user))
datalab.fab.conn.sudo('rstudio-server start')
datalab.fab.conn.sudo('''bash -c 'echo "{0}:{1}" | chpasswd' '''.format(os_user, rstudio_pass))
datalab.fab.conn.sudo("sed -i '/exit 0/d' /etc/rc.local")
datalab.fab.conn.sudo('''bash -c "echo \'sed -i 's/^#SPARK_HOME/SPARK_HOME/' /home/{}/.Renviron\' >> /etc/rc.local"'''.format(os_user))
datalab.fab.conn.sudo("bash -c 'echo exit 0 >> /etc/rc.local'")
datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/rstudio_ensured'.format(os_user))
except:
sys.exit(1)
else:
try:
datalab.fab.conn.sudo('''bash -c 'echo "{0}:{1}" | chpasswd' '''.format(os_user, rstudio_pass))
except:
sys.exit(1)
def ensure_jre_jdk(os_user):
if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/jre_jdk_ensured'):
try:
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk')
manage_pkg('-y install', 'remote', 'java-1.8.0-openjdk-devel')
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/jre_jdk_ensured')
except:
sys.exit(1)
def ensure_sbt(os_user):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/sbt_ensured'.format(os_user)):
try:
datalab.fab.conn.sudo('curl https://bintray.com/sbt/rpm/rpm | sudo tee /etc/yum.repos.d/bintray-sbt-rpm.repo')
manage_pkg('-y install', 'remote', 'sbt')
datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/sbt_ensured'.format(os_user))
except:
sys.exit(1)
def ensure_nginx(datalab_path):
try:
if not exists(conn, '{}tmp/nginx_ensured'.format(datalab_path)):
manage_pkg('-y install', 'remote', 'nginx')
conn.sudo('systemctl restart nginx.service')
conn.sudo('chkconfig nginx on')
conn.sudo('touch {}tmp/nginx_ensured'.format(datalab_path))
except Exception as err:
traceback.print_exc()
print('Failed to ensure Nginx: ', str(err))
sys.exit(1)
def ensure_additional_python_libs(os_user):
if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/additional_python_libs_ensured'):
try:
manage_pkg('clean', 'remote', 'all')
manage_pkg('-y install', 'remote', 'zlib-devel libjpeg-turbo-devel --nogpgcheck')
if os.environ['application'] in ('jupyter', 'zeppelin'):
datalab.fab.conn.sudo('python3.5 -m pip install NumPy=={} SciPy pandas Sympy Pillow sklearn --no-cache-dir'.format(os.environ['notebook_numpy_version']))
if os.environ['application'] in ('tensor', 'deeplearning'):
datalab.fab.conn.sudo('python3.8 -m pip install opencv-python h5py --no-cache-dir')
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/additional_python_libs_ensured')
except:
sys.exit(1)
def ensure_python3_specific_version(python3_version, os_user):
if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/python3_specific_version_ensured'):
try:
manage_pkg('-y install', 'remote', 'yum-utils python34 openssl-devel')
manage_pkg('-y groupinstall', 'remote', 'development --nogpgcheck')
if len(python3_version) < 4:
python3_version = python3_version + ".0"
datalab.fab.conn.sudo('wget https://www.python.org/ftp/python/{0}/Python-{0}.tgz'.format(python3_version))
datalab.fab.conn.sudo('tar xzf Python-{0}.tgz; cd Python-{0}; ./configure --prefix=/usr/local; make altinstall'.format(python3_version))
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_specific_version_ensured')
except:
sys.exit(1)
def install_opencv(os_user):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/opencv_ensured'.format(os_user)):
manage_pkg('-y install', 'remote', 'cmake python34 python34-devel python34-pip gcc gcc-c++')
datalab.fab.conn.sudo('pip3.4 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
datalab.fab.conn.sudo('pip3.5 install numpy=={} --no-cache-dir'.format(os.environ['notebook_numpy_version']))
datalab.fab.conn.run('git clone https://github.com/opencv/opencv.git')
datalab.fab.conn.run('cd /home/{}/opencv/ && git checkout 3.2.0'.format(os_user))
datalab.fab.conn.run('cd /home/{}/opencv/ && mkdir release'.format(os_user))
datalab.fab.conn.run('cd /home/{}/opencv/release/ && cmake -DINSTALL_TESTS=OFF -D CUDA_GENERATION=Auto -D CMAKE_BUILD_TYPE=RELEASE -D CMAKE_INSTALL_PREFIX=$(python2 -c "import sys; print(sys.prefix)") -D PYTHON_EXECUTABLE=$(which python2) ..')
datalab.fab.conn.run('cd /home/{}/opencv/release/ && make -j$(nproc)')
datalab.fab.conn.sudo('''bash -c 'cd /home/{}/opencv/release/ && make install' ''')
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/opencv_ensured')
def ensure_nginx(datalab_path):
try:
if not exists(datalab.fab.conn, datalab_path + 'tmp/nginx_ensured'):
manage_pkg('-y install', 'remote', 'nginx')
datalab.fab.conn.sudo('service nginx restart')
datalab.fab.conn.sudo('update-rc.d nginx defaults')
datalab.fab.conn.sudo('update-rc.d nginx enable')
datalab.fab.conn.sudo('touch ' + datalab_path +
'tmp/nginx_ensured')
except Exception as err:
traceback.print_exc()
print('Failed to ensure Nginx: ', str(err))
sys.exit(1)
def ensure_supervisor():
try:
if not exists(datalab.fab.conn,
os.environ['ssn_datalab_path'] + 'tmp/superv_ensured'):
manage_pkg('-y install', 'remote', 'supervisor')
datalab.fab.conn.sudo('update-rc.d supervisor defaults')
datalab.fab.conn.sudo('update-rc.d supervisor enable')
datalab.fab.conn.sudo('touch ' + os.environ['ssn_datalab_path'] +
'tmp/superv_ensured')
except Exception as err:
traceback.print_exc()
print('Failed to install Supervisor: ', str(err))
sys.exit(1)
def enable_proxy(proxy_host, proxy_port):
try:
proxy_string = "http://%s:%s" % (proxy_host, proxy_port)
proxy_https_string = "https://%s:%s" % (proxy_host, proxy_port)
datalab.fab.conn.sudo('sed -i "/^export http_proxy/d" /etc/profile')
datalab.fab.conn.sudo('sed -i "/^export https_proxy/d" /etc/profile')
datalab.fab.conn.sudo('bash -c "echo export http_proxy=' + proxy_string + ' >> /etc/profile"')
datalab.fab.conn.sudo('bash -c "echo export https_proxy=' + proxy_string + ' >> /etc/profile"')
if exists(datalab.fab.conn, '/etc/yum.conf'):
datalab.fab.conn.sudo('sed -i "/^proxy=/d" /etc/yum.conf')
datalab.fab.conn.sudo('''bash -c "echo 'proxy={}' >> /etc/yum.conf" '''.format(proxy_string))
manage_pkg('clean all', 'remote', '')
except:
sys.exit(1)
def get_available_os_pkgs():
try:
os_pkgs = dict()
manage_pkg('update-minimal --security -y --skip-broken', 'remote', '')
#downgrade_python_version()
yum_names = datalab.fab.conn.sudo("yum list available | grep -v \"Loaded plugins:\|Available Packages\" | xargs -n3 | column -t | awk '{print $1}'").stdout.split('\n')
for pkg in yum_names:
if "." in pkg:
os_pkgs[pkg.split('.')[0]] = 'N/A'
elif pkg != '':
os_pkgs[pkg] = 'N/A'
return os_pkgs
except Exception as err:
append_result("Failed to get available os packages.", str(err))
sys.exit(1)
def ensure_supervisor():
try:
if not exists(
conn, '{}tmp/superv_ensured'.format(
os.environ['ssn_datalab_path'])):
manage_pkg('-y install', 'remote', 'supervisor')
# conn.sudo('pip install supervisor')
conn.sudo('chkconfig supervisord on')
conn.sudo('systemctl start supervisord')
conn.sudo('touch {}tmp/superv_ensured'.format(
os.environ['ssn_datalab_path']))
except Exception as err:
traceback.print_exc()
print('Failed to install supervisor: ', str(err))
sys.exit(1)
def install_tensor(os_user, cuda_version, cuda_file_name,
cudnn_version, cudnn_file_name, tensorflow_version,
templates_dir, nvidia_version):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/tensor_ensured'.format(os_user)):
try:
# install nvidia drivers
datalab.fab.conn.sudo('''bash -c 'echo "blacklist nouveau" >> /etc/modprobe.d/blacklist-nouveau.conf' ''')
datalab.fab.conn.sudo('''bash -c 'echo "options nouveau modeset=0" >> /etc/modprobe.d/blacklist-nouveau.conf' ''')
datalab.fab.conn.sudo('dracut --force')
datalab.fab.conn.sudo('reboot', warn=True)
time.sleep(150)
manage_pkg('-y install', 'remote', 'libglvnd-opengl libglvnd-devel dkms gcc kernel-devel-$(uname -r) kernel-headers-$(uname -r)')
datalab.fab.conn.sudo('wget http://us.download.nvidia.com/XFree86/Linux-x86_64/{0}/NVIDIA-Linux-x86_64-{0}.run -O /home/{1}/NVIDIA-Linux-x86_64-{0}.run'.format(nvidia_version, os_user))
datalab.fab.conn.sudo('/bin/bash /home/{0}/NVIDIA-Linux-x86_64-{1}.run -s --dkms'.format(os_user, nvidia_version))
datalab.fab.conn.sudo('rm -f /home/{0}/NVIDIA-Linux-x86_64-{1}.run'.format(os_user, nvidia_version))
# install cuda
datalab.fab.conn.sudo('python3.5 -m pip install --upgrade pip=={0} wheel numpy=={1} --no-cache-dir'. format(os.environ['conf_pip_version'], os.environ['notebook_numpy_version']))
datalab.fab.conn.sudo('wget -P /opt https://developer.nvidia.com/compute/cuda/{0}/prod/local_installers/{1}'.format(cuda_version, cuda_file_name))
datalab.fab.conn.sudo('sh /opt/{} --silent --toolkit'.format(cuda_file_name))
datalab.fab.conn.sudo('mv /usr/local/cuda-{} /opt/'.format(cuda_version[:-2]))
datalab.fab.conn.sudo('ln -s /opt/cuda-{0} /usr/local/cuda-{0}'.format(cuda_version[:-2]))
datalab.fab.conn.sudo('rm -f /opt/{}'.format(cuda_file_name))
# install cuDNN
datalab.fab.conn.run('wget https://developer.download.nvidia.com/compute/redist/cudnn/v{0}/{1} -O /tmp/{1}'.format(cudnn_version, cudnn_file_name))
datalab.fab.conn.run('tar xvzf /tmp/{} -C /tmp'.format(cudnn_file_name))
datalab.fab.conn.sudo('mkdir -p /opt/cudnn/include')
datalab.fab.conn.sudo('mkdir -p /opt/cudnn/lib64')
datalab.fab.conn.sudo('mv /tmp/cuda/include/cudnn.h /opt/cudnn/include')
datalab.fab.conn.sudo('mv /tmp/cuda/lib64/libcudnn* /opt/cudnn/lib64')
datalab.fab.conn.sudo('chmod a+r /opt/cudnn/include/cudnn.h /opt/cudnn/lib64/libcudnn*')
datalab.fab.conn.run('''bash -l -c 'echo "export LD_LIBRARY_PATH=\"$LD_LIBRARY_PATH:/opt/cudnn/lib64:/usr/local/cuda/lib64\"" >> ~/.bashrc' ''')
# install TensorFlow and run TensorBoard
datalab.fab.conn.sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp27-none-linux_x86_64.whl'.format(tensorflow_version))
datalab.fab.conn.sudo('wget https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl'.format(tensorflow_version))
datalab.fab.conn.sudo('python3.8 -m pip install --upgrade tensorflow_gpu-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(tensorflow_version))
datalab.fab.conn.sudo('rm -rf /home/{}/tensorflow_gpu-*'.format(os_user))
datalab.fab.conn.sudo('mkdir /var/log/tensorboard; chown {0}:{0} -R /var/log/tensorboard'.format(os_user))
datalab.fab.conn.put('{}tensorboard.service'.format(templates_dir), '/tmp/tensorboard.service')
datalab.fab.conn.sudo("sed -i 's|OS_USR|{}|' /tmp/tensorboard.service".format(os_user))
datalab.fab.conn.sudo("chmod 644 /tmp/tensorboard.service")
datalab.fab.conn.sudo('\cp /tmp/tensorboard.service /etc/systemd/system/')
datalab.fab.conn.sudo("systemctl daemon-reload")
datalab.fab.conn.sudo("systemctl enable tensorboard")
datalab.fab.conn.sudo("systemctl start tensorboard")
datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/tensor_ensured'.format(os_user))
except:
sys.exit(1)
def ensure_jenkins(datalab_path):
try:
if not exists(datalab.fab.conn, datalab_path + 'tmp/jenkins_ensured'):
datalab.fab.conn.sudo(
'wget -q -O - https://pkg.jenkins.io/debian/jenkins-ci.org.key | sudo apt-key add -'
)
datalab.fab.conn.sudo(
'echo deb http://pkg.jenkins.io/debian-stable binary/ > /etc/apt/sources.list.d/jenkins.list'
)
manage_pkg('-y update', 'remote', '')
manage_pkg('-y install', 'remote', 'jenkins')
datalab.fab.conn.sudo('touch ' + datalab_path +
'tmp/jenkins_ensured')
except Exception as err:
traceback.print_exc()
print('Failed to ensure Jenkins: ', str(err))
sys.exit(1)
def ensure_python3_libraries(os_user):
if not exists(datalab.fab.conn,'/home/' + os_user + '/.ensure_dir/python3_libraries_ensured'):
try:
manage_pkg('-y install', 'remote', 'https://centos7.iuscommunity.org/ius-release.rpm')
manage_pkg('-y install', 'remote', 'python35u python35u-pip python35u-devel')
datalab.fab.conn.sudo('python3.5 -m pip install -U pip=={} setuptools --no-cache-dir'.format(os.environ['conf_pip_version']))
datalab.fab.conn.sudo('python3.5 -m pip install boto3 --no-cache-dir')
datalab.fab.conn.sudo('python3.5 -m pip install fabvenv fabric-virtualenv future patchwork --no-cache-dir')
try:
datalab.fab.conn.sudo('python3.5 -m pip install tornado=={0} ipython==7.9.0 ipykernel=={1} --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
except:
datalab.fab.conn.sudo('python3.5 -m pip install tornado=={0} ipython==5.0.0 ipykernel=={1} --no-cache-dir' \
.format(os.environ['notebook_tornado_version'], os.environ['notebook_ipykernel_version']))
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/python3_libraries_ensured')
except:
sys.exit(1)
def ensure_jenkins(datalab_path):
try:
if not exists(conn, '{}tmp/jenkins_ensured'.format(datalab_path)):
conn.sudo(
'wget -O /etc/yum.repos.d/jenkins.repo https://pkg.jenkins.io/redhat-stable/jenkins.repo'
)
try:
conn.sudo(
'rpm --import https://pkg.jenkins.io/redhat-stable/jenkins.io.key'
)
except:
pass
manage_pkg('-y install', 'remote', 'jenkins')
manage_pkg('-y install', 'remote', 'policycoreutils-python')
conn.sudo('touch {}tmp/jenkins_ensured'.format(datalab_path))
except Exception as err:
traceback.print_exc()
print('Failed to ensure Jenkins: ', str(err))
sys.exit(1)
def install_caffe2(os_user, caffe2_version, cmake_version):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/caffe2_ensured'.format(os_user)):
env.shell = "/bin/bash -l -c -i"
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install --nogpgcheck', 'remote', 'automake cmake3 gcc gcc-c++ kernel-devel leveldb-devel lmdb-devel libtool protobuf-devel graphviz')
datalab.fab.conn.sudo('pip3.5 install flask graphviz hypothesis jupyter matplotlib=={} numpy=={} protobuf pydot python-nvd3 pyyaml '
'requests scikit-image scipy setuptools tornado future --no-cache-dir'.format(os.environ['notebook_matplotlib_version'], os.environ['notebook_numpy_version']))
datalab.fab.conn.sudo('cp /opt/cudnn/include/* /opt/cuda-8.0/include/')
datalab.fab.conn.sudo('cp /opt/cudnn/lib64/* /opt/cuda-8.0/lib64/')
datalab.fab.conn.sudo('wget https://cmake.org/files/v{2}/cmake-{1}.tar.gz -O /home/{0}/cmake-{1}.tar.gz'.format(
os_user, cmake_version, cmake_version.split('.')[0] + "." + cmake_version.split('.')[1]))
datalab.fab.conn.sudo('tar -zxvf cmake-{}.tar.gz'.format(cmake_version))
datalab.fab.conn.sudo('''bash -c 'cd /home/{}/cmake-{}/ && ./bootstrap --prefix=/usr/local && make && make install' '''.format(os_user, cmake_version))
datalab.fab.conn.sudo('ln -s /usr/local/bin/cmake /bin/cmake{}'.format(cmake_version))
datalab.fab.conn.sudo('git clone https://github.com/pytorch/pytorch.git')
datalab.fab.conn.sudo('''bash -c 'cd /home/{}/pytorch/ && git submodule update --init' '''.format(os_user))
datalab.fab.conn.sudo('''bash -c 'cd /home/{}/pytorch/ && git checkout v{}' '''.format(os_user, caffe2_version), warn=True)
datalab.fab.conn.sudo('''bash -c 'cd /home/{}/pytorch/ && git submodule update --recursive' '''.format(os_user), warn=True)
datalab.fab.conn.sudo('''bash -c 'cd /home/{}/pytorch/ && mkdir build && cd build && cmake{} .. && make "-j$(nproc)" install' '''.format(os_user, cmake_version))
datalab.fab.conn.sudo('touch /home/' + os_user + '/.ensure_dir/caffe2_ensured')
def ensure_mongo():
try:
if not exists(datalab.fab.conn,
os.environ['ssn_datalab_path'] + 'tmp/mongo_ensured'):
datalab.fab.conn.sudo(
'wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | sudo apt-key add -'
)
os_version = datalab.fab.conn.sudo(
'lsb_release -cs').stdout.replace('\n', '')
datalab.fab.conn.sudo(
'echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu {}/mongodb-org/4.4 '
'multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-4.4.list'
.format(os_version))
manage_pkg('update', 'remote', '')
manage_pkg('-y install', 'remote', 'mongodb-org')
datalab.fab.conn.sudo('systemctl enable mongod.service')
datalab.fab.conn.sudo('touch ' + os.environ['ssn_datalab_path'] +
'tmp/mongo_ensured')
except Exception as err:
traceback.print_exc()
print('Failed to install MongoDB: ', str(err))
sys.exit(1)
def install_cntk(os_user, cntk_version):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/cntk_ensured'.format(os_user)):
datalab.fab.conn.sudo('echo "exclude=*.i386 *.i686" >> /etc/yum.conf')
manage_pkg('clean', 'remote', 'all')
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install --nogpgcheck', 'remote', 'openmpi openmpi-devel')
datalab.fab.conn.sudo('pip3.5 install https://cntk.ai/PythonWheel/GPU/cntk-{}-cp35-cp35m-linux_x86_64.whl --no-cache-dir'.format(cntk_version))
datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/cntk_ensured'.format(os_user))
def ensure_r(os_user, r_libs):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/r_ensured'.format(os_user)):
try:
r_repository = 'https://cloud.r-project.org'
manage_pkg('-y install', 'remote', 'cmake')
manage_pkg('-y install', 'remote', 'libcur*')
datalab.fab.conn.sudo('echo -e "[base]\nname=CentOS-7-Base\nbaseurl=http://buildlogs.centos.org/centos/7/os/x86_64-20140704-1/\ngpgcheck=1\ngpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7\npriority=1\nexclude=php mysql" >> /etc/yum.repos.d/CentOS-base.repo')
manage_pkg('-y install', 'remote', 'R R-core R-core-devel R-devel --nogpgcheck')
datalab.fab.conn.sudo('R CMD javareconf')
datalab.fab.conn.sudo('''bash -c 'cd /root; git clone https://github.com/zeromq/zeromq4-x.git; cd zeromq4-x/; mkdir build; cd build; cmake ..; make install; ldconfig' ''')
for i in r_libs:
datalab.fab.conn.sudo('R -e "install.packages(\'{}\',repos=\'{}\')"'.format(i, r_repository))
datalab.fab.conn.sudo('R -e "library(\'devtools\');install.packages(repos=\'{}\',c(\'rzmq\',\'repr\',\'digest\',\'stringr\',\'RJSONIO\',\'functional\',\'plyr\'))"'.format(r_repository))
datalab.fab.conn.sudo('R -e "library(\'devtools\');install_github(\'IRkernel/repr\');install_github(\'IRkernel/IRdisplay\');install_github(\'IRkernel/IRkernel\');"')
datalab.fab.conn.sudo('R -e "library(\'devtools\');install_version(\'keras\', version = \'{}\', repos = \'{}\');"'.format(os.environ['notebook_keras_version'],r_repository))
datalab.fab.conn.sudo('R -e "install.packages(\'RJDBC\',repos=\'{}\',dep=TRUE)"'.format(r_repository))
datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/r_ensured'.format(os_user))
except:
sys.exit(1)
def ensure_docker_daemon(datalab_path, os_user, region):
try:
if not exists(conn,
'{}tmp/docker_daemon_ensured'.format(datalab_path)):
docker_version = os.environ['ssn_docker_version']
if region == 'cn-north-1':
mirror = 'mirror.lzu.edu.cn'
else:
mirror = 'mirror.centos.org'
conn.sudo(
'''bash -c 'cd /etc/yum.repos.d/ && echo "[centosrepo]" > centos.repo' '''
)
conn.sudo(
'''bash -c 'cd /etc/yum.repos.d/ && echo "name=Centos 7 Repository" >> centos.repo' '''
)
conn.sudo(
'''bash -c 'cd /etc/yum.repos.d/ && echo "baseurl=http://{}/centos/7/extras/x86_64/" >> centos.repo' '''
.format(mirror))
conn.sudo(
'''bash -c 'cd /etc/yum.repos.d/ && echo "enabled=1" >> centos.repo' '''
)
conn.sudo(
'''bash -c 'cd /etc/yum.repos.d/ && echo "gpgcheck=1" >> centos.repo' '''
)
conn.sudo(
'''bash -c 'cd /etc/yum.repos.d/ && echo "gpgkey=http://{}/centos/7/os/x86_64/RPM-GPG-KEY-CentOS-7" >> centos.repo' '''
.format(mirror))
conn.sudo(
'yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo'
)
manage_pkg('update-minimal --security -y', 'remote', '')
manage_pkg('-y install', 'remote', 'container-selinux')
manage_pkg('-y install', 'remote',
'docker-ce-{}.ce'.format(docker_version))
conn.sudo('usermod -aG docker {}'.format(os_user))
conn.sudo('systemctl enable docker.service')
conn.sudo('systemctl start docker')
conn.sudo('touch {}tmp/docker_daemon_ensured'.format(datalab_path))
return True
except:
return False
def remove_os_pkg(pkgs):
try:
manage_pkg('remove -y', 'remote', '{}'.format(' '.join(pkgs)))
except:
sys.exit(1)
def install_os_pkg(requisites):
status = list()
error_parser = "Could not|No matching|Error:|failed|Requires:|Errno"
new_pkgs_parser = "Dependency Installed:"
try:
print("Updating repositories and installing requested tools: {}".format(requisites))
manage_pkg('update-minimal --security -y --skip-broken', 'remote', '')
#datalab.fab.conn.sudo('export LC_ALL=C')
for os_pkg in requisites:
name, vers = os_pkg
if vers != '' and vers !='N/A':
version = vers
os_pkg = "{}-{}".format(name, vers)
else:
version = 'N/A'
os_pkg = name
manage_pkg('-y install', 'remote', '{0} --nogpgcheck 2>&1 | tee /tmp/os_install_{2}.tmp; if ! grep -w -E "({1})" '
'/tmp/os_install_{2}.tmp > /tmp/os_install_{2}.log; then echo "no_error" > /tmp/os_install_{2}.log;fi'.format(os_pkg, error_parser, name))
install_output = datalab.fab.conn.sudo('cat /tmp/os_install_{}.tmp'.format(name)).stdout
err = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout.replace('"', "'")
datalab.fab.conn.sudo('cat /tmp/os_install_{0}.tmp | if ! grep -w -E -A 30 "({1})" /tmp/os_install_{0}.tmp > '
'/tmp/os_install_{0}.log; then echo "no_dependencies" > /tmp/os_install_{0}.log;fi'.format(name, new_pkgs_parser))
dep = datalab.fab.conn.sudo('cat /tmp/os_install_{}.log'.format(name)).stdout
if 'no_dependencies' in dep:
dep = []
else:
dep = dep[len(new_pkgs_parser): dep.find("Complete!") - 1].replace(' ', '').strip().split('\r\n')
for n, i in enumerate(dep):
i = i.split('.')[0]
datalab.fab.conn.sudo('yum info {0} 2>&1 | if ! grep Version > /tmp/os_install_{0}.log; then echo "" > /tmp/os_install_{0}.log;fi'.format(i))
dep[n] =sudo('cat /tmp/os_install_{}.log'.format(i)).replace('Version : ', '{} v.'.format(i))
dep = [i for i in dep if i]
versions = []
datalab.fab.conn.sudo(
'yum list installed | if ! grep "{0}\." > /tmp/os_install_{0}.list; then echo "not_installed" > /tmp/os_install_{0}.list;fi'.format(
name))
res = datalab.fab.conn.sudo('cat /tmp/os_install_{}.list '.format(name) + '| awk \'{print $1":"$2}\'').stdout.replace('\n', '')
#res = datalab.fab.conn.sudo('python3 -c "import os,sys,yum; yb = yum.YumBase(); pl = yb.doPackageLists(); print [pkg.vr for pkg in pl.installed if pkg.name == \'{0}\']"'.format(name)).stdout.split('\r\n')[1]
if "no_error" not in err:
status_msg = 'installation_error'
elif "not_installed" not in res:
version = res.split(":")[1]
status_msg = "installed"
if 'No package {} available'.format(os_pkg) in install_output:
versions = datalab.fab.conn.sudo('yum --showduplicates list ' + name + ' | expand | grep ' + name + ' | awk \'{print $2}\'').stdout.replace('\r\n', '')
if versions and versions != 'Error: No matching Packages to list':
versions = versions.split(' ')
status_msg = 'invalid_version'
for n, i in enumerate(versions):
if ':' in i:
versions[n] = i.split(':')[1].split('-')[0]
else:
versions[n] = i.split('-')[0]
else:
versions = []
status_msg = 'invalid_name'
status.append({"group": "os_pkg", "name": name, "version": version, "status": status_msg,
"error_message": err, "add_pkgs": dep, "available_versions": versions})
datalab.fab.conn.sudo('rm /tmp/*{}*'.format(name))
return status
except Exception as err:
for os_pkg in requisites:
name, vers = os_pkg
status.append(
{"group": "os_pkg", "name": name, "version": vers, "status": 'installation_error', "error_message": err})
print("Failed to install OS packages: {}".format(requisites))
return status
def configure_dataengine_service(instance, emr_conf):
emr_conf['instance_ip'] = instance.get('PrivateIpAddress')
try:
logging.info('[CREATING DATALAB SSH USER ON DATAENGINE SERVICE]')
params = "--hostname {} --keyfile {} --initial_user {} --os_user {} --sudo_group {}".format \
(emr_conf['instance_ip'], emr_conf['key_path'], emr_conf['initial_user'],
emr_conf['os_user'], emr_conf['sudo_group'])
try:
subprocess.run("~/scripts/{}.py {}".format('create_ssh_user',
params),
shell=True,
check=True)
except:
traceback.print_exc()
raise Exception
except Exception as err:
datalab.fab.append_result("Failed to create DataLab ssh user.",
str(err))
datalab.actions_lib.terminate_emr(emr_conf['cluster_id'])
sys.exit(1)
# configuring proxy on Data Engine service
try:
logging.info('[CONFIGURE PROXY ON DATAENGINE SERVICE]')
additional_config = {
"proxy_host": emr_conf['edge_instance_hostname'],
"proxy_port": "3128"
}
params = "--hostname {} --instance_name {} --keyfile {} --additional_config '{}' --os_user {}" \
.format(emr_conf['instance_ip'], emr_conf['cluster_name'], emr_conf['key_path'],
json.dumps(additional_config), emr_conf['os_user'])
try:
subprocess.run("~/scripts/{}.py {}".format(
'common_configure_proxy', params),
shell=True,
check=True)
except:
traceback.print_exc()
raise Exception
except Exception as err:
datalab.fab.append_result("Failed to configure proxy.", str(err))
datalab.actions_lib.terminate_emr(emr_conf['cluster_id'])
sys.exit(1)
try:
logging.info('[CONFIGURE DATAENGINE SERVICE]')
try:
datalab.fab.configure_data_engine_service_pip(
emr_conf['instance_ip'], emr_conf['os_user'],
emr_conf['key_path'], True)
global conn
conn = datalab.fab.init_datalab_connection(emr_conf['instance_ip'],
emr_conf['os_user'],
emr_conf['key_path'])
conn.sudo(
'''bash -c 'echo "[main]" > /etc/yum/pluginconf.d/priorities.conf ; echo "enabled = 0" >> /etc/yum/pluginconf.d/priorities.conf' '''
)
manage_pkg('-y install', 'remote', 'R-devel')
conn.close()
except:
traceback.print_exc()
raise Exception
except Exception as err:
datalab.fab.append_result("Failed to configure dataengine service.",
str(err))
datalab.actions_lib.terminate_emr(emr_conf['cluster_id'])
sys.exit(1)
try:
logging.info('[SETUP EDGE REVERSE PROXY TEMPLATE]')
cluster_master_instances = emr_conf['cluster_master_instances']
slaves = []
for idx, instance in enumerate(emr_conf['cluster_core_instances']):
slave = {
'name': 'datanode{}'.format(idx + 1),
'ip': instance.get('PrivateIpAddress'),
'dns': instance.get('PrivateDnsName')
}
slaves.append(slave)
additional_info = {
"computational_name": emr_conf['computational_name'],
"master_ip": cluster_master_instances[0].get('PrivateIpAddress'),
"master_dns": cluster_master_instances[0].get('PrivateDnsName'),
"slaves": slaves,
"tensor": False
}
params = "--edge_hostname {} " \
"--keyfile {} " \
"--os_user {} " \
"--type {} " \
"--exploratory_name {} " \
"--additional_info '{}'"\
.format(emr_conf['edge_instance_hostname'],
emr_conf['key_path'],
emr_conf['os_user'],
'dataengine-service',
emr_conf['exploratory_name'],
json.dumps(additional_info))
try:
subprocess.run("~/scripts/{}.py {}".format(
'common_configure_reverse_proxy', params),
shell=True,
check=True)
except:
datalab.fab.append_result("Failed edge reverse proxy template")
raise Exception
except Exception as err:
datalab.fab.append_result("Failed edge reverse proxy template",
str(err))
datalab.actions_lib.terminate_emr(emr_conf['cluster_id'])
sys.exit(1)
try:
logging.info('[INSTALLING USERs KEY]')
additional_config = {
"user_keyname": emr_conf['user_keyname'],
"user_keydir": os.environ['conf_key_dir']
}
params = "--hostname {} --keyfile {} --additional_config '{}' --user {}".format(
emr_conf['instance_ip'], emr_conf['key_path'],
json.dumps(additional_config), emr_conf['os_user'])
try:
subprocess.run("~/scripts/{}.py {}".format('install_user_key',
params),
shell=True,
check=True)
except:
traceback.print_exc()
raise Exception
except Exception as err:
datalab.fab.append_result("Failed installing users key", str(err))
datalab.actions_lib.terminate_emr(emr_conf['cluster_id'])
sys.exit(1)
def install_nodejs(os_user):
if not exists(datalab.fab.conn,'/home/{}/.ensure_dir/nodejs_ensured'.format(os_user)):
datalab.fab.conn.sudo('curl -sL https://rpm.nodesource.com/setup_6.x | sudo -E bash -')
manage_pkg('-y install', 'remote', 'nodejs')
datalab.fab.conn.sudo('touch /home/{}/.ensure_dir/nodejs_ensured'.format(os_user))
def install_nginx_lua(edge_ip, nginx_version, keycloak_auth_server_url,
keycloak_realm_name, keycloak_client_id,
keycloak_client_secret, user, hostname, step_cert_sans):
try:
if not os.path.exists('/tmp/nginx_installed'):
manage_pkg('-y install', 'remote', 'wget')
conn.sudo(
'wget https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm'
)
try:
conn.sudo('rpm -ivh epel-release-latest-7.noarch.rpm')
except:
print('Looks like EPEL is already installed.')
manage_pkg(
'-y install', 'remote',
'gcc gcc-c++ make zlib-devel pcre-devel openssl-devel git openldap-devel'
)
if os.environ['conf_stepcerts_enabled'] == 'true':
conn.sudo('mkdir -p /home/{0}/keys'.format(user))
conn.sudo(
'''bash -c 'echo "{0}" | base64 --decode > /etc/ssl/certs/root_ca.crt' '''
.format(os.environ['conf_stepcerts_root_ca']))
fingerprint = conn.sudo(
'step certificate fingerprint /etc/ssl/certs/root_ca.crt'
).stdout.replace('\n', '')
conn.sudo('step ca bootstrap --fingerprint {0} --ca-url "{1}"'.
format(fingerprint,
os.environ['conf_stepcerts_ca_url']))
conn.sudo(
'echo "{0}" > /home/{1}/keys/provisioner_password'.format(
os.environ['conf_stepcerts_kid_password'], user))
sans = "--san localhost --san 127.0.0.1 {0}".format(
step_cert_sans)
cn = edge_ip
conn.sudo(
'step ca token {3} --kid {0} --ca-url "{1}" --root /etc/ssl/certs/root_ca.crt '
'--password-file /home/{2}/keys/provisioner_password {4} --output-file /tmp/step_token'
.format(os.environ['conf_stepcerts_kid'],
os.environ['conf_stepcerts_ca_url'], user, cn,
sans))
token = conn.sudo('cat /tmp/step_token').stdout.replace(
'\n', '')
conn.sudo(
'step ca certificate "{0}" /etc/ssl/certs/datalab.crt /etc/ssl/certs/datalab.key '
'--token "{1}" --kty=RSA --size 2048 --provisioner {2} '.
format(cn, token, os.environ['conf_stepcerts_kid']))
conn.sudo('touch /var/log/renew_certificates.log')
conn.put('/root/templates/manage_step_certs.sh',
'/tmp/manage_step_certs.sh')
conn.sudo(
'cp /tmp/manage_step_certs.sh /usr/local/bin/manage_step_certs.sh'
)
conn.sudo('chmod +x /usr/local/bin/manage_step_certs.sh')
conn.sudo(
'sed -i "s|STEP_ROOT_CERT_PATH|/etc/ssl/certs/root_ca.crt|g" '
'/usr/local/bin/manage_step_certs.sh')
conn.sudo(
'sed -i "s|STEP_CERT_PATH|/etc/ssl/certs/datalab.crt|g" /usr/local/bin/manage_step_certs.sh'
)
conn.sudo(
'sed -i "s|STEP_KEY_PATH|/etc/ssl/certs/datalab.key|g" /usr/local/bin/manage_step_certs.sh'
)
conn.sudo(
'sed -i "s|STEP_CA_URL|{0}|g" /usr/local/bin/manage_step_certs.sh'
.format(os.environ['conf_stepcerts_ca_url']))
conn.sudo(
'sed -i "s|RESOURCE_TYPE|edge|g" /usr/local/bin/manage_step_certs.sh'
)
conn.sudo(
'sed -i "s|SANS|{0}|g" /usr/local/bin/manage_step_certs.sh'
.format(sans))
conn.sudo(
'sed -i "s|CN|{0}|g" /usr/local/bin/manage_step_certs.sh'.
format(cn))
conn.sudo(
'sed -i "s|KID|{0}|g" /usr/local/bin/manage_step_certs.sh'.
format(os.environ['conf_stepcerts_kid']))
conn.sudo(
'sed -i "s|STEP_PROVISIONER_PASSWORD_PATH|/home/{0}/keys/provisioner_password|g" '
'/usr/local/bin/manage_step_certs.sh'.format(user))
conn.sudo(
'bash -c \'echo "0 * * * * root /usr/local/bin/manage_step_certs.sh >> '
'/var/log/renew_certificates.log 2>&1" >> /etc/crontab \'')
conn.put('/root/templates/step-cert-manager.service',
'/tmp/step-cert-manager.service')
conn.sudo(
'''bash -c 'cd -f /tmp/step-cert-manager.service /etc/systemd/system/step-cert-manager.service' '''
)
conn.sudo('systemctl daemon-reload')
conn.sudo('systemctl enable step-cert-manager.service')
else:
if os.environ['conf_letsencrypt_enabled'] == 'true':
print(
'Lets Encrypt certificates are not supported for redhat in DataLab. Using self signed certificates'
)
conn.sudo(
'openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout /etc/ssl/certs/datalab.key \
-out /etc/ssl/certs/datalab.crt -subj "/C=US/ST=US/L=US/O=datalab/CN={}"'
.format(hostname))
conn.sudo('mkdir -p /tmp/lua')
conn.sudo('mkdir -p /tmp/src')
conn.sudo(
'''bash -c 'cd /tmp/src/ && wget http://nginx.org/download/nginx-{}.tar.gz' '''
.format(nginx_version))
conn.sudo(
'''bash -c 'cd /tmp/src/ && tar -xzf nginx-{}.tar.gz' '''.
format(nginx_version))
conn.sudo(
'''bash -c 'cd /tmp/src/ && wget https://github.com/openresty/lua-nginx-module/archive/v0.10.15.tar.gz' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/ && tar -xzf v0.10.15.tar.gz' ''')
conn.sudo(
'''bash -c 'cd /tmp/src/ && wget https://github.com/simplresty/ngx_devel_kit/archive/v0.3.1.tar.gz' '''
)
conn.sudo('''bash -c 'cd /tmp/src/ && tar -xzf v0.3.1.tar.gz' ''')
conn.sudo(
'''bash -c 'cd /tmp/src/ && wget http://luajit.org/download/LuaJIT-2.0.5.tar.gz' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/ && tar -xzf LuaJIT-2.0.5.tar.gz' ''')
conn.sudo(
'''bash -c 'cd /tmp/src/ && wget http://keplerproject.github.io/luarocks/releases/luarocks-2.2.2.tar.gz' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/ && tar -xzf luarocks-2.2.2.tar.gz' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/ && ln -sf nginx-{} nginx' '''.format(
nginx_version))
conn.sudo('''bash -c 'cd /tmp/src/LuaJIT-2.0.5/ && make' ''')
conn.sudo(
'''bash -c 'cd /tmp/src/LuaJIT-2.0.5/ && make install' ''')
conn.sudo(
'export LUAJIT_LIB=/usr/local/lib/ LUAJIT_INC=/usr/local/include/luajit-2.0'
)
conn.sudo(
'''bash -l -c 'cd /tmp/src/nginx/ && ./configure --user=nginx --group=nginx --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf --pid-path=/run/nginx.pid --lock-path=/run/lock/subsys/nginx \
--error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log \
--with-http_gzip_static_module --with-http_stub_status_module --with-http_ssl_module --with-pcre \
--with-http_realip_module --with-file-aio --with-ipv6 --with-http_v2_module --with-ld-opt="-Wl,-rpath,$LUAJIT_LIB" \
--without-http_scgi_module --without-http_uwsgi_module --without-http_fastcgi_module --with-http_sub_module \
--add-dynamic-module=/tmp/src/ngx_devel_kit-0.3.1 --add-dynamic-module=/tmp/src/lua-nginx-module-0.10.15' '''
)
conn.sudo('''bash -c 'cd /tmp/src/nginx/ && make' ''')
conn.sudo('''bash -c 'cd /tmp/src/nginx/ && make install' ''')
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && ./configure --with-lua-include=/usr/local/include/luajit-2.0' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && make build' ''')
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && make install' ''')
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-jwt' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-session' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-http' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-openidc' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install luacrypto' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-cjson' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-core' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install random' '''
)
conn.sudo(
'''bash -c 'cd /tmp/src/luarocks-2.2.2/ && luarocks install lua-resty-string' '''
)
conn.sudo('useradd -r nginx')
conn.sudo('rm -f /etc/nginx/nginx.conf')
conn.sudo('mkdir -p /opt/datalab/templates')
conn.local(
'cd /root/templates; tar -zcvf /tmp/templates.tar.gz *')
conn.put('/tmp/templates.tar.gz', '/tmp/templates.tar.gz')
conn.sudo(
'tar -zxvf /tmp/templates.tar.gz -C /opt/datalab/templates/')
conn.sudo(
'sed -i \'s/EDGE_IP/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'
.format(edge_ip))
conn.sudo(
'sed -i \'s|KEYCLOAK_AUTH_URL|{}|g\' /opt/datalab/templates/conf.d/proxy.conf'
.format(keycloak_auth_server_url))
conn.sudo(
'sed -i \'s/KEYCLOAK_REALM_NAME/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'
.format(keycloak_realm_name))
conn.sudo(
'sed -i \'s/KEYCLOAK_CLIENT_ID/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'
.format(keycloak_client_id))
conn.sudo(
'sed -i \'s/KEYCLOAK_CLIENT_SECRET/{}/g\' /opt/datalab/templates/conf.d/proxy.conf'
.format(keycloak_client_secret))
conn.sudo('cp /opt/datalab/templates/nginx.conf /etc/nginx/')
conn.sudo('mkdir /etc/nginx/conf.d')
conn.sudo(
'cp /opt/datalab/templates/conf.d/proxy.conf /etc/nginx/conf.d/'
)
conn.sudo('mkdir /etc/nginx/locations')
conn.sudo(
'cp /opt/datalab/templates/nginx_redhat /etc/init.d/nginx')
conn.sudo('chmod +x /etc/init.d/nginx')
conn.sudo('chkconfig --add nginx')
conn.sudo('chkconfig --level 345 nginx on')
conn.sudo('setsebool -P httpd_can_network_connect 1')
conn.sudo('service nginx start')
conn.sudo('touch /tmp/nginx_installed')
except Exception as err:
print("Failed install nginx with ldap: " + str(err))
sys.exit(1)
