def ubuntu_install_cloudfoundry(master, *args, **kwargs): # DEPS, TODO: @depends(['go', 'bosh', 'vagrant']) command = "go" if cmd_avail(command): local("echo {command} is already installed".format(command=command)) else: install_go() command = "bosh" if cmd_avail(command): local("echo {command} is already installed".format(command=command)) else: ubuntu_actually_install_bosh(master) sudo("gem install bundler") loc_0 = "deployments" run("mkdir {loc}".format(loc=loc_0)) with cd(loc_0): loc_1 = "cf-example" run("mkdir {loc}".format(loc=loc_0)) with cd(loc_1): run("touch Gemfile") run("""source 'https://rubygems.org' ruby "1.9.3" gem "bosh_cli_plugin_aws" """) run("bundle install")
def ubuntu_install_cloudfoundry(master, *args, **kwargs): # DEPS, TODO: @depends(['go', 'bosh', 'vagrant']) command = 'go' if cmd_avail(command): local('echo {command} is already installed'.format(command=command)) else: install_go() command = 'bosh' if cmd_avail(command): local('echo {command} is already installed'.format(command=command)) else: ubuntu_actually_install_bosh(master) sudo('gem install bundler') loc_0 = 'deployments' run('mkdir {loc}'.format(loc=loc_0)) with cd(loc_0): loc_1 = 'cf-example' run('mkdir {loc}'.format(loc=loc_0)) with cd(loc_1): run('touch Gemfile') run('''source 'https://rubygems.org' ruby "1.9.3" gem "bosh_cli_plugin_aws" ''') run('bundle install')
def ubuntu_install_bosh(master, *args, **kwargs): # DEPS, TODO: @depends(['go', 'bosh', 'vagrant']) command = "go" if cmd_avail(command): local("echo {command} is already installed".format(command=command)) else: install_go() command = "bosh" if cmd_avail(command): local("echo {command} is already installed".format(command=command)) else: ubuntu_actually_install_bosh(master)
def ubuntu_install_bosh(master, *args, **kwargs): # DEPS, TODO: @depends(['go', 'bosh', 'vagrant']) command = 'go' if cmd_avail(command): local('echo {command} is already installed'.format(command=command)) else: install_go() command = 'bosh' if cmd_avail(command): local('echo {command} is already installed'.format(command=command)) else: ubuntu_actually_install_bosh(master)
def _serve_bosh(domain, master): command = 'bosh' if not cmd_avail(command): ubuntu_install_bosh(master) run('bosh version') # run('bosh help') '''
def _serve_vulcand(**kwargs): command = 'vulcandctl' if cmd_avail(command): raise EnvironmentError('Install {command} first'.format(command=command)) run('eval `ssh-agent -s`') run('vulcandctl install platform')
def install(version='v2.3.7', *args, **kwargs): command = 'etcd' if cmd_avail(command): installed_version = (lambda s: s[s.rfind(' ') + 1:])(run('etcd --version | head -n 1', quiet=True)) if version[1:] == installed_version: local('echo {command} {version} is already installed'.format(command=command, version=version)) return tempdir = get_tempdir_fab(run_command=run) install_dir = '{tempdir}/{version}/{epoch}'.format( tempdir=tempdir, version=version, epoch=run('date +%s', quiet=True) ) run('mkdir -p {install_dir}'.format(install_dir=install_dir)) ubuntu_install_curl() with cd(install_dir): run( 'curl -OL ' 'https://github.com/coreos/etcd/releases/download/{version}/etcd-{version}-linux-amd64.tar.gz'.format( version=version ) ) run('tar xf etcd-{version}-linux-amd64.tar.gz'.format(version=version)) run('sudo mv etcd-{version}-linux-amd64/etcd /usr/local/bin'.format(version=version)) run('sudo mv etcd-{version}-linux-amd64/etcdctl /usr/local/bin'.format(version=version)) run('logout') # Need to get stuff from /etc/environment on next run
def install0(arch="amd64", *args, **kwargs): version = kwargs.get("GOVERSION", "1.12.9") if cmd_avail("go"): current_version = (run( "go version", quiet=True).rpartition(" ")[0].rpartition(" ")[2][2:]) if current_version == version: return "Go {current_version} already installed".format( current_version=current_version) go_path = kwargs.get("GOPATH", run("echo $HOME/go", quiet=True)) install_loc = "/usr/local" go_tar = "go{version}.{os}-{arch}.tar.gz".format(version=version, os="linux", arch=arch) run("curl -O https://storage.googleapis.com/golang/{go_tar}".format( go_tar=go_tar)) sudo("tar -C {install_loc} -xzf {go_tar}".format(install_loc=install_loc, go_tar=go_tar)) append_path("{install_loc}/go/bin:{go_path}/bin".format( install_loc=install_loc, go_path=go_path)) append("/etc/environment", "GOPATH={go_path}".format(go_path=go_path), use_sudo=True) run("rm {go_tar}".format(go_tar=go_tar)) # run('rm -rf go*') run("mkdir -p {go_path}{subdirs}".format(go_path=go_path, subdirs="/{bin,src}")) return "Installed:{}".format(run("go version", quiet=True))
def _serve_bosh(domain, master): command = "bosh" if not cmd_avail(command): ubuntu_install_bosh(master) run("bosh version") # run('bosh help') """
def install0(*args, **kwargs): apt_depends("git") go.install0() if not cmd_avail("glide"): run("curl https://glide.sh/get | sh") gopath = run('echo "$GOPATH"', quiet=True) to_dir = "{gopath}/src/github.com/bradleyfalzon/gopherci".format(gopath=gopath) clone_or_update( "gopherci", team="SamuelMarks", branch="master", to_dir=to_dir, skip_clean=True, skip_reset=True, ) with cd(to_dir): run("glide i") run("go build && mv gopherci $GOPATH/bin") to_dir = "{gopath}/src/github.com/bradleyfalzon/gopherci-web".format(gopath=gopath) clone_or_update( "gopherci-web", team="SamuelMarks", branch="false-stripes", to_dir=to_dir, skip_clean=True, skip_reset=True, ) with cd(to_dir): run("glide i") run("go build && mv gopherci-web {gopath}/bin".format(gopath=gopath)) return {"installed": ("gopherci", "gopherci-web")}
def install_guac_client(): with cd("Downloads"): if not cmd_avail("mvn"): maven = "apache-maven-3.5.0-bin" run("curl -OL http://apache.melbourneitmirror.net/maven/maven-{ver[0]}/{ver}/binaries/{pkg}.tar.gz" .format(pkg=maven, ver=maven.rpartition("-")[0].rpartition("-")[2])) d = "/opt/javalibs/maven" sudo("mkdir -p {d}".format(d=d)) sudo("tar xf {pkg}.tar.gz -C {d} --strip-components=1".format( pkg=maven, d=d)) with cd(d): sudo( "ln -s '{d}/bin/mvn' '/usr/local/bin/$b'".format(d=d), shell_escape=False, ) guac_client = "0.9.13-incubating" if not exists("{pkg}.tar.gz".format(pkg=guac_client)): run("curl -OL https://github.com/apache/incubator-guacamole-client/archive/{pkg}.tar.gz" .format(pkg=guac_client)) run("tar xf {pkg}.tar.gz".format(pkg=guac_client)) with cd("incubator-guacamole-client-{pkg}".format(pkg=guac_client)): war = "guacamole/target/guacamole-0.9.13-incubating.war" if not exists(war): run("mvn package") sudo("cp {war} $CATALINA_HOME/webapps".format(war=war))
def _serve_vulcand(**kwargs): command = "vulcandctl" if cmd_avail(command): raise EnvironmentError( "Install {command} first".format(command=command)) run("eval `ssh-agent -s`") run("vulcandctl install platform")
def install_plugin(repo_team, repo_name, location=None): apt_depends('git') cmd = 'dokku' if not cmd_avail(cmd): raise EnvironmentError('Install {cmd} before installing plugins'.format(cmd=cmd)) with cd('/var/lib/dokku/plugins'): clone_or_update(team=repo_team, repo=repo_name, to_dir=location or repo_name) run('dokku plugins-install')
def ubuntu_install_flynn(*args, **kwargs): apt_depends("curl") command = "flynn" if cmd_avail(command): local("echo {command} is already installed".format(command=command)) return # run('L=/usr/local/bin/flynn && curl -sSL -A "`uname -sp`" https://dl.flynn.io/cli | zcat >$L && chmod +x $L') run("sudo bash < <(curl -fsSL https://dl.flynn.io/install-flynn)")
def ubuntu_install_flynn(*args, **kwargs): apt_depends('curl') command = 'flynn' if cmd_avail(command): local('echo {command} is already installed'.format(command=command)) return # run('L=/usr/local/bin/flynn && curl -sSL -A "`uname -sp`" https://dl.flynn.io/cli | zcat >$L && chmod +x $L') run('sudo bash < <(curl -fsSL https://dl.flynn.io/install-flynn)')
def build_node_app(kwargs, run_cmd): user = run_cmd("echo $USER", quiet=True) nonroot = run("echo $USER", quiet=True) if user != nonroot: logger.info('user = "******"; nonroot = "{user}"') n_prefix = kwargs.get("N_PREFIX", run_cmd("echo $HOME/n", quiet=True)) if not exists("{n_prefix}/bin".format(n_prefix=n_prefix)): (install_node0 if "Ubuntu" in run("uname -v") else install_node)( node_version=kwargs.get("node_version"), use_sudo=False, node_sudo=False, N_PREFIX=n_prefix, ) with shell_env(PATH="$PATH:{n_prefix}/bin".format(n_prefix=n_prefix)): if not cmd_avail("npm"): return "[Warn]: npm not installed; skipping" npm_tmp = run_cmd("echo $HOME/.npm/_cacache/tmp", quiet=True) install_global_npm_packages1( npm_global_packages=kwargs.get("npm_global_packages"), N_PREFIX=n_prefix, use_sudo=False, node_sudo=False, ) if run_cmd("npm i --unsafe-perm=true", warn_only=True).failed: # sudo('chown -R {user} {npm_tmp}'.format(user=user, npm_tmp=npm_tmp)) run_cmd("chown -R {u} {d} {s}".format( u=nonroot, d=kwargs["GIT_DIR"], s="$(npm config get prefix)/{lib/node_modules,bin,share}", )) sudo("npm i --unsafe-perm=true", user=nonroot) if exists("typings.json"): if cmd_avail("typings"): run_cmd("rm -rf typings") run_cmd("typings i") else: logger.warn("typings not installed; skipping") if "post_npm_step" in kwargs: run_cmd(kwargs["post_npm_step"]) sudo("rm -rf {npm_tmp}".format(npm_tmp=npm_tmp)) return "[node] app built"
def install_plugin(repo_team, repo_name, location=None): apt_depends("git") cmd = "dokku" if not cmd_avail(cmd): raise EnvironmentError( "Install {cmd} before installing plugins".format(cmd=cmd)) with cd("/var/lib/dokku/plugins"): clone_or_update(team=repo_team, repo=repo_name, to_dir=location or repo_name) run("dokku plugins-install")
def install_docker_compose3(*args, **kwargs): version = "1.27.3" if cmd_avail("docker-compose"): return "already installed" executable = "/usr/local/bin/docker-compose" sudo( "curl -L https://github.com/docker/compose/releases/download/{version}/docker-compose-$(uname -s)-$(uname -m)" " -o {executable}".format(executable=executable, version=version)) sudo("chmod +x {executable}".format(executable=executable))
def install_deps0(*args, **kwargs): if not cmd_avail("javac"): sudo("add-apt-repository -y ppa:webupd8team/java") sudo( "echo debconf shared/accepted-oracle-license-v1-1 select true | \ sudo debconf-set-selections" ) apt_depends("oracle-java8-set-default") srcs_list = "/etc/apt/sources.list.d/gocd.list" if not exists(srcs_list): append(srcs_list, "deb https://download.gocd.org /", use_sudo=True) sudo("curl https://download.gocd.org/GOCD-GPG-KEY.asc | sudo apt-key add -")
def ubuntu_install_coreos(*args, **kwargs): # raise NotImplementedError("CoreOS on Ubuntu? - That'll be the day!") command = 'etcd' if cmd_avail(command): local('echo {command} is already installed'.format(command=command)) else: install_etcd() status = run('status etcd2', warn_only=True, quiet=True) if status.succeeded: if not status.endswith('etcd2 stop/waiting'): sudo('stop etcd2') return '[ubuntu_install_coreos] etcd2 service stopped (so the conf can be updated)'
def ubuntu_install_coreos(*args, **kwargs): # raise NotImplementedError("CoreOS on Ubuntu? - That'll be the day!") command = "etcd" if cmd_avail(command): local("echo {command} is already installed".format(command=command)) else: install_etcd() status = run("status etcd2", warn_only=True, quiet=True) if status.succeeded: if not status.endswith("etcd2 stop/waiting"): sudo("stop etcd2") return "[ubuntu_install_coreos] etcd2 service stopped (so the conf can be updated)"
def install_node(node_version="latest", *args, **kwargs): run_cmd = partial(_run_command, sudo=kwargs.get("node_sudo", kwargs.get("use_sudo", False))) n_prefix = kwargs.get("N_PREFIX", run_cmd("echo $HOME/n", quiet=True)) if cmd_avail("n") and not exists( "{n_prefix}/n/versions/node/{version}".format( n_prefix=n_prefix, version=node_version)): run_cmd("n {version}".format(version=node_version)) else: run_cmd("curl -L https://git.io/n-install" " | N_PREFIX='{n_prefix}' bash -s -- -y '{version}'".format( n_prefix=n_prefix, version=node_version)) return {"Node.js version installed": node_version}
def shared_serve(etcd_discovery, size, kwargs): if 'ADVERT_PORT' not in kwargs: kwargs['ADVERT_PORT'] = 2379 if 'PEER_PORT' not in kwargs: kwargs['PEER_PORT'] = kwargs['ADVERT_PORT'] + 1 if 'ADDITIONAL_LISTEN_PORT' not in kwargs: kwargs['ADDITIONAL_LISTEN_PORT'] = 4001 command = 'etcd' if not cmd_avail(command): raise EnvironmentError('Expected {command} to be installed'.format(command=command)) client = get_client() discovery_path = '/'.join((kwargs['cluster_path'], 'discovery')) return etcd_discovery or get_or_set_etcd_discovery_url(client, discovery_path, size)
def install_bazel(from_ppa=True): if not from_ppa: raise NotImplementedError() elif cmd_avail("bazel"): return fname = "/etc/apt/sources.list.d/bazel.list" sudo("touch {}".format(fname)) append( filename=fname, text= "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8", use_sudo=True, ) sudo("curl https://bazel.build/bazel-release.pub.gpg | apt-key add -") apt_depends("openjdk-8-jdk", "bazel")
def install_guac_server(): if cmd_avail("guacenc"): return "Apache Guacamole server already installed" with cd("Downloads"): guac = "guacamole-server-0.9.13-incubating" run("curl -OL 'http://apache.org/dyn/closer.cgi?action=download&filename=incubator/guacamole/0.9.13-incubating/source/{pkg}.tar.gz'" .format(pkg=guac)) run("tar xf {pkg}.tar.gz".format(pkg=guac)) with cd(guac): run("autoreconf -fi") run("./configure --with-init-dir=/etc/init.d") run("make") sudo("make install") sudo("ldconfig") return "installed Apache Guacamole server"
def install_docker0(*args, **kwargs): if cmd_avail("docker"): return kernel_version = sudo("uname -r", quiet=True) if tuple(map(int, kernel_version[:kernel_version.find(".", 2)].split("."))) < ( 3, 10, ): raise NotImplementedError( "Docker for older versions of the Linux kernel") for mnt in sudo('mount | grep "^/dev"'): if "xfs" not in mnt and len(mnt) > 1: print("mnt =", mnt) dev = mnt[:mnt.find(" on")] print("dev =", dev) sudo("mkfs -t xfs -n ftype=1 {dev}".format(dev=dev)) if sudo("lsmod | grep overlay", warn_only=True).failed: append("/etc/modules-load.d/overlay.conf", "overlay", use_sudo=True) sudo("reboot") return "rebooting: RERUN" centos_join = partial( path.join, path.join( path.dirname( resource_filename(modules[__name__].__package__, "__init__.py")), "_centos", ), ) with open(centos_join("docker.yum")) as f: append("/etc/yum.repos.d/docker.repo", f.read(), use_sudo=True) with open(centos_join("override.conf")) as f: p = "/etc/systemd/system/docker.service.d" sudo("mkdir -p {p}".format(p=p)) append("{p}/override.conf".format(p=p), f.read(), use_sudo=True) sudo("yum install -y docker-engine-1.13.1 docker-engine-selinux-1.13.1") sudo("systemctl start docker") sudo("systemctl enable docker") return sudo("docker ps")
def housekeeping0(*args, **kwargs): if sudo("docker ps", warn_only=True, quiet=True).failed: raise EnvironmentError( "Expected Docker. Include offregister-docker in your config.") disable_service("firewalld") sudo("mkdir -p /var/lib/dcos /var/lib/mesos") append("/etc/sudoers", "%wheel ALL=(ALL) NOPASSWD: ALL", use_sudo=True) if not cmd_avail("timedatectl") and run( "timedatectl", quiet=True, warn_only=True): raise EnvironmentError("Expected NTP to be enabled.") yum_depends("tar", "xz", "unzip", "curl", "ipset") append( "/etc/environment", "LANG={enc}\nLC_ALL={enc}".format(enc="en_US.utf-8"), use_sudo=True, ) # <Cluster node> if sudo("grep -F 'SELINUX=permissive' /etc/selinux/config", warn_only=True, quiet=True).failed: sudo( "sed -i s/SELINUX=enforcing/SELINUX=permissive/g /etc/selinux/config" ) did_something = False for group in ("nogroup", "docker"): if sudo("grep -Fq {group} /etc/group".format(group=group)): sudo("groupadd {group}".format(group=group)) did_something = True if did_something: sudo("reboot") # </Cluster node> run("mkdir -p ~/Downloads") with cd("~/Downloads"): sh = "dcos_generate_config.sh" if not exists(sh): run("curl -L https://downloads.dcos.io/dcos/stable/{sh} -o {sh}". format(sh=sh))
def install(c): """ :param c: Connection :type c: ```fabric.connection.Connection``` """ if cmd_avail(c, "certbot"): return "certbot already installed" uname = c.run("uname -v").stdout dist_version = float(c.run("lsb_release -rs", hide=True).stdout) is_debian = "Debian" in uname is_ubuntu = "Ubuntu" in uname if is_debian or is_ubuntu: apt_depends( c, "certbot", "python{}-certbot-nginx".format( "3" if is_ubuntu and dist_version >= 20.04 or is_debian and dist_version >= 10 else ""), ) else: raise NotImplementedError()
def install_docker0(*args, **kwargs): if cmd_avail("docker"): return "Docker is already installed" apt_depends( "apt-transport-https", "ca-certificates", "curl", "gnupg-agent", "software-properties-common", ) sudo( "curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -" ) sudo("apt-key fingerprint 0EBFCD88") distro = run("lsb_release -cs") sudo( 'add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu {distro} stable"' .format(distro=distro)) apt_depends("docker-ce", "docker-ce-cli", "containerd.io") sudo("systemctl enable docker") return "Docker is now installed and will run at boot"
def nginx3(**kwargs): if not kwargs["nginx"]: return "[Warn]: skipping nginx" if not cmd_avail("nginx") and not exists("/etc/nginx"): uname = run("uname -v") sio = StringIO() flavour = None if "Ubuntu" in uname: flavour = "ubuntu" elif "Debian" in uname: flavour = "debian" if flavour is None: raise NotImplementedError() apt_depends("curl", "gnupg2", "ca-certificates", "lsb-release") release = run("lsb_release -cs") sio.write( "deb http://nginx.org/packages/{flavour} {release} nginx".format( flavour=flavour, release=release)) put(sio, "/etc/apt/sources.list.d/nginx.list", use_sudo=True) sudo("apt-get update -qq") sudo( "curl -fsSL https://nginx.org/keys/nginx_signing.key | apt-key add -" ) sudo("apt-key fingerprint ABF5BD827BD9BF62") apt_depends("nginx") # TODO: Move this to an nginx module; usable by other `offregister-` packages sites_avail_local_filepath = kwargs.get( "nginx-sites-available", resource_filename("offregister_app_push", path.join("conf", "nginx.sites-available.conf")), ) proxy_block_local_filepath = kwargs.get( "nginx-proxy-block", resource_filename("offregister_app_push", path.join("conf", "nginx.proxy_block.conf")), ) remote_conf_dir = "/etc/nginx/sites-enabled" if not exists(remote_conf_dir): sudo("mkdir -p {}".format(remote_conf_dir)) sudo( r"sed -i '/include \/etc\/nginx\/conf.d\/\*.conf;/ a\ \ \ \ include {remote_dir}/*;' {fname}" .format(remote_dir=remote_conf_dir, fname="/etc/nginx/nginx.conf")) conf_remote_filename = "/etc/nginx/sites-enabled/{service_name}.conf".format( service_name=kwargs["app_name"]) deque( map( lambda dns_name: append( text="127.0.0.1\t{site_name}".format(site_name=dns_name), filename="/etc/hosts", use_sudo=True, ), kwargs["DNS_NAMES"], ), maxlen=0, ) _send_nginx_conf( conf_remote_filename, proxy_block_local_filepath, sites_avail_local_filepath, kwargs, ) return restart_systemd("nginx")
def install_deps(): # sudo('add-apt-repository ppa:mc3man/trusty-media') apt_depends( "curl", # From Guacamole "libcairo2-dev", # 'libjpeg62-turbo-dev', 'libjpeg62-dev', "libpng12-dev", "libossp-uuid-dev", "libpango-1.0-0", "libpango1.0-dev", "libssh2-1", "libssh2-1-dev", "libpulse-dev", "libssl-dev", "libvorbis-dev", "libwebp-dev", "libfreerdp-dev", # 'ffmpeg' "openjdk-8-jdk", # <- client # From FreeRDP "build-essential", "git-core", "cmake", "xsltproc", "libssl-dev", "libx11-dev", "libxext-dev", "libxinerama-dev", "libxcursor-dev", "libxdamage-dev", "libxv-dev", "libxkbfile-dev", "libasound2-dev", "libcups2-dev", "libxml2", "libxml2-dev", "libxrandr-dev", "libgstreamer0.10-dev", "libgstreamer-plugins-base0.10-dev", "libxi-dev", "libgstreamer-plugins-base1.0-dev", "libavutil-dev", "libavcodec-dev", "libdirectfb-dev", # From libtelnet "autoconf", "libtool", "automake", # From ffmpeg "yasm", ) sudo( "update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java 1" ) sudo( "update-alternatives --install /usr/bin/javac javac /usr/lib/jvm/java-8-openjdk-amd64/bin/javac 1" ) sudo("mkdir -p ~/Downloads") curr_ug = run("echo -n $USER:$GROUP", quiet=True) sudo("chown -R {curr_ug} ~/Downloads".format(curr_ug=curr_ug)) with cd("Downloads"): libjpeg = "libjpeg-turbo-official_1.5.2_amd64.deb" if not exists(libjpeg): run("curl -L 'https://downloads.sourceforge.net/project/libjpeg-turbo/{ver}/{pkg}' -o '{pkg}'" .format(pkg=libjpeg, ver=libjpeg.rpartition("_")[0].partition("_")[2])) sudo("dpkg -i {pkg}".format(pkg=libjpeg)) zlib = "zlib-1.2.11" if not exists(zlib): run("curl -OL http://www.zlib.net/{pkg}.tar.gz".format(pkg=zlib)) run("tar xf {pkg}.tar.gz".format(pkg=zlib)) with cd(zlib): run("./configure") run("make") sudo("make install") libvnc = "LibVNCServer-0.9.11" if not exists("{pkg}.tar.gz".format(pkg=libvnc)): run("curl -OL https://github.com/LibVNC/libvncserver/archive/{pkg}.tar.gz" .format(pkg=libvnc)) run("tar xf {pkg}.tar.gz".format(pkg=libvnc)) with cd("libvncserver-{pkg}".format(pkg=libvnc)): run("mkdir build") with cd("build"): run("cmake ..") run("cmake --build .") sudo("make install") """freerdp = '2.0.0-rc0' run('curl -OL https://github.com/FreeRDP/FreeRDP/archive/{pkg}.tar.gz'.format(pkg=freerdp)) run('tar xf {pkg}.tar.gz'.format(pkg=freerdp)) with cd('FreeRDP-{pkg}'.format(pkg=freerdp)): run('rm -rf build') run('mkdir build') with cd('build'): run('cmake -DCMAKE_BUILD_TYPE=Debug -DWITH_SSE2=ON ..') run('make') sudo('make install') sudo('mkdir -p /etc/ld.so.conf.d') append('/etc/ld.so.conf.d/freerdp.conf', '/usr/local/lib/freerdp', use_sudo=True) sudo('ldconfig')""" libtelnet = "libtelnet-master" if not exists(libtelnet): run("curl -L https://github.com/seanmiddleditch/libtelnet/archive/master.tar.gz -o {pkg}.tar.gz" .format(pkg=libtelnet)) run("tar xf {pkg}.tar.gz".format(pkg=libtelnet)) with cd(libtelnet): run("autoreconf -fi") run('zlib_CFLAGS="-DHAVE_ZLIB=1" zlib_LIBS="-lz" ./configure --disable-util' ) run("make") sudo("make install") sudo("ldconfig") if not cmd_avail("ffmpeg"): ffmpeg = "ffmpeg-3.3.4" run("curl -OL http://ffmpeg.org/releases/{pkg}.tar.bz2".format( pkg=ffmpeg)) run("tar xf {pkg}.tar.bz2".format(pkg=ffmpeg)) with cd(ffmpeg): run("./configure") run("make") sudo("make install") CATALINA_HOME = "/opt/javalibs/tomcat9" if not exists(CATALINA_HOME): sudo( "mkdir -p {CATALINA_HOME}".format(CATALINA_HOME=CATALINA_HOME)) append( "/etc/environment", "export CATALINA_HOME='{CATALINA_HOME}'".format( CATALINA_HOME=CATALINA_HOME), use_sudo=True, ) tomcat = "apache-tomcat-9.0.1" tomcat_ver = tomcat.rpartition("-")[2] run("curl -OL http://mirror.intergrid.com.au/apache/tomcat/tomcat-{ver[0]}/v{ver}/bin/{pkg}.tar.gz" .format(pkg=tomcat, ver=tomcat_ver)) sudo("tar xf {pkg}.tar.gz -C {CATALINA_HOME} --strip-components=1". format(pkg=tomcat, CATALINA_HOME=CATALINA_HOME)) return "installed dependencies"
def ubuntu_install_curl(): command = 'curl' if cmd_avail(command): local('echo {command} is already installed'.format(command=command)) else: sudo('apt-get install -y curl')
def step0(domain, *args, **kwargs): key_file = '/root/.ssh/id_rsa.pub' config = { 'DOKKU_HOSTNAME': ('hostname', domain), 'DOKKU_KEY_FILE': ('key_file', key_file), 'DOKKU_SKIP_KEY_FILE': ('skip_key_file', False), 'DOKKU_VHOST_ENABLE': ('vhost_enable', False), 'DOKKU_WEB_CONFIG': ('web_config', False) } create_static = kwargs.get('create_static_page', True) static_git_url = kwargs.get('static_git', environ.get('DOKKU_STATIC_GIT', environ.get('STATIC_GIT'))) local_pubkey = kwargs.get('PUBLIC_KEY_PATH') or environ.get('DOKKU_PUBLIC_KEY_PATH', environ['PUBLIC_KEY_PATH']) if not cmd_avail('docker'): docker.install_0() # docker.dockeruser_1() docker.serve_2() put(StringIO('pZPlHOkV649DCepEwf9G'), '/tmp/passwd') if not cmd_avail('dokku'): # is_installed('dokku'): run('wget -qN https://packagecloud.io/gpg.key') sudo('apt-key add gpg.key') append('/etc/apt/sources.list.d/dokku.list', 'deb https://packagecloud.io/dokku/dokku/ubuntu/ trusty main', use_sudo=True) put(StringIO('\n'.join('{com} {com}/{var} {type} {val}'.format( com='dokku', var=v[0], val=str(v[1]).lower() if type(v[1]) is BooleanType else v[1], type=( lambda t: {type(True): 'boolean', type(''): 'string', type(unicode): 'string'}.get(t, t))(type(v[1]))) for k, v in config.iteritems() if v[1] is not None) ), '/tmp/dokku-debconf') sudo('debconf-set-selections /tmp/dokku-debconf') if not exists(key_file): sudo('ssh-keygen -t rsa -b 4096 -f {key_file} -N ""'.format(key_file=key_file)) apt_depends('dokku') sudo('dokku plugin:install-dependencies --core') put(local_pubkey, key_file) sudo('sshcommand acl-add dokku domain {key_file}'.format(key_file=key_file)) return 'installed dokku' if create_static: if run('getent passwd static', quiet=True, warn_only=True).failed: sudo('adduser static --disabled-password') sudo('mkdir /home/static/sites/', user='******') upload_template( path.join(path.dirname(resource_filename('offregister_dokku', '__init__.py')), 'data', 'static_sites.conf'), '/etc/nginx/conf.d/static_sites.conf', use_sudo=True ) if sudo('service nginx status').endswith('stop/waiting'): sudo('service nginx start') else: sudo('service nginx reload') # TODO: Abstract this out into a different module, and allow for multiple domains if static_git_url: ipv4 = '/home/static/sites/{public_ipv4}'.format(public_ipv4=kwargs['public_ipv4']) if exists(ipv4): sudo('rm -rf {ipv4}'.format(ipv4=ipv4)) sudo('mkdir -p {ipv4}'.format(ipv4=ipv4), user='******') if domain: domain = '/home/static/sites/{domain}'.format(domain=domain) if not exists(domain): sudo('ln -s {ipv4} {domain}'.format(ipv4=ipv4, domain=domain), user='******') xip = '{ipv4}.xip.io'.format(ipv4=ipv4) if not exists(xip): sudo('ln -s {ipv4} {xip}'.format(ipv4=ipv4, xip=xip), user='******') if static_git_url: apt_depends('git') if isinstance(static_git_url, basestring): clone_or_update(**url_to_git_dict(static_git_url)) else: clone_or_update(to_dir=ipv4, **static_git_url) return 'installed dokku [already]'
def add_cert1(c, domains, email, server="nginx", **kwargs): """add_cert1 gets a new LetsEncrypt HTTPS certificate using certbot. Because we don't trust the nginx module, we do this process: 0. move conf(s) for domain(s) to sites-disabled 1. create new conf(s) for domain(s) in sites-enabled, that point to temporary wwwroot 2. use certbot to request new certificate 3. restore previous conf :param c: Connection :type c: ```fabric.connection.Connection``` """ if server != "nginx": logger.warning( "Not tested with non nginx: {server}".format(server=server)) sites_enabled = kwargs.get("sites-enabled", "/etc/nginx/sites-enabled") sites_disabled = kwargs.get("sites-disabled", "/etc/nginx/sites-disabled") c.sudo("mkdir -p {sites_disabled}".format(sites_disabled=sites_disabled)) cmd = partial( c.sudo, user=kwargs.get("as_user", "root"), # group=kwargs.get("as_group", "root"), ) if not cmd_avail(c, "nginx") and not exists( c, runner=cmd, path="/usr/sbin/nginx"): raise EnvironmentError("nginx is not installed | not found in PATH") if not exists(c, runner=cmd, path=sites_enabled): raise EnvironmentError('No confs found in "{}"'.format(sites_enabled)) _grep_conf = "grep -lER {pat} {sites_enabled}".format( pat=" -e " + " -e ".join("'server_name[[:space:]]+{domain}'".format(domain=domain) for domain in domains) if len(domains) > 0 else "", sites_enabled=sites_enabled, ) confs = cmd(_grep_conf, warn=True) if confs.exited != 0: stderr.write(confs.stderr) raise EnvironmentError( "grep failed with: {}. Failed command: {}".format( confs, _grep_conf)) elif not confs: # print(_grep_conf, file=sys.stderr) raise ReferenceError("No confs found matching domains searched for") confs = [ s.rstrip().replace(sites_enabled + "/", "") for s in confs.stdout.split("\n") ] cmd("mv '{sites_enabled}/'{confs} '{sites_disabled}/'".format( sites_enabled=sites_enabled, confs="{{{confs}}}".format( confs=",".join(confs)) if len(confs) > 1 else "".join(confs), sites_disabled=sites_disabled, )) def apply_conf(domain): root = c.sudo("mktemp -d --suffix .nginx").stdout.strip() assert root != "/" nginx.setup_custom_conf2( c, nginx_conf="static.conf", SERVER_NAME=domain, WWWROOT=root, conf_remote_filename="{}/{}".format(sites_enabled, domain.replace("/", "-")), skip_nginx_restart=True, ) cmd("touch {root}/index.html && chmod -R 755 {root}".format(root=root)) return root static_dirs = tuple(map(apply_conf, domains)) def exclude_valid_certs(domain): cert_details = c.sudo( "certbot certificates --cert-name {domain}".format( domain=domain)).stdout if "Expiry Date" not in cert_details: return domain elif "(VALID" not in cert_details: return domain cert_expiry = next( map( lambda s: s.partition(":")[2].rpartition("(")[0].strip(), filter( lambda s: s.lstrip().startswith("Expiry Date"), cert_details.split("\n"), ), ), None, ) if cert_expiry is None: return domain cert_expiry = datetime.strptime(cert_expiry, "%Y-%m-%d %H:%M:%S+00:00") if (cert_expiry - datetime.now()).days < 30: return domain return None domains = tuple(_f for _f in map(exclude_valid_certs, domains) if _f) if domains: restart_systemd(c, "nginx") cmd("certbot certonly {email} --webroot {webroots} {domains} --agree-tos --no-eff-email" .format( email="-m '{email}'".format(email=email), webroots=" ".join("-w '{}'".format(wr) for wr in static_dirs), domains=" ".join("-d '{}'".format(domain) for domain in domains), )) cmd("rm -rf {}/*nginx".format( static_dirs[0][:static_dirs[0].rfind("/")])) cmd("rm -f {}".format(" ".join("{}/{}".format( sites_enabled, domain if domain.endswith(".conf") else "{domain}.conf".format( domain=domain), ) for domain in domains))) cmd("mv '{sites_disabled}/'{confs} '{sites_enabled}/'".format( sites_enabled=sites_enabled, confs="{{{confs}}}".format(confs=",".join(confs)) if len(confs) > 1 else "{confs}".format(confs="".join(confs)), sites_disabled=sites_disabled, )) return restart_systemd(c, "nginx") # reload didn't work :(
def step0(domain, *args, **kwargs): key_file = "/root/.ssh/id_rsa.pub" config = { "DOKKU_HOSTNAME": ("hostname", domain), "DOKKU_KEY_FILE": ("key_file", key_file), "DOKKU_SKIP_KEY_FILE": ("skip_key_file", False), "DOKKU_VHOST_ENABLE": ("vhost_enable", False), "DOKKU_WEB_CONFIG": ("web_config", False), } create_static = kwargs.get("create_static_page", True) static_git_url = kwargs.get( "static_git", environ.get("DOKKU_STATIC_GIT", environ.get("STATIC_GIT"))) local_pubkey = kwargs.get("PUBLIC_KEY_PATH") or environ.get( "DOKKU_PUBLIC_KEY_PATH", environ["PUBLIC_KEY_PATH"]) if not cmd_avail("docker"): docker.install_0() # docker.dockeruser_1() docker.serve_2() put(StringIO("pZPlHOkV649DCepEwf9G"), "/tmp/passwd") if not cmd_avail("dokku"): # is_installed('dokku'): run("wget -qN https://packagecloud.io/gpg.key") sudo("apt-key add gpg.key") append( "/etc/apt/sources.list.d/dokku.list", "deb https://packagecloud.io/dokku/dokku/ubuntu/ trusty main", use_sudo=True, ) put( StringIO("\n".join("{com} {com}/{var} {type} {val}".format( com="dokku", var=v[0], val=str(v[1]).lower() if type(v[1]) is type(bool) else v[1], type=(lambda t: { type(True): "boolean", type(""): "string", type(str): "string", }.get(t, t))(type(v[1])), ) for k, v in iteritems(config) if v[1] is not None)), "/tmp/dokku-debconf", ) sudo("debconf-set-selections /tmp/dokku-debconf") if not exists(key_file): sudo('ssh-keygen -t rsa -b 4096 -f {key_file} -N ""'.format( key_file=key_file)) apt_depends("dokku") sudo("dokku plugin:install-dependencies --core") put(local_pubkey, key_file) sudo("sshcommand acl-add dokku domain {key_file}".format( key_file=key_file)) return "installed dokku" if create_static: if run("getent passwd static", quiet=True, warn_only=True).failed: sudo("adduser static --disabled-password") sudo("mkdir /home/static/sites/", user="******") upload_template( path.join( path.dirname( resource_filename("offregister_dokku", "__init__.py")), "data", "static_sites.conf", ), "/etc/nginx/conf.d/static_sites.conf", use_sudo=True, ) if sudo("service nginx status").endswith("stop/waiting"): sudo("service nginx start") else: sudo("service nginx reload") # TODO: Abstract this out into a different module, and allow for multiple domains if static_git_url: ipv4 = "/home/static/sites/{public_ipv4}".format( public_ipv4=kwargs["public_ipv4"]) if exists(ipv4): sudo("rm -rf {ipv4}".format(ipv4=ipv4)) sudo("mkdir -p {ipv4}".format(ipv4=ipv4), user="******") if domain: domain = "/home/static/sites/{domain}".format(domain=domain) if not exists(domain): sudo( "ln -s {ipv4} {domain}".format(ipv4=ipv4, domain=domain), user="******", ) xip = "{ipv4}.xip.io".format(ipv4=ipv4) if not exists(xip): sudo("ln -s {ipv4} {xip}".format(ipv4=ipv4, xip=xip), user="******") if static_git_url: apt_depends("git") if isinstance(static_git_url, str): clone_or_update(**url_to_git_dict(static_git_url)) else: clone_or_update(to_dir=ipv4, **static_git_url) return "installed dokku [already]"
def install_configure0(*args, **kwargs): apt_install = False if apt_install: apt_depends("webhook") elif not cmd_avail("webhook"): go.install0() run("go get github.com/adnanh/webhook") if (kwargs.get("HOOK_PORT") == 443 or kwargs.get("HOOK_KEY") or kwargs.get("HOOK_CERT")): kwargs["HOOK_SECURE"] = True if not kwargs.get("HOOK_IP") and kwargs.get("SERVER_NAME"): kwargs["HOOK_IP"] = kwargs["SERVER_NAME"] if not kwargs.get("HOOK_HOOKS"): kwargs["HOOK_HOOKS"] = "/etc/webhook.json" else: sudo('mkdir -p "${' + kwargs["HOOK_HOOKS"] + '##*/}"', shell_escape=False) sio = StringIO() dump(kwargs["HOOK_HOOKS_JSON"], sio) tmp = "{}.tmp".format(kwargs["HOOK_HOOKS"]) put(sio, tmp, use_sudo=True) sudo("cat {tmp} | envsubst > {hooks} && rm {tmp}".format( tmp=tmp, hooks=kwargs["HOOK_HOOKS"])) if "HOOK_NOPANIC" not in kwargs: kwargs["HOOK_NOPANIC"] = "" # true elif not kwargs["HOOK_NOPANIC"]: del kwargs["HOOK_NOPANIC"] upload_template( hook_dir("webhook.service"), "/lib/systemd/system/", context={ "CMD": "/usr/bin/webhook" if apt_install else run( 'echo "$GOPATH/bin/webhook"', quiet=True, shell_escape=False), "ARGS": " ".join("-{cli_arg} '{cli_val}'".format( cli_arg=cli_arg, cli_val=kwargs["HOOK_{}".format(cli_arg.upper())]) for cli_arg in ( "cert", "header", "hooks", "hotreload", "ip", "key", "nopanic", "port", "secure", "template", "verbose", ) if "HOOK_{}".format(cli_arg.upper()) in kwargs).replace( " ''", "").replace(" 'True'", ""), }, use_sudo=True, ) return restart_systemd("webhook")
def _install_backend( backend_root, name, remote_user, backend_virtual_env, team, repo, database_uri, env_vars, install_postgres=True, create_postgres_database=True, use_sudo=True, ): uname = run("uname -v") is_ubuntu = "Ubuntu" in uname run_cmd = partial(_run_command, sudo=use_sudo) if install_postgres: if not is_ubuntu and not cmd_avail("psql"): raise NotImplementedError("Postgres install on {!r}".format(uname)) postgres.install0() if create_postgres_database: parsed_database_uri = urlparse(database_uri) created = postgres.setup_users( create=({ "user": parsed_database_uri.username, "password": parsed_database_uri.password, "dbname": parsed_database_uri.path[1:], }, ), connection_str=database_uri, ) assert created is not None clone_or_update(team=team, repo=repo, use_sudo=use_sudo, to_dir=backend_root, branch="master") offregister_python_ubuntu.install_venv0(virtual_env=backend_virtual_env, python3=True, packages=("gunicorn", "uwsgi")) offregister_python_ubuntu.install_package1(package_directory=backend_root, virtual_env=backend_virtual_env) if not exists("/etc/systemd/system"): raise NotImplementedError("Non SystemD platforms") if run("id {remote_user}".format(remote_user=remote_user), warn_only=True, quiet=True).failed: sudo('adduser {remote_user} --disabled-password --quiet --gecos ""'. format(remote_user=remote_user)) (uid, user), (gid, group) = get_user_group_tuples(remote_user) uwsgi_service = "/etc/systemd/system/{name}-uwsgi.service".format( name=name) upload_template( circus_dir("uwsgi.service"), uwsgi_service, context={ "USER": user, "GROUP": group, "{}_BACK".format(name.upper()): "{}/{}".format(backend_root, name), "UID": uid, "GID": gid, "VENV": backend_virtual_env, "BACKEND_ROOT": backend_root, "SERVICE_USER": "******", "NAME": name, "ENV": env_vars, }, use_sudo=True, backup=False, mode=0o644, ) grp = sudo("id -gn", quiet=True) sudo("chown {grp}:{grp} {uwsgi_service}".format( grp=grp, uwsgi_service=uwsgi_service)) restart_systemd("{name}-uwsgi".format(name=name)) return backend_virtual_env, database_uri
def _nginx_cerbot_setup( domains, https_cert_email, conf_dirs=("/etc/nginx/sites-enabled", ), use_sudo=True, warn_only=True, quiet=True, ): if not cmd_avail("certbot"): install() if domains != "all": raise NotImplementedError("{} for domains".format(domains)) run_cmd = partial(_run_command, sudo=use_sudo) if not run("ls -A '{conf_dir}'".format(conf_dir=conf_dirs[0]), shell_escape=False): return "hosts_d is empty empty; skipping" server_names_t = tuple( chain(*(run_cmd("grep -RF server_name '{conf_dir}'".format( conf_dir=conf_dir)).split("\n") for conf_dir in conf_dirs))) hosts = tuple( l.partition("127.0.0.1")[2].strip() for l in run_cmd("grep -F 127.0.0.1 /etc/hosts").split("\n") if "localhost" not in l) server_names_d = dict((lambda spl: (spl[1].lstrip().rstrip("; \t\r"), spl[ 0][:spl[0].rfind(":")]))(l.split("server_name")) for l in server_names_t) if len(server_names_d) < len(server_names_t): raise NotImplementedError( "Same server_name in multiple files. We don't know what to stop!") hosts_d = { host: server_names_d[host] for host in hosts if host.count(".") > 1 and host in server_names_d and len(host.translate(None, "~^|()?*")) == len(host) } if not hosts_d: return "hosts_d is empty empty; skipping" run_cmd("mkdir -p /etc/nginx/sites-disabled") sites_avail_local_filepath = resource_filename( "offregister_app_push", path.join("conf", "nginx.sites-available.conf")) def certbot_prep(dns_name, conf_loc): run_cmd("mv '{}' '/etc/nginx/sites-disabled/{}'".format( conf_loc, path.split(conf_loc)[1])) wwwroot = "/var/www/static/{dns_name}".format(dns_name=dns_name) if exists(wwwroot): run_cmd("rm -r '{wwwroot}'".format(wwwroot=wwwroot)) run_cmd("mkdir -p '{wwwroot}'".format(wwwroot=wwwroot)) _send_nginx_conf( conf_remote_filename="/etc/nginx/sites-enabled/{dns_name}-certbot". format(dns_name=dns_name), sites_avail_local_filepath=sites_avail_local_filepath, proxy_block_local_filepath=None, conf_vars={ "NGINX_PORT": 80, "DNS_NAMES": (dns_name, ), "DESCRIPTION": "Temporary conf doing certbot for {}".format(dns_name), "WWWPATH": "/", "WWWROOT": wwwroot, }, ) print( 'one("{}", "{}") ='.format(dns_name, conf_loc), "-w '{wwwroot}' -d '{dns_name}' ".format(dns_name=dns_name, wwwroot=wwwroot), ) return "-w '{wwwroot}' -d '{dns_name}' ".format(dns_name=dns_name, wwwroot=wwwroot) secured_already = (frozenset( run_cmd("ls /etc/letsencrypt/live", warn_only=True).splitlines()) if exists("/etc/letsencrypt/live") else tuple()) cerbot_cmds = tuple( "certbot certonly --agree-tos -m {https_cert_email} --webroot {root}". format(https_cert_email=https_cert_email, root=certbot_prep(dns_name, conf_loc)) for dns_name, conf_loc in iteritems(hosts_d) if dns_name not in secured_already) if not cerbot_cmds: return "You must've already secured all your domains. Otherwise clean: /etc/letsencrypt/live" service_name = "nginx" if sudo( "systemctl status -q {service_name} --no-pager --full".format( service_name=service_name), warn_only=True, ).failed: sudo("systemctl start -q {service_name} --no-pager --full".format( service_name=service_name)) else: sudo("systemctl reload -q {service_name} --no-pager --full".format( service_name=service_name)) print("cerbot_cmds =", cerbot_cmds) certbot_res = tuple(map(run_cmd, cerbot_cmds)) sudo("cp /etc/nginx/sites-disabled/* /etc/nginx/sites-enabled") # sudo('rm -r /etc/nginx/sites-disabled') def secure_conf(dns_name, conf_loc, https_header): # print 'secure_conf({!r}, {!r})'.format(dns_name, conf_loc) if run_cmd("grep -Fq 443 {conf_loc}".format(conf_loc=conf_loc), warn_only=True).failed: logger.warning( "Skipping {conf_loc}; 443 already found within".format( conf_loc=conf_loc)) sio = StringIO() get(remote_path=conf_loc, use_sudo=use_sudo, local_path=sio) sio.seek(0) sio_s = sio.read() substr = sio_s[sio_s.find("{", sio_s.find("server")):sio_s.rfind("}") + 2].replace("listen 80", "listen 443", 1) https_header %= { "CA_CERT_PATH": "/etc/letsencrypt/live/{dns_name}/fullchain.pem".format( dns_name=dns_name), "PRIV_KEY_PATH": "/etc/letsencrypt/live/{dns_name}/privkey.pem".format( dns_name=dns_name), } """ # TODO: Address parsing, if not in `listen` keyword sni = substr.find('server_name') sni = substr[sni:substr.find(';', sni)] col = sni.rfind(':') col = col.format(':') if col > -1 else col""" return put( remote_path=conf_loc, use_sudo=use_sudo, local_path=StringIO("{orig}\n\nserver {substr}".format( orig=sio_s, substr=substr.replace( "{dns_name};\n".format(dns_name=dns_name), "{dns_name};\n{https_header}\n".format( dns_name=dns_name, https_header=_indent(https_header, 4)), 1, ), )), ) with open( resource_filename("offregister_app_push", path.join("conf", "nginx.https_header.conf")), "rt", ) as f: https_header = f.read() replaced_confs = tuple( secure_conf(dns_name, conf_loc, https_header) for dns_name, conf_loc in iteritems(hosts_d)) sudo("systemctl reload -q {service_name} --no-pager --full".format( service_name=service_name)) return {"certbot_res": certbot_res, "replaced_confs": replaced_confs}