def install0(*args, **kwargs):
apt_depends("git")
go.install0()
if not cmd_avail("glide"):
run("curl https://glide.sh/get | sh")
gopath = run('echo "$GOPATH"', quiet=True)
to_dir = "{gopath}/src/github.com/bradleyfalzon/gopherci".format(gopath=gopath)
clone_or_update(
"gopherci",
team="SamuelMarks",
branch="master",
to_dir=to_dir,
skip_clean=True,
skip_reset=True,
)
with cd(to_dir):
run("glide i")
run("go build && mv gopherci $GOPATH/bin")
to_dir = "{gopath}/src/github.com/bradleyfalzon/gopherci-web".format(gopath=gopath)
clone_or_update(
"gopherci-web",
team="SamuelMarks",
branch="false-stripes",
to_dir=to_dir,
skip_clean=True,
skip_reset=True,
)
with cd(to_dir):
run("glide i")
run("go build && mv gopherci-web {gopath}/bin".format(gopath=gopath))
return {"installed": ("gopherci", "gopherci-web")}
def _build_and_upload_deb(version):
sudo('apt-get update -qq')
sudo('apt-get upgrade -y')
sudo('apt-get dist-upgrade -y')
apt_depends('openjdk-7-jdk', 'build-essential', 'python-dev', 'python-boto', 'curl',
'libcurl4-nss-dev', 'libsasl2-dev', 'maven', 'libapr1-dev', 'libsvn-dev',
'git', 'autoconf')
install_ruby()
from offregister_fab_utils.apt import is_installed, Package
print '----------------------------------------------------------------------'
print "is_installed(Package('ruby', '2.3')) =", is_installed(Package('ruby', '2.3'))
apt_depends('software-properties-common')
sudo('apt-add-repository -y ppa:brightbox/ruby-ng')
apt_depends('ruby{version}'.format(version='2.3'), 'ruby{version}-dev'.format(version='2.3'))
print '----------------------------------------------------------------------'
clone_or_update(team='deric', repo='mesos-deb-packaging',
branch='master', skip_checkout=True, skip_reset=True)
pkg = 'mesos-{version}-gcc4.8-amd64.deb'.format(version=version)
with cd('mesos-deb-packaging'):
run('bash build_mesos --ref {version} --build-version p1'.format(version=version))
run('mv pkg.deb {pkg}'.format(pkg=pkg))
clone_or_update(team='offscale', repo='packages',
branch='master', skip_checkout=True, skip_reset=True)
with cd('packages'):
run('git add {pkg}'.format(pkg=pkg))
run('git commit -am "Built {pkg}"'.format(pkg=pkg))
run('git push -u origin master')
def pull0(destory_cache=True, **kwargs):
apt_depends("git")
run_cmd = partial(_run_command, sudo=kwargs.get("use_sudo"))
cache = not destory_cache and exists(kwargs["GIT_DIR"])
if cache:
dirnam = run_cmd("mktemp -d --suffix '{name}'".format(
name=kwargs["GIT_DIR"][kwargs["GIT_DIR"].rfind("/") + 1:]))
run_cmd(
"""while read -r l; do [ -e "$l" ] && mv "$l" '{dirnam}' & done <'{git_dir}/.gitignore' """
.format(dirnam=dirnam, git_dir=kwargs["GIT_DIR"]))
clone_or_update(
repo=kwargs["GIT_REPO"],
to_dir=kwargs["GIT_DIR"],
use_sudo=kwargs.get("use_sudo", False),
branch=kwargs.get("GIT_BRANCH", "master"),
skip_reset=kwargs.get("skip_reset", False),
cmd_runner=run_cmd,
)
if cache:
run_cmd(
"""while read -r l; do d="{dirnam}/$l"; [ -e "$d" ] && cp -r "$d" '{git_dir}' & done <'{git_dir}/.gitignore' """
.format(dirnam=dirnam, git_dir=kwargs["GIT_DIR"]))
run_cmd("rm -rf {dirnam}".format(dirnam=dirnam))
return "[git] Updated"
def install_plugin(repo_team, repo_name, location=None):
apt_depends('git')
cmd = 'dokku'
if not cmd_avail(cmd):
raise EnvironmentError('Install {cmd} before installing plugins'.format(cmd=cmd))
with cd('/var/lib/dokku/plugins'):
clone_or_update(team=repo_team, repo=repo_name, to_dir=location or repo_name)
run('dokku plugins-install')
def git_static1(*args, **kwargs):
server_dir = _get_server_dir(**kwargs)
clone_or_update(
repo=kwargs["GIT_REPO"],
to_dir=server_dir,
branch=kwargs.get("GIT_BRANCH", "master"),
use_sudo=True,
)
def install_plugin(repo_team, repo_name, location=None):
apt_depends("git")
cmd = "dokku"
if not cmd_avail(cmd):
raise EnvironmentError(
"Install {cmd} before installing plugins".format(cmd=cmd))
with cd("/var/lib/dokku/plugins"):
clone_or_update(team=repo_team,
repo=repo_name,
to_dir=location or repo_name)
run("dokku plugins-install")
def dl_install_opencv(branch='2.4', extra_cmake_args=''):
apt_depends('build-essential', 'cmake', 'git', 'libgtk2.0-dev', 'pkg-config', 'libavcodec-dev',
'libavformat-dev', 'libswscale-dev', 'python-dev', 'python-numpy', 'libtbb2',
'libtbb-dev', 'libjpeg-dev', 'libpng-dev', 'libtiff-dev', 'libjasper-dev', 'libdc1394-22-dev')
run("mkdir -p '$HOME/repos'", shell_escape=False)
with cd('$HOME/repos'):
clone_or_update(repo='opencv', team='opencv', branch=branch, skip_reset=True)
run('mkdir -p opencv-build')
with cd('opencv-build'):
run('cmake -D CMAKE_BUILD_TYPE=RELEASE '
'-D CMAKE_INSTALL_PREFIX=/usr/local {} ../opencv'.format(extra_cmake_args))
run('make')
sudo('make install')
def push0(c, **kwargs):
apt_depends(c, "git")
clone_or_update(
c,
repo=kwargs["GIT_REPO"],
to_dir=kwargs["GIT_DIR"],
use_sudo=kwargs.get("use_sudo", False),
branch=kwargs.get("GIT_BRANCH", "master"),
skip_reset=kwargs.get("skip_reset", False),
skip_clean=kwargs.get("skip_clean", True),
reset_to_first=kwargs.get("GIT_RESET_TO_FIRST", False),
git_env=kwargs.get("GIT_ENV"),
)
def install0():
sudo(
"""psql "$RDBMS_URI" -c 'create table entries
(id serial primary key,
key varchar(255) not null,
value text not null,
expiration int,
unique(key));' """,
user="******",
shell_escape=False,
)
{
"GIT_DIR": "/var/www/static/glaucoma-risk-calculator-rest-api",
"GIT_REPO": "https://github.com/glaucoma-australia/glaucoma-risk-calculator-rest-api",
"service_name": "glaucoma-risk-calculator-rest-api",
"skip_reset": False,
"destroy_node_modules": True,
"use_sudo": True,
"node_sudo": False,
"node_version": "8.6.0",
"ExecStart": "/bin/bash -c 'PATH={home_dir}/n/bin:$PATH {home_dir}/n/bin/node main.js'",
"nginx": True,
"app_name": "glaucoma-risk-calculator",
"DESCRIPTION": "Glaucoma risk calculator frontend and backend",
"DNS_NAMES": ["paste.complicated.io"],
"PROXY_ROUTE": "/api",
"PROXY_PASS": "******",
"REST_API_PORT": 5465,
"NGINX_PORT": 80,
"WWWROOT": "/var/www/static/glaucoma-risk-calculator-web-frontend-dist/dist",
"WWWPATH": "/",
"nginx_secure": "certbot",
"https_cert_email": "*****@*****.**",
}
clone_or_update(
repo="seejohnrun",
to_dir="~/repos/haste-server",
use_sudo=False,
branch="master",
skip_reset=False,
)
def install_frontend(skip_apt_update=False):
apt_depends('git')
remote_home = run('printf $HOME', quiet=True)
with cd(remote_home):
repo = 'taiga-front'
clone_or_update(team='taigaio', repo=repo)
# Compile it here if you prefer
if not exists('taiga-front/dist'):
clone_or_update(team='taigaio', repo='taiga-front-dist')
run('ln -s $HOME/taiga-front-dist/dist $HOME/taiga-front/dist')
js_conf_dir = '/'.join((repo, 'dist', 'js'))
if not exists('/'.join((js_conf_dir, 'conf.json'))):
run('mkdir -p {conf_dir}'.format(conf_dir=js_conf_dir))
put(taiga_dir('conf.json'), js_conf_dir)
run('mkdir -p $HOME/logs')
install_nginx(taiga_dir('taiga.nginx.conf'), name='taiga',
service_cmd='status', template_vars={'HOME': remote_home})
def install_backend(database=True, database_uri=None):
if database:
if not exists('$HOME/.setup/postgresql'):
remote_user = run('printf $USER')
run('mkdir -p $HOME/.setup')
run('touch $HOME/.setup/postgresql')
install_postgres(dbs=('taiga', remote_user), users=(remote_user,))
elif not database_uri:
raise ValueError('Must create database or provide database_uri')
apt_depends('git')
with cd(run('printf $HOME')):
install_python_taiga_deps(clone_or_update(team='taigaio', repo='taiga-back'))
def core_install_vulcand(*args, **kwargs):
clone_or_update(team='coreos', repo='unit-examples')
def _install_backend(
backend_root,
name,
remote_user,
backend_virtual_env,
team,
repo,
database_uri,
env_vars,
install_postgres=True,
create_postgres_database=True,
use_sudo=True,
):
uname = run("uname -v")
is_ubuntu = "Ubuntu" in uname
run_cmd = partial(_run_command, sudo=use_sudo)
if install_postgres:
if not is_ubuntu and not cmd_avail("psql"):
raise NotImplementedError("Postgres install on {!r}".format(uname))
postgres.install0()
if create_postgres_database:
parsed_database_uri = urlparse(database_uri)
created = postgres.setup_users(
create=({
"user": parsed_database_uri.username,
"password": parsed_database_uri.password,
"dbname": parsed_database_uri.path[1:],
}, ),
connection_str=database_uri,
)
assert created is not None
clone_or_update(team=team,
repo=repo,
use_sudo=use_sudo,
to_dir=backend_root,
branch="master")
offregister_python_ubuntu.install_venv0(virtual_env=backend_virtual_env,
python3=True,
packages=("gunicorn", "uwsgi"))
offregister_python_ubuntu.install_package1(package_directory=backend_root,
virtual_env=backend_virtual_env)
if not exists("/etc/systemd/system"):
raise NotImplementedError("Non SystemD platforms")
if run("id {remote_user}".format(remote_user=remote_user),
warn_only=True,
quiet=True).failed:
sudo('adduser {remote_user} --disabled-password --quiet --gecos ""'.
format(remote_user=remote_user))
(uid, user), (gid, group) = get_user_group_tuples(remote_user)
uwsgi_service = "/etc/systemd/system/{name}-uwsgi.service".format(
name=name)
upload_template(
circus_dir("uwsgi.service"),
uwsgi_service,
context={
"USER": user,
"GROUP": group,
"{}_BACK".format(name.upper()): "{}/{}".format(backend_root, name),
"UID": uid,
"GID": gid,
"VENV": backend_virtual_env,
"BACKEND_ROOT": backend_root,
"SERVICE_USER": "******",
"NAME": name,
"ENV": env_vars,
},
use_sudo=True,
backup=False,
mode=0o644,
)
grp = sudo("id -gn", quiet=True)
sudo("chown {grp}:{grp} {uwsgi_service}".format(
grp=grp, uwsgi_service=uwsgi_service))
restart_systemd("{name}-uwsgi".format(name=name))
return backend_virtual_env, database_uri
def core_install_vulcand(*args, **kwargs):
clone_or_update(team="coreos", repo="unit-examples")
def step0(domain, *args, **kwargs):
key_file = "/root/.ssh/id_rsa.pub"
config = {
"DOKKU_HOSTNAME": ("hostname", domain),
"DOKKU_KEY_FILE": ("key_file", key_file),
"DOKKU_SKIP_KEY_FILE": ("skip_key_file", False),
"DOKKU_VHOST_ENABLE": ("vhost_enable", False),
"DOKKU_WEB_CONFIG": ("web_config", False),
}
create_static = kwargs.get("create_static_page", True)
static_git_url = kwargs.get(
"static_git", environ.get("DOKKU_STATIC_GIT",
environ.get("STATIC_GIT")))
local_pubkey = kwargs.get("PUBLIC_KEY_PATH") or environ.get(
"DOKKU_PUBLIC_KEY_PATH", environ["PUBLIC_KEY_PATH"])
if not cmd_avail("docker"):
docker.install_0()
# docker.dockeruser_1()
docker.serve_2()
put(StringIO("pZPlHOkV649DCepEwf9G"), "/tmp/passwd")
if not cmd_avail("dokku"): # is_installed('dokku'):
run("wget -qN https://packagecloud.io/gpg.key")
sudo("apt-key add gpg.key")
append(
"/etc/apt/sources.list.d/dokku.list",
"deb https://packagecloud.io/dokku/dokku/ubuntu/ trusty main",
use_sudo=True,
)
put(
StringIO("\n".join("{com} {com}/{var} {type} {val}".format(
com="dokku",
var=v[0],
val=str(v[1]).lower() if type(v[1]) is type(bool) else v[1],
type=(lambda t: {
type(True): "boolean",
type(""): "string",
type(str): "string",
}.get(t, t))(type(v[1])),
) for k, v in iteritems(config) if v[1] is not None)),
"/tmp/dokku-debconf",
)
sudo("debconf-set-selections /tmp/dokku-debconf")
if not exists(key_file):
sudo('ssh-keygen -t rsa -b 4096 -f {key_file} -N ""'.format(
key_file=key_file))
apt_depends("dokku")
sudo("dokku plugin:install-dependencies --core")
put(local_pubkey, key_file)
sudo("sshcommand acl-add dokku domain {key_file}".format(
key_file=key_file))
return "installed dokku"
if create_static:
if run("getent passwd static", quiet=True, warn_only=True).failed:
sudo("adduser static --disabled-password")
sudo("mkdir /home/static/sites/", user="******")
upload_template(
path.join(
path.dirname(
resource_filename("offregister_dokku", "__init__.py")),
"data",
"static_sites.conf",
),
"/etc/nginx/conf.d/static_sites.conf",
use_sudo=True,
)
if sudo("service nginx status").endswith("stop/waiting"):
sudo("service nginx start")
else:
sudo("service nginx reload")
# TODO: Abstract this out into a different module, and allow for multiple domains
if static_git_url:
ipv4 = "/home/static/sites/{public_ipv4}".format(
public_ipv4=kwargs["public_ipv4"])
if exists(ipv4):
sudo("rm -rf {ipv4}".format(ipv4=ipv4))
sudo("mkdir -p {ipv4}".format(ipv4=ipv4), user="******")
if domain:
domain = "/home/static/sites/{domain}".format(domain=domain)
if not exists(domain):
sudo(
"ln -s {ipv4} {domain}".format(ipv4=ipv4,
domain=domain),
user="******",
)
xip = "{ipv4}.xip.io".format(ipv4=ipv4)
if not exists(xip):
sudo("ln -s {ipv4} {xip}".format(ipv4=ipv4, xip=xip),
user="******")
if static_git_url:
apt_depends("git")
if isinstance(static_git_url, str):
clone_or_update(**url_to_git_dict(static_git_url))
else:
clone_or_update(to_dir=ipv4, **static_git_url)
return "installed dokku [already]"
def step0(domain, *args, **kwargs):
key_file = '/root/.ssh/id_rsa.pub'
config = {
'DOKKU_HOSTNAME': ('hostname', domain),
'DOKKU_KEY_FILE': ('key_file', key_file),
'DOKKU_SKIP_KEY_FILE': ('skip_key_file', False),
'DOKKU_VHOST_ENABLE': ('vhost_enable', False),
'DOKKU_WEB_CONFIG': ('web_config', False)
}
create_static = kwargs.get('create_static_page', True)
static_git_url = kwargs.get('static_git', environ.get('DOKKU_STATIC_GIT', environ.get('STATIC_GIT')))
local_pubkey = kwargs.get('PUBLIC_KEY_PATH') or environ.get('DOKKU_PUBLIC_KEY_PATH', environ['PUBLIC_KEY_PATH'])
if not cmd_avail('docker'):
docker.install_0()
# docker.dockeruser_1()
docker.serve_2()
put(StringIO('pZPlHOkV649DCepEwf9G'), '/tmp/passwd')
if not cmd_avail('dokku'): # is_installed('dokku'):
run('wget -qN https://packagecloud.io/gpg.key')
sudo('apt-key add gpg.key')
append('/etc/apt/sources.list.d/dokku.list',
'deb https://packagecloud.io/dokku/dokku/ubuntu/ trusty main', use_sudo=True)
put(StringIO('\n'.join('{com} {com}/{var} {type} {val}'.format(
com='dokku', var=v[0], val=str(v[1]).lower() if type(v[1]) is BooleanType else v[1], type=(
lambda t: {type(True): 'boolean', type(''): 'string', type(unicode): 'string'}.get(t, t))(type(v[1])))
for k, v in config.iteritems() if v[1] is not None)
),
'/tmp/dokku-debconf')
sudo('debconf-set-selections /tmp/dokku-debconf')
if not exists(key_file):
sudo('ssh-keygen -t rsa -b 4096 -f {key_file} -N ""'.format(key_file=key_file))
apt_depends('dokku')
sudo('dokku plugin:install-dependencies --core')
put(local_pubkey, key_file)
sudo('sshcommand acl-add dokku domain {key_file}'.format(key_file=key_file))
return 'installed dokku'
if create_static:
if run('getent passwd static', quiet=True, warn_only=True).failed:
sudo('adduser static --disabled-password')
sudo('mkdir /home/static/sites/', user='******')
upload_template(
path.join(path.dirname(resource_filename('offregister_dokku', '__init__.py')), 'data', 'static_sites.conf'),
'/etc/nginx/conf.d/static_sites.conf', use_sudo=True
)
if sudo('service nginx status').endswith('stop/waiting'):
sudo('service nginx start')
else:
sudo('service nginx reload')
# TODO: Abstract this out into a different module, and allow for multiple domains
if static_git_url:
ipv4 = '/home/static/sites/{public_ipv4}'.format(public_ipv4=kwargs['public_ipv4'])
if exists(ipv4):
sudo('rm -rf {ipv4}'.format(ipv4=ipv4))
sudo('mkdir -p {ipv4}'.format(ipv4=ipv4), user='******')
if domain:
domain = '/home/static/sites/{domain}'.format(domain=domain)
if not exists(domain):
sudo('ln -s {ipv4} {domain}'.format(ipv4=ipv4, domain=domain), user='******')
xip = '{ipv4}.xip.io'.format(ipv4=ipv4)
if not exists(xip):
sudo('ln -s {ipv4} {xip}'.format(ipv4=ipv4, xip=xip), user='******')
if static_git_url:
apt_depends('git')
if isinstance(static_git_url, basestring):
clone_or_update(**url_to_git_dict(static_git_url))
else:
clone_or_update(to_dir=ipv4, **static_git_url)
return 'installed dokku [already]'
def dl_install_opencv(branch="master", tag="4.1.1", extra_cmake_args=""):
distro = run("lsb_release -cs", quiet=True)
if run(
"grep '^[[:blank:]]*[^[:blank:]#;]' /etc/apt/sources.list | grep -qF '-security main'",
warn_only=True,
).failed:
sudo(
'add-apt-repository -y "deb http://security.ubuntu.com/ubuntu {distro}-security main"'
.format(distro=distro))
sudo("apt update")
apt_depends(
"build-essential",
"checkinstall",
"cmake",
"pkg-config",
"yasm",
"git",
"gfortran",
"libjpeg8-dev",
"libpng-dev",
"software-properties-common",
"libtiff-dev",
"libavcodec-dev",
"libavformat-dev",
"libswscale-dev",
"libdc1394-22-dev",
"libxine2-dev",
"libv4l-dev",
"libgstreamer1.0-dev",
"libgstreamer-plugins-base1.0-dev",
"libgtk2.0-dev",
"libtbb-dev",
"qt5-default",
"libatlas-base-dev",
"libfaac-dev",
"libmp3lame-dev",
"libtheora-dev",
"libvorbis-dev",
"libxvidcore-dev",
"libopencore-amrnb-dev",
"libopencore-amrwb-dev",
"libavresample-dev",
"x264",
"v4l-utils",
"libprotobuf-dev",
"protobuf-compiler",
"libgoogle-glog-dev",
"libgflags-dev",
"libgphoto2-dev",
"libeigen3-dev",
"libhdf5-dev",
"doxygen",
"python3-dev",
"python3-pip",
"python3-numpy",
"python3-testresources",
"libeigen3-dev",
# Hmm:
"libopencv-dev",
"opencv-data",
)
run("mkdir -p '$HOME/repos'", shell_escape=False)
with cd("$HOME/repos"):
clone_or_update(repo="opencv",
team="opencv",
branch=branch,
tag=tag,
skip_reset=True)
run("mkdir -p opencv-build")
with cd("opencv-build"):
run("cmake -D CMAKE_BUILD_TYPE=RELEASE "
"-D CMAKE_INSTALL_PREFIX=/usr/local {} ../opencv".format(
extra_cmake_args))
run("make")
sudo("make install")