def setup_connection_meta(self, within):
if not self.node.public_ips:
raise Exception("No public IP address, so cannot SSH")
if self.node.state == 2:
# state = 2 is terminated on AWS, need to lookup MAP/enum from lib for general solution. TODO
raise Exception("Node is terminated, so cannot SSH")
if self.node.extra is not None and "ssh_config" in self.node.extra:
# TODO: Get value for `self.env_kwargs['ssh_config_path']` and set that
ssh_config_to_fab = {
"IdentityFile": "key_filename",
"HostName": "host",
"Port": "port",
"User": "******",
}
for ssh_name, fab_name in iteritems(ssh_config_to_fab):
if ssh_name in self.node.extra["ssh_config"]:
setattr(self.env, fab_name,
self.node.extra["ssh_config"][ssh_name])
if "ssh" in self.config_provider:
if "private_key_path" in self.config_provider["ssh"]:
self.env.key_filename = (
self.env.key_filename
or self.config_provider["ssh"]["private_key_path"])
if "node_password" in self.config_provider["ssh"]:
self.env.password = self.config_provider["ssh"][
"node_password"]
assert (self.env.key_filename or self.env.password
), "Set a private key or password to have unattended deploys"
if self.env.user is None or self.env.user == "user":
self.env.user = (self.node.extra["user"] if "user"
in self.node.extra else self.guess_os_username())
if (self.env.password is None or self.env.password
== "unspecified") and "password" in self.node.extra:
self.env.password = self.node.extra["password"]
dir_or_key = (lambda d_or_k: d_or_k.directory or d_or_k.file)(
ProcessNode.get_directory_or_key(self.process_dict, within))
if ip_address(self.node.public_ips[0]).is_private:
self.dns_name = self.node.public_ips[0] # LOL
elif (not self.dns_name
and "skydns2" not in self.process_dict["register"][dir_or_key]
and "consul" not in self.process_dict["register"][dir_or_key]):
# self.dns_name = '{public_ip}.xip.io'.format(public_ip=self.node.public_ips[0])
self.dns_name = self.node.public_ips[0]
# raise Exception('No DNS name and no way of acquiring one')
self.env.hosts = [self.dns_name]
if "no_key_filename" in self.node.extra and self.node.extra[
"no_key_filename"]:
del self.env.key_filename
self.env.use_ssh_config = False
print("<env>")
pp(obj_to_d(self.env))
print("</env>")
def merge_steps(self, merge, res):
for (mod, step_name), out in iteritems(merge):
mod = mod.partition(".")[0]
if mod not in res[self.dns_name]:
res[self.dns_name][mod] = {}
if step_name not in res[self.dns_name][mod]:
res[self.dns_name][mod][step_name] = out
else:
res[self.dns_name][mod][step_name].update(out)
def _environment(kwargs):
home_dir = run("echo $HOME", quiet=True)
kwargs["WorkingDirectory"] = kwargs["GIT_DIR"]
if "{home}" in kwargs["ExecStart"]:
kwargs["ExecStart"] = kwargs["ExecStart"].format(home_dir=home_dir)
kwargs["service_name"] = kwargs["GIT_DIR"][kwargs["GIT_DIR"].rfind("/") +
1:]
kwargs["User"] = kwargs["User"] if "User" in kwargs else "root"
kwargs["Group"] = kwargs["Group"] if "Group" in kwargs else "root"
if kwargs["RDBMS_URI"]:
rdbms_uri = (kwargs["RDBMS_URI"]
if isinstance(kwargs["RDBMS_URI"], str) else "".join(
map(str, kwargs["RDBMS_URI"])))
else:
rdbms_uri = run('echo "$RDBMS_URI"')
kwargs["Environments"] = ("{}\n".format(kwargs["Environments"])
if "Environments" in kwargs else "")
kwargs[
"Environments"] += "Environment='RDBMS_URI={rdbms_uri}'\n" "Environment=PORT={port}\n".format(
rdbms_uri=rdbms_uri, port=kwargs["REST_API_PORT"])
if "DAEMON_ENV" in kwargs and kwargs["DAEMON_ENV"]:
kwargs["Environments"] += "\n".join(
"Environment='{k}={v}'".format(k=k, v=v)
for k, v in iteritems(kwargs["DAEMON_ENV"])
if not k.startswith("$$"))
if "$$ENV_JSON_FILE" in kwargs["DAEMON_ENV"]:
kwargs["Environments"] += "\n" + run(
"""python -c 'import json; f=open("{fname}");{rest}""".format(
fname=kwargs["DAEMON_ENV"]["$$ENV_JSON_FILE"],
rest=
'd=json.load(f);print chr(10).join("Environment={q}{k}={v}{q}".format(q=chr(39), k=k, v=v) for k,v in d.iteritems()); f.close()\'',
))
return kwargs
def _nginx_cerbot_setup(
domains,
https_cert_email,
conf_dirs=("/etc/nginx/sites-enabled", ),
use_sudo=True,
warn_only=True,
quiet=True,
):
if not cmd_avail("certbot"):
install()
if domains != "all":
raise NotImplementedError("{} for domains".format(domains))
run_cmd = partial(_run_command, sudo=use_sudo)
if not run("ls -A '{conf_dir}'".format(conf_dir=conf_dirs[0]),
shell_escape=False):
return "hosts_d is empty empty; skipping"
server_names_t = tuple(
chain(*(run_cmd("grep -RF server_name '{conf_dir}'".format(
conf_dir=conf_dir)).split("\n") for conf_dir in conf_dirs)))
hosts = tuple(
l.partition("127.0.0.1")[2].strip()
for l in run_cmd("grep -F 127.0.0.1 /etc/hosts").split("\n")
if "localhost" not in l)
server_names_d = dict((lambda spl: (spl[1].lstrip().rstrip("; \t\r"), spl[
0][:spl[0].rfind(":")]))(l.split("server_name"))
for l in server_names_t)
if len(server_names_d) < len(server_names_t):
raise NotImplementedError(
"Same server_name in multiple files. We don't know what to stop!")
hosts_d = {
host: server_names_d[host]
for host in hosts if host.count(".") > 1 and host in server_names_d
and len(host.translate(None, "~^|()?*")) == len(host)
}
if not hosts_d:
return "hosts_d is empty empty; skipping"
run_cmd("mkdir -p /etc/nginx/sites-disabled")
sites_avail_local_filepath = resource_filename(
"offregister_app_push", path.join("conf",
"nginx.sites-available.conf"))
def certbot_prep(dns_name, conf_loc):
run_cmd("mv '{}' '/etc/nginx/sites-disabled/{}'".format(
conf_loc,
path.split(conf_loc)[1]))
wwwroot = "/var/www/static/{dns_name}".format(dns_name=dns_name)
if exists(wwwroot):
run_cmd("rm -r '{wwwroot}'".format(wwwroot=wwwroot))
run_cmd("mkdir -p '{wwwroot}'".format(wwwroot=wwwroot))
_send_nginx_conf(
conf_remote_filename="/etc/nginx/sites-enabled/{dns_name}-certbot".
format(dns_name=dns_name),
sites_avail_local_filepath=sites_avail_local_filepath,
proxy_block_local_filepath=None,
conf_vars={
"NGINX_PORT":
80,
"DNS_NAMES": (dns_name, ),
"DESCRIPTION":
"Temporary conf doing certbot for {}".format(dns_name),
"WWWPATH":
"/",
"WWWROOT":
wwwroot,
},
)
print(
'one("{}", "{}") ='.format(dns_name, conf_loc),
"-w '{wwwroot}' -d '{dns_name}' ".format(dns_name=dns_name,
wwwroot=wwwroot),
)
return "-w '{wwwroot}' -d '{dns_name}' ".format(dns_name=dns_name,
wwwroot=wwwroot)
secured_already = (frozenset(
run_cmd("ls /etc/letsencrypt/live", warn_only=True).splitlines())
if exists("/etc/letsencrypt/live") else tuple())
cerbot_cmds = tuple(
"certbot certonly --agree-tos -m {https_cert_email} --webroot {root}".
format(https_cert_email=https_cert_email,
root=certbot_prep(dns_name, conf_loc))
for dns_name, conf_loc in iteritems(hosts_d)
if dns_name not in secured_already)
if not cerbot_cmds:
return "You must've already secured all your domains. Otherwise clean: /etc/letsencrypt/live"
service_name = "nginx"
if sudo(
"systemctl status -q {service_name} --no-pager --full".format(
service_name=service_name),
warn_only=True,
).failed:
sudo("systemctl start -q {service_name} --no-pager --full".format(
service_name=service_name))
else:
sudo("systemctl reload -q {service_name} --no-pager --full".format(
service_name=service_name))
print("cerbot_cmds =", cerbot_cmds)
certbot_res = tuple(map(run_cmd, cerbot_cmds))
sudo("cp /etc/nginx/sites-disabled/* /etc/nginx/sites-enabled")
# sudo('rm -r /etc/nginx/sites-disabled')
def secure_conf(dns_name, conf_loc, https_header):
# print 'secure_conf({!r}, {!r})'.format(dns_name, conf_loc)
if run_cmd("grep -Fq 443 {conf_loc}".format(conf_loc=conf_loc),
warn_only=True).failed:
logger.warning(
"Skipping {conf_loc}; 443 already found within".format(
conf_loc=conf_loc))
sio = StringIO()
get(remote_path=conf_loc, use_sudo=use_sudo, local_path=sio)
sio.seek(0)
sio_s = sio.read()
substr = sio_s[sio_s.find("{", sio_s.find("server")):sio_s.rfind("}") +
2].replace("listen 80", "listen 443", 1)
https_header %= {
"CA_CERT_PATH":
"/etc/letsencrypt/live/{dns_name}/fullchain.pem".format(
dns_name=dns_name),
"PRIV_KEY_PATH":
"/etc/letsencrypt/live/{dns_name}/privkey.pem".format(
dns_name=dns_name),
}
""" # TODO: Address parsing, if not in `listen` keyword
sni = substr.find('server_name')
sni = substr[sni:substr.find(';', sni)]
col = sni.rfind(':')
col = col.format(':') if col > -1 else col"""
return put(
remote_path=conf_loc,
use_sudo=use_sudo,
local_path=StringIO("{orig}\n\nserver {substr}".format(
orig=sio_s,
substr=substr.replace(
"{dns_name};\n".format(dns_name=dns_name),
"{dns_name};\n{https_header}\n".format(
dns_name=dns_name,
https_header=_indent(https_header, 4)),
1,
),
)),
)
with open(
resource_filename("offregister_app_push",
path.join("conf", "nginx.https_header.conf")),
"rt",
) as f:
https_header = f.read()
replaced_confs = tuple(
secure_conf(dns_name, conf_loc, https_header)
for dns_name, conf_loc in iteritems(hosts_d))
sudo("systemctl reload -q {service_name} --no-pager --full".format(
service_name=service_name))
return {"certbot_res": certbot_res, "replaced_confs": replaced_confs}
def offshell(name, load_system_host_keys, ssh_config, etcd):
host, port = etcd.split(":")
node = dict_to_node(
loads(etcd3.client(host=host, port=int(port)).get(name)[0]))
if node.state != NodeState.RUNNING:
raise EnvironmentError(
"Node isn't running, it's {}. Ensure it's ON, then try again.".
format(node.state))
connection_d = {
"hostname":
node.public_ips[0],
"username":
node.extra["user"] if "user" in node.extra else node.extra.get(
"ssh_config", {}).get("User"),
"password":
node.extra.get("password"),
"key_filename":
node.extra.get("ssh_config", {}).get("IdentityFile"),
}
if ssh_config:
if not connection_d["key_filename"]:
root_logger.warn(
"Cannot set password in ssh_config format. You'll still be prompted."
)
tab = " " * 4
if "ssh_config" in node.extra and len(node.extra["ssh_config"]) > 1:
host = node.extra["ssh_config"].pop("Host")
print("Host {host}\n{rest}".format(
host=host,
rest=tab + tab.join(
"{} {}\n".format(k, v[0] if isinstance(v, list) else v)
for k, v in iteritems(node.extra["ssh_config"]))[:-1],
))
else:
print("Host {name}\n"
"{tab}HostName {hostname}\n"
"{tab}User {username}\n"
"{last_line}".format(
name=name.rpartition("/")[2],
hostname=connection_d["hostname"],
username=connection_d["username"],
tab=tab,
last_line="{tab}IdentityFile {key_filename}".format(
tab=tab, key_filename=connection_d["key_filename"])
if connection_d["key_filename"] else "",
))
known_hosts = path.join(expanduser("~"), ".ssh", "known_hosts")
s = ""
if exists(known_hosts):
with open(known_hosts, "rt") as f:
s = f.read()
if connection_d["hostname"] not in s:
print(
"After checking the key fingerprint, add it to your known hosts with:"
"\nssh-keyscan {host} >> {known_hosts}".format(
host=connection_d["hostname"], known_hosts=known_hosts),
file=stderr,
)
return
client = SSHClient()
if load_system_host_keys:
client.load_system_host_keys()
client.connect(**connection_d)
chan = client.invoke_shell()
interactive_shell(chan)
chan.close()
client.close()
def prepare_cluster_obj(self, cluster, res):
cluster_args = cluster["args"] if "args" in cluster else tuple()
cluster_kwargs = update_d(
{
"domain": self.dns_name,
"node_name": self.node_name,
"public_ipv4": self.node.public_ips[-1],
"cache": {},
"cluster_name": cluster.get("cluster_name"),
},
cluster["kwargs"] if "kwargs" in cluster else {},
)
if "source" in cluster:
cluster_kwargs.update(play_source=cluster["source"])
cluster_type = cluster["module"].replace("-", "_")
cluster_path = "/".join(_f for _f in (cluster_type,
cluster_kwargs["cluster_name"])
if _f)
cluster_kwargs.update(cluster_path=cluster_path)
if "cache" not in cluster_kwargs:
cluster_kwargs["cache"] = {}
if ":" in cluster_type:
cluster_type, _, tag = cluster_type.rpartition(":")
del _
else:
tag = None
cluster_kwargs.update(tag=tag)
# create play with tasks
if "play_source" not in cluster_kwargs:
raise NotImplementedError(
"Ansible from module/playbook not implemented yet. Inline your source."
)
cluster_kwargs["play_source"].update(hosts=self.env_namedtuple.hosts)
extra_vars = {
"ansible_host": self.env_namedtuple.host,
"ansible_ssh_host": self.env_namedtuple.host,
"ansible_user": self.env_namedtuple.user,
"ansible_ssh_user": self.env_namedtuple.user,
"ansible_port": self.env_namedtuple.port,
"ansible_ssh_port": self.env_namedtuple.port,
}
if self.env.password is not None:
extra_vars.update(
{"ansible_ssh_pass": self.env_namedtuple.password})
if self.env.key_filename is None:
extra_vars.update({
"ansible_ssh_private_key_file":
self.env_namedtuple.key_filename
})
if (self.env.ssh_config is not None
and "StrictHostKeyChecking" in self.env.ssh_config):
host_key_checking = (self.env_namedtuple.
ssh_config["StrictHostKeyChecking"] == "yes")
extra_vars.update({
"ansible_ssh_extra_args":
"-o " +
" -o ".join('{k}="{v}"'.format(k=k, v=v)
for k, v in iteritems(self.env.ssh_config)
if k not in frozenset(("Port", "User", "Host"))),
"ansible_host_key_checking":
host_key_checking,
})
self.variable_manager.extra_vars = extra_vars
cluster_kwargs["play"] = Play().load(
cluster_kwargs["play_source"],
variable_manager=self.variable_manager,
loader=self.loader,
)
return PreparedClusterObj(
cluster_path=cluster_path,
cluster_type=cluster_type,
cluster_args=cluster_args,
cluster_kwargs=cluster_kwargs,
res=res,
tag=tag,
)
def install_circus2(circus_env=None,
circus_cmd=None,
circus_args=None,
circus_name=None,
circus_home=None,
circus_venv="/opt/venvs/circus",
remote_user="******",
virtual_env=None,
use_sudo=False,
*args,
**kwargs):
if (circus_cmd is None or circus_args is None or circus_name is None
or circus_home is None):
return "insufficient args, skipping circus"
virtual_env = virtual_env or "{home}/venvs/tflow".format(
home=run("echo $HOME", quiet=True))
conf_dir = "/etc/circus/conf.d" # '/'.join((taiga_root, 'config'))
sudo("mkdir -p {conf_dir}".format(conf_dir=conf_dir))
if not use_sudo:
user, group = run("echo $(id -un; id -gn)").split(" ")
sudo("mkdir -p {circus_venv} {virtual_env}".format(
circus_venv=circus_venv, virtual_env=virtual_env))
sudo("chown -R {user}:{group} {circus_venv} {virtual_env} {conf_dir}".
format(
user=user,
group=group,
circus_venv=circus_venv,
virtual_env=virtual_env,
conf_dir=conf_dir,
))
install_venv0(python3=False, virtual_env=circus_venv, use_sudo=use_sudo)
run_cmd = partial(_run_command, sudo=use_sudo)
run_cmd("mkdir -p {circus_home}/logs".format(circus_home=circus_home))
with shell_env(VIRTUAL_ENV=circus_venv,
PATH="{}/bin:$PATH".format(circus_venv)):
run_cmd("pip install circus")
py_ver = run("python --version").partition(" ")[2][:3]
upload_template(
offpy_dir("circus.ini"),
"{conf_dir}/".format(conf_dir=conf_dir),
context={
"ENDPOINT_PORT":
5555,
"WORKING_DIR":
kwargs.get("circus_working_dir", circus_home),
"CMD":
circus_cmd,
"ARGS":
circus_args,
"NAME":
circus_name,
"USER":
remote_user,
"HOME":
circus_home,
"VENV":
virtual_env,
"CIRCUS_ENV":
"" if circus_env is None else "\n".join(
"{}={}".format(k, v) for k, v in iteritems(circus_env)),
"PYTHON_VERSION":
py_ver,
},
use_sudo=use_sudo,
)
circusd_context = {"CONF_DIR": conf_dir, "CIRCUS_VENV": circus_venv}
if exists("/etc/systemd/system"):
upload_template(
offpy_dir("circusd.service"),
"/etc/systemd/system/",
context=circusd_context,
use_sudo=True,
backup=False,
)
sudo("systemctl daemon-reload")
else:
upload_template(
offpy_dir("circusd.conf"),
"/etc/init/",
context=circusd_context,
use_sudo=True,
backup=False,
)
return circus_venv
def step0(domain, *args, **kwargs):
key_file = "/root/.ssh/id_rsa.pub"
config = {
"DOKKU_HOSTNAME": ("hostname", domain),
"DOKKU_KEY_FILE": ("key_file", key_file),
"DOKKU_SKIP_KEY_FILE": ("skip_key_file", False),
"DOKKU_VHOST_ENABLE": ("vhost_enable", False),
"DOKKU_WEB_CONFIG": ("web_config", False),
}
create_static = kwargs.get("create_static_page", True)
static_git_url = kwargs.get(
"static_git", environ.get("DOKKU_STATIC_GIT",
environ.get("STATIC_GIT")))
local_pubkey = kwargs.get("PUBLIC_KEY_PATH") or environ.get(
"DOKKU_PUBLIC_KEY_PATH", environ["PUBLIC_KEY_PATH"])
if not cmd_avail("docker"):
docker.install_0()
# docker.dockeruser_1()
docker.serve_2()
put(StringIO("pZPlHOkV649DCepEwf9G"), "/tmp/passwd")
if not cmd_avail("dokku"): # is_installed('dokku'):
run("wget -qN https://packagecloud.io/gpg.key")
sudo("apt-key add gpg.key")
append(
"/etc/apt/sources.list.d/dokku.list",
"deb https://packagecloud.io/dokku/dokku/ubuntu/ trusty main",
use_sudo=True,
)
put(
StringIO("\n".join("{com} {com}/{var} {type} {val}".format(
com="dokku",
var=v[0],
val=str(v[1]).lower() if type(v[1]) is type(bool) else v[1],
type=(lambda t: {
type(True): "boolean",
type(""): "string",
type(str): "string",
}.get(t, t))(type(v[1])),
) for k, v in iteritems(config) if v[1] is not None)),
"/tmp/dokku-debconf",
)
sudo("debconf-set-selections /tmp/dokku-debconf")
if not exists(key_file):
sudo('ssh-keygen -t rsa -b 4096 -f {key_file} -N ""'.format(
key_file=key_file))
apt_depends("dokku")
sudo("dokku plugin:install-dependencies --core")
put(local_pubkey, key_file)
sudo("sshcommand acl-add dokku domain {key_file}".format(
key_file=key_file))
return "installed dokku"
if create_static:
if run("getent passwd static", quiet=True, warn_only=True).failed:
sudo("adduser static --disabled-password")
sudo("mkdir /home/static/sites/", user="******")
upload_template(
path.join(
path.dirname(
resource_filename("offregister_dokku", "__init__.py")),
"data",
"static_sites.conf",
),
"/etc/nginx/conf.d/static_sites.conf",
use_sudo=True,
)
if sudo("service nginx status").endswith("stop/waiting"):
sudo("service nginx start")
else:
sudo("service nginx reload")
# TODO: Abstract this out into a different module, and allow for multiple domains
if static_git_url:
ipv4 = "/home/static/sites/{public_ipv4}".format(
public_ipv4=kwargs["public_ipv4"])
if exists(ipv4):
sudo("rm -rf {ipv4}".format(ipv4=ipv4))
sudo("mkdir -p {ipv4}".format(ipv4=ipv4), user="******")
if domain:
domain = "/home/static/sites/{domain}".format(domain=domain)
if not exists(domain):
sudo(
"ln -s {ipv4} {domain}".format(ipv4=ipv4,
domain=domain),
user="******",
)
xip = "{ipv4}.xip.io".format(ipv4=ipv4)
if not exists(xip):
sudo("ln -s {ipv4} {xip}".format(ipv4=ipv4, xip=xip),
user="******")
if static_git_url:
apt_depends("git")
if isinstance(static_git_url, str):
clone_or_update(**url_to_git_dict(static_git_url))
else:
clone_or_update(to_dir=ipv4, **static_git_url)
return "installed dokku [already]"