#!/usr/bin/env python
# This script will generate some certificates using flocker-ca and upload them
# to the servers specified in a cluster.yml
import sys
import yaml
# Usage: deploy.py cluster.yml
from utils import Configurator
if __name__ == "__main__":
c = Configurator(configFile=sys.argv[1])
c.run("flocker-ca initialize %s" % (c.config["cluster_name"],))
print "Initialized cluster CA."
c.run("flocker-ca create-control-certificate %s" % (c.config["control_node"],))
print "Created control cert."
node_mapping = {}
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# Created 8eab4b8d-c0a2-4ce2-80aa-0709277a9a7a.crt. Copy ...
uuid = c.run("flocker-ca create-node-certificate").split(".")[0].split(" ")[1]
node_mapping[public_ip] = uuid
print "Generated", uuid, "for", public_ip
for user in c.config["users"]:
c.run("flocker-ca create-api-certificate %s" % (user,))
print "Created user key for", user
print "Uploading keys to respective nodes:"
# Copy cluster cert, and control cert and key to control node.
c.runSSHRaw(c.config["control_node"], "mkdir -p /etc/flocker")
def main(reactor, configFile):
c = Configurator(configFile=configFile)
if c.config["os"] == "ubuntu":
user = "******"
elif c.config["os"] == "centos":
user = "******"
elif c.config["os"] == "amazon":
user = "******"
# Gather IPs of all nodes
nodes = c.config["agent_nodes"]
node_public_ips = [n["public"] for n in nodes]
node_public_ips.append(c.config["control_node"])
# Wait for all nodes to boot
yield gatherResults([verify_socket(ip, 22, timeout=600) for ip in node_public_ips])
log("Generating API certs")
# generate and upload plugin.crt and plugin.key for each node
for public_ip in node_public_ips:
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip,))
log("Generated api certs for", public_ip)
deferreds = []
log("Uploading api certs...")
for public_ip in node_public_ips:
# upload the .crt and .key
for ext in ("crt", "key"):
d = c.scp("%s-plugin.%s" % (public_ip, ext,),
public_ip, "/etc/flocker/api.%s" % (ext,), async=True)
d.addCallback(report_completion, public_ip=public_ip, message=" * Uploaded api cert for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Uploaded api certs")
deferreds = []
for public_ip in node_public_ips:
cmd = """echo
cat > /etc/flocker/env << EOF
FLOCKER_CONTROL_SERVICE_HOST=%s\n
FLOCKER_CONTROL_SERVICE_PORT=4523\n
FLOCKER_CONTROL_SERVICE_CA_FILE=/etc/flocker/cluster.crt\n
FLOCKER_CONTROL_SERVICE_CLIENT_KEY_FILE=/etc/flocker/api.key\n
FLOCKER_CONTROL_SERVICE_CLIENT_CERT_FILE=/etc/flocker/api.crt
EOF""" % c.config['control_node']
d = c.runSSHAsync(public_ip, cmd)
d.addCallback(report_completion, public_ip=public_ip, message="Enabled flocker ENVs for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Uploaded Flocker ENV file.")
deferreds = []
for public_ip in node_public_ips:
cmd = """echo
sed -i -e 's,/usr/bin/kubelet,/root/kubelet,g' /etc/systemd/system/kubelet.service;
sed -i '/\[Service\]/aEnvironmentFile=/etc/flocker/env' /etc/systemd/system/kubelet.service
"""
d = c.runSSHAsync(public_ip, cmd)
d.addCallback(report_completion, public_ip=public_ip, message="Configured flocker ENVs for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Configured Flocker ENV file.")
deferreds = []
for public_ip in node_public_ips:
cmd = """echo
wget https://storage.googleapis.com/kubernetes-release/release/v1.1.1/bin/linux/amd64/kubelet;
chmod +x kubelet;
systemctl daemon-reload;
systemctl restart kubelet;
"""
d = c.runSSHAsync(public_ip, cmd)
d.addCallback(report_completion, public_ip=public_ip, message="Restarted Kubelet for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Restarted Kubelet")
log("Completed")
def main(reactor, configFile):
c = Configurator(configFile=configFile)
control_ip = c.config["control_node"]
log("Generating plugin certs")
# generate and upload plugin.crt and plugin.key for each node
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip,))
log("Generated plugin certs for", public_ip)
def report_completion(result, public_ip, message="Completed plugin install for"):
log(message, public_ip)
return result
deferreds = []
log("Uploading plugin certs...")
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# upload the .crt and .key
for ext in ("crt", "key"):
d = c.scp("%s-plugin.%s" % (public_ip, ext,),
public_ip, "/etc/flocker/plugin.%s" % (ext,), async=True)
d.addCallback(report_completion, public_ip=public_ip, message=" * Uploaded plugin cert for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Uploaded plugin certs")
log("Installing flocker plugin")
# loop each agent and get the plugin installed/running
# clone the plugin and configure an upstart/systemd unit for it to run
deferreds = []
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
log("Using %s => %s" % (public_ip, private_ip))
# the full api path to the control service
controlservice = 'https://%s:4523/v1' % (control_ip,)
# perhaps the user has pre-compiled images with the plugin
# downloaded and installed
if not settings["SKIP_INSTALL_PLUGIN"]:
if c.config["os"] == "ubuntu":
log("Installing plugin for", public_ip, "...")
d = c.runSSHAsync(public_ip,
"apt-get install -y --force-yes clusterhq-flocker-docker-plugin && "
"service flocker-docker-plugin restart")
d.addCallback(report_completion, public_ip=public_ip)
deferreds.append(d)
elif c.config["os"] == "centos":
log("Installing plugin for", public_ip, "...")
d = c.runSSHAsync(public_ip,
"yum install -y clusterhq-flocker-docker-plugin && "
"systemctl enable flocker-docker-plugin && "
"systemctl start flocker-docker-plugin")
d.addCallback(report_completion, public_ip=public_ip)
deferreds.append(d)
else:
log("Skipping installing plugin: %r" % (settings["SKIP_INSTALL_PLUGIN"],))
yield gatherResults(deferreds)
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
# ensure that the /run/docker/plugins
# folder exists
log("Creating the /run/docker/plugins folder")
c.runSSHRaw(public_ip, "mkdir -p /run/docker/plugins")
if c.config["os"] == "coreos":
log("Starting flocker-docker-plugin as docker container on CoreOS on %s" % (public_ip,))
c.runSSH(public_ip, """echo
docker run --restart=always -d --net=host --privileged \\
-e FLOCKER_CONTROL_SERVICE_BASE_URL=%s \\
-e MY_NETWORK_IDENTITY=%s \\
-v /etc/flocker:/etc/flocker \\
-v /run/docker:/run/docker \\
--name=flocker-docker-plugin \\
clusterhq/flocker-docker-plugin""" % (controlservice, private_ip,))
log("Done!")
def main(reactor, configFile):
c = Configurator(configFile=configFile)
control_ip = c.config["control_node"]
log("Generating plugin certs")
# generate and upload plugin.crt and plugin.key for each node
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip, ))
log("Generated plugin certs for", public_ip)
def report_completion(result,
public_ip,
message="Completed plugin install for"):
log(message, public_ip)
return result
deferreds = []
log("Uploading plugin certs...")
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# upload the .crt and .key
for ext in ("crt", "key"):
d = c.scp("%s-plugin.%s" % (
public_ip,
ext,
),
public_ip,
"/etc/flocker/plugin.%s" % (ext, ),
async=True)
d.addCallback(report_completion,
public_ip=public_ip,
message=" * Uploaded plugin cert for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Uploaded plugin certs")
log("Installing flocker plugin")
# loop each agent and get the plugin installed/running
# clone the plugin and configure an upstart/systemd unit for it to run
deferreds = []
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
log("Using %s => %s" % (public_ip, private_ip))
# the full api path to the control service
controlservice = 'https://%s:4523/v1' % (control_ip, )
# perhaps the user has pre-compiled images with the plugin
# downloaded and installed
if not settings["SKIP_INSTALL_PLUGIN"]:
if c.config["os"] == "ubuntu":
log("Installing plugin for", public_ip, "...")
d = c.runSSHAsync(
public_ip,
"apt-get install -y --force-yes clusterhq-flocker-docker-plugin && "
"service flocker-docker-plugin restart")
d.addCallback(report_completion, public_ip=public_ip)
deferreds.append(d)
elif c.config["os"] == "centos":
log("Installing plugin for", public_ip, "...")
d = c.runSSHAsync(
public_ip,
"yum install -y clusterhq-flocker-docker-plugin && "
"systemctl enable flocker-docker-plugin && "
"systemctl start flocker-docker-plugin")
d.addCallback(report_completion, public_ip=public_ip)
deferreds.append(d)
else:
log("Skipping installing plugin: %r" %
(settings["SKIP_INSTALL_PLUGIN"], ))
yield gatherResults(deferreds)
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
# ensure that the /run/docker/plugins
# folder exists
log("Creating the /run/docker/plugins folder")
c.runSSHRaw(public_ip, "mkdir -p /run/docker/plugins")
if c.config["os"] == "coreos":
log("Starting flocker-docker-plugin as docker container on CoreOS on %s"
% (public_ip, ))
c.runSSH(
public_ip, """echo
docker run --restart=always -d --net=host --privileged \\
-e FLOCKER_CONTROL_SERVICE_BASE_URL=%s \\
-e MY_NETWORK_IDENTITY=%s \\
-v /etc/flocker:/etc/flocker \\
-v /run/docker:/run/docker \\
--name=flocker-docker-plugin \\
clusterhq/flocker-docker-plugin""" % (
controlservice,
private_ip,
))
log("Done!")
def main():
c = Configurator(configFile=sys.argv[1])
control_ip = c.config["control_node"]
# download and replace the docker binary on each of the nodes
for node in c.config["agent_nodes"]:
# don't download a new docker for reasons only the user knows
if settings["SKIP_DOCKER_BINARY"]:
break
public_ip = node["public"]
print "Replacing docker binary on %s" % (public_ip, )
# stop the docker service
print "Stopping the docker service on %s - %s" \
% (public_ip, settings['DOCKER_SERVICE_NAME'],)
if c.config["os"] == "ubuntu":
c.runSSHRaw(
public_ip,
"stop %s || true" % (settings['DOCKER_SERVICE_NAME'], ))
elif c.config["os"] == "centos":
c.runSSHRaw(
public_ip, "systemctl stop %s.service || true" %
(settings['DOCKER_SERVICE_NAME'], ))
# download the latest docker binary\
print "Downloading the latest docker binary on %s - %s" \
% (public_ip, settings['DOCKER_BINARY_URL'],)
c.runSSHRaw(
public_ip,
"wget -O /usr/bin/docker %s" % (settings['DOCKER_BINARY_URL'], ))
if c.config["os"] == "ubuntu":
# newer versions of docker insist on AUFS on ubuntu, probably for good reason.
c.runSSHRaw(
public_ip, "DEBIAN_FRONTEND=noninteractive "
"'apt-get install -y linux-image-extra-$(uname -r)'")
# start the docker service
print "Starting the docker service on %s" % (public_ip, )
if c.config["os"] == "ubuntu":
c.runSSHRaw(public_ip,
"start %s" % (settings['DOCKER_SERVICE_NAME'], ))
elif c.config["os"] == "centos":
c.runSSHRaw(
public_ip, "systemctl start %s.service" %
(settings['DOCKER_SERVICE_NAME'], ))
print "Generating plugin certs"
# generate and upload plugin.crt and plugin.key for each node
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip, ))
print "Generated plugin certs for", public_ip
# upload the .crt and .key
for ext in ("crt", "key"):
c.scp("%s-plugin.%s" % (
public_ip,
ext,
), public_ip, "/etc/flocker/plugin.%s" % (ext, ))
print "Uploaded plugin certs for", public_ip
print "Installing flocker plugin"
# loop each agent and get the plugin installed/running
# clone the plugin and configure an upstart/systemd unit for it to run
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
# the full api path to the control service
controlservice = 'https://%s:4523/v1' % (control_ip, )
# perhaps the user has pre-compiled images with the plugin
# downloaded and installed
if not settings["SKIP_INSTALL_PLUGIN"]:
if c.config["os"] == "ubuntu":
print c.runSSHRaw(
public_ip, "apt-get install -y "
"python-pip python-dev build-essential "
"libssl-dev libffi-dev")
elif c.config["os"] == "centos":
print c.runSSHRaw(
public_ip, "yum install -y "
"python-pip python-devel "
"gcc libffi-devel python-devel openssl-devel")
# pip install the plugin
print c.runSSHRaw(
public_ip, "pip install git+%s@%s" % (
settings['PLUGIN_REPO'],
settings['PLUGIN_BRANCH'],
))
else:
print "Skipping installing plugin: %r" % (
settings["SKIP_INSTALL_PLUGIN"], )
# ensure that the /usr/share/docker/plugins
# folder exists
print "Creating the /usr/share/docker/plugins folder"
c.runSSHRaw(public_ip, "mkdir -p /usr/share/docker/plugins")
# configure an upstart job that runs the bash script
if c.config["os"] == "ubuntu":
print "Writing flocker-docker-plugin upstart job to %s" % (
public_ip, )
c.runSSH(
public_ip, """cat <<EOF > /etc/init/flocker-docker-plugin.conf
# flocker-plugin - flocker-docker-plugin job file
description "Flocker Plugin service"
author "ClusterHQ <*****@*****.**>"
respawn
env FLOCKER_CONTROL_SERVICE_BASE_URL=%s
env MY_NETWORK_IDENTITY=%s
exec /usr/local/bin/flocker-docker-plugin
EOF
service flocker-docker-plugin restart
""" % (
controlservice,
private_ip,
))
# configure a systemd job that runs the bash script
elif c.config["os"] == "centos":
print "Writing flocker-docker-plugin systemd job to %s" % (
public_ip, )
c.runSSH(
public_ip, """# writing flocker-docker-plugin systemd
cat <<EOF > /etc/systemd/system/flocker-docker-plugin.service
[Unit]
Description=flocker-plugin - flocker-docker-plugin job file
[Service]
Environment=FLOCKER_CONTROL_SERVICE_BASE_URL=%s
Environment=MY_NETWORK_IDENTITY=%s
ExecStart=/usr/local/bin/flocker-docker-plugin
[Install]
WantedBy=multi-user.target
EOF
systemctl enable flocker-docker-plugin.service
systemctl start flocker-docker-plugin.service
""" % (
controlservice,
private_ip,
))
def main(reactor, configFile):
c = Configurator(configFile=configFile)
if c.config["os"] == "ubuntu":
user = "******"
elif c.config["os"] == "centos":
user = "******"
elif c.config["os"] == "amazon":
user = "******"
# Gather IPs of all nodes
nodes = c.config["agent_nodes"]
node_public_ips = [n["public"] for n in nodes]
node_public_ips.append(c.config["control_node"])
# Wait for all nodes to boot
yield gatherResults(
[verify_socket(ip, 22, timeout=600) for ip in node_public_ips])
log("Generating API certs")
# generate and upload plugin.crt and plugin.key for each node
for public_ip in node_public_ips:
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip, ))
log("Generated api certs for", public_ip)
deferreds = []
log("Uploading api certs...")
for public_ip in node_public_ips:
# upload the .crt and .key
for ext in ("crt", "key"):
d = c.scp("%s-plugin.%s" % (
public_ip,
ext,
),
public_ip,
"/etc/flocker/api.%s" % (ext, ),
async=True)
d.addCallback(report_completion,
public_ip=public_ip,
message=" * Uploaded api cert for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Uploaded api certs")
deferreds = []
for public_ip in node_public_ips:
cmd = """echo
cat > /etc/flocker/env << EOF
FLOCKER_CONTROL_SERVICE_HOST=%s\n
FLOCKER_CONTROL_SERVICE_PORT=4523\n
FLOCKER_CONTROL_SERVICE_CA_FILE=/etc/flocker/cluster.crt\n
FLOCKER_CONTROL_SERVICE_CLIENT_KEY_FILE=/etc/flocker/api.key\n
FLOCKER_CONTROL_SERVICE_CLIENT_CERT_FILE=/etc/flocker/api.crt
EOF""" % c.config['control_node']
d = c.runSSHAsync(public_ip, cmd)
d.addCallback(report_completion,
public_ip=public_ip,
message="Enabled flocker ENVs for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Uploaded Flocker ENV file.")
deferreds = []
for public_ip in node_public_ips:
cmd = """echo
sed -i -e 's,/usr/bin/kubelet,/root/kubelet,g' /etc/systemd/system/kubelet.service;
sed -i '/\[Service\]/aEnvironmentFile=/etc/flocker/env' /etc/systemd/system/kubelet.service
"""
d = c.runSSHAsync(public_ip, cmd)
d.addCallback(report_completion,
public_ip=public_ip,
message="Configured flocker ENVs for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Configured Flocker ENV file.")
deferreds = []
for public_ip in node_public_ips:
cmd = """echo
wget https://storage.googleapis.com/kubernetes-release/release/v1.1.1/bin/linux/amd64/kubelet;
chmod +x kubelet;
systemctl daemon-reload;
systemctl restart kubelet;
"""
d = c.runSSHAsync(public_ip, cmd)
d.addCallback(report_completion,
public_ip=public_ip,
message="Restarted Kubelet for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Restarted Kubelet")
log("Completed")
def main():
c = Configurator(configFile=sys.argv[1])
c.run("flocker-ca initialize %s" % (c.config["cluster_name"],))
print "Initialized cluster CA."
c.run("flocker-ca create-control-certificate %s" % (c.config["control_node"],))
print "Created control cert."
node_mapping = {}
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# Created 8eab4b8d-c0a2-4ce2-80aa-0709277a9a7a.crt. Copy ...
uuid = c.run("flocker-ca create-node-certificate").split(".")[0].split(" ")[1]
node_mapping[public_ip] = uuid
print "Generated", uuid, "for", public_ip
for user in c.config["users"]:
c.run("flocker-ca create-api-certificate %s" % (user,))
print "Created user key for", user
print "Uploading keys to respective nodes:"
# Copy cluster cert, and control cert and key to control node.
c.runSSHRaw(c.config["control_node"], "mkdir -p /etc/flocker")
c.scp("cluster.crt", c.config["control_node"], "/etc/flocker/cluster.crt")
print " * Uploaded cluster cert to control node."
for ext in ("crt", "key"):
c.scp("control-%s.%s" % (c.config["control_node"], ext),
c.config["control_node"], "/etc/flocker/control-service.%s" % (ext,))
print " * Uploaded control cert & key to control node."
# Dump agent_config into a file and scp it to /etc/flocker/agent.yml on the
# nodes.
f = open("agent.yml", "w")
agent_config = yaml.dump(c.config["agent_config"], f)
f.close()
# Record the node mapping for later.
f = open("node_mapping.yml", "w")
agent_config = yaml.dump(node_mapping, f)
f.close()
# Copy cluster cert, and agent cert and key to agent nodes.
for node, uuid in node_mapping.iteritems():
c.runSSHRaw(node, "mkdir -p /etc/flocker")
c.scp("cluster.crt", node, "/etc/flocker/cluster.crt")
c.scp("agent.yml", node, "/etc/flocker/agent.yml")
print " * Uploaded cluster cert to %s." % (node,)
for ext in ("crt", "key"):
c.scp("%s.%s" % (uuid, ext), node, "/etc/flocker/node.%s" % (ext,))
print " * Uploaded node cert and key to %s." % (node,)
for node, uuid in node_mapping.iteritems():
if c.config["os"] == "ubuntu":
c.runSSH(node, """apt-get -y install apt-transport-https software-properties-common
service flocker-container-agent restart
service flocker-dataset-agent restart
""")
elif c.config["os"] == "centos":
c.runSSH(node, """if selinuxenabled; then setenforce 0; fi
systemctl enable docker.service
systemctl start docker.service
""")
if c.config["os"] == "ubuntu":
c.runSSH(c.config["control_node"], """cat <<EOF > /etc/init/flocker-control.override
start on runlevel [2345]
stop on runlevel [016]
EOF
echo 'flocker-control-api 4523/tcp # Flocker Control API port' >> /etc/services
echo 'flocker-control-agent 4524/tcp # Flocker Control Agent port' >> /etc/services
service flocker-control restart
ufw allow flocker-control-api
ufw allow flocker-control-agent
""")
elif c.config["os"] == "centos":
c.runSSH(c.config["control_node"], """systemctl enable flocker-control
systemctl start flocker-control
firewall-cmd --permanent --add-service flocker-control-api
firewall-cmd --add-service flocker-control-api
firewall-cmd --permanent --add-service flocker-control-agent
firewall-cmd --add-service flocker-control-agent
""")
print "Configured and started control service, opened firewall."
if c.config["users"]:
print "\nYou should now be able to communicate with the control service, for example:\n"
prefix = ("curl -s --cacert $PWD/cluster.crt --cert $PWD/%(user)s.crt "
"--key $PWD/%(user)s.key" % dict(user=c.config["users"][0],))
url = "https://%(control_node)s:4523/v1" % dict(control_node=c.config["control_node"],)
header = ' --header "Content-type: application/json"'
print "This should give you a list of your nodes:"
print prefix + " " + url + "/state/nodes | jq ."
print "Try running tutorial.py cluster.yml for more..."
def main(reactor, *args):
c = Configurator(configFile=sys.argv[1])
c.run("flocker-ca initialize %s" % (c.config["cluster_name"],))
log("Initialized cluster CA.")
c.run("flocker-ca create-control-certificate %s" % (c.config["control_node"],))
log("Created control cert.")
node_mapping = {}
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# Created 8eab4b8d-c0a2-4ce2-80aa-0709277a9a7a.crt. Copy ...
uuid = c.run("flocker-ca create-node-certificate").split(".")[0].split(" ")[1]
node_mapping[public_ip] = uuid
log("Generated", uuid, "for", public_ip)
for user in c.config["users"]:
c.run("flocker-ca create-api-certificate %s" % (user,))
log("Created user key for", user)
# Dump agent_config into a file and scp it to /etc/flocker/agent.yml on the
# nodes.
f = open("agent.yml", "w")
yaml.dump(c.config["agent_config"], f)
f.close()
# Record the node mapping for later.
f = open("node_mapping.yml", "w")
yaml.dump(node_mapping, f)
f.close()
log("Making /etc/flocker directory on all nodes")
deferreds = []
for node, uuid in node_mapping.iteritems():
deferreds.append(c.runSSHAsync(node, "mkdir -p /etc/flocker"))
deferreds.append(c.runSSHAsync(c.config["control_node"], "mkdir -p /etc/flocker"))
yield gatherResults(deferreds)
log("Uploading keys to respective nodes:")
deferreds = []
# Copy cluster cert, and control cert and key to control node.
d = c.scp("cluster.crt", c.config["control_node"], "/etc/flocker/cluster.crt", async=True)
d.addCallback(report_completion, public_ip=c.config["control_node"], message=" * Uploaded cluster cert to")
deferreds.append(d)
for ext in ("crt", "key"):
d = c.scp("control-%s.%s" % (c.config["control_node"], ext),
c.config["control_node"], "/etc/flocker/control-service.%s" % (ext,), async=True)
d.addCallback(report_completion, public_ip=c.config["control_node"], message=" * Uploaded control %s to" % (ext,))
deferreds.append(d)
log(" * Uploaded control cert & key to control node.")
# Copy cluster cert, and agent cert and key to agent nodes.
deferreds = []
for node, uuid in node_mapping.iteritems():
d = c.scp("cluster.crt", node, "/etc/flocker/cluster.crt", async=True)
d.addCallback(report_completion, public_ip=node, message=" * Uploaded cluster cert to")
deferreds.append(d)
d = c.scp("agent.yml", node, "/etc/flocker/agent.yml", async=True)
d.addCallback(report_completion, public_ip=node, message=" * Uploaded agent.yml to")
deferreds.append(d)
for ext in ("crt", "key"):
d = c.scp("%s.%s" % (uuid, ext), node, "/etc/flocker/node.%s" % (ext,), async=True)
d.addCallback(report_completion, public_ip=node, message=" * Uploaded node %s to" % (ext,))
deferreds.append(d)
yield gatherResults(deferreds)
deferreds = []
for node, uuid in node_mapping.iteritems():
if c.config["os"] == "ubuntu":
d = c.runSSHAsync(node, """echo "starting flocker-container-agent..."
service flocker-container-agent start
echo "starting flocker-dataset-agent..."
service flocker-dataset-agent start
""")
elif c.config["os"] == "centos":
d = c.runSSHAsync(node, """if selinuxenabled; then setenforce 0; fi
systemctl enable docker.service
systemctl start docker.service
""")
elif c.config["os"] == "coreos":
d = c.runSSHAsync(node, """echo
echo > /tmp/flocker-command-log
docker %(early_docker_prefix)s run --restart=always -d --net=host --privileged \\
-v /etc/flocker:/etc/flocker \\
-v /var/run/docker.sock:/var/run/docker.sock \\
--name=flocker-container-agent \\
clusterhq/flocker-container-agent
docker %(early_docker_prefix)s run --restart=always -d --net=host --privileged \\
-e DEBUG=1 \\
-v /tmp/flocker-command-log:/tmp/flocker-command-log \\
-v /flocker:/flocker -v /:/host -v /etc/flocker:/etc/flocker \\
-v /dev:/dev \\
--name=flocker-dataset-agent \\
clusterhq/flocker-dataset-agent
""" % dict(early_docker_prefix=EARLY_DOCKER_PREFIX))
deferreds.append(d)
if c.config["os"] == "ubuntu":
d = c.runSSHAsync(c.config["control_node"], """cat <<EOF > /etc/init/flocker-control.override
start on runlevel [2345]
stop on runlevel [016]
EOF
echo 'flocker-control-api 4523/tcp # Flocker Control API port' >> /etc/services
echo 'flocker-control-agent 4524/tcp # Flocker Control Agent port' >> /etc/services
service flocker-control restart
ufw allow flocker-control-api
ufw allow flocker-control-agent
""")
elif c.config["os"] == "centos":
d = c.runSSHAsync(c.config["control_node"], """systemctl enable flocker-control
systemctl start flocker-control
firewall-cmd --permanent --add-service flocker-control-api
firewall-cmd --add-service flocker-control-api
firewall-cmd --permanent --add-service flocker-control-agent
firewall-cmd --add-service flocker-control-agent
""")
elif c.config["os"] == "coreos":
d = c.runSSHAsync(c.config["control_node"], """echo
docker %(early_docker_prefix)s run --name=flocker-control-volume -v /var/lib/flocker clusterhq/flocker-control-service true
docker %(early_docker_prefix)s run --restart=always -d --net=host -v /etc/flocker:/etc/flocker --volumes-from=flocker-control-volume --name=flocker-control-service clusterhq/flocker-control-service""" % dict(early_docker_prefix=EARLY_DOCKER_PREFIX))
deferreds.append(d)
yield gatherResults(deferreds)
if c.config["os"] == "ubuntu":
# XXX INSECURE, UNSUPPORTED, UNDOCUMENTED EXPERIMENTAL OPTION
# Usage: `uft-flocker-config --ubuntu-aws --swarm`, I guess
if len(sys.argv) > 2 and sys.argv[2] == "--swarm":
# Install swarm
deferreds = []
clusterid = c.runSSH(c.config["control_node"], """
docker run swarm create""").strip()
log("Created Swarm ID")
for node in c.config["agent_nodes"]:
d = c.runSSHAsync(node['public'], """
service docker stop
docker daemon -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375 >> /tmp/dockerlogs 2>&1 &
""")
# Let daemon come up
time.sleep(3)
d = c.runSSHAsync(node['public'], """
docker run -d swarm join --addr=%s:2375 token://%s
""" % (node['private'], clusterid))
log("Started Swarm Agent for %s" % node['public'])
deferreds.append(d)
d = c.runSSHAsync(c.config["control_node"], """
docker run -d -p 2357:2375 swarm manage token://%s
""" % clusterid)
log("Starting Swarm Master")
deferreds.append(d)
yield gatherResults(deferreds)
log("Swarm Master is at tcp://%s:2357" % c.config["control_node"])
value = os.environ.get(field)
if value is None:
value = settings_defaults[field]
settings[field] = value
if __name__ == "__main__":
c = Configurator(configFile=sys.argv[1])
control_ip = c.config["control_node"]
print "Generating plugin certs"
# generate and upload plugin.crt and plugin.key for each node
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip,))
print "Generated plugin certs for", public_ip
# upload the .crt and .key
for ext in ("crt", "key"):
c.scp("%s-plugin.%s" % (public_ip, ext,),
public_ip, "/etc/flocker/plugin.%s" % (ext,))
print "Uploaded plugin certs for", public_ip
print "Installing flocker plugin"
# loop each agent and get the plugin installed/running
# clone the plugin and configure an upstart/systemd unit for it to run
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
# we need this so we know what folder to cd into
plugin_repo_folder = settings['PLUGIN_REPO'].split('/').pop()
def main():
c = Configurator(configFile=sys.argv[1])
control_ip = c.config["control_node"]
# download and replace the docker binary on each of the nodes
for node in c.config["agent_nodes"]:
# don't download a new docker for reasons only the user knows
if settings["SKIP_DOCKER_BINARY"]:
break
public_ip = node["public"]
print "Replacing docker binary on %s" % (public_ip,)
# stop the docker service
print "Stopping the docker service on %s - %s" \
% (public_ip, settings['DOCKER_SERVICE_NAME'],)
if c.config["os"] == "ubuntu":
c.runSSHRaw(public_ip, "stop %s || true"
% (settings['DOCKER_SERVICE_NAME'],))
elif c.config["os"] == "centos":
c.runSSHRaw(public_ip, "systemctl stop %s.service || true"
% (settings['DOCKER_SERVICE_NAME'],))
# download the latest docker binary\
print "Downloading the latest docker binary on %s - %s" \
% (public_ip, settings['DOCKER_BINARY_URL'],)
c.runSSHRaw(public_ip, "wget -O /usr/bin/docker %s"
% (settings['DOCKER_BINARY_URL'],))
if c.config["os"] == "ubuntu":
# newer versions of docker insist on AUFS on ubuntu, probably for good reason.
c.runSSHRaw(public_ip, "DEBIAN_FRONTEND=noninteractive "
"'apt-get install -y linux-image-extra-$(uname -r)'")
# start the docker service
print "Starting the docker service on %s" % (public_ip,)
if c.config["os"] == "ubuntu":
c.runSSHRaw(public_ip, "start %s"
% (settings['DOCKER_SERVICE_NAME'],))
elif c.config["os"] == "centos":
c.runSSHRaw(public_ip, "systemctl start %s.service"
% (settings['DOCKER_SERVICE_NAME'],))
print "Generating plugin certs"
# generate and upload plugin.crt and plugin.key for each node
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip,))
print "Generated plugin certs for", public_ip
# upload the .crt and .key
for ext in ("crt", "key"):
c.scp("%s-plugin.%s" % (public_ip, ext,),
public_ip, "/etc/flocker/plugin.%s" % (ext,))
print "Uploaded plugin certs for", public_ip
print "Installing flocker plugin"
# loop each agent and get the plugin installed/running
# clone the plugin and configure an upstart/systemd unit for it to run
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
# the full api path to the control service
controlservice = 'https://%s:4523/v1' % (control_ip,)
# perhaps the user has pre-compiled images with the plugin
# downloaded and installed
if not settings["SKIP_INSTALL_PLUGIN"]:
if c.config["os"] == "ubuntu":
print c.runSSHRaw(public_ip,
"apt-get install -y "
"python-pip python-dev build-essential "
"libssl-dev libffi-dev")
elif c.config["os"] == "centos":
print c.runSSHRaw(public_ip,
"yum install -y "
"python-pip python-devel "
"gcc libffi-devel python-devel openssl-devel")
# pip install the plugin
print c.runSSHRaw(public_ip, "pip install git+%s@%s"
% (settings['PLUGIN_REPO'], settings['PLUGIN_BRANCH'],))
else:
print "Skipping installing plugin: %r" % (settings["SKIP_INSTALL_PLUGIN"],)
# ensure that the /usr/share/docker/plugins
# folder exists
print "Creating the /usr/share/docker/plugins folder"
c.runSSHRaw(public_ip, "mkdir -p /usr/share/docker/plugins")
# configure an upstart job that runs the bash script
if c.config["os"] == "ubuntu":
print "Writing flocker-docker-plugin upstart job to %s" % (public_ip,)
c.runSSH(public_ip, """cat <<EOF > /etc/init/flocker-docker-plugin.conf
# flocker-plugin - flocker-docker-plugin job file
description "Flocker Plugin service"
author "ClusterHQ <*****@*****.**>"
respawn
env FLOCKER_CONTROL_SERVICE_BASE_URL=%s
env MY_NETWORK_IDENTITY=%s
exec /usr/local/bin/flocker-docker-plugin
EOF
service flocker-docker-plugin restart
""" % (controlservice, private_ip,))
# configure a systemd job that runs the bash script
elif c.config["os"] == "centos":
print "Writing flocker-docker-plugin systemd job to %s" % (public_ip,)
c.runSSH(public_ip, """# writing flocker-docker-plugin systemd
cat <<EOF > /etc/systemd/system/flocker-docker-plugin.service
[Unit]
Description=flocker-plugin - flocker-docker-plugin job file
[Service]
Environment=FLOCKER_CONTROL_SERVICE_BASE_URL=%s
Environment=MY_NETWORK_IDENTITY=%s
ExecStart=/usr/local/bin/flocker-docker-plugin
[Install]
WantedBy=multi-user.target
EOF
systemctl enable flocker-docker-plugin.service
systemctl start flocker-docker-plugin.service
""" % (controlservice, private_ip,))
def main(reactor, *args):
c = Configurator(configFile=sys.argv[1])
c.run("flocker-ca initialize %s" % (c.config["cluster_name"], ))
log("Initialized cluster CA.")
c.run("flocker-ca create-control-certificate %s" %
(c.config["control_node"], ))
log("Created control cert.")
node_mapping = {}
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# Created 8eab4b8d-c0a2-4ce2-80aa-0709277a9a7a.crt. Copy ...
uuid = c.run("flocker-ca create-node-certificate").split(".")[0].split(
" ")[1]
node_mapping[public_ip] = uuid
log("Generated", uuid, "for", public_ip)
for user in c.config["users"]:
c.run("flocker-ca create-api-certificate %s" % (user, ))
log("Created user key for", user)
# Dump agent_config into a file and scp it to /etc/flocker/agent.yml on the
# nodes.
f = open("agent.yml", "w")
yaml.dump(c.config["agent_config"], f)
f.close()
# Record the node mapping for later.
f = open("node_mapping.yml", "w")
yaml.dump(node_mapping, f)
f.close()
log("Making /etc/flocker directory on all nodes")
deferreds = []
for node, uuid in node_mapping.iteritems():
deferreds.append(c.runSSHAsync(node, "mkdir -p /etc/flocker"))
deferreds.append(
c.runSSHAsync(c.config["control_node"], "mkdir -p /etc/flocker"))
yield gatherResults(deferreds)
log("Uploading keys to respective nodes:")
deferreds = []
# Copy cluster cert, and control cert and key to control node.
d = c.scp("cluster.crt",
c.config["control_node"],
"/etc/flocker/cluster.crt",
async=True)
d.addCallback(report_completion,
public_ip=c.config["control_node"],
message=" * Uploaded cluster cert to")
deferreds.append(d)
for ext in ("crt", "key"):
d = c.scp("control-%s.%s" % (c.config["control_node"], ext),
c.config["control_node"],
"/etc/flocker/control-service.%s" % (ext, ),
async=True)
d.addCallback(report_completion,
public_ip=c.config["control_node"],
message=" * Uploaded control %s to" % (ext, ))
deferreds.append(d)
log(" * Uploaded control cert & key to control node.")
# Copy cluster cert, and agent cert and key to agent nodes.
deferreds = []
for node, uuid in node_mapping.iteritems():
d = c.scp("cluster.crt", node, "/etc/flocker/cluster.crt", async=True)
d.addCallback(report_completion,
public_ip=node,
message=" * Uploaded cluster cert to")
deferreds.append(d)
d = c.scp("agent.yml", node, "/etc/flocker/agent.yml", async=True)
d.addCallback(report_completion,
public_ip=node,
message=" * Uploaded agent.yml to")
deferreds.append(d)
for ext in ("crt", "key"):
d = c.scp("%s.%s" % (uuid, ext),
node,
"/etc/flocker/node.%s" % (ext, ),
async=True)
d.addCallback(report_completion,
public_ip=node,
message=" * Uploaded node %s to" % (ext, ))
deferreds.append(d)
yield gatherResults(deferreds)
deferreds = []
for node, uuid in node_mapping.iteritems():
if c.config["os"] == "ubuntu":
d = c.runSSHAsync(
node, """echo "starting flocker-container-agent..."
service flocker-container-agent start
echo "starting flocker-dataset-agent..."
service flocker-dataset-agent start
""")
elif c.config["os"] == "centos":
d = c.runSSHAsync(
node, """if selinuxenabled; then setenforce 0; fi
systemctl enable docker.service
systemctl start docker.service
""")
elif c.config["os"] == "coreos":
d = c.runSSHAsync(
node, """echo
echo > /tmp/flocker-command-log
docker %(early_docker_prefix)s run --restart=always -d --net=host --privileged \\
-v /etc/flocker:/etc/flocker \\
-v /var/run/docker.sock:/var/run/docker.sock \\
--name=flocker-container-agent \\
clusterhq/flocker-container-agent
docker %(early_docker_prefix)s run --restart=always -d --net=host --privileged \\
-e DEBUG=1 \\
-v /tmp/flocker-command-log:/tmp/flocker-command-log \\
-v /flocker:/flocker -v /:/host -v /etc/flocker:/etc/flocker \\
-v /dev:/dev \\
--name=flocker-dataset-agent \\
clusterhq/flocker-dataset-agent
""" % dict(early_docker_prefix=EARLY_DOCKER_PREFIX))
deferreds.append(d)
if c.config["os"] == "ubuntu":
d = c.runSSHAsync(
c.config["control_node"],
"""cat <<EOF > /etc/init/flocker-control.override
start on runlevel [2345]
stop on runlevel [016]
EOF
echo 'flocker-control-api 4523/tcp # Flocker Control API port' >> /etc/services
echo 'flocker-control-agent 4524/tcp # Flocker Control Agent port' >> /etc/services
service flocker-control restart
ufw allow flocker-control-api
ufw allow flocker-control-agent
""")
elif c.config["os"] == "centos":
d = c.runSSHAsync(
c.config["control_node"], """systemctl enable flocker-control
systemctl start flocker-control
firewall-cmd --permanent --add-service flocker-control-api
firewall-cmd --add-service flocker-control-api
firewall-cmd --permanent --add-service flocker-control-agent
firewall-cmd --add-service flocker-control-agent
""")
elif c.config["os"] == "coreos":
d = c.runSSHAsync(
c.config["control_node"], """echo
docker %(early_docker_prefix)s run --name=flocker-control-volume -v /var/lib/flocker clusterhq/flocker-control-service true
docker %(early_docker_prefix)s run --restart=always -d --net=host -v /etc/flocker:/etc/flocker --volumes-from=flocker-control-volume --name=flocker-control-service clusterhq/flocker-control-service"""
% dict(early_docker_prefix=EARLY_DOCKER_PREFIX))
deferreds.append(d)
yield gatherResults(deferreds)
if c.config["os"] == "ubuntu":
# XXX INSECURE, UNSUPPORTED, UNDOCUMENTED EXPERIMENTAL OPTION
# Usage: `uft-flocker-config --ubuntu-aws --swarm`, I guess
if len(sys.argv) > 2 and sys.argv[2] == "--swarm":
# Install swarm
deferreds = []
clusterid = c.runSSH(c.config["control_node"], """
docker run swarm create""").strip()
log("Created Swarm ID")
for node in c.config["agent_nodes"]:
d = c.runSSHAsync(
node['public'], """
service docker stop
docker daemon -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375 >> /tmp/dockerlogs 2>&1 &
""")
# Let daemon come up
time.sleep(3)
d = c.runSSHAsync(
node['public'], """
docker run -d swarm join --addr=%s:2375 token://%s
""" % (node['private'], clusterid))
log("Started Swarm Agent for %s" % node['public'])
deferreds.append(d)
d = c.runSSHAsync(
c.config["control_node"], """
docker run -d -p 2357:2375 swarm manage token://%s
""" % clusterid)
log("Starting Swarm Master")
deferreds.append(d)
yield gatherResults(deferreds)
log("Swarm Master is at tcp://%s:2357" % c.config["control_node"])
def main():
c = Configurator(configFile=sys.argv[1])
c.run("flocker-ca initialize %s" % (c.config["cluster_name"], ))
print "Initialized cluster CA."
c.run("flocker-ca create-control-certificate %s" %
(c.config["control_node"], ))
print "Created control cert."
node_mapping = {}
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# Created 8eab4b8d-c0a2-4ce2-80aa-0709277a9a7a.crt. Copy ...
uuid = c.run("flocker-ca create-node-certificate").split(".")[0].split(
" ")[1]
node_mapping[public_ip] = uuid
print "Generated", uuid, "for", public_ip
for user in c.config["users"]:
c.run("flocker-ca create-api-certificate %s" % (user, ))
print "Created user key for", user
print "Uploading keys to respective nodes:"
# Copy cluster cert, and control cert and key to control node.
c.runSSHRaw(c.config["control_node"], "mkdir -p /etc/flocker")
c.scp("cluster.crt", c.config["control_node"], "/etc/flocker/cluster.crt")
print " * Uploaded cluster cert to control node."
for ext in ("crt", "key"):
c.scp("control-%s.%s" % (c.config["control_node"], ext),
c.config["control_node"],
"/etc/flocker/control-service.%s" % (ext, ))
print " * Uploaded control cert & key to control node."
# Dump agent_config into a file and scp it to /etc/flocker/agent.yml on the
# nodes.
f = open("agent.yml", "w")
agent_config = yaml.dump(c.config["agent_config"], f)
f.close()
# Record the node mapping for later.
f = open("node_mapping.yml", "w")
agent_config = yaml.dump(node_mapping, f)
f.close()
# Copy cluster cert, and agent cert and key to agent nodes.
for node, uuid in node_mapping.iteritems():
c.runSSHRaw(node, "mkdir -p /etc/flocker")
c.scp("cluster.crt", node, "/etc/flocker/cluster.crt")
c.scp("agent.yml", node, "/etc/flocker/agent.yml")
print " * Uploaded cluster cert to %s." % (node, )
for ext in ("crt", "key"):
c.scp("%s.%s" % (uuid, ext), node,
"/etc/flocker/node.%s" % (ext, ))
print " * Uploaded node cert and key to %s." % (node, )
for node, uuid in node_mapping.iteritems():
if c.config["os"] == "ubuntu":
c.runSSH(
node,
"""apt-get -y install apt-transport-https software-properties-common
service flocker-container-agent restart
service flocker-dataset-agent restart
""")
elif c.config["os"] == "centos":
c.runSSH(
node, """if selinuxenabled; then setenforce 0; fi
systemctl enable docker.service
systemctl start docker.service
""")
if c.config["os"] == "ubuntu":
c.runSSH(
c.config["control_node"],
"""cat <<EOF > /etc/init/flocker-control.override
start on runlevel [2345]
stop on runlevel [016]
EOF
echo 'flocker-control-api 4523/tcp # Flocker Control API port' >> /etc/services
echo 'flocker-control-agent 4524/tcp # Flocker Control Agent port' >> /etc/services
service flocker-control restart
ufw allow flocker-control-api
ufw allow flocker-control-agent
""")
elif c.config["os"] == "centos":
c.runSSH(
c.config["control_node"], """systemctl enable flocker-control
systemctl start flocker-control
firewall-cmd --permanent --add-service flocker-control-api
firewall-cmd --add-service flocker-control-api
firewall-cmd --permanent --add-service flocker-control-agent
firewall-cmd --add-service flocker-control-agent
""")
print "Configured and started control service, opened firewall."
if c.config["users"]:
print "\nYou should now be able to communicate with the control service, for example:\n"
prefix = ("curl -s --cacert $PWD/cluster.crt --cert $PWD/%(user)s.crt "
"--key $PWD/%(user)s.key" %
dict(user=c.config["users"][0], ))
url = "https://%(control_node)s:4523/v1" % dict(
control_node=c.config["control_node"], )
header = ' --header "Content-type: application/json"'
print "This should give you a list of your nodes:"
print prefix + " " + url + "/state/nodes | jq ."
print "Try running tutorial.py cluster.yml for more..."
def main(reactor, args):
c = Configurator(configFile=sys.argv[1])
c.run("flocker-ca initialize %s" % (c.config["cluster_name"],))
log("Initialized cluster CA.")
c.run("flocker-ca create-control-certificate %s" % (c.config["control_node"],))
log("Created control cert.")
node_mapping = {}
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# Created 8eab4b8d-c0a2-4ce2-80aa-0709277a9a7a.crt. Copy ...
uuid = c.run("flocker-ca create-node-certificate").split(".")[0].split(" ")[1]
node_mapping[public_ip] = uuid
log("Generated", uuid, "for", public_ip)
for user in c.config["users"]:
c.run("flocker-ca create-api-certificate %s" % (user,))
log("Created user key for", user)
# Dump agent_config into a file and scp it to /etc/flocker/agent.yml on the
# nodes.
f = open("agent.yml", "w")
yaml.dump(c.config["agent_config"], f)
f.close()
# Record the node mapping for later.
f = open("node_mapping.yml", "w")
yaml.dump(node_mapping, f)
f.close()
log("Making /etc/flocker directory on all nodes")
deferreds = []
for node, uuid in node_mapping.iteritems():
deferreds.append(c.runSSHAsync(node, "mkdir -p /etc/flocker"))
deferreds.append(c.runSSHAsync(c.config["control_node"], "mkdir -p /etc/flocker"))
yield gatherResults(deferreds)
log("Uploading keys to respective nodes:")
deferreds = []
# Copy cluster cert, and control cert and key to control node.
d = c.scp("cluster.crt", c.config["control_node"], "/etc/flocker/cluster.crt", async=True)
d.addCallback(report_completion, public_ip=c.config["control_node"], message=" * Uploaded cluster cert to")
deferreds.append(d)
for ext in ("crt", "key"):
d = c.scp("control-%s.%s" % (c.config["control_node"], ext),
c.config["control_node"], "/etc/flocker/control-service.%s" % (ext,), async=True)
d.addCallback(report_completion, public_ip=c.config["control_node"], message=" * Uploaded control %s to" % (ext,))
deferreds.append(d)
log(" * Uploaded control cert & key to control node.")
# Copy cluster cert, and agent cert and key to agent nodes.
deferreds = []
for node, uuid in node_mapping.iteritems():
d = c.scp("cluster.crt", node, "/etc/flocker/cluster.crt", async=True)
d.addCallback(report_completion, public_ip=node, message=" * Uploaded cluster cert to")
deferreds.append(d)
d = c.scp("agent.yml", node, "/etc/flocker/agent.yml", async=True)
d.addCallback(report_completion, public_ip=node, message=" * Uploaded agent.yml to")
deferreds.append(d)
for ext in ("crt", "key"):
d = c.scp("%s.%s" % (uuid, ext), node, "/etc/flocker/node.%s" % (ext,), async=True)
d.addCallback(report_completion, public_ip=node, message=" * Uploaded node %s to" % (ext,))
deferreds.append(d)
yield gatherResults(deferreds)
deferreds = []
for node, uuid in node_mapping.iteritems():
if c.config["os"] == "ubuntu":
d = c.runSSHAsync(node, """echo "starting flocker-container-agent..."
service flocker-container-agent start
echo "starting flocker-dataset-agent..."
service flocker-dataset-agent start
""")
elif c.config["os"] == "centos":
d = c.runSSHAsync(node, """if selinuxenabled; then setenforce 0; fi
systemctl enable docker.service
systemctl start docker.service
""")
elif c.config["os"] == "coreos":
d = c.runSSHAsync(node, """echo
echo > /tmp/flocker-command-log
docker run --restart=always -d --net=host --privileged \\
-v /etc/flocker:/etc/flocker \\
-v /var/run/docker.sock:/var/run/docker.sock \\
--name=flocker-container-agent \\
clusterhq/flocker-container-agent
docker run --restart=always -d --net=host --privileged \\
-e DEBUG=1 \\
-v /tmp/flocker-command-log:/tmp/flocker-command-log \\
-v /flocker:/flocker -v /:/host -v /etc/flocker:/etc/flocker \\
-v /dev:/dev \\
--name=flocker-dataset-agent \\
clusterhq/flocker-dataset-agent
""")
deferreds.append(d)
if c.config["os"] == "ubuntu":
d = c.runSSHAsync(c.config["control_node"], """cat <<EOF > /etc/init/flocker-control.override
start on runlevel [2345]
stop on runlevel [016]
EOF
echo 'flocker-control-api 4523/tcp # Flocker Control API port' >> /etc/services
echo 'flocker-control-agent 4524/tcp # Flocker Control Agent port' >> /etc/services
service flocker-control restart
ufw allow flocker-control-api
ufw allow flocker-control-agent
""")
elif c.config["os"] == "centos":
d = c.runSSHAsync(c.config["control_node"], """systemctl enable flocker-control
systemctl start flocker-control
firewall-cmd --permanent --add-service flocker-control-api
firewall-cmd --add-service flocker-control-api
firewall-cmd --permanent --add-service flocker-control-agent
firewall-cmd --add-service flocker-control-agent
""")
elif c.config["os"] == "coreos":
d = c.runSSHAsync(c.config["control_node"], """echo
docker run --name=flocker-control-volume -v /var/lib/flocker clusterhq/flocker-control-service true
docker run --restart=always -d --net=host -v /etc/flocker:/etc/flocker --volumes-from=flocker-control-volume --name=flocker-control-service clusterhq/flocker-control-service""")
deferreds.append(d)
yield gatherResults(deferreds)
def main(reactor, configFile):
c = Configurator(configFile=configFile)
control_ip = c.config["control_node"]
# download and replace the docker binary on each of the nodes
for node in c.config["agent_nodes"]:
if c.config["os"] != "coreos":
log(
"Skipping installing new docker binary because we're on",
"ubuntu/centos, assuming we installed a sufficiently recent one",
"already.")
break
# only install new docker binary on coreos. XXX TODO coreos > 801.0.0
# doesn't need newer docker.
if settings["SKIP_DOCKER_BINARY"] or c.config["os"] != "coreos":
break
public_ip = node["public"]
log("Replacing docker binary on %s" % (public_ip, ))
# stop the docker service
log("Stopping the docker service on %s" % (public_ip, ))
if c.config["os"] == "ubuntu":
c.runSSHRaw(
public_ip,
"stop %s || true" % (settings['DOCKER_SERVICE_NAME'], ))
elif c.config["os"] == "centos":
c.runSSHRaw(
public_ip, "systemctl stop %s.service || true" %
(settings['DOCKER_SERVICE_NAME'], ))
elif c.config["os"] == "coreos":
c.runSSHRaw(public_ip, "systemctl stop docker.service || true")
# download the latest docker binary
if c.config["os"] == "coreos":
log("Downloading the latest docker binary on %s - %s" \
% (public_ip, settings['DOCKER_BINARY_URL'],))
c.runSSHRaw(public_ip, "mkdir -p /root/bin")
c.runSSHRaw(
public_ip, "wget -qO /root/bin/docker %s" %
(settings['DOCKER_BINARY_URL'], ))
c.runSSHRaw(public_ip, "chmod +x /root/bin/docker")
c.runSSHRaw(public_ip,
"cp /usr/lib/coreos/dockerd /root/bin/dockerd")
c.runSSHRaw(
public_ip,
"cp /usr/lib/systemd/system/docker.service /etc/systemd/system/"
)
c.runSSHRaw(
public_ip,
"sed -i s@/usr/lib/coreos@/root/bin@g /etc/systemd/system/docker.service"
)
c.runSSHRaw(
public_ip,
"sed -i \\'s@exec docker@exec /root/bin/docker@g\\' /root/bin/dockerd"
)
c.runSSHRaw(public_ip, "systemctl daemon-reload")
else:
log("Downloading the latest docker binary on %s - %s" \
% (public_ip, settings['DOCKER_BINARY_URL'],))
c.runSSHRaw(
public_ip, "wget -O /usr/bin/docker %s" %
(settings['DOCKER_BINARY_URL'], ))
# start the docker service
log("Starting the docker service on %s" % (public_ip, ))
if c.config["os"] == "ubuntu":
c.runSSHRaw(public_ip,
"start %s" % (settings['DOCKER_SERVICE_NAME'], ))
elif c.config["os"] == "centos":
c.runSSHRaw(
public_ip, "systemctl start %s.service" %
(settings['DOCKER_SERVICE_NAME'], ))
elif c.config["os"] == "coreos":
c.runSSHRaw(public_ip, "systemctl start docker.service")
log("Generating plugin certs")
# generate and upload plugin.crt and plugin.key for each node
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip, ))
log("Generated plugin certs for", public_ip)
def report_completion(result,
public_ip,
message="Completed plugin install for"):
log(message, public_ip)
return result
deferreds = []
log("Uploading plugin certs...")
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# upload the .crt and .key
for ext in ("crt", "key"):
d = c.scp("%s-plugin.%s" % (
public_ip,
ext,
),
public_ip,
"/etc/flocker/plugin.%s" % (ext, ),
async=True)
d.addCallback(report_completion,
public_ip=public_ip,
message=" * Uploaded plugin cert for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Uploaded plugin certs")
log("Installing flocker plugin")
# loop each agent and get the plugin installed/running
# clone the plugin and configure an upstart/systemd unit for it to run
deferreds = []
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
log("Using %s => %s" % (public_ip, private_ip))
# the full api path to the control service
controlservice = 'https://%s:4523/v1' % (control_ip, )
# perhaps the user has pre-compiled images with the plugin
# downloaded and installed
if not settings["SKIP_INSTALL_PLUGIN"]:
if c.config["os"] == "ubuntu":
log("Installing plugin for", public_ip, "...")
d = c.runSSHAsync(
public_ip,
"apt-get install -y --force-yes clusterhq-flocker-docker-plugin && "
"service flocker-docker-plugin restart")
d.addCallback(report_completion, public_ip=public_ip)
deferreds.append(d)
elif c.config["os"] == "centos":
log("Installing plugin for", public_ip, "...")
d = c.runSSHAsync(
public_ip,
"yum install -y clusterhq-flocker-docker-plugin && "
"systemctl enable flocker-docker-plugin && "
"systemctl start flocker-docker-plugin")
d.addCallback(report_completion, public_ip=public_ip)
deferreds.append(d)
else:
log("Skipping installing plugin: %r" %
(settings["SKIP_INSTALL_PLUGIN"], ))
yield gatherResults(deferreds)
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
# ensure that the /run/docker/plugins
# folder exists
log("Creating the /run/docker/plugins folder")
c.runSSHRaw(public_ip, "mkdir -p /run/docker/plugins")
if c.config["os"] == "coreos":
log("Starting flocker-docker-plugin as docker container on CoreOS on %s"
% (public_ip, ))
c.runSSH(
public_ip, """echo
/root/bin/docker run --restart=always -d --net=host --privileged \\
-e FLOCKER_CONTROL_SERVICE_BASE_URL=%s \\
-e MY_NETWORK_IDENTITY=%s \\
-v /etc/flocker:/etc/flocker \\
-v /run/docker:/run/docker \\
--name=flocker-docker-plugin \\
clusterhq/flocker-docker-plugin""" % (
controlservice,
private_ip,
))
log("Done!")
def main(reactor, configFile):
c = Configurator(configFile=configFile)
control_ip = c.config["control_node"]
# download and replace the docker binary on each of the nodes
for node in c.config["agent_nodes"]:
if c.config["os"] != "coreos":
log("Skipping installing new docker binary because we're on",
"ubuntu/centos, assuming we installed a sufficiently recent one",
"already.")
break
# only install new docker binary on coreos. XXX TODO coreos > 801.0.0
# doesn't need newer docker.
if settings["SKIP_DOCKER_BINARY"] or c.config["os"] != "coreos":
break
public_ip = node["public"]
log("Replacing docker binary on %s" % (public_ip,))
# stop the docker service
log("Stopping the docker service on %s" % (public_ip,))
if c.config["os"] == "ubuntu":
c.runSSHRaw(public_ip, "stop %s || true"
% (settings['DOCKER_SERVICE_NAME'],))
elif c.config["os"] == "centos":
c.runSSHRaw(public_ip, "systemctl stop %s.service || true"
% (settings['DOCKER_SERVICE_NAME'],))
elif c.config["os"] == "coreos":
c.runSSHRaw(public_ip, "systemctl stop docker.service || true")
# download the latest docker binary
if c.config["os"] == "coreos":
log("Downloading the latest docker binary on %s - %s" \
% (public_ip, settings['DOCKER_BINARY_URL'],))
c.runSSHRaw(public_ip, "mkdir -p /root/bin")
c.runSSHRaw(public_ip, "wget -qO /root/bin/docker %s"
% (settings['DOCKER_BINARY_URL'],))
c.runSSHRaw(public_ip, "chmod +x /root/bin/docker")
c.runSSHRaw(public_ip,
"cp /usr/lib/coreos/dockerd /root/bin/dockerd")
c.runSSHRaw(public_ip,
"cp /usr/lib/systemd/system/docker.service /etc/systemd/system/")
c.runSSHRaw(public_ip,
"sed -i s@/usr/lib/coreos@/root/bin@g /etc/systemd/system/docker.service")
c.runSSHRaw(public_ip,
"sed -i \\'s@exec docker@exec /root/bin/docker@g\\' /root/bin/dockerd")
c.runSSHRaw(public_ip, "systemctl daemon-reload")
else:
log("Downloading the latest docker binary on %s - %s" \
% (public_ip, settings['DOCKER_BINARY_URL'],))
c.runSSHRaw(public_ip, "wget -O /usr/bin/docker %s"
% (settings['DOCKER_BINARY_URL'],))
# start the docker service
log("Starting the docker service on %s" % (public_ip,))
if c.config["os"] == "ubuntu":
c.runSSHRaw(public_ip, "start %s"
% (settings['DOCKER_SERVICE_NAME'],))
elif c.config["os"] == "centos":
c.runSSHRaw(public_ip, "systemctl start %s.service"
% (settings['DOCKER_SERVICE_NAME'],))
elif c.config["os"] == "coreos":
c.runSSHRaw(public_ip, "systemctl start docker.service")
log("Generating plugin certs")
# generate and upload plugin.crt and plugin.key for each node
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# use the node IP to name the local files
# so they do not overwrite each other
c.run("flocker-ca create-api-certificate %s-plugin" % (public_ip,))
log("Generated plugin certs for", public_ip)
def report_completion(result, public_ip, message="Completed plugin install for"):
log(message, public_ip)
return result
deferreds = []
log("Uploading plugin certs...")
for node in c.config["agent_nodes"]:
public_ip = node["public"]
# upload the .crt and .key
for ext in ("crt", "key"):
d = c.scp("%s-plugin.%s" % (public_ip, ext,),
public_ip, "/etc/flocker/plugin.%s" % (ext,), async=True)
d.addCallback(report_completion, public_ip=public_ip, message=" * Uploaded plugin cert for")
deferreds.append(d)
yield gatherResults(deferreds)
log("Uploaded plugin certs")
log("Installing flocker plugin")
# loop each agent and get the plugin installed/running
# clone the plugin and configure an upstart/systemd unit for it to run
deferreds = []
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
log("Using %s => %s" % (public_ip, private_ip))
# the full api path to the control service
controlservice = 'https://%s:4523/v1' % (control_ip,)
# perhaps the user has pre-compiled images with the plugin
# downloaded and installed
if not settings["SKIP_INSTALL_PLUGIN"]:
if c.config["os"] == "ubuntu":
log("Installing plugin for", public_ip, "...")
d = c.runSSHAsync(public_ip,
"apt-get install -y --force-yes clusterhq-flocker-docker-plugin && "
"service flocker-docker-plugin restart")
d.addCallback(report_completion, public_ip=public_ip)
deferreds.append(d)
elif c.config["os"] == "centos":
log("Installing plugin for", public_ip, "...")
d = c.runSSHAsync(public_ip,
"yum install -y clusterhq-flocker-docker-plugin && "
"systemctl enable flocker-docker-plugin && "
"systemctl start flocker-docker-plugin")
d.addCallback(report_completion, public_ip=public_ip)
deferreds.append(d)
else:
log("Skipping installing plugin: %r" % (settings["SKIP_INSTALL_PLUGIN"],))
yield gatherResults(deferreds)
for node in c.config["agent_nodes"]:
public_ip = node["public"]
private_ip = node["private"]
# ensure that the /run/docker/plugins
# folder exists
log("Creating the /run/docker/plugins folder")
c.runSSHRaw(public_ip, "mkdir -p /run/docker/plugins")
if c.config["os"] == "coreos":
log("Starting flocker-docker-plugin as docker container on CoreOS on %s" % (public_ip,))
c.runSSH(public_ip, """echo
/root/bin/docker run --restart=always -d --net=host --privileged \\
-e FLOCKER_CONTROL_SERVICE_BASE_URL=%s \\
-e MY_NETWORK_IDENTITY=%s \\
-v /etc/flocker:/etc/flocker \\
-v /run/docker:/run/docker \\
--name=flocker-docker-plugin \\
clusterhq/flocker-docker-plugin""" % (controlservice, private_ip,))
log("Done!")
