def swarm_join(c):
managers = {}
workers = {}
for k, v in env.hosts.iteritems():
#print(k, v)
if "manager" in v["roles"]:
if "master" not in v["roles"]:
managers[k] = v
if "worker" in v["roles"]:
workers[k] = v
print("managers: %s" % managers)
print("workers: %s" % workers)
mk, mv = utils.get_master(env.hosts)
cmd = "docker-machine ssh {} docker swarm join-token manager".format(mk)
res = c.run(cmd)
cmd_manager = re.search(r'docker\sswarm\sjoin.*:[0-9]{4}',
res.stdout.strip(), re.DOTALL).group()
cmd_manager = re.sub(r'[\\\r\n]', "", cmd_manager)
cmd = "docker-machine ssh {} docker swarm join-token worker".format(mk)
res = c.run(cmd)
cmd_worker = re.search(r'docker\sswarm\sjoin.*:[0-9]{4}',
res.stdout.strip(), re.DOTALL).group()
cmd_worker = re.sub(r'[\\\r\n]', "", cmd_worker)
for k, v in managers.iteritems():
cmd = "docker-machine ssh {} {}".format(k, cmd_manager)
utils.run_with_exit(c, cmd)
for k, v in workers.iteritems():
cmd = "docker-machine ssh {} {}".format(k, cmd_worker)
utils.run_with_exit(c, cmd)
def cmd(c, cmd, host=""):
if host:
rcmd = "ssh -o StrictHostKeyChecking=no {}@{} {}".format(env.SKP_USER, env.hosts[host]["ipv4"], cmd)
utils.run_with_exit(c, rcmd)
else:
for k, v in env.hosts.iteritems():
rcmd = "ssh -o StrictHostKeyChecking=no {}@{} {}".format(env.SKP_USER, v["ipv4"], cmd)
utils.run_with_exit(c, rcmd)
def swarm_label(c):
labels = {}
for k, v in env.hosts.iteritems():
#print(k, v)
labels[k] = ""
for lk, lv in v["labels"].iteritems():
labels[k] += " --label-add %s=%s " % (lk, lv)
for k, v in labels.iteritems():
cmd = "docker node update {} {}".format(v, k)
utils.run_with_exit(c, cmd)
def machine_install(c):
uname = utils.run_with_exit(c, "uname")
print("uname: {}".format(uname))
# reference: https://docs.docker.com/machine/install-machine/#install-bash-completion-scripts
cmd = "curl -L https://github.com/docker/machine/releases/download/v0.14.0/docker-machine-$(uname -s)-$(uname -m) >/tmp/docker-machine && sudo install /tmp/docker-machine /usr/local/bin/docker-machine"
utils.run_with_exit(c, cmd)
if uname.find('Linux') > -1:
cmd = "sudo usermod -aG docker $USER"
utils.run_with_exit(c, cmd)
def exportfs(c, name=""):
volumes = utils.get_volumes(env.volumes, name)
for vk, vv in volumes.iteritems():
if vv["type"] != "nfs":
continue
cmd = "mkdir -p {}".format(os.path.expandvars(vv["path"]))
utils.run_with_exit(c, cmd)
cmds = ["sudo exportfs -a", "showmount -e"]
for cmd in cmds:
utils.run_with_exit(c, cmd)
def install(c):
cmds = [
"wget -q https://www.virtualbox.org/download/oracle_vbox_2016.asc -O- | sudo apt-key add -",
"wget -q https://www.virtualbox.org/download/oracle_vbox.asc -O- | sudo apt-key add -",
'sudo add-apt-repository "deb [arch=amd64] http://download.virtualbox.org/virtualbox/debian $(lsb_release -cs) contrib"',
"sudo apt update",
"sudo apt install virtualbox virtualbox-ext-pack",
# "sudo apt install vagrant",
"wget -c https://releases.hashicorp.com/vagrant/2.0.3/vagrant_2.0.3_x86_64.deb -P /tmp",
"sudo dpkg -i /tmp/vagrant_2.0.3_x86_64.deb",
"vagrant plugin install vagrant-disksize"
]
for cmd in cmds:
utils.run_with_exit(c, cmd)
def install(c):
uname = utils.run_with_exit(c, "uname")
print("uname: {}".format(uname))
cmds = [
"sudo apt-get update",
"sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common",
"curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -",
'\'sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"\'',
"sudo apt-get update", "sudo apt-get install -y docker-ce",
"sudo usermod -aG docker $USER"
]
for hk, hv in env.hosts.iteritems():
for cmd in cmds:
rcmd = "ssh -o StrictHostKeyChecking=no {}@{} {}".format(
env.SKP_USER, hv["ipv4"], cmd)
utils.run_with_exit(c, rcmd)
def install(c):
cmds = [
"sudo apt-get update",
"sudo apt-get install -y --no-install-recommends automake autotools-dev g++ git libcurl4-gnutls-dev libssl-dev libxml2-dev make pkg-config fuse libfuse-dev",
"sudo apt-get install -y --no-install-recommends sshfs",
"rm -rf s3fs-fuse",
"git clone https://github.com/s3fs-fuse/s3fs-fuse.git",
"\"cd s3fs-fuse && ./autogen.sh && ./configure && make && sudo make install\"",
"rm -rf s3fs-fuse",
"wget https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb",
"sudo dpkg -i packages-microsoft-prod.deb",
"rm -rf packages-microsoft-prod.deb",
"sudo apt-get update",
"sudo apt-get install -y --no-install-recommends blobfuse"
]
for hk, hv in env.hosts.iteritems():
for cmd in cmds:
rcmd = "ssh -o StrictHostKeyChecking=no {}@{} {}".format(env.SKP_USER, hv["ipv4"], cmd)
utils.run_with_exit(c, rcmd)
def install(c, role, locale="en_US.UTF-8"):
if locale == 'en_US.UTF-8':
pack = 'language-pack-en'
elif locale == 'ko_KR.UTF-8':
pack = 'language-pack-ko'
else:
print("only support en/ko langauge pack")
sys.exit(-1)
for k, v in env.hosts.iteritems():
if role not in v["roles"]:
continue
cmd = "ssh -o StrictHostKeyChecking=no {}@{} ".format(
env.SKP_USER, v["ipv4"])
cmd += "sudo apt-get install --no-install-recommends -y locales {} ".format(
pack)
utils.run_with_exit(c, cmd)
cmd = "ssh -o StrictHostKeyChecking=no {}@{} ".format(
env.SKP_USER, v["ipv4"])
cmd += '"sudo sed -i -e \'s/# {} UTF-8/{} UTF-8/\' /etc/locale.gen && sudo locale-gen"'.format(
locale, locale)
utils.run_with_exit(c, cmd)
cmd = "ssh -o StrictHostKeyChecking=no {}@{} ".format(
env.SKP_USER, v["ipv4"])
cmd += "sudo update-locale LC_ALL={} LANG={} LC_MESSAGES=POSIX".format(
locale, locale)
utils.run_with_exit(c, cmd)
def install(c):
ports = [2049]
cmd = "sudo apt-get install -y nfs-kernel-server portmap"
utils.run_with_exit(c, cmd)
for k, v in env.hosts.iteritems():
for port in ports:
cmd = "ssh -o StrictHostKeyChecking=no {}@{} sudo ufw allow {}".format(
env.SKP_USER, v["ipv4"], port)
utils.run_with_exit(c, cmd)
cmd = "ssh -o StrictHostKeyChecking=no {}@{} sudo apt-get install -y nfs-common".format(
env.SKP_USER, v["ipv4"])
utils.run_with_exit(c, cmd)
def machine_create(c):
ports = [22, 2376, 2377, 7946, 4789]
for k, v in env.hosts.iteritems():
for port in ports:
cmd = "ssh -o StrictHostKeyChecking=no {}@{} sudo ufw allow {}".format(
env.SKP_USER, v["ipv4"], port)
utils.run_with_exit(c, cmd)
cmd = "docker-machine create --driver generic --generic-ip-address {} --generic-ssh-key $HOME/.ssh/id_rsa --generic-ssh-user {} {}".format(
v["ipv4"], env.SKP_USER, k)
utils.run_with_exit(c, cmd)
cmd = "ssh -o StrictHostKeyChecking=no {}@{} sudo usermod -aG docker {}".format(
env.SKP_USER, v["ipv4"], env.SKP_USER)
utils.run_with_exit(c, cmd)
def image_build(c, name=""):
images = utils.get_images(env.images, name)
for ik, iv in images.iteritems():
cmd = "docker build "
cmd += " ".join(iv["opts"])
cmd += " {}/etc/docker-file/{} --tag {}:{} ".format(
env.SKP_HOME, iv["path"], iv["image"], iv["tag"])
utils.run_with_exit(c, cmd)
cmd = "docker image tag {}:{} {}/{}:{}".format(iv["image"], iv["tag"],
iv["registry"],
iv["image"], iv["tag"])
utils.run_with_exit(c, cmd)
cmd = "docker push {}/{}:{}".format(iv["registry"], iv["image"],
iv["tag"])
utils.run_with_exit(c, cmd)
def run(c, name):
sk, sv = utils.get_service(env.services, name)
hosts = utils.get_hosts(env.hosts, sv["label"])
cmd = ""
if sv["label"] == "registry":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "jupyter":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "spark-master":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "spark-worker":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "mysql":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} --default-authentication-plugin=mysql_native_password ".format(
sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "pgsql":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} ".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "airflow":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "portainer":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "portainer-agent":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "zookeeper":
zid = 0
for k, v in hosts.iteritems():
zid += 1
cmd = "docker rm -f {}-{}".format(sk, zid)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
with open('/tmp/zoo.cfg', 'w') as f:
lines = [
'tickTime=2000', 'dataDir=/opt/zookeeper/data',
'clientPort=2181', 'initLimit=5', 'syncLimit=2'
]
zid = 0
for k, v in hosts.iteritems():
zid += 1
lines.append("server.{}={}-{}:2888:3888".format(zid, sk, zid))
for line in lines:
f.write("{}\n".format(line))
zid = 0
for k, v in hosts.iteritems():
zid += 1
with open('/tmp/myid', 'w') as f:
f.write("{}\n".format(zid))
path_z = os.path.expandvars(sv["path"])
path_conf = "{}/conf".format(path_z)
cmd = "sudo mkdir -p {} ".format(path_conf)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
path_data = "{}/data".format(path_z)
cmd = "sudo mkdir -p {} ".format(path_data)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
cmd = "docker-machine scp /tmp/zoo.cfg {}:/tmp/zoo.cfg ".format(k)
utils.run_with_exit(c, cmd)
cmd = "docker-machine ssh {} sudo cp /tmp/zoo.cfg {} ".format(
k, path_conf)
utils.run_with_exit(c, cmd)
cmd = "docker-machine scp /tmp/myid {}:/tmp/myid ".format(k)
utils.run_with_exit(c, cmd)
cmd = "docker-machine ssh {} sudo cp /tmp/myid {} ".format(
k, path_data)
utils.run_with_exit(c, cmd)
cmd = "docker run -i -t -d --privileged --restart=always --name {}-{} ".format(
sk, zid)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
elif sv["label"] == "kafka":
_docker_rm(c, hosts, sk)
zsk, zsv = utils.get_service(env.services, sv["zookeeper"])
zhosts = utils.get_hosts(env.hosts, "zookeeper")
zid = 0
zconns = []
for zk, zv in zhosts.iteritems():
zid += 1
zconns.append("{}-{}:2181".format(zsk, zid))
zconn = ",".join(zconns)
print("Zookeeper Conn: {}".format(zconn))
bid = 0
for k, v in hosts.iteritems():
bid += 1
cmd = "docker-machine ip {} ".format(k)
res = c.run(cmd)
ip = res.stdout.strip()
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += '-e KAFKA_BROKER_ID="{}" '.format(bid)
cmd += '-e KAFKA_ADVERTISED_HOST_NAME="{}" '.format(k)
cmd += '-e KAFKA_ZOOKEEPER_CONNECT="{}" '.format(zconn)
cmd += '-e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP="INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" '
cmd += '-e KAFKA_ADVERTISED_LISTENERS="INSIDE://:9092,OUTSIDE://{}:9094" '.format(
k)
cmd += '-e KAFKA_LISTENERS="INSIDE://:9092,OUTSIDE://:9094" '
cmd += '-e KAFKA_INTER_BROKER_LISTENER_NAME="INSIDE" '
cmd += '-e KAFKA_ADVERTISED_PORT="9092" '
cmd += "{}".format(sv["image"])
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_with_exit(c, rcmd)
else:
print("Unkown Label: {}".format(cv["label"]))
sys.exit(-1)
def _docker_ssh(c, hosts, cmd):
for k, v in hosts.iteritems():
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_with_exit(c, rcmd)
def copy_id(c, passwd):
SKP_PASSWD = passwd
mk, mv = utils.get_master(env.hosts)
SKP_MIP = mv['ipv4']
for k, v in env.hosts.iteritems():
cmd = "sshpass -p {} ssh -o StrictHostKeyChecking=no {}@{} sudo apt-get update".format(SKP_PASSWD, env.SKP_USER, v["ipv4"])
utils.run_with_exit(c, cmd)
cmd = "sshpass -p {} ssh -o StrictHostKeyChecking=no {}@{} sudo apt-get install -y sshpass".format(SKP_PASSWD, env.SKP_USER, v["ipv4"])
utils.run_with_exit(c, cmd)
cmd = "sshpass -p {} ssh -o StrictHostKeyChecking=no {}@{} sudo sed -i -e 's/#AuthorizedKeysFile/AuthorizedKeysFile/g' /etc/ssh/sshd_config".format(SKP_PASSWD, env.SKP_USER, v["ipv4"])
utils.run_with_exit(c, cmd)
for k, v in env.hosts.iteritems():
# Generate Keys
cmd = "sshpass -p {} ssh -o StrictHostKeyChecking=no {}@{} ".format(SKP_PASSWD, env.SKP_USER, v["ipv4"])
cmd += '"mkdir -p $HOME/.ssh;rm -f $HOME/.ssh/id_rsa;rm -f $HOME/.ssh/id_rsa.pub;'
cmd += 'ssh-keygen -t rsa -N'
cmd += " '' "
cmd += '-f $HOME/.ssh/id_rsa"'
utils.run_with_exit(c, cmd)
# Send Master Public Key to All Hosts
cmd = "sshpass -p {} ssh-copy-id -f -i $HOME/.ssh/id_rsa.pub -o StrictHostKeyChecking=no {}@{}".format(SKP_PASSWD, env.SKP_USER, v["ipv4"])
utils.run_with_exit(c, cmd)
for k, v in env.hosts.iteritems():
# Send All Node's Public Key to Master
cmd = "sshpass -p {} ssh -o StrictHostKeyChecking=no {}@{} ".format(SKP_PASSWD, env.SKP_USER, v["ipv4"])
cmd += "sshpass -p {} ssh-copy-id -f -i $HOME/.ssh/id_rsa.pub -o StrictHostKeyChecking=no {}@{}".format(SKP_PASSWD, env.SKP_USER, SKP_MIP)
utils.run_with_exit(c, cmd)
def machine_rm(c):
for k, v in env.hosts.iteritems():
cmd = "docker-machine rm -f {}".format(k)
utils.run_with_exit(c, cmd)
def image_prune(c):
for k, v in env.hosts.iteritems():
cmd = "docker-machine ssh {} docker image prune ".format(k)
utils.run_with_exit(c, cmd)
def network_rm(c, name=""):
networks = utils.get_networks(env.networks, name)
for nk, nv in networks.iteritems():
cmd = "docker network rm {}".format(nk)
utils.run_with_exit(c, cmd)
def network_create(c, name=""):
networks = utils.get_networks(env.networks, name)
for nk, nv in networks.iteritems():
opts = " ".join(nv["opts"])
cmd = "docker network create {} {}".format(opts, nk)
utils.run_with_exit(c, cmd)
def node_ls(c):
cmd = "docker node ls"
utils.run_with_exit(c, cmd)
def image_rm(c, image, tag):
for k, v in env.hosts.iteritems():
cmd = "docker-machine ssh {} docker image rm -f {}:{}".format(
k, image, tag)
utils.run_with_exit(c, cmd)
def network_ls(c):
cmd = "docker network ls"
utils.run_with_exit(c, cmd)
def machine_ls(c):
cmd = "docker-machine ls"
utils.run_with_exit(c, cmd)
def restart(c):
cmd = "sudo service nfs-kernel-server restart"
utils.run_with_exit(c, cmd)
def swarm_init(c):
mk, mv = utils.get_master(env.hosts)
SKP_MIP = mv['ipv4']
cmd = "docker swarm init --advertise-addr {}".format(SKP_MIP)
utils.run_with_exit(c, cmd)
def volume_create(c, name=""):
volumes = utils.get_volumes(env.volumes, name)
for hk, hv in env.hosts.iteritems():
for vk, vv in volumes.iteritems():
if not _is_matched_label(hv, vv):
continue
print("type: {}".format(vv["type"]))
opts = os.path.expandvars(" ".join(vv["opts"]))
if vv["type"] == "nfs":
cmds = ["docker volume create --name {} {} ".format(vk, opts)]
elif vv["type"] == "sshfs":
cmds = [
"docker volume create -d vieux/sshfs --name {} {} ".format(
vk, opts)
]
elif vv["type"] == "blobfs":
config_file = "$HOME/tmp/volumes/{}/blob-passwd".format(vk)
tmp_path = "$HOME/tmp/volumes/{}/blobfusetmp".format(vk)
cmds = [
"mkdir -p $HOME/tmp/volumes/{}".format(vk),
"rm -rf {}".format(config_file),
"\"echo 'accountName {}' >> {}\"".format(
vv["account_name"],
config_file), "\"echo 'accountKey {}' >> {}\"".format(
vv["account_key"], config_file),
"\"echo 'containerName {}' >> {}\"".format(
vv["container_name"],
config_file), "chmod 600 {}".format(config_file),
"mkdir -p {}".format(os.path.expandvars(vv["path"])),
"mkdir -p {}".format(tmp_path),
"sudo blobfuse {} --tmp-path={} -o attr_timeout=240 -o entry_timeout=240 -o negative_timeout=120 --config-file={}"
.format(os.path.expandvars(vv["path"]), tmp_path,
config_file),
"docker volume create --name {} {} ".format(vk, opts)
]
elif vv["type"] == "s3fs":
config_file = "$HOME/tmp/volumes/{}/s3-passwd".format(vk)
cmds = [
"mkdir -p $HOME/tmp/volumes/{}".format(vk),
"rm -rf {}".format(config_file),
"\"echo '{}:{}' >> {}\"".format(vv["account_id"],
vv["account_key"],
config_file),
"chmod 600 {}".format(config_file),
"mkdir -p {}".format(os.path.expandvars(vv["path"])),
"sudo s3fs {} {} -o passwd_file={}".format(
vv["bucket"], os.path.expandvars(vv["path"]),
config_file),
"docker volume create --name {} {} ".format(vk, opts)
]
elif vv["type"] == "fs":
cmds = [
"mkdir -p {}".format(os.path.expandvars(vv["path"])),
"docker volume create --name {} {} ".format(vk, opts)
]
else:
print("Unknown Volume Type: {}".format(vv["type"]))
sys.exit(-1)
for cmd in cmds:
rcmd = "docker-machine ssh {} {}".format(hk, cmd)
utils.run_with_exit(c, rcmd)
def ps(c):
for k, v in env.hosts.iteritems():
cmd = "docker-machine ssh {} docker ps".format(k)
utils.run_with_exit(c, cmd)
def masquerade(c, eth):
cmd = "sudo /sbin/iptables -t nat -A POSTROUTING -o {} -j MASQUERADE".format(
eth)
utils.run_with_exit(c, cmd)
cmd = "sudo /sbin/iptables -A FORWARD -i {} -j ACCEPT".format(eth)
utils.run_with_exit(c, cmd)
