def rm(c, name):
sk, sv = utils.get_service(env.services, name)
hosts = utils.get_hosts(env.hosts, sv["label"])
if sv["label"] == "zookeeper":
zid = 0
for k, v in hosts.iteritems():
zid += 1
cmd = "docker rm -f {}-{}".format(sk, zid)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
else:
_docker_rm(c, hosts, sk)
def airflow_init_db(c, host, name):
ck, cv = utils.get_service(env.services, name)
cmd = "docker rm -f {} ".format(name)
rcmd = "docker-machine ssh {} {}".format(host, cmd)
utils.run_without_exit(c, rcmd)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
cv["docker"], ck)
cmd += "--network {} ".format(" --network ".join(cv["networks"]))
cmd += "-p {} ".format(" -p ".join(cv["ports"]))
cmd += "-v {} ".format(os.path.expandvars(" -v ".join(cv["volumes"])))
cmd += "-e {} ".format(os.path.expandvars(" -e ".join(cv["environments"])))
cmd += "{} {}".format(
cv["image"],
os.path.expandvars("$SKP_SHOME/volume/bin/init_airflow.sh"))
rcmd = "ssh -o StrictHostKeyChecking=no {}@{} -t {}".format(
env.SKP_USER, env.hosts[host]["ipv4"], cmd)
res = c.run(rcmd, pty=True)
def volume_rm(c, name=""):
volumes = utils.get_volumes(env.volumes, name)
for hk, hv in env.hosts.iteritems():
for vk, vv in volumes.iteritems():
if not _is_matched_label(hv, vv):
continue
if vv["type"] == "blobfs":
cmds = [
"sudo umount {}".format(os.path.expandvars(vv["path"])),
"docker volume rm -f {} ".format(vk)
]
elif vv["type"] == "s3fs":
cmds = [
"sudo umount {}".format(os.path.expandvars(vv["path"])),
"docker volume rm -f {} ".format(vk)
]
else:
cmds = ["docker volume rm -f {} ".format(vk)]
for cmd in cmds:
rcmd = "docker-machine ssh {} {}".format(hk, cmd)
utils.run_without_exit(c, rcmd)
def swarm_leave(c):
for k, v in env.hosts.iteritems():
cmd = "docker swarm leave --force"
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
def run(c, name):
sk, sv = utils.get_service(env.services, name)
hosts = utils.get_hosts(env.hosts, sv["label"])
cmd = ""
if sv["label"] == "registry":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "jupyter":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "spark-master":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "spark-worker":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "mysql":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} --default-authentication-plugin=mysql_native_password ".format(
sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "pgsql":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} ".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "airflow":
_docker_rm(c, hosts, sk)
cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format(
sv["docker"], sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "portainer":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"]))
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "portainer-agent":
_docker_rm(c, hosts, sk)
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
_docker_ssh(c, hosts, cmd)
elif sv["label"] == "zookeeper":
zid = 0
for k, v in hosts.iteritems():
zid += 1
cmd = "docker rm -f {}-{}".format(sk, zid)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
with open('/tmp/zoo.cfg', 'w') as f:
lines = [
'tickTime=2000', 'dataDir=/opt/zookeeper/data',
'clientPort=2181', 'initLimit=5', 'syncLimit=2'
]
zid = 0
for k, v in hosts.iteritems():
zid += 1
lines.append("server.{}={}-{}:2888:3888".format(zid, sk, zid))
for line in lines:
f.write("{}\n".format(line))
zid = 0
for k, v in hosts.iteritems():
zid += 1
with open('/tmp/myid', 'w') as f:
f.write("{}\n".format(zid))
path_z = os.path.expandvars(sv["path"])
path_conf = "{}/conf".format(path_z)
cmd = "sudo mkdir -p {} ".format(path_conf)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
path_data = "{}/data".format(path_z)
cmd = "sudo mkdir -p {} ".format(path_data)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
cmd = "docker-machine scp /tmp/zoo.cfg {}:/tmp/zoo.cfg ".format(k)
utils.run_with_exit(c, cmd)
cmd = "docker-machine ssh {} sudo cp /tmp/zoo.cfg {} ".format(
k, path_conf)
utils.run_with_exit(c, cmd)
cmd = "docker-machine scp /tmp/myid {}:/tmp/myid ".format(k)
utils.run_with_exit(c, cmd)
cmd = "docker-machine ssh {} sudo cp /tmp/myid {} ".format(
k, path_data)
utils.run_with_exit(c, cmd)
cmd = "docker run -i -t -d --privileged --restart=always --name {}-{} ".format(
sk, zid)
cmd += _get_opts(sv)
cmd += "{}".format(sv["image"])
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
elif sv["label"] == "kafka":
_docker_rm(c, hosts, sk)
zsk, zsv = utils.get_service(env.services, sv["zookeeper"])
zhosts = utils.get_hosts(env.hosts, "zookeeper")
zid = 0
zconns = []
for zk, zv in zhosts.iteritems():
zid += 1
zconns.append("{}-{}:2181".format(zsk, zid))
zconn = ",".join(zconns)
print("Zookeeper Conn: {}".format(zconn))
bid = 0
for k, v in hosts.iteritems():
bid += 1
cmd = "docker-machine ip {} ".format(k)
res = c.run(cmd)
ip = res.stdout.strip()
cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format(
sk)
cmd += _get_opts(sv)
cmd += '-e KAFKA_BROKER_ID="{}" '.format(bid)
cmd += '-e KAFKA_ADVERTISED_HOST_NAME="{}" '.format(k)
cmd += '-e KAFKA_ZOOKEEPER_CONNECT="{}" '.format(zconn)
cmd += '-e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP="INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" '
cmd += '-e KAFKA_ADVERTISED_LISTENERS="INSIDE://:9092,OUTSIDE://{}:9094" '.format(
k)
cmd += '-e KAFKA_LISTENERS="INSIDE://:9092,OUTSIDE://:9094" '
cmd += '-e KAFKA_INTER_BROKER_LISTENER_NAME="INSIDE" '
cmd += '-e KAFKA_ADVERTISED_PORT="9092" '
cmd += "{}".format(sv["image"])
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_with_exit(c, rcmd)
else:
print("Unkown Label: {}".format(cv["label"]))
sys.exit(-1)
def _docker_ssh_without_exit(c, hosts, cmd):
for k, v in hosts.iteritems():
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
def _docker_rm(c, hosts, name):
for k, v in hosts.iteritems():
cmd = "docker rm -f {} ".format(name)
rcmd = "docker-machine ssh {} {}".format(k, cmd)
utils.run_without_exit(c, rcmd)
def volume_ls(c, name=""):
volumes = utils.get_volumes(env.volumes, name)
for hk, hv in env.hosts.iteritems():
cmd = "docker volume ls"
rcmd = "docker-machine ssh {} {}".format(hk, cmd)
utils.run_without_exit(c, rcmd)