def rm(c, name): sk, sv = utils.get_service(env.services, name) hosts = utils.get_hosts(env.hosts, sv["label"]) if sv["label"] == "zookeeper": zid = 0 for k, v in hosts.iteritems(): zid += 1 cmd = "docker rm -f {}-{}".format(sk, zid) rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_without_exit(c, rcmd) else: _docker_rm(c, hosts, sk)
def airflow_init_db(c, host, name): ck, cv = utils.get_service(env.services, name) cmd = "docker rm -f {} ".format(name) rcmd = "docker-machine ssh {} {}".format(host, cmd) utils.run_without_exit(c, rcmd) cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format( cv["docker"], ck) cmd += "--network {} ".format(" --network ".join(cv["networks"])) cmd += "-p {} ".format(" -p ".join(cv["ports"])) cmd += "-v {} ".format(os.path.expandvars(" -v ".join(cv["volumes"]))) cmd += "-e {} ".format(os.path.expandvars(" -e ".join(cv["environments"]))) cmd += "{} {}".format( cv["image"], os.path.expandvars("$SKP_SHOME/volume/bin/init_airflow.sh")) rcmd = "ssh -o StrictHostKeyChecking=no {}@{} -t {}".format( env.SKP_USER, env.hosts[host]["ipv4"], cmd) res = c.run(rcmd, pty=True)
def volume_rm(c, name=""): volumes = utils.get_volumes(env.volumes, name) for hk, hv in env.hosts.iteritems(): for vk, vv in volumes.iteritems(): if not _is_matched_label(hv, vv): continue if vv["type"] == "blobfs": cmds = [ "sudo umount {}".format(os.path.expandvars(vv["path"])), "docker volume rm -f {} ".format(vk) ] elif vv["type"] == "s3fs": cmds = [ "sudo umount {}".format(os.path.expandvars(vv["path"])), "docker volume rm -f {} ".format(vk) ] else: cmds = ["docker volume rm -f {} ".format(vk)] for cmd in cmds: rcmd = "docker-machine ssh {} {}".format(hk, cmd) utils.run_without_exit(c, rcmd)
def swarm_leave(c): for k, v in env.hosts.iteritems(): cmd = "docker swarm leave --force" rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_without_exit(c, rcmd)
def run(c, name): sk, sv = utils.get_service(env.services, name) hosts = utils.get_hosts(env.hosts, sv["label"]) cmd = "" if sv["label"] == "registry": _docker_rm(c, hosts, sk) cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format( sk) cmd += _get_opts(sv) cmd += "{}".format(sv["image"]) _docker_ssh(c, hosts, cmd) elif sv["label"] == "jupyter": _docker_rm(c, hosts, sk) cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format( sv["docker"], sk) cmd += _get_opts(sv) cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"])) _docker_ssh(c, hosts, cmd) elif sv["label"] == "spark-master": _docker_rm(c, hosts, sk) cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format( sv["docker"], sk) cmd += _get_opts(sv) cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"])) _docker_ssh(c, hosts, cmd) elif sv["label"] == "spark-worker": _docker_rm(c, hosts, sk) cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format( sv["docker"], sk) cmd += _get_opts(sv) cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"])) _docker_ssh(c, hosts, cmd) elif sv["label"] == "mysql": _docker_rm(c, hosts, sk) cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format( sk) cmd += _get_opts(sv) cmd += "{} --default-authentication-plugin=mysql_native_password ".format( sv["image"]) _docker_ssh(c, hosts, cmd) elif sv["label"] == "pgsql": _docker_rm(c, hosts, sk) cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format( sk) cmd += _get_opts(sv) cmd += "{} ".format(sv["image"]) _docker_ssh(c, hosts, cmd) elif sv["label"] == "airflow": _docker_rm(c, hosts, sk) cmd = "{} run -i -t -d --privileged --restart=always --name {} ".format( sv["docker"], sk) cmd += _get_opts(sv) cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"])) _docker_ssh(c, hosts, cmd) elif sv["label"] == "portainer": _docker_rm(c, hosts, sk) cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format( sk) cmd += _get_opts(sv) cmd += "{} {}".format(sv["image"], os.path.expandvars(sv["cmd"])) _docker_ssh(c, hosts, cmd) elif sv["label"] == "portainer-agent": _docker_rm(c, hosts, sk) cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format( sk) cmd += _get_opts(sv) cmd += "{}".format(sv["image"]) _docker_ssh(c, hosts, cmd) elif sv["label"] == "zookeeper": zid = 0 for k, v in hosts.iteritems(): zid += 1 cmd = "docker rm -f {}-{}".format(sk, zid) rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_without_exit(c, rcmd) with open('/tmp/zoo.cfg', 'w') as f: lines = [ 'tickTime=2000', 'dataDir=/opt/zookeeper/data', 'clientPort=2181', 'initLimit=5', 'syncLimit=2' ] zid = 0 for k, v in hosts.iteritems(): zid += 1 lines.append("server.{}={}-{}:2888:3888".format(zid, sk, zid)) for line in lines: f.write("{}\n".format(line)) zid = 0 for k, v in hosts.iteritems(): zid += 1 with open('/tmp/myid', 'w') as f: f.write("{}\n".format(zid)) path_z = os.path.expandvars(sv["path"]) path_conf = "{}/conf".format(path_z) cmd = "sudo mkdir -p {} ".format(path_conf) rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_without_exit(c, rcmd) path_data = "{}/data".format(path_z) cmd = "sudo mkdir -p {} ".format(path_data) rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_without_exit(c, rcmd) cmd = "docker-machine scp /tmp/zoo.cfg {}:/tmp/zoo.cfg ".format(k) utils.run_with_exit(c, cmd) cmd = "docker-machine ssh {} sudo cp /tmp/zoo.cfg {} ".format( k, path_conf) utils.run_with_exit(c, cmd) cmd = "docker-machine scp /tmp/myid {}:/tmp/myid ".format(k) utils.run_with_exit(c, cmd) cmd = "docker-machine ssh {} sudo cp /tmp/myid {} ".format( k, path_data) utils.run_with_exit(c, cmd) cmd = "docker run -i -t -d --privileged --restart=always --name {}-{} ".format( sk, zid) cmd += _get_opts(sv) cmd += "{}".format(sv["image"]) rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_without_exit(c, rcmd) elif sv["label"] == "kafka": _docker_rm(c, hosts, sk) zsk, zsv = utils.get_service(env.services, sv["zookeeper"]) zhosts = utils.get_hosts(env.hosts, "zookeeper") zid = 0 zconns = [] for zk, zv in zhosts.iteritems(): zid += 1 zconns.append("{}-{}:2181".format(zsk, zid)) zconn = ",".join(zconns) print("Zookeeper Conn: {}".format(zconn)) bid = 0 for k, v in hosts.iteritems(): bid += 1 cmd = "docker-machine ip {} ".format(k) res = c.run(cmd) ip = res.stdout.strip() cmd = "docker run -i -t -d --privileged --restart=always --name {} ".format( sk) cmd += _get_opts(sv) cmd += '-e KAFKA_BROKER_ID="{}" '.format(bid) cmd += '-e KAFKA_ADVERTISED_HOST_NAME="{}" '.format(k) cmd += '-e KAFKA_ZOOKEEPER_CONNECT="{}" '.format(zconn) cmd += '-e KAFKA_LISTENER_SECURITY_PROTOCOL_MAP="INSIDE:PLAINTEXT,OUTSIDE:PLAINTEXT" ' cmd += '-e KAFKA_ADVERTISED_LISTENERS="INSIDE://:9092,OUTSIDE://{}:9094" '.format( k) cmd += '-e KAFKA_LISTENERS="INSIDE://:9092,OUTSIDE://:9094" ' cmd += '-e KAFKA_INTER_BROKER_LISTENER_NAME="INSIDE" ' cmd += '-e KAFKA_ADVERTISED_PORT="9092" ' cmd += "{}".format(sv["image"]) rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_with_exit(c, rcmd) else: print("Unkown Label: {}".format(cv["label"])) sys.exit(-1)
def _docker_ssh_without_exit(c, hosts, cmd): for k, v in hosts.iteritems(): rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_without_exit(c, rcmd)
def _docker_rm(c, hosts, name): for k, v in hosts.iteritems(): cmd = "docker rm -f {} ".format(name) rcmd = "docker-machine ssh {} {}".format(k, cmd) utils.run_without_exit(c, rcmd)
def volume_ls(c, name=""): volumes = utils.get_volumes(env.volumes, name) for hk, hv in env.hosts.iteritems(): cmd = "docker volume ls" rcmd = "docker-machine ssh {} {}".format(hk, cmd) utils.run_without_exit(c, rcmd)