def delete_nodes(client, kind, count):
print('Deleting %d %s server node(s) to cluster...' % (count, kind))
prev_count = util.get_previous_count(client, kind)
if prev_count < count:
print('There are only %d %s server node(s)' % (prev_count, kind))
return
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' + kind).items
pods_to_delete = pods[:count]
deleted_pod_ips = []
for pod in pods_to_delete:
deleted_pod_ips.append(pod.status.pod_ip)
podname = pod.metadata.name
client.delete_namespaced_pod(name=podname,
namespace=util.NAMESPACE,
body=k8s.client.V1DeleteOptions())
client.delete_node(name=hostname, body=k8s.client.V1DeleteOptions())
util.run_process(['./modify_ig.sh', ntype,
str(prev_count - count)], 'kops')
# Notify routers about deleted key nodes
if kind == 'keynode':
deregister(client, deleted_pod_ips)
def remove_node(ip, ntype):
client, _ = util.init_k8s()
pod = util.get_pod_from_ip(client, ip)
hostname = 'ip-%s.ec2.internal' % (ip.replace('.', '-'))
podname = pod.metadata.name
client.delete_namespaced_pod(name=podname, namespace=util.NAMESPACE,
body=k8s.client.V1DeleteOptions())
client.delete_node(name=hostname, body=k8s.client.V1DeleteOptions())
prev_count = util.get_previous_count(client, ntype)
util.run_process(['./modify_ig.sh', ntype, str(prev_count - 1)])
def add_nodes(client, apps_client, cfile, kind, count, aws_key_id=None,
aws_key=None, create=False, prefix=None, branch="master"):
print('Adding %d %s server node(s) to cluster...' % (count, kind))
prev_count = util.get_previous_count(client, kind)
util.run_process(['./modify_ig.sh', kind, str(count + prev_count)], 'kops')
util.run_process(['./validate_cluster.sh'], 'kops')
if create:
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname, prefix)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'BRANCH', branch)
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
if kind == "tasc":
routing_svc = util.get_service_address(client, 'routing-service')
util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc)
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
worker_svc = util.get_service_address(client, 'worker-service')
util.replace_yaml_val(env, 'WORKER_ILB', worker_svc)
if kind == "keynode":
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
if kind == 'worker':
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
routing_svc = util.get_service_address(client, 'routing-service')
util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# Wait until all pods of this kind are running
res = []
while len(res) != count:
res = util.get_pod_ips(client, 'role=' + kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
# Send kube config to lb
if kind == 'lb':
kubecfg = os.path.join(os.environ['HOME'], '.kube/config')
for pod in pods:
cname = pod.spec.containers[0].name
util.copy_file_to_pod(client, kubecfg, pod.metadata.name,
'/root/.kube', cname)
# Generate list of all recently created pods.
created_pod_ips = []
for pod in pods:
created_pod_ips.append(pod.status.pod_ip)
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
# Copy the KVS config into all recently created pods.
cfile_name = './tasc-config.yml' if kind != 'routing' else './anna-config.yml'
cfile_dir = '/go/src/github.com/saurav-c/tasc/config' if kind != 'routing' else 'hydro/anna/conf'
os.system(str('cp %s ' + cfile_name) % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(client, cfile_name[2:], pname,
cfile_dir, cname)
os.system('rm ' + cfile_name)
def add_nodes(client,
apps_client,
cfile,
kinds,
counts,
management_ip,
aws_key_id=None,
aws_key=None,
create=False,
prefix=None):
for i in range(len(kinds)):
print('Adding %d %s server node(s) to cluster...' %
(counts[i], kinds[i]))
prev_count = util.get_previous_count(client, kinds[i])
util.run_process(
['./modify_ig.sh', kinds[i],
str(counts[i] + prev_count)])
util.run_process(['./validate_cluster.sh'])
replica_str = ' '.join(util.get_node_ips(client, 'role=aft'))
# Create should only be true when the DaemonSet is being created for the
# first time -- i.e., when this is called from create_cluster. After that,
# we can basically ignore this because the DaemonSet will take care of
# adding pods to created nodes.
if create:
for i in range(len(kinds)):
kind = kinds[i]
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname, prefix)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'REPLICA_IPS', replica_str)
util.replace_yaml_val(env, 'MANAGER', management_ip)
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# Wait until all pods of this kind are running
res = []
while len(res) != counts[i]:
res = util.get_pod_ips(client, 'role=' + kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
# Generate list of all recently created pods.
for pod in pods:
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
pod.metadata.labels['aftReady'] = 'isready'
client.patch_namespaced_pod(pod.metadata.name,
util.NAMESPACE, pod)
# Copy the KVS config into all recently created pods.
os.system('cp %s ./aft-config.yml' % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(
client, 'aft-config.yml', pname,
'/go/src/github.com/Alchem-Lab/aft/config', cname)
os.system('rm ./aft-config.yml')
def add_nodes(client, apps_client, cfile, kinds, counts, create=False):
for i in range(len(kinds)):
print('Adding %d %s server node(s) to cluster...' %
(counts[i], kinds[i]))
# get the previous number of nodes of type kind that are running
prev_count = util.get_previous_count(client, kinds[i])
# we only add new nodes if we didn't pass in a node IP
util.run_process(['./modify_ig.sh', kinds[i],
str(counts[i] + prev_count)])
util.run_process(['./validate_cluster.sh'])
kops_ip = util.get_pod_ips(client, 'role=kops')[0]
route_ips = util.get_pod_ips(client, 'role=routing')
if len(route_ips) > 0:
seed_ip = random.choice(route_ips)
else:
seed_ip = ''
mon_str = ' '.join(util.get_pod_ips(client, 'role=monitoring'))
route_str = ' '.join(route_ips)
sched_str = ' '.join(util.get_pod_ips(client, 'role=scheduler'))
route_addr = util.get_service_address(client, 'routing-service')
function_addr = util.get_service_address(client, 'function-service')
# create should only be true when the DaemonSet is being created for the
# first time -- i.e., when this is called from create_cluster
if create:
for i in range(len(kinds)):
kind = kinds[i]
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'ROUTING_IPS', route_str)
util.replace_yaml_val(env, 'ROUTE_ADDR', route_addr)
util.replace_yaml_val(env, 'SCHED_IPS', sched_str)
util.replace_yaml_val(env, 'FUNCTION_ADDR', function_addr)
util.replace_yaml_val(env, 'MON_IPS', mon_str)
util.replace_yaml_val(env, 'MGMT_IP', kops_ip)
util.replace_yaml_val(env, 'SEED_IP', seed_ip)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# wait until all pods of this kind are running
res = []
while len(res) != counts[i]:
res = util.get_pod_ips(client, 'role='+kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
for pod in pods:
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
os.system('cp %s ./kvs-config.yml' % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(client, 'kvs-config.yml', pname,
'/fluent/conf/', cname)
os.system('rm ./kvs-config.yml')