def run():
context = zmq.Context(1)
client = util.init_k8s()
node_add_socket = context.socket(zmq.PULL)
node_add_socket.bind('ipc:///tmp/node_add')
node_remove_socket = context.socket(zmq.PULL)
node_remove_socket.bind('ipc:///tmp/node_remove')
poller = zmq.Poller()
poller.register(node_add_socket, zmq.POLLIN)
poller.register(node_remove_socket, zmq.POLLIN)
cfile = '/fluent/conf/kvs-base.yml'
while True:
socks = dict(poller.poll(timeout=1000))
if node_add_socket in socks and socks[node_add_socket] == zmq.POLLIN:
msg = node_add_socket.recv_string()
args = msg.split(':')
ntype = args[0]
num = int(args[1])
logging.info('Adding %d new %s node(s)...' % (num, ntype))
mon_ips = util.get_pod_ips(client, 'role=monitoring')
route_ips = util.get_pod_ips(client, 'role=routing')
scheduler_ips = util.get_pod_ips(client, 'role=scheduler')
route_addr = util.get_service_address(client, 'routing-service')
add_nodes(client,
cfile, [ntype], [num],
mon_ips,
route_ips=route_ips,
route_addr=route_addr,
scheduler_ips=scheduler_ips)
logging.info('Successfully added %d %s node(s).' % (num, ntype))
if node_remove_socket in socks and socks[node_remove_socket] == \
zmq.POLLIN:
msg = node_remove_socket.recv_string()
args = msg.split(':')
ntype = args[0]
ip = args[1]
remove_node(ip, ntype)
logging.info('Successfully removed node %s.' % (ip))
def add_nodes(client, apps_client, cfile, kind, count, aws_key_id=None,
aws_key=None, create=False, prefix=None, branch="master"):
print('Adding %d %s server node(s) to cluster...' % (count, kind))
prev_count = util.get_previous_count(client, kind)
util.run_process(['./modify_ig.sh', kind, str(count + prev_count)], 'kops')
util.run_process(['./validate_cluster.sh'], 'kops')
if create:
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname, prefix)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'BRANCH', branch)
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
if kind == "tasc":
routing_svc = util.get_service_address(client, 'routing-service')
util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc)
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
worker_svc = util.get_service_address(client, 'worker-service')
util.replace_yaml_val(env, 'WORKER_ILB', worker_svc)
if kind == "keynode":
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
if kind == 'worker':
monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0]
util.replace_yaml_val(env, 'MONITOR', monitor_ip)
routing_svc = util.get_service_address(client, 'routing-service')
util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# Wait until all pods of this kind are running
res = []
while len(res) != count:
res = util.get_pod_ips(client, 'role=' + kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
# Send kube config to lb
if kind == 'lb':
kubecfg = os.path.join(os.environ['HOME'], '.kube/config')
for pod in pods:
cname = pod.spec.containers[0].name
util.copy_file_to_pod(client, kubecfg, pod.metadata.name,
'/root/.kube', cname)
# Generate list of all recently created pods.
created_pod_ips = []
for pod in pods:
created_pod_ips.append(pod.status.pod_ip)
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
# Copy the KVS config into all recently created pods.
cfile_name = './tasc-config.yml' if kind != 'routing' else './anna-config.yml'
cfile_dir = '/go/src/github.com/saurav-c/tasc/config' if kind != 'routing' else 'hydro/anna/conf'
os.system(str('cp %s ' + cfile_name) % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(client, cfile_name[2:], pname,
cfile_dir, cname)
os.system('rm ' + cfile_name)
def create_cluster(mem_count, ebs_count, func_count, sched_count, route_count,
bench_count, cfile, ssh_key, cluster_name, kops_bucket,
aws_key_id, aws_key):
# create the cluster object with kops
util.run_process(
['./create_cluster_object.sh', cluster_name, kops_bucket, ssh_key])
client, apps_client = util.init_k8s()
# create the kops pod
print('Creating management pods...')
kops_spec = util.load_yaml('yaml/pods/kops-pod.yml')
env = kops_spec['spec']['containers'][0]['env']
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
util.replace_yaml_val(env, 'KOPS_STATE_STORE', kops_bucket)
util.replace_yaml_val(env, 'FLUENT_CLUSTER_NAME', cluster_name)
client.create_namespaced_pod(namespace=util.NAMESPACE, body=kops_spec)
# wait for the kops pod to start
kops_ip = util.get_pod_ips(client, 'role=kops', is_running=True)[0]
# copy kube config file to kops pod, so it can execute kubectl commands
kops_podname = kops_spec['metadata']['name']
kcname = kops_spec['spec']['containers'][0]['name']
os.system('cp %s kvs-config.yml' % cfile)
util.copy_file_to_pod(client, '/home/ubuntu/.kube/config', kops_podname,
'/root/.kube/', kcname)
util.copy_file_to_pod(client, ssh_key, kops_podname, '/root/.ssh/', kcname)
util.copy_file_to_pod(client, ssh_key + '.pub', kops_podname,
'/root/.ssh/', kcname)
util.copy_file_to_pod(client, 'kvs-config.yml', kops_podname,
'/fluent/conf/', kcname)
# start the monitoring pod
mon_spec = util.load_yaml('yaml/pods/monitoring-pod.yml')
util.replace_yaml_val(mon_spec['spec']['containers'][0]['env'], 'MGMT_IP',
kops_ip)
client.create_namespaced_pod(namespace=util.NAMESPACE, body=mon_spec)
util.get_pod_ips(client, 'role=monitoring')
# copy config file into monitoring pod -- wait till we create routing pods,
# so we're sure that the monitoring nodes are up and running
util.copy_file_to_pod(client, 'kvs-config.yml',
mon_spec['metadata']['name'], '/fluent/conf/',
mon_spec['spec']['containers'][0]['name'])
os.system('rm kvs-config.yml')
print('Creating %d routing nodes...' % (route_count))
add_nodes(client, apps_client, cfile, ['routing'], [route_count], True)
util.get_pod_ips(client, 'role=routing')
print('Creating %d memory, %d ebs node(s)...' % (mem_count, ebs_count))
add_nodes(client, apps_client, cfile, ['memory', 'ebs'],
[mem_count, ebs_count], True)
print('Creating routing service...')
service_spec = util.load_yaml('yaml/services/routing.yml')
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Adding %d scheduler nodes...' % (sched_count))
add_nodes(client, apps_client, cfile, ['scheduler'], [sched_count], True)
util.get_pod_ips(client, 'role=scheduler')
print('Adding %d function serving nodes...' % (func_count))
add_nodes(client, apps_client, cfile, ['function'], [func_count], True)
print('Creating function service...')
service_spec = util.load_yaml('yaml/services/function.yml')
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Adding %d benchmark nodes...' % (bench_count))
add_nodes(client, apps_client, cfile, ['benchmark'], [bench_count], True)
print('Finished creating all pods...')
os.system('touch setup_complete')
util.copy_file_to_pod(client, 'setup_complete', kops_podname, '/fluent',
kcname)
os.system('rm setup_complete')
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print('Authorizing ports for routing service...')
permission = [{
'FromPort': 6200,
'IpProtocol': 'tcp',
'ToPort': 6203,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
routing_svc_addr = util.get_service_address(client, 'routing-service')
function_svc_addr = util.get_service_address(client, 'function-service')
print('The routing service can be accessed here: \n\t%s' %
(routing_svc_addr))
print('The function service can be accessed here: \n\t%s' %
(function_svc_addr))
def add_nodes(client, apps_client, cfile, kinds, counts, create=False):
for i in range(len(kinds)):
print('Adding %d %s server node(s) to cluster...' %
(counts[i], kinds[i]))
# get the previous number of nodes of type kind that are running
prev_count = util.get_previous_count(client, kinds[i])
# we only add new nodes if we didn't pass in a node IP
util.run_process(['./modify_ig.sh', kinds[i],
str(counts[i] + prev_count)])
util.run_process(['./validate_cluster.sh'])
kops_ip = util.get_pod_ips(client, 'role=kops')[0]
route_ips = util.get_pod_ips(client, 'role=routing')
if len(route_ips) > 0:
seed_ip = random.choice(route_ips)
else:
seed_ip = ''
mon_str = ' '.join(util.get_pod_ips(client, 'role=monitoring'))
route_str = ' '.join(route_ips)
sched_str = ' '.join(util.get_pod_ips(client, 'role=scheduler'))
route_addr = util.get_service_address(client, 'routing-service')
function_addr = util.get_service_address(client, 'function-service')
# create should only be true when the DaemonSet is being created for the
# first time -- i.e., when this is called from create_cluster
if create:
for i in range(len(kinds)):
kind = kinds[i]
fname = 'yaml/ds/%s-ds.yml' % kind
yml = util.load_yaml(fname)
for container in yml['spec']['template']['spec']['containers']:
env = container['env']
util.replace_yaml_val(env, 'ROUTING_IPS', route_str)
util.replace_yaml_val(env, 'ROUTE_ADDR', route_addr)
util.replace_yaml_val(env, 'SCHED_IPS', sched_str)
util.replace_yaml_val(env, 'FUNCTION_ADDR', function_addr)
util.replace_yaml_val(env, 'MON_IPS', mon_str)
util.replace_yaml_val(env, 'MGMT_IP', kops_ip)
util.replace_yaml_val(env, 'SEED_IP', seed_ip)
apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE,
body=yml)
# wait until all pods of this kind are running
res = []
while len(res) != counts[i]:
res = util.get_pod_ips(client, 'role='+kind, is_running=True)
created_pods = []
pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector='role=' +
kind).items
for pod in pods:
pname = pod.metadata.name
for container in pod.spec.containers:
cname = container.name
created_pods.append((pname, cname))
os.system('cp %s ./kvs-config.yml' % cfile)
for pname, cname in created_pods:
util.copy_file_to_pod(client, 'kvs-config.yml', pname,
'/fluent/conf/', cname)
os.system('rm ./kvs-config.yml')
def create_cluster(txn_count, keynode_count, rtr_count, worker_count, lb_count,
benchmark_count, config_file, branch_name, ssh_key,
cluster_name, kops_bucket, aws_key_id, aws_key,
anna_config_file):
prefix = './'
util.run_process(['./create_cluster_object.sh', kops_bucket, ssh_key],
'kops')
client, apps_client = util.init_k8s()
print('Creating Monitor Node...')
add_nodes(client, apps_client, config_file, "monitor", 1, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating %d Anna Routing Nodes...' % (rtr_count))
add_nodes(client, apps_client, anna_config_file, "routing", rtr_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating routing service...')
service_spec = util.load_yaml('yaml/services/routing.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
util.get_service_address(client, 'routing-service')
print('Creating %d Key Nodes...' % (keynode_count))
add_nodes(client, apps_client, config_file, "keynode", keynode_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating %d Worker Nodes...' % (worker_count))
add_nodes(client, apps_client, config_file, "worker", worker_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating Worker Service...')
service_spec = util.load_yaml('yaml/services/worker.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
util.get_service_address(client, 'worker-service')
print('Creating %d TASC nodes...' % (txn_count))
add_nodes(client, apps_client, config_file, 'tasc', txn_count, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating %d Load Balancers...' % (lb_count))
add_nodes(client, apps_client, config_file, 'lb', lb_count, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating TASC Load Balancing service...')
service_spec = util.load_yaml('yaml/services/tasc.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Creating %d Benchmark nodes...' % (benchmark_count))
add_nodes(client, apps_client, config_file, 'benchmark', benchmark_count,
aws_key_id, aws_key, True, prefix, branch_name)
benchmark_ips = util.get_node_ips(client, 'role=benchmark', 'ExternalIP')
with open('../cmd/benchmark/benchmarks.txt', 'w+') as f:
for ip in benchmark_ips:
f.write(ip + '\n')
print('Finished creating all pods...')
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print("Authorizing Ports for TASC...")
permission = [{
'FromPort': 0,
'IpProtocol': 'tcp',
'ToPort': 65535,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
print('Registering Key Nodes...')
keynode_pod_ips = util.get_pod_ips(client, 'role=keynode', is_running=True)
register(client, keynode_pod_ips)
print("\nThe TASC ELB Endpoint: " +
util.get_service_address(client, "tasc-service") + "\n")
print('Finished!')