def run(): context = zmq.Context(1) client = util.init_k8s() node_add_socket = context.socket(zmq.PULL) node_add_socket.bind('ipc:///tmp/node_add') node_remove_socket = context.socket(zmq.PULL) node_remove_socket.bind('ipc:///tmp/node_remove') poller = zmq.Poller() poller.register(node_add_socket, zmq.POLLIN) poller.register(node_remove_socket, zmq.POLLIN) cfile = '/fluent/conf/kvs-base.yml' while True: socks = dict(poller.poll(timeout=1000)) if node_add_socket in socks and socks[node_add_socket] == zmq.POLLIN: msg = node_add_socket.recv_string() args = msg.split(':') ntype = args[0] num = int(args[1]) logging.info('Adding %d new %s node(s)...' % (num, ntype)) mon_ips = util.get_pod_ips(client, 'role=monitoring') route_ips = util.get_pod_ips(client, 'role=routing') scheduler_ips = util.get_pod_ips(client, 'role=scheduler') route_addr = util.get_service_address(client, 'routing-service') add_nodes(client, cfile, [ntype], [num], mon_ips, route_ips=route_ips, route_addr=route_addr, scheduler_ips=scheduler_ips) logging.info('Successfully added %d %s node(s).' % (num, ntype)) if node_remove_socket in socks and socks[node_remove_socket] == \ zmq.POLLIN: msg = node_remove_socket.recv_string() args = msg.split(':') ntype = args[0] ip = args[1] remove_node(ip, ntype) logging.info('Successfully removed node %s.' % (ip))
def add_nodes(client, apps_client, cfile, kind, count, aws_key_id=None, aws_key=None, create=False, prefix=None, branch="master"): print('Adding %d %s server node(s) to cluster...' % (count, kind)) prev_count = util.get_previous_count(client, kind) util.run_process(['./modify_ig.sh', kind, str(count + prev_count)], 'kops') util.run_process(['./validate_cluster.sh'], 'kops') if create: fname = 'yaml/ds/%s-ds.yml' % kind yml = util.load_yaml(fname, prefix) for container in yml['spec']['template']['spec']['containers']: env = container['env'] util.replace_yaml_val(env, 'BRANCH', branch) util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id) util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key) if kind == "tasc": routing_svc = util.get_service_address(client, 'routing-service') util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc) monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0] util.replace_yaml_val(env, 'MONITOR', monitor_ip) worker_svc = util.get_service_address(client, 'worker-service') util.replace_yaml_val(env, 'WORKER_ILB', worker_svc) if kind == "keynode": monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0] util.replace_yaml_val(env, 'MONITOR', monitor_ip) if kind == 'worker': monitor_ip = util.get_node_ips(client, 'role=monitor', 'ExternalIP')[0] util.replace_yaml_val(env, 'MONITOR', monitor_ip) routing_svc = util.get_service_address(client, 'routing-service') util.replace_yaml_val(env, 'ROUTING_ILB', routing_svc) apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE, body=yml) # Wait until all pods of this kind are running res = [] while len(res) != count: res = util.get_pod_ips(client, 'role=' + kind, is_running=True) created_pods = [] pods = client.list_namespaced_pod(namespace=util.NAMESPACE, label_selector='role=' + kind).items # Send kube config to lb if kind == 'lb': kubecfg = os.path.join(os.environ['HOME'], '.kube/config') for pod in pods: cname = pod.spec.containers[0].name util.copy_file_to_pod(client, kubecfg, pod.metadata.name, '/root/.kube', cname) # Generate list of all recently created pods. created_pod_ips = [] for pod in pods: created_pod_ips.append(pod.status.pod_ip) pname = pod.metadata.name for container in pod.spec.containers: cname = container.name created_pods.append((pname, cname)) # Copy the KVS config into all recently created pods. cfile_name = './tasc-config.yml' if kind != 'routing' else './anna-config.yml' cfile_dir = '/go/src/github.com/saurav-c/tasc/config' if kind != 'routing' else 'hydro/anna/conf' os.system(str('cp %s ' + cfile_name) % cfile) for pname, cname in created_pods: util.copy_file_to_pod(client, cfile_name[2:], pname, cfile_dir, cname) os.system('rm ' + cfile_name)
def create_cluster(mem_count, ebs_count, func_count, sched_count, route_count, bench_count, cfile, ssh_key, cluster_name, kops_bucket, aws_key_id, aws_key): # create the cluster object with kops util.run_process( ['./create_cluster_object.sh', cluster_name, kops_bucket, ssh_key]) client, apps_client = util.init_k8s() # create the kops pod print('Creating management pods...') kops_spec = util.load_yaml('yaml/pods/kops-pod.yml') env = kops_spec['spec']['containers'][0]['env'] util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id) util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key) util.replace_yaml_val(env, 'KOPS_STATE_STORE', kops_bucket) util.replace_yaml_val(env, 'FLUENT_CLUSTER_NAME', cluster_name) client.create_namespaced_pod(namespace=util.NAMESPACE, body=kops_spec) # wait for the kops pod to start kops_ip = util.get_pod_ips(client, 'role=kops', is_running=True)[0] # copy kube config file to kops pod, so it can execute kubectl commands kops_podname = kops_spec['metadata']['name'] kcname = kops_spec['spec']['containers'][0]['name'] os.system('cp %s kvs-config.yml' % cfile) util.copy_file_to_pod(client, '/home/ubuntu/.kube/config', kops_podname, '/root/.kube/', kcname) util.copy_file_to_pod(client, ssh_key, kops_podname, '/root/.ssh/', kcname) util.copy_file_to_pod(client, ssh_key + '.pub', kops_podname, '/root/.ssh/', kcname) util.copy_file_to_pod(client, 'kvs-config.yml', kops_podname, '/fluent/conf/', kcname) # start the monitoring pod mon_spec = util.load_yaml('yaml/pods/monitoring-pod.yml') util.replace_yaml_val(mon_spec['spec']['containers'][0]['env'], 'MGMT_IP', kops_ip) client.create_namespaced_pod(namespace=util.NAMESPACE, body=mon_spec) util.get_pod_ips(client, 'role=monitoring') # copy config file into monitoring pod -- wait till we create routing pods, # so we're sure that the monitoring nodes are up and running util.copy_file_to_pod(client, 'kvs-config.yml', mon_spec['metadata']['name'], '/fluent/conf/', mon_spec['spec']['containers'][0]['name']) os.system('rm kvs-config.yml') print('Creating %d routing nodes...' % (route_count)) add_nodes(client, apps_client, cfile, ['routing'], [route_count], True) util.get_pod_ips(client, 'role=routing') print('Creating %d memory, %d ebs node(s)...' % (mem_count, ebs_count)) add_nodes(client, apps_client, cfile, ['memory', 'ebs'], [mem_count, ebs_count], True) print('Creating routing service...') service_spec = util.load_yaml('yaml/services/routing.yml') client.create_namespaced_service(namespace=util.NAMESPACE, body=service_spec) print('Adding %d scheduler nodes...' % (sched_count)) add_nodes(client, apps_client, cfile, ['scheduler'], [sched_count], True) util.get_pod_ips(client, 'role=scheduler') print('Adding %d function serving nodes...' % (func_count)) add_nodes(client, apps_client, cfile, ['function'], [func_count], True) print('Creating function service...') service_spec = util.load_yaml('yaml/services/function.yml') client.create_namespaced_service(namespace=util.NAMESPACE, body=service_spec) print('Adding %d benchmark nodes...' % (bench_count)) add_nodes(client, apps_client, cfile, ['benchmark'], [bench_count], True) print('Finished creating all pods...') os.system('touch setup_complete') util.copy_file_to_pod(client, 'setup_complete', kops_podname, '/fluent', kcname) os.system('rm setup_complete') sg_name = 'nodes.' + cluster_name sg = ec2_client.describe_security_groups(Filters=[{ 'Name': 'group-name', 'Values': [sg_name] }])['SecurityGroups'][0] print('Authorizing ports for routing service...') permission = [{ 'FromPort': 6200, 'IpProtocol': 'tcp', 'ToPort': 6203, 'IpRanges': [{ 'CidrIp': '0.0.0.0/0' }] }] ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'], IpPermissions=permission) routing_svc_addr = util.get_service_address(client, 'routing-service') function_svc_addr = util.get_service_address(client, 'function-service') print('The routing service can be accessed here: \n\t%s' % (routing_svc_addr)) print('The function service can be accessed here: \n\t%s' % (function_svc_addr))
def add_nodes(client, apps_client, cfile, kinds, counts, create=False): for i in range(len(kinds)): print('Adding %d %s server node(s) to cluster...' % (counts[i], kinds[i])) # get the previous number of nodes of type kind that are running prev_count = util.get_previous_count(client, kinds[i]) # we only add new nodes if we didn't pass in a node IP util.run_process(['./modify_ig.sh', kinds[i], str(counts[i] + prev_count)]) util.run_process(['./validate_cluster.sh']) kops_ip = util.get_pod_ips(client, 'role=kops')[0] route_ips = util.get_pod_ips(client, 'role=routing') if len(route_ips) > 0: seed_ip = random.choice(route_ips) else: seed_ip = '' mon_str = ' '.join(util.get_pod_ips(client, 'role=monitoring')) route_str = ' '.join(route_ips) sched_str = ' '.join(util.get_pod_ips(client, 'role=scheduler')) route_addr = util.get_service_address(client, 'routing-service') function_addr = util.get_service_address(client, 'function-service') # create should only be true when the DaemonSet is being created for the # first time -- i.e., when this is called from create_cluster if create: for i in range(len(kinds)): kind = kinds[i] fname = 'yaml/ds/%s-ds.yml' % kind yml = util.load_yaml(fname) for container in yml['spec']['template']['spec']['containers']: env = container['env'] util.replace_yaml_val(env, 'ROUTING_IPS', route_str) util.replace_yaml_val(env, 'ROUTE_ADDR', route_addr) util.replace_yaml_val(env, 'SCHED_IPS', sched_str) util.replace_yaml_val(env, 'FUNCTION_ADDR', function_addr) util.replace_yaml_val(env, 'MON_IPS', mon_str) util.replace_yaml_val(env, 'MGMT_IP', kops_ip) util.replace_yaml_val(env, 'SEED_IP', seed_ip) apps_client.create_namespaced_daemon_set(namespace=util.NAMESPACE, body=yml) # wait until all pods of this kind are running res = [] while len(res) != counts[i]: res = util.get_pod_ips(client, 'role='+kind, is_running=True) created_pods = [] pods = client.list_namespaced_pod(namespace=util.NAMESPACE, label_selector='role=' + kind).items for pod in pods: pname = pod.metadata.name for container in pod.spec.containers: cname = container.name created_pods.append((pname, cname)) os.system('cp %s ./kvs-config.yml' % cfile) for pname, cname in created_pods: util.copy_file_to_pod(client, 'kvs-config.yml', pname, '/fluent/conf/', cname) os.system('rm ./kvs-config.yml')
def create_cluster(txn_count, keynode_count, rtr_count, worker_count, lb_count, benchmark_count, config_file, branch_name, ssh_key, cluster_name, kops_bucket, aws_key_id, aws_key, anna_config_file): prefix = './' util.run_process(['./create_cluster_object.sh', kops_bucket, ssh_key], 'kops') client, apps_client = util.init_k8s() print('Creating Monitor Node...') add_nodes(client, apps_client, config_file, "monitor", 1, aws_key_id, aws_key, True, prefix, branch_name) print('Creating %d Anna Routing Nodes...' % (rtr_count)) add_nodes(client, apps_client, anna_config_file, "routing", rtr_count, aws_key_id, aws_key, True, prefix, branch_name) print('Creating routing service...') service_spec = util.load_yaml('yaml/services/routing.yml', prefix) client.create_namespaced_service(namespace=util.NAMESPACE, body=service_spec) util.get_service_address(client, 'routing-service') print('Creating %d Key Nodes...' % (keynode_count)) add_nodes(client, apps_client, config_file, "keynode", keynode_count, aws_key_id, aws_key, True, prefix, branch_name) print('Creating %d Worker Nodes...' % (worker_count)) add_nodes(client, apps_client, config_file, "worker", worker_count, aws_key_id, aws_key, True, prefix, branch_name) print('Creating Worker Service...') service_spec = util.load_yaml('yaml/services/worker.yml', prefix) client.create_namespaced_service(namespace=util.NAMESPACE, body=service_spec) util.get_service_address(client, 'worker-service') print('Creating %d TASC nodes...' % (txn_count)) add_nodes(client, apps_client, config_file, 'tasc', txn_count, aws_key_id, aws_key, True, prefix, branch_name) print('Creating %d Load Balancers...' % (lb_count)) add_nodes(client, apps_client, config_file, 'lb', lb_count, aws_key_id, aws_key, True, prefix, branch_name) print('Creating TASC Load Balancing service...') service_spec = util.load_yaml('yaml/services/tasc.yml', prefix) client.create_namespaced_service(namespace=util.NAMESPACE, body=service_spec) print('Creating %d Benchmark nodes...' % (benchmark_count)) add_nodes(client, apps_client, config_file, 'benchmark', benchmark_count, aws_key_id, aws_key, True, prefix, branch_name) benchmark_ips = util.get_node_ips(client, 'role=benchmark', 'ExternalIP') with open('../cmd/benchmark/benchmarks.txt', 'w+') as f: for ip in benchmark_ips: f.write(ip + '\n') print('Finished creating all pods...') sg_name = 'nodes.' + cluster_name sg = ec2_client.describe_security_groups(Filters=[{ 'Name': 'group-name', 'Values': [sg_name] }])['SecurityGroups'][0] print("Authorizing Ports for TASC...") permission = [{ 'FromPort': 0, 'IpProtocol': 'tcp', 'ToPort': 65535, 'IpRanges': [{ 'CidrIp': '0.0.0.0/0' }] }] ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'], IpPermissions=permission) print('Registering Key Nodes...') keynode_pod_ips = util.get_pod_ips(client, 'role=keynode', is_running=True) register(client, keynode_pod_ips) print("\nThe TASC ELB Endpoint: " + util.get_service_address(client, "tasc-service") + "\n") print('Finished!')