def run():
context = zmq.Context(1)
restart_pull_socket = context.socket(zmq.REP)
restart_pull_socket.bind('tcp://*:7000')
churn_pull_socket = context.socket(zmq.PULL)
churn_pull_socket.bind('tcp://*:7001')
poller = zmq.Poller()
poller.register(restart_pull_socket, zmq.POLLIN)
poller.register(churn_pull_socket, zmq.POLLIN)
# waits until the kubecfg file gets copied into the pod -- this might be
# brittle if we try to move to a non-Ubuntu setting, but I'm not worried
# about that for now
while not os.path.isfile('/root/.kube/config'):
pass
client = util.init_k8s()
while True:
socks = dict(poller.poll())
if churn_pull_socket in socks and socks[churn_pull_socket] == \
zmq.POLLIN:
msg = churn_pull_socket.recv_string()
args = msg.split(':')
if args[0] == 'add':
num = int(args[1])
ntype = args[2]
logging.info('Adding %d new %s node(s)...' % (num, ntype))
mon_ips = util.get_pod_ips(client, 'role=monitoring')
route_ips = util.get_pod_ips(client, 'role=routing')
add_nodes(client, [ntype], [num], mon_ips, route_ips)
logging.info('Successfully added %d %s node(s).' %
(num, ntype))
elif args[0] == 'remove':
ip = args[1]
ntype = args[2]
remove_node(ip, ntype)
logging.info('Successfully removed node %s.' % (ip))
if restart_pull_socket in socks and socks[restart_pull_socket] == \
zmq.POLLIN:
msg = restart_pull_socket.recv_string()
args = msg.split(':')
ip = args[1]
pod = util.get_pod_from_ip(client, ip)
count = str(pod.status.container_statuses[0].restart_count)
logging.info('Returning restart count ' + count + ' for IP ' + ip +
'.')
restart_pull_socket.send_string(count)
def check_unused_nodes(client):
kinds = ['ebs', 'memory']
for kind in kinds:
selector = 'role=' + kind
nodes = {}
for node in client.list_node(label_selector=selector).items:
ip = list(
filter(lambda address: address.type == 'InternalIP',
node.status.addresses))[0].address
nodes[ip] = node
node_ips = nodes.keys()
pod_ips = util.get_pod_ips(client, selector)
unallocated = set(node_ips) - set(pod_ips)
mon_ips = util.get_pod_ips(client, 'role=monitoring')
route_ips = util.get_pod_ips(client, 'role=routing')
logging.info('Found %d unallocated %s nodes.' %
(len(unallocated), kind))
for node_ip in unallocated:
# note that the last argument is a list of lists
add_nodes(client, [kind], [1], mon_ips, route_ips,
[[nodes[node_ip]]])
def add(count):
prev_ips = util.get_pod_ips(client, 'role=aft', is_running=True)
add_nodes(client, apps_client, BASE_CONFIG_FILE, ['aft'], [count], "",
aws_key_id, aws_key, False, './')
new_ips = util.get_pod_ips(client, 'role=aft', is_running=True)
while len(new_ips) - len(prev_ips) != count:
new_ips = util.get_pod_ips(client, 'role=aft', is_running=True)
print('New AFT Nodes created')
# Restart all AFT nodes
restart_all()
def run():
context = zmq.Context(1)
client = util.init_k8s()
node_add_socket = context.socket(zmq.PULL)
node_add_socket.bind('ipc:///tmp/node_add')
node_remove_socket = context.socket(zmq.PULL)
node_remove_socket.bind('ipc:///tmp/node_remove')
poller = zmq.Poller()
poller.register(node_add_socket, zmq.POLLIN)
poller.register(node_remove_socket, zmq.POLLIN)
cfile = '/fluent/conf/kvs-base.yml'
while True:
socks = dict(poller.poll(timeout=1000))
if node_add_socket in socks and socks[node_add_socket] == zmq.POLLIN:
msg = node_add_socket.recv_string()
args = msg.split(':')
ntype = args[0]
num = int(args[1])
logging.info('Adding %d new %s node(s)...' % (num, ntype))
mon_ips = util.get_pod_ips(client, 'role=monitoring')
route_ips = util.get_pod_ips(client, 'role=routing')
scheduler_ips = util.get_pod_ips(client, 'role=scheduler')
route_addr = util.get_service_address(client, 'routing-service')
add_nodes(client,
cfile, [ntype], [num],
mon_ips,
route_ips=route_ips,
route_addr=route_addr,
scheduler_ips=scheduler_ips)
logging.info('Successfully added %d %s node(s).' % (num, ntype))
if node_remove_socket in socks and socks[node_remove_socket] == \
zmq.POLLIN:
msg = node_remove_socket.recv_string()
args = msg.split(':')
ntype = args[0]
ip = args[1]
remove_node(ip, ntype)
logging.info('Successfully removed node %s.' % (ip))
def add(kind, count):
prev_pod_ips = None
if kind == 'keynode':
prev_pod_ips = util.get_pod_ips(client,
'role=' + kind,
is_running=True)
add_nodes(client, apps_client, BASE_CONFIG_FILE, kind, count, aws_key_id,
aws_key, True, './', 'master')
if prev_pod_ips is not None:
pod_ips = util.get_pod_ips(client, 'role=' + kind, is_running=True)
while len(pod_ips) != len(prev_pod_ips) + count:
pod_ips = util.get_pod_ips(client, 'role=' + kind, is_running=True)
created_pod_ips = list(set(pod_ips) - set(prev_pod_ips))
# Register new keynodes with routers
if kind == 'keynodes':
register(client, created_pod_ips)
def create_cluster(mem_count, ebs_count, func_count, sched_count, route_count,
bench_count, cfile, ssh_key, cluster_name, kops_bucket,
aws_key_id, aws_key):
# create the cluster object with kops
util.run_process(
['./create_cluster_object.sh', cluster_name, kops_bucket, ssh_key])
client, apps_client = util.init_k8s()
# create the kops pod
print('Creating management pods...')
kops_spec = util.load_yaml('yaml/pods/kops-pod.yml')
env = kops_spec['spec']['containers'][0]['env']
util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
util.replace_yaml_val(env, 'KOPS_STATE_STORE', kops_bucket)
util.replace_yaml_val(env, 'FLUENT_CLUSTER_NAME', cluster_name)
client.create_namespaced_pod(namespace=util.NAMESPACE, body=kops_spec)
# wait for the kops pod to start
kops_ip = util.get_pod_ips(client, 'role=kops', is_running=True)[0]
# copy kube config file to kops pod, so it can execute kubectl commands
kops_podname = kops_spec['metadata']['name']
kcname = kops_spec['spec']['containers'][0]['name']
os.system('cp %s kvs-config.yml' % cfile)
util.copy_file_to_pod(client, '/home/ubuntu/.kube/config', kops_podname,
'/root/.kube/', kcname)
util.copy_file_to_pod(client, ssh_key, kops_podname, '/root/.ssh/', kcname)
util.copy_file_to_pod(client, ssh_key + '.pub', kops_podname,
'/root/.ssh/', kcname)
util.copy_file_to_pod(client, 'kvs-config.yml', kops_podname,
'/fluent/conf/', kcname)
# start the monitoring pod
mon_spec = util.load_yaml('yaml/pods/monitoring-pod.yml')
util.replace_yaml_val(mon_spec['spec']['containers'][0]['env'], 'MGMT_IP',
kops_ip)
client.create_namespaced_pod(namespace=util.NAMESPACE, body=mon_spec)
util.get_pod_ips(client, 'role=monitoring')
# copy config file into monitoring pod -- wait till we create routing pods,
# so we're sure that the monitoring nodes are up and running
util.copy_file_to_pod(client, 'kvs-config.yml',
mon_spec['metadata']['name'], '/fluent/conf/',
mon_spec['spec']['containers'][0]['name'])
os.system('rm kvs-config.yml')
print('Creating %d routing nodes...' % (route_count))
add_nodes(client, apps_client, cfile, ['routing'], [route_count], True)
util.get_pod_ips(client, 'role=routing')
print('Creating %d memory, %d ebs node(s)...' % (mem_count, ebs_count))
add_nodes(client, apps_client, cfile, ['memory', 'ebs'],
[mem_count, ebs_count], True)
print('Creating routing service...')
service_spec = util.load_yaml('yaml/services/routing.yml')
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Adding %d scheduler nodes...' % (sched_count))
add_nodes(client, apps_client, cfile, ['scheduler'], [sched_count], True)
util.get_pod_ips(client, 'role=scheduler')
print('Adding %d function serving nodes...' % (func_count))
add_nodes(client, apps_client, cfile, ['function'], [func_count], True)
print('Creating function service...')
service_spec = util.load_yaml('yaml/services/function.yml')
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Adding %d benchmark nodes...' % (bench_count))
add_nodes(client, apps_client, cfile, ['benchmark'], [bench_count], True)
print('Finished creating all pods...')
os.system('touch setup_complete')
util.copy_file_to_pod(client, 'setup_complete', kops_podname, '/fluent',
kcname)
os.system('rm setup_complete')
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print('Authorizing ports for routing service...')
permission = [{
'FromPort': 6200,
'IpProtocol': 'tcp',
'ToPort': 6203,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
routing_svc_addr = util.get_service_address(client, 'routing-service')
function_svc_addr = util.get_service_address(client, 'function-service')
print('The routing service can be accessed here: \n\t%s' %
(routing_svc_addr))
print('The function service can be accessed here: \n\t%s' %
(function_svc_addr))
def create_cluster(mem_count, ebs_count, route_count, bench_count, ssh_key,
cluster_name, kops_bucket, aws_key_id, aws_key):
# create the cluster object with kops
run_process(
['./create_cluster_object.sh', cluster_name, kops_bucket, ssh_key])
client = init_k8s()
# create the kops pod
print('Creating management pods...')
kops_spec = load_yaml('yaml/pods/kops-pod.yml')
env = kops_spec['spec']['containers'][0]['env']
replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
replace_yaml_val(env, 'KOPS_STATE_STORE', kops_bucket)
replace_yaml_val(env, 'NAME', cluster_name)
client.create_namespaced_pod(namespace=NAMESPACE, body=kops_spec)
# wait for the kops pod to start
kops_pod = client.list_namespaced_pod(namespace=NAMESPACE,
label_selector='role=kops').items[0]
while kops_pod.status.phase != 'Running':
kops_pod = client.list_namespaced_pod(
namespace=NAMESPACE, label_selector='role=kops').items[0]
kops_ip = kops_pod.status.pod_ip
# copy kube config file to kops pod, so it can execute kubectl commands
kops_podname = kops_spec['metadata']['name']
copy_file_to_pod(client, '/home/ubuntu/.kube/config', kops_podname,
'/root/.kube/')
copy_file_to_pod(client, ssh_key, kops_podname, '/root/.ssh/')
copy_file_to_pod(client, ssh_key + '.pub', kops_podname, '/root/.ssh/')
# start the monitoring pod
mon_spec = load_yaml('yaml/pods/monitoring-pod.yml')
replace_yaml_val(mon_spec['spec']['containers'][0]['env'], 'MGMT_IP',
kops_ip)
client.create_namespaced_pod(namespace=NAMESPACE, body=mon_spec)
mon_ips = get_pod_ips(client, 'role=monitoring')
print('Creating %d routing nodes...' % (route_count))
add_nodes(client, ['routing'], [route_count], mon_ips)
route_ips = get_pod_ips(client, 'role=routing')
print('Creating %d memory, %d ebs, and %d benchmark node(s)...' %
(mem_count, ebs_count, bench_count))
add_nodes(client, ['memory', 'ebs', 'benchmark'],
[mem_count, ebs_count, bench_count], mon_ips, route_ips)
print('Finished creating all pods...')
os.system('touch setup_complete')
copy_file_to_pod(client, 'setup_complete', kops_podname, '/fluent')
os.system('rm setup_complete')
print('Creating routing service...')
service_spec = load_yaml('yaml/services/routing.yml')
client.create_namespaced_service(namespace=NAMESPACE, body=service_spec)
service_name = service_spec['metadata']['name']
service = client.read_namespaced_service(namespace=NAMESPACE,
name=service_name)
while service.status.load_balancer.ingress == None or \
service.status.load_balancer.ingress[0].hostname == None:
service = client.read_namespaced_service(namespace=NAMESPACE,
name=service_name)
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
permissions = []
for i in range(4):
port = 6200 + i
permission = {
'FromPort': port,
'IpProtocol': 'tcp',
'ToPort': port,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}
permissions.append(permission)
print('Authorizing ports for routing service...')
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permissions)
print('The service can be accessed via the following address: \n\t%s' %
(service.status.load_balancer.ingress[0].hostname))
def create_cluster(replica_count, gc_count, lb_count, bench_count, cfile,
ssh_key, cluster_name, kops_bucket, aws_key_id, aws_key):
prefix = './'
util.run_process(['./create_cluster_object.sh', kops_bucket, ssh_key])
client, apps_client = util.init_k8s()
print('Creating management pod')
# management_spec = util.load_yaml('yaml/pods/management-pod.yml')
# env = management_spec['spec']['containers'][0]['env']
# util.replace_yaml_val(env, 'AWS_ACCESS_KEY_ID', aws_key_id)
# util.replace_yaml_val(env, 'AWS_SECRET_ACCESS_KEY', aws_key)
#
# client.create_namespaced_pod(namespace=util.NAMESPACE,
# body=management_spec)
# management_ip = util.get_pod_ips(client, 'role=management',
# is_running=True)[0]
management_ip = ""
print('Creating standby replicas...')
util.run_process(['./modify_ig.sh', 'standby', '1'])
util.run_process(['./validate_cluster.sh'])
print('Creating %d load balancer, %d GC replicas...' %
(lb_count, gc_count))
add_nodes(client, apps_client, cfile, ['lb', 'gc'], [lb_count, gc_count],
management_ip, aws_key_id, aws_key, True, prefix)
lb_pods = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=lb").items
kubecfg = os.path.join(os.environ['HOME'], '.kube/config')
for pod in lb_pods:
util.copy_file_to_pod(client, kubecfg, pod.metadata.name,
'/root/.kube', 'lb-container')
replica_ips = util.get_node_ips(client, 'role=gc', 'ExternalIP')
with open('gcs.txt', 'w') as f:
for ip in replica_ips:
f.write(ip + '\n')
# Wait until the monitoring pod is finished creating to get its IP address
# and then copy KVS config into the monitoring pod.
print('Creating %d Aft replicas...' % (replica_count))
add_nodes(client, apps_client, cfile, ['aft'], [replica_count],
management_ip, aws_key_id, aws_key, True, prefix)
util.get_pod_ips(client, 'role=aft')
replica_ips = util.get_node_ips(client, 'role=aft', 'ExternalIP')
with open('replicas.txt', 'w') as f:
for ip in replica_ips:
f.write(ip + '\n')
os.system('cp %s aft-config.yml' % cfile)
management_pname = management_spec['metadata']['name']
management_cname = management_spec['spec']['containers'][0]['name']
util.copy_file_to_pod(client, 'aft-config.yml', management_pname,
'/go/src/github.com/tajshaik24/aft/config',
management_cname)
util.copy_file_to_pod(client, 'replicas.txt', management_pname,
'/go/src/github.com/tajshaik24/aft',
management_cname)
util.copy_file_to_pod(client, 'gcs.txt', management_pname,
'/go/src/github.com/tajshaik24/aft',
management_cname)
util.copy_file_to_pod(client, kubecfg, management_pname, '/root/.kube/',
management_cname)
os.system('rm aft-config.yml')
os.system('rm gcs.txt')
# Copy replicas.txt to all Aft pods.
aft_pod_list = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=aft").items
aft_pod_list = list(map(lambda pod: pod.metadata.name, aft_pod_list))
for pname in aft_pod_list:
util.copy_file_to_pod(client, 'replicas.txt', pname,
'/go/src/github.com/tajshaik24/aft',
'aft-container')
gc_pod_list = client.list_namespaced_pod(namespace=util.NAMESPACE,
label_selector="role=gc").items
gc_pod_list = list(map(lambda pod: pod.metadata.name, gc_pod_list))
for pname in gc_pod_list:
util.copy_file_to_pod(client, 'replicas.txt', pname,
'/go/src/github.com/tajshaik24/aft',
'gc-container')
os.system('rm replicas.txt')
print('Adding %d benchmark nodes...' % (bench_count))
add_nodes(client, apps_client, cfile, ['benchmark'], [bench_count],
management_ip, aws_key_id, aws_key, True, prefix)
print('Finished creating all pods...')
print('Creating Aft service...')
service_spec = util.load_yaml('yaml/services/aft.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print('Authorizing ports for Aft replicas...')
permission = [{
'FromPort': 7654,
'IpProtocol': 'tcp',
'ToPort': 7656,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}, {
'FromPort': 7777,
'IpProtocol': 'tcp',
'ToPort': 7782,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}, {
'FromPort': 8000,
'IpProtocol': 'tcp',
'ToPort': 8003,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
print('Finished!')
def run():
context = zmq.Context(1)
restart_pull_socket = context.socket(zmq.REP)
restart_pull_socket.bind('tcp://*:7000')
churn_pull_socket = context.socket(zmq.PULL)
churn_pull_socket.bind('tcp://*:7001')
extant_caches_socket = context.socket(zmq.REP)
extant_caches_socket.bind('tcp://*:7002')
func_pull_socket = context.socket(zmq.PULL)
func_pull_socket.bind('tcp://*:7003')
poller = zmq.Poller()
poller.register(restart_pull_socket, zmq.POLLIN)
poller.register(churn_pull_socket, zmq.POLLIN)
poller.register(func_pull_socket, zmq.POLLIN)
poller.register(extant_caches_socket, zmq.POLLIN)
cfile = '/fluent/conf/kvs-base.yml'
# waits until the kubecfg file gets copied into the pod -- this might be
# brittle if we try to move to a non-Ubuntu setting, but I'm not worried
# about that for now
while not os.path.isfile('/root/.kube/config'):
pass
client = util.init_k8s()
func_occ_map = {}
start = time.time()
while True:
socks = dict(poller.poll(timeout=1000))
if churn_pull_socket in socks and socks[churn_pull_socket] == \
zmq.POLLIN:
msg = churn_pull_socket.recv_string()
args = msg.split(':')
if args[0] == 'add':
num = int(args[1])
ntype = args[2]
logging.info('Adding %d new %s node(s)...' % (num, ntype))
if len(args) > 3:
num_threads = args[3]
else:
num_threads = 3
mon_ips = util.get_pod_ips(client, 'role=monitoring')
route_ips = util.get_pod_ips(client, 'role=routing')
os.system('sed -i "s|%s: [0-9][0-9]*|%s: %d|g" %s' % (ntype,
ntype, num_threads, cfile))
os.system('sed -i "s|%s-cap: [0-9][0-9]*|%s: %d|g" %s' % (ntype,
ntype, num_threads * 15, cfile))
add_nodes(client, cfile, [ntype], [num], mon_ips, route_ips)
logging.info('Successfully added %d %s node(s).' % (num, ntype))
elif args[0] == 'remove':
ip = args[1]
ntype = args[2]
remove_node(ip, ntype)
logging.info('Successfully removed node %s.' % (ip))
if restart_pull_socket in socks and socks[restart_pull_socket] == \
zmq.POLLIN:
msg = restart_pull_socket.recv_string()
args = msg.split(':')
ip = args[1]
pod = util.get_pod_from_ip(client, ip)
count = str(pod.status.container_statuses[0].restart_count)
logging.info('Returning restart count ' + count + ' for IP ' + ip + '.')
restart_pull_socket.send_string(count)
if extant_caches_socket in socks and socks[extant_caches_socket] == \
zmq.POLLIN:
# It doesn't matter what is in this message
msg = extant_caches_socket.recv_string()
ks = KeySet()
for ip in util.get_pod_ips(clinet, 'role=function'):
ks.add_keys(ip)
extant_caches_socket.send_string(ks.SerializeToString())
if func_pull_socket in socks and socks[func_pull_socket] == zmq.POLLIN:
msg = func_pull_socket.recv_string()
args = msg.split('|')
ip, mutil = args[0], float(args[1])
logging.info('Received node occupancy of %.2f%% from IP %s.' %
(mutil * 100, ip))
func_occ_map[ip] = mutil
end = time.time()
if end - start > THRESHOLD:
logging.info('Checking hash ring...')
check_hash_ring(client, context)
logging.info('Checking for extra nodes...')
check_unused_nodes(client)
if func_occ_map.values():
avg_focc = reduce(lambda a, b: a + b, func_occ_map.values(), \
0) / len(func_occ_map)
else:
avg_focc = 0
logging.info('Average node occupancy is %f%%...' % (avg_focc * 100))
if avg_focc > FOCC_THRESHOLD:
mon_ips = util.get_pod_ips(client, 'role=monitoring')
route_addr = get_service_address(client, 'routing-service')
add_nodes(client, ['function'], [1], mon_ips,
route_addr=route_addr)
start = time.time()
def create_cluster(txn_count, keynode_count, rtr_count, worker_count, lb_count,
benchmark_count, config_file, branch_name, ssh_key,
cluster_name, kops_bucket, aws_key_id, aws_key,
anna_config_file):
prefix = './'
util.run_process(['./create_cluster_object.sh', kops_bucket, ssh_key],
'kops')
client, apps_client = util.init_k8s()
print('Creating Monitor Node...')
add_nodes(client, apps_client, config_file, "monitor", 1, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating %d Anna Routing Nodes...' % (rtr_count))
add_nodes(client, apps_client, anna_config_file, "routing", rtr_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating routing service...')
service_spec = util.load_yaml('yaml/services/routing.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
util.get_service_address(client, 'routing-service')
print('Creating %d Key Nodes...' % (keynode_count))
add_nodes(client, apps_client, config_file, "keynode", keynode_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating %d Worker Nodes...' % (worker_count))
add_nodes(client, apps_client, config_file, "worker", worker_count,
aws_key_id, aws_key, True, prefix, branch_name)
print('Creating Worker Service...')
service_spec = util.load_yaml('yaml/services/worker.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
util.get_service_address(client, 'worker-service')
print('Creating %d TASC nodes...' % (txn_count))
add_nodes(client, apps_client, config_file, 'tasc', txn_count, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating %d Load Balancers...' % (lb_count))
add_nodes(client, apps_client, config_file, 'lb', lb_count, aws_key_id,
aws_key, True, prefix, branch_name)
print('Creating TASC Load Balancing service...')
service_spec = util.load_yaml('yaml/services/tasc.yml', prefix)
client.create_namespaced_service(namespace=util.NAMESPACE,
body=service_spec)
print('Creating %d Benchmark nodes...' % (benchmark_count))
add_nodes(client, apps_client, config_file, 'benchmark', benchmark_count,
aws_key_id, aws_key, True, prefix, branch_name)
benchmark_ips = util.get_node_ips(client, 'role=benchmark', 'ExternalIP')
with open('../cmd/benchmark/benchmarks.txt', 'w+') as f:
for ip in benchmark_ips:
f.write(ip + '\n')
print('Finished creating all pods...')
sg_name = 'nodes.' + cluster_name
sg = ec2_client.describe_security_groups(Filters=[{
'Name': 'group-name',
'Values': [sg_name]
}])['SecurityGroups'][0]
print("Authorizing Ports for TASC...")
permission = [{
'FromPort': 0,
'IpProtocol': 'tcp',
'ToPort': 65535,
'IpRanges': [{
'CidrIp': '0.0.0.0/0'
}]
}]
ec2_client.authorize_security_group_ingress(GroupId=sg['GroupId'],
IpPermissions=permission)
print('Registering Key Nodes...')
keynode_pod_ips = util.get_pod_ips(client, 'role=keynode', is_running=True)
register(client, keynode_pod_ips)
print("\nThe TASC ELB Endpoint: " +
util.get_service_address(client, "tasc-service") + "\n")
print('Finished!')
def run():
context = zmq.Context(1)
restart_pull_socket = context.socket(zmq.REP)
restart_pull_socket.bind('tcp://*:7000')
churn_pull_socket = context.socket(zmq.PULL)
churn_pull_socket.bind('tcp://*:7001')
poller = zmq.Poller()
poller.register(restart_pull_socket, zmq.POLLIN)
poller.register(churn_pull_socket, zmq.POLLIN)
cfile = '/fluent/conf/kvs-base.yml'
# waits until the kubecfg file gets copied into the pod -- this might be
# brittle if we try to move to a non-Ubuntu setting, but I'm not worried
# about that for now
while not os.path.isfile('/root/.kube/config'):
pass
client = util.init_k8s()
start = time.time()
while True:
socks = dict(poller.poll(timeout=1000))
if churn_pull_socket in socks and socks[churn_pull_socket] == \
zmq.POLLIN:
msg = churn_pull_socket.recv_string()
args = msg.split(':')
if args[0] == 'add':
num = int(args[1])
ntype = args[2]
logging.info('Adding %d new %s node(s)...' % (num, ntype))
if len(args) > 3:
num_threads = args[3]
else:
num_threads = 3
mon_ips = util.get_pod_ips(client, 'role=monitoring')
route_ips = util.get_pod_ips(client, 'role=routing')
os.system('sed -i "s|%s: [0-9][0-9]*|%s: %d|g" %s' %
(ntype, ntype, num_threads, cfile))
os.system('sed -i "s|%s-cap: [0-9][0-9]*|%s: %d|g" %s' %
(ntype, ntype, num_threads * 15, cfile))
add_nodes(client, cfile, [ntype], [num], mon_ips, route_ips)
logging.info('Successfully added %d %s node(s).' %
(num, ntype))
elif args[0] == 'remove':
ip = args[1]
ntype = args[2]
remove_node(ip, ntype)
logging.info('Successfully removed node %s.' % (ip))
if restart_pull_socket in socks and socks[restart_pull_socket] == \
zmq.POLLIN:
msg = restart_pull_socket.recv_string()
args = msg.split(':')
ip = args[1]
pod = util.get_pod_from_ip(client, ip)
count = str(pod.status.container_statuses[0].restart_count)
logging.info('Returning restart count ' + count + ' for IP ' + ip +
'.')
restart_pull_socket.send_string(count)
end = time.time()
if end - start > THRESHOLD:
logging.info('Checking hash ring...')
check_hash_ring(client, context)
logging.info('Checking for extra nodes...')
check_unused_nodes(client)
start = time.time()