def find_first_service(s):
'''find the FIRST listed service with tag post-fix s
NOTE: We implicitly assume that the list of AWS services is fixed.
We should fix that by identifying the leader nodes among each service
type.
'''
for v in self.services:
if v.tags[0]['Value']==get_tag(s):
return(v.private_ip_address)
def find_first_service(s):
'''find the FIRST listed service with tag post-fix s
NOTE: We implicitly assume that the list of AWS services is fixed.
We should fix that by identifying the leader nodes among each service
type.
'''
for v in self.services:
if v.tags[0]['Value'] == get_tag(s):
return (v.private_ip_address)
def __init__(self):
ec2 = boto3.resource('ec2')
client = boto3.client('ec2')
filt = [{
'Name': 'instance-state-name',
'Values': ['running']
}, {
'Name': 'tag-value',
'Values': [get_tag(s + '-node') for s in services]
}]
self.services = []
for v in ec2.instances.filter(Filters=filt):
self.services.append(v)
def make_config_file(self, filename):
'''create a straw config file for AWS'''
def find_first_service(s):
'''find the FIRST listed service with tag post-fix s
NOTE: We implicitly assume that the list of AWS services is fixed.
We should fix that by identifying the leader nodes among each service
type.
'''
for v in self.services:
if v.tags[0]['Value'] == get_tag(s):
return (v.private_ip_address)
with open(filename, "w") as f:
header = """#
# config for straw.storm application
#
"""
f.write(header)
elasticsearch = """
# elasticsearch settings
elasticsearch_host={0}
elasticsearch_port=9300
elasticsearch_cluster_name={1}
index_name=documents
document_type=document
""".format(find_first_service("elasticsearch-node"),
get_tag("elasticsearch-cluster"))
f.write(elasticsearch)
kafka = """
# kafka settings
zookeeper_host={0}
zookeeper_port=2181
kafka_query_topic=queries
kafka_document_topic=documents
""".format(find_first_service("kafka-node"))
f.write(kafka)
redis = """
# redis
redis_host={0}
redis_port=6379
""".format(find_first_service("flask-node"))
f.write(redis)
print("Wrote config file {0}.".format(f.name))
def make_config_file(self, filename):
'''create a straw config file for AWS'''
def find_first_service(s):
'''find the FIRST listed service with tag post-fix s
NOTE: We implicitly assume that the list of AWS services is fixed.
We should fix that by identifying the leader nodes among each service
type.
'''
for v in self.services:
if v.tags[0]['Value']==get_tag(s):
return(v.private_ip_address)
with open(filename,"w") as f:
header = """#
# config for straw.storm application
#
"""
f.write(header)
elasticsearch = """
# elasticsearch settings
elasticsearch_host={0}
elasticsearch_port=9300
elasticsearch_cluster_name={1}
index_name=documents
document_type=document
""".format(find_first_service("elasticsearch-node"), get_tag("elasticsearch-cluster"))
f.write(elasticsearch)
kafka = """
# kafka settings
zookeeper_host={0}
zookeeper_port=2181
kafka_query_topic=queries
kafka_document_topic=documents
""".format(find_first_service("kafka-node"))
f.write(kafka)
redis = """
# redis
redis_host={0}
redis_port=6379
""".format(find_first_service("flask-node"))
f.write(redis)
print("Wrote config file {0}.".format(f.name))
# Configure Kafka on ec2 instances
#
import boto3, os, argparse, sys
sys.path.append("..")
from botocore.exceptions import ClientError as BotoClientError
from time import sleep
from create_clusters import get_tag, keyfile
from config_utils import quiet_wrap
# configuration
my_instances_filters = [{
'Name': 'instance-state-name',
'Values': ['running']
}, {
'Name': 'tag-value',
'Values': [get_tag('storm-node')]
}]
if __name__ == "__main__":
# argument help
parser = argparse.ArgumentParser(
description='Configure the storm cluster.')
parser.add_argument('--elasticsearch',
help='Collocate elasticsearch with Storm cluster.',
action='store_true')
args = parser.parse_args()
# find all the host nodes
ec2 = boto3.resource('ec2')
hosts = []
#!/usr/bin/python3
#
# Configure Kafka on ec2 instances
#
import boto3, os, sys
sys.path.append("..")
from botocore.exceptions import ClientError as BotoClientError
from time import sleep
from create_clusters import get_tag, keyfile
# configuration
my_instances_filters = [{ 'Name': 'instance-state-name', 'Values': ['running']}, {'Name':'tag-value', 'Values':[get_tag('elasticsearch-node')]}]
if __name__=="__main__":
# find all the host nodes
ec2 = boto3.resource('ec2')
hosts = []
private_ips = []
reservations = ec2.instances.filter( Filters = my_instances_filters )
for instance in reservations:
print("ID: {0:<15}\tIP: {1:<15}".format(instance.instance_id, instance.public_ip_address))
hosts.append(instance.public_ip_address)
private_ips.append(instance.private_ip_address)
if len(hosts) != len(private_ips):
raise(RuntimeError("Host and private ips not consistent!"))
if len(hosts) == 0:
raise(RuntimeError("No hosts found."))
#!/usr/bin/python3
#
# Configure Kafka on ec2 instances
#
import boto3, os, sys
sys.path.append("..")
from botocore.exceptions import ClientError as BotoClientError
from time import sleep
from create_clusters import get_tag, keyfile
from config_utils import quiet_wrap
# configuration
my_instances_filters = [
{"Name": "instance-state-name", "Values": ["running"]},
{"Name": "tag-value", "Values": [get_tag("flask-node")]},
]
if __name__ == "__main__":
# find all the host nodes
ec2 = boto3.resource("ec2")
hosts = []
private_ips = []
public_dns = []
reservations = ec2.instances.filter(Filters=my_instances_filters)
for instance in reservations:
print("ID: {0:<15}\tIP: {1:<15}".format(instance.instance_id, instance.public_ip_address))
hosts.append(instance.public_ip_address)
private_ips.append(instance.private_ip_address)
public_dns.append(instance.public_dns_name)
#!/usr/bin/python3
#
# Configure Kafka on ec2 instances
#
import boto3, os, sys
sys.path.append("..")
from botocore.exceptions import ClientError as BotoClientError
from time import sleep
from create_clusters import get_tag, keyfile
# configuration
my_instances_filters = [
{"Name": "instance-state-name", "Values": ["running"]},
{"Name": "tag-value", "Values": [get_tag("elasticsearch-node")]},
]
if __name__ == "__main__":
# find all the host nodes
ec2 = boto3.resource("ec2")
hosts = []
private_ips = []
reservations = ec2.instances.filter(Filters=my_instances_filters)
for instance in reservations:
print("ID: {0:<15}\tIP: {1:<15}".format(instance.instance_id, instance.public_ip_address))
hosts.append(instance.public_ip_address)
private_ips.append(instance.private_ip_address)
if len(hosts) != len(private_ips):
raise (RuntimeError("Host and private ips not consistent!"))
#!/usr/bin/python3
#
# Configure Kafka on ec2 instances
#
import boto3, os, sys
from botocore.exceptions import ClientError as BotoClientError
from time import sleep
sys.path.append("..")
from create_clusters import get_tag, keyfile
# configuration
my_instances_filters = [{ 'Name': 'instance-state-name', 'Values': ['running']}, {'Name':'tag-value', 'Values':[get_tag('spark-node')]}]
if __name__=="__main__":
# find all the host nodes
ec2 = boto3.resource('ec2')
hosts = []
private_ips = []
reservations = ec2.instances.filter( Filters = my_instances_filters )
for instance in reservations:
print("ID: {0:<15}\tIP: {1:<15}".format(instance.instance_id, instance.public_ip_address))
hosts.append(instance.public_ip_address)
private_ips.append(instance.private_ip_address)
if len(hosts) != len(private_ips):
raise(RuntimeError("Host and private ips not consistent!"))
if len(hosts) == 0:
raise(RuntimeError("No hosts found."))
def __init__(self):
ec2 = boto3.resource('ec2')
client= boto3.client('ec2')
filt=[{'Name': 'instance-state-name', 'Values': ['running']},{'Name':'tag-value','Values':[get_tag(s+'-node') for s in services]}]
self.services = []
for v in ec2.instances.filter(Filters=filt):
self.services.append(v)
#!/usr/bin/python3
#
# Configure Kafka on ec2 instances
#
import boto3, os, argparse, sys
sys.path.append("..")
from botocore.exceptions import ClientError as BotoClientError
from time import sleep
from create_clusters import get_tag, keyfile
from config_utils import quiet_wrap
# configuration
my_instances_filters = [{ 'Name': 'instance-state-name', 'Values': ['running']}, {'Name':'tag-value', 'Values':[get_tag('storm-node')]}]
if __name__=="__main__":
# argument help
parser = argparse.ArgumentParser(description='Configure the storm cluster.')
parser.add_argument('--elasticsearch', help='Collocate elasticsearch with Storm cluster.', action='store_true')
args = parser.parse_args()
# find all the host nodes
ec2 = boto3.resource('ec2')
hosts = []
private_ips = []
reservations = ec2.instances.filter( Filters = my_instances_filters )
for instance in reservations:
print("ID: {0:<15}\tIP: {1:<15}".format(instance.instance_id, instance.public_ip_address))
hosts.append(instance.public_ip_address)
private_ips.append(instance.private_ip_address)
#!/usr/bin/python3
#
# Configure Kafka on ec2 instances
#
import boto3, os, sys
sys.path.append("..")
from botocore.exceptions import ClientError as BotoClientError
from time import sleep
from create_clusters import get_tag, keyfile
from config_utils import quiet_wrap
# configuration
my_instances_filters = [{ 'Name': 'instance-state-name', 'Values': ['running']}, {'Name':'tag-value', 'Values':[get_tag('flask-node')]}]
if __name__=="__main__":
# find all the host nodes
ec2 = boto3.resource('ec2')
hosts = []
private_ips = []
public_dns = []
reservations = ec2.instances.filter( Filters = my_instances_filters )
for instance in reservations:
print("ID: {0:<15}\tIP: {1:<15}".format(instance.instance_id, instance.public_ip_address))
hosts.append(instance.public_ip_address)
private_ips.append(instance.private_ip_address)
public_dns.append(instance.public_dns_name)
if len(hosts) != len(private_ips):
raise(RuntimeError("Host and private ips not consistent!"))
#!/usr/bin/python3
#
# Configure Kafka on ec2 instances
#
import boto3, os, sys
from botocore.exceptions import ClientError as BotoClientError
from time import sleep
sys.path.append("..")
from create_clusters import get_tag, keyfile
# configuration
my_instances_filters = [
{"Name": "instance-state-name", "Values": ["running"]},
{"Name": "tag-value", "Values": [get_tag("kafka-node")]},
]
if __name__ == "__main__":
# find all the host nodes
ec2 = boto3.resource("ec2")
hosts = []
private_ips = []
reservations = ec2.instances.filter(Filters=my_instances_filters)
for instance in reservations:
print("ID: {0:<15}\tIP: {1:<15}".format(instance.instance_id, instance.public_ip_address))
hosts.append(instance.public_ip_address)
private_ips.append(instance.private_ip_address)
if len(hosts) != len(private_ips):
