def test_ec2_client(self):
ec2 = boto.ec2.connect_to_region("us-west-2")
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2)
ec2.get_all_instances()
spans = self.pop_spans()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
self.assertEqual(span.get_tag("aws.operation"), "DescribeInstances")
assert_span_http_status_code(span, 200)
self.assertEqual(span.get_tag(http.METHOD), "POST")
self.assertEqual(span.get_tag("aws.region"), "us-west-2")
self.assertIsNone(span.get_metric(ANALYTICS_SAMPLE_RATE_KEY))
# Create an instance
ec2.run_instances(21)
spans = self.pop_spans()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
assert_is_measured(span)
self.assertEqual(span.get_tag("aws.operation"), "RunInstances")
assert_span_http_status_code(span, 200)
self.assertEqual(span.get_tag(http.METHOD), "POST")
self.assertEqual(span.get_tag("aws.region"), "us-west-2")
self.assertEqual(span.service, "test-boto-tracing.ec2")
self.assertEqual(span.resource, "ec2.runinstances")
self.assertEqual(span.name, "ec2.command")
self.assertEqual(span.span_type, "http")
def test_ec2_client(self):
ec2 = boto.ec2.connect_to_region("us-west-2")
tracer = get_dummy_tracer()
writer = tracer.writer
Pin(service=self.TEST_SERVICE, tracer=tracer).onto(ec2)
ec2.get_all_instances()
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag('aws.operation'), "DescribeInstances")
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "POST")
eq_(span.get_tag('aws.region'), "us-west-2")
# Create an instance
ec2.run_instances(21)
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag('aws.operation'), "RunInstances")
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "POST")
eq_(span.get_tag('aws.region'), "us-west-2")
eq_(span.service, "test-boto-tracing.ec2")
eq_(span.resource, "ec2.runinstances")
eq_(span.name, "ec2.command")
def trackRWorkers():
print "Tracking workers"
while True:
reservations = ec2.get_all_reservations(filters={"instance-state-code" : [0, 16]})
max_workers = 16
#dynamic
#print "Number of requests " + str(requests.count()) + " and number of reservations " + str(len(reservations))
if requests.count() > 0 and len(reservations) == 2: #there are requests, but no workers
print "Starting a worker"
newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro',
security_groups=['launch-wizard-2'])
time.sleep(30)
elif len(reservations) < max_workers:
if requests.count() % 2 == 0 and len(reservations)-2 < requests.count():
print "Starting a worker"
newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro',
security_groups=['launch-wizard-2'])
time.sleep(30)
elif requests.count() % 2 == 1 and len(reservations)-2 < requests.count()-1:
print "Starting a worker"
newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro',
security_groups=['launch-wizard-2'])
time.sleep(30)
def test_ec2_client(self):
ec2 = boto.ec2.connect_to_region("us-west-2")
ec2.get_all_instances()
spans = self.memory_exporter.get_finished_spans()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
self.assertEqual(span.attributes["aws.operation"], "DescribeInstances")
assert_span_http_status_code(span, 200)
self.assertEqual(span.attributes["http.method"], "POST")
self.assertEqual(span.attributes["aws.region"], "us-west-2")
# Create an instance
ec2.run_instances(21)
spans = self.memory_exporter.get_finished_spans()
assert spans
self.assertEqual(len(spans), 2)
span = spans[1]
self.assertEqual(span.attributes["aws.operation"], "RunInstances")
assert_span_http_status_code(span, 200)
self.assertEqual(
span.resource,
Resource(attributes={
"endpoint": "ec2",
"http_method": "runinstances"
}),
)
self.assertEqual(span.attributes["http.method"], "POST")
self.assertEqual(span.attributes["aws.region"], "us-west-2")
self.assertEqual(span.name, "ec2.command")
def test_ec2_client(self):
ec2 = boto.ec2.connect_to_region("us-west-2")
tracer = get_dummy_tracer()
writer = tracer.writer
Pin(service=self.TEST_SERVICE, tracer=tracer).onto(ec2)
ec2.get_all_instances()
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag('aws.operation'), "DescribeInstances")
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "POST")
eq_(span.get_tag('aws.region'), "us-west-2")
# Create an instance
ec2.run_instances(21)
spans = writer.pop()
assert spans
eq_(len(spans), 1)
span = spans[0]
eq_(span.get_tag('aws.operation'), "RunInstances")
eq_(span.get_tag(http.STATUS_CODE), "200")
eq_(span.get_tag(http.METHOD), "POST")
eq_(span.get_tag('aws.region'), "us-west-2")
eq_(span.service, "test-boto-tracing.ec2")
eq_(span.resource, "ec2.runinstances")
eq_(span.name, "ec2.command")
def test_ec2_client(self):
ec2 = boto.ec2.connect_to_region('us-west-2')
writer = self.tracer.writer
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2)
ec2.get_all_instances()
spans = writer.pop()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances')
self.assertEqual(span.get_tag(http.STATUS_CODE), '200')
self.assertEqual(span.get_tag(http.METHOD), 'POST')
self.assertEqual(span.get_tag('aws.region'), 'us-west-2')
self.assertIsNone(span.get_metric(ANALYTICS_SAMPLE_RATE_KEY))
# Create an instance
ec2.run_instances(21)
spans = writer.pop()
assert spans
self.assertEqual(len(spans), 1)
span = spans[0]
self.assertEqual(span.get_tag('aws.operation'), 'RunInstances')
self.assertEqual(span.get_tag(http.STATUS_CODE), '200')
self.assertEqual(span.get_tag(http.METHOD), 'POST')
self.assertEqual(span.get_tag('aws.region'), 'us-west-2')
self.assertEqual(span.service, 'test-boto-tracing.ec2')
self.assertEqual(span.resource, 'ec2.runinstances')
self.assertEqual(span.name, 'ec2.command')
self.assertEqual(span.span_type, 'http')
def spawn(hints, ec2):
logging.info("About to launch {0} instance(s). Press Control-C to abort!".format(hints['count']))
for t in range(5,0,-1):
logging.info("Launching in {0}...".format(t))
time.sleep(1)
profile_suffix = '' if hints['tags']['environment'] == 'prod' else '_' + hints['tags']['environment']
instance_profile = hints['tags']['application'] + "_profile" + profile_suffix
logging.info("OK, sending launch request!")
# Make the reservation call.
if hints['public_ip']:
# http://stackoverflow.com/questions/19029588/how-to-auto-assign-public-ip-to-ec2-instance-with-boto
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id=hints['subnet_id'],
groups=hints['security_group_ids'],
associate_public_ip_address=True)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface)
reservation = ec2.run_instances(
hints['ami_id'],
instance_type = hints['instance_type'],
network_interfaces=interfaces,
instance_profile_name = instance_profile
)
else:
reservation = ec2.run_instances(
hints['ami_id'],
min_count = hints['count'],
max_count = hints['count'],
instance_type = hints['instance_type'],
subnet_id = hints['subnet_id'],
security_group_ids = hints['security_group_ids'],
instance_profile_name = instance_profile,
)
# Sleep a little bit.
logging.info("Launch request sent. Waiting for instances to be created...")
time.sleep(5)
reservation_id = reservation.id
sentinel = 0
while len(reservation.instances) != hints['count'] or [i for i in reservation.instances if i.private_ip_address is None]:
print '.'
time.sleep(2)
sentinel += 1
if sentinel > 30:
logging.error("Giving up!")
exit(11)
reservation = ec2.get_all_reservations(filters={"reservation-id":reservation_id})
hints['ips'] = {}
# Apply tags to the instances.
for idx in range(0, len(hints['names'])):
instance = reservation.instances[idx]
logging.info ("Tagging instance {0} as {1}".format(instance.id, hints['names'][idx]))
instance.add_tags(hints['tags'])
instance.add_tag('Name', value = hints['names'][idx])
# Store private IP by name
hints['ips'][hints['names'][idx]] = instance.private_ip_address
def run_instances(location, ami, num, key, sg, p, pg, it):
ec2 = boto.ec2.connect_to_region(location)
ec2.run_instances(ami,
min_count=num,
max_count=num,
key_name=key,
security_groups=[sg],
placement=p,
placement_group=pg,
instance_type=it)
def launch_instance(tag, key_name, group_name, inst_type, ami_name, user_data,
wait=True, returninfo=None):
"""
Launch a testing instance. Doesn't actually attempt to connect as
it can take quite a while between 'running' and connectability
"""
ec2 = boto.ec2.connect_to_region(AWS_REGION)
failures = 0
max_failures = 10
while True:
try:
reservation = ec2.run_instances(ami_name,
key_name=key_name,
security_groups=[group_name],
instance_type=inst_type,
user_data=None)
break
except Exception, err:
# Failed to get instance; wait 15 seconds and then try again (up to
# 10 total times)
errortext = str(err)
if errortext.find("Not authorized for images") >= 0:
print "**************************************"
print "* Error from AWS suggests that the AMI code in"
print "* CloudSetup.py is deprecated. Please go to"
print "* https://aws.amazon.com/marketplace/ and search for"
print "* \"Ubuntu server lts hvm\", selecting the most recent"
print "* version. Click \"Continue\", \"Manual Launch\","
print "* and then copy the AMI ID for the US East region."
print "* Copy that to the AMI_NAME value in CloudSetup.py"
print "* and re-run."
print "***************************************"
print "* (Full text of error):"
print errortext
print "***************************************"
return None
elif errortext.find("accept terms and subscribe") >= 0:
print "**************************************"
print "* Error from AWS suggests that you have never used this"
print "* AMI before and need to accept its terms and"
print "* subscribe to it. Please follow the link in the below"
print "* error text. Click \"Continue\", \"Manual Launch\","
print "* and \"Accept Terms\". After receiving email"
print "* confirmation, you can re-run the code."
print "**************************************"
print "* (Full text of error):"
print errortext
print "**************************************"
return None
failures += 1
if failures == max_failures:
print "**************************************"
print "* Maximum number of instance launch failures reached."
print "* (Full text of error):"
print errortext
print "**************************************"
return None
print " ** ec2.run_instances failed for tag", tag, "; waiting 15"
print " ** seconds and then trying again..."
time.sleep(15)
def start_machine(ami, instance):
ec2 = boto.ec2.connect_to_region(settings.aws_region,
aws_access_key_id=settings.aws_access,
aws_secret_access_key=settings.aws_secret)
if not ec2:
logging.error('Cannot connect to region %s' % settings.aws_region)
return None, None
worker_id = 'jm-%s' % uuid.uuid4()
logging.info('Request workerID = %s' % worker_id)
try:
reservation = ec2.run_instances(
ami,
security_groups=[settings.aws_seqgrp],
instance_type=instance,
user_data=base64.b64encode(worker_id),
instance_initiated_shutdown_behavior='terminate',
)
logging.info('Reservation %s for worker %s' %
(reservation.id, worker_id))
return worker_id, reservation.id
except Exception, e:
logging.exception('Cannot reserve instance %s for type %s (%s)' %
(ami, instance, e))
return None, None
def launch_instance(tag="mqlibtest1",key_name="mqlibtest",group_name="mqlibtest",
wait=True, returnInfo=None):
"""
Launch a testing instance. Doesn't actually attempt to connect as
it can take quite a while between 'running' and connectability
"""
ec2 = boto.ec2.connect_to_region(AWS_REGION)
failures = 0
max_failures = 10
while True:
try:
reservation = ec2.run_instances(AMI_NAME,
key_name=key_name,
security_groups=[group_name],
instance_type=INST_TYPE,
user_data=None)
break
except Exception, err:
# Failed to get instance; wait 15 seconds and then try again (up to
# 10 total times)
errorText = str(err)
if errorText.find("Not authorized for images") >= 0:
print "**************************************"
print "* Error from AWS suggests that the AMI code in"
print "* CloudSetup.py is deprecated. Please go to"
print "* https://aws.amazon.com/marketplace/ and search for"
print "* \"Ubuntu server lts hvm\", selecting the most recent"
print "* version. Click \"Continue\", \"Manual Launch\","
print "* and then copy the AMI ID for the US East region."
print "* Copy that to the AMI_NAME value in CloudSetup.py"
print "* and re-run."
print "***************************************"
print "* (Full text of error):"
print errorText
print "***************************************"
return None
elif errorText.find("accept terms and subscribe") >= 0:
print "**************************************"
print "* Error from AWS suggests that you have never used this"
print "* AMI before and need to accept its terms and"
print "* subscribe to it. Please follow the link in the below"
print "* error text. Click \"Continue\", \"Manual Launch\","
print "* and \"Accept Terms\". After receiving email"
print "* confirmation, you can re-run the code."
print "**************************************"
print "* (Full text of error):"
print errorText
print "**************************************"
return None
failures += 1
if failures == max_failures:
print "**************************************"
print "* Maximum number of instance launch failures reached."
print "* (Full text of error):"
print errorText
print "**************************************"
return None
print " ** ec2.run_instances failed for tag", tag, "; waiting 15"
print " ** seconds and then trying again..."
time.sleep(15)
def create_box():
old_ids = set(i.id for i in ec2.get_only_instances())
machine = ec2.run_instances(AMI_ID, key_name=KEYNAME, min_count=NODE_COUNT, max_count=NODE_COUNT,
security_groups=[SECURITY_GROUP,], instance_type=os.environ.get('EC2_INSTANCE_TYPE', 'm3.medium'))
new_instances = [i for i in ec2.get_only_instances() if i.id not in old_ids]
for new_instance in new_instances:
print("new instance:", new_instance.id)
ec2.create_tags([new_instance.id], {"billingProject": "aei"})
is_running = [False] * len(new_instances)
while not all(is_running):
for count, newinstance in enumerate(new_instances):
is_running[count] = new_instance.state == u'running'
time.sleep(3)
for new_instance in new_instances:
new_instance.update()
is_reachable = [False] * len(new_instances)
while not all(is_reachable):
instance_ids=[new_instance.id for new_instance in new_instances]
inst_statuses = ec2.get_all_instance_status(instance_ids=instance_ids)
is_reachable = [inst_status.system_status.details['reachability'] != 'passed' for inst_status in inst_statuses]
time.sleep(3)
time.sleep(1)
for new_instance in new_instances:
assert new_instance.public_dns_name
print("public dns name:", new_instance.public_dns_name)
return new_instances
def _create_compute(config, name):
info("[CREATE COMPUTE] " + name)
# user_data from template
j2_env = jinja2.Environment(
loader=jinja2.FileSystemLoader('/opt/necofs/templates'))
USER_DATA = j2_env.get_template('compute.user_data').render(
local_ipv4=local_ipv4)
# create a NetworkInterface in our VPC
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(
subnet_id=config['SUBNET_ID'],
groups=config['SECURITY_GROUP_IDS'],
associate_public_ip_address=True)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(
interface)
reservation = ec2.run_instances(
image_id=config['IMAGE_ID'],
key_name=config['KEY_NAME'],
user_data=USER_DATA,
instance_type=config['COMPUTE_INSTANCE_TYPE'],
instance_initiated_shutdown_behavior='terminate',
placement_group=config[
'PLACEMENT_GROUP'], # only for high level machines, not for testing
network_interfaces=interfaces,
instance_profile_name=config['IAM_ROLE'])
info("[CREATE_COMPUTE]: reservation " + reservation.id + " for " + name)
time.sleep(
30
) # -- was checking before instance was created, wait a few for Amazon to notice it's own machine
for instance in reservation.instances:
status = instance.update()
while status == 'pending':
time.sleep(5)
status = instance.update()
instance.add_tag('Name', config['INSTANCE_NAME'] + "_compute_" + name)
return reservation
def runInstance(keyName, imageId, securityGroup, instanceType): reservation = ec2.run_instances(imageId, key_name=keyName, instance_type=instanceType, security_groups=[securityGroup]) instances = reservation.instances if(len(instances) < 1): return None instance = instances[0] return instance
def launch(key_name=None, region='us-west-2', image_id='ami-5189a661',
instance_type='t2.micro', security_groups='launch-wizard-1',
user_data=None, initial_check=False):
'''
'''
if not isinstance(security_groups, list):
security_groups = [security_groups]
ec2 = boto.ec2.connect_to_region(region)
reserve = ec2.run_instances(image_id, key_name=key_name,
instance_type=instance_type,
security_groups=security_groups,
user_data=user_data)
inst = reserve.instances[0]
while inst.state == u'pending':
time.sleep(10)
inst.update()
if initial_check:
# Wait for the status checks first
status = ec2.get_all_instance_status(instance_ids=[inst.id])[0]
check_stat = "Status:initializing"
while str(status.system_status) == check_stat and str(status.instance_status) == check_stat:
time.sleep(10)
status = ec2.get_all_instance_status(instance_ids=[inst.id])[0]
return inst
def start_instance(name, data):
print('reserving instance for %s' % name)
print(data)
reserve_instance = lambda: ec2.run_instances(aws_image, key_name='sandy', user_data=data, instance_type=size, instance_profile_name=name)
reservation = while_not_try(reserve_instance)
instance = reservation.instances[0]
print(instance)
return instance
def launch_instance(region, image_id, instance_type, subnet_id, security_group): ec2 = boto.ec2.connect_to_region(region) reservation = ec2.run_instances(image_id=image_id, min_count=1, max_count=1, instance_type=instance_type, subnet_id=subnet_id, security_group_ids=[security_group]) print reservation return reservation
def createInstance(ec2,ami,nb_nodes,placement,instance_type,key,sg,user_data=None):
"""
Create a new EC2 instance with specific parameters
SecurityGroup (sg) and KeyPair (key) have to be previously created (see cassandgo initSG and cassandgo initKP)
"""
reservation = ec2.run_instances(ami,min_count=nb_nodes,max_count=nb_nodes,placement = placement,key_name=key,security_groups=[sg],instance_type=instance_type,user_data=user_data)
instance = reservation.instances[0]
return instance
def test_ec2_client_ot(self):
"""OpenTracing compatibility check of the test_ec2_client test."""
ec2 = boto.ec2.connect_to_region("us-west-2")
tracer = get_dummy_tracer()
ot_tracer = init_tracer('my_svc', tracer)
writer = tracer.writer
Pin(service=self.TEST_SERVICE, tracer=tracer).onto(ec2)
with ot_tracer.start_active_span('ot_span'):
ec2.get_all_instances()
spans = writer.pop()
assert spans
eq_(len(spans), 2)
ot_span, dd_span = spans
# confirm the parenting
eq_(ot_span.parent_id, None)
eq_(dd_span.parent_id, ot_span.span_id)
eq_(ot_span.resource, "ot_span")
eq_(dd_span.get_tag('aws.operation'), "DescribeInstances")
eq_(dd_span.get_tag(http.STATUS_CODE), "200")
eq_(dd_span.get_tag(http.METHOD), "POST")
eq_(dd_span.get_tag('aws.region'), "us-west-2")
with ot_tracer.start_active_span('ot_span'):
ec2.run_instances(21)
spans = writer.pop()
assert spans
eq_(len(spans), 2)
ot_span, dd_span = spans
# confirm the parenting
eq_(ot_span.parent_id, None)
eq_(dd_span.parent_id, ot_span.span_id)
eq_(dd_span.get_tag('aws.operation'), "RunInstances")
eq_(dd_span.get_tag(http.STATUS_CODE), "200")
eq_(dd_span.get_tag(http.METHOD), "POST")
eq_(dd_span.get_tag('aws.region'), "us-west-2")
eq_(dd_span.service, "test-boto-tracing.ec2")
eq_(dd_span.resource, "ec2.runinstances")
eq_(dd_span.name, "ec2.command")
def test_ec2_client_ot(self):
"""OpenTracing compatibility check of the test_ec2_client test."""
ec2 = boto.ec2.connect_to_region('us-west-2')
ot_tracer = init_tracer('my_svc', self.tracer)
writer = self.tracer.writer
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2)
with ot_tracer.start_active_span('ot_span'):
ec2.get_all_instances()
spans = writer.pop()
assert spans
self.assertEqual(len(spans), 2)
ot_span, dd_span = spans
# confirm the parenting
self.assertIsNone(ot_span.parent_id)
self.assertEqual(dd_span.parent_id, ot_span.span_id)
self.assertEqual(ot_span.resource, 'ot_span')
self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances')
self.assertEqual(dd_span.get_tag(http.STATUS_CODE), '200')
self.assertEqual(dd_span.get_tag(http.METHOD), 'POST')
self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2')
with ot_tracer.start_active_span('ot_span'):
ec2.run_instances(21)
spans = writer.pop()
assert spans
self.assertEqual(len(spans), 2)
ot_span, dd_span = spans
# confirm the parenting
self.assertIsNone(ot_span.parent_id)
self.assertEqual(dd_span.parent_id, ot_span.span_id)
self.assertEqual(dd_span.get_tag('aws.operation'), 'RunInstances')
self.assertEqual(dd_span.get_tag(http.STATUS_CODE), '200')
self.assertEqual(dd_span.get_tag(http.METHOD), 'POST')
self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2')
self.assertEqual(dd_span.service, 'test-boto-tracing.ec2')
self.assertEqual(dd_span.resource, 'ec2.runinstances')
self.assertEqual(dd_span.name, 'ec2.command')
def clone_instances(inst, num=1, delete_image=False, start_instance=False):
'''
num: number of clone
delete_image: delete image after clone
start_instance: start_instance after clone
'''
return ec2.run_instances(
image_id='ami-f52f1dc5',
instance_type='t2.micro',
key_name='aws-ec2')
def test_ec2_client_ot(self):
"""OpenTracing compatibility check of the test_ec2_client test."""
ec2 = boto.ec2.connect_to_region("us-west-2")
ot_tracer = init_tracer("my_svc", self.tracer)
Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2)
with ot_tracer.start_active_span("ot_span"):
ec2.get_all_instances()
spans = self.pop_spans()
assert spans
self.assertEqual(len(spans), 2)
ot_span, dd_span = spans
# confirm the parenting
self.assertIsNone(ot_span.parent_id)
self.assertEqual(dd_span.parent_id, ot_span.span_id)
self.assertEqual(ot_span.resource, "ot_span")
self.assertEqual(dd_span.get_tag("aws.operation"), "DescribeInstances")
assert_span_http_status_code(dd_span, 200)
self.assertEqual(dd_span.get_tag(http.METHOD), "POST")
self.assertEqual(dd_span.get_tag("aws.region"), "us-west-2")
with ot_tracer.start_active_span("ot_span"):
ec2.run_instances(21)
spans = self.pop_spans()
assert spans
self.assertEqual(len(spans), 2)
ot_span, dd_span = spans
# confirm the parenting
self.assertIsNone(ot_span.parent_id)
self.assertEqual(dd_span.parent_id, ot_span.span_id)
self.assertEqual(dd_span.get_tag("aws.operation"), "RunInstances")
assert_span_http_status_code(dd_span, 200)
self.assertEqual(dd_span.get_tag(http.METHOD), "POST")
self.assertEqual(dd_span.get_tag("aws.region"), "us-west-2")
self.assertEqual(dd_span.service, "test-boto-tracing.ec2")
self.assertEqual(dd_span.resource, "ec2.runinstances")
self.assertEqual(dd_span.name, "ec2.command")
def create_submaster():
global subMasterCounter
subMasterCounter = int(subMasterCounter) + 1
ec2 = boto.connect_ec2()
reservation = ec2.run_instances(image_id='ami-d05223b8', key_name='keyy',
subnet_id='subnet-aaeb39f3',
instance_type='t2.micro',
instance_profile_name='workerProfile',
user_data=str(subMasterCounter))
ec2.create_tags(reservation.instances[0].id,{"Name":str(subMasterCounter)+'Submaster',"type":'Submaster',"idnumber":subMasterCounter})
def clone_instance(instance):
new_bdm = None
ec2 = instance.connection
if instance.block_device_mapping:
root_device_name = instance.get_attribute('rootDeviceName')['rootDeviceName']
user_data = instance.get_attribute('userData')['userData']
# user_data comes back base64 encoded. Need to decode it so it
# can get re-encoded by run_instance !
user_data = base64.b64decode(user_data)
new_bdm = BlockDeviceMapping()
for dev in instance.block_device_mapping:
# if this entry is about the root device, skip it
if dev != root_device_name:
bdt = instance.block_device_mapping[dev]
if bdt.volume_id:
volume = ec2.get_all_volumes([bdt.volume_id])[0]
snaps = volume.snapshots()
if len(snaps) == 0:
print 'No snapshots available for %s' % volume.id
else:
# sort the list of snapshots, newest is at the end now
snaps.sort(key=lambda snap: snap.start_time)
latest_snap = snaps[-1]
new_bdt = BlockDeviceType()
new_bdt.snapshot_id = latest_snap.id
new_bdm[dev] = new_bdt
return ec2.run_instances(
instance.image_id,
key_name=instance.key_name,
security_groups=[g.name for g in instance.groups],
user_data=user_data,
instance_type=instance.instance_type,
kernel_id=instance.kernel,
ramdisk_id=instance.ramdisk,
monitoring_enabled=instance.monitored,
placement=instance.placement,
block_device_map=new_bdm
).instances[0]
def create_workers(subMasterID,workers_no):
ec2 = boto.connect_ec2()
reservation = ec2.run_instances(image_id='ami-ea552482', min_count=workers_no, max_count=workers_no,key_name='keyy',
subnet_id='subnet-aaeb39f3',
instance_type='t2.micro',
instance_profile_name='workerProfile',
user_data=str(subMasterID))
ids = []
for instance in reservation.instances:
ids.append(instance.id)
print ids
#for instance in reservation:
ec2.create_tags(ids,{"Name":str(subMasterID)+'Worker',"type":'Worker',"idnumber":subMasterID})
def ensure_council():
count = 3
g1 = security_group('counselor', 'Security Group for Council').id
g2 = security_group('counseled',
'Security Group for Instances Following to Council').id
role_and_profile('counselor')
insts = get_instances()
counselors = [
i for i in insts if 'counselor' in [g.name for g in i.groups]
]
counselors = [i for i in counselors if i.state in ['running', 'pending']]
is_bootstrap = (len(counselors) == 0)
user_data = '''#!/bin/bash
sudo apt-get update
sudo apt-get install -y python-dev python-setuptools
sudo easy_install pip
sudo pip install boto
sudo pip install carlcm
sudo /usr/local/bin/carlcm-counselor 3'''
if is_bootstrap:
user_data += ' bootstrap'
user_data += '\n\n'
num_to_launch = count - len(counselors)
d = {
'image_id': 'ami-870a2fb7',
'key_name': 'carl-ssh-2015-02-14',
'user_data': user_data,
'security_group_ids': [g1, g2],
'subnet_id': 'subnet-7af8590d',
'instance_type': 't2.micro',
'instance_profile_name': 'counselor',
'min_count': num_to_launch,
'max_count': num_to_launch,
}
if num_to_launch > 0:
ec2.run_instances(**d)
return
def clone_instance(instance):
new_bdm = None
ec2 = instance.connection
if instance.block_device_mapping:
root_device_name = instance.get_attribute(
'rootDeviceName')['rootDeviceName']
user_data = instance.get_attribute('userData')['userData']
# user_data comes back base64 encoded. Need to decode it so it
# can get re-encoded by run_instance !
user_data = base64.b64decode(user_data)
new_bdm = BlockDeviceMapping()
for dev in instance.block_device_mapping:
# if this entry is about the root device, skip it
if dev != root_device_name:
bdt = instance.block_device_mapping[dev]
if bdt.volume_id:
volume = ec2.get_all_volumes([bdt.volume_id])[0]
snaps = volume.snapshots()
if len(snaps) == 0:
print 'No snapshots available for %s' % volume.id
else:
# sort the list of snapshots, newest is at the end now
snaps.sort(key=lambda snap: snap.start_time)
latest_snap = snaps[-1]
new_bdt = BlockDeviceType()
new_bdt.snapshot_id = latest_snap.id
new_bdm[dev] = new_bdt
return ec2.run_instances(instance.image_id,
key_name=instance.key_name,
security_groups=[g.name for g in instance.groups],
user_data=user_data,
instance_type=instance.instance_type,
kernel_id=instance.kernel,
ramdisk_id=instance.ramdisk,
monitoring_enabled=instance.monitored,
placement=instance.placement,
block_device_map=new_bdm).instances[0]
def create_box():
old_ids = set(i.id for i in ec2.get_only_instances())
machine = ec2.run_instances(AMI_ID, key_name=KEYNAME,
security_groups=[os.environ.get('SECURITY_GROUP', 'memex-explorer-prod'),], instance_type='m3.2xlarge')
new_instance = [i for i in ec2.get_only_instances() if i.id not in old_ids][0]
print(new_instance.id)
while new_instance.state != u'running':
time.sleep(3)
new_instance.update()
while ec2.get_all_instance_status(instance_ids=[new_instance.id])[0].system_status.details['reachability'] != 'passed':
time.sleep(3)
time.sleep(1)
assert new_instance.public_dns_name
print(new_instance.public_dns_name)
return new_instance
def start_vm_openstack(num_vms):
""" Create a VM instance on OpenStack using the ec2 boto interface commands
Args:
Set the EC2_ACCESS_KEY and EC2_SECRET_KEY environment variables
num_vms: Number of vms to spawn
Returns:
"""
access_key = os.getenv("EC2_ACCESS_KEY")
secret_key = os.getenv("EC2_SECRET_KEY")
ec2=None
try:
region = boto.regioninfo.RegionInfo(name="openstack", endpoint="206.117.53.134")
ec2 = boto.connect_ec2(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
is_secure=False,
region=region,
port=8773,
path="/services/Cloud")
except:
print 'Error connection to openstack'
return
## the machine image
ami = "ami-00000001"
## start a bunch of vms
for count in range(0,6):
reservations=ec2.run_instances(ami,instance_type='m1.tiny',max_count=1)
print "Launching vm ",count," of 256"
## create a chain (ie. Make in iterator that returns elements fromt the first
## iterable untils it is exhaused, then proceeds to the next iterable
chain = itertools.chain.from_iterable
existing_instances = list(chain([res.instances for res in ec2.get_all_instances()]))
status = [e.update() for e in existing_instances]
## wait until all the instances have transitioned from 'pending' to 'running' status
while 'pending' in status:
existing_instances = list(chain([res.instances for res in ec2.get_all_instances()]))
status = [e.update() for e in existing_instances]
status_dict = dict(zip(existing_instances,status))
print [(i,status_dict[i]) for i in status_dict.keys() if status_dict[i] != 'terminated']
print 'Still pending'
time.sleep(3)
print 'Complete'
return
def launch_instance(ec2, ami, instance_type, region, groups, key_name ):
print 'Launch instance [ami=%s, type=%s, groups=%s, key=%s'%(ami, instance_type, groups, key_name)
reservation = ec2.run_instances(ami, key_name=key_name, security_groups=groups, instance_type=instance_type, placement=region )
instance = reservation.instances[0]
# Wait for instance state to change to 'running'.
print 'waiting for instance'
while instance.state != 'running':
print '.'
time.sleep(5)
instance.update()
print 'done'
return instance
def create_instances(image_id='ami-2a31bf1a',
key_name='jthomas',
security_groups=None,
count=1, tags=None, subnet_id=None):
counter = 0
while (counter < count):
reservation = ec2.run_instances(image_id, key_name=key_name,
security_groups=security_groups,
subnet_id=subnet_id)
instance = reservation.instances[0]
print instance
if not tags is None:
for key, value in tags.iteritems():
instance.add_tag(key, value)
counter = counter + 1
def run_instance(obs_worker_id):
global print_only
if not print_only:
# WARNING: Do NOT start more than 1 instance at a time here!
# Multilaunching is managed at higher level to create unique tags etc.
instances_min = 1
instances_max = 1
worker_customization = "#!/bin/sh\nhostname " + obs_worker_id +\
";echo " + obs_worker_id + " > /etc/HOSTNAME;"
global aws_ami_image
global aws_keypair_name
global aws_security_group
global aws_instance_type
reservation = ec2.run_instances(image_id=aws_ami_image,
min_count=instances_min,
max_count=instances_max,
key_name=aws_keypair_name,
security_groups=[aws_security_group],
user_data=worker_customization,
instance_type=aws_instance_type)
#placement='eu-west-1b')
instance = reservation.instances[0]
ec2.create_tags([instance.id], {'Name': obs_worker_id})
global elastic_build_hosts
elastic_build_hosts.append({
'instance_name': obs_worker_id,
'instance_id': instance.id,
'instance_type': instance.instance_type,
'cpu': "-",
'disk': "-",
'instance_state': "SPAWNING",
'launch_time': 0,
'time_left': 60,
'ip': "-",
'workers': []
})
return
def main(config_path, name_prefix, tag):
with open(config_path) as f:
config = json.load(f)
ec2 = boto.ec2.connect_to_region(
INSTANCE_CONFIG['region'],
aws_access_key_id=config['access_key_id'],
aws_secret_access_key=config['secret_access_key'])
name = name_prefix + "-" + datetime.utcnow().isoformat()
# Assume that ssh key is uploaded
group = ec2.create_security_group(name, 'A group that allows SSH access')
group.authorize('tcp', 22, 22, "0.0.0.0/0")
reservation = ec2.run_instances(
INSTANCE_CONFIG['ami'],
key_name=os.path.basename(config['certificate_path']).split(".")[0],
instance_type=INSTANCE_CONFIG['type'],
security_groups=[name])
# Find the actual Instance object inside the Reservation object
# returned by EC2.
instance = reservation.instances[0]
# The instance has been launched but it's not yet up and
# running. Let's wait for it's state to change to 'running'.
print 'waiting for instance'
while instance.state != 'running':
print '.',
time.sleep(1)
instance.update()
print 'done'
instance.add_tag(tag)
print "DoNe! To connect use:"
print "ssh -i {} ubuntu@{}".format(config['certificate_path'],
instance.public_dns_name)
def createInstance(instance_name):
ec2 = boto.ec2.connect_to_region('us-west-2', **auth)
try:
# ami id of the instance as image to spin new ec2 (eg ami-g284cd96)
image_id = '<ami-id>'
# create key and security grp in advance
reservations = ec2.run_instances(
image_id,
key_name='<key_name>',
instance_type='t2.micro',
security_groups=['<security-group-name>'])
instance = reservations.instances[0]
while instance.state != 'running':
time.sleep(5)
instance.update()
ec2.create_tags([instance.id], {'Name': instance_name})
return instance.public_dns_name
except Exception, e3:
error3 = "Error3: %s" % str(e3)
return error3
def run_instance(obs_worker_id):
global print_only
if not print_only:
# WARNING: Do NOT start more than 1 instance at a time here!
# Multilaunching is managed at higher level to create unique tags etc.
instances_min = 1
instances_max = 1
worker_customization = "#!/bin/sh\nhostname " + obs_worker_id +\
";echo " + obs_worker_id + " > /etc/HOSTNAME;"
global aws_ami_image
global aws_keypair_name
global aws_security_group
global aws_instance_type
reservation = ec2.run_instances(
image_id = aws_ami_image,
min_count = instances_min,
max_count = instances_max,
key_name = aws_keypair_name,
security_groups = [aws_security_group],
user_data = worker_customization,
instance_type = aws_instance_type)
#placement='eu-west-1b')
instance = reservation.instances[0]
ec2.create_tags([instance.id],{'Name':obs_worker_id})
global elastic_build_hosts
elastic_build_hosts.append({'instance_name':obs_worker_id,
'instance_id':instance.id,
'instance_type':instance.instance_type,
'cpu':"-", 'disk':"-",
'instance_state':"SPAWNING",
'launch_time':0,
'time_left':60, 'ip':"-",
'workers':[]})
return
def ec2_run_instances(ami_name, tag_name):
ec2 = _get_ec2_connection()
ami = fabtaskit.amis.get(ami_name)
if ami is None:
abort("Couldn't find ami conf with name: {}".format(ami_name))
kwargs = ami.copy()
kwargs['key_name'] = os.environ['AWS_KEYPAIR_NAME']
reservation = ec2.run_instances(**kwargs)
instance = reservation.instances[0]
ec2.create_tags([instance.id], {'Name': tag_name})
tries = 5
booted = False
while not booted and tries != 0:
time.sleep(5)
for r in ec2.get_all_instances():
if r.id == reservation.id:
print_ec2(r.instances[0])
booted = True
break
tries -= 1
if not booted:
abort("Couldn't find if instance booted")
def launch(key_name=None,
region='us-west-2',
image_id='ami-5189a661',
instance_type='t2.micro',
security_groups='launch-wizard-1',
user_data=None,
initial_check=False):
'''
'''
if not isinstance(security_groups, list):
security_groups = [security_groups]
ec2 = boto.ec2.connect_to_region(region)
reserve = ec2.run_instances(image_id,
key_name=key_name,
instance_type=instance_type,
security_groups=security_groups,
user_data=user_data)
inst = reserve.instances[0]
while inst.state == u'pending':
time.sleep(10)
inst.update()
if initial_check:
# Wait for the status checks first
status = ec2.get_all_instance_status(instance_ids=[inst.id])[0]
check_stat = "Status:initializing"
while str(status.system_status) == check_stat and str(
status.instance_status) == check_stat:
time.sleep(10)
status = ec2.get_all_instance_status(instance_ids=[inst.id])[0]
return inst
def real_launch(self, region, item_to_vote, instances=1):
startup = script % {'ITEM': item_to_vote}
ips = []
print "launching in {0}: instances: {1}".format(region, instances)
# print startup
try:
ec2 = boto.ec2.connect_to_region(region, aws_access_key_id=self.key, aws_secret_access_key=self.secret)
try:
key_pair = ec2.create_key_pair(self.ec2_key_name) # only needs to be done once
key_pair.save(SSH_PATH)
except boto.exception.EC2ResponseError as e:
#print e
pass
reservation = ec2.run_instances(image_id=amis_per_regions[region],
min_count=instances, max_count=instances,
key_name=self.ec2_key_name,
user_data=startup)
ips.extend(self._insert_instances(reservation, ec2))
except Exception as e:
print " !!!! not supported. \n{0}".format(e)
return ips
def createInstance(ec2,
ami,
nb_nodes,
placement,
instance_type,
key,
sg,
user_data=None):
"""
Create a new EC2 instance with specific parameters
SecurityGroup (sg) and KeyPair (key) have to be previously created (see cassandgo initSG and cassandgo initKP)
"""
reservation = ec2.run_instances(ami,
min_count=nb_nodes,
max_count=nb_nodes,
placement=placement,
key_name=key,
security_groups=[sg],
instance_type=instance_type,
user_data=user_data)
instance = reservation.instances[0]
return instance
def _launch_instance(type, role):
if not type:
type = 't1.micro'
puts("Creating instance...")
ec2 = _get_conn()
reservation = ec2.run_instances('ami-3fec7956',
key_name='hermes',
security_groups=['hermes'],
instance_type=type)
instance = reservation.instances[0]
while True:
time.sleep(5)
instance.update()
puts("%s (%s)" % (instance.state, instance.state_code))
if instance.state_code != 0:
break
puts("Instance created " + instance.public_dns_name)
instance.add_tag('Role', role)
instance.add_tag('Section', 'hermes')
return instance.public_dns_name, instance.id
def start_machine(ami, instance):
ec2 = boto.ec2.connect_to_region(settings.aws_region,
aws_access_key_id=settings.aws_access,
aws_secret_access_key=settings.aws_secret)
if not ec2:
logging.error('Cannot connect to region %s' % settings.aws_region)
return None, None
worker_id = 'jm-%s' % uuid.uuid4()
logging.info('Request workerID = %s' % worker_id)
try:
reservation = ec2.run_instances(
ami,
security_groups=[settings.aws_seqgrp],
instance_type=instance,
user_data=base64.b64encode(worker_id),
instance_initiated_shutdown_behavior='terminate',
)
logging.info('Reservation %s for worker %s' % (reservation.id, worker_id))
return worker_id, reservation.id
except Exception, e:
logging.exception('Cannot reserve instance %s for type %s (%s)' % (ami, instance, e))
return None, None
params.update(dict(
min_count = count_remaining,
max_count = count_remaining,
client_token = id,
placement_group = placement_group,
))
else:
params.update(dict(
min_count = count_remaining,
max_count = count_remaining,
client_token = id,
placement_group = placement_group,
private_ip_address = private_ip,
))
res = ec2.run_instances(**params)
instids = [ i.id for i in res.instances ]
while True:
try:
ec2.get_all_instances(instids)
break
except boto.exception.EC2ResponseError as e:
if "<Code>InvalidInstanceID.NotFound</Code>" in str(e):
# there's a race between start and get an instance
continue
else:
module.fail_json(msg = str(e))
# The instances returned through ec2.run_instances above can be in
# terminated state due to idempotency. See commit 7f11c3d for a complete
# explanation.
def launch_and_configure(ec2_args):
"""
Creates an sqs queue, launches an ec2 instance,
configures it and creates an AMI. Polls
SQS for updates
"""
print "{:<40}".format(
"Creating SQS queue and launching instance for {}:".format(run_id))
print
for k, v in ec2_args.iteritems():
if k != 'user_data':
print " {:<25}{}".format(k, v)
print
global sqs_queue
global instance_id
sqs_queue = sqs.create_queue(run_id)
sqs_queue.set_message_class(RawMessage)
res = ec2.run_instances(**ec2_args)
inst = res.instances[0]
instance_id = inst.id
print "{:<40}".format(
"Waiting for instance {} to reach running status:".format(instance_id)),
status_start = time.time()
for _ in xrange(EC2_RUN_TIMEOUT):
res = ec2.get_all_instances(instance_ids=[instance_id])
if res[0].instances[0].state == 'running':
status_delta = time.time() - status_start
run_summary.append(('EC2 Launch', status_delta))
print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(
status_delta / 60,
status_delta % 60)
break
else:
time.sleep(1)
else:
raise Exception("Timeout waiting for running status: {} ".format(
instance_id))
print "{:<40}".format("Waiting for system status:"),
system_start = time.time()
for _ in xrange(EC2_STATUS_TIMEOUT):
status = ec2.get_all_instance_status(inst.id)
if status[0].system_status.status == u'ok':
system_delta = time.time() - system_start
run_summary.append(('EC2 Status Checks', system_delta))
print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(
system_delta / 60,
system_delta % 60)
break
else:
time.sleep(1)
else:
raise Exception("Timeout waiting for status checks: {} ".format(
instance_id))
print
print "{:<40}".format(
"Waiting for user-data, polling sqs for Ansible events:")
(ansible_delta, task_report) = poll_sqs_ansible()
run_summary.append(('Ansible run', ansible_delta))
print
print "{} longest Ansible tasks (seconds):".format(NUM_TASKS)
for task in sorted(
task_report, reverse=True,
key=lambda k: k['DELTA'])[:NUM_TASKS]:
print "{:0>3.0f} {}".format(task['DELTA'], task['TASK'])
print " - {}".format(task['INVOCATION'])
print
print "{:<40}".format("Creating AMI:"),
ami_start = time.time()
ami = create_ami(instance_id, run_id, run_id)
ami_delta = time.time() - ami_start
print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(
ami_delta / 60,
ami_delta % 60)
run_summary.append(('AMI Build', ami_delta))
total_time = time.time() - start_time
all_stages = sum(run[1] for run in run_summary)
if total_time - all_stages > 0:
run_summary.append(('Other', total_time - all_stages))
run_summary.append(('Total', total_time))
return run_summary, ami
print ("4. Stop instance")
print ("5. Start instance")
print ("6. Reboot instance")
print ("7. Terminate instance")
print ("8. Exit")
print (30 * '-')
try :
choice = int ( raw_input('Enter your choice [1-8] : ') )
except ValueError, e :
print ("'%s' is not a valid integer." % e.args[0].split(": ")[1])
if choice == 1:
reservation = ec2.run_instances('ami-c5f30081',
key_name='california',
instance_type='t2.micro',
security_groups=['default'])
instance = reservation.instances[0]
print 'waiting for instance'
while instance.state != 'running':
print '.'
time.sleep(5)
instance.update()
print 'done'
instance.add_tag('Name','Wordpress-Server')
elif choice == 2:
reservation = ec2.run_instances('ami-df6a8b9b',
key_name='california',
instance_type='t2.micro',
group_name, 'A group that allows VPN access')
else:
raise
try:
group.authorize('tcp', ssh_port, ssh_port, cidr)
group.authorize('udp', vpn_port, vpn_port, cidr)
except ec2.ResponseError, e:
if e.code == 'InvalidPermission.Duplicate':
"""print ('Security group %s already exists') % group_name"""
else:
raise
reservation = ec2.run_instances(ami,
key_name=key_name,
security_groups=[group_name],
instance_type=instance_type,
user_data=user_data)
instance = reservation.instances[0]
while instance.state != 'running':
time.sleep(30)
instance.update()
instance.add_tag(tag)
global host
host = instance.ip_address
print "%s" % host
else:
raise
#Add SSH rule to Security group
try:
group.authorize('tcp',ssh_port,ssh_port, cidr)
except ec2.ResponseError, e:
if e.code == 'InvalidPermission.Duplicate':
print 'Security group %s already authorized' % group_name
else:
raise
# Now start up the instance. The run_instances method
# has many, many parameters but these are all we need
# for now.
reservation = ec2.run_instances(ami,
key_name=key_name,
security_groups=[group_name],
instance_type=instance_type,
user_data=user_data)
# Find the actual Instance object inside the Reservation object
# returned by EC2.
instance = reservation.instances[0]
# The instance has been launched but it's not yet up and
# running. Let's wait for its state to change to 'running'.
print 'Waiting for the instance'
while instance.state != 'running':
print '.'
time.sleep(5)
instance.update()
print 'done'
print 'scaling policy created'
##cloud watch
cloudwatch=boto.ec2.cloudwatch.connect_to_region('us-east-1')
alarm_dimensions={"AutoScalingGroupName":'jianGroup'}
#scale up
scale_up_alarm=MetricAlarm(name='scale_up_on_cpu',namespace='AWS/EC2',metric='CPUUtilization',statistic='Average',comparison='>',threshold='80',period='60',evaluation_periods=1,alarm_actions=[scale_up_policy.policy_arn],dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_up_alarm)
#scale down
scale_down_alarm = MetricAlarm(name='scale_down_on_cpu', namespace='AWS/EC2',metric='CPUUtilization', statistic='Average',comparison='<', threshold='60',period='60', evaluation_periods=1,alarm_actions=[scale_down_policy.policy_arn],dimensions=alarm_dimensions)
cloudwatch.create_alarm(scale_down_alarm)
print 'both clock watch created'
#load generator
ec2=boto.ec2.connect_to_region("us-east-1")
reservation=ec2.run_instances('ami-7aba0c12',key_name='jj',instance_type='m3.medium',security_groups=['http'])
time.sleep(60)
instance=reservation.instances[0]
id=instance.id
#loadDns=instance.public_dns_name
#get load dns
groupw=autoscale.get_all_groups(names=['jianGroup'])[0]
instances=ec2.get_only_instances(instance_ids=[id])
loadDns=instances[0].public_dns_name
print 'load generator dns is :%s'%(loadDns)
time.sleep(10)
#authenticate
response = urllib2.urlopen('http://'+loadDns+'/username?username=jianj')
print 'load generator authenticated'
if role_name is not None and len(roles) != 0:
should_add = True
def security_group(name, description):
try:
group = ec2.get_all_security_groups(groupnames=[name])[0]
except boto.exception.EC2ResponseError:
group = None
if group is None:
ec2.create_security_group(name, description)
group = ec2.get_all_security_groups(groupnames=[name])[0]
return group
'''
# ubuntu 14.04 cloudinit http://cloud-images.ubuntu.com/releases/14.04/release-20150209.1/
ec2.run_instances(image_id='ami-870a2fb7', key_name='carl-ssh-2015-02-14', user_data='#!/bin/bash\nsudo apt-get update\nsudo apt-get install -y python-dev python-setuptools\nsudo easy_install pip\nsudo pip install boto\nsudo pip install carlcm\n/usr/local/bin/carlcm-counselor 3', security_group_ids=['sg-233a1f46', 'sg-c03a1fa5'], subnet_id='subnet-7af8590d', instance_type='t2.micro', instance_profile_name='counselor')
'''
def get_instances():
return reduce(lambda a, b: a + b,
[a.instances for a in ec2.get_all_reservations()], [])
def ensure_council():
count = 3
g1 = security_group('counselor', 'Security Group for Council').id
g2 = security_group('counseled',
'Security Group for Instances Following to Council').id
botoVersionMismatch
if (botoVersionArr[1] < 13):
botoVersionMismatch
if (debug):
boto.set_stream_logger('h2o-ec2')
ec2 = boto.ec2.connect_to_region(regionName, debug=debug)
print 'Launching', numInstancesToLaunch, 'instances.'
reservation = ec2.run_instances(
image_id=amiId,
min_count=numInstancesToLaunch,
max_count=numInstancesToLaunch,
key_name=keyName,
instance_type=instanceType,
security_groups=[securityGroupName],
instance_profile_arn=iam_profile_resource_name,
instance_profile_name=iam_profile_name,
dry_run=dryRun
)
for i in range(numInstancesToLaunch):
instance = reservation.instances[i]
print 'Waiting for instance', i+1, 'of', numInstancesToLaunch, '...'
instance.update()
while instance.state != 'running':
print ' .'
time.sleep(1)
instance.update()
print ' instance', i+1, 'of', numInstancesToLaunch, 'is up.'
with indent(4):
for k, v in tags.iteritems():
puts(columns(
[colored.blue(k), col1],
[v, col2]))
else:
puts("")
puts(colored.red("For real mode"))
puts("")
with indent(4, quote=colored.blue(' >')):
puts("Starting instance")
reservation = ec2.run_instances(
image_id=args.ami,
key_name=args.key_pair,
user_data=render(args.build_script, template_dict),
security_groups=args.security_group.split(','),
instance_type=args.instance
)
# add tags
for instance in reservation.instances:
for k, v in tags.iteritems():
instance.add_tag(k, v)
for instance in reservation.instances:
s3_key = Key(bucket)
s3_key.key = '%s._cc_' % instance.id
s3_key.set_contents_from_string(
'running', {'Content-Type': 'text/plain'}, replace=True)
if opts.instance_initiated_shutdown_behavior:
run_args[
'instance_initiated_shutdown_behavior'] = opts.instance_initiated_shutdown_behavior
if opts.placement_group:
run_args['placement_group'] = opts.placement_group
if opts.private_ip_address:
run_args['private_ip_address'] = opts.private_ip_address
if opts.kernel:
run_args['kernel_id'] = opts.kernel
if opts.ramdisk:
run_args['ramdisk_id'] = opts.ramdisk
if opts.subnet:
run_args['subnet_id'] = opts.subnet
run_args['image_id'] = resolve_ami(args[0])
r = ec2.run_instances(**run_args)
inst = firstinstance([r])
return str(inst.id)
def ec2run_completers(self, event):
cmd_param = event.line.split()
if event.line.endswith(' '):
cmd_param.append('')
arg = cmd_param.pop()
arg = cmd_param.pop()
if arg in ('-t', '--instance-type'):
return [
'm1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge',
def create(args, config):
instance_type = args['--type']
subnet = args['--subnet']
num = int(args['--num'])
type_template = configuration.get_type_template(instance_type, config)
# Launch Instances per env
for env in args['<env>']:
e = configuration.get_env(env, config)
if not e:
puts(colored.red('%s is not defined in your Nyfile' % e))
else:
ec2 = connection.create(config)
if not ec2:
raise Exception('Unable to connect to EC2')
groups = configuration.security_group_names_to_ids(
names=type_template.security_groups,
env=env,
config=config)
if not groups or not len(groups):
raise Exception('Unable to determine security groups')
if subnet == SUBNET_ALTERNATE:
reservations = ec2.get_all_instances()
instances = [i for r in reservations for i in r.instances]
all_subnets = configuration.get_type_subnets(
type=instance_type,
env=env,
config=config)
if not all_subnets:
all_subnets = configuration.get_env_subnets(env, config)
subnets = Counter(all_subnets)
vpc_subnets = []
if all_subnets:
subnets.update([i.subnet_id for i in instances])
for s in subnets.items():
if s[0] in all_subnets:
vpc_subnets.append(s[0])
subnets = Counter((tuple(vpc_subnets)*num)[:num])
else:
subnets = Counter((subnet,)*num)
reservations = []
tags = configuration.get_tags(instance_type, config)
# Try to get a bootscript
bootscript_paths = configuration.get_bootscript_paths(
instance_type=instance_type,
env=env,
config=config)
for sub in subnets.items():
# Parse the bootscripts once per subnet
parsed_bootscripts = []
render_args = {
'type': instance_type,
'env': env,
'subnet': subnet,
'security_groups': groups,
'image_id': type_template.image_id,
'tags': (';'.join(tags) if tags else []),
'key': e.key,
'config': AttrDict(config.items()),
}
for p in bootscript_paths:
with open(p) as tmp:
puts(colored.green('Parsing Bootscript: %s' % p))
parsed_bootscripts.append(
Template(tmp.read()).render(render_args))
reservations.append(
ec2.run_instances(
min_count=sub[1],
image_id=type_template.image_id,
instance_type=type_template.type,
key_name=e.key,
subnet_id=sub[0],
security_group_ids=groups,
user_data='\n'.join(parsed_bootscripts)))
puts(colored.green(
'Started %s (%i x instance)' % (
reservations[-1:][0], int(sub[1]))))
puts("\n")
puts("Waiting on instances to start: \n")
with distraction():
for reservation in reservations:
for instance in reservation.instances:
status = instance.update()
while status == 'pending':
time.sleep(10)
status = instance.update()
if status == 'running':
puts(colored.green(
'Instance %s is up' % instance))
tags = configuration.get_tags(
instance_type,config)
if tags:
for key,val in tags.items():
instance.add_tag(key, val)
run('git fetch origin master')
instance.add_tag('ny_env', env)
instance.add_tag('ny_type', instance_type)
instance.add_tag('repo_version', run(
'git rev-parse origin/master').std_out[:8])
else:
puts(colored.red(
'Instance status: ' + status))
# If we get an InvalidGroup.NotFound error back from EC2,
# it means that it doesn't exist and we need to create it.
try:
group = ec2.get_all_security_groups(groupnames=[group_name])[0]
except ec2.ResponseError, e:
if e.code == 'InvalidGroup.NotFound':
print 'Invalid Security Group specified' % group_name
else:
raise
# Now start up the instance. The run_instances method
# has many, many parameters but these are all we need
# for now.
reservation = ec2.run_instances(ami,
key_name=key_name,
security_groups=[group_name],
instance_type=instance_type,
placement='us-west-1a',
user_data=user_data)
# Find the actual Instance object inside the Reservation object
# returned by EC2.
instance = reservation.instances[0]
print ("Starting %s Instance Type, id= %s" %(instance_type,instance.id))
# The instance has been launched but it's not yet up and
# running. Let's wait for it's state to change to 'running'.
print ('Waiting for instance'),
while instance.state != 'running':
sys.stdout.write(".")
sys.stdout.flush()
botoVersionMismatch
if (botoVersionArr[1] < 13):
botoVersionMismatch
if (debug):
boto.set_stream_logger('h2o-ec2')
ec2 = boto.ec2.connect_to_region(regionName, debug=debug)
print 'Launching', numInstancesToLaunch, 'instances.'
if spotBid is None:
reservation = ec2.run_instances(
image_id=amiId,
min_count=numInstancesToLaunch,
max_count=numInstancesToLaunch,
key_name=keyName,
instance_type=instanceType,
security_groups=[securityGroupName],
instance_profile_arn=iam_profile_resource_name,
instance_profile_name=iam_profile_name,
dry_run=dryRun)
else:
spotRequests = ec2.request_spot_instances(
price=spotBid,
image_id=amiId,
count=numInstancesToLaunch,
key_name=keyName,
instance_type=instanceType,
security_groups=[securityGroupName],
instance_profile_arn=iam_profile_resource_name,
instance_profile_name=iam_profile_name,
dry_run=dryRun,
for p in processes:
p.join()
#workers.close()
server.join()
elif sys.argv[3] == '-rw':
port = int(sys.argv[2])
server = Process(target=startserv, args=(port,))
server.start()
#trw = Process(target=trackRWorkers, args=())
#trw.start()
#for testing purposes
num_of_workers = 1 #change according to test
for i in range(0, num_of_workers):
print "Starting a worker"
newWorker = ec2.run_instances('ami-9bc394ab', key_name='cs553', instance_type='t2.micro',
security_groups=['launch-wizard-2'])
#trw.join()
server.join()
else:
print "invalid arguments"
sys.exit(1)
else:
print "invalid arguments"
isys.exit(1)
#!/usr/bin/env python
import boto.ec2
aws_access_key = 'xxx'
aws_secret_key = 'xxx'
ec2 = boto.ec2.connect_to_region('ap-northeast-1', aws_access_key_id= aws_access_key, aws_secret_access_key =aws_secret_key)
i=11
ip = '10.0.0.' + str(i)
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id='subnet-20ac0857', groups=['sg-0454d761'], associate_public_ip_address=True, private_ip_address=ip)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface)
t = ec2.run_instances('ami-b1b458b1', instance_type='t2.micro', min_count=1, key_name='testkey', network_interfaces=interfaces )
i=11
ip = '10.0.1.' + str(i)
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id='subnet-17ac0860', groups=['sg-e554d780'] , private_ip_address=ip)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface)
t = ec2.run_instances('ami-b1b458b1', instance_type='t2.micro', min_count=1, key_name='testkey', network_interfaces=interfaces )
i=11
ip = '10.0.2.' + str(i)
interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id='subnet-12ac0865', groups=['sg-e654d783'] , private_ip_address=ip)
interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface)
t = ec2.run_instances('ami-b1b458b1', instance_type='t2.micro', min_count=1, key_name='testkey', network_interfaces=interfaces )
#ec2.associate_address(instance_id='i-a194c052')
def launch_and_configure(ec2_args):
"""
Creates an sqs queue, launches an ec2 instance,
configures it and creates an AMI. Polls
SQS for updates
"""
print "{:<40}".format(
"Creating SQS queue and launching instance for {}:".format(run_id))
print
for k, v in ec2_args.iteritems():
if k != 'user_data':
print " {:<25}{}".format(k, v)
print
global sqs_queue
global instance_id
sqs_queue = sqs.create_queue(run_id)
sqs_queue.set_message_class(RawMessage)
res = ec2.run_instances(**ec2_args)
inst = res.instances[0]
instance_id = inst.id
print "{:<40}".format(
"Waiting for instance {} to reach running status:".format(
instance_id)),
status_start = time.time()
for _ in xrange(EC2_RUN_TIMEOUT):
res = ec2.get_all_instances(instance_ids=[instance_id])
if res[0].instances[0].state == 'running':
status_delta = time.time() - status_start
run_summary.append(('EC2 Launch', status_delta))
print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(status_delta / 60,
status_delta % 60)
break
else:
time.sleep(1)
else:
raise Exception(
"Timeout waiting for running status: {} ".format(instance_id))
print "{:<40}".format("Waiting for system status:"),
system_start = time.time()
for _ in xrange(EC2_STATUS_TIMEOUT):
status = ec2.get_all_instance_status(inst.id)
if status[0].system_status.status == u'ok':
system_delta = time.time() - system_start
run_summary.append(('EC2 Status Checks', system_delta))
print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(system_delta / 60,
system_delta % 60)
break
else:
time.sleep(1)
else:
raise Exception(
"Timeout waiting for status checks: {} ".format(instance_id))
print
print "{:<40}".format(
"Waiting for user-data, polling sqs for Ansible events:")
(ansible_delta, task_report) = poll_sqs_ansible()
run_summary.append(('Ansible run', ansible_delta))
print
print "{} longest Ansible tasks (seconds):".format(NUM_TASKS)
for task in sorted(task_report, reverse=True,
key=lambda k: k['DELTA'])[:NUM_TASKS]:
print "{:0>3.0f} {}".format(task['DELTA'], task['TASK'])
print " - {}".format(task['INVOCATION'])
print
print "{:<40}".format("Creating AMI:"),
ami_start = time.time()
ami = create_ami(instance_id, run_id, run_id)
ami_delta = time.time() - ami_start
print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(ami_delta / 60, ami_delta % 60)
run_summary.append(('AMI Build', ami_delta))
total_time = time.time() - start_time
all_stages = sum(run[1] for run in run_summary)
if total_time - all_stages > 0:
run_summary.append(('Other', total_time - all_stages))
run_summary.append(('Total', total_time))
return run_summary, ami
params.update(dict(
min_count = count_remaining,
max_count = count_remaining,
client_token = id,
placement_group = placement_group,
))
else:
params.update(dict(
min_count = count_remaining,
max_count = count_remaining,
client_token = id,
placement_group = placement_group,
private_ip_address = private_ip,
))
res = ec2.run_instances(**params)
instids = [ i.id for i in res.instances ]
while True:
try:
ec2.get_all_instances(instids)
break
except boto.exception.EC2ResponseError as e:
if "<Code>InvalidInstanceID.NotFound</Code>" in str(e):
# there's a race between start and get an instance
continue
else:
module.fail_json(msg = str(e))
# The instances returned through ec2.run_instances above can be in
# terminated state due to idempotency. See commit 7f11c3d for a complete
# explanation.
