def test_ec2_client(self): ec2 = boto.ec2.connect_to_region("us-west-2") Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2) ec2.get_all_instances() spans = self.pop_spans() assert spans self.assertEqual(len(spans), 1) span = spans[0] self.assertEqual(span.get_tag("aws.operation"), "DescribeInstances") assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), "POST") self.assertEqual(span.get_tag("aws.region"), "us-west-2") self.assertIsNone(span.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) # Create an instance ec2.run_instances(21) spans = self.pop_spans() assert spans self.assertEqual(len(spans), 1) span = spans[0] assert_is_measured(span) self.assertEqual(span.get_tag("aws.operation"), "RunInstances") assert_span_http_status_code(span, 200) self.assertEqual(span.get_tag(http.METHOD), "POST") self.assertEqual(span.get_tag("aws.region"), "us-west-2") self.assertEqual(span.service, "test-boto-tracing.ec2") self.assertEqual(span.resource, "ec2.runinstances") self.assertEqual(span.name, "ec2.command") self.assertEqual(span.span_type, "http")
def test_ec2_client(self): ec2 = boto.ec2.connect_to_region("us-west-2") tracer = get_dummy_tracer() writer = tracer.writer Pin(service=self.TEST_SERVICE, tracer=tracer).onto(ec2) ec2.get_all_instances() spans = writer.pop() assert spans eq_(len(spans), 1) span = spans[0] eq_(span.get_tag('aws.operation'), "DescribeInstances") eq_(span.get_tag(http.STATUS_CODE), "200") eq_(span.get_tag(http.METHOD), "POST") eq_(span.get_tag('aws.region'), "us-west-2") # Create an instance ec2.run_instances(21) spans = writer.pop() assert spans eq_(len(spans), 1) span = spans[0] eq_(span.get_tag('aws.operation'), "RunInstances") eq_(span.get_tag(http.STATUS_CODE), "200") eq_(span.get_tag(http.METHOD), "POST") eq_(span.get_tag('aws.region'), "us-west-2") eq_(span.service, "test-boto-tracing.ec2") eq_(span.resource, "ec2.runinstances") eq_(span.name, "ec2.command")
def trackRWorkers(): print "Tracking workers" while True: reservations = ec2.get_all_reservations(filters={"instance-state-code" : [0, 16]}) max_workers = 16 #dynamic #print "Number of requests " + str(requests.count()) + " and number of reservations " + str(len(reservations)) if requests.count() > 0 and len(reservations) == 2: #there are requests, but no workers print "Starting a worker" newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro', security_groups=['launch-wizard-2']) time.sleep(30) elif len(reservations) < max_workers: if requests.count() % 2 == 0 and len(reservations)-2 < requests.count(): print "Starting a worker" newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro', security_groups=['launch-wizard-2']) time.sleep(30) elif requests.count() % 2 == 1 and len(reservations)-2 < requests.count()-1: print "Starting a worker" newWorker = ec2.run_instances('ami-e1bfe8d1', key_name='cs553', instance_type='t2.micro', security_groups=['launch-wizard-2']) time.sleep(30)
def test_ec2_client(self): ec2 = boto.ec2.connect_to_region("us-west-2") ec2.get_all_instances() spans = self.memory_exporter.get_finished_spans() assert spans self.assertEqual(len(spans), 1) span = spans[0] self.assertEqual(span.attributes["aws.operation"], "DescribeInstances") assert_span_http_status_code(span, 200) self.assertEqual(span.attributes["http.method"], "POST") self.assertEqual(span.attributes["aws.region"], "us-west-2") # Create an instance ec2.run_instances(21) spans = self.memory_exporter.get_finished_spans() assert spans self.assertEqual(len(spans), 2) span = spans[1] self.assertEqual(span.attributes["aws.operation"], "RunInstances") assert_span_http_status_code(span, 200) self.assertEqual( span.resource, Resource(attributes={ "endpoint": "ec2", "http_method": "runinstances" }), ) self.assertEqual(span.attributes["http.method"], "POST") self.assertEqual(span.attributes["aws.region"], "us-west-2") self.assertEqual(span.name, "ec2.command")
def test_ec2_client(self): ec2 = boto.ec2.connect_to_region('us-west-2') writer = self.tracer.writer Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2) ec2.get_all_instances() spans = writer.pop() assert spans self.assertEqual(len(spans), 1) span = spans[0] self.assertEqual(span.get_tag('aws.operation'), 'DescribeInstances') self.assertEqual(span.get_tag(http.STATUS_CODE), '200') self.assertEqual(span.get_tag(http.METHOD), 'POST') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertIsNone(span.get_metric(ANALYTICS_SAMPLE_RATE_KEY)) # Create an instance ec2.run_instances(21) spans = writer.pop() assert spans self.assertEqual(len(spans), 1) span = spans[0] self.assertEqual(span.get_tag('aws.operation'), 'RunInstances') self.assertEqual(span.get_tag(http.STATUS_CODE), '200') self.assertEqual(span.get_tag(http.METHOD), 'POST') self.assertEqual(span.get_tag('aws.region'), 'us-west-2') self.assertEqual(span.service, 'test-boto-tracing.ec2') self.assertEqual(span.resource, 'ec2.runinstances') self.assertEqual(span.name, 'ec2.command') self.assertEqual(span.span_type, 'http')
def spawn(hints, ec2): logging.info("About to launch {0} instance(s). Press Control-C to abort!".format(hints['count'])) for t in range(5,0,-1): logging.info("Launching in {0}...".format(t)) time.sleep(1) profile_suffix = '' if hints['tags']['environment'] == 'prod' else '_' + hints['tags']['environment'] instance_profile = hints['tags']['application'] + "_profile" + profile_suffix logging.info("OK, sending launch request!") # Make the reservation call. if hints['public_ip']: # http://stackoverflow.com/questions/19029588/how-to-auto-assign-public-ip-to-ec2-instance-with-boto interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id=hints['subnet_id'], groups=hints['security_group_ids'], associate_public_ip_address=True) interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface) reservation = ec2.run_instances( hints['ami_id'], instance_type = hints['instance_type'], network_interfaces=interfaces, instance_profile_name = instance_profile ) else: reservation = ec2.run_instances( hints['ami_id'], min_count = hints['count'], max_count = hints['count'], instance_type = hints['instance_type'], subnet_id = hints['subnet_id'], security_group_ids = hints['security_group_ids'], instance_profile_name = instance_profile, ) # Sleep a little bit. logging.info("Launch request sent. Waiting for instances to be created...") time.sleep(5) reservation_id = reservation.id sentinel = 0 while len(reservation.instances) != hints['count'] or [i for i in reservation.instances if i.private_ip_address is None]: print '.' time.sleep(2) sentinel += 1 if sentinel > 30: logging.error("Giving up!") exit(11) reservation = ec2.get_all_reservations(filters={"reservation-id":reservation_id}) hints['ips'] = {} # Apply tags to the instances. for idx in range(0, len(hints['names'])): instance = reservation.instances[idx] logging.info ("Tagging instance {0} as {1}".format(instance.id, hints['names'][idx])) instance.add_tags(hints['tags']) instance.add_tag('Name', value = hints['names'][idx]) # Store private IP by name hints['ips'][hints['names'][idx]] = instance.private_ip_address
def run_instances(location, ami, num, key, sg, p, pg, it): ec2 = boto.ec2.connect_to_region(location) ec2.run_instances(ami, min_count=num, max_count=num, key_name=key, security_groups=[sg], placement=p, placement_group=pg, instance_type=it)
def launch_instance(tag, key_name, group_name, inst_type, ami_name, user_data, wait=True, returninfo=None): """ Launch a testing instance. Doesn't actually attempt to connect as it can take quite a while between 'running' and connectability """ ec2 = boto.ec2.connect_to_region(AWS_REGION) failures = 0 max_failures = 10 while True: try: reservation = ec2.run_instances(ami_name, key_name=key_name, security_groups=[group_name], instance_type=inst_type, user_data=None) break except Exception, err: # Failed to get instance; wait 15 seconds and then try again (up to # 10 total times) errortext = str(err) if errortext.find("Not authorized for images") >= 0: print "**************************************" print "* Error from AWS suggests that the AMI code in" print "* CloudSetup.py is deprecated. Please go to" print "* https://aws.amazon.com/marketplace/ and search for" print "* \"Ubuntu server lts hvm\", selecting the most recent" print "* version. Click \"Continue\", \"Manual Launch\"," print "* and then copy the AMI ID for the US East region." print "* Copy that to the AMI_NAME value in CloudSetup.py" print "* and re-run." print "***************************************" print "* (Full text of error):" print errortext print "***************************************" return None elif errortext.find("accept terms and subscribe") >= 0: print "**************************************" print "* Error from AWS suggests that you have never used this" print "* AMI before and need to accept its terms and" print "* subscribe to it. Please follow the link in the below" print "* error text. Click \"Continue\", \"Manual Launch\"," print "* and \"Accept Terms\". After receiving email" print "* confirmation, you can re-run the code." print "**************************************" print "* (Full text of error):" print errortext print "**************************************" return None failures += 1 if failures == max_failures: print "**************************************" print "* Maximum number of instance launch failures reached." print "* (Full text of error):" print errortext print "**************************************" return None print " ** ec2.run_instances failed for tag", tag, "; waiting 15" print " ** seconds and then trying again..." time.sleep(15)
def start_machine(ami, instance): ec2 = boto.ec2.connect_to_region(settings.aws_region, aws_access_key_id=settings.aws_access, aws_secret_access_key=settings.aws_secret) if not ec2: logging.error('Cannot connect to region %s' % settings.aws_region) return None, None worker_id = 'jm-%s' % uuid.uuid4() logging.info('Request workerID = %s' % worker_id) try: reservation = ec2.run_instances( ami, security_groups=[settings.aws_seqgrp], instance_type=instance, user_data=base64.b64encode(worker_id), instance_initiated_shutdown_behavior='terminate', ) logging.info('Reservation %s for worker %s' % (reservation.id, worker_id)) return worker_id, reservation.id except Exception, e: logging.exception('Cannot reserve instance %s for type %s (%s)' % (ami, instance, e)) return None, None
def launch_instance(tag="mqlibtest1",key_name="mqlibtest",group_name="mqlibtest", wait=True, returnInfo=None): """ Launch a testing instance. Doesn't actually attempt to connect as it can take quite a while between 'running' and connectability """ ec2 = boto.ec2.connect_to_region(AWS_REGION) failures = 0 max_failures = 10 while True: try: reservation = ec2.run_instances(AMI_NAME, key_name=key_name, security_groups=[group_name], instance_type=INST_TYPE, user_data=None) break except Exception, err: # Failed to get instance; wait 15 seconds and then try again (up to # 10 total times) errorText = str(err) if errorText.find("Not authorized for images") >= 0: print "**************************************" print "* Error from AWS suggests that the AMI code in" print "* CloudSetup.py is deprecated. Please go to" print "* https://aws.amazon.com/marketplace/ and search for" print "* \"Ubuntu server lts hvm\", selecting the most recent" print "* version. Click \"Continue\", \"Manual Launch\"," print "* and then copy the AMI ID for the US East region." print "* Copy that to the AMI_NAME value in CloudSetup.py" print "* and re-run." print "***************************************" print "* (Full text of error):" print errorText print "***************************************" return None elif errorText.find("accept terms and subscribe") >= 0: print "**************************************" print "* Error from AWS suggests that you have never used this" print "* AMI before and need to accept its terms and" print "* subscribe to it. Please follow the link in the below" print "* error text. Click \"Continue\", \"Manual Launch\"," print "* and \"Accept Terms\". After receiving email" print "* confirmation, you can re-run the code." print "**************************************" print "* (Full text of error):" print errorText print "**************************************" return None failures += 1 if failures == max_failures: print "**************************************" print "* Maximum number of instance launch failures reached." print "* (Full text of error):" print errorText print "**************************************" return None print " ** ec2.run_instances failed for tag", tag, "; waiting 15" print " ** seconds and then trying again..." time.sleep(15)
def create_box(): old_ids = set(i.id for i in ec2.get_only_instances()) machine = ec2.run_instances(AMI_ID, key_name=KEYNAME, min_count=NODE_COUNT, max_count=NODE_COUNT, security_groups=[SECURITY_GROUP,], instance_type=os.environ.get('EC2_INSTANCE_TYPE', 'm3.medium')) new_instances = [i for i in ec2.get_only_instances() if i.id not in old_ids] for new_instance in new_instances: print("new instance:", new_instance.id) ec2.create_tags([new_instance.id], {"billingProject": "aei"}) is_running = [False] * len(new_instances) while not all(is_running): for count, newinstance in enumerate(new_instances): is_running[count] = new_instance.state == u'running' time.sleep(3) for new_instance in new_instances: new_instance.update() is_reachable = [False] * len(new_instances) while not all(is_reachable): instance_ids=[new_instance.id for new_instance in new_instances] inst_statuses = ec2.get_all_instance_status(instance_ids=instance_ids) is_reachable = [inst_status.system_status.details['reachability'] != 'passed' for inst_status in inst_statuses] time.sleep(3) time.sleep(1) for new_instance in new_instances: assert new_instance.public_dns_name print("public dns name:", new_instance.public_dns_name) return new_instances
def _create_compute(config, name): info("[CREATE COMPUTE] " + name) # user_data from template j2_env = jinja2.Environment( loader=jinja2.FileSystemLoader('/opt/necofs/templates')) USER_DATA = j2_env.get_template('compute.user_data').render( local_ipv4=local_ipv4) # create a NetworkInterface in our VPC interface = boto.ec2.networkinterface.NetworkInterfaceSpecification( subnet_id=config['SUBNET_ID'], groups=config['SECURITY_GROUP_IDS'], associate_public_ip_address=True) interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection( interface) reservation = ec2.run_instances( image_id=config['IMAGE_ID'], key_name=config['KEY_NAME'], user_data=USER_DATA, instance_type=config['COMPUTE_INSTANCE_TYPE'], instance_initiated_shutdown_behavior='terminate', placement_group=config[ 'PLACEMENT_GROUP'], # only for high level machines, not for testing network_interfaces=interfaces, instance_profile_name=config['IAM_ROLE']) info("[CREATE_COMPUTE]: reservation " + reservation.id + " for " + name) time.sleep( 30 ) # -- was checking before instance was created, wait a few for Amazon to notice it's own machine for instance in reservation.instances: status = instance.update() while status == 'pending': time.sleep(5) status = instance.update() instance.add_tag('Name', config['INSTANCE_NAME'] + "_compute_" + name) return reservation
def runInstance(keyName, imageId, securityGroup, instanceType): reservation = ec2.run_instances(imageId, key_name=keyName, instance_type=instanceType, security_groups=[securityGroup]) instances = reservation.instances if(len(instances) < 1): return None instance = instances[0] return instance
def launch(key_name=None, region='us-west-2', image_id='ami-5189a661', instance_type='t2.micro', security_groups='launch-wizard-1', user_data=None, initial_check=False): ''' ''' if not isinstance(security_groups, list): security_groups = [security_groups] ec2 = boto.ec2.connect_to_region(region) reserve = ec2.run_instances(image_id, key_name=key_name, instance_type=instance_type, security_groups=security_groups, user_data=user_data) inst = reserve.instances[0] while inst.state == u'pending': time.sleep(10) inst.update() if initial_check: # Wait for the status checks first status = ec2.get_all_instance_status(instance_ids=[inst.id])[0] check_stat = "Status:initializing" while str(status.system_status) == check_stat and str(status.instance_status) == check_stat: time.sleep(10) status = ec2.get_all_instance_status(instance_ids=[inst.id])[0] return inst
def start_instance(name, data): print('reserving instance for %s' % name) print(data) reserve_instance = lambda: ec2.run_instances(aws_image, key_name='sandy', user_data=data, instance_type=size, instance_profile_name=name) reservation = while_not_try(reserve_instance) instance = reservation.instances[0] print(instance) return instance
def launch_instance(region, image_id, instance_type, subnet_id, security_group): ec2 = boto.ec2.connect_to_region(region) reservation = ec2.run_instances(image_id=image_id, min_count=1, max_count=1, instance_type=instance_type, subnet_id=subnet_id, security_group_ids=[security_group]) print reservation return reservation
def createInstance(ec2,ami,nb_nodes,placement,instance_type,key,sg,user_data=None): """ Create a new EC2 instance with specific parameters SecurityGroup (sg) and KeyPair (key) have to be previously created (see cassandgo initSG and cassandgo initKP) """ reservation = ec2.run_instances(ami,min_count=nb_nodes,max_count=nb_nodes,placement = placement,key_name=key,security_groups=[sg],instance_type=instance_type,user_data=user_data) instance = reservation.instances[0] return instance
def test_ec2_client_ot(self): """OpenTracing compatibility check of the test_ec2_client test.""" ec2 = boto.ec2.connect_to_region("us-west-2") tracer = get_dummy_tracer() ot_tracer = init_tracer('my_svc', tracer) writer = tracer.writer Pin(service=self.TEST_SERVICE, tracer=tracer).onto(ec2) with ot_tracer.start_active_span('ot_span'): ec2.get_all_instances() spans = writer.pop() assert spans eq_(len(spans), 2) ot_span, dd_span = spans # confirm the parenting eq_(ot_span.parent_id, None) eq_(dd_span.parent_id, ot_span.span_id) eq_(ot_span.resource, "ot_span") eq_(dd_span.get_tag('aws.operation'), "DescribeInstances") eq_(dd_span.get_tag(http.STATUS_CODE), "200") eq_(dd_span.get_tag(http.METHOD), "POST") eq_(dd_span.get_tag('aws.region'), "us-west-2") with ot_tracer.start_active_span('ot_span'): ec2.run_instances(21) spans = writer.pop() assert spans eq_(len(spans), 2) ot_span, dd_span = spans # confirm the parenting eq_(ot_span.parent_id, None) eq_(dd_span.parent_id, ot_span.span_id) eq_(dd_span.get_tag('aws.operation'), "RunInstances") eq_(dd_span.get_tag(http.STATUS_CODE), "200") eq_(dd_span.get_tag(http.METHOD), "POST") eq_(dd_span.get_tag('aws.region'), "us-west-2") eq_(dd_span.service, "test-boto-tracing.ec2") eq_(dd_span.resource, "ec2.runinstances") eq_(dd_span.name, "ec2.command")
def test_ec2_client_ot(self): """OpenTracing compatibility check of the test_ec2_client test.""" ec2 = boto.ec2.connect_to_region('us-west-2') ot_tracer = init_tracer('my_svc', self.tracer) writer = self.tracer.writer Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2) with ot_tracer.start_active_span('ot_span'): ec2.get_all_instances() spans = writer.pop() assert spans self.assertEqual(len(spans), 2) ot_span, dd_span = spans # confirm the parenting self.assertIsNone(ot_span.parent_id) self.assertEqual(dd_span.parent_id, ot_span.span_id) self.assertEqual(ot_span.resource, 'ot_span') self.assertEqual(dd_span.get_tag('aws.operation'), 'DescribeInstances') self.assertEqual(dd_span.get_tag(http.STATUS_CODE), '200') self.assertEqual(dd_span.get_tag(http.METHOD), 'POST') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') with ot_tracer.start_active_span('ot_span'): ec2.run_instances(21) spans = writer.pop() assert spans self.assertEqual(len(spans), 2) ot_span, dd_span = spans # confirm the parenting self.assertIsNone(ot_span.parent_id) self.assertEqual(dd_span.parent_id, ot_span.span_id) self.assertEqual(dd_span.get_tag('aws.operation'), 'RunInstances') self.assertEqual(dd_span.get_tag(http.STATUS_CODE), '200') self.assertEqual(dd_span.get_tag(http.METHOD), 'POST') self.assertEqual(dd_span.get_tag('aws.region'), 'us-west-2') self.assertEqual(dd_span.service, 'test-boto-tracing.ec2') self.assertEqual(dd_span.resource, 'ec2.runinstances') self.assertEqual(dd_span.name, 'ec2.command')
def clone_instances(inst, num=1, delete_image=False, start_instance=False): ''' num: number of clone delete_image: delete image after clone start_instance: start_instance after clone ''' return ec2.run_instances( image_id='ami-f52f1dc5', instance_type='t2.micro', key_name='aws-ec2')
def test_ec2_client_ot(self): """OpenTracing compatibility check of the test_ec2_client test.""" ec2 = boto.ec2.connect_to_region("us-west-2") ot_tracer = init_tracer("my_svc", self.tracer) Pin(service=self.TEST_SERVICE, tracer=self.tracer).onto(ec2) with ot_tracer.start_active_span("ot_span"): ec2.get_all_instances() spans = self.pop_spans() assert spans self.assertEqual(len(spans), 2) ot_span, dd_span = spans # confirm the parenting self.assertIsNone(ot_span.parent_id) self.assertEqual(dd_span.parent_id, ot_span.span_id) self.assertEqual(ot_span.resource, "ot_span") self.assertEqual(dd_span.get_tag("aws.operation"), "DescribeInstances") assert_span_http_status_code(dd_span, 200) self.assertEqual(dd_span.get_tag(http.METHOD), "POST") self.assertEqual(dd_span.get_tag("aws.region"), "us-west-2") with ot_tracer.start_active_span("ot_span"): ec2.run_instances(21) spans = self.pop_spans() assert spans self.assertEqual(len(spans), 2) ot_span, dd_span = spans # confirm the parenting self.assertIsNone(ot_span.parent_id) self.assertEqual(dd_span.parent_id, ot_span.span_id) self.assertEqual(dd_span.get_tag("aws.operation"), "RunInstances") assert_span_http_status_code(dd_span, 200) self.assertEqual(dd_span.get_tag(http.METHOD), "POST") self.assertEqual(dd_span.get_tag("aws.region"), "us-west-2") self.assertEqual(dd_span.service, "test-boto-tracing.ec2") self.assertEqual(dd_span.resource, "ec2.runinstances") self.assertEqual(dd_span.name, "ec2.command")
def create_submaster(): global subMasterCounter subMasterCounter = int(subMasterCounter) + 1 ec2 = boto.connect_ec2() reservation = ec2.run_instances(image_id='ami-d05223b8', key_name='keyy', subnet_id='subnet-aaeb39f3', instance_type='t2.micro', instance_profile_name='workerProfile', user_data=str(subMasterCounter)) ec2.create_tags(reservation.instances[0].id,{"Name":str(subMasterCounter)+'Submaster',"type":'Submaster',"idnumber":subMasterCounter})
def clone_instance(instance): new_bdm = None ec2 = instance.connection if instance.block_device_mapping: root_device_name = instance.get_attribute('rootDeviceName')['rootDeviceName'] user_data = instance.get_attribute('userData')['userData'] # user_data comes back base64 encoded. Need to decode it so it # can get re-encoded by run_instance ! user_data = base64.b64decode(user_data) new_bdm = BlockDeviceMapping() for dev in instance.block_device_mapping: # if this entry is about the root device, skip it if dev != root_device_name: bdt = instance.block_device_mapping[dev] if bdt.volume_id: volume = ec2.get_all_volumes([bdt.volume_id])[0] snaps = volume.snapshots() if len(snaps) == 0: print 'No snapshots available for %s' % volume.id else: # sort the list of snapshots, newest is at the end now snaps.sort(key=lambda snap: snap.start_time) latest_snap = snaps[-1] new_bdt = BlockDeviceType() new_bdt.snapshot_id = latest_snap.id new_bdm[dev] = new_bdt return ec2.run_instances( instance.image_id, key_name=instance.key_name, security_groups=[g.name for g in instance.groups], user_data=user_data, instance_type=instance.instance_type, kernel_id=instance.kernel, ramdisk_id=instance.ramdisk, monitoring_enabled=instance.monitored, placement=instance.placement, block_device_map=new_bdm ).instances[0]
def create_workers(subMasterID,workers_no): ec2 = boto.connect_ec2() reservation = ec2.run_instances(image_id='ami-ea552482', min_count=workers_no, max_count=workers_no,key_name='keyy', subnet_id='subnet-aaeb39f3', instance_type='t2.micro', instance_profile_name='workerProfile', user_data=str(subMasterID)) ids = [] for instance in reservation.instances: ids.append(instance.id) print ids #for instance in reservation: ec2.create_tags(ids,{"Name":str(subMasterID)+'Worker',"type":'Worker',"idnumber":subMasterID})
def ensure_council(): count = 3 g1 = security_group('counselor', 'Security Group for Council').id g2 = security_group('counseled', 'Security Group for Instances Following to Council').id role_and_profile('counselor') insts = get_instances() counselors = [ i for i in insts if 'counselor' in [g.name for g in i.groups] ] counselors = [i for i in counselors if i.state in ['running', 'pending']] is_bootstrap = (len(counselors) == 0) user_data = '''#!/bin/bash sudo apt-get update sudo apt-get install -y python-dev python-setuptools sudo easy_install pip sudo pip install boto sudo pip install carlcm sudo /usr/local/bin/carlcm-counselor 3''' if is_bootstrap: user_data += ' bootstrap' user_data += '\n\n' num_to_launch = count - len(counselors) d = { 'image_id': 'ami-870a2fb7', 'key_name': 'carl-ssh-2015-02-14', 'user_data': user_data, 'security_group_ids': [g1, g2], 'subnet_id': 'subnet-7af8590d', 'instance_type': 't2.micro', 'instance_profile_name': 'counselor', 'min_count': num_to_launch, 'max_count': num_to_launch, } if num_to_launch > 0: ec2.run_instances(**d) return
def clone_instance(instance): new_bdm = None ec2 = instance.connection if instance.block_device_mapping: root_device_name = instance.get_attribute( 'rootDeviceName')['rootDeviceName'] user_data = instance.get_attribute('userData')['userData'] # user_data comes back base64 encoded. Need to decode it so it # can get re-encoded by run_instance ! user_data = base64.b64decode(user_data) new_bdm = BlockDeviceMapping() for dev in instance.block_device_mapping: # if this entry is about the root device, skip it if dev != root_device_name: bdt = instance.block_device_mapping[dev] if bdt.volume_id: volume = ec2.get_all_volumes([bdt.volume_id])[0] snaps = volume.snapshots() if len(snaps) == 0: print 'No snapshots available for %s' % volume.id else: # sort the list of snapshots, newest is at the end now snaps.sort(key=lambda snap: snap.start_time) latest_snap = snaps[-1] new_bdt = BlockDeviceType() new_bdt.snapshot_id = latest_snap.id new_bdm[dev] = new_bdt return ec2.run_instances(instance.image_id, key_name=instance.key_name, security_groups=[g.name for g in instance.groups], user_data=user_data, instance_type=instance.instance_type, kernel_id=instance.kernel, ramdisk_id=instance.ramdisk, monitoring_enabled=instance.monitored, placement=instance.placement, block_device_map=new_bdm).instances[0]
def create_box(): old_ids = set(i.id for i in ec2.get_only_instances()) machine = ec2.run_instances(AMI_ID, key_name=KEYNAME, security_groups=[os.environ.get('SECURITY_GROUP', 'memex-explorer-prod'),], instance_type='m3.2xlarge') new_instance = [i for i in ec2.get_only_instances() if i.id not in old_ids][0] print(new_instance.id) while new_instance.state != u'running': time.sleep(3) new_instance.update() while ec2.get_all_instance_status(instance_ids=[new_instance.id])[0].system_status.details['reachability'] != 'passed': time.sleep(3) time.sleep(1) assert new_instance.public_dns_name print(new_instance.public_dns_name) return new_instance
def start_vm_openstack(num_vms): """ Create a VM instance on OpenStack using the ec2 boto interface commands Args: Set the EC2_ACCESS_KEY and EC2_SECRET_KEY environment variables num_vms: Number of vms to spawn Returns: """ access_key = os.getenv("EC2_ACCESS_KEY") secret_key = os.getenv("EC2_SECRET_KEY") ec2=None try: region = boto.regioninfo.RegionInfo(name="openstack", endpoint="206.117.53.134") ec2 = boto.connect_ec2(aws_access_key_id=access_key, aws_secret_access_key=secret_key, is_secure=False, region=region, port=8773, path="/services/Cloud") except: print 'Error connection to openstack' return ## the machine image ami = "ami-00000001" ## start a bunch of vms for count in range(0,6): reservations=ec2.run_instances(ami,instance_type='m1.tiny',max_count=1) print "Launching vm ",count," of 256" ## create a chain (ie. Make in iterator that returns elements fromt the first ## iterable untils it is exhaused, then proceeds to the next iterable chain = itertools.chain.from_iterable existing_instances = list(chain([res.instances for res in ec2.get_all_instances()])) status = [e.update() for e in existing_instances] ## wait until all the instances have transitioned from 'pending' to 'running' status while 'pending' in status: existing_instances = list(chain([res.instances for res in ec2.get_all_instances()])) status = [e.update() for e in existing_instances] status_dict = dict(zip(existing_instances,status)) print [(i,status_dict[i]) for i in status_dict.keys() if status_dict[i] != 'terminated'] print 'Still pending' time.sleep(3) print 'Complete' return
def launch_instance(ec2, ami, instance_type, region, groups, key_name ): print 'Launch instance [ami=%s, type=%s, groups=%s, key=%s'%(ami, instance_type, groups, key_name) reservation = ec2.run_instances(ami, key_name=key_name, security_groups=groups, instance_type=instance_type, placement=region ) instance = reservation.instances[0] # Wait for instance state to change to 'running'. print 'waiting for instance' while instance.state != 'running': print '.' time.sleep(5) instance.update() print 'done' return instance
def create_instances(image_id='ami-2a31bf1a', key_name='jthomas', security_groups=None, count=1, tags=None, subnet_id=None): counter = 0 while (counter < count): reservation = ec2.run_instances(image_id, key_name=key_name, security_groups=security_groups, subnet_id=subnet_id) instance = reservation.instances[0] print instance if not tags is None: for key, value in tags.iteritems(): instance.add_tag(key, value) counter = counter + 1
def run_instance(obs_worker_id): global print_only if not print_only: # WARNING: Do NOT start more than 1 instance at a time here! # Multilaunching is managed at higher level to create unique tags etc. instances_min = 1 instances_max = 1 worker_customization = "#!/bin/sh\nhostname " + obs_worker_id +\ ";echo " + obs_worker_id + " > /etc/HOSTNAME;" global aws_ami_image global aws_keypair_name global aws_security_group global aws_instance_type reservation = ec2.run_instances(image_id=aws_ami_image, min_count=instances_min, max_count=instances_max, key_name=aws_keypair_name, security_groups=[aws_security_group], user_data=worker_customization, instance_type=aws_instance_type) #placement='eu-west-1b') instance = reservation.instances[0] ec2.create_tags([instance.id], {'Name': obs_worker_id}) global elastic_build_hosts elastic_build_hosts.append({ 'instance_name': obs_worker_id, 'instance_id': instance.id, 'instance_type': instance.instance_type, 'cpu': "-", 'disk': "-", 'instance_state': "SPAWNING", 'launch_time': 0, 'time_left': 60, 'ip': "-", 'workers': [] }) return
def main(config_path, name_prefix, tag): with open(config_path) as f: config = json.load(f) ec2 = boto.ec2.connect_to_region( INSTANCE_CONFIG['region'], aws_access_key_id=config['access_key_id'], aws_secret_access_key=config['secret_access_key']) name = name_prefix + "-" + datetime.utcnow().isoformat() # Assume that ssh key is uploaded group = ec2.create_security_group(name, 'A group that allows SSH access') group.authorize('tcp', 22, 22, "0.0.0.0/0") reservation = ec2.run_instances( INSTANCE_CONFIG['ami'], key_name=os.path.basename(config['certificate_path']).split(".")[0], instance_type=INSTANCE_CONFIG['type'], security_groups=[name]) # Find the actual Instance object inside the Reservation object # returned by EC2. instance = reservation.instances[0] # The instance has been launched but it's not yet up and # running. Let's wait for it's state to change to 'running'. print 'waiting for instance' while instance.state != 'running': print '.', time.sleep(1) instance.update() print 'done' instance.add_tag(tag) print "DoNe! To connect use:" print "ssh -i {} ubuntu@{}".format(config['certificate_path'], instance.public_dns_name)
def createInstance(instance_name): ec2 = boto.ec2.connect_to_region('us-west-2', **auth) try: # ami id of the instance as image to spin new ec2 (eg ami-g284cd96) image_id = '<ami-id>' # create key and security grp in advance reservations = ec2.run_instances( image_id, key_name='<key_name>', instance_type='t2.micro', security_groups=['<security-group-name>']) instance = reservations.instances[0] while instance.state != 'running': time.sleep(5) instance.update() ec2.create_tags([instance.id], {'Name': instance_name}) return instance.public_dns_name except Exception, e3: error3 = "Error3: %s" % str(e3) return error3
def run_instance(obs_worker_id): global print_only if not print_only: # WARNING: Do NOT start more than 1 instance at a time here! # Multilaunching is managed at higher level to create unique tags etc. instances_min = 1 instances_max = 1 worker_customization = "#!/bin/sh\nhostname " + obs_worker_id +\ ";echo " + obs_worker_id + " > /etc/HOSTNAME;" global aws_ami_image global aws_keypair_name global aws_security_group global aws_instance_type reservation = ec2.run_instances( image_id = aws_ami_image, min_count = instances_min, max_count = instances_max, key_name = aws_keypair_name, security_groups = [aws_security_group], user_data = worker_customization, instance_type = aws_instance_type) #placement='eu-west-1b') instance = reservation.instances[0] ec2.create_tags([instance.id],{'Name':obs_worker_id}) global elastic_build_hosts elastic_build_hosts.append({'instance_name':obs_worker_id, 'instance_id':instance.id, 'instance_type':instance.instance_type, 'cpu':"-", 'disk':"-", 'instance_state':"SPAWNING", 'launch_time':0, 'time_left':60, 'ip':"-", 'workers':[]}) return
def ec2_run_instances(ami_name, tag_name): ec2 = _get_ec2_connection() ami = fabtaskit.amis.get(ami_name) if ami is None: abort("Couldn't find ami conf with name: {}".format(ami_name)) kwargs = ami.copy() kwargs['key_name'] = os.environ['AWS_KEYPAIR_NAME'] reservation = ec2.run_instances(**kwargs) instance = reservation.instances[0] ec2.create_tags([instance.id], {'Name': tag_name}) tries = 5 booted = False while not booted and tries != 0: time.sleep(5) for r in ec2.get_all_instances(): if r.id == reservation.id: print_ec2(r.instances[0]) booted = True break tries -= 1 if not booted: abort("Couldn't find if instance booted")
def launch(key_name=None, region='us-west-2', image_id='ami-5189a661', instance_type='t2.micro', security_groups='launch-wizard-1', user_data=None, initial_check=False): ''' ''' if not isinstance(security_groups, list): security_groups = [security_groups] ec2 = boto.ec2.connect_to_region(region) reserve = ec2.run_instances(image_id, key_name=key_name, instance_type=instance_type, security_groups=security_groups, user_data=user_data) inst = reserve.instances[0] while inst.state == u'pending': time.sleep(10) inst.update() if initial_check: # Wait for the status checks first status = ec2.get_all_instance_status(instance_ids=[inst.id])[0] check_stat = "Status:initializing" while str(status.system_status) == check_stat and str( status.instance_status) == check_stat: time.sleep(10) status = ec2.get_all_instance_status(instance_ids=[inst.id])[0] return inst
def real_launch(self, region, item_to_vote, instances=1): startup = script % {'ITEM': item_to_vote} ips = [] print "launching in {0}: instances: {1}".format(region, instances) # print startup try: ec2 = boto.ec2.connect_to_region(region, aws_access_key_id=self.key, aws_secret_access_key=self.secret) try: key_pair = ec2.create_key_pair(self.ec2_key_name) # only needs to be done once key_pair.save(SSH_PATH) except boto.exception.EC2ResponseError as e: #print e pass reservation = ec2.run_instances(image_id=amis_per_regions[region], min_count=instances, max_count=instances, key_name=self.ec2_key_name, user_data=startup) ips.extend(self._insert_instances(reservation, ec2)) except Exception as e: print " !!!! not supported. \n{0}".format(e) return ips
def createInstance(ec2, ami, nb_nodes, placement, instance_type, key, sg, user_data=None): """ Create a new EC2 instance with specific parameters SecurityGroup (sg) and KeyPair (key) have to be previously created (see cassandgo initSG and cassandgo initKP) """ reservation = ec2.run_instances(ami, min_count=nb_nodes, max_count=nb_nodes, placement=placement, key_name=key, security_groups=[sg], instance_type=instance_type, user_data=user_data) instance = reservation.instances[0] return instance
def _launch_instance(type, role): if not type: type = 't1.micro' puts("Creating instance...") ec2 = _get_conn() reservation = ec2.run_instances('ami-3fec7956', key_name='hermes', security_groups=['hermes'], instance_type=type) instance = reservation.instances[0] while True: time.sleep(5) instance.update() puts("%s (%s)" % (instance.state, instance.state_code)) if instance.state_code != 0: break puts("Instance created " + instance.public_dns_name) instance.add_tag('Role', role) instance.add_tag('Section', 'hermes') return instance.public_dns_name, instance.id
params.update(dict( min_count = count_remaining, max_count = count_remaining, client_token = id, placement_group = placement_group, )) else: params.update(dict( min_count = count_remaining, max_count = count_remaining, client_token = id, placement_group = placement_group, private_ip_address = private_ip, )) res = ec2.run_instances(**params) instids = [ i.id for i in res.instances ] while True: try: ec2.get_all_instances(instids) break except boto.exception.EC2ResponseError as e: if "<Code>InvalidInstanceID.NotFound</Code>" in str(e): # there's a race between start and get an instance continue else: module.fail_json(msg = str(e)) # The instances returned through ec2.run_instances above can be in # terminated state due to idempotency. See commit 7f11c3d for a complete # explanation.
def launch_and_configure(ec2_args): """ Creates an sqs queue, launches an ec2 instance, configures it and creates an AMI. Polls SQS for updates """ print "{:<40}".format( "Creating SQS queue and launching instance for {}:".format(run_id)) print for k, v in ec2_args.iteritems(): if k != 'user_data': print " {:<25}{}".format(k, v) print global sqs_queue global instance_id sqs_queue = sqs.create_queue(run_id) sqs_queue.set_message_class(RawMessage) res = ec2.run_instances(**ec2_args) inst = res.instances[0] instance_id = inst.id print "{:<40}".format( "Waiting for instance {} to reach running status:".format(instance_id)), status_start = time.time() for _ in xrange(EC2_RUN_TIMEOUT): res = ec2.get_all_instances(instance_ids=[instance_id]) if res[0].instances[0].state == 'running': status_delta = time.time() - status_start run_summary.append(('EC2 Launch', status_delta)) print "[ OK ] {:0>2.0f}:{:0>2.0f}".format( status_delta / 60, status_delta % 60) break else: time.sleep(1) else: raise Exception("Timeout waiting for running status: {} ".format( instance_id)) print "{:<40}".format("Waiting for system status:"), system_start = time.time() for _ in xrange(EC2_STATUS_TIMEOUT): status = ec2.get_all_instance_status(inst.id) if status[0].system_status.status == u'ok': system_delta = time.time() - system_start run_summary.append(('EC2 Status Checks', system_delta)) print "[ OK ] {:0>2.0f}:{:0>2.0f}".format( system_delta / 60, system_delta % 60) break else: time.sleep(1) else: raise Exception("Timeout waiting for status checks: {} ".format( instance_id)) print print "{:<40}".format( "Waiting for user-data, polling sqs for Ansible events:") (ansible_delta, task_report) = poll_sqs_ansible() run_summary.append(('Ansible run', ansible_delta)) print print "{} longest Ansible tasks (seconds):".format(NUM_TASKS) for task in sorted( task_report, reverse=True, key=lambda k: k['DELTA'])[:NUM_TASKS]: print "{:0>3.0f} {}".format(task['DELTA'], task['TASK']) print " - {}".format(task['INVOCATION']) print print "{:<40}".format("Creating AMI:"), ami_start = time.time() ami = create_ami(instance_id, run_id, run_id) ami_delta = time.time() - ami_start print "[ OK ] {:0>2.0f}:{:0>2.0f}".format( ami_delta / 60, ami_delta % 60) run_summary.append(('AMI Build', ami_delta)) total_time = time.time() - start_time all_stages = sum(run[1] for run in run_summary) if total_time - all_stages > 0: run_summary.append(('Other', total_time - all_stages)) run_summary.append(('Total', total_time)) return run_summary, ami
print ("4. Stop instance") print ("5. Start instance") print ("6. Reboot instance") print ("7. Terminate instance") print ("8. Exit") print (30 * '-') try : choice = int ( raw_input('Enter your choice [1-8] : ') ) except ValueError, e : print ("'%s' is not a valid integer." % e.args[0].split(": ")[1]) if choice == 1: reservation = ec2.run_instances('ami-c5f30081', key_name='california', instance_type='t2.micro', security_groups=['default']) instance = reservation.instances[0] print 'waiting for instance' while instance.state != 'running': print '.' time.sleep(5) instance.update() print 'done' instance.add_tag('Name','Wordpress-Server') elif choice == 2: reservation = ec2.run_instances('ami-df6a8b9b', key_name='california', instance_type='t2.micro',
group_name, 'A group that allows VPN access') else: raise try: group.authorize('tcp', ssh_port, ssh_port, cidr) group.authorize('udp', vpn_port, vpn_port, cidr) except ec2.ResponseError, e: if e.code == 'InvalidPermission.Duplicate': """print ('Security group %s already exists') % group_name""" else: raise reservation = ec2.run_instances(ami, key_name=key_name, security_groups=[group_name], instance_type=instance_type, user_data=user_data) instance = reservation.instances[0] while instance.state != 'running': time.sleep(30) instance.update() instance.add_tag(tag) global host host = instance.ip_address print "%s" % host
else: raise #Add SSH rule to Security group try: group.authorize('tcp',ssh_port,ssh_port, cidr) except ec2.ResponseError, e: if e.code == 'InvalidPermission.Duplicate': print 'Security group %s already authorized' % group_name else: raise # Now start up the instance. The run_instances method # has many, many parameters but these are all we need # for now. reservation = ec2.run_instances(ami, key_name=key_name, security_groups=[group_name], instance_type=instance_type, user_data=user_data) # Find the actual Instance object inside the Reservation object # returned by EC2. instance = reservation.instances[0] # The instance has been launched but it's not yet up and # running. Let's wait for its state to change to 'running'. print 'Waiting for the instance' while instance.state != 'running': print '.' time.sleep(5) instance.update() print 'done'
print 'scaling policy created' ##cloud watch cloudwatch=boto.ec2.cloudwatch.connect_to_region('us-east-1') alarm_dimensions={"AutoScalingGroupName":'jianGroup'} #scale up scale_up_alarm=MetricAlarm(name='scale_up_on_cpu',namespace='AWS/EC2',metric='CPUUtilization',statistic='Average',comparison='>',threshold='80',period='60',evaluation_periods=1,alarm_actions=[scale_up_policy.policy_arn],dimensions=alarm_dimensions) cloudwatch.create_alarm(scale_up_alarm) #scale down scale_down_alarm = MetricAlarm(name='scale_down_on_cpu', namespace='AWS/EC2',metric='CPUUtilization', statistic='Average',comparison='<', threshold='60',period='60', evaluation_periods=1,alarm_actions=[scale_down_policy.policy_arn],dimensions=alarm_dimensions) cloudwatch.create_alarm(scale_down_alarm) print 'both clock watch created' #load generator ec2=boto.ec2.connect_to_region("us-east-1") reservation=ec2.run_instances('ami-7aba0c12',key_name='jj',instance_type='m3.medium',security_groups=['http']) time.sleep(60) instance=reservation.instances[0] id=instance.id #loadDns=instance.public_dns_name #get load dns groupw=autoscale.get_all_groups(names=['jianGroup'])[0] instances=ec2.get_only_instances(instance_ids=[id]) loadDns=instances[0].public_dns_name print 'load generator dns is :%s'%(loadDns) time.sleep(10) #authenticate response = urllib2.urlopen('http://'+loadDns+'/username?username=jianj') print 'load generator authenticated'
if role_name is not None and len(roles) != 0: should_add = True def security_group(name, description): try: group = ec2.get_all_security_groups(groupnames=[name])[0] except boto.exception.EC2ResponseError: group = None if group is None: ec2.create_security_group(name, description) group = ec2.get_all_security_groups(groupnames=[name])[0] return group ''' # ubuntu 14.04 cloudinit http://cloud-images.ubuntu.com/releases/14.04/release-20150209.1/ ec2.run_instances(image_id='ami-870a2fb7', key_name='carl-ssh-2015-02-14', user_data='#!/bin/bash\nsudo apt-get update\nsudo apt-get install -y python-dev python-setuptools\nsudo easy_install pip\nsudo pip install boto\nsudo pip install carlcm\n/usr/local/bin/carlcm-counselor 3', security_group_ids=['sg-233a1f46', 'sg-c03a1fa5'], subnet_id='subnet-7af8590d', instance_type='t2.micro', instance_profile_name='counselor') ''' def get_instances(): return reduce(lambda a, b: a + b, [a.instances for a in ec2.get_all_reservations()], []) def ensure_council(): count = 3 g1 = security_group('counselor', 'Security Group for Council').id g2 = security_group('counseled', 'Security Group for Instances Following to Council').id
botoVersionMismatch if (botoVersionArr[1] < 13): botoVersionMismatch if (debug): boto.set_stream_logger('h2o-ec2') ec2 = boto.ec2.connect_to_region(regionName, debug=debug) print 'Launching', numInstancesToLaunch, 'instances.' reservation = ec2.run_instances( image_id=amiId, min_count=numInstancesToLaunch, max_count=numInstancesToLaunch, key_name=keyName, instance_type=instanceType, security_groups=[securityGroupName], instance_profile_arn=iam_profile_resource_name, instance_profile_name=iam_profile_name, dry_run=dryRun ) for i in range(numInstancesToLaunch): instance = reservation.instances[i] print 'Waiting for instance', i+1, 'of', numInstancesToLaunch, '...' instance.update() while instance.state != 'running': print ' .' time.sleep(1) instance.update() print ' instance', i+1, 'of', numInstancesToLaunch, 'is up.'
with indent(4): for k, v in tags.iteritems(): puts(columns( [colored.blue(k), col1], [v, col2])) else: puts("") puts(colored.red("For real mode")) puts("") with indent(4, quote=colored.blue(' >')): puts("Starting instance") reservation = ec2.run_instances( image_id=args.ami, key_name=args.key_pair, user_data=render(args.build_script, template_dict), security_groups=args.security_group.split(','), instance_type=args.instance ) # add tags for instance in reservation.instances: for k, v in tags.iteritems(): instance.add_tag(k, v) for instance in reservation.instances: s3_key = Key(bucket) s3_key.key = '%s._cc_' % instance.id s3_key.set_contents_from_string( 'running', {'Content-Type': 'text/plain'}, replace=True)
if opts.instance_initiated_shutdown_behavior: run_args[ 'instance_initiated_shutdown_behavior'] = opts.instance_initiated_shutdown_behavior if opts.placement_group: run_args['placement_group'] = opts.placement_group if opts.private_ip_address: run_args['private_ip_address'] = opts.private_ip_address if opts.kernel: run_args['kernel_id'] = opts.kernel if opts.ramdisk: run_args['ramdisk_id'] = opts.ramdisk if opts.subnet: run_args['subnet_id'] = opts.subnet run_args['image_id'] = resolve_ami(args[0]) r = ec2.run_instances(**run_args) inst = firstinstance([r]) return str(inst.id) def ec2run_completers(self, event): cmd_param = event.line.split() if event.line.endswith(' '): cmd_param.append('') arg = cmd_param.pop() arg = cmd_param.pop() if arg in ('-t', '--instance-type'): return [ 'm1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge',
def create(args, config): instance_type = args['--type'] subnet = args['--subnet'] num = int(args['--num']) type_template = configuration.get_type_template(instance_type, config) # Launch Instances per env for env in args['<env>']: e = configuration.get_env(env, config) if not e: puts(colored.red('%s is not defined in your Nyfile' % e)) else: ec2 = connection.create(config) if not ec2: raise Exception('Unable to connect to EC2') groups = configuration.security_group_names_to_ids( names=type_template.security_groups, env=env, config=config) if not groups or not len(groups): raise Exception('Unable to determine security groups') if subnet == SUBNET_ALTERNATE: reservations = ec2.get_all_instances() instances = [i for r in reservations for i in r.instances] all_subnets = configuration.get_type_subnets( type=instance_type, env=env, config=config) if not all_subnets: all_subnets = configuration.get_env_subnets(env, config) subnets = Counter(all_subnets) vpc_subnets = [] if all_subnets: subnets.update([i.subnet_id for i in instances]) for s in subnets.items(): if s[0] in all_subnets: vpc_subnets.append(s[0]) subnets = Counter((tuple(vpc_subnets)*num)[:num]) else: subnets = Counter((subnet,)*num) reservations = [] tags = configuration.get_tags(instance_type, config) # Try to get a bootscript bootscript_paths = configuration.get_bootscript_paths( instance_type=instance_type, env=env, config=config) for sub in subnets.items(): # Parse the bootscripts once per subnet parsed_bootscripts = [] render_args = { 'type': instance_type, 'env': env, 'subnet': subnet, 'security_groups': groups, 'image_id': type_template.image_id, 'tags': (';'.join(tags) if tags else []), 'key': e.key, 'config': AttrDict(config.items()), } for p in bootscript_paths: with open(p) as tmp: puts(colored.green('Parsing Bootscript: %s' % p)) parsed_bootscripts.append( Template(tmp.read()).render(render_args)) reservations.append( ec2.run_instances( min_count=sub[1], image_id=type_template.image_id, instance_type=type_template.type, key_name=e.key, subnet_id=sub[0], security_group_ids=groups, user_data='\n'.join(parsed_bootscripts))) puts(colored.green( 'Started %s (%i x instance)' % ( reservations[-1:][0], int(sub[1])))) puts("\n") puts("Waiting on instances to start: \n") with distraction(): for reservation in reservations: for instance in reservation.instances: status = instance.update() while status == 'pending': time.sleep(10) status = instance.update() if status == 'running': puts(colored.green( 'Instance %s is up' % instance)) tags = configuration.get_tags( instance_type,config) if tags: for key,val in tags.items(): instance.add_tag(key, val) run('git fetch origin master') instance.add_tag('ny_env', env) instance.add_tag('ny_type', instance_type) instance.add_tag('repo_version', run( 'git rev-parse origin/master').std_out[:8]) else: puts(colored.red( 'Instance status: ' + status))
# If we get an InvalidGroup.NotFound error back from EC2, # it means that it doesn't exist and we need to create it. try: group = ec2.get_all_security_groups(groupnames=[group_name])[0] except ec2.ResponseError, e: if e.code == 'InvalidGroup.NotFound': print 'Invalid Security Group specified' % group_name else: raise # Now start up the instance. The run_instances method # has many, many parameters but these are all we need # for now. reservation = ec2.run_instances(ami, key_name=key_name, security_groups=[group_name], instance_type=instance_type, placement='us-west-1a', user_data=user_data) # Find the actual Instance object inside the Reservation object # returned by EC2. instance = reservation.instances[0] print ("Starting %s Instance Type, id= %s" %(instance_type,instance.id)) # The instance has been launched but it's not yet up and # running. Let's wait for it's state to change to 'running'. print ('Waiting for instance'), while instance.state != 'running': sys.stdout.write(".") sys.stdout.flush()
botoVersionMismatch if (botoVersionArr[1] < 13): botoVersionMismatch if (debug): boto.set_stream_logger('h2o-ec2') ec2 = boto.ec2.connect_to_region(regionName, debug=debug) print 'Launching', numInstancesToLaunch, 'instances.' if spotBid is None: reservation = ec2.run_instances( image_id=amiId, min_count=numInstancesToLaunch, max_count=numInstancesToLaunch, key_name=keyName, instance_type=instanceType, security_groups=[securityGroupName], instance_profile_arn=iam_profile_resource_name, instance_profile_name=iam_profile_name, dry_run=dryRun) else: spotRequests = ec2.request_spot_instances( price=spotBid, image_id=amiId, count=numInstancesToLaunch, key_name=keyName, instance_type=instanceType, security_groups=[securityGroupName], instance_profile_arn=iam_profile_resource_name, instance_profile_name=iam_profile_name, dry_run=dryRun,
for p in processes: p.join() #workers.close() server.join() elif sys.argv[3] == '-rw': port = int(sys.argv[2]) server = Process(target=startserv, args=(port,)) server.start() #trw = Process(target=trackRWorkers, args=()) #trw.start() #for testing purposes num_of_workers = 1 #change according to test for i in range(0, num_of_workers): print "Starting a worker" newWorker = ec2.run_instances('ami-9bc394ab', key_name='cs553', instance_type='t2.micro', security_groups=['launch-wizard-2']) #trw.join() server.join() else: print "invalid arguments" sys.exit(1) else: print "invalid arguments" isys.exit(1)
#!/usr/bin/env python import boto.ec2 aws_access_key = 'xxx' aws_secret_key = 'xxx' ec2 = boto.ec2.connect_to_region('ap-northeast-1', aws_access_key_id= aws_access_key, aws_secret_access_key =aws_secret_key) i=11 ip = '10.0.0.' + str(i) interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id='subnet-20ac0857', groups=['sg-0454d761'], associate_public_ip_address=True, private_ip_address=ip) interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface) t = ec2.run_instances('ami-b1b458b1', instance_type='t2.micro', min_count=1, key_name='testkey', network_interfaces=interfaces ) i=11 ip = '10.0.1.' + str(i) interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id='subnet-17ac0860', groups=['sg-e554d780'] , private_ip_address=ip) interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface) t = ec2.run_instances('ami-b1b458b1', instance_type='t2.micro', min_count=1, key_name='testkey', network_interfaces=interfaces ) i=11 ip = '10.0.2.' + str(i) interface = boto.ec2.networkinterface.NetworkInterfaceSpecification(subnet_id='subnet-12ac0865', groups=['sg-e654d783'] , private_ip_address=ip) interfaces = boto.ec2.networkinterface.NetworkInterfaceCollection(interface) t = ec2.run_instances('ami-b1b458b1', instance_type='t2.micro', min_count=1, key_name='testkey', network_interfaces=interfaces ) #ec2.associate_address(instance_id='i-a194c052')
def launch_and_configure(ec2_args): """ Creates an sqs queue, launches an ec2 instance, configures it and creates an AMI. Polls SQS for updates """ print "{:<40}".format( "Creating SQS queue and launching instance for {}:".format(run_id)) print for k, v in ec2_args.iteritems(): if k != 'user_data': print " {:<25}{}".format(k, v) print global sqs_queue global instance_id sqs_queue = sqs.create_queue(run_id) sqs_queue.set_message_class(RawMessage) res = ec2.run_instances(**ec2_args) inst = res.instances[0] instance_id = inst.id print "{:<40}".format( "Waiting for instance {} to reach running status:".format( instance_id)), status_start = time.time() for _ in xrange(EC2_RUN_TIMEOUT): res = ec2.get_all_instances(instance_ids=[instance_id]) if res[0].instances[0].state == 'running': status_delta = time.time() - status_start run_summary.append(('EC2 Launch', status_delta)) print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(status_delta / 60, status_delta % 60) break else: time.sleep(1) else: raise Exception( "Timeout waiting for running status: {} ".format(instance_id)) print "{:<40}".format("Waiting for system status:"), system_start = time.time() for _ in xrange(EC2_STATUS_TIMEOUT): status = ec2.get_all_instance_status(inst.id) if status[0].system_status.status == u'ok': system_delta = time.time() - system_start run_summary.append(('EC2 Status Checks', system_delta)) print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(system_delta / 60, system_delta % 60) break else: time.sleep(1) else: raise Exception( "Timeout waiting for status checks: {} ".format(instance_id)) print print "{:<40}".format( "Waiting for user-data, polling sqs for Ansible events:") (ansible_delta, task_report) = poll_sqs_ansible() run_summary.append(('Ansible run', ansible_delta)) print print "{} longest Ansible tasks (seconds):".format(NUM_TASKS) for task in sorted(task_report, reverse=True, key=lambda k: k['DELTA'])[:NUM_TASKS]: print "{:0>3.0f} {}".format(task['DELTA'], task['TASK']) print " - {}".format(task['INVOCATION']) print print "{:<40}".format("Creating AMI:"), ami_start = time.time() ami = create_ami(instance_id, run_id, run_id) ami_delta = time.time() - ami_start print "[ OK ] {:0>2.0f}:{:0>2.0f}".format(ami_delta / 60, ami_delta % 60) run_summary.append(('AMI Build', ami_delta)) total_time = time.time() - start_time all_stages = sum(run[1] for run in run_summary) if total_time - all_stages > 0: run_summary.append(('Other', total_time - all_stages)) run_summary.append(('Total', total_time)) return run_summary, ami