def test_ec2_factory(self): os.environ['GRAPHLAB_TEST_AMI_ID'] = 'foo' os.environ['GRAPHLAB_TEST_ENGINE_URL'] = 'bar' os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] = 'biz' os.environ['AWS_ACCESS_KEY_ID'] = 'fake' os.environ['AWS_SECRET_ACCESS_KEY'] = 'totally fake' os.environ['GRAPHLAB_TEST_HASH_KEY'] = 'embarrassingly_fake' ret = _ec2_factory('m3.xlarge') self.assertIsNotNone(ret.instance) self.assertIsNotNone(ret.instance.private_dns_name) self.assertIsNotNone(ret.instance_id) self.assertIsNotNone(ret.public_dns_name) self.assertEqual(ret.region, 'us-west-2') self.assertEqual(ret.instance.image_id, 'foo') self.assertEqual(ret.instance.instance_type, 'm3.xlarge') self.assertEqual(ret.instance.key_name, 'biz') # Upgrading to moto 0.3.7 broke this test. It worked with moto 0.3.5 # I've verified this functionality is correct in our 1.0 release. #ret.instance.update() # tags were applied before the last update #self.assertEquals(ret.instance.tags, {u'GraphLab': u'None'}) self.assertEquals(ret.instance.state, u'running') ret.stop() ret.instance.update() self.assertEquals(ret.instance.state, u'terminated')
def _start_commander_host(env_name, config, s3_folder_path, num_hosts, additional_packages, idle_shutdown_timeout): @_file_util.retry(tries=240, delay=2, retry_exception=_requests.exceptions.ConnectionError) def _wait_for_host_to_start_up(): response = _requests.get("http://%s:9004/ping" % commander.public_dns_name) if not response: raise RuntimeError() credentials = config.get_credentials() # Set user data for cluster controller num_children = num_hosts - 1 user_data = { 'auth_token': '', 'AWS_ACCESS_KEY_ID': credentials['aws_access_key_id'], 'AWS_SECRET_ACCESS_KEY': credentials['aws_secret_access_key'], 'daemon': True, 'is_cluster_controller': True, 'num_children_host': num_children, 's3_folder_path': s3_folder_path, 'additional_packages': additional_packages, 'idle_shutdown_timeout': idle_shutdown_timeout } # Propagating debug environment variables to user data if('GRAPHLAB_TEST_AMI_ID' in _os.environ and 'GRAPHLAB_TEST_ENGINE_URL' in _os.environ and 'GRAPHLAB_TEST_OS_URL' in _os.environ and 'GRAPHLAB_TEST_HASH_KEY' in _os.environ): user_data['GRAPHLAB_TEST_AMI_ID'] = _os.environ['GRAPHLAB_TEST_AMI_ID'] user_data['GRAPHLAB_TEST_ENGINE_URL'] = _os.environ['GRAPHLAB_TEST_ENGINE_URL'] user_data['GRAPHLAB_TEST_OS_URL'] = _os.environ['GRAPHLAB_TEST_OS_URL'] user_data['GRAPHLAB_TEST_HASH_KEY'] = _os.environ['GRAPHLAB_TEST_HASH_KEY'] if('GRAPHLAB_TEST_EC2_KEYPAIR' in _os.environ): user_data['GRAPHLAB_TEST_EC2_KEYPAIR'] = _os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] # Launch the cluster controller tags = _copy(config.tags) tags['Name'] = env_name commander, security_group, subnet_id = _ec2_factory(config.instance_type, region = config.region, CIDR_rule = config.cidr_ip, security_group_name = config.security_group, tags = tags, user_data = user_data, credentials = credentials, product_type = _ProductType.DatoDistributed) # Log message explaining the additional hosts will not be launched by us. if num_children == 1: __LOGGER__.info("One additional host will be launched by %s" % commander.instance_id) elif num_children > 1: __LOGGER__.info("%d additional hosts will be launched by %s" % (num_children, commander.instance_id)) # Wait for cluster_controller_daemon __LOGGER__.info("Waiting for %s to start up." % commander.instance_id) try: _wait_for_host_to_start_up() except: raise RuntimeError('Unable to start host(s). Please terminate ' 'manually from the AWS console.') return commander.public_dns_name
def test_launching_multiple_hosts(self): os.environ['GRAPHLAB_TEST_AMI_ID'] = 'foo' os.environ['GRAPHLAB_TEST_ENGINE_URL'] = 'bar' os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] = 'biz' os.environ['AWS_ACCESS_KEY_ID'] = 'fake' os.environ['AWS_SECRET_ACCESS_KEY'] = 'totally fake' os.environ['GRAPHLAB_TEST_HASH_KEY'] = 'embarrassingly_fake' ret = _ec2_factory('m3.xlarge', num_hosts = 2) self.assertIsInstance(ret, list) self.assertEquals(len(ret), 2) self.assertIsInstance(ret[0], EC2._Ec2Instance) self.assertIsInstance(ret[1], EC2._Ec2Instance)
def _start_commander_host(environment, credentials): user_data = { 'daemon': True, 'num_children_host': environment.num_hosts - 1, 'aws_access_key': credentials['aws_access_key_id'], 'aws_secret_key': credentials['aws_secret_access_key'], 'auth_token': '' } if('GRAPHLAB_TEST_AMI_ID' in _os.environ and 'GRAPHLAB_TEST_ENGINE_URL' in _os.environ): user_data['GRAPHLAB_TEST_AMI_ID'] = _os.environ['GRAPHLAB_TEST_AMI_ID'] user_data['GRAPHLAB_TEST_ENGINE_URL'] = _os.environ['GRAPHLAB_TEST_ENGINE_URL'] if('GRAPHLAB_TEST_EC2_KEYPAIR' in _os.environ): user_data['GRAPHLAB_TEST_EC2_KEYPAIR'] = _os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] # Launch the (commander) EC2 host commander = _ec2_factory(environment.instance_type, region = environment.region, CIDR_rule = environment.CIDR_rule, security_group_name = environment.security_group, tags = environment.tags, user_data = user_data, credentials = credentials) return commander
def launch(name, ec2_config, s3_predictive_object_path, admin_key, ssl_credentials, aws_credentials, started): s3_log_path = "s3://%s/%s%s_logs" % ( ec2_config.s3_bucket, ec2_config.s3_log_folder_path, name, ) # s3_log_folder_path always ends in '/' user_data = { "log_path": s3_log_path, "aws_access_key": aws_credentials["aws_access_key_id"], "aws_secret_key": aws_credentials["aws_secret_access_key"], "s3_predictive_service": s3_predictive_object_path, } _logger.info("[Step 1/5]: Launching EC2 instances.") # Start the hosts up. ec2_hosts = _ec2_factory( ec2_config.instance_type, region=ec2_config.region, CIDR_rule=ec2_config.CIDR_rule, security_group_name=ec2_config.security_group, tags=ec2_config.tags, user_data=user_data, credentials=aws_credentials, num_hosts=ec2_config.num_hosts, ami_service_parameters={"service": "predictive"}, ) lb = None try: # Determine host ids and availability zones used. zones, host_ids, is_vpc = set(), [], False if ec2_config.num_hosts == 1: host_ids = [ec2_hosts.instance_id] zones.add(ec2_hosts.instance.placement) is_vpc = bool(ec2_hosts.instance.vpc_id) else: for i in ec2_hosts: zones.add(i.instance.placement) host_ids.append(i.instance_id) is_vpc = bool(ec2_hosts[0].instance.vpc_id) certificate_name, certificate_is_self_signed = None, None if ssl_credentials: # Using HTTPS (private_key_path, public_certificate_path, certificate_is_self_signed) = ssl_credentials certificate_name = name certificate_id = Ec2PredictiveServiceEnvironment._upload_ssl_info( certificate_name, private_key_path, public_certificate_path, aws_credentials ) listener_tuple = (443, Ec2PredictiveServiceEnvironment._PORT_NUM, "https", certificate_id) else: # Using HTTP listener_tuple = (80, Ec2PredictiveServiceEnvironment._PORT_NUM, "http") conn = boto.ec2.elb.connect_to_region(ec2_config.region, **aws_credentials) # Create the load balancer. _logger.info("[Step 2/5]: Launching Load Balancer.") while (datetime.now() - started).total_seconds() < MAX_CREATE_TIMEOUT_SECS: try: lb = conn.create_load_balancer(name, zones, [listener_tuple]) break except BotoServerError as e: # We just uploaded the certificate, so there's a good chance it will not be found, yet. if "<Code>CertificateNotFound</Code>" not in str(e): raise e sleep(1) else: raise RuntimeError( "Unable to successfully create load balancer. Please confirm in AWS Management Console" ) # If we're running in a VPC (i.e. not "EC2-Classic") we need to attach a security group. if is_vpc: # Look up security group id security_group = None while (datetime.now() - started).total_seconds() < MAX_CREATE_TIMEOUT_SECS: ec2_conn = boto.ec2.connect_to_region(ec2_config.region, **aws_credentials) for sg in ec2_conn.get_all_security_groups(): if GRAPHLAB_NAME == sg.name: security_group = sg break # found it if security_group: break # no more eventual consistency else: sleep(1) else: raise RuntimeError("Unable to find expected security group needed to configure load balancer.") lb.apply_security_groups([sg.id]) _logger.info("[Step 3/5]: Configuring Load Balancer.") # Configure healthCheck health_target = "HTTP:%d/control/healthcheck" % PredictiveServiceEnvironment._PORT_NUM health_check = boto.ec2.elb.HealthCheck( interval=20, healthy_threshold=3, unhealthy_threshold=5, target=health_target ) lb.configure_health_check(health_check) # Add EC2 instances to the load balancer. lb.register_instances(host_ids) except: _logger.error("Could not create or configure the load balancer, terminating EC2 instances.") _stop_instances([h.instance_id for h in ec2_hosts], ec2_config.region, credentials=aws_credentials) if lb: _logger.info("Deleting the load balancer.") lb.delete() raise return Ec2PredictiveServiceEnvironment( lb.dns_name, ec2_config.region, s3_log_path, admin_key, certificate_name, certificate_is_self_signed, aws_credentials, )
def _start_commander_host(env_name, config, s3_folder_path, num_hosts, additional_packages, idle_shutdown_timeout): @_file_util.retry(tries=240, delay=2, retry_exception=_requests.exceptions.ConnectionError) def _wait_for_host_to_start_up(): response = _requests.get("http://%s:9004/ping" % commander.public_dns_name) if not response: raise RuntimeError() credentials = config.get_credentials() # Set user data for cluster controller num_children = num_hosts - 1 user_data = { 'auth_token': '', 'AWS_ACCESS_KEY_ID': credentials['aws_access_key_id'], 'AWS_SECRET_ACCESS_KEY': credentials['aws_secret_access_key'], 'daemon': True, 'is_cluster_controller': True, 'num_children_host': num_children, 's3_folder_path': s3_folder_path, 'additional_packages': additional_packages, 'idle_shutdown_timeout': idle_shutdown_timeout } # Propagating debug environment variables to user data if ('GRAPHLAB_TEST_AMI_ID' in _os.environ and 'GRAPHLAB_TEST_ENGINE_URL' in _os.environ and 'GRAPHLAB_TEST_OS_URL' in _os.environ and 'GRAPHLAB_TEST_HASH_KEY' in _os.environ): user_data['GRAPHLAB_TEST_AMI_ID'] = _os.environ[ 'GRAPHLAB_TEST_AMI_ID'] user_data['GRAPHLAB_TEST_ENGINE_URL'] = _os.environ[ 'GRAPHLAB_TEST_ENGINE_URL'] user_data['GRAPHLAB_TEST_OS_URL'] = _os.environ[ 'GRAPHLAB_TEST_OS_URL'] user_data['GRAPHLAB_TEST_HASH_KEY'] = _os.environ[ 'GRAPHLAB_TEST_HASH_KEY'] if ('GRAPHLAB_TEST_EC2_KEYPAIR' in _os.environ): user_data['GRAPHLAB_TEST_EC2_KEYPAIR'] = _os.environ[ 'GRAPHLAB_TEST_EC2_KEYPAIR'] # Launch the cluster controller tags = _copy(config.tags) tags['Name'] = env_name commander, security_group, subnet_id = _ec2_factory( config.instance_type, region=config.region, CIDR_rule=config.cidr_ip, security_group_name=config.security_group, tags=tags, user_data=user_data, credentials=credentials, product_type=_ProductType.TuriDistributed, subnet_id=config.subnet_id, security_group_id=config.security_group_id) # Log message explaining the additional hosts will not be launched by us. if num_children == 1: __LOGGER__.info("One additional host will be launched by %s" % commander.instance_id) elif num_children > 1: __LOGGER__.info("%d additional hosts will be launched by %s" % (num_children, commander.instance_id)) # Wait for cluster_controller_daemon __LOGGER__.info("Waiting for %s to start up." % commander.instance_id) try: _wait_for_host_to_start_up() except: raise RuntimeError('Unable to start host(s). Please terminate ' 'manually from the AWS console.') return commander.public_dns_name