def test_ec2_factory(self):
os.environ['GRAPHLAB_TEST_AMI_ID'] = 'foo'
os.environ['GRAPHLAB_TEST_ENGINE_URL'] = 'bar'
os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] = 'biz'
os.environ['AWS_ACCESS_KEY_ID'] = 'fake'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'totally fake'
os.environ['GRAPHLAB_TEST_HASH_KEY'] = 'embarrassingly_fake'
ret = _ec2_factory('m3.xlarge')
self.assertIsNotNone(ret.instance)
self.assertIsNotNone(ret.instance.private_dns_name)
self.assertIsNotNone(ret.instance_id)
self.assertIsNotNone(ret.public_dns_name)
self.assertEqual(ret.region, 'us-west-2')
self.assertEqual(ret.instance.image_id, 'foo')
self.assertEqual(ret.instance.instance_type, 'm3.xlarge')
self.assertEqual(ret.instance.key_name, 'biz')
# Upgrading to moto 0.3.7 broke this test. It worked with moto 0.3.5
# I've verified this functionality is correct in our 1.0 release.
#ret.instance.update() # tags were applied before the last update
#self.assertEquals(ret.instance.tags, {u'GraphLab': u'None'})
self.assertEquals(ret.instance.state, u'running')
ret.stop()
ret.instance.update()
self.assertEquals(ret.instance.state, u'terminated')
def _start_commander_host(env_name, config, s3_folder_path, num_hosts, additional_packages,
idle_shutdown_timeout):
@_file_util.retry(tries=240, delay=2, retry_exception=_requests.exceptions.ConnectionError)
def _wait_for_host_to_start_up():
response = _requests.get("http://%s:9004/ping" % commander.public_dns_name)
if not response:
raise RuntimeError()
credentials = config.get_credentials()
# Set user data for cluster controller
num_children = num_hosts - 1
user_data = {
'auth_token': '', 'AWS_ACCESS_KEY_ID': credentials['aws_access_key_id'],
'AWS_SECRET_ACCESS_KEY': credentials['aws_secret_access_key'], 'daemon': True,
'is_cluster_controller': True, 'num_children_host': num_children,
's3_folder_path': s3_folder_path, 'additional_packages': additional_packages,
'idle_shutdown_timeout': idle_shutdown_timeout
}
# Propagating debug environment variables to user data
if('GRAPHLAB_TEST_AMI_ID' in _os.environ and 'GRAPHLAB_TEST_ENGINE_URL' in _os.environ
and 'GRAPHLAB_TEST_OS_URL' in _os.environ and 'GRAPHLAB_TEST_HASH_KEY' in _os.environ):
user_data['GRAPHLAB_TEST_AMI_ID'] = _os.environ['GRAPHLAB_TEST_AMI_ID']
user_data['GRAPHLAB_TEST_ENGINE_URL'] = _os.environ['GRAPHLAB_TEST_ENGINE_URL']
user_data['GRAPHLAB_TEST_OS_URL'] = _os.environ['GRAPHLAB_TEST_OS_URL']
user_data['GRAPHLAB_TEST_HASH_KEY'] = _os.environ['GRAPHLAB_TEST_HASH_KEY']
if('GRAPHLAB_TEST_EC2_KEYPAIR' in _os.environ):
user_data['GRAPHLAB_TEST_EC2_KEYPAIR'] = _os.environ['GRAPHLAB_TEST_EC2_KEYPAIR']
# Launch the cluster controller
tags = _copy(config.tags)
tags['Name'] = env_name
commander, security_group, subnet_id = _ec2_factory(config.instance_type, region = config.region,
CIDR_rule = config.cidr_ip,
security_group_name = config.security_group,
tags = tags, user_data = user_data,
credentials = credentials,
product_type = _ProductType.DatoDistributed)
# Log message explaining the additional hosts will not be launched by us.
if num_children == 1:
__LOGGER__.info("One additional host will be launched by %s" % commander.instance_id)
elif num_children > 1:
__LOGGER__.info("%d additional hosts will be launched by %s"
% (num_children, commander.instance_id))
# Wait for cluster_controller_daemon
__LOGGER__.info("Waiting for %s to start up." % commander.instance_id)
try:
_wait_for_host_to_start_up()
except:
raise RuntimeError('Unable to start host(s). Please terminate '
'manually from the AWS console.')
return commander.public_dns_name
def test_launching_multiple_hosts(self):
os.environ['GRAPHLAB_TEST_AMI_ID'] = 'foo'
os.environ['GRAPHLAB_TEST_ENGINE_URL'] = 'bar'
os.environ['GRAPHLAB_TEST_EC2_KEYPAIR'] = 'biz'
os.environ['AWS_ACCESS_KEY_ID'] = 'fake'
os.environ['AWS_SECRET_ACCESS_KEY'] = 'totally fake'
os.environ['GRAPHLAB_TEST_HASH_KEY'] = 'embarrassingly_fake'
ret = _ec2_factory('m3.xlarge', num_hosts = 2)
self.assertIsInstance(ret, list)
self.assertEquals(len(ret), 2)
self.assertIsInstance(ret[0], EC2._Ec2Instance)
self.assertIsInstance(ret[1], EC2._Ec2Instance)
def _start_commander_host(environment, credentials):
user_data = {
'daemon': True, 'num_children_host': environment.num_hosts - 1,
'aws_access_key': credentials['aws_access_key_id'],
'aws_secret_key': credentials['aws_secret_access_key'],
'auth_token': ''
}
if('GRAPHLAB_TEST_AMI_ID' in _os.environ and 'GRAPHLAB_TEST_ENGINE_URL' in _os.environ):
user_data['GRAPHLAB_TEST_AMI_ID'] = _os.environ['GRAPHLAB_TEST_AMI_ID']
user_data['GRAPHLAB_TEST_ENGINE_URL'] = _os.environ['GRAPHLAB_TEST_ENGINE_URL']
if('GRAPHLAB_TEST_EC2_KEYPAIR' in _os.environ):
user_data['GRAPHLAB_TEST_EC2_KEYPAIR'] = _os.environ['GRAPHLAB_TEST_EC2_KEYPAIR']
# Launch the (commander) EC2 host
commander = _ec2_factory(environment.instance_type, region = environment.region,
CIDR_rule = environment.CIDR_rule,
security_group_name = environment.security_group,
tags = environment.tags, user_data = user_data,
credentials = credentials)
return commander
def launch(name, ec2_config, s3_predictive_object_path, admin_key, ssl_credentials, aws_credentials, started):
s3_log_path = "s3://%s/%s%s_logs" % (
ec2_config.s3_bucket,
ec2_config.s3_log_folder_path,
name,
) # s3_log_folder_path always ends in '/'
user_data = {
"log_path": s3_log_path,
"aws_access_key": aws_credentials["aws_access_key_id"],
"aws_secret_key": aws_credentials["aws_secret_access_key"],
"s3_predictive_service": s3_predictive_object_path,
}
_logger.info("[Step 1/5]: Launching EC2 instances.")
# Start the hosts up.
ec2_hosts = _ec2_factory(
ec2_config.instance_type,
region=ec2_config.region,
CIDR_rule=ec2_config.CIDR_rule,
security_group_name=ec2_config.security_group,
tags=ec2_config.tags,
user_data=user_data,
credentials=aws_credentials,
num_hosts=ec2_config.num_hosts,
ami_service_parameters={"service": "predictive"},
)
lb = None
try:
# Determine host ids and availability zones used.
zones, host_ids, is_vpc = set(), [], False
if ec2_config.num_hosts == 1:
host_ids = [ec2_hosts.instance_id]
zones.add(ec2_hosts.instance.placement)
is_vpc = bool(ec2_hosts.instance.vpc_id)
else:
for i in ec2_hosts:
zones.add(i.instance.placement)
host_ids.append(i.instance_id)
is_vpc = bool(ec2_hosts[0].instance.vpc_id)
certificate_name, certificate_is_self_signed = None, None
if ssl_credentials:
# Using HTTPS
(private_key_path, public_certificate_path, certificate_is_self_signed) = ssl_credentials
certificate_name = name
certificate_id = Ec2PredictiveServiceEnvironment._upload_ssl_info(
certificate_name, private_key_path, public_certificate_path, aws_credentials
)
listener_tuple = (443, Ec2PredictiveServiceEnvironment._PORT_NUM, "https", certificate_id)
else:
# Using HTTP
listener_tuple = (80, Ec2PredictiveServiceEnvironment._PORT_NUM, "http")
conn = boto.ec2.elb.connect_to_region(ec2_config.region, **aws_credentials)
# Create the load balancer.
_logger.info("[Step 2/5]: Launching Load Balancer.")
while (datetime.now() - started).total_seconds() < MAX_CREATE_TIMEOUT_SECS:
try:
lb = conn.create_load_balancer(name, zones, [listener_tuple])
break
except BotoServerError as e:
# We just uploaded the certificate, so there's a good chance it will not be found, yet.
if "<Code>CertificateNotFound</Code>" not in str(e):
raise e
sleep(1)
else:
raise RuntimeError(
"Unable to successfully create load balancer. Please confirm in AWS Management Console"
)
# If we're running in a VPC (i.e. not "EC2-Classic") we need to attach a security group.
if is_vpc:
# Look up security group id
security_group = None
while (datetime.now() - started).total_seconds() < MAX_CREATE_TIMEOUT_SECS:
ec2_conn = boto.ec2.connect_to_region(ec2_config.region, **aws_credentials)
for sg in ec2_conn.get_all_security_groups():
if GRAPHLAB_NAME == sg.name:
security_group = sg
break # found it
if security_group:
break # no more eventual consistency
else:
sleep(1)
else:
raise RuntimeError("Unable to find expected security group needed to configure load balancer.")
lb.apply_security_groups([sg.id])
_logger.info("[Step 3/5]: Configuring Load Balancer.")
# Configure healthCheck
health_target = "HTTP:%d/control/healthcheck" % PredictiveServiceEnvironment._PORT_NUM
health_check = boto.ec2.elb.HealthCheck(
interval=20, healthy_threshold=3, unhealthy_threshold=5, target=health_target
)
lb.configure_health_check(health_check)
# Add EC2 instances to the load balancer.
lb.register_instances(host_ids)
except:
_logger.error("Could not create or configure the load balancer, terminating EC2 instances.")
_stop_instances([h.instance_id for h in ec2_hosts], ec2_config.region, credentials=aws_credentials)
if lb:
_logger.info("Deleting the load balancer.")
lb.delete()
raise
return Ec2PredictiveServiceEnvironment(
lb.dns_name,
ec2_config.region,
s3_log_path,
admin_key,
certificate_name,
certificate_is_self_signed,
aws_credentials,
)
def _start_commander_host(env_name, config, s3_folder_path, num_hosts,
additional_packages, idle_shutdown_timeout):
@_file_util.retry(tries=240,
delay=2,
retry_exception=_requests.exceptions.ConnectionError)
def _wait_for_host_to_start_up():
response = _requests.get("http://%s:9004/ping" %
commander.public_dns_name)
if not response:
raise RuntimeError()
credentials = config.get_credentials()
# Set user data for cluster controller
num_children = num_hosts - 1
user_data = {
'auth_token': '',
'AWS_ACCESS_KEY_ID': credentials['aws_access_key_id'],
'AWS_SECRET_ACCESS_KEY': credentials['aws_secret_access_key'],
'daemon': True,
'is_cluster_controller': True,
'num_children_host': num_children,
's3_folder_path': s3_folder_path,
'additional_packages': additional_packages,
'idle_shutdown_timeout': idle_shutdown_timeout
}
# Propagating debug environment variables to user data
if ('GRAPHLAB_TEST_AMI_ID' in _os.environ
and 'GRAPHLAB_TEST_ENGINE_URL' in _os.environ
and 'GRAPHLAB_TEST_OS_URL' in _os.environ
and 'GRAPHLAB_TEST_HASH_KEY' in _os.environ):
user_data['GRAPHLAB_TEST_AMI_ID'] = _os.environ[
'GRAPHLAB_TEST_AMI_ID']
user_data['GRAPHLAB_TEST_ENGINE_URL'] = _os.environ[
'GRAPHLAB_TEST_ENGINE_URL']
user_data['GRAPHLAB_TEST_OS_URL'] = _os.environ[
'GRAPHLAB_TEST_OS_URL']
user_data['GRAPHLAB_TEST_HASH_KEY'] = _os.environ[
'GRAPHLAB_TEST_HASH_KEY']
if ('GRAPHLAB_TEST_EC2_KEYPAIR' in _os.environ):
user_data['GRAPHLAB_TEST_EC2_KEYPAIR'] = _os.environ[
'GRAPHLAB_TEST_EC2_KEYPAIR']
# Launch the cluster controller
tags = _copy(config.tags)
tags['Name'] = env_name
commander, security_group, subnet_id = _ec2_factory(
config.instance_type,
region=config.region,
CIDR_rule=config.cidr_ip,
security_group_name=config.security_group,
tags=tags,
user_data=user_data,
credentials=credentials,
product_type=_ProductType.TuriDistributed,
subnet_id=config.subnet_id,
security_group_id=config.security_group_id)
# Log message explaining the additional hosts will not be launched by us.
if num_children == 1:
__LOGGER__.info("One additional host will be launched by %s" %
commander.instance_id)
elif num_children > 1:
__LOGGER__.info("%d additional hosts will be launched by %s" %
(num_children, commander.instance_id))
# Wait for cluster_controller_daemon
__LOGGER__.info("Waiting for %s to start up." % commander.instance_id)
try:
_wait_for_host_to_start_up()
except:
raise RuntimeError('Unable to start host(s). Please terminate '
'manually from the AWS console.')
return commander.public_dns_name
