def test_secure_communication(self, mock_logging):
SERVER_PUBLIC_KEY = "Ee4##T$OmI4]hzyKqZT@H&Fixt95^.72&%MK!UR."
SERVER_SECRET_KEY = "lIn2Szq0.mpPiB<N)t6fR2/4^4&wYnFs-x72HlTz"
# Non-error case
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
server = create_server(ipc_addr, public_key=SERVER_PUBLIC_KEY, secret_key=SERVER_SECRET_KEY)
server.start()
glconnect.launch(server_addr=ipc_addr, server_public_key=SERVER_PUBLIC_KEY)
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
# Tests with bogus key
BOGUS_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
server = create_server(ipc_addr, public_key=BOGUS_KEY, secret_key=SERVER_SECRET_KEY)
try:
server.start()
except:
pass
else:
self.fail("Server started with bogus key.")
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
server = create_server(ipc_addr, public_key=SERVER_PUBLIC_KEY, secret_key=BOGUS_KEY)
try:
server.start()
except:
pass
else:
self.fail("Server started with bogus key.")
def test_launch_to_tcp(self):
auth_token = 'graphlab_awesome'
tcp_server = start_test_tcp_server(auth_token)
#launch with remote server tcp address
glconnect.launch(tcp_server.get_server_addr(), auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
tcp_server.stop()
def test_launch_to_tcp(self):
auth_token = 'graphlab_awesome'
tcp_server = start_test_tcp_server(auth_token)
#launch with remote server tcp address
glconnect.launch(tcp_server.get_server_addr(), auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(),
server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
tcp_server.stop()
def test_launch_with_exception(self, mock_logging):
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
auth_token = 'graphlab_awesome'
ipc_server = create_server(ipc_addr, auth_token)
ipc_server.start()
#default launch without stopping
glconnect.launch(server_addr=ipc_addr)
glconnect.launch()
self.assertTrue(mock_logging.warning.called_once_with(SubstringMatcher(containing="existing server")))
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
# launch with bogus server path (path is not listend by server)
with tempfile.NamedTemporaryFile() as f:
glconnect.launch(server_addr=('ipc://' + f.name))
self.assertTrue(mock_logging.warning.called_once_with(SubstringMatcher(containing="communication failure")))
self.assertFalse(glconnect.is_connected())
def test_launch_to_ipc(self):
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
auth_token = 'graphlab_awesome'
ipc_server = create_server(ipc_addr, auth_token)
ipc_server.start()
#default local launch
glconnect.launch()
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with remote server ipc address
glconnect.launch(ipc_addr, auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(),
server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with remote server addr, and server_bin(ignored)
glconnect.launch(ipc_addr,
os.getenv("GRAPHLAB_UNITY"),
auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(),
server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
ipc_server.stop()
def test_launch_to_ipc(self):
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
auth_token = 'graphlab_awesome'
ipc_server = create_server(ipc_addr, auth_token)
ipc_server.start()
#default local launch
glconnect.launch()
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with remote server ipc address
glconnect.launch(ipc_addr, auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with remote server addr, and server_bin(ignored)
glconnect.launch(ipc_addr, os.getenv("GRAPHLAB_UNITY"), auth_token=auth_token)
self.assertTrue(glconnect.is_connected())
self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
glconnect.stop()
self.assertFalse(glconnect.is_connected())
ipc_server.stop()
def test_launch(self):
#default launch
glconnect.launch()
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with server address
tmpname = tempfile.NamedTemporaryFile().name
tmpaddr = 'ipc://' + tmpname
glconnect.launch(tmpaddr)
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#check that the ipc file gets deleted
self.assertFalse(os.path.exists(tmpname))
#launch address and binary
graphlab_bin = os.getenv("GRAPHLAB_UNITY")
glconnect.launch(server_addr=tmpaddr,
server_bin=graphlab_bin)
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
self.assertFalse(os.path.exists(tmpname))
def test_secure_communication(self, mock_logging):
SERVER_PUBLIC_KEY = "Ee4##T$OmI4]hzyKqZT@H&Fixt95^.72&%MK!UR."
SERVER_SECRET_KEY = "lIn2Szq0.mpPiB<N)t6fR2/4^4&wYnFs-x72HlTz"
# Non-error case
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
server = create_server(ipc_addr,
public_key=SERVER_PUBLIC_KEY,
secret_key=SERVER_SECRET_KEY)
server.start()
glconnect.launch(server_addr=ipc_addr,
server_public_key=SERVER_PUBLIC_KEY)
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
# Tests with bogus key
BOGUS_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
server = create_server(ipc_addr,
public_key=BOGUS_KEY,
secret_key=SERVER_SECRET_KEY)
try:
server.start()
except:
pass
else:
self.fail("Server started with bogus key.")
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
server = create_server(ipc_addr,
public_key=SERVER_PUBLIC_KEY,
secret_key=BOGUS_KEY)
try:
server.start()
except:
pass
else:
self.fail("Server started with bogus key.")
def test_launch_with_exception(self, mock_logging):
ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
auth_token = 'graphlab_awesome'
ipc_server = create_server(ipc_addr, auth_token)
ipc_server.start()
#default launch without stopping
glconnect.launch(server_addr=ipc_addr)
glconnect.launch()
self.assertTrue(
mock_logging.warning.called_once_with(
SubstringMatcher(containing="existing server")))
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
# launch with bogus server path (path is not listend by server)
with tempfile.NamedTemporaryFile() as f:
glconnect.launch(server_addr=('ipc://' + f.name))
self.assertTrue(
mock_logging.warning.called_once_with(
SubstringMatcher(containing="communication failure")))
self.assertFalse(glconnect.is_connected())
def test_launch_with_exception(self, mock_logging):
# Assert warning logged when launching without stopping
glconnect.launch()
glconnect.launch()
self.assertTrue(
mock_logging.warning.called_once_with(
SubstringMatcher(containing="existing server")))
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
# launch with bogus server binary (path is not executable)
with tempfile.NamedTemporaryFile() as f:
random_server_bin = f.name
glconnect.launch(server_bin=random_server_bin)
self.assertTrue(
mock_logging.error.called_once_with(
SubstringMatcher(containing="Invalid server binary")))
self.assertFalse(glconnect.is_connected())
#launch with server address without permission
tmpaddr = 'ipc:///root/bad_server'
glconnect.launch(tmpaddr)
self.assertTrue(
mock_logging.warning.called_once_with(
SubstringMatcher(containing="communication error")))
self.assertFalse(glconnect.is_connected())
glconnect.stop()
# launch with binary that does not exist
tmpname = tempfile.NamedTemporaryFile().name
glconnect.launch(server_bin=tmpname)
self.assertTrue(
mock_logging.error.called_once_with(
SubstringMatcher(containing="Invalid server binary")))
self.assertFalse(glconnect.is_connected())
# launch with bogus server binary (path is a faked executable)
with tempfile.NamedTemporaryFile() as f:
os.chmod(f.name, stat.S_IXUSR)
random_server_bin = f.name
glconnect.launch(server_bin=random_server_bin)
self.assertTrue(
mock_logging.error.called_once_with(
SubstringMatcher(containing="Invalid server binary")))
self.assertFalse(glconnect.is_connected())
def test_launch(self):
#default launch
glconnect.launch()
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#launch with server address
tmpname = tempfile.NamedTemporaryFile().name
tmpaddr = 'ipc://' + tmpname
glconnect.launch(tmpaddr)
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
#check that the ipc file gets deleted
self.assertFalse(os.path.exists(tmpname))
#launch address and binary
graphlab_bin = os.getenv("GRAPHLAB_UNITY")
glconnect.launch(server_addr=tmpaddr, server_bin=graphlab_bin)
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
self.assertFalse(os.path.exists(tmpname))
def test_launch_with_exception(self, mock_logging):
# Assert warning logged when launching without stopping
glconnect.launch()
glconnect.launch()
self.assertTrue(mock_logging.warning.called_once_with(SubstringMatcher(containing="existing server")))
self.assertTrue(glconnect.is_connected())
glconnect.stop()
self.assertFalse(glconnect.is_connected())
# launch with bogus server binary (path is not executable)
with tempfile.NamedTemporaryFile() as f:
random_server_bin = f.name
glconnect.launch(server_bin=random_server_bin)
self.assertTrue(mock_logging.error.called_once_with(SubstringMatcher(containing="Invalid server binary")))
self.assertFalse(glconnect.is_connected())
#launch with server address without permission
tmpaddr = 'ipc:///root/bad_server'
glconnect.launch(tmpaddr)
self.assertTrue(mock_logging.warning.called_once_with(SubstringMatcher(containing="communication error")))
self.assertFalse(glconnect.is_connected())
glconnect.stop()
# launch with binary that does not exist
tmpname = tempfile.NamedTemporaryFile().name
glconnect.launch(server_bin=tmpname)
self.assertTrue(mock_logging.error.called_once_with(SubstringMatcher(containing="Invalid server binary")))
self.assertFalse(glconnect.is_connected())
# launch with bogus server binary (path is a faked executable)
with tempfile.NamedTemporaryFile() as f:
os.chmod(f.name, stat.S_IXUSR)
random_server_bin = f.name
glconnect.launch(server_bin=random_server_bin)
self.assertTrue(mock_logging.error.called_once_with(SubstringMatcher(containing="Invalid server binary")))
self.assertFalse(glconnect.is_connected())
def launch_EC2(instance_type=DEFAULT_INSTANCE_TYPE, region=None, CIDR_rule=None, auth_token=None,
security_group=None, tags=None, use_secure_connection=True):
"""
Launches an EC2 instance. All subsequent GraphLab operations will be
executed on that host. Prior to calling this function, AWS credentials must
be set as environment variables (see
:py:func:`~graphlab.aws.set_credentials`).
Parameters
----------
instance_type : string, optional
The EC2 instance type to launch. We support `all instance types
<http://aws.amazon.com/ec2/instance-types/#Instance_Types>`_ that are not
micros, smalls or mediums.
region : string, optional
The `AWS region
<http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region>`_
in which to launch your instance. Default is 'us-west-2'.
CIDR_rule : string or list[string], optional
The Classless Inter-Domain Routing rule(s) to use for the instance.
Useful for restricting the IP Address Range for a client. Default is no
restriction. If you specify CIDR_rule(s), you must also specify a
security group to use.
auth_token : string, optional
The Authentication Token to be used by the instance. By default a
randomly generated token is used.
security_group : string, optional
The name of the security group for the EC2 instance to use.
tags : dict, optional
A dictionary containing the name/value tag pairs to be assigned to the
instance. If you want to create only a tag name, the value for that tag
should be the empty string (i.e. ''). In addition to these specified
tags, a 'GraphLab' tag will also be assigned.
use_secure_connection : bool, optional
Determine whether the communication, between your computer and the EC2
instance, should be encrypted.
See Also
--------
terminate_EC2
Examples
--------
>>> # Launch a general purpose xlarge host in Oregon.
>>> graphlab.aws.launch_EC2()
>>> # Launch an xlarge compute optimized host in the US East Region.
>>> graphlab.aws.launch_EC2(instance_type='c3.xlarge', region='us-east-1')
"""
# Check existing connection
if glconnect.is_connected():
__LOGGER__.warning('You have GraphLab objects instantiated on the current server. If you want to \n'
'launch a new server, you must first stop the current server connection by running: \n'
'\'graphlab.connect.main.stop()\'. Be warned: you will lose these instantiated objects.')
return
assert not glconnect.is_connected()
(server_public_key, server_secret_key, client_public_key, client_secret_key) = ("", "", "", "")
if use_secure_connection:
(server_public_key, server_secret_key) = get_public_secret_key_pair()
(client_public_key, client_secret_key) = get_public_secret_key_pair()
__LOGGER__.debug('Launching server with public key: %s' % server_public_key)
try:
server = _Ec2GraphLabServer(instance_type, region, CIDR_rule, auth_token, security_group, tags,
server_public_key, server_secret_key)
try:
# Once the EC2 instance starts up, the client still need to wait for graphlab engine to start up.
# So allow for a larger than normal number of ping failures. This is especially important when launching
# in other AWS regions (since longer latency and longer download from S3 for engine binary)
server.start(num_tolerable_ping_failures=120)
except Exception as e:
__LOGGER__.error("Unable to successfully connect to GraphLab Server on EC2 instance: '%s'."
" Please check AWS Console to make sure any EC2 instances launched have"
" been terminated." % e)
server.stop()
raise e
except LicenseValidationException as e:
# catch exception and print license check hint message here instead of raise
__LOGGER__.info(e)
return
# Create the client
num_tolerable_ping_failures = 3
client = Client([], server.get_server_addr(), num_tolerable_ping_failures, public_key=client_public_key,
secret_key=client_secret_key, server_public_key=server_public_key)
auth_token = server.get_auth_token()
if auth_token:
client.add_auth_method_token(auth_token)
client.start()
glconnect._assign_server_and_client(server, client)
def launch_EC2(instance_type=DEFAULT_INSTANCE_TYPE,
region=None,
CIDR_rule=None,
auth_token=None,
security_group=None,
tags=None,
use_secure_connection=True):
"""
Launches an EC2 instance. All subsequent GraphLab operations will be
executed on that host. Prior to calling this function, AWS credentials must
be set as environment variables (see
:py:func:`~graphlab.aws.set_credentials`).
Parameters
----------
instance_type : string, optional
The EC2 instance type to launch. We support `all instance types
<http://aws.amazon.com/ec2/instance-types/#Instance_Types>`_ that are not
micros, smalls or mediums.
region : string, optional
The `AWS region
<http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region>`_
in which to launch your instance. Default is 'us-west-2'.
CIDR_rule : string or list[string], optional
The Classless Inter-Domain Routing rule(s) to use for the instance.
Useful for restricting the IP Address Range for a client. Default is no
restriction. If you specify CIDR_rule(s), you must also specify a
security group to use.
auth_token : string, optional
The Authentication Token to be used by the instance. By default a
randomly generated token is used.
security_group : string, optional
The name of the security group for the EC2 instance to use.
tags : dict, optional
A dictionary containing the name/value tag pairs to be assigned to the
instance. If you want to create only a tag name, the value for that tag
should be the empty string (i.e. ''). In addition to these specified
tags, a 'GraphLab' tag will also be assigned.
use_secure_connection : bool, optional
Determine whether the communication, between your computer and the EC2
instance, should be encrypted.
See Also
--------
terminate_EC2
Examples
--------
>>> # Launch a general purpose xlarge host in Oregon.
>>> graphlab.aws.launch_EC2()
>>> # Launch an xlarge compute optimized host in the US East Region.
>>> graphlab.aws.launch_EC2(instance_type='c3.xlarge', region='us-east-1')
"""
# Check existing connection
if glconnect.is_connected():
__LOGGER__.error(
'You have GraphLab objects instantiated on the current server. If you want to \n'
'launch a new server, you must first stop the current server connection by running: \n'
'\'graphlab.connect.main.stop()\'. Be warned: you will lose these instantiated objects.'
)
return
assert not glconnect.is_connected()
(server_public_key, server_secret_key, client_public_key,
client_secret_key) = ("", "", "", "")
if use_secure_connection:
(server_public_key, server_secret_key) = get_public_secret_key_pair()
(client_public_key, client_secret_key) = get_public_secret_key_pair()
__LOGGER__.debug('Launching server with public key: %s' %
server_public_key)
try:
server = _Ec2GraphLabServer(instance_type, region, CIDR_rule,
auth_token, security_group, tags,
server_public_key, server_secret_key)
try:
# Once the EC2 instance starts up, the client still need to wait for graphlab engine to start up.
# So allow for a larger than normal number of ping failures. This is especially important when launching
# in other AWS regions (since longer latency and longer download from S3 for engine binary)
server.start(num_tolerable_ping_failures=120)
except Exception as e:
__LOGGER__.error(
"Unable to successfully connect to GraphLab Server on EC2 instance: '%s'."
" Please check AWS Console to make sure any EC2 instances launched have"
" been terminated." % e)
server.stop()
raise e
except LicenseValidationException as e:
# catch exception and print license check hint message here instead of raise
__LOGGER__.info(e)
return
# Create the client
num_tolerable_ping_failures = 3
client = Client([],
server.get_server_addr(),
num_tolerable_ping_failures,
public_key=client_public_key,
secret_key=client_secret_key,
server_public_key=server_public_key)
auth_token = server.get_auth_token()
if auth_token:
client.add_auth_method_token(auth_token)
client.start()
glconnect._assign_server_and_client(server, client)
def _ec2_factory(instance_type, region=None, availability_zone=None,
CIDR_rule=None, security_group_name=None, tags=None,
user_data = {}, credentials = {}, ami_service_parameters = {},
num_hosts = 1, additional_port_to_open = None,
product_type = None,
subnet_id = None, security_group_id = None):
'''
This function does everything necessary to bring up EC2 host(s): create a security group (if
nessary), determine arguments to start up the EC2 instance (i.e. AMI id and user data),
actually start up the EC2 instance, wait for it, and applies AWS tags.
'''
from graphlab.connect.main import get_unity, is_connected, ENGINE_START_ERROR_MESSAGE
from graphlab.product_key import get_product_key
# Before launching EC2 instances we want to make sure the product key is valid. So make sure
# the server has started.
get_unity()
assert is_connected(), ENGINE_START_ERROR_MESSAGE
product_key = get_product_key()
# Set default values for parameters.
if(region is None):
region = _get_region_from_config()
if(region is None):
region = 'us-west-2'
else:
__LOGGER__.info('Read region from config file.')
if (region not in VALID_REGIONS):
raise Exception("%s is not a valid region." % region)
security_group, subnet_id = _setup_security_group(region = region, CIDR_rule = CIDR_rule,
security_group_name = security_group_name, credentials = credentials,
additional_port_to_open = additional_port_to_open, product_type = product_type,
subnet_id = subnet_id, security_group_id = security_group_id)
if ('GRAPHLAB_TEST_AMI_ID' in os.environ and 'GRAPHLAB_TEST_ENGINE_URL' in os.environ and 'GRAPHLAB_TEST_HASH_KEY' in os.environ):
# unit-test mode, don't involve webservice to retrieve AMI, instead use environment variables
ami_id = os.environ['GRAPHLAB_TEST_AMI_ID']
engine_url = os.environ['GRAPHLAB_TEST_ENGINE_URL']
__LOGGER__.info("UNIT mode, using AMI: '%s' and engine url: '%s' when launching EC2 instance." % (ami_id, engine_url))
json_blob = json.loads('{}')
json_blob['ami_id'] = ami_id
json_blob['engine_url'] = engine_url
json_blob['hash_key'] = os.environ['GRAPHLAB_TEST_HASH_KEY']
else:
# Get the info to start a EC2 from the GraphLab Server
json_blob_path = JSON_BLOB_PATH_FORMAT % (instance_type, graphlab.version, region, product_key)
for (param_name, param_value) in ami_service_parameters.items():
json_blob_path += "&%s=%s" % (str(param_name), str(param_value))
json_blob_url = config.graphlab_server + json_blob_path
try:
# set specific timeout for this web service request, lots of time spent in SSL negotiation
# for staging server allows a little more time
timeout_in_seconds = 10 if config.mode == 'PROD' else 60
graphlab_server_response = urlopen(json_blob_url, timeout=timeout_in_seconds)
json_blob = json.loads(graphlab_server_response.read().decode('utf-8'))
except:
raise Exception('Unable to successfully retrieve correct EC2 image to launch for this '
'version. This could be a temporary problem. Please try again in a few '
'minutes.')
__LOGGER__.debug("web service return: %s" % json_blob)
if json_blob.get('error'):
raise LicenseValidationException(json_blob.get('error'))
if 'ami_id' not in json_blob or json_blob['ami_id'] is None:
raise Exception("Unable to successfully retrieve correct EC2 image to launch. Please try "
"again later. Error received:'%s'"
% json_blob.get('message'))
ami_id = json_blob['ami_id']
# Add json_blob to user_data and set the product key and hash key
user_data.update(json_blob)
user_data['product_key'] = product_key
user_data['hash_key'] = json_blob.get('hash_key', 'NO_HASH_VALUE')
# Check for empty os_url
if user_data.get('os_url') is None or len(user_data.get('os_url')) == 0:
user_data['os_url'] = 'NO_OS_URL'
# Check for testing override of os_url param.
if ('GRAPHLAB_TEST_OS_URL' in os.environ):
user_data['os_url'] = os.environ['GRAPHLAB_TEST_OS_URL']
run_instances_args = {
'security_group_ids' : [ security_group.id ],
'user_data' : json.dumps(user_data),
'instance_type' : instance_type,
'placement' : availability_zone,
'subnet_id' : subnet_id
}
if num_hosts != 1:
run_instances_args['min_count'] = num_hosts
run_instances_args['max_count'] = num_hosts
if 'GRAPHLAB_TEST_EC2_KEYPAIR' in os.environ:
keypair = os.environ['GRAPHLAB_TEST_EC2_KEYPAIR']
__LOGGER__.info("Using keypair: '%s' when launching EC2 instance" % (keypair))
run_instances_args['key_name'] = keypair
run_instances_args['block_device_map'] = get_block_device_mapping(instance_type)
# Actually launch the EC2 instance(s) and wait for them to start running.
instances = None
try:
conn = boto.vpc.connect_to_region(region, **credentials)
response = conn.run_instances(ami_id, **run_instances_args)
instances = response.instances
if(len(response.instances) != num_hosts):
raise Exception
# Report
for i in instances:
__LOGGER__.info("Launching an %s instance in the %s availability zone, with id: %s."
" You will be responsible for the cost of this instance."
% (i.instance_type, i.placement, i.id))
# Wait for all host(s) to say they're done starting up.
while True:
try:
for i in instances:
# Rarely an instance can a reach temp state before going into pending. We check for
# 'running' right away to make unit tests work.
while not i.update() in ['pending', 'running', 'failed']:
time.sleep(1)
while i.update() == 'pending':
time.sleep(1)
if i.update() == 'failed':
raise RuntimeError("Instance %s startup failed" % i.id)
break
except EC2ResponseError as e:
# EC2 is eventual consistence so sometimes it complains that it
# cannot find the instance, in that case, we will retry
__LOGGER__.debug("Ignoring EC2ResponseError: %s" % e.message)
# Add tags to this instance(s).
if(tags is None):
tags = {}
if product_type is _ProductType.TuriDistributed:
security_group_default_name = TURI_DISTRIBUTED_NAME
tags[security_group_default_name] = ''
for i in instances:
conn.create_tags(i.id, tags)
results = []
for i in instances:
results.append(_Ec2Instance(i.ip_address, i.private_ip_address, i.id, i, region))
if num_hosts == 1:
# for backward compatibility
return [results[0], security_group, subnet_id]
return [results, security_group, subnet_id]
except Exception as e:
if instances:
_stop_instances([i.id for i in instances] , region)
raise Exception("Unable to launch EC2 instance: '%s'. Please check AWS Console to make"
" sure any EC2 instances launched have been terminated." % e)
