Example #1
0
    def test_secure_communication(self, mock_logging):
        SERVER_PUBLIC_KEY = "Ee4##T$OmI4]hzyKqZT@H&Fixt95^.72&%MK!UR."
        SERVER_SECRET_KEY = "lIn2Szq0.mpPiB<N)t6fR2/4^4&wYnFs-x72HlTz"

        # Non-error case
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        server = create_server(ipc_addr, public_key=SERVER_PUBLIC_KEY, secret_key=SERVER_SECRET_KEY)
        server.start()
        glconnect.launch(server_addr=ipc_addr, server_public_key=SERVER_PUBLIC_KEY)
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        # Tests with bogus key
        BOGUS_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        server = create_server(ipc_addr, public_key=BOGUS_KEY, secret_key=SERVER_SECRET_KEY)
        try:
            server.start()
        except:
            pass
        else: 
            self.fail("Server started with bogus key.")
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        server = create_server(ipc_addr, public_key=SERVER_PUBLIC_KEY, secret_key=BOGUS_KEY)
        try:
            server.start()
        except:
            pass
        else: 
            self.fail("Server started with bogus key.")
Example #2
0
    def test_launch_to_tcp(self):
        auth_token = 'graphlab_awesome'
        tcp_server = start_test_tcp_server(auth_token)

        #launch with remote server tcp address
        glconnect.launch(tcp_server.get_server_addr(), auth_token=auth_token)
        self.assertTrue(glconnect.is_connected())
        self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())
        tcp_server.stop()
Example #3
0
    def test_launch_to_tcp(self):
        auth_token = 'graphlab_awesome'
        tcp_server = start_test_tcp_server(auth_token)

        #launch with remote server tcp address
        glconnect.launch(tcp_server.get_server_addr(), auth_token=auth_token)
        self.assertTrue(glconnect.is_connected())
        self.assertTrue(isinstance(glconnect.get_server(),
                                   server.RemoteServer))
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())
        tcp_server.stop()
Example #4
0
    def test_launch_with_exception(self, mock_logging):
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        auth_token = 'graphlab_awesome'
        ipc_server = create_server(ipc_addr, auth_token)
        ipc_server.start()

        #default launch without stopping
        glconnect.launch(server_addr=ipc_addr)
        glconnect.launch()
        self.assertTrue(mock_logging.warning.called_once_with(SubstringMatcher(containing="existing server")))
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        # launch with bogus server path (path is not listend by server)
        with tempfile.NamedTemporaryFile() as f:
            glconnect.launch(server_addr=('ipc://' + f.name))
            self.assertTrue(mock_logging.warning.called_once_with(SubstringMatcher(containing="communication failure")))
            self.assertFalse(glconnect.is_connected())
Example #5
0
    def test_launch_to_ipc(self):
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        auth_token = 'graphlab_awesome'
        ipc_server = create_server(ipc_addr, auth_token)
        ipc_server.start()
        #default local launch
        glconnect.launch()
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        #launch with remote server ipc address
        glconnect.launch(ipc_addr, auth_token=auth_token)
        self.assertTrue(glconnect.is_connected())
        self.assertTrue(isinstance(glconnect.get_server(),
                                   server.RemoteServer))
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        #launch with remote server addr, and server_bin(ignored)
        glconnect.launch(ipc_addr,
                         os.getenv("GRAPHLAB_UNITY"),
                         auth_token=auth_token)
        self.assertTrue(glconnect.is_connected())
        self.assertTrue(isinstance(glconnect.get_server(),
                                   server.RemoteServer))
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())
        ipc_server.stop()
Example #6
0
    def test_launch_to_ipc(self):
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        auth_token = 'graphlab_awesome'
        ipc_server = create_server(ipc_addr, auth_token)
        ipc_server.start()
        #default local launch
        glconnect.launch()
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        #launch with remote server ipc address
        glconnect.launch(ipc_addr, auth_token=auth_token)
        self.assertTrue(glconnect.is_connected())
        self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        #launch with remote server addr, and server_bin(ignored)
        glconnect.launch(ipc_addr, os.getenv("GRAPHLAB_UNITY"), auth_token=auth_token)
        self.assertTrue(glconnect.is_connected())
        self.assertTrue(isinstance(glconnect.get_server(), server.RemoteServer))
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())
        ipc_server.stop()
Example #7
0
    def test_launch(self):
        #default launch
        glconnect.launch()
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        #launch with server address
        tmpname = tempfile.NamedTemporaryFile().name
        tmpaddr = 'ipc://' + tmpname
        glconnect.launch(tmpaddr)
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())
        #check that the ipc file gets deleted
        self.assertFalse(os.path.exists(tmpname))

        #launch address and binary
        graphlab_bin = os.getenv("GRAPHLAB_UNITY")
        glconnect.launch(server_addr=tmpaddr,
                         server_bin=graphlab_bin)
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())
        self.assertFalse(os.path.exists(tmpname))
Example #8
0
    def test_secure_communication(self, mock_logging):
        SERVER_PUBLIC_KEY = "Ee4##T$OmI4]hzyKqZT@H&Fixt95^.72&%MK!UR."
        SERVER_SECRET_KEY = "lIn2Szq0.mpPiB<N)t6fR2/4^4&wYnFs-x72HlTz"

        # Non-error case
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        server = create_server(ipc_addr,
                               public_key=SERVER_PUBLIC_KEY,
                               secret_key=SERVER_SECRET_KEY)
        server.start()
        glconnect.launch(server_addr=ipc_addr,
                         server_public_key=SERVER_PUBLIC_KEY)
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        # Tests with bogus key
        BOGUS_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        server = create_server(ipc_addr,
                               public_key=BOGUS_KEY,
                               secret_key=SERVER_SECRET_KEY)
        try:
            server.start()
        except:
            pass
        else:
            self.fail("Server started with bogus key.")
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        server = create_server(ipc_addr,
                               public_key=SERVER_PUBLIC_KEY,
                               secret_key=BOGUS_KEY)
        try:
            server.start()
        except:
            pass
        else:
            self.fail("Server started with bogus key.")
Example #9
0
    def test_launch_with_exception(self, mock_logging):
        ipc_addr = 'ipc://' + tempfile.NamedTemporaryFile().name
        auth_token = 'graphlab_awesome'
        ipc_server = create_server(ipc_addr, auth_token)
        ipc_server.start()

        #default launch without stopping
        glconnect.launch(server_addr=ipc_addr)
        glconnect.launch()
        self.assertTrue(
            mock_logging.warning.called_once_with(
                SubstringMatcher(containing="existing server")))
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        # launch with bogus server path (path is not listend by server)
        with tempfile.NamedTemporaryFile() as f:
            glconnect.launch(server_addr=('ipc://' + f.name))
            self.assertTrue(
                mock_logging.warning.called_once_with(
                    SubstringMatcher(containing="communication failure")))
            self.assertFalse(glconnect.is_connected())
Example #10
0
    def test_launch_with_exception(self, mock_logging):
        # Assert warning logged when launching without stopping
        glconnect.launch()
        glconnect.launch()
        self.assertTrue(
            mock_logging.warning.called_once_with(
                SubstringMatcher(containing="existing server")))
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        # launch with bogus server binary (path is not executable)
        with tempfile.NamedTemporaryFile() as f:
            random_server_bin = f.name
            glconnect.launch(server_bin=random_server_bin)
            self.assertTrue(
                mock_logging.error.called_once_with(
                    SubstringMatcher(containing="Invalid server binary")))
            self.assertFalse(glconnect.is_connected())

        #launch with server address without permission
        tmpaddr = 'ipc:///root/bad_server'
        glconnect.launch(tmpaddr)
        self.assertTrue(
            mock_logging.warning.called_once_with(
                SubstringMatcher(containing="communication error")))
        self.assertFalse(glconnect.is_connected())
        glconnect.stop()

        # launch with binary that does not exist
        tmpname = tempfile.NamedTemporaryFile().name
        glconnect.launch(server_bin=tmpname)
        self.assertTrue(
            mock_logging.error.called_once_with(
                SubstringMatcher(containing="Invalid server binary")))
        self.assertFalse(glconnect.is_connected())

        # launch with bogus server binary (path is a faked executable)
        with tempfile.NamedTemporaryFile() as f:
            os.chmod(f.name, stat.S_IXUSR)
            random_server_bin = f.name
            glconnect.launch(server_bin=random_server_bin)
            self.assertTrue(
                mock_logging.error.called_once_with(
                    SubstringMatcher(containing="Invalid server binary")))
            self.assertFalse(glconnect.is_connected())
Example #11
0
    def test_launch(self):
        #default launch
        glconnect.launch()
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        #launch with server address
        tmpname = tempfile.NamedTemporaryFile().name
        tmpaddr = 'ipc://' + tmpname
        glconnect.launch(tmpaddr)
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())
        #check that the ipc file gets deleted
        self.assertFalse(os.path.exists(tmpname))

        #launch address and binary
        graphlab_bin = os.getenv("GRAPHLAB_UNITY")
        glconnect.launch(server_addr=tmpaddr, server_bin=graphlab_bin)
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())
        self.assertFalse(os.path.exists(tmpname))
Example #12
0
    def test_launch_with_exception(self, mock_logging):
        # Assert warning logged when launching without stopping
        glconnect.launch()
        glconnect.launch()
        self.assertTrue(mock_logging.warning.called_once_with(SubstringMatcher(containing="existing server")))
        self.assertTrue(glconnect.is_connected())
        glconnect.stop()
        self.assertFalse(glconnect.is_connected())

        # launch with bogus server binary (path is not executable)
        with tempfile.NamedTemporaryFile() as f:
            random_server_bin = f.name
            glconnect.launch(server_bin=random_server_bin)
            self.assertTrue(mock_logging.error.called_once_with(SubstringMatcher(containing="Invalid server binary")))
            self.assertFalse(glconnect.is_connected())

        #launch with server address without permission
        tmpaddr = 'ipc:///root/bad_server'
        glconnect.launch(tmpaddr)
        self.assertTrue(mock_logging.warning.called_once_with(SubstringMatcher(containing="communication error")))
        self.assertFalse(glconnect.is_connected())
        glconnect.stop()

        # launch with binary that does not exist
        tmpname = tempfile.NamedTemporaryFile().name
        glconnect.launch(server_bin=tmpname)
        self.assertTrue(mock_logging.error.called_once_with(SubstringMatcher(containing="Invalid server binary")))
        self.assertFalse(glconnect.is_connected())

        # launch with bogus server binary (path is a faked executable)
        with tempfile.NamedTemporaryFile() as f:
            os.chmod(f.name, stat.S_IXUSR)
            random_server_bin = f.name
            glconnect.launch(server_bin=random_server_bin)
            self.assertTrue(mock_logging.error.called_once_with(SubstringMatcher(containing="Invalid server binary")))
            self.assertFalse(glconnect.is_connected())
def launch_EC2(instance_type=DEFAULT_INSTANCE_TYPE, region=None, CIDR_rule=None, auth_token=None,
               security_group=None, tags=None, use_secure_connection=True):
    """
    Launches an EC2 instance. All subsequent GraphLab operations will be
    executed on that host. Prior to calling this function, AWS credentials must
    be set as environment variables (see
    :py:func:`~graphlab.aws.set_credentials`).

    Parameters
    ----------
    instance_type : string, optional
        The EC2 instance type to launch. We support `all instance types
        <http://aws.amazon.com/ec2/instance-types/#Instance_Types>`_ that are not
        micros, smalls or mediums.

    region : string, optional
        The `AWS region
        <http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region>`_
        in which to launch your instance. Default is 'us-west-2'.

    CIDR_rule : string or list[string], optional
        The Classless Inter-Domain Routing rule(s) to use for the instance.
        Useful for restricting the IP Address Range for a client. Default is no
        restriction. If you specify CIDR_rule(s), you must also specify a
        security group to use.

    auth_token : string, optional
        The Authentication Token to be used by the instance. By default a
        randomly generated token is used.

    security_group : string, optional
        The name of the security group for the EC2 instance to use.

    tags : dict, optional
        A dictionary containing the name/value tag pairs to be assigned to the
        instance. If you want to create only a tag name, the value for that tag
        should be the empty string (i.e. ''). In addition to these specified
        tags, a 'GraphLab' tag will also be assigned.

    use_secure_connection : bool, optional
       Determine whether the communication, between your computer and the EC2
       instance, should be encrypted.

    See Also
    --------
    terminate_EC2

    Examples
    --------
    >>> # Launch a general purpose xlarge host in Oregon.
    >>> graphlab.aws.launch_EC2()

    >>> # Launch an xlarge compute optimized host in the US East Region.
    >>> graphlab.aws.launch_EC2(instance_type='c3.xlarge', region='us-east-1')
    """

    # Check existing connection
    if glconnect.is_connected():
        __LOGGER__.warning('You have GraphLab objects instantiated on the current server. If you want to \n'
                           'launch a new server, you must first stop the current server connection by running: \n'
                           '\'graphlab.connect.main.stop()\'. Be warned: you will lose these instantiated objects.')
        return

    assert not glconnect.is_connected()

    (server_public_key, server_secret_key, client_public_key, client_secret_key) = ("", "", "", "")
    if use_secure_connection:
        (server_public_key, server_secret_key) = get_public_secret_key_pair()
        (client_public_key, client_secret_key) = get_public_secret_key_pair()
        __LOGGER__.debug('Launching server with public key: %s' % server_public_key)

    try: 
        server = _Ec2GraphLabServer(instance_type, region, CIDR_rule, auth_token, security_group, tags,
                                server_public_key, server_secret_key)

        try:
            # Once the EC2 instance starts up, the client still need to wait for graphlab engine to start up.
            # So allow for a larger than normal number of ping failures. This is especially important when launching
            # in other AWS regions (since longer latency and longer download from S3 for engine binary)
            server.start(num_tolerable_ping_failures=120)
    
        except Exception as e:
            __LOGGER__.error("Unable to successfully connect to GraphLab Server on EC2 instance: '%s'."
                         " Please check AWS Console to make sure any EC2 instances launched have"
                         " been terminated." % e)
            server.stop()
            raise e
    
    except LicenseValidationException as e:
        # catch exception and print license check hint message here instead of raise
        __LOGGER__.info(e)
        return 
    # Create the client
    num_tolerable_ping_failures = 3
    client = Client([], server.get_server_addr(), num_tolerable_ping_failures, public_key=client_public_key,
                secret_key=client_secret_key, server_public_key=server_public_key)
    auth_token = server.get_auth_token()
    if auth_token:
        client.add_auth_method_token(auth_token)
    client.start()
    glconnect._assign_server_and_client(server, client)
Example #14
0
def launch_EC2(instance_type=DEFAULT_INSTANCE_TYPE,
               region=None,
               CIDR_rule=None,
               auth_token=None,
               security_group=None,
               tags=None,
               use_secure_connection=True):
    """
    Launches an EC2 instance. All subsequent GraphLab operations will be
    executed on that host. Prior to calling this function, AWS credentials must
    be set as environment variables (see
    :py:func:`~graphlab.aws.set_credentials`).

    Parameters
    ----------
    instance_type : string, optional
        The EC2 instance type to launch. We support `all instance types
        <http://aws.amazon.com/ec2/instance-types/#Instance_Types>`_ that are not
        micros, smalls or mediums.

    region : string, optional
        The `AWS region
        <http://docs.aws.amazon.com/general/latest/gr/rande.html#ec2_region>`_
        in which to launch your instance. Default is 'us-west-2'.

    CIDR_rule : string or list[string], optional
        The Classless Inter-Domain Routing rule(s) to use for the instance.
        Useful for restricting the IP Address Range for a client. Default is no
        restriction. If you specify CIDR_rule(s), you must also specify a
        security group to use.

    auth_token : string, optional
        The Authentication Token to be used by the instance. By default a
        randomly generated token is used.

    security_group : string, optional
        The name of the security group for the EC2 instance to use.

    tags : dict, optional
        A dictionary containing the name/value tag pairs to be assigned to the
        instance. If you want to create only a tag name, the value for that tag
        should be the empty string (i.e. ''). In addition to these specified
        tags, a 'GraphLab' tag will also be assigned.

    use_secure_connection : bool, optional
       Determine whether the communication, between your computer and the EC2
       instance, should be encrypted.

    See Also
    --------
    terminate_EC2

    Examples
    --------
    >>> # Launch a general purpose xlarge host in Oregon.
    >>> graphlab.aws.launch_EC2()

    >>> # Launch an xlarge compute optimized host in the US East Region.
    >>> graphlab.aws.launch_EC2(instance_type='c3.xlarge', region='us-east-1')
    """

    # Check existing connection
    if glconnect.is_connected():
        __LOGGER__.error(
            'You have GraphLab objects instantiated on the current server. If you want to \n'
            'launch a new server, you must first stop the current server connection by running: \n'
            '\'graphlab.connect.main.stop()\'. Be warned: you will lose these instantiated objects.'
        )
        return

    assert not glconnect.is_connected()

    (server_public_key, server_secret_key, client_public_key,
     client_secret_key) = ("", "", "", "")
    if use_secure_connection:
        (server_public_key, server_secret_key) = get_public_secret_key_pair()
        (client_public_key, client_secret_key) = get_public_secret_key_pair()
        __LOGGER__.debug('Launching server with public key: %s' %
                         server_public_key)

    try:
        server = _Ec2GraphLabServer(instance_type, region, CIDR_rule,
                                    auth_token, security_group, tags,
                                    server_public_key, server_secret_key)

        try:
            # Once the EC2 instance starts up, the client still need to wait for graphlab engine to start up.
            # So allow for a larger than normal number of ping failures. This is especially important when launching
            # in other AWS regions (since longer latency and longer download from S3 for engine binary)
            server.start(num_tolerable_ping_failures=120)

        except Exception as e:
            __LOGGER__.error(
                "Unable to successfully connect to GraphLab Server on EC2 instance: '%s'."
                " Please check AWS Console to make sure any EC2 instances launched have"
                " been terminated." % e)
            server.stop()
            raise e

    except LicenseValidationException as e:
        # catch exception and print license check hint message here instead of raise
        __LOGGER__.info(e)
        return
    # Create the client
    num_tolerable_ping_failures = 3
    client = Client([],
                    server.get_server_addr(),
                    num_tolerable_ping_failures,
                    public_key=client_public_key,
                    secret_key=client_secret_key,
                    server_public_key=server_public_key)
    auth_token = server.get_auth_token()
    if auth_token:
        client.add_auth_method_token(auth_token)
    client.start()
    glconnect._assign_server_and_client(server, client)
Example #15
0
def _ec2_factory(instance_type, region=None, availability_zone=None,
                 CIDR_rule=None, security_group_name=None, tags=None,
                 user_data = {}, credentials = {}, ami_service_parameters = {},
                 num_hosts = 1, additional_port_to_open = None,
                 product_type = None,
                 subnet_id = None, security_group_id = None):

    '''
    This function does everything necessary to bring up EC2 host(s): create a security group (if
    nessary), determine arguments to start up the EC2 instance (i.e. AMI id and user data),
    actually start up the EC2 instance, wait for it, and applies AWS tags.
    '''
    from graphlab.connect.main import get_unity, is_connected, ENGINE_START_ERROR_MESSAGE
    from graphlab.product_key import get_product_key

    # Before launching EC2 instances we want to make sure the product key is valid. So make sure
    # the server has started.
    get_unity()
    assert is_connected(), ENGINE_START_ERROR_MESSAGE
    product_key = get_product_key()

    # Set default values for parameters.
    if(region is None):
        region = _get_region_from_config()
        if(region is None):
            region = 'us-west-2'
        else:
            __LOGGER__.info('Read region from config file.')

    if (region not in VALID_REGIONS):
        raise Exception("%s is not a valid region." % region)

    security_group, subnet_id = _setup_security_group(region = region, CIDR_rule = CIDR_rule,
        security_group_name = security_group_name, credentials = credentials,
        additional_port_to_open = additional_port_to_open, product_type = product_type,
        subnet_id = subnet_id, security_group_id = security_group_id)

    if ('GRAPHLAB_TEST_AMI_ID' in os.environ and 'GRAPHLAB_TEST_ENGINE_URL' in os.environ and 'GRAPHLAB_TEST_HASH_KEY' in os.environ):
        # unit-test mode, don't involve webservice to retrieve AMI, instead use environment variables
        ami_id = os.environ['GRAPHLAB_TEST_AMI_ID']
        engine_url = os.environ['GRAPHLAB_TEST_ENGINE_URL']
        __LOGGER__.info("UNIT mode, using AMI: '%s' and engine url: '%s' when launching EC2 instance." % (ami_id, engine_url))
        json_blob = json.loads('{}')
        json_blob['ami_id'] = ami_id
        json_blob['engine_url'] = engine_url
        json_blob['hash_key'] = os.environ['GRAPHLAB_TEST_HASH_KEY']
    else:
        # Get the info to start a EC2 from the GraphLab Server
        json_blob_path = JSON_BLOB_PATH_FORMAT % (instance_type, graphlab.version, region, product_key)
        for (param_name, param_value) in ami_service_parameters.items():
            json_blob_path += "&%s=%s" % (str(param_name), str(param_value))
        json_blob_url = config.graphlab_server + json_blob_path

        try:
            # set specific timeout for this web service request, lots of time spent in SSL negotiation
            # for staging server allows a little more time
            timeout_in_seconds = 10 if config.mode == 'PROD' else 60
            graphlab_server_response = urlopen(json_blob_url, timeout=timeout_in_seconds)
            json_blob = json.loads(graphlab_server_response.read().decode('utf-8'))
        except:
            raise Exception('Unable to successfully retrieve correct EC2 image to launch for this '
                    'version. This could be a temporary problem. Please try again in a few '
                    'minutes.')
        __LOGGER__.debug("web service return: %s" % json_blob)

        if json_blob.get('error'):
            raise LicenseValidationException(json_blob.get('error'))

    if 'ami_id' not in json_blob or json_blob['ami_id'] is None:
        raise Exception("Unable to successfully retrieve correct EC2 image to launch. Please try "
                "again later. Error received:'%s'"
                % json_blob.get('message'))
    ami_id = json_blob['ami_id']

    # Add json_blob to user_data and set the product key and hash key
    user_data.update(json_blob)
    user_data['product_key'] = product_key

    user_data['hash_key'] = json_blob.get('hash_key', 'NO_HASH_VALUE')

    # Check for empty os_url
    if user_data.get('os_url') is None or len(user_data.get('os_url')) == 0:
        user_data['os_url'] = 'NO_OS_URL'

    # Check for testing override of os_url param.
    if ('GRAPHLAB_TEST_OS_URL' in os.environ):
        user_data['os_url'] = os.environ['GRAPHLAB_TEST_OS_URL']

    run_instances_args =  {
            'security_group_ids' : [ security_group.id ],
            'user_data' : json.dumps(user_data),
            'instance_type' : instance_type,
            'placement' : availability_zone,
            'subnet_id' : subnet_id
    }

    if num_hosts != 1:
        run_instances_args['min_count'] = num_hosts
        run_instances_args['max_count'] = num_hosts

    if 'GRAPHLAB_TEST_EC2_KEYPAIR' in os.environ:
        keypair = os.environ['GRAPHLAB_TEST_EC2_KEYPAIR']
        __LOGGER__.info("Using keypair: '%s' when launching EC2 instance" % (keypair))
        run_instances_args['key_name'] = keypair

    run_instances_args['block_device_map'] = get_block_device_mapping(instance_type)

    # Actually launch the EC2 instance(s) and wait for them to start running.
    instances = None
    try:
        conn = boto.vpc.connect_to_region(region, **credentials)
        response = conn.run_instances(ami_id, **run_instances_args)
        instances = response.instances
        if(len(response.instances) != num_hosts):
            raise Exception

        # Report
        for i in instances:
            __LOGGER__.info("Launching an %s instance in the %s availability zone, with id: %s."
                    " You will be responsible for the cost of this instance."
                    % (i.instance_type, i.placement, i.id))

        # Wait for all host(s) to say they're done starting up.
        while True:
            try:
                for i in instances:
                    # Rarely an instance can a reach temp state before going into pending. We check for
                    # 'running' right away to make unit tests work.
                    while not i.update() in ['pending', 'running', 'failed']:
                        time.sleep(1)
                    while i.update() == 'pending':
                        time.sleep(1)
                    if i.update() == 'failed':
                        raise RuntimeError("Instance %s startup failed" % i.id)
                break
            except EC2ResponseError as e:
                # EC2 is eventual consistence so sometimes it complains that it
                # cannot find the instance, in that case, we will retry
                __LOGGER__.debug("Ignoring EC2ResponseError: %s" % e.message)

        # Add tags to this instance(s).
        if(tags is None):
            tags = {}
        if product_type is _ProductType.TuriDistributed:
            security_group_default_name = TURI_DISTRIBUTED_NAME
        tags[security_group_default_name] = ''
        for i in instances:
            conn.create_tags(i.id, tags)

        results = []
        for i in instances:
            results.append(_Ec2Instance(i.ip_address, i.private_ip_address, i.id, i, region))

        if num_hosts == 1:
            # for backward compatibility
            return [results[0], security_group, subnet_id]
        return [results, security_group, subnet_id]

    except Exception as e:
        if instances:
            _stop_instances([i.id for i in instances] , region)
        raise Exception("Unable to launch EC2 instance: '%s'. Please check AWS Console to make"
                " sure any EC2 instances launched have been terminated." % e)