示例#1
0
def test_start(load_config_dict_mock, pre_add_host_mock,
               fire_provisioned_even_mock, get_instance_size_mapping_mock,
               dbm, valid_ami):
    """
    Test ResourceAdapter.start() workflow
    """

    get_instance_size_mapping_mock.return_value = 8

    load_config_dict_mock.return_value = {
        'awsaccesskey': 'the_key',
        'awssecretkey': 'the_secret',
        'keypair': 'the_keypair',
        'ami': valid_ami,
        'use_instance_hostname': 'true',
        'instancetype': 'the_instancetype'
    }

    with dbm.session() as session:
        adapter = Aws(addHostSession='123EXAMPLE')

        # override default sleep time
        adapter.LAUNCH_INITIAL_SLEEP_TIME = 0.0

        hardwareprofile = HardwareProfilesDbHandler().getHardwareProfile(
            session, 'aws2'
        )

        softwareprofile = SoftwareProfilesDbHandler().getSoftwareProfile(
            session, 'compute'
        )

        addNodesRequest = {
            'count': 2,
            'hardwareProfile': hardwareprofile.name,
            'softwareProfile': softwareprofile.name,
        }

        nodes = adapter.start(
            addNodesRequest, session, hardwareprofile,
            dbSoftwareProfile=softwareprofile
        )

        assert nodes and isinstance(nodes, list) and \
            isinstance(nodes[0], Node)

        assert nodes[0].instance.instance

        if len(nodes) > 1:
            assert nodes[1].instance.instance

    pre_add_host_mock.assert_called()

    fire_provisioned_even_mock.assert_called()
示例#2
0
def test_deleteNode(load_config_dict_mock, dbm):
    load_config_dict_mock.return_value = {
        'awsaccesskey': 'the_key',
        'awssecretkey': 'the_secret'
    }

    with mock_ec2_deprecated():
        with dbm.session() as session:
            adapter = Aws()

            node = session.query(Node).filter(
                Node.name == 'ip-10-10-10-1.ec2.internal').one()

            adapter.deleteNode([node])
示例#3
0
def test_installer_public_ipaddress():
    with mock.patch(
            'tortuga.resourceAdapter.aws.Aws.installer_public_ipaddress',
            new_callable=mock.PropertyMock) \
            as installer_public_ipaddress_mock:
        installer_public_ipaddress_mock.return_value = '1.2.3.4'

        assert Aws()._get_installer_ip() == '1.2.3.4'
示例#4
0
def test_instantiation_with_addHostSession():
    """
    Simple test to ensure resource adapter can be instantiated
    """

    adapter = Aws(addHostSession=123)

    assert adapter.addHostSession == 123
示例#5
0
def test_boto3_conn_setup(proxy_host, proxy_port, proxy_user, proxy_pass):
    """Test setup of boto3 connection"""

    # Construct configDict
    configDict = {
        'awsaccesskey': 'the_key',
        'awssecretkey': 'the_secret',
        'region': 'us-east-1',
    }
    if proxy_host:
        configDict['proxy_host'] = proxy_host
        configDict['proxy_port'] = proxy_port
        if proxy_user:
            configDict['proxy_user'] = proxy_user
            configDict['proxy_pass'] = proxy_pass

    # Set up adapter and process configDict
    adapter = Aws()
    adapter.process_config(configDict)

    # Get boto3 connection
    session_cls = 'tortuga.resourceAdapter.aws.aws.boto3.Session'
    config_cls = 'tortuga.resourceAdapter.aws.aws.Config'
    with mock.patch(session_cls) as boto3_session_mock, \
         mock.patch(config_cls) as botocore_config_mock:
        conn3 = adapter.getEC2Connection3(configDict)

    # Test session call args/kwargs
    session_call_kwargs = boto3_session_mock.call_args[1]
    assert len(session_call_kwargs) == 3
    assert session_call_kwargs['aws_access_key_id'] == \
        configDict['awsaccesskey']
    assert session_call_kwargs['aws_secret_access_key'] == \
        configDict['awssecretkey']
    assert session_call_kwargs['region_name'] == configDict['region']

    # Test config call args/kwargs if proxy used
    if proxy_host is not None:
        config_call_kwargs = botocore_config_mock.call_args[1]
        proxy_dict = config_call_kwargs['proxies']
        assert len(proxy_dict) == 1
        assert 'http' in proxy_dict
        proxy_url = f'{proxy_host}:{proxy_port}'
        if proxy_user is not None:
            proxy_url = f'{proxy_user}:{proxy_pass}@{proxy_url}'
        assert proxy_dict['http'] == proxy_url
示例#6
0
def test_boto3_validate_launch_args():
    """Test validation of launch args for boto3 connection"""
    configDict = {
        'awsaccesskey': 'the_key',
        'awssecretkey': 'the_secret',
        'region': 'us-east-1',
        'placementgroup': 'testgroup',
    }

    # Set up adapter and process configDict
    adapter = Aws()
    adapter.process_config(configDict)

    # Get boto3 connection
    conn3 = adapter.getEC2Connection3(configDict)

    # We have to mock this directly since moto doesn't implement
    # a mock of create_placement_group at present (Oct. 2019)
    with mock.patch.object(conn3, 'create_placement_group') as mock_cpg:
        adapter._validate_ec2_launch_args(conn3, configDict)

    # Check results
    call_kwargs = mock_cpg.call_args[1]
    assert len(call_kwargs) == 2
    assert call_kwargs['GroupName'] == configDict['placementgroup']
    assert call_kwargs['Strategy'] == 'cluster'
示例#7
0
def test_installer_public_ipaddress_with_hardwareprofile():
    class DummyNic:
        def __init__(self, ip):
            self.ip = ip

    class DummyHardwareProfile:
        def __init__(self):
            self.nics = [
                DummyNic('1.2.3.4'),
                DummyNic('2.3.4.5'),
            ]

    ip = Aws()._get_installer_ip(hardwareprofile=DummyHardwareProfile())

    assert ip == '1.2.3.4'
示例#8
0
def test_scale_set_tags(name_tag):
    """Test scale set tag generation"""
    configDict = {
        'installer_ip': '123.456.7.89',
        'use_instance_hostname': True,
        'tags': {
            'other tag': 'value'
        }
    }
    if name_tag:
        configDict['tags']['Name'] = name_tag
    group_name = 'fake_group'
    hardware_profile = 'hwp'
    software_profile = 'swp'
    adapter = Aws()
    adapter.process_config(configDict)

    # Get tags
    tags = adapter._get_scale_set_tags(group_name, configDict,
                                       hardware_profile, software_profile)

    # Check basic tag properties
    assert len(tags) == 6
    for tag in tags:
        assert tag.resource_id == group_name
        assert tag.resource_type == 'auto-scaling-group'
        assert tag.propagate_at_launch

    # Convert to dict for more specific testing
    tag_dict = {tag.key: tag for tag in tags}
    assert tag_dict['tortuga-hardwareprofile'].value == hardware_profile
    assert tag_dict['tortuga-softwareprofile'].value == software_profile
    assert tag_dict['tortuga-installer_hostname'].value == \
        adapter._sanitze_tag_value(adapter.installer_public_hostname)
    assert tag_dict['tortuga-installer_ipaddress'].value == \
        adapter._sanitze_tag_value(configDict['installer_ip'])
    assert tag_dict['other tag'].value == configDict['tags']['other tag']

    # Check name
    expected_name = name_tag if name_tag else 'Tortuga compute node'
    assert tag_dict['Name'].value == expected_name
示例#9
0
def test_instantiation():
    """
    Simple test to ensure resource adapter can be instantiated
    """

    assert Aws()
示例#10
0
def test_start_update_node(load_config_dict_mock, pre_add_host_mock,
                           fire_provisioned_event_mock, dbm, valid_ami):
    configDict = {
        'awsaccesskey': 'the_key',
        'awssecretkey': 'the_secret',
        'ami': valid_ami,
        'use_instance_hostname': 'true',
    }

    load_config_dict_mock.return_value = configDict

    with dbm.session() as session:
        addHostSession = '123EXAMPLE'

        adapter = Aws(addHostSession=addHostSession)

        # override default sleep time
        adapter.LAUNCH_INITIAL_SLEEP_TIME = 0.0

        count = 3

        hardwareprofile = HardwareProfilesDbHandler().getHardwareProfile(
            session, 'aws2'
        )

        softwareprofile = SoftwareProfilesDbHandler().getSoftwareProfile(
            session, 'compute'
        )

        # create instances to be associated with nodes
        conn = boto.connect_ec2(configDict['awsaccesskey'],
                                configDict['awssecretkey'])

        conn.run_instances(
            configDict['ami'],
            min_count=count,
            max_count=count
        )

        # get newly created instances
        instances = conn.get_only_instances()

        # intialize 'addNodesRequest'
        addNodesRequest = {
            'nodeDetails': [],
        }

        for instance in instances:
            addNodesRequest['nodeDetails'].append({
                'name': instance.private_dns_name,
                'metadata': {
                    'ec2_instance_id': instance.id,
                    'ec2_ipaddress': instance.private_ip_address,
                }
            })

        # call Aws.start() with instance metadata
        nodes = adapter.start(
            addNodesRequest, session, hardwareprofile,
            dbSoftwareProfile=softwareprofile
        )

        assert nodes and len(nodes) == count

        assert isinstance(nodes[0], Node)

        assert nodes[0].softwareprofile.name == softwareprofile.name

        assert nodes[0].hardwareprofile.name == hardwareprofile.name

        assert nodes[0].addHostSession == addHostSession

        fire_provisioned_event_mock.assert_called()

        pre_add_host_mock.assert_called()
示例#11
0
def test_launch_EC2(valid_ami):
    """Test full EC2 launch process"""
    # Set up configDict
    configDict = {
        'keypair': 'keypair_name',
        'instancetype': 't2.large',
        'region': 'us-east-1',
        'awsaccesskey': 'the_key',
        'awssecretkey': 'the_secret',
        'zone': 'fake_zone',
        'use_instance_hostname': False,
        'block_device_map': '/dev/sda1=:30:true:io1:500:encrypted',
        'ami': valid_ami,
        'aki': 'fake_kernel_id',
        'ari': 'fake_ramdisk_id',
        'ebs_optimized': True,
        'monitoring_enabled': False,
        'iam_instance_profile_name': 'fake_profile_name',
        'subnet_id': None,
        'securitygroup': ['sg-1234'],
        'associate_public_ip_address': True,
        'tags': {'key1': 'value1', 'key2': 'value2'},
    }

    # Set up adapter and process configDict
    adapter = Aws()
    adapter.process_config(configDict)

    # Set up a mock node
    node = mock.Mock(spec=Node)
    node.hardwareprofile.name = 'hwp_name'
    node.softwareprofile.name = 'swp_name'
    node.name = 'node_name'

    # Run __launchEC2
    with mock_ec2():
        # Get boto3 connection
        conn3 = adapter.getEC2Connection3(configDict)

        # Create mock vpc and subnet
        vpc = conn3.create_vpc(CidrBlock='10.0.0.0/16')
        subnet = conn3.create_subnet(CidrBlock='10.0.0.0/18', VpcId=vpc.id)
        configDict['subnet_id'] = subnet.id

        # Launch instances
        result = adapter._Aws__launchEC2(conn3, configDict, count=1, node=node)

    # Check results
    assert len(result) == 1
    instance = result[0]
    assert instance.image.id == configDict['ami']
    assert instance.key_pair.key_name == configDict['keypair']
    assert instance.instance_type == configDict['instancetype']
    assert instance.vpc_id == vpc.id
    assert instance.subnet_id == subnet.id
    assert len(instance.network_interfaces) == 1
    assert instance.network_interfaces[0].groups[0]['GroupName'] == \
        configDict['securitygroup'][0]
    # NOTE: moto does not give the correct result for these two. I have tested
    # manually on AWS and confirmed that they work as expected.
    #assert instance.ebs_optimized == configDict['ebs_optimized']
    #assert instance.monitoring['state'] == 'disabled'

    # Check tags
    instance_tags = {d['Key']: d['Value'] for d in instance.tags}
    assert instance_tags['Name'] == node.name
    assert instance_tags['key1'] == 'value1'
    assert instance_tags['key2'] == 'value2'
    assert instance_tags['tortuga-softwareprofile'] == \
        node.softwareprofile.name
    assert instance_tags['tortuga-hardwareprofile'] == \
        node.hardwareprofile.name
    assert instance_tags['tortuga-installer_hostname'] == \
        adapter.installer_public_hostname
    # Have to sanitize this one
    assert instance_tags['tortuga-installer_ipaddress'] == \
        adapter._sanitze_tag_value(adapter.installer_public_ipaddress)
示例#12
0
def test_get_common_launch_args3(subnet_id, valid_ami):
    """Test construction of launch args dict for run_instances with boto3"""
    # Set up configDict
    configDict = {
        'keypair': 'keypair_name',
        'instancetype': 't2.large',
        'region': 'us-east-1',
        'zone': 'fake_zone',
        'installer_ip': '127.0.0.1',
        'use_instance_hostname': False,
        'placementgroup': 'fake_placementgroup',
        'cloud_init': None,
        'ami': valid_ami,
        'aki': 'fake_kernel_id',
        'ari': 'fake_ramdisk_id',
        'ebs_optimized': True,
        'monitoring_enabled': False,
        'iam_instance_profile_name': 'fake_profile_name',
        'subnet_id': subnet_id,
        'securitygroup': ['sg-1234'],
        'associate_public_ip_address': True,
        'use_tags': True,
        'tags': {'key1': 'value1', 'key2': 'value2'},
    }

    # Set up adapter and process configDict
    adapter = Aws()

    # Get boto3 connection
    conn3 = adapter.getEC2Connection3(configDict)

    # Set up a mock node
    node = mock.Mock(spec=Node)
    node.hardwareprofile.name = 'hwp_name'
    node.softwareprofile.name = 'swp_name'
    node.name = 'node_name'

    # Execute function
    with mock_ec2():
        run_args = adapter._Aws__get_common_launch_args3(conn3, configDict,
                                                         node=node)

    # Check results
    assert run_args['EbsOptimized'] == configDict['ebs_optimized']
    assert run_args['IamInstanceProfile']['Name'] == \
        configDict['iam_instance_profile_name']
    assert run_args['InstanceType'] == configDict['instancetype']
    assert run_args['KernelId'] == configDict['aki']
    assert run_args['KeyName'] == configDict['keypair']
    assert run_args['Monitoring']['Enabled'] == \
        configDict['monitoring_enabled']
    assert run_args['Placement']['AvailabilityZone'] == configDict['zone']
    assert run_args['Placement']['GroupName'] == configDict['placementgroup']
    assert run_args['RamdiskId'] == configDict['ari']

    # Depends on subnet id
    if subnet_id is not None:
        assert 'NetworkInterfaces' in run_args
        assert len(run_args['NetworkInterfaces']) == 1
        ni = run_args['NetworkInterfaces'][0]
        assert ni['AssociatePublicIpAddress'] == \
            configDict['associate_public_ip_address']
        assert ni['Groups'] == configDict['securitygroup']
        assert ni['SubnetId'] == configDict['subnet_id']
    else:
        assert 'SecurityGroupIds' in run_args
        assert run_args['SecurityGroupIds'] == configDict['securitygroup']

    # Don't need to check instance tags here, since it's done in another test
    tag_specs = \
        {d['ResourceType']: d['Tags'] for d in run_args['TagSpecifications']}
    volume_tag_specs = {d['Key']: d['Value'] for d in tag_specs['volume']}

    # 5 extra tags for tortuga:* tags and Name
    assert len(volume_tag_specs) == len(configDict['tags']) + 5
    for k in configDict['tags']:
        assert volume_tag_specs[k] == configDict['tags'][k]
    assert volume_tag_specs['tortuga-softwareprofile'] == \
        node.softwareprofile.name
    assert volume_tag_specs['tortuga-hardwareprofile'] == \
        node.hardwareprofile.name
    assert volume_tag_specs['tortuga-installer_hostname'] == \
        adapter._sanitze_tag_value(adapter.installer_public_hostname)
    assert volume_tag_specs['tortuga-installer_ipaddress'] == \
        adapter._sanitze_tag_value(configDict['installer_ip'])
    assert volume_tag_specs['Name'] == node.name
示例#13
0
def test_get_tags_for_instance_creation(use_instance_hostname, name_tag,
                                        use_node, use_addNodesRequest):
    """Test tags generated for instance creation"""
    # Set up configDict
    configDict = {
        'region': 'us-east-1',
        'installer_ip': '127.0.0.1',
        'tags': {'key1': 'value1', 'key2': 'value2'},
        'use_instance_hostname': use_instance_hostname,
    }
    if name_tag:
        configDict['tags']['Name'] = name_tag

    # Set up node mock, if needed
    node = None
    if use_node:
        node = mock.Mock(spec=Node)
        node.name = 'node_name'
        node.hardwareprofile.name = 'node_hwp'
        node.softwareprofile.name = 'node_swp'

    # Set up addNodesRequest, if needed
    addNodesRequest = {}
    if use_addNodesRequest:
        addNodesRequest = {'softwareProfile': 'swp', 'hardwareProfile': 'hwp'}

    # Set up adapter - no need to process configDict since the tags
    # are already in processed form
    adapter = Aws()

    # Get boto3 connection
    conn3 = adapter.getEC2Connection3(configDict)

    # Get tags
    tags = adapter._Aws__get_tags_for_instance_creation(
        configDict,
        node=node,
        addNodesRequest=addNodesRequest
    )

    # Manually generate expected contents
    expected_hwp_name = node.hardwareprofile.name if use_node \
        else addNodesRequest['hardwareProfile']
    expected_swp_name = node.softwareprofile.name if use_node \
        else addNodesRequest['softwareProfile']
    expected_name = configDict.get('tags').get('Name', None)
    if use_instance_hostname:
        if expected_name is None:
            expected_name = 'Tortuga compute node'
    elif node:
        expected_name = node.name
    expected_num_tags = 4 + int(expected_name is not None) + \
        (len(configDict['tags']) - int(bool(name_tag)))

    # Check results
    assert len(tags) == expected_num_tags
    assert tags['tortuga-installer_ipaddress'] == \
        adapter._sanitze_tag_value(configDict['installer_ip'])
    assert tags['tortuga-installer_hostname'] == \
        adapter._sanitze_tag_value(adapter.installer_public_hostname)
    assert tags['tortuga-softwareprofile'] == expected_swp_name
    assert tags['tortuga-hardwareprofile'] == expected_hwp_name
    for k,v in configDict['tags'].items():
        # 'Name' tag will not *always* match - we check it below
        if k != 'Name':
            assert tags[k] == v
    if expected_name is None:
        assert 'Name' not in tags
    else:
        assert tags['Name'] == expected_name