Exemplo n.º 1
0
def run_network_test(profile=None, provider=None, credentials=None):
    """
    Test network management on the given provider.
    """

    # Get the client for this test
    client = cloudless.Client(profile, provider, credentials)

    # Get somewhat unique network names
    network1_name = generate_unique_name("network1")
    network2_name = generate_unique_name("network2")

    # Create two private cloud networks
    network1 = client.network.create(name=network1_name,
                                     blueprint=NETWORK_BLUEPRINT)
    network2 = client.network.create(name=network2_name)
    assert isinstance(network1, Network)
    assert isinstance(network2, Network)
    assert network2 != network1
    # pylint: disable=comparison-with-itself
    assert network1 == network1
    # pylint: disable=comparison-with-itself
    assert network2 == network2

    # Make sure we can discover them again
    assert client.network.get(network1_name) == network1
    assert client.network.get(network2_name) == network2

    # Destroy them and make sure they no longer exist
    client.network.destroy(network1)
    client.network.destroy(network2)
    assert not client.network.get(network1_name)
    assert not client.network.get(network2_name)
Exemplo n.º 2
0
def print_graph(provider):
    """
    Print a graph of all services for provider.
    """
    if provider == "aws":
        client = cloudless.Client("aws", {})
    elif provider == "gce":
        client = cloudless.Client(
            "gce", {
                "user_id": os.environ['CLOUDLESS_GCE_USER_ID'],
                "key": os.environ['CLOUDLESS_GCE_CREDENTIALS_PATH'],
                "project": os.environ['CLOUDLESS_GCE_PROJECT_NAME']
            })
    else:
        raise NotImplementedError("Provider %s not supported" % provider)
    print(client.graph())
Exemplo n.º 3
0
def run_firewall_model_test(profile=None, provider=None, credentials=None):
    """
    Test model management on the given provider.
    """
    client = cloudless.Client(profile, provider, credentials)

    # Get a unique "namespace" for testing
    network_name = generate_unique_name("network")

    # Create our models
    network_dict = {"name": network_name, "version": "0.0.0"}
    network = NetworkModel.fromdict(network_dict)
    firewall = Firewall.fromdict({
        "name": network_name,
        "version": "0.0.0",
        "network": {
            "name": network.name
        }
    })

    # Make sure our models are registered
    assert "Firewall" in client.model.resources()
    assert "Network" in client.model.resources()

    # Create a network and do some sanity checks
    network_create = client.model.create("Network", network)

    network_get = client.model.get("Network", network)
    assert len(network_get) == 1
    assert network_get[0] == network_create
    assert network_get[0].name == network.name

    # Create a firewall and do some sanity checks
    firewall_create = client.model.create("Firewall", firewall)

    firewall_get = client.model.get("Firewall", firewall)
    assert len(firewall_get) == 1
    assert firewall_get[0] == firewall_create
    assert firewall_get[0].name == firewall.name

    # Delete the firewall
    firewall_delete = client.model.delete("Firewall", firewall)
    assert not firewall_delete

    firewall_get = client.model.get("Firewall", firewall)
    assert not firewall_get

    # Delete the network
    network_delete = client.model.delete("Network", network)
    assert not network_delete

    network_get = client.model.get("Network", network)
    assert not network_get
Exemplo n.º 4
0
def run_subnet_model_test(profile=None, provider=None, credentials=None):
    """
    Test model management on the given provider.
    """
    client = cloudless.Client(profile, provider, credentials)

    # Get a unique "namespace" for testing
    network_name = generate_unique_name("network")

    # Create our models
    network_dict = {"name": network_name, "version": "0.0.0"}
    network = NetworkModel.fromdict(network_dict)
    subnet = SubnetModel.fromdict({
        "name": network_name,
        "version": "0.0.0",
        "network": network_dict,
        "size": 256
    })

    # Make sure our models are registered
    assert "Subnet" in client.model.resources()
    assert "Network" in client.model.resources()

    # Create a network and do some sanity checks
    network_create = client.model.create("Network", network)

    network_get = client.model.get("Network", network)
    assert len(network_get) == 1
    assert network_get[0] == network_create
    assert network_get[0].name == network.name

    # Create an subnet and do some sanity checks
    subnet_create = client.model.create("Subnet", subnet)

    subnet_get = client.model.get("Subnet", subnet)
    assert len(subnet_get) == 1
    assert subnet_get[0] == subnet_create
    assert subnet_get[0].name == subnet.name

    # Delete the subnet
    subnet_delete = client.model.delete("Subnet", subnet)
    assert not subnet_delete

    subnet_get = client.model.get("Subnet", subnet)
    assert not subnet_get

    # Delete the network
    network_delete = client.model.delete("Network", network)
    assert not network_delete

    network_get = client.model.get("Network", network)
    assert not network_get
Exemplo n.º 5
0
def run_paths_test(profile=None, provider=None, credentials=None):
    """
    Test that the path management works against the given provider.
    """

    # Get the client for this test
    client = cloudless.Client(profile, provider, credentials)

    # Get a somewhat unique network name
    network_name = generate_unique_name("unittest")

    # Provision all the resources
    test_network = client.network.create(network_name,
                                         blueprint=NETWORK_BLUEPRINT)
    if client.provider in ["aws", "mock-aws"]:
        lb_service = client.service.create(test_network, "web-lb",
                                           AWS_SERVICE_BLUEPRINT, {})
        web_service = client.service.create(test_network, "web",
                                            AWS_SERVICE_BLUEPRINT, {})
    else:
        assert client.provider == "gce"
        lb_service = client.service.create(test_network, "web-lb",
                                           GCE_SERVICE_BLUEPRINT, {})
        web_service = client.service.create(test_network, "web",
                                            GCE_SERVICE_BLUEPRINT, {})

    # Create CIDR block object for the paths API
    internet = cloudless.paths.CidrBlock("0.0.0.0/0")

    assert not client.paths.has_access(lb_service, web_service, 80)
    assert not client.paths.internet_accessible(lb_service, 80)

    client.paths.add(lb_service, web_service, 80)
    client.paths.add(internet, lb_service, 80)
    for path in client.paths.list():
        assert isinstance(path, Path)
    client.graph()

    assert client.paths.has_access(lb_service, web_service, 80)
    assert client.paths.internet_accessible(lb_service, 80)

    client.paths.remove(lb_service, web_service, 80)
    assert not client.paths.has_access(lb_service, web_service, 80)

    client.paths.remove(internet, lb_service, 80)
    assert not client.paths.internet_accessible(lb_service, 80)

    client.service.destroy(lb_service)
    client.service.destroy(web_service)
    client.network.destroy(test_network)
Exemplo n.º 6
0
def handle_profile_for_cli(ctx):
    """
    Loads the profile and sets the proper fields on the context object for the command line.
    """
    profile = cloudless.profile.load_profile(ctx.obj['PROFILE'])
    if not profile:
        click.echo("Profile: \"%s\" not found." % ctx.obj['PROFILE'])
        click.echo("Try running \"cldls --profile %s init\"." %
                   ctx.obj['PROFILE'])
        sys.exit(1)
    ctx.obj['PROVIDER'] = profile["provider"]
    ctx.obj['CREDENTIALS'] = profile["credentials"]
    ctx.obj['CLIENT'] = cloudless.Client(provider=ctx.obj['PROVIDER'],
                                         credentials=ctx.obj['CREDENTIALS'])
def run_blueprint_tester_test(provider, credentials):
    """
    Test blueprint test framework with the given provider.
    """

    # Get the client for this test
    client = cloudless.Client(provider, credentials)

    # Load the configuration because we use it directly
    test_config_obj = BlueprintTestConfiguration(BLUEPRINT_TEST_CONFIGURATION)

    # First make sure we have no state if we teardown
    do_teardown(client, BLUEPRINT_TEST_CONFIGURATION)
    state = get_state(test_config_obj)
    assert not state

    # Then, make sure the full run works
    run_all(client, BLUEPRINT_TEST_CONFIGURATION)

    # Now make sure the state got cleared
    state = get_state(test_config_obj)
    assert not state

    # Now do just the setup and teardown
    do_setup(client, BLUEPRINT_TEST_CONFIGURATION)
    state = get_state(test_config_obj)
    assert state
    assert client.network.get(state["network_name"])
    assert client.service.get(client.network.get(state["network_name"]),
                              state["service_name"])
    assert state["setup_info"]
    do_teardown(client, BLUEPRINT_TEST_CONFIGURATION)
    state = get_state(test_config_obj)
    assert not state

    # Now do setup, multiple verifies and teardown
    do_setup(client, BLUEPRINT_TEST_CONFIGURATION)
    state = get_state(test_config_obj)
    assert state
    assert client.network.get(state["network_name"])
    assert client.service.get(client.network.get(state["network_name"]),
                              state["service_name"])
    assert state["setup_info"]
    do_verify(client, BLUEPRINT_TEST_CONFIGURATION)
    do_verify(client, BLUEPRINT_TEST_CONFIGURATION)
    do_teardown(client, BLUEPRINT_TEST_CONFIGURATION)
    state = get_state(test_config_obj)
    assert not state
Exemplo n.º 8
0
def handle_profile_for_cli(ctx):
    """
    Loads the profile and sets the proper fields on the context object for the command line.

    Note, the API does handle the profile as well, but this is an attempt to return nice errors to
    the user.  It would be nice to make this cleaner/deduplicate code.
    """
    profile = cloudless.profile.load_profile(ctx.obj['PROFILE'])
    if not profile:
        click.echo("Profile: \"%s\" not found." % ctx.obj['PROFILE'])
        click.echo("Try running \"cldls --profile %s init\"." %
                   ctx.obj['PROFILE'])
        sys.exit(1)
    ctx.obj['PROVIDER'] = profile["provider"]
    ctx.obj['CREDENTIALS'] = profile["credentials"]
    ctx.obj['CLIENT'] = cloudless.Client(provider=ctx.obj['PROVIDER'],
                                         credentials=ctx.obj['CREDENTIALS'])
Exemplo n.º 9
0
def run_image_model_test(profile=None, provider=None, credentials=None):
    """
    Test image model on the given provider.
    """
    client = cloudless.Client(profile, provider, credentials)

    # Create Image model
    image = ImageModel.fromdict({
        "name": "*ubuntu*xenial*",
        "version": "0.0.0",
        "creation_date": "latest"
    })

    # Make sure the image model is registered
    assert "Image" in client.model.resources()

    image_get = client.model.get("Image", image)
    assert len(image_get) == 1
    assert "ubuntu" in image_get[0].name
Exemplo n.º 10
0
def run_instance_fitter_test(profile=None, provider=None, credentials=None):
    """
    Test that we get the proper instance sizes for the given provider.
    """

    # Get the client for this test
    client = cloudless.Client(profile, provider, credentials)

    # If no memory, cpu, or storage is passed in, find the cheapest.
    if client.provider == "aws":
        assert get_fitting_instance(client.service,
                                    ServiceBlueprint(SMALL_INSTANCE_BLUEPRINT)) == "t2.small"
        assert get_fitting_instance(client.service,
                                    ServiceBlueprint(LARGE_INSTANCE_BLUEPRINT)) == "m5.xlarge"
    if client.provider == "gce":
        assert get_fitting_instance(client.service,
                                    ServiceBlueprint(SMALL_INSTANCE_BLUEPRINT)) == "n1-highcpu-4"
        assert get_fitting_instance(client.service,
                                    ServiceBlueprint(LARGE_INSTANCE_BLUEPRINT)) == "n1-highmem-4"
Exemplo n.º 11
0
def test_image_builder(mock_filesystem, mock_image_build_configuration):
    """
    Test image builder using the mock-aws provider.
    """
    client = cloudless.Client("mock-aws", {})
    image_builder = ImageBuilder(client,
                                 config=mock_image_build_configuration,
                                 filesystem=mock_filesystem)

    # Make sure we can actually deploy the service.
    deployed_service, state = image_builder.deploy()
    network = client.network.get(state["network"])
    assert network
    assert network.name == deployed_service.network.name
    service = client.service.get(network, state["service"])
    assert service
    assert service.name == deployed_service.name

    # Make sure configure works and properly finds errors.
    image_builder.configure()
    mock_image_build_configuration.set_configure_script_path(
        BAD_CONFIGURE_FILE)
    with pytest.raises(Exception):
        image_builder.configure()
        pytest.fail("Expected exception on bad configure file")
    mock_image_build_configuration.set_configure_script_path(CONFIGURE_FILE)
    image_builder.configure()

    # Make sure check works and properly finds errors.
    image_builder.check()
    mock_image_build_configuration.set_check_script_path(BAD_CHECK_FILE)
    with pytest.raises(Exception):
        image_builder.check()
        pytest.fail("Expected exception on bad check file")
    mock_image_build_configuration.set_check_script_path(CHECK_FILE)
    image_builder.check()

    # Make sure things are gone when we clean up.
    image_builder.cleanup()
    network = client.network.get(state["network"])
    assert not network

    image_builder.run(mock=True)
Exemplo n.º 12
0
def load_client(profile_name):
    """
    Load the client using the given profile.

    Need to do this outside our client because of
    https://github.com/getcloudless/cloudless/issues/56
    """
    if profile_name:
        profile_name = profile_name
    elif "CLOUDLESS_PROFILE" in os.environ:
        profile_name = os.environ["CLOUDLESS_PROFILE"]
    else:
        profile_name = "default"
    profile = cloudless.profile.load_profile(profile_name)
    if not profile:
        click.echo("Profile: \"%s\" not found." % profile_name)
        click.echo("Try running \"cldls --profile %s init\"." % profile_name)
        sys.exit(1)
    return cloudless.Client(provider=profile['provider'],
                            credentials=profile['credentials'])
Exemplo n.º 13
0
def run_image_test(provider, credentials):
    """
    Test that the instance management works against the given provider.
    """

    # Get the client for this test
    client = cloudless.Client(provider, credentials)

    # Get a somewhat unique network name
    network_name = generate_unique_name("unittest")
    image_name = generate_unique_name("unittest")

    # Provision a service with one instance
    test_network = client.network.create(network_name,
                                         blueprint=NETWORK_BLUEPRINT)
    if provider in ["aws", "mock-aws"]:
        service = client.service.create(test_network,
                                        "service",
                                        AWS_SERVICE_BLUEPRINT, {},
                                        count=1)
    else:
        assert provider == "gce"
        service = client.service.create(test_network,
                                        "service",
                                        GCE_SERVICE_BLUEPRINT, {},
                                        count=1)

    def validate_service(network, service, count):
        discovered_service = client.service.get(network, service.name)
        assert discovered_service.network == network
        assert discovered_service.name == service.name
        assert discovered_service == service
        assert isinstance(discovered_service, Service)
        assert isinstance(service, Service)
        instances = []
        for subnetwork in discovered_service.subnetworks:
            instances.extend(subnetwork.instances)
        assert len(instances) == count
        assert instances == client.service.get_instances(service)

    validate_service(test_network, service, 1)

    # Create an image from this service
    client.image.create(image_name, service)

    # List images
    images = client.image.list()
    assert images
    for image in client.image.list():
        if image.name == image_name:
            found_image = True
    assert found_image

    # Get image
    assert client.image.get(image_name)

    # Destroy image
    assert client.image.destroy(client.image.get(image_name))

    # Get image
    assert not client.image.get(image_name)
    if provider == "mock-aws":
        assert not client.image.list()

    # Clean up everything else
    client.service.destroy(service)
    client.network.destroy(test_network)
Exemplo n.º 14
0
def run_ssh_test(profile=None, provider=None, credentials=None):
    """
    Test that the instance management works against the given provider.
    """

    # Get the client for this test
    client = cloudless.Client(profile, provider, credentials)

    # Get a somewhat unique network name
    network_name = generate_unique_name("unittest")

    # Get a somewhat unique service name
    service_name = generate_unique_name("unittest")

    # Get a keypair to use
    key_pair = generate_ssh_keypair()

    # Provision all the resources
    test_network = client.network.create(network_name,
                                         blueprint=NETWORK_BLUEPRINT)
    if client.provider in ["aws", "mock-aws"]:
        test_service = client.service.create(
            test_network,
            service_name,
            AWS_SERVICE_BLUEPRINT,
            template_vars={
                "cloudless_image_build_ssh_key": key_pair.public_key,
                "cloudless_image_build_ssh_username": "******"
            },
            count=1)
    else:
        assert client.provider == "gce"
        test_service = client.service.create(
            test_network,
            service_name,
            GCE_SERVICE_BLUEPRINT,
            template_vars={
                "cloudless_image_build_ssh_key": key_pair.public_key,
                "cloudless_image_build_ssh_username": "******"
            },
            count=1)

    def validate_service(network, service, count):
        discovered_service = client.service.get(network, service.name)
        assert discovered_service.network == network
        assert discovered_service.name == service.name
        assert discovered_service == service
        assert isinstance(discovered_service, Service)
        assert isinstance(service, Service)
        instances = []
        for subnetwork in discovered_service.subnetworks:
            instances.extend(subnetwork.instances)
        assert len(instances) == count
        assert instances == client.service.get_instances(service)

    # Check that our service is provisioned properly
    validate_service(test_network, test_service, 1)

    # Add a path for SSH
    internet = CidrBlock("0.0.0.0/0")
    client.paths.add(internet, test_service, 22)

    if client.provider != "mock-aws":
        # Test that we can connect with the given key
        def attempt_connection():
            ssh = paramiko.SSHClient()
            ssh_key = paramiko.RSAKey(file_obj=StringIO(key_pair.private_key))
            public_ip = [
                i.public_ip for i in client.service.get_instances(test_service)
            ][0]
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
            ssh.connect(hostname=public_ip, username="******", pkey=ssh_key)
            return ssh

        call_with_retries(attempt_connection, int(10), float(1.0))
        ssh = attempt_connection()
        _, ssh_stdout, ssh_stderr = ssh.exec_command("whoami")
        assert ssh_stdout.read().decode().strip() == "cloudless"
        assert ssh_stderr.read().decode().strip() == ""

    # Make sure they are gone when I destroy them
    client.service.destroy(test_service)

    # Clean up the VPC
    client.network.destroy(test_network)
Exemplo n.º 15
0
def run_instances_test(profile=None, provider=None, credentials=None):
    """
    Test that the instance management works against the given provider.
    """

    # Get the client for this test
    client = cloudless.Client(profile, provider, credentials)

    # Get a somewhat unique network name
    network_name = generate_unique_name("unittest")

    # Provision all the resources
    test_network = client.network.create(network_name,
                                         blueprint=NETWORK_BLUEPRINT)
    if client.provider in ["aws", "mock-aws"]:
        lb_service = client.service.create(test_network, "web-lb",
                                           AWS_SERVICE_BLUEPRINT, {})
        web_service = client.service.create(test_network,
                                            "web",
                                            AWS_SERVICE_BLUEPRINT, {},
                                            count=6)
    else:
        assert client.provider == "gce"
        lb_service = client.service.create(test_network, "web-lb",
                                           GCE_SERVICE_BLUEPRINT, {})
        web_service = client.service.create(test_network,
                                            "web",
                                            GCE_SERVICE_BLUEPRINT, {},
                                            count=6)

    def validate_service(network, service, count):
        discovered_service = client.service.get(network, service.name)
        assert discovered_service.network == network
        assert discovered_service.name == service.name
        assert discovered_service == service
        assert isinstance(discovered_service, Service)
        assert isinstance(service, Service)
        instances = []
        for subnetwork in discovered_service.subnetworks:
            assert subnetwork.instances
            instances.extend(subnetwork.instances)
        assert len(instances) == count
        assert instances == client.service.get_instances(service)

    validate_service(test_network, lb_service, 1)
    validate_service(test_network, web_service, 6)

    if client.provider in ["mock-aws"]:
        # Networking
        ec2 = boto3.client("ec2")
        dc_id = client.network.get(network_name).network_id
        subnets = ec2.describe_subnets(Filters=[{
            'Name': 'vpc-id',
            'Values': [dc_id]
        }])
        route_tables = ec2.describe_route_tables(Filters=[{
            'Name': 'vpc-id',
            'Values': [dc_id]
        }])
        # The blueprints we are using to test only create one availability zone right now.
        assert len(route_tables["RouteTables"]) == 3
        assert len(subnets["Subnets"]) == 2

        # AutoScalingGroup
        autoscaling = boto3.client("autoscaling")
        web_asgs = autoscaling.describe_auto_scaling_groups(
            AutoScalingGroupNames=["%s.web" % network_name])
        assert len(web_asgs["AutoScalingGroups"]) == 1
        web_asg = web_asgs["AutoScalingGroups"][0]
        assert web_asg["AutoScalingGroupName"] == "%s.web" % network_name
        assert web_asg["LaunchConfigurationName"] == "%s.web" % network_name
        web_lb_asgs = autoscaling.describe_auto_scaling_groups(
            AutoScalingGroupNames=["%s.web-lb" % network_name])
        assert len(web_lb_asgs["AutoScalingGroups"]) == 1
        web_lb_asg = web_lb_asgs["AutoScalingGroups"][0]
        assert web_lb_asg["AutoScalingGroupName"] == "%s.web-lb" % network_name
        assert web_lb_asg[
            "LaunchConfigurationName"] == "%s.web-lb" % network_name

        # Make sure subnets don't overlap
        web_asg = web_asgs["AutoScalingGroups"][0]
        web_asg_subnet_ids = web_asg["VPCZoneIdentifier"].split(",")
        assert len(web_asg_subnet_ids) == 1

        web_lb_asg = web_lb_asgs["AutoScalingGroups"][0]
        web_lb_asg_subnet_ids = web_lb_asg["VPCZoneIdentifier"].split(",")
        assert len(web_lb_asg_subnet_ids) == 1

        web_asg_subnets = ec2.describe_subnets(SubnetIds=web_asg_subnet_ids)
        assert len(web_asg_subnets["Subnets"]) == 1

        web_lb_asg_subnets = ec2.describe_subnets(
            SubnetIds=web_lb_asg_subnet_ids)
        assert len(web_lb_asg_subnets["Subnets"]) == 1

        for web_asg_subnet in web_asg_subnets["Subnets"]:
            web_asg_cidr = ipaddress.ip_network(
                str(web_asg_subnet["CidrBlock"]))
            for web_lb_asg_subnet in web_lb_asg_subnets["Subnets"]:
                web_lb_asg_cidr = ipaddress.ip_network(
                    str(web_lb_asg_subnet["CidrBlock"]))
                assert not web_asg_cidr.overlaps(web_lb_asg_cidr)

        # Make sure they got allocated in the same VPC
        web_asg_vpc_id = None
        for web_asg_subnet in web_asg_subnets["Subnets"]:
            if not web_asg_vpc_id:
                web_asg_vpc_id = web_asg_subnet["VpcId"]
            assert web_asg_subnet["VpcId"] == web_asg_vpc_id

        web_lb_asg_vpc_id = None
        for web_lb_asg_subnet in web_lb_asg_subnets["Subnets"]:
            if not web_lb_asg_vpc_id:
                web_lb_asg_vpc_id = web_lb_asg_subnet["VpcId"]
            assert web_lb_asg_subnet["VpcId"] == web_lb_asg_vpc_id

        assert web_asg_vpc_id == web_lb_asg_vpc_id

    # Make sure they are gone when I destroy them
    client.service.destroy(lb_service)

    if client.provider in ["mock-aws"]:
        # Networking
        ec2 = boto3.client("ec2")
        subnets = ec2.describe_subnets(Filters=[{
            'Name': 'vpc-id',
            'Values': [dc_id]
        }])
        route_tables = ec2.describe_route_tables(Filters=[{
            'Name': 'vpc-id',
            'Values': [dc_id]
        }])
        assert len(route_tables["RouteTables"]) == 2
        assert len(subnets["Subnets"]) == 1

        # AutoScalingGroup
        autoscaling = boto3.client("autoscaling")
        asgs = autoscaling.describe_auto_scaling_groups(
            AutoScalingGroupNames=["%s.web" % network_name])
        assert len(asgs["AutoScalingGroups"]) == 1
        asg = asgs["AutoScalingGroups"][0]
        assert asg["AutoScalingGroupName"] == "%s.web" % network_name
        assert asg["LaunchConfigurationName"] == "%s.web" % network_name

    # Now destroy the rest
    client.service.destroy(web_service)

    if client.provider in ["mock-aws"]:
        # AutoScalingGroups
        autoscaling = boto3.client("autoscaling")
        asgs = autoscaling.describe_auto_scaling_groups(
            AutoScalingGroupNames=["%s.web" % network_name])
        assert not asgs["AutoScalingGroups"]
        asgs = autoscaling.describe_auto_scaling_groups(
            AutoScalingGroupNames=["%s.web-lb" % network_name])
        assert not asgs["AutoScalingGroups"]

    # Clean up the VPC
    client.network.destroy(test_network)