def setup_after_tested_service(self, network, service, setup_info):
"""
Do any setup that must happen after the service under test has been
created.
"""
internet = CidrBlock("0.0.0.0/0")
self.client.paths.add(internet, service, 80)
def internet_accessible(self, service, port):
"""
Return true if the given service is accessible on the internet.
"""
for public_block in get_public_blocks():
if self.has_access(CidrBlock(public_block), service, port):
return True
return False
def setup_after_tested_service(self, network, service, setup_info):
"""
Do any setup that must happen after the service under test has been
created.
"""
my_ip = requests.get("http://ipinfo.io/ip")
test_machine = CidrBlock(my_ip.content.decode("utf-8").strip())
self.client.paths.add(test_machine, service, 9090)
def setup_after_tested_service(self, network, service, setup_info):
"""
Do any setup that must happen after the service under test has been
created.
"""
internal_service_name = setup_info.deployment_info["service_name"]
internal_service = self.client.service.get(network,
internal_service_name)
internet = CidrBlock("0.0.0.0/0")
self.client.paths.add(service, internal_service, 80)
self.client.paths.add(internet, service, 80)
def internet_accessible(self, service, port):
"""
Return true if the given network is internet accessible.
"""
paths = self.list()
for public_block in get_public_blocks():
source = CidrBlock(public_block)
self._validate_args(source, service)
if self._has_access(paths, source, service, port):
return True
return False
def setup_before_tested_service(self, network):
"""
Create the dependent services needed to test this service.
"""
# Create consul since our web server needs it to pull API keys.
service_name = "consul"
service = self.client.service.create(network,
service_name,
SERVICE_BLUEPRINT,
count=1)
# For the test framework, check if these environment variables are set.
use_sslmate = 'SSLMATE_API_KEY' in os.environ
use_datadog = 'DATADOG_API_KEY' in os.environ
check_environment(use_sslmate=use_sslmate, use_datadog=use_datadog)
# Get our domain name
# Unfortunately we have to do this manually because Cloudless doesn't have a native way to
# configure parameters. See https://github.com/getcloudless/cloudless/issues/78
# We also can't do a random domain, because generating the certificate is done out of band.
if 'STATIC_SITE_TEST_DOMAIN' in os.environ:
domain_name = os.environ['STATIC_SITE_TEST_DOMAIN']
else:
domain_name = "getcloudless.com"
# Now let's add any necessary API keys to Consul.
my_ip = requests.get("http://ipinfo.io/ip")
test_machine = CidrBlock(my_ip.content.decode("utf-8").strip())
self.client.paths.add(test_machine, service, 8500)
consul_ips = [
i.public_ip for s in service.subnetworks for i in s.instances
]
setup_consul(consul_ips,
domain_name,
use_sslmate=use_sslmate,
use_datadog=use_datadog)
self.client.paths.remove(test_machine, service, 8500)
blueprint_variables = {
"consul_ips":
[i.private_ip for s in service.subnetworks for i in s.instances],
"jekyll_site_github_url":
"https://github.com/getcloudless/getcloudless.com.git",
"jekyll_site_domain":
domain_name
}
if use_sslmate:
blueprint_variables["use_sslmate"] = True
if use_datadog:
blueprint_variables["use_datadog"] = True
return SetupInfo({"service_name": service_name}, blueprint_variables)
def setup_after_tested_service(self, network, service, setup_info):
"""
Do any setup that must happen after the service under test has been
created.
"""
consul_service_name = setup_info.deployment_info["service_name"]
consul_service = self.client.service.get(network, consul_service_name)
my_ip = requests.get("http://ipinfo.io/ip")
test_machine = CidrBlock(my_ip.content.decode("utf-8").strip())
self.client.paths.add(test_machine, service, 80)
self.client.paths.add(test_machine, service, 443)
# Add this last because we want to make sure that our service can handle a delay before
# getting connectivity to consul.
self.client.paths.add(service, consul_service, 8500)
def deploy(network_name, consul_name, service_name, domain, git_url,
expected_content, profile_name, count):
"""Deploy the service in the given network in the given profile with health checks."""
client = load_client(profile_name)
network = client.network.get(network_name)
if not network:
click.echo("Network: \"%s\" not found." % network_name)
sys.exit(1)
service = client.service.get(network, service_name)
if service:
print("Service: %s already exists!" % service_name)
else:
print("Service: %s not found! Creating." % service_name)
consul_service = client.service.get(network, consul_name)
consul_instances = client.service.get_instances(consul_service)
blueprint_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "blueprint.yml")
blueprint_vars = {
"jekyll_site_domain": domain,
"jekyll_site_github_url": git_url,
"consul_ips":
[instance.private_ip for instance in consul_instances],
"use_sslmate": True,
"use_datadog": True
}
internet = CidrBlock("0.0.0.0/0")
service = client.service.create(network,
service_name,
blueprint=blueprint_path,
count=count,
template_vars=blueprint_vars)
client.paths.add(service, consul_service, 8500)
client.paths.add(internet, service, 80)
client.paths.add(internet, service, 443)
print("Created service: %s!" % service_name)
check_health(service,
expected_content,
use_datadog=True,
use_sslmate=True)
print("")
print("Deploy Successful!")
print("")
print("Public IPs: %s" %
[i.public_ip for i in client.service.get_instances(service)])
print("Use 'cldls service get %s %s' for more info." %
(network_name, service_name))
def deploy(self):
"""
Deploy: Deploys the image on the cloud provider.
"""
# 1. Deploy a temporary network
state = {}
state["network"] = generate_unique_name("image-build")
state["service"] = "image-build"
# Save state first in case something fails
if self._load_state():
raise DisallowedOperationException(
"Called deploy but found existing state %s in state file %s" %
(state, self.state_file))
self._save_state(state)
network = self.client.network.create(state["network"],
NETWORK_BLUEPRINT)
# 2. Create temporary ssh keys
keypair = generate_ssh_keypair()
self._save_keypair(keypair)
state["ssh_username"] = "******"
state["ssh_public_key"] = self.public_key_path
state["ssh_private_key"] = self.private_key_path
self._save_state(state)
# 3. Deploy a service with one instance in that network
template_vars = {
"cloudless_image_build_ssh_key": keypair.public_key,
"cloudless_image_build_ssh_username": state["ssh_username"]
}
service = self.client.service.create(network,
state["service"],
self.config.get_blueprint_path(),
template_vars=template_vars,
count=1)
# 4. Allow port 22 to that instance
internet = CidrBlock("0.0.0.0/0")
self.client.paths.add(internet, service, 22)
return service, state
def setup(client, config):
"""
Create all the boilerplate to spin up the service, and the service itself.
"""
logger.debug("Running setup to test: %s", config)
config_obj = BlueprintTestConfiguration(config)
state = get_state(config_obj)
if state:
raise DisallowedOperationException("Found non empty state file: %s" %
state)
network_name = generate_unique_name("test-network")
service_name = generate_unique_name("test-service")
key_pair = generate_ssh_keypair()
state = {
"network_name": network_name,
"service_name": service_name,
"public_key": key_pair.public_key,
"private_key": key_pair.private_key
}
logger.debug("Saving state: %s now in case something fails", state)
save_state(state, config_obj)
save_key_pair(key_pair, config_obj)
logger.debug("Creating test network: %s", network_name)
network = client.network.create(network_name, NETWORK_BLUEPRINT)
logger.debug("Calling the pre service setup in test fixture")
blueprint_tester = get_blueprint_tester(
client, config_obj.get_config_dir(),
config_obj.get_create_fixture_type(),
config_obj.get_create_fixture_options())
setup_info = blueprint_tester.setup_before_tested_service(network)
if not isinstance(setup_info, SetupInfo):
raise DisallowedOperationException(
"Test fixture must return cloudless.testutils.fixture.SetupInfo object!"
"Found: %s" % setup_info)
state["setup_info"] = {
"deployment_info": setup_info.deployment_info,
"blueprint_vars": setup_info.blueprint_vars,
}
state["ssh_username"] = "******"
logger.debug("Saving full state: %s", state)
save_state(state, config_obj)
# Add SSH key to the instance using reserved variables
if "cloudless_test_framework_ssh_key" in setup_info.blueprint_vars:
raise DisallowedOperationException(
"cloudless_test_framework_ssh_key is a parameter reserved by the test framework "
"and cannot be returned by the test fixture. Found: %s" %
(setup_info.blueprint_vars))
setup_info.blueprint_vars[
"cloudless_test_framework_ssh_key"] = key_pair.public_key
if "cloudless_test_framework_ssh_username" in setup_info.blueprint_vars:
raise DisallowedOperationException(
"cloudless_test_framework_ssh_username is a parameter reserved by the test "
"framework and cannot be returned by the test fixture. Found: %s"
% (setup_info.blueprint_vars))
setup_info.blueprint_vars["cloudless_test_framework_ssh_username"] = state[
"ssh_username"]
logger.debug("Creating services using the blueprint under test")
service = client.service.create(network,
service_name,
config_obj.get_blueprint_path(),
setup_info.blueprint_vars,
count=config_obj.get_count())
logger.debug("Calling the post service setup in test fixture")
blueprint_tester.setup_after_tested_service(network, service, setup_info)
logger.debug("Allowing SSH to test service")
internet = CidrBlock("0.0.0.0/0")
client.paths.add(internet, service, 22)
logger.debug("Test service instances: %s",
client.service.get_instances(service))
return (service, state["ssh_username"], private_key_path(config_obj))
def run_ssh_test(profile=None, provider=None, credentials=None):
"""
Test that the instance management works against the given provider.
"""
# Get the client for this test
client = cloudless.Client(profile, provider, credentials)
# Get a somewhat unique network name
network_name = generate_unique_name("unittest")
# Get a somewhat unique service name
service_name = generate_unique_name("unittest")
# Get a keypair to use
key_pair = generate_ssh_keypair()
# Provision all the resources
test_network = client.network.create(network_name,
blueprint=NETWORK_BLUEPRINT)
if client.provider in ["aws", "mock-aws"]:
test_service = client.service.create(
test_network,
service_name,
AWS_SERVICE_BLUEPRINT,
template_vars={
"cloudless_image_build_ssh_key": key_pair.public_key,
"cloudless_image_build_ssh_username": "******"
},
count=1)
else:
assert client.provider == "gce"
test_service = client.service.create(
test_network,
service_name,
GCE_SERVICE_BLUEPRINT,
template_vars={
"cloudless_image_build_ssh_key": key_pair.public_key,
"cloudless_image_build_ssh_username": "******"
},
count=1)
def validate_service(network, service, count):
discovered_service = client.service.get(network, service.name)
assert discovered_service.network == network
assert discovered_service.name == service.name
assert discovered_service == service
assert isinstance(discovered_service, Service)
assert isinstance(service, Service)
instances = []
for subnetwork in discovered_service.subnetworks:
instances.extend(subnetwork.instances)
assert len(instances) == count
assert instances == client.service.get_instances(service)
# Check that our service is provisioned properly
validate_service(test_network, test_service, 1)
# Add a path for SSH
internet = CidrBlock("0.0.0.0/0")
client.paths.add(internet, test_service, 22)
if client.provider != "mock-aws":
# Test that we can connect with the given key
def attempt_connection():
ssh = paramiko.SSHClient()
ssh_key = paramiko.RSAKey(file_obj=StringIO(key_pair.private_key))
public_ip = [
i.public_ip for i in client.service.get_instances(test_service)
][0]
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=public_ip, username="******", pkey=ssh_key)
return ssh
call_with_retries(attempt_connection, int(10), float(1.0))
ssh = attempt_connection()
_, ssh_stdout, ssh_stderr = ssh.exec_command("whoami")
assert ssh_stdout.read().decode().strip() == "cloudless"
assert ssh_stderr.read().decode().strip() == ""
# Make sure they are gone when I destroy them
client.service.destroy(test_service)
# Clean up the VPC
client.network.destroy(test_network)