def test_prune_repos(configured, aws_credentials):
config_name = configured
ck.refresh_clients()
repo0 = ck.aws.DockerRepo(name="test-prune-0")
repo1 = ck.aws.DockerRepo(name="test-prune-1")
repo2 = ck.aws.DockerRepo(name="test-prune-2")
ck.aws.clients["ecr"].delete_repository(repositoryName=repo0.name)
ck.config.prune_repos()
config = configparser.ConfigParser()
config.read(config_name)
repo_section = config[repo0._section_name]
assert repo0.name not in repo_section
assert repo1.name in repo_section
assert repo2.name in repo_section
ck.config.add_resource(repo1._section_name, repo1.name, "0123")
ck.config.prune()
config = configparser.ConfigParser()
config.read(config_name)
repo_section = config[repo0._section_name]
assert repo1.name not in repo_section
assert repo2.name in repo_section
def test_is_valid_stack(configured, aws_credentials):
ck.refresh_clients()
pars = ck.Pars(name="test-valid-stack")
assert ck.config.is_valid_stack(stack_id=pars.stack_id)
ck.aws.clients["cloudformation"].delete_stack(StackName=pars.stack_id)
assert not ck.config.is_valid_stack(stack_id=pars.stack_id)
def test_prune_stacks(configured, aws_credentials):
config_name = configured
ck.refresh_clients()
pars0 = ck.Pars(name="test-prune-0")
pars1 = ck.Pars(name="test-prune-1")
ck.aws.clients["cloudformation"].delete_stack(StackName=pars0.stack_id)
ck.config.prune_stacks()
config = configparser.ConfigParser()
config.read(config_name)
assert pars0.pars_name not in config.sections()
assert pars1.pars_name in config.sections()
def test_get_tags(aws_credentials):
ck.refresh_clients()
name = "test-name"
tags_with_name_only = ck.aws.get_tags(name=name)
ref_name = {"Key": "Name", "Value": name}
ref_owner = {"Key": "Owner", "Value": ck.aws.get_user()}
ref_env = {"Key": "Environment", "Value": "cloudknot"}
ref_dicts = [ref_name, ref_owner, ref_env]
def is_eq_list_of_dicts(lst0, lst1):
sort0 = sorted(lst0, key=lambda d: d["Key"])
sort1 = sorted(lst1, key=lambda d: d["Key"])
return all([pair0 == pair1 for pair0, pair1 in zip(sort0, sort1)])
assert is_eq_list_of_dicts(ref_dicts, tags_with_name_only)
add_list = [{"Key": "Project", "Value": "unit-testing"}]
tags_with_add_list = ck.aws.get_tags(name=name, additional_tags=add_list)
assert is_eq_list_of_dicts(ref_dicts + add_list, tags_with_add_list)
add_dict = {"Project": "unit-testing"}
tags_with_add_dict = ck.aws.get_tags(name=name, additional_tags=add_dict)
assert is_eq_list_of_dicts(ref_dicts + add_list, tags_with_add_dict)
add_name = {"Key": "Name", "Value": "custom-name"}
tags_add_name = ck.aws.get_tags(name=name, additional_tags=[add_name])
assert is_eq_list_of_dicts([add_name, ref_owner, ref_env], tags_add_name)
add_env = {"Key": "Environment", "Value": "custom-env"}
tags_add_env = ck.aws.get_tags(name=name, additional_tags=[add_env])
assert is_eq_list_of_dicts([ref_name, ref_owner, add_env], tags_add_env)
add_owner = {"Key": "Owner", "Value": "custom-env"}
tags_add_owner = ck.aws.get_tags(name=name, additional_tags=[add_owner])
assert is_eq_list_of_dicts([ref_name, add_owner, ref_env], tags_add_owner)
with pytest.raises(ValueError):
ck.aws.get_tags(name=name, additional_tags=42)
with pytest.raises(ValueError):
ck.aws.get_tags(name=name, additional_tags=[{"Foo": "Bar"}])
with pytest.raises(ValueError):
ck.aws.get_tags(name=name, additional_tags={"Key": 42})
with pytest.raises(ValueError):
ck.aws.get_tags(name=name, additional_tags={"Value": 42})
def test_set_profile(bucket_cleanup):
old_credentials_file = os.environ.get("AWS_SHARED_CREDENTIALS_FILE")
old_aws_config_file = os.environ.get("AWS_CONFIG_FILE")
old_ck_config_file = os.environ.get("CLOUDKNOT_CONFIG_FILE")
ref_dir = op.join(data_path, "profiles_ref_data")
ck_config_file = op.join(ref_dir, "cloudknot_without_profile")
shutil.copy(ck_config_file, ck_config_file + ".bak")
try:
os.environ["CLOUDKNOT_CONFIG_FILE"] = ck_config_file
config_file = op.join(ref_dir, "config")
os.environ["AWS_CONFIG_FILE"] = config_file
cred_file = op.join(ref_dir, "credentials_without_default")
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = cred_file
with pytest.raises(ck.aws.CloudknotInputError):
ck.set_profile(profile_name="not_in_list_of_profiles")
profile = "name-5"
ck.set_profile(profile_name=profile)
assert ck.get_profile() == profile
finally:
shutil.move(ck_config_file + ".bak", ck_config_file)
if old_credentials_file:
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = old_credentials_file
else:
os.environ.pop("AWS_SHARED_CREDENTIALS_FILE")
if old_aws_config_file:
os.environ["AWS_CONFIG_FILE"] = old_aws_config_file
else:
os.environ.pop("AWS_CONFIG_FILE")
if old_ck_config_file:
os.environ["CLOUDKNOT_CONFIG_FILE"] = old_ck_config_file
else:
os.environ.pop("CLOUDKNOT_CONFIG_FILE")
ck.refresh_clients()
def test_set_region(bucket_cleanup):
ck.refresh_clients()
with pytest.raises(ck.aws.CloudknotInputError):
ck.set_region(region="not a valid region name")
old_region = ck.get_region()
try:
old_config_file = os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
old_config_file = None
try:
with tempfile.NamedTemporaryFile() as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
region = "us-west-1"
ck.set_region(region)
assert ck.get_region() == region
for service, client in ck.aws.clients.items():
if service == "iam":
assert client.meta.region_name == "aws-global"
else:
assert client.meta.region_name == region
finally:
ck.set_region(old_region)
if old_config_file:
os.environ["CLOUDKNOT_CONFIG_FILE"] = old_config_file
else:
try:
del os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
pass
ck.refresh_clients()
def test_pars_errors(cleanup):
ck.refresh_clients()
name = get_testing_name()
# Confirm name input validation
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=42)
# Assert ck.aws.CloudknotInputError on long name
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name="a" * 46)
# Confirm batch_service_role_name input validation
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=name, batch_service_role_name=42)
# Confirm error on redundant VPC input
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=name, ipv4_cidr="172.31.0.0/16")
# Confirm error on invalid VPC CIDR
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=name, use_default_vpc=False, ipv4_cidr=42)
# Confirm error on invalid VPC instance tenancy
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=name, use_default_vpc=False, instance_tenancy=42)
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=name, use_default_vpc=False, policies=42)
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=name, use_default_vpc=False, policies=[42, 42])
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=name, use_default_vpc=False, policies=["foo"])
def test_DockerImage(cleanup_repos):
ck.refresh_clients()
config = configparser.ConfigParser()
config_file = ck.config.get_config_file()
ecr = ck.aws.clients["ecr"]
try:
correct_pip_imports = {
"boto3",
"six",
"dask",
"docker",
"pytest",
"cloudpickle",
}
# First, test a DockerImage instance with `func` input
# ----------------------------------------------------
di = ck.DockerImage(
name=unit_testing_func.__name__.replace("_", "-"), func=unit_testing_func
)
assert di.name == unit_testing_func.__name__.replace("_", "-")
import_names = set([d["name"] for d in di.pip_imports])
assert import_names == correct_pip_imports
assert di.missing_imports == []
assert di.username == "cloudknot-user"
assert di.func == unit_testing_func
py_dir = "py3"
# Compare the created files with the reference files
correct_dir = op.join(data_path, "docker_reqs_ref_data", py_dir, "ref1")
correct_req_path = op.join(correct_dir, "requirements.txt")
correct_dockerfile = op.join(correct_dir, "Dockerfile")
correct_script_path = op.join(correct_dir, "unit-testing-func.py")
with open(correct_req_path) as f:
correct_reqs = set([s.split("=")[0] for s in f.readlines()])
with open(di.req_path) as f:
created_reqs = set([s.split("=")[0] for s in f.readlines()])
assert created_reqs == correct_reqs
assert filecmp.cmp(di.docker_path, correct_dockerfile, shallow=False)
assert filecmp.cmp(di.script_path, correct_script_path, shallow=False)
# Confirm that the docker image is in the config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert "docker-image " + di.name in config.sections()
# Next, retrieve another instance with the same name, confirm that it
# retrieves the same info from the config file
di2 = ck.DockerImage(name=di.name)
assert di2.build_path == di.build_path
assert di2.docker_path == di.docker_path
assert di2.images == di.images
assert di2.missing_imports == di.missing_imports
assert di2.name == di.name
assert di2.pip_imports == di.pip_imports
assert di2.repo_uri == di.repo_uri
assert di2.req_path == di.req_path
assert di2.script_path == di.script_path
assert di2.username == di.username
# Clobber and confirm that it deleted all the created files and dirs
di2.clobber()
assert not op.isfile(di.req_path)
assert not op.isfile(di.docker_path)
assert not op.isfile(di.script_path)
assert not op.isdir(di.build_path)
# Assert that it was removed from the config file
# If we just re-read the config file, config will keep the union
# of the in memory values and the file values, updating the
# intersection of the two with the file values. So we must clear
# config and then re-read the file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert "docker-image " + di.name not in config.sections()
# Second, test a DockerImage with a func and a dir_name
# -----------------------------------------------------
dir_name = tempfile.mkdtemp(dir=os.getcwd())
di = ck.DockerImage(func=unit_testing_func, dir_name=dir_name)
assert di.name == unit_testing_func.__name__.replace("_", "-")
import_names = set([d["name"] for d in di.pip_imports])
assert import_names == correct_pip_imports
assert di.missing_imports == []
assert di.username == "cloudknot-user"
assert di.func == unit_testing_func
with open(di.req_path) as f:
created_reqs = set([s.split("=")[0] for s in f.readlines()])
assert created_reqs == correct_reqs
assert filecmp.cmp(di.docker_path, correct_dockerfile, shallow=False)
assert filecmp.cmp(di.script_path, correct_script_path, shallow=False)
# Confirm that the docker image is in the config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert "docker-image " + di.name in config.sections()
# Clobber and confirm that it deleted all the created files and dirs
di.clobber()
assert not op.isfile(di.req_path)
assert not op.isfile(di.docker_path)
assert not op.isfile(di.script_path)
assert not op.isdir(di.build_path)
# Assert that it was removed from the config file
# If we just re-read the config file, config will keep the union
# of the in memory values and the file values, updating the
# intersection of the two with the file values. So we must clear
# config and then re-read the file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert "docker-image " + di.name not in config.sections()
# Third, test a DockerImage with script_path and dir_name input
# -------------------------------------------------------------
correct_dir = op.join(data_path, "docker_reqs_ref_data", py_dir, "ref2")
script_path = op.join(correct_dir, "test-func-input.py")
# Put the results in a temp dir with a pre-existing file
dir_name = tempfile.mkdtemp(dir=os.getcwd())
_, tmp_file_name = tempfile.mkstemp(dir=dir_name)
di = ck.DockerImage(
script_path=script_path,
dir_name=dir_name,
username="******",
pin_pip_versions=True,
)
assert di.name == op.splitext(op.basename(script_path))[0].replace("_", "-")
import_names = set([d["name"] for d in di.pip_imports])
assert import_names == correct_pip_imports
assert di.missing_imports == []
assert di.username == "unit-test-username"
assert di.func is None
assert di.build_path == dir_name
assert di.script_path == script_path
# Compare the created files with the reference files
correct_dir = op.join(data_path, "docker_reqs_ref_data", py_dir, "ref2")
correct_req_path = op.join(correct_dir, "requirements.txt")
correct_dockerfile = op.join(correct_dir, "Dockerfile")
with open(correct_req_path) as f:
correct_reqs = set([s.split("=")[0] for s in f.readlines()])
with open(di.req_path) as f:
created_reqs = set([s.split("=")[0] for s in f.readlines()])
assert created_reqs == correct_reqs
assert filecmp.cmp(di.docker_path, correct_dockerfile, shallow=False)
# Confirm that the docker image is in the config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert "docker-image " + di.name in config.sections()
# Assert ck.aws.CloudknotInputError on name plus other input
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage(name=di.name, script_path="Foo")
# Clobber and confirm that it deleted all the created files
di.clobber()
assert not op.isfile(di.req_path)
assert not op.isfile(di.docker_path)
# But since we had a pre-existing file in the build_path, it should not
# have deleted the build_path or the input python script
assert op.isfile(di.script_path)
assert op.isfile(tmp_file_name)
assert op.isdir(di.build_path)
# Now delete them to clean up after ourselves
os.remove(tmp_file_name)
os.rmdir(di.build_path)
# Assert that it was removed from the config file
# If we just re-read the config file, config will keep the union
# of the in memory values and the file values, updating the
# intersection of the two with the file values. So we must clear
# config and then re-read the file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert "docker-image " + di.name not in config.sections()
# Test for exception handling of incorrect input
# ----------------------------------------------
# Assert ck.aws.CloudknotInputError on no input
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage()
# Assert ck.aws.CloudknotInputError on non-string name input
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage(name=42)
# Assert ck.aws.CloudknotInputError on redundant input
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage(
func=unit_testing_func,
script_path=correct_script_path,
dir_name=os.getcwd(),
)
# Assert ck.aws.CloudknotInputError on invalid script path
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage(script_path=str(uuid.uuid4()), dir_name=os.getcwd())
# Assert ck.aws.CloudknotInputError on invalid dir name
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage(script_path=correct_script_path, dir_name=str(uuid.uuid4()))
correct_dir = op.join(data_path, "docker_reqs_ref_data", py_dir, "ref1")
# Assert CloudknotInputError to prevent overwriting existing script
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage(func=unit_testing_func, dir_name=correct_dir)
# Assert CloudknotInputError to prevent overwriting existing Dockerfile
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage(script_path=correct_script_path)
# Assert CloudknotInputError to prevent overwriting existing
# requirements.txt
# First, avoid the existing Dockerfile error by renaming the Dockerfile
old_dockerfile = op.join(op.dirname(correct_script_path), "Dockerfile")
new_dockerfile = op.join(op.dirname(correct_script_path), "tmpdockerfile")
os.rename(old_dockerfile, new_dockerfile)
# Assert the ck.aws.CloudknotInputError
with pytest.raises(ck.aws.CloudknotInputError):
ck.DockerImage(script_path=correct_script_path)
# Clean up our mess by renaming to the old Dockerfile
os.rename(new_dockerfile, old_dockerfile)
# Finally, test the build and push methods
# ----------------------------------------
# Make one last DockerImage instance with the simple_unit_testing_func
di = ck.DockerImage(func=simple_unit_testing_func)
# Create a repo to which to push this image
response = ecr.create_repository(repositoryName=get_testing_name())
repo_name = response["repository"]["repositoryName"]
repo_uri = response["repository"]["repositoryUri"]
repo = ck.aws.DockerRepo(name=repo_name)
# Assert ck.aws.CloudknotInputError on push without args
with pytest.raises(ck.aws.CloudknotInputError):
di.push()
# Assert ck.aws.CloudknotInputError on over-specified input
with pytest.raises(ck.aws.CloudknotInputError):
di.push(repo="input doesn't matter here", repo_uri=str(repo_uri))
# Assert ck.aws.CloudknotInputError on push before build
with pytest.raises(ck.aws.CloudknotInputError):
di.push(repo_uri=str(repo_uri))
# Assert ck.aws.CloudknotInputError on incorrect build args
with pytest.raises(ck.aws.CloudknotInputError):
di.build(tags=[42, -42])
# Assert ck.aws.CloudknotInputError on 'latest' in tags
with pytest.raises(ck.aws.CloudknotInputError):
di.build(tags=["testing", "latest"])
tags = ["testing", ["testing1", "testing2"]]
image_names = [None, "testing_image"]
for idx, (tag, n) in enumerate(zip(tags, image_names)):
di.build(tags=tag, image_name=n)
n = n if n else "cloudknot/" + di.name
if isinstance(tag, str):
tag = [tag]
images = [{"name": n, "tag": t} for t in tag]
for im in images:
assert im in di.images
if idx % 2:
di.push(repo_uri=str(repo_uri))
else:
di.push(repo=repo)
assert repo_uri in di.repo_uri
# Assert ck.aws.CloudknotInputError on push with invalid repo
with pytest.raises(ck.aws.CloudknotInputError):
di.push(repo=42)
# Assert ck.aws.CloudknotInputError on push with invalid repo_uri
with pytest.raises(ck.aws.CloudknotInputError):
di.push(repo_uri=42)
di.clobber()
# Assert error on build after clobber
with pytest.raises(ck.aws.ResourceClobberedException):
di.build(tags=["testing"])
# Assert ck.aws.CloudknotInputError on push with invalid repo_uri
with pytest.raises(ck.aws.ResourceClobberedException):
di.push(repo=repo)
except Exception as e:
# Get all local images with unit test prefix in any of the repo tags
c = docker.from_env().api
unit_test_images = [
im
for im in c.images()
if any(
("unit-testing-func" in tag or "test-func-input" in tag)
for tag in im["RepoTags"]
)
]
# Remove local images
for im in unit_test_images:
for tag in im["RepoTags"]:
c.remove_image(tag, force=True)
# Clean up config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
for name in list(config.sections()):
if name in [
"docker-image unit-testing-func",
"docker-image test-func-input",
]:
config.remove_section(name)
try:
section_name = "docker-repos" + ck.aws.get_region()
for option in config.options(section_name):
if UNIT_TEST_PREFIX in option:
config.remove_option(section_name, option)
except configparser.NoSectionError:
pass
with open(config_file, "w") as f:
config.write(f)
raise e
def test_pars_with_new_vpc(cleanup):
ck.refresh_clients()
name = get_testing_name()
p = ck.Pars(name=name, use_default_vpc=False)
response = ck.aws.clients["cloudformation"].describe_stacks(
StackName=name + "-pars"
)
stack_id = response.get("Stacks")[0]["StackId"]
assert stack_id == p.stack_id
response = ck.aws.clients["iam"].list_roles()
response = ck.aws.clients["iam"].get_role(RoleName=name + "-batch-service-role")
bsr_arn = response.get("Role")["Arn"]
assert bsr_arn == p.batch_service_role
response = ck.aws.clients["iam"].get_role(RoleName=name + "-ecs-instance-role")
ecs_arn = response.get("Role")["Arn"]
assert ecs_arn == p.ecs_instance_role
response = ck.aws.clients["iam"].get_role(RoleName=name + "-spot-fleet-role")
sfr_arn = response.get("Role")["Arn"]
assert sfr_arn == p.spot_fleet_role
response = ck.aws.clients["iam"].list_instance_profiles_for_role(
RoleName=name + "-ecs-instance-role"
)
ecs_profile_arn = response.get("InstanceProfiles")[0]["Arn"]
assert ecs_profile_arn == p.ecs_instance_profile
# Check for a VPC with the tag "Name: name"
response = ck.aws.clients["ec2"].describe_tags(
Filters=[
{"Name": "resource-type", "Values": ["vpc"]},
{"Name": "key", "Values": ["Name"]},
{"Name": "value", "Values": [p.name]},
]
)
vpc_id = response.get("Tags")[0]["ResourceId"]
assert vpc_id == p.vpc
response = ck.aws.clients["ec2"].describe_subnets(
Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
)
subnet_ids = [d["SubnetId"] for d in response.get("Subnets")]
assert set(subnet_ids) == set(p.subnets)
response = ck.aws.clients["ec2"].describe_security_groups(
Filters=[
{"Name": "vpc-id", "Values": [vpc_id]},
{"Name": "tag-key", "Values": ["Name"]},
{"Name": "tag-value", "Values": [p.name]},
]
)
sg_id = response.get("SecurityGroups")[0]["GroupId"]
assert sg_id == p.security_group
# Now, confirm input validation for pre-existing PARS
with pytest.raises(ck.aws.CloudknotInputError):
ck.Pars(name=name, batch_service_role_name="error-test")
p = ck.Pars(name=name)
assert stack_id == p.stack_id
assert bsr_arn == p.batch_service_role
assert ecs_arn == p.ecs_instance_role
assert sfr_arn == p.spot_fleet_role
assert ecs_profile_arn == p.ecs_instance_profile
assert vpc_id == p.vpc
assert set(subnet_ids) == set(p.subnets)
assert sg_id == p.security_group
p.clobber()
assert p.clobbered
# Clobbering twice shouldn't be a problem
p.clobber()
response = ck.aws.clients["cloudformation"].describe_stacks(StackName=stack_id)
status = response.get("Stacks")[0]["StackStatus"]
assert status in ["DELETE_IN_PROGRESS", "DELETE_COMPLETE"]
waiter = ck.aws.clients["cloudformation"].get_waiter("stack_delete_complete")
waiter.wait(StackName=stack_id, WaiterConfig={"Delay": 10})
# Confirm that clobber deleted the stack from the config file
config_file = ck.config.get_config_file()
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert p.pars_name not in config.sections()
name = get_testing_name()
instance_tenancy = "dedicated"
cidr = "172.32.0.0/16"
p = ck.Pars(
name=name,
use_default_vpc=False,
ipv4_cidr=cidr,
instance_tenancy=instance_tenancy,
)
response = ck.aws.clients["ec2"].describe_vpcs(VpcIds=[p.vpc])
assert instance_tenancy == response.get("Vpcs")[0]["InstanceTenancy"]
assert cidr == response.get("Vpcs")[0]["CidrBlock"]
ck.aws.clients["cloudformation"].delete_stack(StackName=p.stack_id)
# Change the stack-id in the config file to get an error
config_file = ck.config.get_config_file()
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
stack_id = config.get(p.pars_name, "stack-id")
stack_id = stack_id.split("/")
stack_id[1] = get_testing_name()
stack_id = "/".join(stack_id)
config.set(p.pars_name, "stack-id", stack_id)
with open(config_file, "w") as f:
config.write(f)
# Confirm error on retrieving the nonexistent stack
with pytest.raises(ck.aws.ResourceDoesNotExistException) as e:
ck.Pars(name=name)
assert e.value.resource_id == stack_id
# Confirm that the previous error deleted the stack from the config file
config_file = ck.config.get_config_file()
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert p.pars_name not in config.sections()
def test_pars_with_default_vpc(cleanup):
ck.refresh_clients()
name = get_testing_name()
batch_service_role_name = "ck-unit-test-batch-service-role-1"
ecs_instance_role_name = "ck-unit-test-ecs-instance-role-1"
spot_fleet_role_name = "ck-unit-test-spot-fleet-role-1"
try:
p = ck.Pars(
name=name,
batch_service_role_name=batch_service_role_name,
ecs_instance_role_name=ecs_instance_role_name,
spot_fleet_role_name=spot_fleet_role_name,
)
response = ck.aws.clients["cloudformation"].describe_stacks(
StackName=name + "-pars"
)
stack_id = response.get("Stacks")[0]["StackId"]
assert stack_id == p.stack_id
response = ck.aws.clients["iam"].get_role(RoleName=batch_service_role_name)
bsr_arn = response.get("Role")["Arn"]
assert bsr_arn == p.batch_service_role
response = ck.aws.clients["iam"].get_role(RoleName=ecs_instance_role_name)
ecs_arn = response.get("Role")["Arn"]
assert ecs_arn == p.ecs_instance_role
response = ck.aws.clients["iam"].get_role(RoleName=spot_fleet_role_name)
sfr_arn = response.get("Role")["Arn"]
assert sfr_arn == p.spot_fleet_role
response = ck.aws.clients["iam"].list_instance_profiles_for_role(
RoleName=ecs_instance_role_name
)
ecs_profile_arn = response.get("InstanceProfiles")[0]["Arn"]
assert ecs_profile_arn == p.ecs_instance_profile
# Check for a default VPC
response = ck.aws.clients["ec2"].describe_vpcs(
Filters=[{"Name": "isDefault", "Values": ["true"]}]
)
vpc_id = response.get("Vpcs")[0]["VpcId"]
assert vpc_id == p.vpc
response = ck.aws.clients["ec2"].describe_subnets(
Filters=[{"Name": "vpc-id", "Values": [vpc_id]}]
)
subnet_ids = [d["SubnetId"] for d in response.get("Subnets")]
assert set(subnet_ids) == set(p.subnets)
response = ck.aws.clients["ec2"].describe_security_groups(
Filters=[
{"Name": "vpc-id", "Values": [vpc_id]},
{"Name": "tag-key", "Values": ["Name"]},
{"Name": "tag-value", "Values": [p.name]},
]
)
sg_id = response.get("SecurityGroups")[0]["GroupId"]
assert sg_id == p.security_group
# Delete the stack using boto3 to check for an error from Pars
# on reinstantiation
ck.aws.clients["cloudformation"].delete_stack(StackName=p.stack_id)
waiter = ck.aws.clients["cloudformation"].get_waiter("stack_delete_complete")
waiter.wait(StackName=p.stack_id, WaiterConfig={"Delay": 10})
# Confirm error on retrieving the deleted stack
with pytest.raises(ck.aws.ResourceDoesNotExistException) as e:
ck.Pars(name=name)
assert e.value.resource_id == p.stack_id
# Confirm that the previous error deleted
# the stack from the config file
config_file = ck.config.get_config_file()
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert p.pars_name not in config.sections()
except ck.aws.CannotCreateResourceException:
# Cannot create a default VPC in this account
# Ignore test
pass
def test_DockerRepo(bucket_cleanup):
ck.refresh_clients()
ecr = ck.aws.clients["ecr"]
config = configparser.ConfigParser()
config_file = ck.config.get_config_file()
repo_section_name = "docker-repos " + ck.get_profile(
) + " " + ck.get_region()
try:
name = get_testing_name()
# Use boto3 to create an ECR repo
response = ecr.create_repository(repositoryName=name)
repo_name = response["repository"]["repositoryName"]
repo_uri = response["repository"]["repositoryUri"]
repo_registry_id = response["repository"]["registryId"]
# Retrieve that same repo with cloudknot
dr = ck.aws.DockerRepo(name=name)
assert dr.name == repo_name
assert dr.repo_uri == repo_uri
assert dr.repo_registry_id == repo_registry_id
# Confirm that the docker repo is in the config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert name in config.options(repo_section_name)
# Clobber the docker repo
dr.clobber()
retry = tenacity.Retrying(
wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(180),
retry=tenacity.retry_unless_exception_type((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)),
)
# Assert that it was removed from AWS
with pytest.raises((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)):
retry.call(ecr.describe_repositories, repositoryNames=[name])
# Assert that it was removed from the config file
# If we just re-read the config file, config will keep the union
# of the in memory values and the file values, updating the
# intersection of the two with the file values. So we must clear
# config and then re-read the file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert name not in config.options(repo_section_name)
# Now create a new repo using only cloudknot
name = get_testing_name()
dr = ck.aws.DockerRepo(name=name)
# Confirm that it exists on AWS and retrieve its properties
retry = tenacity.Retrying(
wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(60),
retry=tenacity.retry_if_exception_type((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)),
)
response = retry.call(ecr.describe_repositories,
repositoryNames=[name])
repo_name = response["repositories"][0]["repositoryName"]
repo_uri = response["repositories"][0]["repositoryUri"]
repo_registry_id = response["repositories"][0]["registryId"]
assert dr.name == repo_name
assert dr.repo_uri == repo_uri
assert dr.repo_registry_id == repo_registry_id
# Confirm that the docker repo is in the config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert name in config.options(repo_section_name)
# Delete the repo from AWS before clobbering
ecr.delete_repository(registryId=repo_registry_id,
repositoryName=repo_name,
force=True)
# Clobber the docker repo
dr.clobber()
retry = tenacity.Retrying(
wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(180),
retry=tenacity.retry_unless_exception_type((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)),
)
# Assert that it was removed from AWS
with pytest.raises((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)):
retry.call(ecr.describe_repositories, repositoryNames=[name])
# Assert that it was removed from the config file
# If we just re-read the config file, config will keep the union
# of the in memory values and the file values, updating the
# intersection of the two with the file values. So we must clear
# config and then re-read the file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert name not in config.options(repo_section_name)
except Exception as e:
response = ecr.describe_repositories()
# Get all repos with unit test prefix in the name
repos = [
r for r in response.get("repositories")
if UNIT_TEST_PREFIX in r["repositoryName"]
]
# Delete the AWS ECR repo
for r in repos:
ecr.delete_repository(
registryId=r["registryId"],
repositoryName=r["repositoryName"],
force=True,
)
# Clean up config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
try:
for name in config.options(repo_section_name):
if UNIT_TEST_PREFIX in name:
config.remove_option(repo_section_name, name)
except configparser.NoSectionError:
pass
with open(config_file, "w") as f:
config.write(f)
raise e
def test_get_region(bucket_cleanup):
ck.refresh_clients()
# Save environment variables for restoration later
try:
old_region_env = os.environ["AWS_DEFAULT_REGION"]
except KeyError:
old_region_env = None
old_region = ck.get_region()
try:
old_config_file = os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
old_config_file = None
try:
# With empty config file, get_region should return the
# environment variable AWS_DEFAULT_REGION
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
region = "test-region-0"
os.environ["AWS_DEFAULT_REGION"] = region
assert ck.get_region() == region
del os.environ["AWS_DEFAULT_REGION"]
# With region in a temporary config file, region should simply
# read the config file
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
region = "test-region-1"
tmp.file.write("[aws]\n")
tmp.file.write("region = {region:s}\n".format(region=region))
tmp.file.flush()
os.fsync(tmp.file.fileno())
assert ck.get_region() == region
# With no cloudknot config file and no environment variable
# get_region should return region in aws config file
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
aws_config_file = op.join(op.expanduser("~"), ".aws", "config")
try:
if op.isfile(aws_config_file):
if op.isfile(aws_config_file + ".bak"):
raise Exception(
"Backup aws config file already exists.")
shutil.move(aws_config_file, aws_config_file + ".bak")
assert ck.get_region() == "us-east-1"
finally:
if op.isfile(aws_config_file + ".bak"):
shutil.move(aws_config_file + ".bak", aws_config_file)
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
aws_config_file = op.join(op.expanduser("~"), ".aws", "config")
try:
if op.isfile(aws_config_file):
if op.isfile(aws_config_file + ".bak"):
raise Exception(
"Backup aws config file already exists.")
shutil.move(aws_config_file, aws_config_file + ".bak")
else:
# Create the config directory if it doesn't exist
aws_config_dir = op.dirname(aws_config_file)
try:
os.makedirs(aws_config_dir)
except OSError as e:
pre_existing = e.errno == errno.EEXIST and op.isdir(
aws_config_dir)
if pre_existing:
pass
else:
raise e
region = "test-region-2"
with open(aws_config_file, "w") as f:
f.write("[default]\n")
f.write("region = {region:s}\n".format(region=region))
f.flush()
os.fsync(f.fileno())
assert ck.get_region() == region
finally:
if op.isfile(aws_config_file + ".bak"):
shutil.move(aws_config_file + ".bak", aws_config_file)
elif op.isfile(aws_config_file):
os.remove(aws_config_file)
finally:
ck.set_region(old_region)
# Restore old environment variables
if old_config_file:
os.environ["CLOUDKNOT_CONFIG_FILE"] = old_config_file
else:
try:
del os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
pass
if old_region_env:
os.environ["AWS_DEFAULT_REGION"] = old_region_env
else:
try:
del os.environ["AWS_DEFAULT_REGION"]
except KeyError:
pass
ck.refresh_clients()
def test_get_profile(bucket_cleanup):
try:
old_credentials_file = os.environ['AWS_SHARED_CREDENTIALS_FILE']
except KeyError:
old_credentials_file = None
try:
old_aws_config_file = os.environ['AWS_CONFIG_FILE']
except KeyError:
old_aws_config_file = None
try:
old_ck_config_file = os.environ['CLOUDKNOT_CONFIG_FILE']
except KeyError:
old_ck_config_file = None
ref_dir = op.join(data_path, 'profiles_ref_data')
ck_config_with_profile = op.join(ref_dir, 'cloudknot_with_profile')
ck_config_without_profile = op.join(ref_dir, 'cloudknot_without_profile')
shutil.copy(ck_config_with_profile, ck_config_with_profile + '.bak')
shutil.copy(ck_config_without_profile, ck_config_without_profile + '.bak')
try:
os.environ['CLOUDKNOT_CONFIG_FILE'] = ck_config_with_profile
assert ck.get_profile() == 'profile_from_cloudknot_config'
os.environ['CLOUDKNOT_CONFIG_FILE'] = ck_config_without_profile
config_file = op.join(ref_dir, 'config')
os.environ['AWS_CONFIG_FILE'] = config_file
cred_file = op.join(ref_dir, 'credentials_without_default')
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = cred_file
assert ck.get_profile(fallback=None) is None
assert ck.get_profile() == 'from-env'
cred_file = op.join(ref_dir, 'credentials_with_default')
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = cred_file
assert ck.get_profile() == 'default'
finally:
shutil.move(ck_config_with_profile + '.bak', ck_config_with_profile)
shutil.move(ck_config_without_profile + '.bak',
ck_config_without_profile)
if old_credentials_file:
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = old_credentials_file
else:
try:
del os.environ['AWS_SHARED_CREDENTIALS_FILE']
except KeyError:
pass
if old_aws_config_file:
os.environ['AWS_CONFIG_FILE'] = old_aws_config_file
else:
try:
del os.environ['AWS_CONFIG_FILE']
except KeyError:
pass
if old_ck_config_file:
os.environ['CLOUDKNOT_CONFIG_FILE'] = old_ck_config_file
else:
try:
del os.environ['CLOUDKNOT_CONFIG_FILE']
except KeyError:
pass
ck.refresh_clients()
def test_get_region(bucket_cleanup):
# Save environment variables for restoration later
try:
old_region_env = os.environ['AWS_DEFAULT_REGION']
except KeyError:
old_region_env = None
old_region = ck.get_region()
try:
old_config_file = os.environ['CLOUDKNOT_CONFIG_FILE']
except KeyError:
old_config_file = None
try:
# With empty config file, get_region should return the
# environment variable AWS_DEFAULT_REGION
with tempfile.NamedTemporaryFile(mode='w+') as tmp:
os.environ['CLOUDKNOT_CONFIG_FILE'] = tmp.name
region = 'test-region-0'
os.environ['AWS_DEFAULT_REGION'] = region
assert ck.get_region() == region
del os.environ['AWS_DEFAULT_REGION']
# With region in a temporary config file, region should simply
# read the config file
with tempfile.NamedTemporaryFile(mode='w+') as tmp:
os.environ['CLOUDKNOT_CONFIG_FILE'] = tmp.name
region = 'test-region-1'
tmp.file.write('[aws]\n')
tmp.file.write('region = {region:s}\n'.format(region=region))
tmp.file.flush()
os.fsync(tmp.file.fileno())
assert ck.get_region() == region
# With no cloudknot config file and no environment variable
# get_region should return region in aws config file
with tempfile.NamedTemporaryFile(mode='w+') as tmp:
os.environ['CLOUDKNOT_CONFIG_FILE'] = tmp.name
aws_config_file = op.join(op.expanduser('~'), '.aws', 'config')
try:
if op.isfile(aws_config_file):
if op.isfile(aws_config_file + '.bak'):
raise Exception(
'Backup aws config file already exists.')
shutil.move(aws_config_file, aws_config_file + '.bak')
assert ck.get_region() == 'us-east-1'
finally:
if op.isfile(aws_config_file + '.bak'):
shutil.move(aws_config_file + '.bak', aws_config_file)
with tempfile.NamedTemporaryFile(mode='w+') as tmp:
os.environ['CLOUDKNOT_CONFIG_FILE'] = tmp.name
aws_config_file = op.join(op.expanduser('~'), '.aws', 'config')
try:
if op.isfile(aws_config_file):
if op.isfile(aws_config_file + '.bak'):
raise Exception(
'Backup aws config file already exists.')
shutil.move(aws_config_file, aws_config_file + '.bak')
else:
# Create the config directory if it doesn't exist
aws_config_dir = op.dirname(aws_config_file)
try:
os.makedirs(aws_config_dir)
except OSError as e:
pre_existing = (e.errno == errno.EEXIST
and op.isdir(aws_config_dir))
if pre_existing:
pass
else:
raise e
region = 'test-region-2'
with open(aws_config_file, 'w') as f:
f.write('[default]\n')
f.write('region = {region:s}\n'.format(region=region))
f.flush()
os.fsync(f.fileno())
assert ck.get_region() == region
finally:
if op.isfile(aws_config_file + '.bak'):
shutil.move(aws_config_file + '.bak', aws_config_file)
elif op.isfile(aws_config_file):
os.remove(aws_config_file)
finally:
ck.set_region(old_region)
# Restore old environment variables
if old_config_file:
os.environ['CLOUDKNOT_CONFIG_FILE'] = old_config_file
else:
try:
del os.environ['CLOUDKNOT_CONFIG_FILE']
except KeyError:
pass
if old_region_env:
os.environ['AWS_DEFAULT_REGION'] = old_region_env
else:
try:
del os.environ['AWS_DEFAULT_REGION']
except KeyError:
pass
ck.refresh_clients()
def test_knot(cleanup_repos):
config_file = ck.config.get_config_file()
knot = None
ck.refresh_clients()
try:
ec2 = ck.aws.clients["ec2"]
instance = ec2.run_instances(MaxCount=1, MinCount=1)["Instances"][0]
ec2.create_image(
BlockDeviceMappings=[{
"DeviceName": "/dev/xvda",
"Ebs": {
"DeleteOnTermination": True,
"VolumeSize": 30,
"VolumeType": "gp2",
"Encrypted": False,
},
}],
Description="amazon linux ami 2 x86_64 ecs hvm gp2",
Name="unit-test-ecs-optimized-ami",
InstanceId=instance["InstanceId"],
)
pars = ck.Pars(name=get_testing_name(), use_default_vpc=False)
name = get_testing_name()
knot = ck.Knot(name=name, pars=pars, func=unit_testing_func)
# Now remove the images and repo-uri from the docker-image
# Forcing the next call to Knot to rebuild and re-push the image.
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
config.set("docker-image " + knot.docker_image.name, "images", "")
config.set("docker-image " + knot.docker_image.name, "repo-uri",
"")
with open(config_file, "w") as f:
config.write(f)
# Re-instantiate the knot so that it retrieves from config
# with AWS resources that already exist
knot = ck.Knot(name=name)
knot.docker_image._clobber_script = True
# Assert properties are as expected
assert knot.name == name
assert knot.knot_name == "knot " + name
assert knot.pars.name == pars.name
func_name = unit_testing_func.__name__.replace("_", "-")
assert knot.docker_image.name == func_name
assert knot.docker_repo.name == "cloudknot"
pre = name + "-ck-"
assert knot.job_definition.name == pre + "jd"
# Delete the stack using boto3 to check for an error from Pars
# on reinstantiation
ck.aws.clients["cloudformation"].delete_stack(StackName=knot.stack_id)
waiter = ck.aws.clients["cloudformation"].get_waiter(
"stack_delete_complete")
waiter.wait(StackName=knot.stack_id, WaiterConfig={"Delay": 10})
# Confirm error on retrieving the deleted stack
with pytest.raises(ck.aws.ResourceDoesNotExistException) as e:
ck.Knot(name=name)
assert e.value.resource_id == knot.stack_id
# Confirm that the previous error deleted
# the stack from the config file
config_file = ck.config.get_config_file()
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert knot.knot_name not in config.sections()
name = get_testing_name()
knot = ck.Knot(name=name, func=unit_testing_func)
knot.docker_image._clobber_script = True
knot.clobber(clobber_pars=True, clobber_image=True, clobber_repo=True)
assert knot.clobbered
# Clobbering twice shouldn't be a problem
knot.clobber()
response = ck.aws.clients["cloudformation"].describe_stacks(
StackName=knot.stack_id)
status = response.get("Stacks")[0]["StackStatus"]
assert status in ["DELETE_IN_PROGRESS", "DELETE_COMPLETE"]
waiter = ck.aws.clients["cloudformation"].get_waiter(
"stack_delete_complete")
waiter.wait(StackName=knot.stack_id, WaiterConfig={"Delay": 10})
# Confirm that clobber deleted the stack from the config file
config_file = ck.config.get_config_file()
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert knot.knot_name not in config.sections()
except Exception as e:
try:
if knot:
knot.clobber(clobber_pars=True,
clobber_image=True,
clobber_repo=True)
except Exception:
pass
raise e
def test_knot_errors(cleanup_repos):
ck.refresh_clients()
# Test Exceptions on invalid input
# --------------------------------
# Assert ck.aws.CloudknotInputError on invalid name
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(name=42)
# Assert ck.aws.CloudknotInputError on long name
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(name="a" * 56)
# Assert ck.aws.CloudknotInputError on invalid pars input
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(func=unit_testing_func, pars=42)
# Assert ck.aws.CloudknotInputError on redundant docker_image and func
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(func=unit_testing_func, docker_image=42)
# Assert ck.aws.CloudknotInputError on invalid docker_image input
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(docker_image=42)
# Assert ck.aws.CloudknotInputError on invalid retries
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(retries=0)
# Assert ck.aws.CloudknotInputError on invalid retries
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(retries=11)
# Assert ck.aws.CloudknotInputError on invalid memory
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(memory=0)
# Assert ck.aws.CloudknotInputError on invalid job_def_vcpus
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(job_def_vcpus=-42)
# Assert ck.aws.CloudknotInputError on invalid priority
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(priority=-42)
# Assert ck.aws.CloudknotInputError on invalid min_vcpus
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(min_vcpus=-1)
# Assert ck.aws.CloudknotInputError on invalid desired_vcpus
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(desired_vcpus=-1)
# Assert ck.aws.CloudknotInputError on invalid max_vcpus
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(max_vcpus=-1)
# Assert ck.aws.CloudknotInputError on invalid instance_types
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(instance_types=[42])
# Assert ck.aws.CloudknotInputError on invalid instance_types
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(instance_types="not a valid instance")
# Assert ck.aws.CloudknotInputError on invalid instance_types
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(instance_types=["not", "a", "valid", "instance"])
# Assert ck.aws.CloudknotInputError on invalid image_id
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(image_id=42)
# Assert ck.aws.CloudknotInputError on invalid ec2_key_pair
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(ec2_key_pair=42)
# Assert ck.aws.CloudknotInputError on invalid volume_size
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(volume_size="string")
# Assert ck.aws.CloudknotInputError on volume_size < 1
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(volume_size=0)
# Assert ck.aws.CloudknotInputError when providing both image_id and volume_size
with pytest.raises(ck.aws.CloudknotInputError):
ck.Knot(image_id="test-string", volume_size=30)
def test_get_profile(bucket_cleanup):
try:
old_credentials_file = os.environ["AWS_SHARED_CREDENTIALS_FILE"]
except KeyError:
old_credentials_file = None
try:
old_aws_config_file = os.environ["AWS_CONFIG_FILE"]
except KeyError:
old_aws_config_file = None
try:
old_ck_config_file = os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
old_ck_config_file = None
ref_dir = op.join(data_path, "profiles_ref_data")
ck_config_with_profile = op.join(ref_dir, "cloudknot_with_profile")
ck_config_without_profile = op.join(ref_dir, "cloudknot_without_profile")
shutil.copy(ck_config_with_profile, ck_config_with_profile + ".bak")
shutil.copy(ck_config_without_profile, ck_config_without_profile + ".bak")
try:
os.environ["CLOUDKNOT_CONFIG_FILE"] = ck_config_with_profile
assert ck.get_profile() == "profile_from_cloudknot_config"
os.environ["CLOUDKNOT_CONFIG_FILE"] = ck_config_without_profile
config_file = op.join(ref_dir, "config")
os.environ["AWS_CONFIG_FILE"] = config_file
cred_file = op.join(ref_dir, "credentials_without_default")
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = cred_file
assert ck.get_profile(fallback=None) is None
assert ck.get_profile() == "from-env"
cred_file = op.join(ref_dir, "credentials_with_default")
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = cred_file
assert ck.get_profile() == "default"
finally:
shutil.move(ck_config_with_profile + ".bak", ck_config_with_profile)
shutil.move(ck_config_without_profile + ".bak",
ck_config_without_profile)
if old_credentials_file:
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = old_credentials_file
else:
try:
del os.environ["AWS_SHARED_CREDENTIALS_FILE"]
except KeyError:
pass
if old_aws_config_file:
os.environ["AWS_CONFIG_FILE"] = old_aws_config_file
else:
try:
del os.environ["AWS_CONFIG_FILE"]
except KeyError:
pass
if old_ck_config_file:
os.environ["CLOUDKNOT_CONFIG_FILE"] = old_ck_config_file
else:
try:
del os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
pass
ck.refresh_clients()
