def cleanup_repos(bucket_cleanup):
yield None
ecr = ck.aws.clients["ecr"]
config_file = ck.config.get_config_file()
section_suffix = ck.get_profile() + " " + ck.get_region()
repos_section_name = "docker-repos " + section_suffix
# Clean up repos from AWS
# -----------------------
# Get all repos with unit test prefix in the name
response = ecr.describe_repositories()
repos = [
r for r in response.get("repositories")
if ("unit_testing_func" in r["repositoryName"] or "test_func_input" in
r["repositoryName"] or "simple_unit_testing_func" in
r["repositoryName"] or UNIT_TEST_PREFIX in r["repositoryName"])
]
# Delete the AWS ECR repo
for r in repos:
ecr.delete_repository(registryId=r["registryId"],
repositoryName=r["repositoryName"],
force=True)
# Clean up repos from config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
for repo_name in config.options(repos_section_name):
if UNIT_TEST_PREFIX in repo_name:
config.remove_option(repos_section_name, repo_name)
with open(config_file, "w") as f:
config.write(f)
def cleanup_repos(bucket_cleanup):
yield None
ecr = ck.aws.clients['ecr']
config_file = ck.config.get_config_file()
section_suffix = ck.get_profile() + ' ' + ck.get_region()
repos_section_name = 'docker-repos ' + section_suffix
# Clean up repos from AWS
# -----------------------
# Get all repos with unit test prefix in the name
response = ecr.describe_repositories()
repos = [r for r in response.get('repositories')
if ('unit-testing-func' in r['repositoryName']
or 'test-func-input' in r['repositoryName']
or 'simple-unit-testing-func' in r['repositoryName']
or UNIT_TEST_PREFIX in r['repositoryName'])]
# Delete the AWS ECR repo
for r in repos:
ecr.delete_repository(
registryId=r['registryId'],
repositoryName=r['repositoryName'],
force=True
)
# Clean up repos from config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
for repo_name in config.options(repos_section_name):
if UNIT_TEST_PREFIX in repo_name:
config.remove_option(repos_section_name, repo_name)
with open(config_file, 'w') as f:
config.write(f)
def test_set_profile(bucket_cleanup):
old_credentials_file = os.environ.get("AWS_SHARED_CREDENTIALS_FILE")
old_aws_config_file = os.environ.get("AWS_CONFIG_FILE")
old_ck_config_file = os.environ.get("CLOUDKNOT_CONFIG_FILE")
ref_dir = op.join(data_path, "profiles_ref_data")
ck_config_file = op.join(ref_dir, "cloudknot_without_profile")
shutil.copy(ck_config_file, ck_config_file + ".bak")
try:
os.environ["CLOUDKNOT_CONFIG_FILE"] = ck_config_file
config_file = op.join(ref_dir, "config")
os.environ["AWS_CONFIG_FILE"] = config_file
cred_file = op.join(ref_dir, "credentials_without_default")
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = cred_file
with pytest.raises(ck.aws.CloudknotInputError):
ck.set_profile(profile_name="not_in_list_of_profiles")
profile = "name-5"
ck.set_profile(profile_name=profile)
assert ck.get_profile() == profile
finally:
shutil.move(ck_config_file + ".bak", ck_config_file)
if old_credentials_file:
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = old_credentials_file
else:
os.environ.pop("AWS_SHARED_CREDENTIALS_FILE")
if old_aws_config_file:
os.environ["AWS_CONFIG_FILE"] = old_aws_config_file
else:
os.environ.pop("AWS_CONFIG_FILE")
if old_ck_config_file:
os.environ["CLOUDKNOT_CONFIG_FILE"] = old_ck_config_file
else:
os.environ.pop("CLOUDKNOT_CONFIG_FILE")
ck.refresh_clients()
def test_DockerRepo(bucket_cleanup):
ck.refresh_clients()
ecr = ck.aws.clients["ecr"]
config = configparser.ConfigParser()
config_file = ck.config.get_config_file()
repo_section_name = "docker-repos " + ck.get_profile(
) + " " + ck.get_region()
try:
name = get_testing_name()
# Use boto3 to create an ECR repo
response = ecr.create_repository(repositoryName=name)
repo_name = response["repository"]["repositoryName"]
repo_uri = response["repository"]["repositoryUri"]
repo_registry_id = response["repository"]["registryId"]
# Retrieve that same repo with cloudknot
dr = ck.aws.DockerRepo(name=name)
assert dr.name == repo_name
assert dr.repo_uri == repo_uri
assert dr.repo_registry_id == repo_registry_id
# Confirm that the docker repo is in the config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert name in config.options(repo_section_name)
# Clobber the docker repo
dr.clobber()
retry = tenacity.Retrying(
wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(180),
retry=tenacity.retry_unless_exception_type((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)),
)
# Assert that it was removed from AWS
with pytest.raises((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)):
retry.call(ecr.describe_repositories, repositoryNames=[name])
# Assert that it was removed from the config file
# If we just re-read the config file, config will keep the union
# of the in memory values and the file values, updating the
# intersection of the two with the file values. So we must clear
# config and then re-read the file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert name not in config.options(repo_section_name)
# Now create a new repo using only cloudknot
name = get_testing_name()
dr = ck.aws.DockerRepo(name=name)
# Confirm that it exists on AWS and retrieve its properties
retry = tenacity.Retrying(
wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(60),
retry=tenacity.retry_if_exception_type((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)),
)
response = retry.call(ecr.describe_repositories,
repositoryNames=[name])
repo_name = response["repositories"][0]["repositoryName"]
repo_uri = response["repositories"][0]["repositoryUri"]
repo_registry_id = response["repositories"][0]["registryId"]
assert dr.name == repo_name
assert dr.repo_uri == repo_uri
assert dr.repo_registry_id == repo_registry_id
# Confirm that the docker repo is in the config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert name in config.options(repo_section_name)
# Delete the repo from AWS before clobbering
ecr.delete_repository(registryId=repo_registry_id,
repositoryName=repo_name,
force=True)
# Clobber the docker repo
dr.clobber()
retry = tenacity.Retrying(
wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(180),
retry=tenacity.retry_unless_exception_type((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)),
)
# Assert that it was removed from AWS
with pytest.raises((
ecr.exceptions.RepositoryNotFoundException,
botocore.exceptions.ClientError,
)):
retry.call(ecr.describe_repositories, repositoryNames=[name])
# Assert that it was removed from the config file
# If we just re-read the config file, config will keep the union
# of the in memory values and the file values, updating the
# intersection of the two with the file values. So we must clear
# config and then re-read the file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
assert name not in config.options(repo_section_name)
except Exception as e:
response = ecr.describe_repositories()
# Get all repos with unit test prefix in the name
repos = [
r for r in response.get("repositories")
if UNIT_TEST_PREFIX in r["repositoryName"]
]
# Delete the AWS ECR repo
for r in repos:
ecr.delete_repository(
registryId=r["registryId"],
repositoryName=r["repositoryName"],
force=True,
)
# Clean up config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
try:
for name in config.options(repo_section_name):
if UNIT_TEST_PREFIX in name:
config.remove_option(repo_section_name, name)
except configparser.NoSectionError:
pass
with open(config_file, "w") as f:
config.write(f)
raise e
def test_get_profile(bucket_cleanup):
try:
old_credentials_file = os.environ["AWS_SHARED_CREDENTIALS_FILE"]
except KeyError:
old_credentials_file = None
try:
old_aws_config_file = os.environ["AWS_CONFIG_FILE"]
except KeyError:
old_aws_config_file = None
try:
old_ck_config_file = os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
old_ck_config_file = None
ref_dir = op.join(data_path, "profiles_ref_data")
ck_config_with_profile = op.join(ref_dir, "cloudknot_with_profile")
ck_config_without_profile = op.join(ref_dir, "cloudknot_without_profile")
shutil.copy(ck_config_with_profile, ck_config_with_profile + ".bak")
shutil.copy(ck_config_without_profile, ck_config_without_profile + ".bak")
try:
os.environ["CLOUDKNOT_CONFIG_FILE"] = ck_config_with_profile
assert ck.get_profile() == "profile_from_cloudknot_config"
os.environ["CLOUDKNOT_CONFIG_FILE"] = ck_config_without_profile
config_file = op.join(ref_dir, "config")
os.environ["AWS_CONFIG_FILE"] = config_file
cred_file = op.join(ref_dir, "credentials_without_default")
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = cred_file
assert ck.get_profile(fallback=None) is None
assert ck.get_profile() == "from-env"
cred_file = op.join(ref_dir, "credentials_with_default")
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = cred_file
assert ck.get_profile() == "default"
finally:
shutil.move(ck_config_with_profile + ".bak", ck_config_with_profile)
shutil.move(ck_config_without_profile + ".bak",
ck_config_without_profile)
if old_credentials_file:
os.environ["AWS_SHARED_CREDENTIALS_FILE"] = old_credentials_file
else:
try:
del os.environ["AWS_SHARED_CREDENTIALS_FILE"]
except KeyError:
pass
if old_aws_config_file:
os.environ["AWS_CONFIG_FILE"] = old_aws_config_file
else:
try:
del os.environ["AWS_CONFIG_FILE"]
except KeyError:
pass
if old_ck_config_file:
os.environ["CLOUDKNOT_CONFIG_FILE"] = old_ck_config_file
else:
try:
del os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
pass
ck.refresh_clients()
def test_get_profile(bucket_cleanup):
try:
old_credentials_file = os.environ['AWS_SHARED_CREDENTIALS_FILE']
except KeyError:
old_credentials_file = None
try:
old_aws_config_file = os.environ['AWS_CONFIG_FILE']
except KeyError:
old_aws_config_file = None
try:
old_ck_config_file = os.environ['CLOUDKNOT_CONFIG_FILE']
except KeyError:
old_ck_config_file = None
ref_dir = op.join(data_path, 'profiles_ref_data')
ck_config_with_profile = op.join(ref_dir, 'cloudknot_with_profile')
ck_config_without_profile = op.join(ref_dir, 'cloudknot_without_profile')
shutil.copy(ck_config_with_profile, ck_config_with_profile + '.bak')
shutil.copy(ck_config_without_profile, ck_config_without_profile + '.bak')
try:
os.environ['CLOUDKNOT_CONFIG_FILE'] = ck_config_with_profile
assert ck.get_profile() == 'profile_from_cloudknot_config'
os.environ['CLOUDKNOT_CONFIG_FILE'] = ck_config_without_profile
config_file = op.join(ref_dir, 'config')
os.environ['AWS_CONFIG_FILE'] = config_file
cred_file = op.join(ref_dir, 'credentials_without_default')
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = cred_file
assert ck.get_profile(fallback=None) is None
assert ck.get_profile() == 'from-env'
cred_file = op.join(ref_dir, 'credentials_with_default')
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = cred_file
assert ck.get_profile() == 'default'
finally:
shutil.move(ck_config_with_profile + '.bak', ck_config_with_profile)
shutil.move(ck_config_without_profile + '.bak',
ck_config_without_profile)
if old_credentials_file:
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = old_credentials_file
else:
try:
del os.environ['AWS_SHARED_CREDENTIALS_FILE']
except KeyError:
pass
if old_aws_config_file:
os.environ['AWS_CONFIG_FILE'] = old_aws_config_file
else:
try:
del os.environ['AWS_CONFIG_FILE']
except KeyError:
pass
if old_ck_config_file:
os.environ['CLOUDKNOT_CONFIG_FILE'] = old_ck_config_file
else:
try:
del os.environ['CLOUDKNOT_CONFIG_FILE']
except KeyError:
pass
ck.refresh_clients()
def cleanup(bucket_cleanup):
"""Use this fixture to delete all unit testing resources
regardless of of the failure or success of the test"""
yield None
iam = ck.aws.clients['iam']
ec2 = ck.aws.clients['ec2']
batch = ck.aws.clients['batch']
ecs = ck.aws.clients['ecs']
config_file = ck.config.get_config_file()
section_suffix = ck.get_profile() + ' ' + ck.get_region()
jq_section_name = 'job-queues ' + section_suffix
ce_section_name = 'compute-environments ' + section_suffix
jd_section_name = 'job-definitions ' + section_suffix
roles_section_name = 'roles ' + ck.get_profile() + ' global'
vpc_section_name = 'vpc ' + section_suffix
sg_section_name = 'security-groups ' + section_suffix
retry = tenacity.Retrying(wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(120),
retry=tenacity.retry_if_exception_type(
batch.exceptions.ClientException))
# Clean up job queues from AWS
# ----------------------------
# Find all unit testing job queues
response = batch.describe_job_queues()
job_queues = [{
'name': d['jobQueueName'],
'arn': d['jobQueueArn'],
'state': d['state'],
'status': d['status']
} for d in response.get('jobQueues')]
while response.get('nextToken'):
response = batch.describe_job_queues(
nextToken=response.get('nextToken'))
job_queues = job_queues + [{
'name': d['jobQueueName'],
'arn': d['jobQueueArn'],
'state': d['state'],
'status': d['status']
} for d in response.get('jobQueues')]
unit_test_JQs = list(
filter(lambda d: UNIT_TEST_PREFIX in d['name'], job_queues))
enabled = list(filter(lambda d: d['state'] == 'ENABLED', unit_test_JQs))
for jq in enabled:
ck.aws.wait_for_job_queue(name=jq['name'], max_wait_time=180)
retry.call(batch.update_job_queue,
jobQueue=jq['arn'],
state='DISABLED')
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
requires_deletion = list(
filter(lambda d: d['status'] not in ['DELETED', 'DELETING'],
unit_test_JQs))
for jq in requires_deletion:
ck.aws.wait_for_job_queue(name=jq['name'], max_wait_time=180)
# Finally, delete the job queue
retry.call(batch.delete_job_queue, jobQueue=jq['arn'])
# Clean up config file
try:
config.remove_option(jq_section_name, jq['name'])
except configparser.NoSectionError:
pass
with open(config_file, 'w') as f:
config.write(f)
# Clean up compute environments from AWS
# --------------------------------------
# Find all unit testing compute environments
response = batch.describe_compute_environments()
comp_envs = [{
'name': d['computeEnvironmentName'],
'arn': d['computeEnvironmentArn'],
'state': d['state'],
'status': d['status']
} for d in response.get('computeEnvironments')]
while response.get('nextToken'):
response = batch.describe_compute_environments(
nextToken=response.get('nextToken'))
comp_envs = comp_envs + [{
'name': d['computeEnvironmentName'],
'arn': d['computeEnvironmentArn'],
'state': d['state'],
'status': d['status']
} for d in response.get('computeEnvironments')]
unit_test_CEs = list(
filter(lambda d: UNIT_TEST_PREFIX in d['name'], comp_envs))
enabled = list(filter(lambda d: d['state'] == 'ENABLED', unit_test_CEs))
for ce in enabled:
ck.aws.wait_for_compute_environment(arn=ce['arn'],
name=ce['name'],
log=False)
# Set the compute environment state to 'DISABLED'
retry.call(batch.update_compute_environment,
computeEnvironment=ce['arn'],
state='DISABLED')
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
for ce in unit_test_CEs:
# Then disassociate from any job queues
response = batch.describe_job_queues()
associated_queues = list(
filter(
lambda q: ce['arn'] in [
c['computeEnvironment']
for c in q['computeEnvironmentOrder']
], response.get('jobQueues')))
for queue in associated_queues:
arn = queue['jobQueueArn']
name = queue['jobQueueName']
# Disable submissions to the queue
if queue['state'] == 'ENABLED':
ck.aws.wait_for_job_queue(name=name,
log=True,
max_wait_time=180)
retry.call(batch.update_job_queue,
jobQueue=arn,
state='DISABLED')
# Delete the job queue
if queue['status'] not in ['DELETED', 'DELETING']:
ck.aws.wait_for_job_queue(name=name,
log=True,
max_wait_time=180)
retry.call(batch.delete_job_queue, jobQueue=arn)
# Clean up config file
try:
config.remove_option(jq_section_name, name)
except configparser.NoSectionError:
pass
requires_deletion = list(
filter(lambda d: d['status'] not in ['DELETED', 'DELETING'],
unit_test_CEs))
for ce in requires_deletion:
# Now get the associated ECS cluster
response = batch.describe_compute_environments(
computeEnvironments=[ce['arn']])
cluster_arn = response.get('computeEnvironments')[0]['ecsClusterArn']
# Get container instances
response = ecs.list_container_instances(cluster=cluster_arn, )
instances = response.get('containerInstanceArns')
for i in instances:
ecs.deregister_container_instance(cluster=cluster_arn,
containerInstance=i,
force=True)
retry_if_exception = tenacity.Retrying(
wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(120),
retry=tenacity.retry_if_exception_type())
retry_if_exception.call(ecs.delete_cluster, cluster=cluster_arn)
ck.aws.wait_for_compute_environment(arn=ce['arn'],
name=ce['name'],
log=False)
retry.call(batch.delete_compute_environment,
computeEnvironment=ce['arn'])
# Clean up config file
try:
config.remove_option(ce_section_name, ce['name'])
except configparser.NoSectionError:
pass
with open(config_file, 'w') as f:
config.write(f)
# Clean up job definitions from AWS
# ---------------------------------
# Find all unit testing job definitions
response = batch.describe_job_definitions(status='ACTIVE')
jds = [{
'name': d['jobDefinitionName'],
'arn': d['jobDefinitionArn']
} for d in response.get('jobDefinitions')]
unit_test_jds = list(filter(lambda d: UNIT_TEST_PREFIX in d['name'], jds))
while response.get('nextToken'):
response = batch.describe_job_definitions(
status='ACTIVE', nextToken=response.get('nextToken'))
jds = [{
'name': d['jobDefinitionName'],
'arn': d['jobDefinitionArn']
} for d in response.get('jobDefinitions')]
unit_test_jds = unit_test_jds + list(
filter(lambda d: UNIT_TEST_PREFIX in d['name'], jds))
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
for jd in unit_test_jds:
# Deregister the job definition
retry.call(batch.deregister_job_definition,
jobDefinition=jd['arn'])
# Clean up config file
try:
config.remove_option(jd_section_name, jd['name'])
except configparser.NoSectionError:
pass
with open(config_file, 'w') as f:
config.write(f)
# Clean up security_groups from AWS
# ---------------------------------
# Find all unit test security groups
ec2_retry = tenacity.Retrying(wait=tenacity.wait_exponential(max=16),
stop=tenacity.stop_after_delay(60),
retry=tenacity.retry_if_exception_type(
ec2.exceptions.ClientError))
response = ec2.describe_security_groups()
sgs = [{
'name': d['GroupName'],
'id': d['GroupId']
} for d in response.get('SecurityGroups')]
unit_test_sgs = filter(lambda d: UNIT_TEST_PREFIX in d['name'], sgs)
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
for sg in unit_test_sgs:
# Delete role
ec2_retry.call(ec2.delete_security_group, GroupId=sg['id'])
# Clean up config file
try:
config.remove_option(sg_section_name, sg['id'])
except configparser.NoSectionError:
pass
with open(config_file, 'w') as f:
config.write(f)
# Clean up VPCs from AWS
# ----------------------
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
# Find all VPCs with a Name tag key
response = ec2.describe_vpcs(Filters=[{
'Name': 'tag-key',
'Values': ['Name']
}])
for vpc in response.get('Vpcs'):
# Test if the unit-test prefix is in the name
if UNIT_TEST_PREFIX in [
d for d in vpc['Tags'] if d['Key'] == 'Name'
][0]['Value']:
# Retrieve and delete subnets
response = ec2.describe_subnets(
Filters=[{
'Name': 'vpc-id',
'Values': [vpc['VpcId']]
}])
subnets = [d['SubnetId'] for d in response.get('Subnets')]
for subnet_id in subnets:
ec2_retry.call(ec2.delete_subnet, SubnetId=subnet_id)
response = ec2.describe_network_acls(
Filters=[{
'Name': 'vpc-id',
'Values': [vpc['VpcId']]
}, {
'Name': 'default',
'Values': ['false']
}])
network_acl_ids = [
n['NetworkAclId'] for n in response.get('NetworkAcls')
]
# Delete the network ACL
for net_id in network_acl_ids:
ec2_retry.call(ec2.delete_network_acl, NetworkAclId=net_id)
response = ec2.describe_route_tables(
Filters=[{
'Name': 'vpc-id',
'Values': [vpc['VpcId']]
}, {
'Name': 'association.main',
'Values': ['false']
}])
route_table_ids = [
rt['RouteTableId'] for rt in response.get('RouteTables')
]
# Delete the route table
for rt_id in route_table_ids:
ec2_retry.call(ec2.delete_route_table, RouteTableId=rt_id)
# Detach and delete the internet gateway
response = ec2.describe_internet_gateways(
Filters=[{
'Name': 'attachment.vpc-id',
'Values': [vpc['VpcId']]
}])
gateway_ids = [
g['InternetGatewayId']
for g in response.get('InternetGateways')
]
for gid in gateway_ids:
ec2_retry.call(ec2.detach_internet_gateway,
InternetGatewayId=gid,
VpcId=vpc['VpcId'])
ec2_retry.call(ec2.delete_internet_gateway,
InternetGatewayId=gid)
# delete the VPC
ec2_retry.call(ec2.delete_vpc, VpcId=vpc['VpcId'])
# Clean up config file
try:
config.remove_option(vpc_section_name, vpc['VpcId'])
except configparser.NoSectionError:
pass
with open(config_file, 'w') as f:
config.write(f)
# Clean up roles from AWS
# -----------------------
# Find all unit test roles
response = iam.list_roles()
role_names = [d['RoleName'] for d in response.get('Roles')]
unit_test_roles = filter(lambda n: UNIT_TEST_PREFIX in n, role_names)
for role_name in unit_test_roles:
# Remove instance profiles
response = iam.list_instance_profiles_for_role(RoleName=role_name)
for ip in response.get('InstanceProfiles'):
iam.remove_role_from_instance_profile(
InstanceProfileName=ip['InstanceProfileName'],
RoleName=role_name)
iam.delete_instance_profile(
InstanceProfileName=ip['InstanceProfileName'])
# Detach policies from role
response = iam.list_attached_role_policies(RoleName=role_name)
for policy in response.get('AttachedPolicies'):
iam.detach_role_policy(RoleName=role_name,
PolicyArn=policy['PolicyArn'])
# Delete role
iam.delete_role(RoleName=role_name)
# Clean up config file
config = configparser.ConfigParser()
with ck.config.rlock:
config.read(config_file)
for role_name in config.options(roles_section_name):
if UNIT_TEST_PREFIX in role_name:
config.remove_option(roles_section_name, role_name)
with open(config_file, 'w') as f:
config.write(f)