def test_set_region(bucket_cleanup):
ck.refresh_clients()
with pytest.raises(ck.aws.CloudknotInputError):
ck.set_region(region="not a valid region name")
old_region = ck.get_region()
try:
old_config_file = os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
old_config_file = None
try:
with tempfile.NamedTemporaryFile() as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
region = "us-west-1"
ck.set_region(region)
assert ck.get_region() == region
for service, client in ck.aws.clients.items():
if service == "iam":
assert client.meta.region_name == "aws-global"
else:
assert client.meta.region_name == region
finally:
ck.set_region(old_region)
if old_config_file:
os.environ["CLOUDKNOT_CONFIG_FILE"] = old_config_file
else:
try:
del os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
pass
ck.refresh_clients()
def test_get_region(bucket_cleanup):
# Save environment variables for restoration later
try:
old_region_env = os.environ['AWS_DEFAULT_REGION']
except KeyError:
old_region_env = None
old_region = ck.get_region()
try:
old_config_file = os.environ['CLOUDKNOT_CONFIG_FILE']
except KeyError:
old_config_file = None
try:
# With empty config file, get_region should return the
# environment variable AWS_DEFAULT_REGION
with tempfile.NamedTemporaryFile(mode='w+') as tmp:
os.environ['CLOUDKNOT_CONFIG_FILE'] = tmp.name
region = 'test-region-0'
os.environ['AWS_DEFAULT_REGION'] = region
assert ck.get_region() == region
del os.environ['AWS_DEFAULT_REGION']
# With region in a temporary config file, region should simply
# read the config file
with tempfile.NamedTemporaryFile(mode='w+') as tmp:
os.environ['CLOUDKNOT_CONFIG_FILE'] = tmp.name
region = 'test-region-1'
tmp.file.write('[aws]\n')
tmp.file.write('region = {region:s}\n'.format(region=region))
tmp.file.flush()
os.fsync(tmp.file.fileno())
assert ck.get_region() == region
# With no cloudknot config file and no environment variable
# get_region should return region in aws config file
with tempfile.NamedTemporaryFile(mode='w+') as tmp:
os.environ['CLOUDKNOT_CONFIG_FILE'] = tmp.name
aws_config_file = op.join(op.expanduser('~'), '.aws', 'config')
try:
if op.isfile(aws_config_file):
if op.isfile(aws_config_file + '.bak'):
raise Exception(
'Backup aws config file already exists.')
shutil.move(aws_config_file, aws_config_file + '.bak')
assert ck.get_region() == 'us-east-1'
finally:
if op.isfile(aws_config_file + '.bak'):
shutil.move(aws_config_file + '.bak', aws_config_file)
with tempfile.NamedTemporaryFile(mode='w+') as tmp:
os.environ['CLOUDKNOT_CONFIG_FILE'] = tmp.name
aws_config_file = op.join(op.expanduser('~'), '.aws', 'config')
try:
if op.isfile(aws_config_file):
if op.isfile(aws_config_file + '.bak'):
raise Exception(
'Backup aws config file already exists.')
shutil.move(aws_config_file, aws_config_file + '.bak')
else:
# Create the config directory if it doesn't exist
aws_config_dir = op.dirname(aws_config_file)
try:
os.makedirs(aws_config_dir)
except OSError as e:
pre_existing = (e.errno == errno.EEXIST
and op.isdir(aws_config_dir))
if pre_existing:
pass
else:
raise e
region = 'test-region-2'
with open(aws_config_file, 'w') as f:
f.write('[default]\n')
f.write('region = {region:s}\n'.format(region=region))
f.flush()
os.fsync(f.fileno())
assert ck.get_region() == region
finally:
if op.isfile(aws_config_file + '.bak'):
shutil.move(aws_config_file + '.bak', aws_config_file)
elif op.isfile(aws_config_file):
os.remove(aws_config_file)
finally:
ck.set_region(old_region)
# Restore old environment variables
if old_config_file:
os.environ['CLOUDKNOT_CONFIG_FILE'] = old_config_file
else:
try:
del os.environ['CLOUDKNOT_CONFIG_FILE']
except KeyError:
pass
if old_region_env:
os.environ['AWS_DEFAULT_REGION'] = old_region_env
else:
try:
del os.environ['AWS_DEFAULT_REGION']
except KeyError:
pass
ck.refresh_clients()
def test_get_region(bucket_cleanup):
ck.refresh_clients()
# Save environment variables for restoration later
try:
old_region_env = os.environ["AWS_DEFAULT_REGION"]
except KeyError:
old_region_env = None
old_region = ck.get_region()
try:
old_config_file = os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
old_config_file = None
try:
# With empty config file, get_region should return the
# environment variable AWS_DEFAULT_REGION
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
region = "test-region-0"
os.environ["AWS_DEFAULT_REGION"] = region
assert ck.get_region() == region
del os.environ["AWS_DEFAULT_REGION"]
# With region in a temporary config file, region should simply
# read the config file
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
region = "test-region-1"
tmp.file.write("[aws]\n")
tmp.file.write("region = {region:s}\n".format(region=region))
tmp.file.flush()
os.fsync(tmp.file.fileno())
assert ck.get_region() == region
# With no cloudknot config file and no environment variable
# get_region should return region in aws config file
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
aws_config_file = op.join(op.expanduser("~"), ".aws", "config")
try:
if op.isfile(aws_config_file):
if op.isfile(aws_config_file + ".bak"):
raise Exception(
"Backup aws config file already exists.")
shutil.move(aws_config_file, aws_config_file + ".bak")
assert ck.get_region() == "us-east-1"
finally:
if op.isfile(aws_config_file + ".bak"):
shutil.move(aws_config_file + ".bak", aws_config_file)
with tempfile.NamedTemporaryFile(mode="w+") as tmp:
os.environ["CLOUDKNOT_CONFIG_FILE"] = tmp.name
aws_config_file = op.join(op.expanduser("~"), ".aws", "config")
try:
if op.isfile(aws_config_file):
if op.isfile(aws_config_file + ".bak"):
raise Exception(
"Backup aws config file already exists.")
shutil.move(aws_config_file, aws_config_file + ".bak")
else:
# Create the config directory if it doesn't exist
aws_config_dir = op.dirname(aws_config_file)
try:
os.makedirs(aws_config_dir)
except OSError as e:
pre_existing = e.errno == errno.EEXIST and op.isdir(
aws_config_dir)
if pre_existing:
pass
else:
raise e
region = "test-region-2"
with open(aws_config_file, "w") as f:
f.write("[default]\n")
f.write("region = {region:s}\n".format(region=region))
f.flush()
os.fsync(f.fileno())
assert ck.get_region() == region
finally:
if op.isfile(aws_config_file + ".bak"):
shutil.move(aws_config_file + ".bak", aws_config_file)
elif op.isfile(aws_config_file):
os.remove(aws_config_file)
finally:
ck.set_region(old_region)
# Restore old environment variables
if old_config_file:
os.environ["CLOUDKNOT_CONFIG_FILE"] = old_config_file
else:
try:
del os.environ["CLOUDKNOT_CONFIG_FILE"]
except KeyError:
pass
if old_region_env:
os.environ["AWS_DEFAULT_REGION"] = old_region_env
else:
try:
del os.environ["AWS_DEFAULT_REGION"]
except KeyError:
pass
ck.refresh_clients()
`here <https://wiki.humanconnectome.org/display/PublicData/How+To+Connect+to+Connectome+Data+via+AWS>`_.
We will use the ``Cloudknot`` library to run our AFQ analysis in the AWS
Batch service (see also
`this example <http://yeatmanlab.github.io/pyAFQ/auto_examples/cloudknot_example.html>`_).
In the following we will use ``Cloudknot`` to run multiple
configurations of pyAFQ on the HCP dataset. Specifically, here we will run
pyAFQ with different tractography seeding strategies.
"""
##########################################################################
# Import cloudknot and set the correct region. The HCP data is stored in `us-east-1`, so it's best
# to analyze it there.
import configparser
import itertools
import cloudknot as ck
ck.set_region('us-east-1')
##########################################################################
# Define a function to run. This function allows us to pass in the subject ID for the subjects we would
# like to analyze , as well as strategies for seeding tractography (different masks and/or different
# numbers of seeds per voxel).
def afq_process_subject(subject, seed_mask, n_seeds,
aws_access_key, aws_secret_key):
# define a function that each job will run
# In this case, each process does a single subject
import logging
import s3fs
# all imports must be at the top of the function
# cloudknot installs the appropriate packages from pip