def testAWSProvisionerUtils(self):
clusterName = 'cluster-utils-test' + str(uuid.uuid4())
try:
system([
self.toilMain, 'launch-cluster', '--nodeType=t2.micro',
'--keyPairName=jenkins@jenkins-master', clusterName,
'--provisioner=aws'
])
finally:
system([
self.toilMain, 'destroy-cluster', '--provisioner=aws',
clusterName
])
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
# launch preemptable master with same name
system([
self.toilMain, 'launch-cluster', '--nodeType=m3.medium:0.2',
'--keyPairName=jenkins@jenkins-master', clusterName,
'--provisioner=aws', '--logLevel=DEBUG'
])
system([
self.toilMain, 'ssh-cluster', '--provisioner=aws', clusterName
])
testStrings = [
"'foo'", '"foo"', ' foo', '$PATH', '"', "'", '\\', '| cat',
'&& cat', '; cat'
]
for test in testStrings:
logger.info('Testing SSH with special string: %s', test)
compareTo = "import sys; assert sys.argv[1]==%r" % test
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['python', '-', test],
input=compareTo)
try:
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['nonsenseShouldFail'])
except RuntimeError:
pass
else:
self.fail(
'The remote command failed silently where it should have '
'raised an error')
finally:
system([
self.toilMain, 'destroy-cluster', '--provisioner=aws',
clusterName
])
def testAWSProvisionerUtils(self):
clusterName = 'cluster-utils-test' + str(uuid.uuid4())
try:
system([self.toilMain, 'launch-cluster', '--nodeType=t2.micro', '--keyPairName=jenkins@jenkins-master',
clusterName, '--provisioner=aws'])
finally:
system([self.toilMain, 'destroy-cluster', '--provisioner=aws', clusterName])
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
# launch preemptable master with same name
system([self.toilMain, 'launch-cluster', '--nodeType=m3.medium:0.2', '--keyPairName=jenkins@jenkins-master',
clusterName, '--provisioner=aws', '--logLevel=DEBUG'])
system([self.toilMain, 'ssh-cluster', '--provisioner=aws', clusterName])
testStrings = ["'foo'",
'"foo"',
' foo',
'$PATH',
'"',
"'",
'\\',
'| cat',
'&& cat',
'; cat'
]
for test in testStrings:
logger.info('Testing SSH with special string: %s', test)
compareTo = "import sys; assert sys.argv[1]==%r" % test
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['python', '-', test],
input=compareTo)
try:
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['nonsenseShouldFail'])
except RuntimeError:
pass
else:
self.fail('The remote command failed silently where it should have '
'raised an error')
finally:
system([self.toilMain, 'destroy-cluster', '--provisioner=aws', clusterName])
def testAWSProvisionerUtils(self):
clusterName = 'cluster-utils-test' + str(uuid.uuid4())
keyName = os.getenv('TOIL_AWS_KEYNAME')
try:
# --provisioner flag should default to aws, so we're not explicitly
# specifying that here
system([self.toilMain, 'launch-cluster', '--nodeType=t2.micro',
'--keyPairName=' + keyName, clusterName])
finally:
system([self.toilMain, 'destroy-cluster', '--provisioner=aws', clusterName])
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
userTags = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
tags = {'Name': clusterName, 'Owner': keyName}
tags.update(userTags)
# launch preemptable master with same name
system([self.toilMain, 'launch-cluster', '-t', 'key1=value1', '-t', 'key2=value2', '--tag', 'key3=value3',
'--nodeType=m3.medium:0.2', '--keyPairName=' + keyName, clusterName,
'--provisioner=aws', '--logLevel=DEBUG'])
# test leader tags
leaderTags = AWSProvisioner._getLeader(clusterName).tags
self.assertEqual(tags, leaderTags)
# Test strict host key checking
# Doesn't work when run locally.
if(keyName == 'jenkins@jenkins-master'):
try:
AWSProvisioner.sshLeader(clusterName=clusterName, strict=True)
except RuntimeError:
pass
else:
self.fail("Host key verification passed where it should have failed")
# Add the host key to known_hosts so that the rest of the tests can
# pass without choking on the verification prompt.
AWSProvisioner.sshLeader(clusterName=clusterName, strict=True, sshOptions=['-oStrictHostKeyChecking=no'])
system([self.toilMain, 'ssh-cluster', '--provisioner=aws', clusterName])
testStrings = ["'foo'",
'"foo"',
' foo',
'$PATH',
'"',
"'",
'\\',
'| cat',
'&& cat',
'; cat'
]
for test in testStrings:
logger.info('Testing SSH with special string: %s', test)
compareTo = "import sys; assert sys.argv[1]==%r" % test
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['python', '-', test],
input=compareTo)
try:
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['nonsenseShouldFail'])
except RuntimeError:
pass
else:
self.fail('The remote command failed silently where it should have '
'raised an error')
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['python', '-c', "import os; assert os.environ['TOIL_WORKDIR']=='/var/lib/toil'"])
# `toil rsync-cluster`
# Testing special characters - string.punctuation
fname = '!"#$%&\'()*+,-.;<=>:\ ?@[\\\\]^_`{|}~'
testData = os.urandom(3 * (10**6))
with tempfile.NamedTemporaryFile(suffix=fname) as tmpFile:
relpath = os.path.basename(tmpFile.name)
tmpFile.write(testData)
tmpFile.flush()
# Upload file to leader
AWSProvisioner.rsyncLeader(clusterName=clusterName, args=[tmpFile.name, ":"])
# Ensure file exists
AWSProvisioner.sshLeader(clusterName=clusterName, args=["test", "-e", relpath])
tmpDir = tempfile.mkdtemp()
# Download the file again and make sure it's the same file
# `--protect-args` needed because remote bash chokes on special characters
AWSProvisioner.rsyncLeader(clusterName=clusterName, args=["--protect-args", ":" + relpath, tmpDir])
with open(os.path.join(tmpDir, relpath), "r") as f:
self.assertEqual(f.read(), testData, "Downloaded file does not match original file")
finally:
system([self.toilMain, 'destroy-cluster', '--provisioner=aws', clusterName])
try:
shutil.rmtree(tmpDir)
except NameError:
pass
def testAWSProvisionerUtils(self):
clusterName = 'cluster-utils-test' + str(uuid.uuid4())
keyName = os.getenv('TOIL_AWS_KEYNAME')
try:
# --provisioner flag should default to aws, so we're not explicitly
# specifying that here
system([
self.toilMain, 'launch-cluster', '--leaderNodeType=t2.micro',
'--keyPairName=' + keyName, clusterName
])
finally:
system([
self.toilMain, 'destroy-cluster', '--provisioner=aws',
clusterName
])
try:
from toil.provisioners.aws.awsProvisioner import AWSProvisioner
userTags = {'key1': 'value1', 'key2': 'value2', 'key3': 'value3'}
tags = {'Name': clusterName, 'Owner': keyName}
tags.update(userTags)
# launch preemptable master with same name
system([
self.toilMain, 'launch-cluster', '-t', 'key1=value1', '-t',
'key2=value2', '--tag', 'key3=value3',
'--leaderNodeType=m3.medium:0.2', '--keyPairName=' + keyName,
clusterName, '--provisioner=aws', '--logLevel=DEBUG'
])
# test leader tags
leaderTags = AWSProvisioner._getLeader(clusterName).tags
self.assertEqual(tags, leaderTags)
# Test strict host key checking
# Doesn't work when run locally.
if (keyName == 'jenkins@jenkins-master'):
try:
AWSProvisioner.sshLeader(clusterName=clusterName,
strict=True)
except RuntimeError:
pass
else:
self.fail(
"Host key verification passed where it should have failed"
)
# Add the host key to known_hosts so that the rest of the tests can
# pass without choking on the verification prompt.
AWSProvisioner.sshLeader(clusterName=clusterName,
strict=True,
sshOptions=['-oStrictHostKeyChecking=no'])
system([
self.toilMain, 'ssh-cluster', '--provisioner=aws', clusterName
])
testStrings = [
"'foo'", '"foo"', ' foo', '$PATH', '"', "'", '\\', '| cat',
'&& cat', '; cat'
]
for test in testStrings:
logger.info('Testing SSH with special string: %s', test)
compareTo = "import sys; assert sys.argv[1]==%r" % test
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['python', '-', test],
input=compareTo)
try:
AWSProvisioner.sshLeader(clusterName=clusterName,
args=['nonsenseShouldFail'])
except RuntimeError:
pass
else:
self.fail(
'The remote command failed silently where it should have '
'raised an error')
AWSProvisioner.sshLeader(
clusterName=clusterName,
args=[
'python', '-c',
"import os; assert os.environ['TOIL_WORKDIR']=='/var/lib/toil'"
])
# `toil rsync-cluster`
# Testing special characters - string.punctuation
fname = '!"#$%&\'()*+,-.;<=>:\ ?@[\\\\]^_`{|}~'
testData = os.urandom(3 * (10**6))
with tempfile.NamedTemporaryFile(suffix=fname) as tmpFile:
relpath = os.path.basename(tmpFile.name)
tmpFile.write(testData)
tmpFile.flush()
# Upload file to leader
AWSProvisioner.rsyncLeader(clusterName=clusterName,
args=[tmpFile.name, ":"])
# Ensure file exists
AWSProvisioner.sshLeader(clusterName=clusterName,
args=["test", "-e", relpath])
tmpDir = tempfile.mkdtemp()
# Download the file again and make sure it's the same file
# `--protect-args` needed because remote bash chokes on special characters
AWSProvisioner.rsyncLeader(
clusterName=clusterName,
args=["--protect-args", ":" + relpath, tmpDir])
with open(os.path.join(tmpDir, relpath), "r") as f:
self.assertEqual(
f.read(), testData,
"Downloaded file does not match original file")
finally:
system([
self.toilMain, 'destroy-cluster', '--provisioner=aws',
clusterName
])
try:
shutil.rmtree(tmpDir)
except NameError:
pass
