def _getVIPNameAndAddress(self, interfaceId):
"""
Return a tuple that is the Name tag value and the private IP address of
the AWS::EC2::NetworkInterface with the given interfaceId
The given interfaceId is an physical resource ID for an AWS::EC2::NetworkInterface resource.
This method is a helper for _getVIPs().
"""
interface = self.ec2.NetworkInterface(interfaceId)
privateIP = interface.private_ip_address
tags = interface.tag_set
name = ''
for tag in tags:
key = tag.get('Key')
if (key == 'Name'):
name = tag.get('Value')
break
#endIf
#endFor
if (not name):
raise ICPInstallationException("NetworkInterface: %s is expected to have a Name tag." % interfaceId)
#endIf
return (name, privateIP)
def _configureMgmtServices(self,configParameters,optionalServices,excludedServices):
"""
Walk through the optionalServices list and set all that are not in the excludedServivces
list to "enabled" in the configParameters dictionary.
The incoming excludedServices parameter is assumed to have been "regularized" by the
_transformExcludeMgmtServices() method prior to the invocation of this method.
The names in the optionalServices list are mapped to names that match the corresponding
parameter names used in the configParameters dictionary.
NOTE: This method is used to support the inclusion/exclusion of management services in the
config.yaml file for ICP v3.1.0 or later.
"""
methodName = "_configureMgmtServices"
for service in optionalServices:
serviceParameterName = ServiceNameParameterMap.get(service)
if (not serviceParameterName):
raise ICPInstallationException("Missing service name parameter in ServiceNameParameterMap for service: %s" % service)
#endIf
if (excludedServices and service in excludedServices):
# Set the service parameter to be disabled in the configParmaeters
configParameters[serviceParameterName] = 'disabled'
else:
configParameters[serviceParameterName] = 'enabled'
#endIf
if (TR.isLoggable(Level.FINEST)):
TR.finest(methodName,"Management Service: %s: %s" % (serviceParameterName,configParameters[serviceParameterName]))
def loadICPImages(self):
"""
Load the IBM Cloud Pak images from the installation tar archive.
Loading the ICP installation images on each node prior to kicking off the
inception install is an expediency that speeds up the installation process
dramatically.
The AWS CloudFormation template downlaods the ICP installation tar ball from
an S3 bucket to /tmp/icp-install-archive.tgz of each cluster node. It turns
out that download is very fast: typically 3 to 4 minutes.
"""
methodName = "loadICPImages"
TR.info(methodName, "STARTED docker load of ICP installation images.")
retcode = call(
"tar -zxvf /tmp/icp-install-archive.tgz -O | docker load | tee /root/logs/load-icp-images.log",
shell=True)
if (retcode != 0):
raise ICPInstallationException(
"Error calling: 'tar -zxvf /tmp/icp-install-archive.tgz -O | docker load' - Return code: %s"
% retcode)
#endIf
TR.info(methodName,
"COMPLETED Docker load of ICP installation images.")
def getBootNodePublicKey(self):
"""
Return the authorized key entry for the ~/.ssh/authorized_keys file.
The returned string is intended to include the RSA public key as well as the root user
and IP address of the boot node. The returned string can be added directly to the
authorized_keys file.
NOTE: It is possible that the a given cluster node may be checking for the authorized
key from the boot node, before the boot node has published it in its parameter. When
that happens a ParameterNotFound exception is raised by ssm.get_parameter(). That
exception is reported in the log, but ignored.
"""
methodName = "getBootNodePublicKey"
authorizedKeyEntry = ""
parameterKey = "/%s/boot-public-key" % self.stackName
tryCount = 1
response = None
while not response and tryCount <= 100:
time.sleep(GetParameterSleepTime)
TR.info(methodName,"Try: %d for getting parameter: %s" % (tryCount,parameterKey))
try:
response = self.ssm.get_parameter(Name=parameterKey)
except ClientError as e:
etext = "%s" % e
if (etext.find('ParameterNotFound') >= 0):
if (TR.isLoggable(Level.FINEST)):
TR.finest(methodName,"Ignoring ParameterNotFound ClientError on ssm.get_parameter() invocation")
#endIf
else:
raise ICPInstallationException("Unexpected ClientError on ssm.get_parameter() invocation: %s" % etext)
#endIf
#endTry
tryCount += 1
#endWhile
if (response and TR.isLoggable(Level.FINEST)):
TR.finest(methodName,"Response: %s" % response)
#endIf
if (not response):
TR.warning(methodName, "Failed to get a response for get_parameter: %s" % parameterKey)
else:
parameter = response.get('Parameter')
if (not parameter):
raise Exception("get_parameter response returned an empty Parameter.")
#endIf
authorizedKeyEntry = parameter.get('Value')
#endIf
return authorizedKeyEntry
def loadInstallMap(self, version=None, region=None):
"""
Return a dictionary that holds all the installation image information needed to
retrieve the installation images from S3.
Which install images to use is driven by the ICP version.
Which S3 bucket to use is driven by the AWS region of the deployment.
The source of the information is icp-install-artifact-map.yaml packaged with the
boostrap script package. The yaml file holds the specifics regarding which bucket
to use and the S3 path for the ICP and Docker images as well as the Docker image
name and the inception commands to use for the installation.
"""
methodName = "loadInstallMap"
if (not version):
raise MissingArgumentException("The ICP version must be provided.")
#endIf
if (not region):
raise MissingArgumentException("The AWS region must be provided.")
#endIf
installDocPath = os.path.join(self.home, "maps",
"icp-install-artifact-map.yaml")
with open(installDocPath, 'r') as installDocFile:
installDoc = yaml.load(installDocFile)
#endWith
if (TR.isLoggable(Level.FINEST)):
TR.finest(methodName, "Install doc: %s" % installDoc)
#endIf
installMap = installDoc.get(version)
if (not installMap):
raise ICPInstallationException(
"No ICP or Docker installation images defined for ICP version: %s"
% version)
#endIf
# The version is needed to get to the proper folder in the region bucket.
installMap['version'] = version
installMap['s3bucket'] = self.ICPArchiveBucketName
return installMap
def createConfigFile(self, configFilePath, icpVersion):
"""
Select the proper method to create the configuration file based on the
ICP version. The differences in the format and content of the config.yaml
from one version of ICP to the next are sufficient to warrant a specialized
method for the creation of the configuration file. This may settle out
as the product matures.
"""
if (icpVersion.startswith('2.1.')):
self.createConfigFile_21(configFilePath)
elif (icpVersion.startswith('3.1.')):
self.createConfigFile_31(configFilePath)
else:
raise ICPInstallationException("Unexpected version of ICP: %s" % icpVersion)
#endIf
if (self.MasterNodeCount > 1):
self.configureEtcHosts()
def _transformExcludedMgmtServices(self,excludedServices):
"""
Return a list of strings that are the names of the services to be excluded.
The incoming excludedServices parameter may be a list of strings or the string
representation of a list using commas to delimit the items in the list.
(The value of ExcludedMgmtServices in the AWS CF template is a CommaDelimitedList
which is just such a string.)
The items in the incoming list are converted to all lowercase characters and trimmed.
If the incoming value in excludedServices is the empty string, then an empty list
is returned.
NOTE: This method is used to support the exclusion of management services in the
ICP v2.1 config.yaml file. It is not used for excluding management services in
versions of ICP 3.1.0 and later.
"""
result = []
if (excludedServices):
if (type(excludedServices) != type([])):
# assume excludedServices is a string
excludedServices = [x.strip() for x in excludedServices.split(',')]
#endIf
excludedServices = [x.lower() for x in excludedServices]
for x in excludedServices:
if (x not in OptionalManagementServices_21):
raise ICPInstallationException("Service: %s is not an optional management service. It must be one of: %s" % (x,OptionalManagementServices_21))
#endIf
#endFor
result = excludedServices
#endIf
return result
def getSSMParameterValue(self,parameterKey,expectedValue=None):
"""
Return the value from the given SSM parameter key.
If an expectedValue is provided, then the wait loop for the SSM get_parameter()
will continue until the expected value is seen or the try count is exceeded.
NOTE: It is possible that the parameter is not present in the SSM parameter
cache when this method is invoked. When that happens a ParameterNotFound
exception is raised by ssm.get_parameter(). Depending on the trace level,
that exception is reported in the log, but ignored.
"""
methodName = "getSSMParameterValue"
parameterValue = None
tryCount = 1
gotit = False
while (not gotit and tryCount <= GetParameterMaxTryCount):
if (expectedValue == None):
TR.info(methodName,"Try: %d for getting parameter: %s" % (tryCount,parameterKey))
else:
TR.info(methodName,"Try: %d for getting parameter: %s with expected value: %s" % (tryCount,parameterKey,expectedValue))
#endIf
try:
response = self.ssm.get_parameter(Name=parameterKey)
if (not response):
if (TR.isLoggable(Level.FINEST)):
TR.finest(methodName, "Failed to get a response for SSM get_parameter(): %s" % parameterKey)
#endIf
else:
if (TR.isLoggable(Level.FINEST)):
TR.finest(methodName,"Response: %s" % response)
#endIf
parameter = response.get('Parameter')
if (not parameter):
raise Exception("SSM get_parameter() response returned an empty Parameter.")
#endIf
parameterValue = parameter.get('Value')
if (expectedValue == None):
gotit = True
break
else:
if (parameterValue == expectedValue):
gotit = True
break
else:
if (TR.isLoggable(Level.FINER)):
TR.finer(methodName,"For key: %s ignoring value: %s waiting on value: %s" % (parameterKey,parameterValue,expectedValue))
#endIf
#endIf
#endIf
#endIf
except ClientError as e:
etext = "%s" % e
if (etext.find('ParameterNotFound') >= 0):
if (TR.isLoggable(Level.FINEST)):
TR.finest(methodName,"Ignoring ParameterNotFound ClientError on ssm.get_parameter() invocation")
#endIf
else:
raise ICPInstallationException("Unexpected ClientError on ssm.get_parameter() invocation: %s" % etext)
#endIf
#endTry
time.sleep(GetParameterSleepTime)
tryCount += 1
#endWhile
if (not gotit):
if (expectedValue == None):
raise ICPInstallationException("Failed to get parameter: %s " % parameterKey)
else:
raise ICPInstallationException("Failed to get parameter: %s with expected value: %s" % (parameterKey,expectedValue))
#endIf
#endIf
return parameterValue
def _init(self, stackIds=None, configTemplatePath=None, **restArgs):
"""
Helper for the __init__() constructor.
All the heavy lifting for initialization of the class occurs in this method.
"""
methodName = '_init'
global StackParameters, StackParameterNames
if (not stackIds):
raise MissingArgumentException("The CloudFormation stack resource IDs must be provided.")
#endIf
self.rootStackId = stackIds[0]
if (not configTemplatePath):
raise MissingArgumentException("The path to the config.yaml template file must be provided.")
#endIf
self.configTemplatePath = configTemplatePath
self.etcHostsPlaybookPath = restArgs.get('etcHostsPlaybookPath')
self.SensitiveParameters = restArgs.get('sensitiveParameters')
StackParameters = restArgs.get('stackParameters')
if (not StackParameters):
raise MissingArgumentException("The stack parameters must be provided.")
#endIf
StackParameterNames = StackParameters.keys()
configParms = self.getConfigParameters(StackParameters)
if (TR.isLoggable(Level.FINEST)):
cleaned = Scrubber.dreplace(configParms, self.SensitiveParameters)
TR.finest(methodName,"Scrubbed parameters defined in the stack:\n\t%s" % cleaned)
#endIf
self.configParameters = self.fillInDefaultValues(**configParms)
self.masterELBAddress = self.getLoadBalancerIPAddress(stackIds,elbName="MasterNodeLoadBalancer")
if (not self.masterELBAddress):
raise ICPInstallationException("An ELB with a Name tag of MasterNodeLoadBalancer was not found.")
#endIf
# This next block supports different ways to set up the WhichClusterLBAddress.
# It is a debugging ploy. I got tired of changing the script to try out different options.
if (self.WhichClusterLBAddress == 'UseMasterELBAddress'):
# NOTE: ICP 2.1.0.3 can't handle a DNS name in the cluster_lb_address config.yaml attribute.
masterELB = self.masterELBAddress
elif (self.WhichClusterLBAddress == 'UseMasterELBName'):
masterELB = self.getLoadBalancerDNSName(stackIds,elbName="MasterNodeLoadBalancer")
elif (self.WhichClusterLBAddress == 'UseClusterName'):
# In the root CloudFormation template, an alias entry is created in the Route53 DNS
# that maps the master ELB public DNS name to the cluster CN, i.e., the ClusterName.VPCDomain.
# Setting the cluster_lb_address to the cluster_CA_domain avoids OAuth issues in mgmt console.
masterELB = self.ClusterDNSName
else:
masterELB = self.ClusterDNSName
#endIf
self.configParameters['ClusterLBAddress'] = masterELB
self.proxyELBAddress = self.getLoadBalancerIPAddress(stackIds,elbName="ProxyNodeLoadBalancer")
if (not self.proxyELBAddress):
raise ICPInstallationException("An ELB with a Name tag of ProxyNodeLoadBalancer was not found.")
#endIf
if (self.WhichProxyLBAddress == 'UseProxyELBAddress'):
# NOTE: ICP 2.1.0.3 can't handle a DNS name in the proxy_lb_address config.yaml attribute.
proxyELB = self.proxyELBAddress
elif (self.WhichProxyLBAddress == 'UseProxyELBName'):
proxyELB = self.getLoadBalancerDNSName(stackIds,elbName="ProxyNodeLoadBalancer")
elif (self.WhichProxyLBAddress == 'UsePrimaryAppDomain'):
# In the root CloudFormation template, an alias entry is created in the Route53 DNS
# that maps the proxy ELB public DNS name to the primary application domain.
# The primary app domain is the first entry in the list of ApplicationDomains passed
# into the root stack.
proxyELB = self.getPrimaryAppDomain()
else:
proxyELB = self.getPrimaryAppDomain()
#endIf
self.configParameters['ProxyLBAddress'] = proxyELB
self.configParameters['ClusterCADomain'] = self.ClusterDNSName
# VIPs are not supposed to be needed when load balancers are used.
# WARNING: For an AWS deployment, VIPs have never worked.
# Using an EC2::NetworkInterface to get an extra IP doesn't work.
#self.vips = self._getVIPs(self.rootStackId)
#self.configParameters['ClusterVIP'] = self.getVIPAddress("MasterVIP")
#self.configParameters['ProxyVIP'] = self.getVIPAddress("ProxyVIP")
self.configParameterNames = self.configParameters.keys()
if (TR.isLoggable(Level.FINEST)):
cleaned = Scrubber.dreplace(self.configParameters,self.SensitiveParameters)
TR.finest(methodName,"All configuration parameters, including defaults:\n\t%s" % cleaned)
def addNodeToCluster(self):
gotit = False
tryCount = 1
methodName = "addNodeToCluster"
randomTime = random.randrange(60, 600)
TR.info(methodName, "Sleeping for %s" % randomTime)
time.sleep(randomTime)
while (not gotit):
parameterKey = "/%s/configureNodeStatus" % self.stackName
TR.info(
methodName,
"Try: %d for getting parameter: %s" % (tryCount, parameterKey))
try:
response = self.getSSMParameterValue(parameterKey, count=3)
TR.info(methodName, "Response returned = %s" % response)
if (response == "True"):
time.sleep(GetParameterSleepTime * 5)
elif (response == "False"):
gotit = True
except ICPInstallationException as e:
etext = "%s" % e
if (etext.find('Failed to get parameter') >= 0):
if (TR.isLoggable(Level.FINEST)):
TR.finest(
methodName,
"Ignoring ParameterNotFound ClientError on ssm.get_parameter() invocation"
)
gotit = True
#endIf
else:
raise ICPInstallationException(
"Unexpected ClientError on ssm.get_parameter() invocation: %s"
% etext)
#endIf
#endTry
tryCount += 1
#endWhile
if (gotit):
ipValue = self.fqdn.split("-", 1)[1].split(".")
workerNodeIP = re.sub("-", ".", ipValue[0])
os.chdir(self.home + "/docker")
os.chmod('icp-install-docker.bin', stat.S_IEXEC)
call(
'./icp-install-docker.bin --install | tee /root/logs/install-docker.log',
shell=True)
os.chdir(self.home)
# call ssm send command to execute docker run
TR.info(methodName, "Execute docker Run script in bootnode")
response = self.ssm.send_command(
Targets=[{
"Key": "instanceids",
"Values": [self.bootInstanceId]
}],
DocumentName="AWS-RunShellScript",
Parameters={
"commands": [
"python configureNode.py %s %s %s" %
(workerNodeIP, self.ICPVersion, self.role)
],
"executionTimeout": ["1200"],
"workingDirectory": ["/root"]
},
Comment=
"Execute script in bootnode to add newly inducted node to cluster",
TimeoutSeconds=1500)
time.sleep(GetParameterSleepTime * 10)
command = response.get("Command")
commandId = command.get("CommandId")
TR.info(methodName, "Command Id : %s" % commandId)
response = self.getExecutionStatus(commandId, self.bootInstanceId)
TR.info(methodName, "response :%s" % response)
if (response == "Success"):
if (TR.isLoggable(Level.FINE)):
TR.fine(
methodName,
"Executed docker Run script in bootnode successfully")
else:
TR.fine(
methodName,
"Executed docker Run script in bootnode failed with error %s"
% response.get("StandardErrorContent"))
parameterKey = "/%s/boot-public-key" % self.stackName
TR.fine(methodName,
"delete ssm parameter key %s post setup" % parameterKey)
self.ssm.delete_parameter(Name=parameterKey)