def _invokeCommand(self, cmdList=[], cmdString=""): """ Invoke a single command as specified by cmdList and cmdString Helper for invokeCommands() NOTE: From experiments, it was determined that the call() method needs to be invoked with shell=True. """ methodName = "_invokeCommand" if (not cmdList): raise MissingArgumentException( "A non-empty command list must be provided.") #endIf if (not cmdString): raise MissingArgumentException( "A non-empty command string must be provided.") #endIf TR.info(methodName, "Invoking: %s" % cmdString) #retcode = call(cmdList) retcode = call(cmdString, shell=True) if (retcode != 0): raise Exception("Invoking: '%s' Return code: %s" % (cmdString, retcode))
def createYamlFile(self, filePath, x): """ Dump a yaml representation of the given object (x) to the given filePath. """ if (not filePath): raise MissingArgumentException( "The file path (filePath) cannot be empty or None.") #endIf if (not x): raise MissingArgumentException( "The object (x) to be dumped to the file cannot be empty or None." ) #endIf destDirPath = os.path.dirname(filePath) if (not os.path.exists(destDirPath)): os.makedirs(destDirPath) #endIf with open(filePath, 'w') as yamlFile: yaml.dump(x, yamlFile, default_flow_style=False) #endWith return filePath
def getStackOutput(self, stackId, outputKey): """ For the given stack ID return the value of the given output key. """ methodName = "getStackOutput" if (not stackId): raise MissingArgumentException( "A stack resource ID must be provided.") #endIf if (not outputKey): raise MissingArgumentException("An output key must be provided.") #endIf response = self.cfnClient.describe_stacks(StackName=stackId) if (not response): raise AWSStackResourceException( "Empty result for CloudFormation describe_stacks for stack: %s" % stackId) #endIf stacks = response.get('Stacks') if (len(stacks) != 1): raise AWSStackResourceException( "Unexpected number of stacks: %d, from describe_stacks for stack: %s" % (len(stacks), stackId)) #endIf myStack = stacks[0] outputs = myStack.get('Outputs') if (not outputs): raise AWSStackResourceException( "Expecting output with key: %s, defined for stack: %s, but no outputs defined." % (outputKey, stackId)) #endIf result = None for output in outputs: key = output.get('OutputKey') if (key == outputKey): result = output.get('OutputValue') if (TR.isLoggable(Level.FINEST)): TR.finest( methodName, "Got output for key: %s with value: %s" % (outputKey, result)) #endIf break #endIf #endFor # Output value types are strings or other simple types. # The actual value of an output will never be None. if (result == None): TR.warning( methodName, "For stack: %s, no output found for key: %s" % (stackId, outputKey)) #endIf return result
def __init__(self, stackId=None, intrinsicVariables=None, configPath=None): """ stackId - the resource ID of the CloudFormation stack to be used for processing security groups. Typically, all security groups are defined in a particular CloudFormation stack used to define security profiles, roles and security groups. intrinsicVariables - dictionary of key-value pairs that are predefined for the given deployment and may be used for substitution macros in the security configuration file(s). Optional parameters: configPath - file system path to a security configuration YAML file or directory of YAML files """ object.__init__(self) if (not stackId): raise MissingArgumentException("A stack ID must be provided.") #endIf self.stackId = stackId if (not intrinsicVariables): raise MissingArgumentException("A collection of intrinsic variables must be provided.") #endIf self.IntrinsicVariables = intrinsicVariables.copy() self.IntrinsicVariableNames = self.IntrinsicVariables.keys() self.configPath = configPath self.cfnClient = boto3.client('cloudformation') self.ec2Resource = boto3.resource('ec2') self.ec2Client = boto3.client('ec2')
def getLoadBalancerDNSName(self,stackIds,elbName=None): """ Return the DNSName for the Elastic Load Balancer V2 with the given name as the value of its Name tag. The stackIds parameter holds the list of all the stacks in the CFN deployment. It is assumed there is only 1 ELB in all of those stacks with the given name. (The DNSName of the first one found with the given name gets returned.) The boto3 API for ELBs is rather baroque. The tags are gotten using the describe_tags() method. We need to look at the tags in order to find the ELB with Name tag value for the given name (elbName). The response from the describe_tags() call also includes the ARN (reource Id) for the ELB with the set of tags. Once we have the ELB ARN, we can get its DNSName with a call to describe_load_balancers() followed by getting the IP address attribute from the load balancer instance. """ if (not stackIds): raise MissingArgumentException("A list of stack IDs (stackIds) is required.") #endIf if (not elbName): raise MissingArgumentException("The ELB name must be provided.") #endIf dnsName = "" for stackId in stackIds: elbIId = self.getELBResourceIdForName(stackId, elbName=elbName) if (elbIId): response = self.elbv2Client.describe_load_balancers(LoadBalancerArns=[elbIId]) if (not response): raise AWSStackResourceException("Empty response for ELBv2 Client describe_load_balancers() call for ELB with ARN: %s" % elbIId) #endIf loadBalancers = response.get('LoadBalancers') if (not loadBalancers): raise AWSStackResourceException("No LoadBalancers in response for ELBv2 Client describe_load_balancers() call for ELB with ARN: %s" % elbIId) #endIf if (len(loadBalancers) != 1): raise AWSStackResourceException("Unexpected number of LoadBalancers from ELBv2 Client describe_load_balancers() call for ELB with ARN: %s" % elbIId) #endIf loadBalancer = loadBalancers[0] dnsName = loadBalancer.get('DNSName') if (not dnsName): raise AWSStackResourceException("Empty DNSName attribute for ELB with ARN: %s" % elbIId) #endIf break #endIf #endFor return dnsName
def fillInDefaultValues(self, parameterNames=None, defaultValues=None, **restArgs): """ Return a dictionary with values for each parameter in parameterNames that is the value in restArgs or the default value in defaultValues. Ff the parameter is not defined in restArgs the default value is used. """ result = {} if (not parameterNames): raise MissingArgumentException( "A parameter names list must be provided.") #endIf if (not defaultValues): raise MissingArgumentException( "A dictionary of default values must be provided.") #endIf for parmName in parameterNames: parmValue = restArgs.get(parmName, defaultValues.get(parmName)) if (parmValue): result[parmName] = parmValue #endIf #endFor return result
def publishReadiness(self, stackName, fqdn): """ Put a parameter in /stackName/fqdn indicating readiness for ICP installation to proceed. """ methodName = "publishReadiness" if (not stackName): raise MissingArgumentException( "The stack name (stackName) must be provided and cannot be empty." ) #endIf if (not fqdn): raise MissingArgumentException( "The FQDN for this node must be provided and cannot be empty.") #endIf parameterKey = "/%s/%s" % (stackName, fqdn) TR.info(methodName, "Putting READY to SSM parameter: %s" % parameterKey) self.ssm.put_parameter(Name=parameterKey, Description="Cluster node: %s is READY" % fqdn, Value="READY", Type='String', Overwrite=True) TR.info(methodName, "Node: %s is READY has been published." % fqdn)
def getS3Object(self, bucket=None, s3Path=None, destPath=None): """ Return destPath which is the local file path provided as the destination of the download. A pre-signed URL is created and used to download the object from the given S3 bucket with the given S3 key (s3Path) to the given local file system destination (destPath). The destination path is assumed to be a full path to the target destination for the object. If the directory of the destPath does not exist it is created. It is assumed the objects to be gotten are large binary objects. For details on how to download a large file with the requests package see: https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py """ methodName = "getS3Object" if (not bucket): raise MissingArgumentException("An S3 bucket name (bucket) must be provided.") #endIf if (not s3Path): raise MissingArgumentException("An S3 object key (s3Path) must be provided.") #endIf if (not destPath): raise MissingArgumentException("A file destination path (destPath) must be provided.") #endIf TR.info(methodName, "STARTED download of object: %s from bucket: %s, to: %s" % (s3Path,bucket,destPath)) s3url = self.s3.generate_presigned_url(ClientMethod='get_object',Params={'Bucket': bucket, 'Key': s3Path}) if (TR.isLoggable(Level.FINE)): TR.fine(methodName,"Getting S3 object with pre-signed URL: %s" % s3url) #endIf destDir = os.path.dirname(destPath) if (not os.path.exists(destDir)): os.makedirs(destDir) TR.info(methodName,"Created object destination directory: %s" % destDir) #endIf r = requests.get(s3url, stream=True) with open(destPath, 'wb') as destFile: shutil.copyfileobj(r.raw, destFile) #endWith TR.info(methodName, "COMPLETED download from bucket: %s, object: %s, to: %s" % (bucket,s3Path,destPath)) return destPath
def getELBResourceIdForName(self,stackId,elbName=None): """ Return the Elastic Load Balancer ARN with the given name as the value of its Name tag. If no ELB is found with the given name in its Name tag then the empty string is returned. """ if (not stackId): raise MissingArgumentException("A stack ID (stackId) is required.") #endIf if (not elbName): raise MissingArgumentException("An Elastic Load Balancer Name (elbName) must be provided.") #endIf elbResourceId = "" elbIIds = self.listELBResoures(stackId) if (elbIIds): for elbIId in elbIIds: response = self.elbv2Client.describe_tags(ResourceArns=[elbIId]) if (not response): raise AWSStackResourceException("Empty response for ELBv2 Client describe_tags() for Elastic Load Balancer with ARN: %s" % elbIId) #endIf tagDescriptions = response.get('TagDescriptions') if (len(tagDescriptions) != 1): raise AWSStackResourceException("Unexpected number of TagDescriptions in describe_tags() response from ELB with ARN: %s" % elbIId) #endIf tagDescription = tagDescriptions[0] tags = tagDescription.get('Tags') if (not tags): raise AWSStackResourceException("All Elastic Load Balancers must have at least a Name tag. No tags found for ELB with ARN: %s" % elbIId) #endIf for tag in tags: if (tag.get('Key') == 'Name'): if (tag.get('Value') == elbName): elbResourceId = tagDescription.get('ResourceArn') break #endIf #endIf #endFor if (elbResourceId): break #endFor #endIf return elbResourceId
def _processSecurityGroup(self, sg): """ Process an object of type: EC2_SecurityGroup If there is a security group defined with the same name as the given security group, then the given security group is checked for any new ingress or egress rules to be added to the existing security group. If there is no security group already defined with the name of the given security group then a new security group is created as defined by the given security group. """ methodName = "_processSecurityGroup" if (not sg): raise MissingArgumentException("A security group object (securityGroup) must be provided.") #endIf if TR.isLoggable(Level.FINER): TR.finer(methodName,"Processing security group: %s" % sg.group_name) #endIf # existing_sg is an instance of the boto3 EC2.SecurityGroup existing_sg = self.getSecurityGroup(self.stackId, sg.group_name) if (not existing_sg): self._createSecurityGroup(sg) else: self._updateSecurityGroup(existing_sg,sg)
def __init__(self, commandPath=None, intrinsicVariables=None, **restArgs): """ Constructor commandPath - path to directory that holds the command definition yaml files instrinsicVariables - dictionary of name-value pairs for variables defined by the framework restArgs: restArgs may have values for custom variables that override the custom variable values in the variables yaml """ object.__init__(self) if (not commandPath): raise MissingArgumentException("The command path must be provided.") #endIf self.commandPath = commandPath if (intrinsicVariables): self.IntrinsicVariables = intrinsicVariables.copy() self.IntrinsicVariableNames = self.IntrinsicVariables.keys() #endIf sensitiveVariables = restArgs.get('sensitiveVariables') if (sensitiveVariables): self.SensitiveVariables = sensitiveVariables #endIf self.home = os.path.expanduser('~') self.variableValues = self.__initVariableValues(self.commandPath, **restArgs) self.metadata = self.__initMetaData(self.commandPath)
def listELBResoures(self,stackId): """ Return a list of ELB resource instance IDs from the given stack. An empty list is returned if there are no ELB instances in the given stack. """ if (not stackId): raise MissingArgumentException("A stack ID (stackId) is required.") #endIf response = self.cfnClient.list_stack_resources(StackName=stackId) if (not response): raise AWSStackResourceException("Empty result for CloudFormation list_stack_resources for stack: %s" % stackId) #endIf stackResources = response.get('StackResourceSummaries') if (not stackResources): raise AWSStackResourceException("Empty StackResourceSummaries in response from CloudFormation list_stack_resources for stack: %s." % stackId) #endIf elbIIDs = [] for resource in stackResources: resourceType = resource.get('ResourceType') if (resourceType == 'AWS::ElasticLoadBalancingV2::LoadBalancer'): elbInstanceId = resource.get('PhysicalResourceId') elbIIDs.append(elbInstanceId) #endIf #endFor return elbIIDs
def getConfigParameters(self, parameters): """ Return a dictionary with configuration parameter name-value pairs extracted from the given parameters dictionary. Only parameters with names in the ConfigurationParameterNames list are included in the result set. """ methodName = "getConfigParameters" if (not parameters): raise MissingArgumentException("The dictionary of parameters from which to get the configuration parameters must be provided.") #endIf if (TR.isLoggable(Level.FINEST)): TR.finest(methodName,"ConfigurationParameterNames: %s" % ConfigurationParameterNames) #endIf result = {} for parmName in parameters.keys(): if (parmName in ConfigurationParameterNames): result[parmName] = parameters[parmName] #endIf #endFor return result
def __init__(self, commandSetsPath=None, intrinsicVariables=None, sensitiveVariables=None): """ Constructor """ object.__init__(self) if (not commandSetsPath): raise MissingArgumentException( "A file system path to the command sets directory must be provided." ) #endIf self.commandSetsPath = commandSetsPath if (intrinsicVariables): self.IntrinsicVariables = intrinsicVariables.copy() self.IntrinsicVariableNames = self.IntrinsicVariables.keys() #endIf if (sensitiveVariables): self.SensitiveVariables = sensitiveVariables #endIf self.rc = 0
def mountEFSVolumes(self, volumes): """ Mount the EFS storage volumes for the audit log and the Docker registry. volumes is either a singleton instance of EFSVolume or a list of instances of EFSVolume. EFSVolume has everything needed to mount the volume on a given mount point. NOTE: It is assumed that nfs-utils (RHEL) or nfs-common (Ubuntu) has been installed on the nodes were EFS mounts are implemented. Depending on what EFS example you look at the options to the mount command vary. The options used in this method are from this AWS documentation: https://docs.aws.amazon.com/efs/latest/ug/wt1-test.html Step 3.3 has the mount command template and the options are: nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport We explicitly add the following default mount options: rw,suid,dev,exec,auto,nouser """ methodName = "mountEFSVolumes" if (not volumes): raise MissingArgumentException( "One or more EFS volumes must be provided.") #endIf if (type(volumes) != type([])): volumes = [volumes] #endIf # See method doc above for AWS source for mount options used in the loop body below. options = "rw,suid,dev,exec,auto,nouser,nfsvers=4.1,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport" for volume in volumes: if (not os.path.exists(volume.mountPoint)): os.makedirs(volume.mountPoint) TR.info( methodName, "Created directory for EFS mount point: %s" % volume.mountPoint) elif (not os.path.isdir(volume.mountPoint)): raise Exception( "EFS mount point path: %s exists but is not a directory." % volume.mountPoint) else: TR.info( methodName, "EFS mount point: %s already exists." % volume.mountPoint) #endIf retcode = call("mount -t nfs4 -o %s %s:/ %s" % (options, volume.efsServer, volume.mountPoint), shell=True) if (retcode != 0): raise Exception( "Error return code: %s mounting to EFS server: %s with mount point: %s" % (retcode, volume.efsServer, volume.mountPoint)) #endIf TR.info( methodName, "%s mounted on EFS server: %s:/ with options: %s" % (volume.mountPoint, volume.efsServer, options))
def __init__(self, **restArgs): """ Constructor Required arguments: ClusterDNSName - the DNS name used for the CN of the cluster cert and used to access the ICP admin console HelmKeyPath - the path to the user key file, e.g., ~/.kube/kubecfg.key or <icp_home>/cluster/cfc-certs/helm/admin.key HelmCertPath - the path to the user cert file, e.g., ~/.kube/kubecfg.cert or <icp_home>/cluster/cfc-certs/helm/admin.crt ClusterCertPath - the path to the file that holds the CA cert. The cluster CA cert is usually used for this. """ object.__init__(self) self.ClusterDNSName = restArgs.get('ClusterDNSName') if (not self.ClusterDNSName): raise MissingArgumentException( "The cluster DNS name to be used to access the ICP master must be provided." ) #endIf self.HelmKeyPath = restArgs.get('HelmKeyPath') if (not self.HelmKeyPath): raise MissingArgumentException( "The file path to the Helm user key must be provided.") #endIf self.HelmCertPath = restArgs.get('HelmCertPath') if (not self.HelmCertPath): raise MissingArgumentException( "The file path to the Helm user certificate must be provided.") #endIf self.ClusterCertPath = restArgs.get('ClusterCertPath') if (not self.ClusterCertPath): raise MissingArgumentException( "The file path to the cluster certificate must be provided.") #endIf HelmHome = restArgs.get('HelmHome') if (not HelmHome): HelmHome = os.path.join(os.path.expanduser('~'), ".helm") #endIf self.HelmHome = HelmHome self.UserKeyPath = os.path.join(self.HelmHome, "key.pem") self.UserCertPath = os.path.join(self.HelmHome, "cert.pem") self.CACertPath = os.path.join(self.HelmHome, "ca.pem")
def loadInstallMap(self, version=None, region=None): """ Return a dictionary that holds all the installation image information needed to retrieve the installation images from S3. Which install images to use is driven by the ICP version. Which S3 bucket to use is driven by the AWS region of the deployment. The source of the information is icp-install-artifact-map.yaml packaged with the boostrap script package. The yaml file holds the specifics regarding which bucket to use and the S3 path for the ICP and Docker images as well as the Docker image name and the inception commands to use for the installation. """ methodName = "loadInstallMap" if (not version): raise MissingArgumentException("The ICP version must be provided.") #endIf if (not region): raise MissingArgumentException("The AWS region must be provided.") #endIf installDocPath = os.path.join(self.home, "maps", "icp-install-artifact-map.yaml") with open(installDocPath, 'r') as installDocFile: installDoc = yaml.load(installDocFile) #endWith if (TR.isLoggable(Level.FINEST)): TR.finest(methodName, "Install doc: %s" % installDoc) #endIf installMap = installDoc.get(version) if (not installMap): raise ICPInstallationException( "No ICP or Docker installation images defined for ICP version: %s" % version) #endIf # The version is needed to get to the proper folder in the region bucket. installMap['version'] = version installMap['s3bucket'] = self.ICPArchiveBucketName return installMap
def scpGetFile_broken(self, fromPath=None, toPath=None): """ Use scp to copy a file from the master node to the local file system WARNING: This never worked due to some issue with parameko and pycryptodome I didn't have time to sort it out. I think parmeko needs to use Pycrypto, but Pycrypto is no longer maintained and has security holes. """ methodName = "scpGetFile" if (not self.masterNode): raise FileTransferException( "The master host IP address or host name must be defined when the instance is created." ) #endIf if (not fromPath): raise MissingArgumentException( "The source path (fromPath) on the remote file system must be provided." ) #endIf if (not toPath): raise MissingArgumentException( "The destination path (toPath) on the local file system must be provided." ) #endIf if (TR.isLoggable(Level.FINEST)): TR.finest(methodName, "Creating an ssh connection to: %s" % self.masterNode) #endIf self.ssh.connect(self.masterNode) scp = SCPClient(self.ssh.get_transport()) if (TR.isLoggable(Level.FINER)): TR.finer( methodName, "From host: %s, copying: %s to %s" % (self.masterNode, fromPath, toPath)) #endIf scp.get(fromPath, local_path=toPath) if (TR.isLoggable(Level.FINEST)): TR.finest(methodName, "Copy completed")
def __init__(self,region=None, bucket=None, keyPrefix='logs', role=None, fqdn=None): """ Constructor region - the AWS region name bucket - the S3 bucket name. The bucket gets created if it does not exist. keyPrefix - the S3 key prefix to be used for each log export to S3, e.g., logs/<stackname> where <stackname> is the name of the CloudFormation stack associated with the root template for a given deployment. The root stack name is unique. Using logs as the beginning of the prefix keeps all logs in a separate "folder" of the bucket. role - the role of the node (VM) exporting the logs fqdn - fully qualified domain name of the node exporting the logs The FQDN provides uniqueness as there may be more than one node with a given role. """ object.__init__(self) if (not region): raise MissingArgumentException("The AWS region name must be provided.") #endIf self.region = region if (not bucket): raise MissingArgumentException("The S3 bucket name for the exported logs must be provided.") #endIf self.bucket = bucket self.keyPrefix = keyPrefix if (not role): raise MissingArgumentException("The role of the node exporting the logs must be provided.") #endIf self.role = role if (not fqdn): raise MissingArgumentException("The FQDN of the node exporting the logs must be provided.") #endIf self.fqdn = fqdn self.s3Helper = S3Helper(region=region) if (not self.s3Helper.bucketExists(bucket)): self.s3Helper.createBucket(bucket,region=region)
def scpGetFile(self, fromPath=None, toPath=None): """ Use scp to copy a file from the master node to the local file system This implementation uses a call out to bash to do the scp. """ methodName = "scpGetFile" if (not self.masterNode): raise FileTransferException( "The master host IP address or host name must be defined when the instance is created." ) #endIf if (not fromPath): raise MissingArgumentException( "The source path (fromPath) on the remote file system must be provided." ) #endIf if (not toPath): raise MissingArgumentException( "The destination path (toPath) on the local file system must be provided." ) #endIf if (TR.isLoggable(Level.FINER)): TR.finer( methodName, "From host: %s, copying: %s to %s" % (self.masterNode, fromPath, toPath)) #endIf retcode = call(["scp", "%s:%s" % (self.masterNode, fromPath), toPath]) if (retcode != 0): raise Exception("Error calling scp. Return code: %s" % retcode) #endIf if (TR.isLoggable(Level.FINEST)): TR.finest(methodName, "Copy completed")
def __init__(self, region=None, stackId=None, **restArgs): """ Constructor The region input is the AWS region where the EFS provisioner is running. The stackId input parameter is expected to be a AWS stack resource ID. The stackId is used to get the stack parameters among which is: EFSDNSName ApplicationStorageMountPoint EFSFileSystemId ClusterDNSName The restArgs keyword arguments include the following required parameters: playbookPath - the path to the playbook to use to configure EFS varTemplatePath - the path to the EFS configuration variable template manifestTemplatePath - the path to the EFS provisioner manifest YAML rbacTemplatePath - the path to the EFS provisioner RBAC YAML serviceAccountPath - the path to the EFS service account YAML """ object.__init__(self) if (not region): raise MissingArgumentException( "The AWS region name must be provided.") #endIf self.AWSRegion = region if (not stackId): raise MissingArgumentException( "The CloudFormation boot stack ID (stackId) must be provided.") #endIf self.stackId = stackId self.cfnResource = boto3.resource('cloudformation') self.home = os.path.expanduser("~") self._init(stackId, **restArgs)
def configureSecurity(self, configPath=None): """ Process one or more security configuration files, either given in the configPath or associated with the instance at the time it was created. """ if (not configPath): configPath = self.configPath else: self.configPath = configPath #endIf if (not configPath): raise MissingArgumentException("The security configuration file or directory path must be provided either at instance creation or on the method call.") #endIf if (os.path.isdir(configPath)): templateFiles = self.getYaml(configPath) if (not templateFiles): raise InvalidArgumentException("No .yaml files found in directory path: %s" % configPath) #endIf else: if (not os.path.isfile(configPath)): raise InvalidArgumentException("The given path is not a file: %s" % configPath) #endIf templateFiles = [configPath] #endIf stagingDir = os.path.join(os.getcwd(),'staging') # Configuration files get created in the staging directory. if (not os.path.exists(stagingDir)): os.mkdir(stagingDir) #endIf for template in templateFiles: baseName = os.path.basename(template) rootName,ext = os.path.splitext(baseName) configFilePath = os.path.join(stagingDir,"%s-config%s" % (rootName,ext)) self.createConfigFile(configFilePath=configFilePath, templateFilePath=template, parameters=self.IntrinsicVariables ) self._configureSecurity(configFilePath) #endFor #endDef #endClass
def invokeCommands(self, cmdDocs, start, **kwargs): """ Process command docs to invoke each command in sequence that is of kind s3. Processing of cmdDocs stops as soon as a doc kind that is not s3 is encountered. All cmdDocs that are processed are marked with a status attribute with the value PROCESSED. cmdDocs - a list of 1 or more YAML documents loaded from yaml.load_all() by the caller. start - index where to start processing in the cmdDocs list. NOTE: The method for each command is responsible for pulling out the arguments for the underlying S3 method. The S3 client methods only accept the arguments in the signature. Extraneous keyword arguments cause an exception to be raised. """ if (not cmdDocs): raise MissingArgumentException( "A non-empty list of command documents (cmdDocs) must be provided." ) #endIf for i in range(start, len(cmdDocs)): doc = cmdDocs[i] kind = doc.get('kind') if (not kind or kind != 's3'): break # done command = doc.get('command') if (not command): raise InvalidArgumentException( "A helm command document: %s, must have a command attribute." % doc) #endIf getattr(self, command)(**doc) doc['status'] = 'PROCESSED' #endFor #endDef #endClass
def invokeCommands(self, cmdDocs, start, **kwargs): """ Process command docs to invoke each command in sequence that is of kind docker. Processing of cmdDocs stops as soon as a doc kind that is not docker is encountered. All cmdDocs that are processed are marked with a status attribute with the value PROCESSED. cmdDocs - a list of 1 or more YAML documents loaded from yaml.load_all() by the caller. start - index where to start processing in the cmdDocs list. """ if (not cmdDocs): raise MissingArgumentException( "A non-empty list of command documents (cmdDocs) must be provided." ) #endIf for i in range(start, len(cmdDocs)): doc = cmdDocs[i] kind = doc.get('kind') if (not kind or kind != 'docker'): break # done command = doc.get('command') if (not command): raise InvalidArgumentException( "A docker command document: %s, must have a command attribute." % doc) #endIf getattr(self, command)(**doc) doc['status'] = 'PROCESSED' #endFor #endDef #endClass
def __init__(self, user='******', clusterName=None, masterNode=None): """ Constructor """ object.__init__(self) self.user = user self.masterNode = masterNode self.home = os.path.expanduser('~') self.kube = os.path.join(self.home, ".kube") if (not clusterName): raise MissingArgumentException( "The cluster name must be provided.") #endIf self.clusterName = clusterName
def _getVIPs(self, stackId): """ Return a dictionary where the key of each entry is the VIP name and the value associated with each key is the private IP address. The stack with the given stackId is expected to have two EC2::NetworkInterface resources that have a private IP address assigned to them. One IP address is used for the master VIP and the other is used for the proxy VIP. Each NetworkInterface has a Name tag that identifies which IP is to be used for the MasterVIP and which for the ProxyVIP. The specific IP that is used for a given VIP does not matter, but for the purpose of knowing which is being used for what, the NetworkInterface resources are named. """ vips = {} if (not stackId): raise MissingArgumentException("A stack ID (stackId) is required.") #endIf response = self.cfnClient.list_stack_resources(StackName=stackId) if (not response): raise AWSStackResourceException("Empty result for CloudFormation list_stack_resources for stack: %s" % stackId) #endIf stackResources = response.get('StackResourceSummaries') if (not stackResources): raise AWSStackResourceException("Empty StackResourceSummaries in response from CloudFormation list_stack_resources for stack: %s." % stackId) #endIf for resource in stackResources: resourceType = resource.get('ResourceType') if (resourceType == 'AWS::EC2::NetworkInterface'): interfaceId = resource.get('PhysicalResourceId') vipName,vipAddress = self._getVIPNameAndAddress(interfaceId) vips[vipName] = vipAddress #endIf #endFor return vips
def _getRequiredArgs(self, method, **kwargs): """ Return a list of required arguments for the given method """ requiredArgs = [] argNames = S3ClientMethodRequiredArgs.get(method) if (argNames): for argName in argNames: argValue = kwargs.get(argName) if (argValue == None): raise MissingArgumentException( "The S3 client method: '%s' requires a '%s' argument." % (method, argName)) #endIf requiredArgs.append(argValue) #endFor #endIF return requiredArgs
def runAnsiblePlaybook(self, playbook=None, extraVars=None, inventory="/etc/ansible/hosts"): """ Invoke a shell script to run an Ansible playbook with the given arguments. extraVars can be a list of argument values or a single string with space separated argument values. list example: [ "target_nodes=icp", "host_addres=9.876.54.32", "host_name=mycluster.example.com" ] string example: "target_nodes=icp host_addres=9.876.54.32 host_name=mycluster.example.com" """ methodName = "runAnsiblePlaybook" if (not playbook): raise MissingArgumentException("The playbook path must be provided.") #endIf try: if (extraVars): if (type(extraVars) != type([])): # Assume extraVars is a string with space separated values extraVars = extraVars.split() #endIf cmd = ["ansible-playbook", playbook, "--inventory", inventory] for var in extraVars: cmd.extend(["-e", "%s" % var]) #endFor TR.info(methodName, "Executing: cmd: %s" % cmd) retcode = call(cmd) else: TR.info(methodName, 'Executing: ansible-playbook %s, --inventory %s.' % (playbook,inventory)) retcode = call(["ansible-playbook", playbook, "--inventory", inventory ] ) #endIf if (retcode != 0): raise Exception("Error calling ansible-playbook. Return code: %s" % retcode) else: TR.info(methodName,"ansible-playbook: %s completed." % playbook) #endIf except Exception as e: TR.error(methodName,"Error calling ansible-playbook: %s" % e, e) raise
def createBucket(self, bucketName, region=None): """ Return an instance of S3 bucket either for a bucket that already exists or for a newly created bucket in the given region. NOTE: Region is required, either on the method call or to the S3Helper instance. """ methodName = "createBucket" bucket = None if (self.bucketExists(bucketName)): bucket = self.s3Resource.Bucket(bucketName) else: if (region): response = self.s3Client.create_bucket( Bucket=bucketName, CreateBucketConfiguration={'LocationConstraint': region}) elif (self.region): response = self.s3Client.create_bucket( Bucket=bucketName, CreateBucketConfiguration={ 'LocationConstraint': self.region }) else: raise MissingArgumentException( "The AWS region name for the bucket must be provided either to the S3Helper instance or in the createBucket() arguments." ) #endIf if (TR.isLoggable(Level.FINE)): TR.fine( methodName, "Bucket: %s created in region: %s" % (bucketName, response.get('Location'))) #endIf bucket = self.s3Resource.Bucket(bucketName) #endIf return bucket
def mountEFSVolumes(volumes, options=""): """ Mount the EFS storage volumes. volumes is either a singleton instance of EFSVolume or a list of instances of EFSVolume. EFSVolume has everything needed to mount the volume on a given mount point. NOTE: The default instance mount options are very likely correct. Don't mess with the mount options unless you know what you are doing. """ if (not volumes): raise MissingArgumentException( "One or more EFS volumes must be provided.") #endIf if (type(volumes) != type([])): volumes = [volumes] #endIf for volume in volumes: volume.mount(options)