def __init__(self, region=None, stackId=None, **restArgs):
"""
Constructor
The region input is the AWS region where the EFS provisioner is running.
The stackId input parameter is expected to be a AWS stack resource ID.
The stackId is used to get the stack parameters among which is:
EFSDNSName
ApplicationStorageMountPoint
EFSFileSystemId
ClusterDNSName
The restArgs keyword arguments include the following required parameters:
playbookPath - the path to the playbook to use to configure EFS
varTemplatePath - the path to the EFS configuration variable template
manifestTemplatePath - the path to the EFS provisioner manifest YAML
rbacTemplatePath - the path to the EFS provisioner RBAC YAML
serviceAccountPath - the path to the EFS service account YAML
"""
object.__init__(self)
if (not region):
raise MissingArgumentException("The AWS region name must be provided.")
#endIf
self.AWSRegion = region
if (not stackId):
raise MissingArgumentException("The CloudFormation boot stack ID (stackId) must be provided.")
#endIf
self.stackId = stackId
self.cfnResource = boto3.resource('cloudformation',region)
self.home = os.path.expanduser("~")
self._init(stackId, **restArgs)
def fillInDefaultValues(self, parameterNames=None, defaultValues=None, **restArgs):
"""
Return a dictionary with values for each parameter in parameterNames that is
the value in restArgs or the default value in defaultValues. Ff the parameter
is not defined in restArgs the default value is used.
"""
result = {}
if (not parameterNames):
raise MissingArgumentException("A parameter names list must be provided.")
#endIf
if (not defaultValues):
raise MissingArgumentException("A dictionary of default values must be provided.")
#endIf
for parmName in parameterNames:
parmValue = restArgs.get(parmName,defaultValues.get(parmName))
if (parmValue):
result[parmName] = parmValue
#endIf
#endFor
return result
def getS3Object(self, bucket=None, s3Path=None, destPath=None):
"""
Return destPath which is the local file path provided as the destination of the download.
A pre-signed URL is created and used to download the object from the given S3 bucket
with the given S3 key (s3Path) to the given local file system destination (destPath).
The destination path is assumed to be a full path to the target destination for
the object.
If the directory of the destPath does not exist it is created.
It is assumed the objects to be gotten are large binary objects.
For details on how to download a large file with the requests package see:
https://stackoverflow.com/questions/16694907/how-to-download-large-file-in-python-with-requests-py
"""
methodName = "getS3Object"
if (not bucket):
raise MissingArgumentException("An S3 bucket name (bucket) must be provided.")
#endIf
if (not s3Path):
raise MissingArgumentException("An S3 object key (s3Path) must be provided.")
#endIf
if (not destPath):
raise MissingArgumentException("A file destination path (destPath) must be provided.")
#endIf
TR.info(methodName, "STARTED download of object: %s from bucket: %s, to: %s" % (s3Path,bucket,destPath))
s3url = self.s3.generate_presigned_url(ClientMethod='get_object',Params={'Bucket': bucket, 'Key': s3Path},ExpiresIn=60)
TR.fine(methodName,"Getting S3 object with pre-signed URL: %s" % s3url)
#endIf
destDir = os.path.dirname(destPath)
if (not os.path.exists(destDir)):
os.makedirs(destDir)
TR.info(methodName,"Created object destination directory: %s" % destDir)
#endIf
r = requests.get(s3url, stream=True)
with open(destPath, 'wb') as destFile:
shutil.copyfileobj(r.raw, destFile)
#endWith
TR.info(methodName, "COMPLETED download from bucket: %s, object: %s, to: %s" % (bucket,s3Path,destPath))
return destPath
def __init__(self, region=None, bucket=None, keyPrefix='logs', fqdn=None):
"""
Constructor
region - the AWS region name
bucket - the S3 bucket name. The bucket gets created if it does not exist.
keyPrefix - the S3 key prefix to be used for each log export to S3,
e.g., logs/<stackname> where <stackname> is the name of the CloudFormation
stack associated with the root template for a given deployment.
The root stack name is unique.
Using logs as the beginning of the prefix keeps all logs in a
separate "folder" of the bucket.
fqdn - fully qualified domain name of the node exporting the logs
The FQDN provides uniqueness as there may be more than one node
with a given role.
"""
object.__init__(self)
if (not region):
raise MissingArgumentException(
"The AWS region name must be provided.")
#endIf
self.region = region
if (not bucket):
raise MissingArgumentException(
"The S3 bucket name for the exported logs must be provided.")
#endIf
self.bucket = bucket
self.keyPrefix = keyPrefix
if (not fqdn):
raise MissingArgumentException(
"The FQDN of the node exporting the logs must be provided.")
#endIf
self.fqdn = fqdn
self.s3Helper = S3Helper(region=region)
if (not self.s3Helper.bucketExists(bucket)):
self.s3Helper.createBucket(bucket, region=region)
def invokeCommands(self, cmdDocs, start, **kwargs):
"""
Process command docs to invoke each command in sequence that is of kind s3.
Processing of cmdDocs stops as soon as a doc kind that is not s3 is encountered.
All cmdDocs that are processed are marked with a status attribute with the value PROCESSED.
cmdDocs - a list of 1 or more YAML documents loaded from yaml.load_all()
by the caller.
start - index where to start processing in the cmdDocs list.
NOTE: The method for each command is responsible for pulling out the arguments for the
underlying S3 method. The S3 client methods only accept the arguments in the signature.
Extraneous keyword arguments cause an exception to be raised.
"""
if (not cmdDocs):
raise MissingArgumentException(
"A non-empty list of command documents (cmdDocs) must be provided."
)
#endIf
for i in range(start, len(cmdDocs)):
doc = cmdDocs[i]
kind = doc.get('kind')
if (not kind or kind != 's3'):
break
# done
command = doc.get('command')
if (not command):
raise InvalidArgumentException(
"A helm command document: %s, must have a command attribute."
% doc)
#endIf
getattr(self, command)(**doc)
doc['status'] = 'PROCESSED'
#endFor
#endDef
#endClass
def _getRequiredArgs(self, method, **kwargs):
"""
Return a list of required arguments for the given method
"""
requiredArgs = []
argNames = S3ClientMethodRequiredArgs.get(method)
if (argNames):
for argName in argNames:
argValue = kwargs.get(argName)
if (argValue == None):
raise MissingArgumentException(
"The S3 client method: '%s' requires a '%s' argument."
% (method, argName))
#endIf
requiredArgs.append(argValue)
#endFor
#endIF
return requiredArgs
def createBucket(self, bucketName, region=None):
"""
Return an instance of S3 bucket either for a bucket that already
exists or for a newly created bucket in the given region.
NOTE: Region is required, either on the method call or to the S3Helper instance.
"""
methodName = "createBucket"
bucket = None
if (self.bucketExists(bucketName)):
bucket = self.s3Resource.Bucket(bucketName)
else:
if (region):
response = self.s3Client.create_bucket(
Bucket=bucketName,
CreateBucketConfiguration={'LocationConstraint': region})
elif (self.region):
response = self.s3Client.create_bucket(
Bucket=bucketName,
CreateBucketConfiguration={
'LocationConstraint': self.region
})
else:
raise MissingArgumentException(
"The AWS region name for the bucket must be provided either to the S3Helper instance or in the createBucket() arguments."
)
#endIf
if (TR.isLoggable(Level.FINE)):
TR.fine(
methodName, "Bucket: %s created in region: %s" %
(bucketName, response.get('Location')))
#endIf
bucket = self.s3Resource.Bucket(bucketName)
#endIf
return bucket
def runAnsiblePlaybook(self, playbook=None, extraVars="efs-config-vars.yaml", inventory="/etc/ansible/hosts"):
"""
Invoke a shell script to run an Ansible playbook with the given arguments.
NOTE: Work-around because I can't get the Ansible Python libraries figured out on Unbuntu.
"""
methodName = "runAnsiblePlaybook"
if (not playbook):
raise MissingArgumentException("The playbook path must be provided.")
#endIf
try:
TR.info(methodName,"Executing: ansible-playbook %s, --extra-vars @%s --inventory %s." % (playbook,extraVars,inventory))
retcode = call(["ansible-playbook", playbook, "--extra-vars", "@%s" % extraVars, "--inventory", inventory ] )
if (retcode != 0):
raise Exception("Error calling ansible-playbook. Return code: %s" % retcode)
else:
TR.info(methodName,"ansible-playbook: %s completed." % playbook)
#endIf
except Exception as e:
TR.error(methodName,"Error calling ansible-playbook: %s" % e, e)
raise
def createConfigFile(self, configFilePath=None, templateFilePath=None, **restArgs):
"""
Using the template file, fill in all the variable strings in the template
using the given parameters.
Required restArgs:
parameters: Dictionary with the actual values of the parameters
The parameter values will be substituted for the corresponding
macro in the template file to create the configuration file.
keywordMap: The mapping of parameter names to macro names (keywords) in the
template file.
Optional restArgs:
multipleAppearences: List of parameters that appear more than once in the
template file. Defaults to the empty list.
specialValues: Dictionary used for macro name that requires a special format
string to be used when doing the macro substitution.
Defaults to an empty dictionary.
Comment lines in the template file are written immediately to the config file.
Parameters that appear in more than one line in the template file need to be
in the given mulitpleAppearances list.
NOTE: It is assumed that a line in the configuration template file has at most
one parameter defined in it.
A macro in the template file is delimited by ${} with the parameter name in the {}.
"""
methodName = "createConfigFile"
if (not configFilePath):
raise MissingArgumentException("The configuration file path must be provided.")
#endIf
if (not templateFilePath):
raise MissingArgumentException("The template file path must be provided.")
#endIf
parameters = restArgs.get('parameters')
if (not parameters):
raise MissingArgumentException("Parameters must be provided.")
#endIf
keywordMap = restArgs.get('keywordMap')
if (not keywordMap):
raise MissingArgumentException("Keyword mappings must be provided.")
#endIf
multipleAppearances = restArgs.get('multipleAppearances',[])
specialValues = restArgs.get('specialValues',{})
specialValueNames = specialValues.keys()
parameterNames = parameters.keys()
try:
with open(templateFilePath,'r') as templateFile, open(configFilePath,'w') as configFile:
for line in templateFile:
# Need to strip at least the newline character(s)
line = line.rstrip()
if (line.startswith('#')):
configFile.write("%s\n" % line)
else:
parmName,macroName = self.checkForParm(line,parameterNames,keywordMap)
if (not parmName):
configFile.write("%s\n" % line)
else:
parmValue = parameters[parmName]
if (specialValueNames and macroName in specialValueNames):
specialFormat = specialValues.get(macroName)
parmValue = specialFormat.format(parmValue)
#endIf
macro = "${%s}" % macroName
if (TR.isLoggable(Level.FINEST)):
TR.finest(methodName,"LINE: %s\n\tReplacing: %s with: %s" % (line,macro,parmValue))
#endIf
newline = line.replace(macro,"%s" % parmValue)
if (TR.isLoggable(Level.FINEST)):
TR.finest(methodName,"NEW LINE: %s" % newline)
#endIf
configFile.write("%s\n" % newline)
if (macroName not in multipleAppearances):
# Only remove parmName from parameterNames when macroName
# does not appear more than once in the template file.
parameterNames.remove(parmName)
#endIf
#endIf
#endIf
#endFor
#endWith
except IOError as e:
TR.error(methodName,"IOError creating configuration variable file: %s from template file: %s" % (configFilePath,templateFilePath), e)
raise
def _init(self, stackId, **restArgs):
"""
Instance initialization constructor helper.
The stackIds input parameter is expected to be a list of AWS stack resource IDs.
The first stack ID in the list is assumed to be the root stack.
The restArgs are keyword arguments that include the following:
playbookPath - the path to the playbook to use
varTemplatePath - the path to the EFS configuration variable template
"""
global StackParameters, StackParameterNames
playbookPath = restArgs.get('playbookPath')
if (not playbookPath):
raise MissingArgumentException("A ploybook path (playbookPath) must be provided.")
#endIf
self.playbookPath = playbookPath
varTemplatePath = restArgs.get('varTemplatePath')
if (not varTemplatePath):
raise MissingArgumentException("An EFS configuration variable template file path (varTemplatePath) must be provided.")
#endIf
self.varTemplatePath = varTemplatePath
self.varFilePath = os.path.join(self.home,"efs-config-vars.yaml")
manifestTemplatePath = restArgs.get('manifestTemplatePath')
if (not manifestTemplatePath):
raise MissingArgumentException("The file path to the YAML defining the EFS provisioner (manifestTemplatePath) must be provided.")
#endIf
self.manifestTemplatePath = manifestTemplatePath
rbacTemplatePath = restArgs.get('rbacTemplatePath')
if (not rbacTemplatePath):
raise MissingArgumentException("The file path to the YAML defining the EFS provisioner RBAC (rbacTemplatePath) must be provided.")
#endIf
self.rbacTemplatePath = rbacTemplatePath
serviceAccountPath = restArgs.get('serviceAccountPath')
if (not serviceAccountPath):
raise MissingArgumentException("The file path to YAML defining the EFS service account must be provided.")
#endIf
self.serviceAccountPath = serviceAccountPath
StackParameters = self.getStackParameters(stackId)
StackParameterNames = StackParameters.keys()
efsParms = self.getEFSParameters()
TR.info("_init","%s "% efsParms)
self.efsParameters = self.fillInDefaultValues(parameterNames=EFSParameterNames,defaultValues=EFSDefaultParameterValues,**efsParms)
self.efsParameterNames = self.efsParameters.keys()
self.efsProvPath = os.path.join(self.home,"efs-provisioner.yaml")
self.provisionerParameters = self.getProvisionerParameters()
self.efsRBACPath = os.path.join(self.home,"efs-rbac.yaml")
self.rbacParameters = self.getRBACParameters()