def test_GatewayALL(self,
target,
aconnect,
acct,
acctName,
global_accts,
targetAPI=None):
print("test_GatewayALL")
Acct_yaml = "Ansible"
opt = "main_%s.yaml" % (global_accts[acct]['all'])
rootFolder = "%s/%s" % (self.temp, target) # roles/xx
src = "%s/%s/%s" % (rootFolder, 'defaults', opt
) # roles/xx/defaults/main_bla.yaml
fileIN = None
default = loadYaml(src)
integratedAPIs, stages, final_MODELS, final_AUTHS = define(
'*', '*', aconnect, target, targetAPI)
print(" look for Yaml...%s" % (src))
apis = default[target]['api_gw']
missing = []
for api in apis:
path = api['path']
method = api['httpMethod']
found = False
for inter in integratedAPIs:
if inter['path'] == path and inter['httpMethod'] == method:
found = True
break
if not found:
missing.append(api)
build4Options = False
for missed in missing:
if missed['httpMethod'] == 'OPTIONS':
build4Options = True
break
if build4Options: # replace the skipping section
skipping = {
"methods": True,
"models": True,
"resources": True,
"options": False,
"stage": False
}
default[target]['skipping'] = skipping
writeYaml(default, src, targetString)
def load_playbook(bucket, rootkey, time, target, output):
s3 = boto3.resource('s3')
tasks = '%s/%s/%s/tasks_main.yaml' % (rootkey, target, time)
defaults = '%s/%s/%s/defaults_main.yaml' % (rootkey, target, time)
ansibleRoot = '/tmp/root_ansible/'
move_resources('root_ansible', '/tmp')
print("============ WRITTING TO TEMP -=[%s]=========" % (ansibleRoot))
s3.meta.client.download_file(
bucket, key, '%sroles/%s/defaults/main.yaml' % (ansibleRoot, target))
s3.meta.client.download_file(
bucket, key, '%sroles/%s/tasks/main.yaml' % (ansibleRoot, target))
targets = ['%s' % target]
rootYML = [{
"name": "nano modler for gateway-%s" % target,
"hosts": "dev",
"remote_user": "******",
"roles": targets
}]
#ansibleRoot
writeYaml(rootYML, ansibleRoot, "CR-%s" % target)
print("============ WRITTING ROOT FILE -==========")
def lambda_describe(self, target, aconnect, accountOrigin, accounts=[], types=[], sendto=None, targetAPI=None, isFullUpdate=False):
if len(types) == 0:
types = ['apigw']
else:
if 's3' in types:
print(
'[W] warning buckets will not be created only updated with triggers found')
sts = None
acctTitle = None
self.origin = accountOrigin
acctID = accountOrigin['account']
assumeRole = accountOrigin['assume_role']
lambdaM, zipName = self.method_describe(target, aconnect)
NETWORK_MAP = loadServicesMap(accountOrigin['services_map'], 'RDS')
TOKEN_MAP = loadServicesMap(accountOrigin['services_map'], 'token')
COGNITO_MAP = loadServicesMap(accountOrigin['services_map'], 'cognito')
XACT_MAP = loadServicesMap(accountOrigin['services_map'], 'xact')
BUCKET_MAP = loadServicesMap(accountOrigin['services_map'], 'S3')
SLACK_MAP = loadServicesMap(accountOrigin['services_map'], 'slack')
SIGNER_MAP = loadServicesMap(accountOrigin['services_map'], 'signer')
DOMAIN_MAP = loadServicesMap(accountOrigin['services_map'], 'domains')
CFRONT_MAP = loadServicesMap(
accountOrigin['services_map'], 'cloudfront')
skipping = error_path = None
if 'error_path' in accountOrigin:
error_path = accountOrigin['error_path']
if 'skipping' in accountOrigin:
skipping = accountOrigin['skipping']
# error_path: /Users/astro_sk/Documents/TFS/Ansible_Deployer
if not skipping:
skipping = {
"methods": False,
"options": False,
"models": False,
"stage": False,
"resources": False
}
roles, resourceRole = describe_role(
lambdaM.lrole, aconnect, self.origin['account'], True if 'api' in types else False)
layers = self.layer_describe(lambdaM.layers, aconnect)
if "python" not in lambdaM.runtime:
print(" ##### WARNING #####")
print(" [W] runtime using [%s] is depreciated please use [Python 3]!!!" % (
lambdaM.runtime))
print(" ##### END WARNING. #####")
print(" [W] ROLE being used for API(s) %s" % resourceRole)
apis = models = authorizeDict = authorizers = authLambdas = None
if 'api' in types:
try:
if targetAPI:
apis, stages, models, authorizeDict = self.describe_gateway(
target, 'lambda', aconnect, resourceRole, targetAPI)
# apis, stages, models, authorizeDict = self.describe_gateway('*','lambda', aconnect, resourceRole, targetAPI)
except Exception as e:
msg = "%s :: :: %s" % (e, traceback.format_exc())
print(msg)
print(" COULD it be a different API tree %s" % (targetAPI))
raise e
# print (authorizeDict)
# raise
events = None
buckets = None
test = True
if not test:
if 'cloudwatch' in types:
events = self.describe_cloudevents(
target, lambdaM.farn, aconnect)
if 's3' in types:
buckets = self.describe_s3events(target, aconnect)
raise
dynoTriggers = None
if 'dynamodb' in types:
dynoTriggers = self.describe_dynoTriggers(target, aconnect)
# print(dynoTriggers)
# raise
taskMain, rootFolder, targetLabel = ansibleSetup(
self.temp, target, isFullUpdate)
if layers:
print("Layer functions currently not supported!!")
# taskMain.insert(2, {"import_tasks": "../aws/layers.yml", "vars": {"project": '{{ project }}'}})
###########################
# AUTH LAMBDAS
###########################
if authorizeDict:
authorizers = authorizeDict
authLambdas = []
print(authorizeDict)
for authIN in authorizeDict:
if authIN['authType'] in 'custom':
l_arn = authIN['authorizerUri'].split(":function:")[1]
authTARGET = l_arn.split("/invocations")[0]
authLam, authZipName = self.method_describe(
authTARGET, aconnect)
authRoles, authResourceRole = describe_role(
authLam.lrole, aconnect, self.origin['account'], True)
zipFile = "%s.zip" % authTARGET
zipLambda = "%s/%s/%s" % (rootFolder, 'files', zipFile)
copyfile(authZipName, zipLambda)
rolename = authLam.lrole.split(":role/")[1]
aLambda = {
"name": authTARGET,
"state": "present",
"runtime": authLam.runtime,
"timeout": authLam.timeout,
"description": authLam.description,
"role": rolename,
"handler": authLam.handler,
"memory_size": authLam.memory,
"alias": authLam.alias,
# "code_location": "", ### this is needed to prevent issues with the bundled lambda
"zip_file": "{{ role_path }}/files/%s" % zipFile
}
if not authLam.envars is None:
aLambda.update(
{"environment_variables": authLam.envars['Variables']})
authLambdas.append(aLambda)
###########################
# AUTH LAMBDAS
###########################
if apis:
if authorizeDict:
taskMain.append(
{"import_tasks": "../aws/agw_authorizer.yml", "vars": {"project": '{{ project }}'}})
taskMain.append({"import_tasks": "../aws/agw_model.yml",
"vars": {"project": '{{ project }}'}})
taskMain.append({"import_tasks": "../aws/_agw.yml",
"vars": {"project": '{{ project }}'}})
if events:
taskMain.append({"import_tasks": "../aws/cldwatch_rule.yml",
"vars": {"project": '{{ project }}'}})
if buckets:
taskMain.append({"import_tasks": "../aws/s3.yml",
"vars": {"project": '{{ project }}'}})
if dynoTriggers:
print("in triggers for dynamo")
taskMain.append({"import_tasks": "../aws/lambda_dyno_triggers.yml",
"vars": {"project": '{{ project }}'}})
# raise
#############################################
#############################################
# ####### write YAML to file in tasks #######
#############################################
#############################################
option = "main"
mainIn = "%s/%s/%s" % (rootFolder, 'tasks', option)
writeYaml(taskMain, mainIn)
#############################################
# ########## END WRITE ####################
#############################################
#############################################
# print(NETWORK_MAP)
# raise ValueError(" stopping now for check...")
# CREATE DEFAULT in "defaults" VARS
if 'services_map' in accountOrigin:
mapfile = accountOrigin['services_map']
serviceMap = loadServicesMap(mapfile, None)
for akey, account in accounts.items():
networkObj = NETWORK_MAP[akey]
bucketObj = BUCKET_MAP[akey]
cfrontObj = CFRONT_MAP[akey]
domainObj = DOMAIN_MAP[akey]
cognitoObj = COGNITO_MAP[akey]
xactObj = XACT_MAP[akey]
tokenObj = TOKEN_MAP[akey]
slackObj = SLACK_MAP[akey]
signerObj = SIGNER_MAP[akey]
if akey == acctID:
acctTitle = account['title']
eID = serviceID(akey, None, account['all'], serviceMap)
accDetail = {
"account_id": akey,
"error_path": error_path,
"skipping": skipping,
"env": account['title'],
"role_duration": 3600,
"region": "us-east-1",
"eid": eID,
"roles": [],
"policies": []
}
if assumeRole:
accDetail.update({"cross_acct_role": account['role']})
defaultVar = {targetLabel: accDetail}
# add Policies
role_list = []
role_policies = []
for role in roles:
rName = role['name']
rData = role['data']
rDescribe = "CR-Default no description found"
if 'Description' in rData:
rDescribe = rData['Description']
rNamePlcy = "%s_%s" % (rName, "trust")
trustIn = "%s/%s/%s" % (rootFolder, 'files', rNamePlcy)
# print ".... dude.....look up......"
rfile = writeJSON(rData['AssumeRolePolicyDocument'], trustIn)
#print (role)
# exit()
roleIn = {
"name": rName,
"trust_policy_filepath": "{{ role_path }}/files/%s" % rfile,
"type": "role",
"state": "present",
"path": rData['Path'],
"description": rDescribe
}
# polices are in seperate list!!!!!
plcies = role['policies']
plcyNames = []
for rp in plcies:
rpName = rp['PolicyName']
rpDoc = rp['PolicyDocument']
rpDescription = rp['Description']
rpPath = rp['Path']
fpIn = "%s/%s/%s" % (rootFolder, 'files', rpName)
pfile = writeJSON(rpDoc, fpIn)
plcyNames.append(rpName)
rPolicy = {
"name": rpName,
"state": "present",
"type": "policy",
"policy_document": "{{ role_path }}/files/%s" % pfile
}
role_policies.append(rPolicy)
roleIn.update({"action_policy_labels": plcyNames})
role_list.append(roleIn)
# CREATE POLICIES
defaultVar[targetLabel].update({"policies": role_policies})
defaultVar[targetLabel].update({"roles": role_list})
########################################################
#################### LAMBDAS ########################
########################################################
tempRname = role_list[0]['name']
# move zip file
zipFile = "%s.zip" % target
zipLambda = "%s/%s/%s" % (rootFolder, 'files', zipFile)
#os.rename("code.zip", zipLambda)
copyfile(zipName, zipLambda)
layer_list = []
for listLayer in layers:
lObj, zipLayer = listLayer
layerIn = {
"name": lObj.name,
"state": "present",
"version": lObj.version,
"arn": lObj.LayerArn,
"version_arn": lObj.LayerVersionArn,
"runtimes": lObj.CompatibleRuntimes,
"license": lObj.LicenseInfo,
"description": lObj.description,
"zip_file": "{{ role_path }}/files/%s" % zipLayer
}
layer_list.append(layerIn)
defaultVar[targetLabel].update({"lambda_layers": layer_list})
if not tempRname in lambdaM.lrole:
print("[E] role doesn't match %s with %s" %
(tempRname, lambdaM.lrole))
oLambda = {
"name": target,
"state": "present",
"runtime": lambdaM.runtime,
"timeout": lambdaM.timeout,
"description": lambdaM.description,
"role": tempRname,
"handler": lambdaM.handler,
"memory_size": lambdaM.memory,
"alias": lambdaM.alias,
# "code_location": "", ### this is needed to prevent issues with the bundled lambda
"zip_file": "{{ role_path }}/files/%s" % zipFile
}
if lambdaM.layers is not None:
oLambda.update({"layers": lambdaM.alias})
if lambdaM.envars is not None:
oLambda.update(
{"environment_variables": lambdaM.envars['Variables']})
if lambdaM.vpcs is not None:
if len(lambdaM.vpcs['SecurityGroupIds']) != 0:
#oLambda.update({"vpc_security_group_ids": lambdaM.vpcs['SecurityGroupIds']})
oLambda.update(
{"vpc_security_group_ids": networkObj['vpc_security_group_ids']})
if len(lambdaM.vpcs['SubnetIds']) != 0:
#oLambda.update({"vpc_subnet_ids": lambdaM.vpcs['SubnetIds']})
oLambda.update(
{"vpc_subnet_ids": networkObj['vpc_subnet_ids']})
if 'VpcId' in lambdaM.vpcs:
if len(lambdaM.vpcs['VpcId']) != 0:
#oLambda.update({"vpc": lambdaM.vpcs['VpcId']})
oLambda.update({"vpc": networkObj['vpc']})
lambdas = [oLambda]
if authorizers:
lambdas = lambdas + authLambdas
defaultVar[targetLabel].update({"lambdas": lambdas})
########################################################
#################### EVENTS ########################
########################################################
if not events is None: # NEW
crules = []
for rule in events:
description = "%s - Description by microMolder" % (
rule['Name'])
if 'Description' in rule:
description = rule['Description']
crules.append({
"name": rule['Name'],
"schedule_expression": rule['ScheduleExpression'],
"description": description,
"state": rule['State'],
"targets": rule['targets']
})
defaultVar[targetLabel].update({"cloud_rules": crules})
########################################################
############## S3 bucket Triggers ###################
########################################################
# print (". ---->>> BUCKETS. &^*&^*&^^&*^*&^&*%^&%^^")
bucket_list = []
if buckets: # bucketObj
for bkt in buckets:
# print (bkt)
# print (" .... #@@@#")
s3Trigger = {
'Id': bkt['config']['Id'],
# 'bucket': bkt['bucket'],
'bucket': bucketObj['bucket'],
'lambdaArn': bkt['config']['LambdaFunctionArn'],
'state': 'present',
# 'alias':bkt
'events': bkt['config']['Events']
}
if 'Filter' in bkt['config']:
s3Filter = bkt['config']['Filter']['Key']['FilterRules']
s3dict = {n['Name']: n['Value'] for n in s3Filter}
s3Trigger.update({'filter': s3dict})
bucket_list.append(s3Trigger)
defaultVar[targetLabel].update({"buckets": bucket_list})
########################################################
# ############# Dynamo DB Triggers ###################
########################################################
dyno_list = []
if dynoTriggers: # bucketObj
for trigger in dynoTriggers:
dyno_list.append(trigger)
defaultVar[targetLabel].update({"triggers_dynamo": dyno_list})
########################################################
############# API GATEWAY METHODS ###################
########################################################
#print (" A P I. see below. ......===---->>>")
api_list = []
stage_list = [] #
model_list = models # [] #
auth_list = authorizers # [] #
# stages.update({apiStage:{'stage':stageLabel,'api':apiName}})
if not apis is None:
# for mk,mv in models.items():
# model_list.append(mv)
for sk, sv in stages.items():
stage_list.append(sv)
for api in apis:
oApi = {
'name': api['name'],
'id': api['id'],
'credentials': "%s" % api['credentials'],
'authorizationType': api['authorizationType'],
'apiKeyRequired': api['apiKeyRequired'],
'type': api['type'],
'path': api['path'],
'operational_name': api['operationlabel'],
'request_valid': api['requestvalidator'],
'request_params': api['requestparameters'],
'auth_scope': api['authscope'],
'authName': api['authName'],
'request_models': api['requestmodels'],
'response_models': api['responsemodels'],
'httpMethod': api['httpMethod'],
'parentid': api['parentid'],
'method_response': api['methodResponse'],
'method_integration': api['methodIn'],
'state': api['state']
}
api_list.append(oApi)
defaultVar[targetLabel].update({"api_gw": api_list})
defaultVar[targetLabel].update({"api_stages": stage_list})
defaultVar[targetLabel].update({"api_models": model_list})
if auth_list:
defaultVar[targetLabel].update(
{"api_authorizers": auth_list})
#defaultVar[targetLabel].update({ "api_domains": stage_list })
#defaultVar[targetLabel].update({ "api_usage": stage_list })
option = "main_%s" % account['all']
mainIn = "%s/%s/%s" % (rootFolder, 'defaults', option)
writeYaml(defaultVar, mainIn)
account_replace("%s.yaml" % mainIn, str(acctID), str(akey))
for key, value in DOMAIN_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(
value), str(domainObj[key]))
for key, value in BUCKET_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(
value), str(bucketObj[key]))
for key, value in TOKEN_MAP[acctID].items():
account_replace("%s.yaml" %
mainIn, str(value), str(tokenObj[key]))
for key, value in NETWORK_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(
value), str(networkObj[key]))
for key, value in COGNITO_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(
value), str(cognitoObj[key]))
for key, value in XACT_MAP[acctID].items():
account_replace("%s.yaml" %
mainIn, str(value), str(xactObj[key]))
for key, value in SLACK_MAP[acctID].items():
account_replace("%s.yaml" %
mainIn, str(value), str(slackObj[key]))
for key, value in SIGNER_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(
value), str(signerObj[key]))
for key, value in CFRONT_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(
value), str(cfrontObj[key]))
print(" .... creating a main.yaml for ansible using dev")
opt = "main_%s.yaml" % accountOrigin['all']
src = "%s/%s/%s" % (rootFolder, 'defaults', opt)
opt2 = "main.yaml"
dst = "%s/%s/%s" % (rootFolder, 'defaults', opt2)
copyfile(src, dst)
print(" -------==------===---- COPY START....")
print(" sending to %s. from %s" % (sendto, rootFolder))
distutils.dir_util.copy_tree(rootFolder, sendto)
print(" -------==------===---- FINAL YAML file....")
ansibleRoot = sendto.split('roles/')[0]
targets = ['%s' % target]
rootYML = [{"name": "micro modler for lambda-%s" % target,
"hosts": "dev",
"remote_user": "******",
"roles": targets}]
# ansibleRoot
writeYaml(rootYML, ansibleRoot, target)
return acctID, target, acctTitle, True
def define(self, target, aconnect, accountOrigin, accounts=[], sendto=None):
self.origin = accountOrigin
acctID = accountOrigin['account']
assumeRole = accountOrigin['assume_role']
tableObj, triggers = self.behavior_describe(target, aconnect)
# for trigger in triggers:
# trigger
# acctTitle = None
skipping = error_path = None
if 'error_path' in accountOrigin:
error_path = accountOrigin['error_path']
if 'skipping' in accountOrigin:
skipping = accountOrigin['skipping']
# error_path: /Users/astro_sk/Documents/TFS/Ansible_Deployer
if not skipping:
skipping = {
"methods": False,
"options": False,
"models": False,
"stage": False,
"resources": False
}
taskMain, rootFolder, targetLabel = ansibleSetup(
self.temp, target, True)
taskWithFiles = [
{"import_tasks": "../aws/sts.yml", "vars": {"project": '{{ project }}'}},
{"import_tasks": "../aws/cr_dynamodb.yml",
"vars": {"project": '{{ project }}'}}
]
taskRaw = taskMain[0]
taskMain = [taskRaw] + taskWithFiles
#############################################
#############################################
# ####### write YAML to file in tasks #######
#############################################
#############################################
option = "main"
mainIn = "%s/%s/%s" % (rootFolder, 'tasks', option)
writeYaml(taskMain, mainIn)
#############################################
# ########## END WRITE ####################
#############################################
#############################################
if 'services_map' in accountOrigin:
mapfile = accountOrigin['services_map']
serviceMap = loadServicesMap(mapfile, None)
for akey, account in accounts.items():
if acctID == akey:
acctTitle = account['title']
eID = serviceID(akey, None, account['all'], serviceMap)
accDetail = {
"account_id": akey,
"env": account['title'],
"error_path": error_path,
"skipping": skipping,
"role_duration": 3600,
"region": "us-east-1",
"eid": eID
}
if assumeRole:
accDetail.update({"cross_acct_role": account['role']})
defaultVar = {targetLabel: accDetail}
defaultVar[targetLabel].update({"dynamodbs": [tableObj]})
if triggers:
defaultVar[targetLabel].update({"dynamo_triggers": triggers})
option = "main_%s" % account['all']
mainIn = "%s/%s/%s" % (rootFolder, 'defaults', option)
writeYaml(defaultVar, mainIn)
account_replace("%s.yaml" % mainIn, str(acctID), str(akey))
if sendto:
print(" .... creating a main.yaml for ansible using dev")
opt = "main_%s.yaml" % accountOrigin['all']
src = "%s/%s/%s" % (rootFolder, 'defaults', opt)
opt2 = "main.yaml"
dst = "%s/%s/%s" % (rootFolder, 'defaults', opt2)
copyfile(src, dst)
print(" -------==------===---- COPY START....")
print(" sending to %s. from %s" % (sendto, rootFolder))
distutils.dir_util.copy_tree(rootFolder, sendto)
ansibleRoot = sendto.split('roles/')[0]
targets = ['%s' % target]
rootYML = [{"name": "micro modler for lambda-%s" % target,
"hosts": "dev",
"remote_user": "******",
"roles": targets}]
# ansibleRoot
writeYaml(rootYML, ansibleRoot, target)
return acctID, target, acctTitle, True
def describe_GatewayALL(self,
target,
aconnect,
accountOrigin,
accounts=[],
types=[],
sendto=None,
targetAPI=None,
isFullUpdate=False):
print("describe_GatewayALL")
#describe_gateway(self, resourceNname, resourceType, aconnect , resourceRole=None,targetAPI=None):
isFullUpdate = False
directorysNeeded = False
skipFiles = True
#tmp="/tmp"
self.origin = accountOrigin
acctID = accountOrigin['account']
assumeRole = accountOrigin['assume_role']
#self.origin['account']
roles, resourceRole = describe_role("CR-Lambda-VPC", aconnect, acctID,
True if 'api' in types else False)
# (target,'lambda', aconnect, resourceRole, targetAPI)
apis, stages, models, auths = self.describe_gateway(
'*', '*', aconnect, resourceRole, targetAPI)
taskMain, rootFolder, targetLabel = ansibleSetup(
self.temp, target, isFullUpdate, skipFiles)
taskMain.append(
{"import_tasks": "../aws/agw_model.yml project={{ project }}"})
taskMain.append(
{"import_tasks": "../aws/_agw.yml project={{ project }}"})
#ONLY needs two files default definition and tasks
#############################################
#############################################
######## write YAML to file in tasks #######
#############################################
#############################################
#rootFolder=tmp
option = "tasks_main"
#mainIn = "%s/%s/%s"%(rootFolder,'tasks',option)
mainIn = "%s/%s" % (rootFolder, option)
writeYaml(taskMain, mainIn)
file_tasks = "%s.yaml" % mainIn
file_defaults = None
for akey, account in accounts.items():
#if not account in acctID:
if not acctID in akey:
continue
accDetail = {
"account_id": akey,
"env": account['title'],
"role_duration": 3600,
"region": "us-east-1",
"eid": account['eID'],
"roles": [],
"policies": []
}
if assumeRole:
accDetail.update({"cross_acct_role": account['role']})
defaultVar = {targetLabel: accDetail}
########################################################
############# API GATEWAY METHODS ###################
########################################################
#print (" A P I. see below. ......===---->>>")
api_list = []
stage_list = [] #
model_list = models #[] #
# stages.update({apiStage:{'stage':stageLabel,'api':apiName}})
if not apis is None:
#for mk,mv in models.items():
# model_list.append(mv)
for sk, sv in stages.items():
stage_list.append(sv)
for api in apis:
oApi = {
'name': api['name'],
'id': api['id'],
'credentials': "%s" % api['credentials'],
'authorizationType': api['authorizationType'],
'apiKeyRequired': api['apiKeyRequired'],
'type': api['type'],
'path': api['path'],
'operational_name': api['operationlabel'],
'request_valid': api['requestvalidator'],
'request_params': api['requestparameters'],
'auth_scope': api['authscope'],
'authName': api['authName'],
'request_models': api['requestmodels'],
'response_models': api['responsemodels'],
'httpMethod': api['httpMethod'],
'parentid': api['parentid'],
'method_response': api['methodResponse'],
'method_integration': api['methodIn'],
'state': api['state']
}
api_list.append(oApi)
defaultVar[targetLabel].update({"api_gw": api_list})
defaultVar[targetLabel].update({"api_stages": stage_list})
defaultVar[targetLabel].update({"api_models": model_list})
defaultVar[targetLabel].update({"api_authorizers": auths})
#defaultVar[targetLabel].update({ "api_domains": stage_list })
#defaultVar[targetLabel].update({ "api_usage": stage_list })
#option = "defaults_main%s"%account['all']
option = "defaults_main"
mainIn = "%s/%s" % (rootFolder, option)
#mainIn = "%s/%s/%s"%(rootFolder,'defaults',option)
file_defaults = "%s.yaml" % mainIn
#CREATE default with all vars
writeYaml(defaultVar, mainIn)
account_replace(file_defaults, str(acctID), str(akey))
if directorysNeeded:
if not sendto is None:
print(" .... creating a main.yaml for ansible using dev")
opt = "main_%s.yaml" % accountOrigin['all']
src = "%s/%s/%s" % (rootFolder, 'defaults', opt)
opt2 = "main.yaml"
dst = "%s/%s/%s" % (rootFolder, 'defaults', opt2)
copyfile(src, dst)
print(" -------==------===---- COPY START....")
print(" sending to %s. from %s" % (sendto, rootFolder))
distutils.dir_util.copy_tree(rootFolder, sendto)
print(" -------==------===---- FINAL YAML file....")
ansibleRoot = sendto.split('roles/')[0]
targets = ['%s' % target]
rootYML = [{
"name": "micro modler for lambda-%s" % target,
"hosts": "dev",
"remote_user": "******",
"roles": targets
}]
#ansibleRoot
writeYaml(rootYML, ansibleRoot, target)
print(file_tasks)
return file_tasks, file_defaults
def make_ansible(self,
sendto,
zipName,
testObjs,
targetRole,
accountOrigin,
accounts,
role_alias=None):
main_role = None
errors = []
if role_alias:
main_role = role_alias
else:
main_role = targetRole
label_main_role = role_yaml_clean(main_role)
label_sub_role = role_yaml_clean(targetRole)
taskMain = []
if self.incremented == 0:
defaultVar = {label_main_role: {label_sub_role: {}}}
# get initial setup and add as you need to before writting to file system
taskMain, rootFolder, targetLabel = ansibleSetup(
self.temp, main_role, True)
if role_alias: # already exists grab latest from file
taskMain_plus, defaultVar, rootFolder, targetLabel = ansible_describe_setup(
self.temp, main_role, targetRole)
taskMain = taskMain + taskMain_plus
defaultVar.update({label_sub_role: {}})
if self.incremented == 0:
taskMain = taskMain[0:2]
self.zip_files_move(testObjs, targetRole, rootFolder, zipName)
defaultVar[label_main_role][label_sub_role].update(testObjs)
#############################################
#############################################
acctID = accountOrigin['account']
m_acctID = acctID.split("_")[0]
assumeRole = accountOrigin['assume_role']
NETWORK_MAP = loadServicesMap(accountOrigin['services_map'], 'RDS')
TOKEN_MAP = loadServicesMap(accountOrigin['services_map'], 'token')
COGNITO_MAP = loadServicesMap(accountOrigin['services_map'], 'cognito')
XACT_MAP = loadServicesMap(accountOrigin['services_map'], 'xact')
BUCKET_MAP = loadServicesMap(accountOrigin['services_map'], 'S3')
SLACK_MAP = loadServicesMap(accountOrigin['services_map'], 'slack')
SIGNER_MAP = loadServicesMap(accountOrigin['services_map'], 'signer')
DOMAIN_MAP = loadServicesMap(accountOrigin['services_map'], 'domains')
CFRONT_MAP = loadServicesMap(accountOrigin['services_map'],
'cloudfront')
skipping = error_path = None
acctTitle = "devlpmnt"
#############################################
#############################################
# ####### write YAML to file in tasks #######
#############################################
#############################################
# BELOW combine service and action to pull the correct Ansible module
# use for test sequence in order
test_actions = ['pre', 'run', 'assert', 'post']
for tk, tv in testObjs.items():
print(" adding module for %s" % (tk))
for state in test_actions:
if state in tv:
print("- - -- - ")
print(" ==K:%s V:%s" %
(tv[state]['service'], tv[state]['action']))
taskMain.append({
"import_tasks":
"../aws/test_%s_%s.yml" %
(tv[state]['service'], tv[state]['action']),
"vars": {
"project_local":
'{{ project.%s.%s.%s }}' %
(label_sub_role, tk, state)
}
})
else:
print("[W] state:%s NOT found in test file: %s" %
(state, tk))
option = "main"
mainIn = "%s/%s/%s" % (rootFolder, 'tasks', option)
writeYaml(taskMain, mainIn)
#############################################
# ########## END WRITE ####################
#############################################
#############################################
if 'services_map' in accountOrigin:
mapfile = accountOrigin['services_map']
serviceMap = loadServicesMap(mapfile, None)
for akey, account in accounts.items():
m_account = akey.split("_")[0]
if akey not in BUCKET_MAP:
print("[W] account:%s not found in MAP" % (akey))
continue
networkObj = NETWORK_MAP[akey]
bucketObj = BUCKET_MAP[akey]
cfrontObj = CFRONT_MAP[akey]
domainObj = DOMAIN_MAP[akey]
cognitoObj = COGNITO_MAP[akey]
xactObj = XACT_MAP[akey]
tokenObj = TOKEN_MAP[akey]
slackObj = SLACK_MAP[akey]
signerObj = SIGNER_MAP[akey]
if akey == acctID:
acctTitle = account['title']
eID = serviceID(akey, None, account['all'], serviceMap)
accDetail = {
"account_id": akey,
"error_path": error_path,
"skipping": skipping,
"env": acctTitle,
"role_duration": 3600,
"region": "us-east-1",
"eid": eID,
"roles": [],
"policies": []
}
if assumeRole:
accDetail.update({"cross_acct_role": account['role']})
print("*******************")
print(targetLabel)
print(defaultVar)
if defaultVar[targetLabel]:
defaultVar[targetLabel].update(accDetail)
else:
defaultVar = {targetLabel: accDetail}
option = "main_%s" % account['all']
mainIn = "%s/%s/%s" % (rootFolder, 'defaults', option)
writeYaml(defaultVar, mainIn)
account_replace("%s.yaml" % mainIn, str(m_acctID), str(m_account))
for key, value in DOMAIN_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(domainObj[key]))
for key, value in BUCKET_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(bucketObj[key]))
for key, value in TOKEN_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(tokenObj[key]))
for key, value in NETWORK_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(networkObj[key]))
for key, value in COGNITO_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(cognitoObj[key]))
for key, value in XACT_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(xactObj[key]))
for key, value in SLACK_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(slackObj[key]))
for key, value in SIGNER_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(signerObj[key]))
for key, value in CFRONT_MAP[acctID].items():
account_replace("%s.yaml" % mainIn, str(value),
str(cfrontObj[key]))
print(" .... creating a main.yaml for ansible using dev")
opt = "main_%s.yaml" % accountOrigin['all']
src = "%s/%s/%s" % (rootFolder, 'defaults', opt)
opt2 = "main.yaml"
dst = "%s/%s/%s" % (rootFolder, 'defaults', opt2)
copyfile(src, dst) # copy [dev] as main.yaml
print(" -------==------===---- COPY START....")
print(" sending to %s. from %s" % (sendto, rootFolder))
distutils.dir_util.copy_tree(rootFolder, sendto)
print(" -------==------===---- FINAL YAML file....")
ansibleRoot = sendto.split('roles/')[0]
# all tests if added below would require more loops per ROLE
targets = ['%s' % main_role]
rootYML = [{
"name": "micro modler for lambda-%s" % main_role,
"hosts": "dev",
"remote_user": "******",
"roles": targets
}]
# ansibleRoot
writeYaml(rootYML, ansibleRoot, main_role)
return errors
def cfront_describe(self,
target,
aconnect,
accountOrigin,
accounts=[],
sendto=None):
self.origin = accountOrigin
acctID = accountOrigin['account']
assumeRole = accountOrigin['assume_role']
cfrontM = self.behavior_describe(target, aconnect)
acctTitle = None
taskMain, rootFolder, targetLabel = ansibleSetup(
self.temp, target, True)
#############################################
#############################################
######## write YAML to file in tasks #######
#############################################
#############################################
option = "main"
mainIn = "%s/%s/%s" % (rootFolder, 'tasks', option)
writeYaml(taskMain, mainIn)
skipping = error_path = None
if 'error_path' in accountOrigin:
error_path = accountOrigin['error_path']
if 'skipping' in accountOrigin:
skipping = accountOrigin['skipping']
# error_path: /Users/astro_sk/Documents/TFS/Ansible_Deployer
if not skipping:
skipping = {
"methods": False,
"options": False,
"models": False,
"stage": False,
"resources": False
}
#############################################
########### END WRITE ####################
#############################################
#############################################
for akey, account in accounts.items():
if acctID == akey:
acctTitle = account['title']
accDetail = {
"account_id": akey,
"error_path": error_path,
"skipping": skipping,
"env": account['title'],
"role_duration": 3600,
"region": "us-east-1",
"eid": account['eID']
}
defaultVar = {targetLabel: accDetail}
cfront = {
'id': cfrontM.id,
'target': cfrontM.target,
's3': cfrontM.s3,
'status': cfrontM.status,
'arn': cfrontM.arn,
'activeSigners': cfrontM.activeSigners,
'distribution_config': cfrontM.distribution_config
}
defaultVar[targetLabel].update({"cloudfront": cfront})
option = "main_%s" % account['all']
mainIn = "%s/%s/%s" % (rootFolder, 'defaults', option)
writeYaml(defaultVar, mainIn)
account_replace("%s.yaml" % mainIn, str(acctID), str(akey))
if not sendto is None:
print(" .... creating a main.yaml for ansible using dev")
opt = "main_%s.yaml" % accountOrigin['all']
src = "%s/%s/%s" % (rootFolder, 'defaults', opt)
opt2 = "main.yaml"
dst = "%s/%s/%s" % (rootFolder, 'defaults', opt2)
copyfile(src, dst)
print(" -------==------===---- COPY START....")
print(" sending to %s. from %s" % (sendto, rootFolder))
distutils.dir_util.copy_tree(rootFolder, sendto)
return acctID, target, acctTitle, True