def _resolve_tfref_from_dict(tfref_var):
if "component" in tfref_var and "terraform" in tfref_var and ("paramName" in tfref_var or "jmespath" in tfref_var):
with Git() as git:
current_branch = git.get_current_branch()
if "branch" in tfref_var:
branch = tfref_var["branch"]
else:
branch = current_branch
tf_key = (tfref_var["component"], tfref_var["terraform"], branch)
if tf_key in terraforms:
terraform = terraforms[tf_key]
else:
root = "."
if branch != current_branch:
root = git.export_branch(branch)
terraform = pull_state(tfref_var["component"], tfref_var["terraform"], root=root)
terraforms[tf_key] = terraform
if "paramName" in tfref_var:
flat_state_dict = flat_state(terraform)
if tfref_var["paramName"] in flat_state_dict:
return flat_state_dict[tfref_var["paramName"]]
else:
return None
if "jmespath" in tfref_var:
return jmespath_var(terraform, tfref_var["jmespath"])
else:
return None
def __init__(self, root=".", branch=None):
if not branch:
self.branch = Git().get_current_branch()
else:
self.branch = branch
self.componets = []
self.root = root if root else guess_project_root()
self.all_subcomponents = []
def resolve_docker_uri(component, uriParam, image_branch, git):
if not git:
git = Git()
with git:
if uriParam in os.environ:
return os.environ[uriParam]
docker = uriParam[14:]
docker_params = load_parameters(component=component, docker=docker, branch=image_branch, git=git)
return repo_uri(docker_params['DOCKER_NAME'])
def resolve_ami(component_params, component, image, imagebranch, branch, git):
if not git:
git = Git()
with git:
if "paramAmi" + image in os.environ:
return { "ImageId": os.environ["paramAmi" + image],
"Name": os.environ["paramAmiName" + image] \
if "paramAmiName" + image in os.environ else "Unknown" }
images = []
image_params = {}
job = ""
if "IMAGE_JOB" in os.environ and not image:
job = re.sub(r'\W', '_', os.environ["IMAGE_JOB"])
else:
image_params = load_parameters(component=component, image=image, branch=imagebranch, git=git)
if "JOB_NAME" in image_params:
job = re.sub(r'\W', '_', image_params["JOB_NAME"])
else:
prefix = ""
prefix = image_params["JENKINS_JOB_PREFIX"]
job = prefix + "_" + component + "_bake"
if image:
job = job + "_" + image
job = re.sub(r'\W', '_', job)
build_param = "paramAmi" + image + "Build"
latest_baked = build_param in component_params and component_params[build_param] == 'latest'
if latest_baked:
# get current branch latest images
images = get_images(job)
if build_param in component_params and component_params[build_param] != 'latest':
# resolve with a specifically set image build number
build = component_params[build_param]
image_tag = job + "_" + build
job_tag_func = lambda image, image_name_prefix: len([tag for tag in image["Tags"] if tag["Value"] == image_tag]) > 0
images = get_images(job, job_tag_function=job_tag_func)
elif imagebranch != branch and not latest_baked:
# resolve promote job
suffix = "_bake"
repl_suffix = "_promote"
if image:
suffix += "_" + image
repl_suffix += "_" + image
if not image_params:
image_params = load_parameters(component=component, image=image, branch=imagebranch, git=git)
this_branch_prefix = re.sub(r'\W', '_', component_params["JENKINS_JOB_PREFIX"] + "_")
image_branch_prefix = re.sub(r'\W', '_', image_params["JENKINS_JOB_PREFIX"] + "_")
job = lreplace(image_branch_prefix, this_branch_prefix, job)
job = rreplace(suffix, repl_suffix, job)
images = get_images(job)
else:
# get current branch latest images
images = get_images(job)
if images:
return images[0]
else:
return None
def guess_project_root():
for guess in [
".",
Git().get_git_root(), "..", "../..", "../../..", "../../../.."
]:
if len(Project(root=guess).get_all_subcomponents()) > 0:
if guess == ".":
return guess
else:
return path.abspath(guess)
def cli_list_components():
""" Prints the components in a branch, by default the current branch """
parser = get_parser()
parser.add_argument("-j", "--json", action="store_true", help="Print in json format.")
parser.add_argument("-b", "--branch", help="The branch to get components from. Default is to process current branch").completer = \
ChoicesCompleter(Git().get_branches())
argcomplete.autocomplete(parser)
args = parser.parse_args()
ret = list_components(**vars(args))
if args.json:
print(json.dumps(ret, indent=2))
else:
print("\n".join(ret))
def cli_list_jobs():
""" Prints a line for every runnable job in this git repository, in all branches and
optionally exports the properties for each under '$root/job-properties/"""
parser = get_parser()
parser.add_argument("-e", "--export-job-properties", action="store_true",
help="Set if you want the properties of all jobs into files under job-properties/")
parser.add_argument("-j", "--json", action="store_true", help="Print in json format. Optionally " \
"exported parameters will be in the json document")
parser.add_argument("-b", "--branch", help="The branch to process. Default is to process all branches").completer = \
ChoicesCompleter(Git().get_branches())
parser.add_argument("-c", "--component", help="Component to process. Default is to process all components").completer = \
branch_components
argcomplete.autocomplete(parser)
args = parser.parse_args()
ret = list_jobs(**vars(args))
if args.json:
print(json.dumps(ret, indent=2))
else:
print("\n".join(ret))
def cli_load_parameters():
""" Load parameters from infra*.properties files in the order:
infra.properties,
infra-[branch].properties,
[component]/infra.properties,
[component]/infra-[branch].properties,
[component]/[subcomponent-type]-[subcomponent]/infra.properties,
[component]/[subcomponent-type]-[subcomponent]/infra-[branch].properties
Last parameter defined overwrites ones defined before in the files. Supports parameter expansion
and bash -like transformations. Namely:
${PARAM##prefix} # strip prefix greedy
${PARAM%%suffix} # strip suffix greedy
${PARAM#prefix} # strip prefix not greedy
${PARAM%suffix} # strip suffix not greedy
${PARAM:-default} # default if empty
${PARAM:4:2} # start:len
${PARAM/substr/replace}
${PARAM^} # upper initial
${PARAM,} # lower initial
${PARAM^^} # upper
${PARAM,,} # lower
Comment lines start with '#'
Lines can be continued by adding '\' at the end
See https://www.tldp.org/LDP/Bash-Beginners-Guide/html/sect_10_03.html
(arrays not supported)
"""
parser = get_parser(formatter=argparse.RawDescriptionHelpFormatter)
parser.add_argument("component", nargs="?", help="Compenent to descend into").completer = \
ChoicesCompleter([c.name for c in Project().get_components()])
parser.add_argument("--branch", "-b", help="Branch to get active parameters for").completer = \
ChoicesCompleter(Git().get_branches())
parser.add_argument("--resolve-images", "-r", action="store_true", help="Also resolve subcomponent AMI IDs and docker repo urls")
subcomponent_group = parser.add_mutually_exclusive_group()
subcomponent_group.add_argument("--stack", "-s", help="CloudFormation subcomponent to descent into").completer = \
lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("stack", prefix, parsed_args, **kwargs)
subcomponent_group.add_argument("--serverless", "-l", help="Serverless subcomponent to descent into").completer = \
lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("serverless", prefix, parsed_args, **kwargs)
subcomponent_group.add_argument("--docker", "-d", help="Docker image subcomponent to descent into").completer = \
lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("docker", prefix, parsed_args, **kwargs)
subcomponent_group.add_argument("--image", "-i", const="", nargs="?", help="AMI image subcomponent to descent into").completer = \
lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("image", prefix, parsed_args, **kwargs)
subcomponent_group.add_argument("--cdk", "-c", help="CDK subcomponent to descent into").completer = \
lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("cdk", prefix, parsed_args, **kwargs)
subcomponent_group.add_argument("--terraform", "-t", help="Terraform subcomponent to descent into").completer = \
lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("terraform", prefix, parsed_args, **kwargs)
format_group = parser.add_mutually_exclusive_group()
format_group.add_argument("--json", "-j", action="store_true", help="JSON format output (default)")
format_group.add_argument("--yaml", "-y", action="store_true", help="YAML format output")
format_group.add_argument("--properties", "-p", action="store_true", help="properties file format output")
format_group.add_argument("--export-statements", "-e", action="store_true",
help="Output as eval-able export statements")
argcomplete.autocomplete(parser)
args = parser.parse_args()
transform = json.dumps
if args.export_statements:
transform = map_to_exports
if args.properties:
transform = map_to_properties
if args.yaml:
transform = yaml.dump
del args.export_statements
del args.yaml
del args.json
del args.properties
if (args.stack or args.serverless or args.docker or not isinstance(args.image, NoneType)) \
and not args.component:
parser.error("image, stack, doker or serverless do not make sense without component")
print(transform(load_parameters(**vars(args))), end="")
def load_parameters(component=None, stack=None, serverless=None, docker=None, image=None,
cdk=None, terraform=None, branch=None, resolve_images=False,
git=None):
subc_name = ""
if stack:
subc_name = "stack=" + stack
if serverless:
subc_name = "serverless=" + serverless
if docker:
subc_name = "docker=" + docker
if isinstance(image, six.string_types):
subc_name = "image=" + image
if cdk:
subc_name = "cdk=" + cdk
if terraform:
subc_name = "terraform=" + terraform
if not git:
git = Git()
with git:
current_branch = git.get_current_branch()
if not branch:
branch = current_branch
branch = branch.strip().split("origin/")[-1:][0]
params_key = (component, subc_name, branch)
if params_key in parameters:
return parameters[params_key]
ret = {
"GIT_BRANCH": branch
}
account = resolve_account()
if account:
ret["ACCOUNT_ID"] = account
if component:
ret["COMPONENT"] = component
prefix = ""
if current_branch != branch:
prefix = git.export_branch(branch) + os.sep
files = [prefix + "infra.properties", prefix + "infra-" + branch + ".properties"]
if component:
files.append(prefix + component + os.sep + "infra.properties")
files.append(prefix + component + os.sep + "infra-" + branch + ".properties")
_add_subcomponent_file(prefix + component, branch, "stack", stack, files)
_add_subcomponent_file(prefix + component, branch, "serverless", serverless, files)
_add_subcomponent_file(prefix + component, branch, "cdk", cdk, files)
_add_subcomponent_file(prefix + component, branch, "terraform", terraform, files)
_add_subcomponent_file(prefix + component, branch, "docker", docker, files)
_add_subcomponent_file(prefix + component, branch, "image", image, files)
if isinstance(image, six.string_types):
files.append(prefix + component + os.sep + "image" + os.sep + "infra.properties")
files.append(prefix + component + os.sep + "image" + os.sep + "infra-" + branch + ".properties")
initial_resolve = ret.copy()
os.environ["DO_NOT_RESOLVE_EXTERNAL_REFS"] = "true"
for file in files:
if os.path.exists(file):
import_parameter_file(file, initial_resolve)
del os.environ["DO_NOT_RESOLVE_EXTERNAL_REFS"]
if "REGION" not in initial_resolve:
ret["REGION"] = region()
else:
ret["REGION"] = initial_resolve["REGION"]
if not "AWS_DEFAULT_REGION" in os.environ:
os.environ["AWS_DEFAULT_REGION"] = ret["REGION"]
if "paramEnvId" not in initial_resolve:
ret["paramEnvId"] = branch
else:
ret["paramEnvId"] = initial_resolve["paramEnvId"]
for file in files:
if os.path.exists(file):
import_parameter_file(file, ret)
if (serverless or stack or cdk or terraform) and resolve_images:
image_branch = branch
if 'BAKE_IMAGE_BRANCH' in ret:
image_branch = ret['BAKE_IMAGE_BRANCH']
for docker in [dockerdir.split("/docker-")[1] for dockerdir in glob(component + os.sep + "docker-*")]:
try:
ret['paramDockerUri' + docker] = resolve_docker_uri(component, 'paramDockerUri' + docker, image_branch, git)
except ClientError:
# Best effor to load docker uris, but ignore errors since the repo might not
# actually be in use. Missing and used uris will result in an error later.
pass
for image_name in [imagedir.split("/image")[1].replace("-", "").lower() for imagedir in glob(component + os.sep + "image*")]:
try:
image = resolve_ami(ret, component, image_name, image_branch, branch, git)
if image:
ret['paramAmi' + image_name] = image['ImageId']
ret['paramAmiName' + image_name] = image['Name']
env_param_name = "AMI_ID"
if image_name:
env_param_name += "_" + image_name.upper()
ret[env_param_name] = image['ImageId']
except ClientError:
# Best effor to load ami info, but ignore errors since the image might not
# actually be in use. Missing and used images will result in an error later.
pass
if "REGION" not in ret:
ret["REGION"] = region()
if "paramEnvId" not in ret:
ret["paramEnvId"] = branch
if "ORIG_STACK_NAME" in os.environ:
ret["ORIG_STACK_NAME"] = os.environ["ORIG_STACK_NAME"]
if "STACK_NAME" not in ret:
ret["STACK_NAME"] = component + "-" + ret["ORIG_STACK_NAME"] + "-" + ret["paramEnvId"]
for k, v in list(os.environ.items()):
if k.startswith("ORIG_") and k.endswith("_NAME"):
ret[k] = v
if "ORIG_DOCKER_NAME" in os.environ:
if "DOCKER_NAME" not in ret:
ret["DOCKER_NAME"] = component + "/" + ret["paramEnvId"] + "-" + ret["ORIG_DOCKER_NAME"]
if "JENKINS_JOB_PREFIX" not in ret:
ret["JENKINS_JOB_PREFIX"] = "ndt" + ret["paramEnvId"]
parameters[params_key] = ret
return ret
def list_jobs(export_job_properties=False,
branch=None,
json=False,
component=None):
ret = {"branches": []}
arr = []
param_files = {}
with Git() as git:
current_project = Project(root=guess_project_root())
if branch:
branches = [branch]
else:
branches = git.get_branches()
components = []
for c_branch in branches:
branch_obj = {"name": c_branch, "components": []}
ret["branches"].append(branch_obj)
if c_branch == git.get_current_branch():
project = current_project
else:
root = git.export_branch(c_branch)
project = Project(root=root, branch=c_branch)
if component:
c_component = project.get_component(component)
if not c_component:
print("No matching components")
if json:
return {}
else:
return []
branch_obj["components"].append({
"name": c_component.name,
"subcomponents": []
})
components.append(c_component)
else:
for c_component in project.get_components():
branch_obj["components"].append({
"name": c_component.name,
"subcomponents": []
})
components.append(c_component)
if not json and export_job_properties:
try:
mkdir(current_project.root + sep + "job-properties")
except OSError as err:
# Directory already exists is ok
if err.errno == 17:
pass
else:
raise err
if json:
_collect_json(components, ret, export_job_properties, git)
else:
arr, param_files = _collect_prop_files(components,
export_job_properties,
current_project.root, git)
if export_job_properties:
_write_prop_files(param_files)
if json:
return ret
else:
return arr