def _resolve_tfref_from_dict(tfref_var): if "component" in tfref_var and "terraform" in tfref_var and ("paramName" in tfref_var or "jmespath" in tfref_var): with Git() as git: current_branch = git.get_current_branch() if "branch" in tfref_var: branch = tfref_var["branch"] else: branch = current_branch tf_key = (tfref_var["component"], tfref_var["terraform"], branch) if tf_key in terraforms: terraform = terraforms[tf_key] else: root = "." if branch != current_branch: root = git.export_branch(branch) terraform = pull_state(tfref_var["component"], tfref_var["terraform"], root=root) terraforms[tf_key] = terraform if "paramName" in tfref_var: flat_state_dict = flat_state(terraform) if tfref_var["paramName"] in flat_state_dict: return flat_state_dict[tfref_var["paramName"]] else: return None if "jmespath" in tfref_var: return jmespath_var(terraform, tfref_var["jmespath"]) else: return None
def __init__(self, root=".", branch=None): if not branch: self.branch = Git().get_current_branch() else: self.branch = branch self.componets = [] self.root = root if root else guess_project_root() self.all_subcomponents = []
def resolve_docker_uri(component, uriParam, image_branch, git): if not git: git = Git() with git: if uriParam in os.environ: return os.environ[uriParam] docker = uriParam[14:] docker_params = load_parameters(component=component, docker=docker, branch=image_branch, git=git) return repo_uri(docker_params['DOCKER_NAME'])
def resolve_ami(component_params, component, image, imagebranch, branch, git): if not git: git = Git() with git: if "paramAmi" + image in os.environ: return { "ImageId": os.environ["paramAmi" + image], "Name": os.environ["paramAmiName" + image] \ if "paramAmiName" + image in os.environ else "Unknown" } images = [] image_params = {} job = "" if "IMAGE_JOB" in os.environ and not image: job = re.sub(r'\W', '_', os.environ["IMAGE_JOB"]) else: image_params = load_parameters(component=component, image=image, branch=imagebranch, git=git) if "JOB_NAME" in image_params: job = re.sub(r'\W', '_', image_params["JOB_NAME"]) else: prefix = "" prefix = image_params["JENKINS_JOB_PREFIX"] job = prefix + "_" + component + "_bake" if image: job = job + "_" + image job = re.sub(r'\W', '_', job) build_param = "paramAmi" + image + "Build" latest_baked = build_param in component_params and component_params[build_param] == 'latest' if latest_baked: # get current branch latest images images = get_images(job) if build_param in component_params and component_params[build_param] != 'latest': # resolve with a specifically set image build number build = component_params[build_param] image_tag = job + "_" + build job_tag_func = lambda image, image_name_prefix: len([tag for tag in image["Tags"] if tag["Value"] == image_tag]) > 0 images = get_images(job, job_tag_function=job_tag_func) elif imagebranch != branch and not latest_baked: # resolve promote job suffix = "_bake" repl_suffix = "_promote" if image: suffix += "_" + image repl_suffix += "_" + image if not image_params: image_params = load_parameters(component=component, image=image, branch=imagebranch, git=git) this_branch_prefix = re.sub(r'\W', '_', component_params["JENKINS_JOB_PREFIX"] + "_") image_branch_prefix = re.sub(r'\W', '_', image_params["JENKINS_JOB_PREFIX"] + "_") job = lreplace(image_branch_prefix, this_branch_prefix, job) job = rreplace(suffix, repl_suffix, job) images = get_images(job) else: # get current branch latest images images = get_images(job) if images: return images[0] else: return None
def guess_project_root(): for guess in [ ".", Git().get_git_root(), "..", "../..", "../../..", "../../../.." ]: if len(Project(root=guess).get_all_subcomponents()) > 0: if guess == ".": return guess else: return path.abspath(guess)
def cli_list_components(): """ Prints the components in a branch, by default the current branch """ parser = get_parser() parser.add_argument("-j", "--json", action="store_true", help="Print in json format.") parser.add_argument("-b", "--branch", help="The branch to get components from. Default is to process current branch").completer = \ ChoicesCompleter(Git().get_branches()) argcomplete.autocomplete(parser) args = parser.parse_args() ret = list_components(**vars(args)) if args.json: print(json.dumps(ret, indent=2)) else: print("\n".join(ret))
def cli_list_jobs(): """ Prints a line for every runnable job in this git repository, in all branches and optionally exports the properties for each under '$root/job-properties/""" parser = get_parser() parser.add_argument("-e", "--export-job-properties", action="store_true", help="Set if you want the properties of all jobs into files under job-properties/") parser.add_argument("-j", "--json", action="store_true", help="Print in json format. Optionally " \ "exported parameters will be in the json document") parser.add_argument("-b", "--branch", help="The branch to process. Default is to process all branches").completer = \ ChoicesCompleter(Git().get_branches()) parser.add_argument("-c", "--component", help="Component to process. Default is to process all components").completer = \ branch_components argcomplete.autocomplete(parser) args = parser.parse_args() ret = list_jobs(**vars(args)) if args.json: print(json.dumps(ret, indent=2)) else: print("\n".join(ret))
def cli_load_parameters(): """ Load parameters from infra*.properties files in the order: infra.properties, infra-[branch].properties, [component]/infra.properties, [component]/infra-[branch].properties, [component]/[subcomponent-type]-[subcomponent]/infra.properties, [component]/[subcomponent-type]-[subcomponent]/infra-[branch].properties Last parameter defined overwrites ones defined before in the files. Supports parameter expansion and bash -like transformations. Namely: ${PARAM##prefix} # strip prefix greedy ${PARAM%%suffix} # strip suffix greedy ${PARAM#prefix} # strip prefix not greedy ${PARAM%suffix} # strip suffix not greedy ${PARAM:-default} # default if empty ${PARAM:4:2} # start:len ${PARAM/substr/replace} ${PARAM^} # upper initial ${PARAM,} # lower initial ${PARAM^^} # upper ${PARAM,,} # lower Comment lines start with '#' Lines can be continued by adding '\' at the end See https://www.tldp.org/LDP/Bash-Beginners-Guide/html/sect_10_03.html (arrays not supported) """ parser = get_parser(formatter=argparse.RawDescriptionHelpFormatter) parser.add_argument("component", nargs="?", help="Compenent to descend into").completer = \ ChoicesCompleter([c.name for c in Project().get_components()]) parser.add_argument("--branch", "-b", help="Branch to get active parameters for").completer = \ ChoicesCompleter(Git().get_branches()) parser.add_argument("--resolve-images", "-r", action="store_true", help="Also resolve subcomponent AMI IDs and docker repo urls") subcomponent_group = parser.add_mutually_exclusive_group() subcomponent_group.add_argument("--stack", "-s", help="CloudFormation subcomponent to descent into").completer = \ lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("stack", prefix, parsed_args, **kwargs) subcomponent_group.add_argument("--serverless", "-l", help="Serverless subcomponent to descent into").completer = \ lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("serverless", prefix, parsed_args, **kwargs) subcomponent_group.add_argument("--docker", "-d", help="Docker image subcomponent to descent into").completer = \ lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("docker", prefix, parsed_args, **kwargs) subcomponent_group.add_argument("--image", "-i", const="", nargs="?", help="AMI image subcomponent to descent into").completer = \ lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("image", prefix, parsed_args, **kwargs) subcomponent_group.add_argument("--cdk", "-c", help="CDK subcomponent to descent into").completer = \ lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("cdk", prefix, parsed_args, **kwargs) subcomponent_group.add_argument("--terraform", "-t", help="Terraform subcomponent to descent into").completer = \ lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("terraform", prefix, parsed_args, **kwargs) format_group = parser.add_mutually_exclusive_group() format_group.add_argument("--json", "-j", action="store_true", help="JSON format output (default)") format_group.add_argument("--yaml", "-y", action="store_true", help="YAML format output") format_group.add_argument("--properties", "-p", action="store_true", help="properties file format output") format_group.add_argument("--export-statements", "-e", action="store_true", help="Output as eval-able export statements") argcomplete.autocomplete(parser) args = parser.parse_args() transform = json.dumps if args.export_statements: transform = map_to_exports if args.properties: transform = map_to_properties if args.yaml: transform = yaml.dump del args.export_statements del args.yaml del args.json del args.properties if (args.stack or args.serverless or args.docker or not isinstance(args.image, NoneType)) \ and not args.component: parser.error("image, stack, doker or serverless do not make sense without component") print(transform(load_parameters(**vars(args))), end="")
def load_parameters(component=None, stack=None, serverless=None, docker=None, image=None, cdk=None, terraform=None, branch=None, resolve_images=False, git=None): subc_name = "" if stack: subc_name = "stack=" + stack if serverless: subc_name = "serverless=" + serverless if docker: subc_name = "docker=" + docker if isinstance(image, six.string_types): subc_name = "image=" + image if cdk: subc_name = "cdk=" + cdk if terraform: subc_name = "terraform=" + terraform if not git: git = Git() with git: current_branch = git.get_current_branch() if not branch: branch = current_branch branch = branch.strip().split("origin/")[-1:][0] params_key = (component, subc_name, branch) if params_key in parameters: return parameters[params_key] ret = { "GIT_BRANCH": branch } account = resolve_account() if account: ret["ACCOUNT_ID"] = account if component: ret["COMPONENT"] = component prefix = "" if current_branch != branch: prefix = git.export_branch(branch) + os.sep files = [prefix + "infra.properties", prefix + "infra-" + branch + ".properties"] if component: files.append(prefix + component + os.sep + "infra.properties") files.append(prefix + component + os.sep + "infra-" + branch + ".properties") _add_subcomponent_file(prefix + component, branch, "stack", stack, files) _add_subcomponent_file(prefix + component, branch, "serverless", serverless, files) _add_subcomponent_file(prefix + component, branch, "cdk", cdk, files) _add_subcomponent_file(prefix + component, branch, "terraform", terraform, files) _add_subcomponent_file(prefix + component, branch, "docker", docker, files) _add_subcomponent_file(prefix + component, branch, "image", image, files) if isinstance(image, six.string_types): files.append(prefix + component + os.sep + "image" + os.sep + "infra.properties") files.append(prefix + component + os.sep + "image" + os.sep + "infra-" + branch + ".properties") initial_resolve = ret.copy() os.environ["DO_NOT_RESOLVE_EXTERNAL_REFS"] = "true" for file in files: if os.path.exists(file): import_parameter_file(file, initial_resolve) del os.environ["DO_NOT_RESOLVE_EXTERNAL_REFS"] if "REGION" not in initial_resolve: ret["REGION"] = region() else: ret["REGION"] = initial_resolve["REGION"] if not "AWS_DEFAULT_REGION" in os.environ: os.environ["AWS_DEFAULT_REGION"] = ret["REGION"] if "paramEnvId" not in initial_resolve: ret["paramEnvId"] = branch else: ret["paramEnvId"] = initial_resolve["paramEnvId"] for file in files: if os.path.exists(file): import_parameter_file(file, ret) if (serverless or stack or cdk or terraform) and resolve_images: image_branch = branch if 'BAKE_IMAGE_BRANCH' in ret: image_branch = ret['BAKE_IMAGE_BRANCH'] for docker in [dockerdir.split("/docker-")[1] for dockerdir in glob(component + os.sep + "docker-*")]: try: ret['paramDockerUri' + docker] = resolve_docker_uri(component, 'paramDockerUri' + docker, image_branch, git) except ClientError: # Best effor to load docker uris, but ignore errors since the repo might not # actually be in use. Missing and used uris will result in an error later. pass for image_name in [imagedir.split("/image")[1].replace("-", "").lower() for imagedir in glob(component + os.sep + "image*")]: try: image = resolve_ami(ret, component, image_name, image_branch, branch, git) if image: ret['paramAmi' + image_name] = image['ImageId'] ret['paramAmiName' + image_name] = image['Name'] env_param_name = "AMI_ID" if image_name: env_param_name += "_" + image_name.upper() ret[env_param_name] = image['ImageId'] except ClientError: # Best effor to load ami info, but ignore errors since the image might not # actually be in use. Missing and used images will result in an error later. pass if "REGION" not in ret: ret["REGION"] = region() if "paramEnvId" not in ret: ret["paramEnvId"] = branch if "ORIG_STACK_NAME" in os.environ: ret["ORIG_STACK_NAME"] = os.environ["ORIG_STACK_NAME"] if "STACK_NAME" not in ret: ret["STACK_NAME"] = component + "-" + ret["ORIG_STACK_NAME"] + "-" + ret["paramEnvId"] for k, v in list(os.environ.items()): if k.startswith("ORIG_") and k.endswith("_NAME"): ret[k] = v if "ORIG_DOCKER_NAME" in os.environ: if "DOCKER_NAME" not in ret: ret["DOCKER_NAME"] = component + "/" + ret["paramEnvId"] + "-" + ret["ORIG_DOCKER_NAME"] if "JENKINS_JOB_PREFIX" not in ret: ret["JENKINS_JOB_PREFIX"] = "ndt" + ret["paramEnvId"] parameters[params_key] = ret return ret
def list_jobs(export_job_properties=False, branch=None, json=False, component=None): ret = {"branches": []} arr = [] param_files = {} with Git() as git: current_project = Project(root=guess_project_root()) if branch: branches = [branch] else: branches = git.get_branches() components = [] for c_branch in branches: branch_obj = {"name": c_branch, "components": []} ret["branches"].append(branch_obj) if c_branch == git.get_current_branch(): project = current_project else: root = git.export_branch(c_branch) project = Project(root=root, branch=c_branch) if component: c_component = project.get_component(component) if not c_component: print("No matching components") if json: return {} else: return [] branch_obj["components"].append({ "name": c_component.name, "subcomponents": [] }) components.append(c_component) else: for c_component in project.get_components(): branch_obj["components"].append({ "name": c_component.name, "subcomponents": [] }) components.append(c_component) if not json and export_job_properties: try: mkdir(current_project.root + sep + "job-properties") except OSError as err: # Directory already exists is ok if err.errno == 17: pass else: raise err if json: _collect_json(components, ret, export_job_properties, git) else: arr, param_files = _collect_prop_files(components, export_job_properties, current_project.root, git) if export_job_properties: _write_prop_files(param_files) if json: return ret else: return arr