def _resolve_tfref_from_dict(tfref_var):
    if "component" in tfref_var and "terraform" in tfref_var and ("paramName" in tfref_var or "jmespath" in tfref_var):
        with Git() as git:
            current_branch = git.get_current_branch()
            if "branch" in tfref_var:
                branch = tfref_var["branch"]
            else:
                branch = current_branch
            tf_key = (tfref_var["component"], tfref_var["terraform"], branch)
            if tf_key in terraforms:
                terraform = terraforms[tf_key]
            else:
                root = "."
                if branch != current_branch:
                    root = git.export_branch(branch)
                terraform = pull_state(tfref_var["component"], tfref_var["terraform"], root=root)
                terraforms[tf_key] = terraform
            if "paramName" in tfref_var:
                flat_state_dict = flat_state(terraform)
                if tfref_var["paramName"] in flat_state_dict:
                    return flat_state_dict[tfref_var["paramName"]]
                else:
                    return None
            if "jmespath" in tfref_var:
                return jmespath_var(terraform, tfref_var["jmespath"])
    else:
        return None
 def __init__(self, root=".", branch=None):
     if not branch:
         self.branch = Git().get_current_branch()
     else:
         self.branch = branch
     self.componets = []
     self.root = root if root else guess_project_root()
     self.all_subcomponents = []
def resolve_docker_uri(component, uriParam, image_branch, git):
    if not git:
        git = Git()
    with git:
        if uriParam in os.environ:
            return os.environ[uriParam]
        docker = uriParam[14:]
        docker_params = load_parameters(component=component, docker=docker, branch=image_branch, git=git)
        return repo_uri(docker_params['DOCKER_NAME'])
def resolve_ami(component_params, component, image, imagebranch, branch, git):
    if not git:
        git = Git()
    with git:
        if "paramAmi" + image in os.environ:
            return { "ImageId": os.environ["paramAmi" + image],
                    "Name": os.environ["paramAmiName" + image] \
                    if "paramAmiName" + image in os.environ else "Unknown" }
        images = []
        image_params = {}
        job = ""
        if "IMAGE_JOB" in os.environ and not image:
            job = re.sub(r'\W', '_', os.environ["IMAGE_JOB"])
        else:
            image_params = load_parameters(component=component, image=image, branch=imagebranch, git=git)
            if "JOB_NAME" in image_params:
                job = re.sub(r'\W', '_', image_params["JOB_NAME"])
            else:
                prefix = ""
                prefix = image_params["JENKINS_JOB_PREFIX"]
                job = prefix + "_" + component + "_bake"
                if image:
                    job = job + "_" + image
                job = re.sub(r'\W', '_', job)
        build_param = "paramAmi" + image + "Build"
        latest_baked = build_param in component_params and component_params[build_param] == 'latest'
        if latest_baked:
            # get current branch latest images
            images = get_images(job)
        if build_param in component_params and component_params[build_param] != 'latest':
            # resolve with a specifically set image build number
            build = component_params[build_param]
            image_tag = job + "_" + build
            job_tag_func = lambda image, image_name_prefix: len([tag for tag in image["Tags"] if tag["Value"] == image_tag]) > 0
            images = get_images(job, job_tag_function=job_tag_func)
        elif imagebranch != branch and not latest_baked:
            # resolve promote job
            suffix = "_bake"
            repl_suffix = "_promote"
            if image:
                suffix += "_" + image 
                repl_suffix += "_" + image
            if not image_params:
                image_params = load_parameters(component=component, image=image, branch=imagebranch, git=git)
            this_branch_prefix = re.sub(r'\W', '_', component_params["JENKINS_JOB_PREFIX"] + "_")
            image_branch_prefix = re.sub(r'\W', '_', image_params["JENKINS_JOB_PREFIX"] + "_")
            job = lreplace(image_branch_prefix, this_branch_prefix, job)
            job = rreplace(suffix, repl_suffix, job)
            images = get_images(job)
        else:
            # get current branch latest images
            images = get_images(job)
        if images:
            return images[0]
        else:
            return None
def guess_project_root():

    for guess in [
            ".",
            Git().get_git_root(), "..", "../..", "../../..", "../../../.."
    ]:
        if len(Project(root=guess).get_all_subcomponents()) > 0:
            if guess == ".":
                return guess
            else:
                return path.abspath(guess)
Exemplo n.º 6
0
def cli_list_components():
    """ Prints the components in a branch, by default the current branch """
    parser = get_parser()
    parser.add_argument("-j", "--json", action="store_true", help="Print in json format.")
    parser.add_argument("-b", "--branch", help="The branch to get components from. Default is to process current branch").completer = \
        ChoicesCompleter(Git().get_branches())
    argcomplete.autocomplete(parser)
    args = parser.parse_args()
    ret = list_components(**vars(args))
    if args.json:
        print(json.dumps(ret, indent=2))
    else:
        print("\n".join(ret))
Exemplo n.º 7
0
def cli_list_jobs():
    """ Prints a line for every runnable job in this git repository, in all branches and
    optionally exports the properties for each under '$root/job-properties/"""
    parser = get_parser()
    parser.add_argument("-e", "--export-job-properties", action="store_true",
                        help="Set if you want the properties of all jobs into files under job-properties/")
    parser.add_argument("-j", "--json", action="store_true", help="Print in json format. Optionally " \
                                                                  "exported parameters will be in the json document")
    parser.add_argument("-b", "--branch", help="The branch to process. Default is to process all branches").completer = \
        ChoicesCompleter(Git().get_branches())
    parser.add_argument("-c", "--component", help="Component to process. Default is to process all components").completer = \
        branch_components
    argcomplete.autocomplete(parser)
    args = parser.parse_args()
    ret = list_jobs(**vars(args))
    if args.json:
        print(json.dumps(ret, indent=2))
    else:
        print("\n".join(ret))
Exemplo n.º 8
0
def cli_load_parameters():
    """ Load parameters from infra*.properties files in the order:
    infra.properties,
    infra-[branch].properties,
    [component]/infra.properties,
    [component]/infra-[branch].properties,
    [component]/[subcomponent-type]-[subcomponent]/infra.properties,
    [component]/[subcomponent-type]-[subcomponent]/infra-[branch].properties

    Last parameter defined overwrites ones defined before in the files. Supports parameter expansion
    and bash -like transformations. Namely:

    ${PARAM##prefix} # strip prefix greedy
    ${PARAM%%suffix} # strip suffix greedy
    ${PARAM#prefix} # strip prefix not greedy
    ${PARAM%suffix} # strip suffix not greedy
    ${PARAM:-default} # default if empty
    ${PARAM:4:2} # start:len
    ${PARAM/substr/replace}
    ${PARAM^} # upper initial
    ${PARAM,} # lower initial
    ${PARAM^^} # upper
    ${PARAM,,} # lower

    Comment lines start with '#'
    Lines can be continued by adding '\' at the end

    See https://www.tldp.org/LDP/Bash-Beginners-Guide/html/sect_10_03.html
    (arrays not supported)
    """
    parser = get_parser(formatter=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("component", nargs="?", help="Compenent to descend into").completer = \
        ChoicesCompleter([c.name for c in Project().get_components()])
    parser.add_argument("--branch", "-b", help="Branch to get active parameters for").completer = \
        ChoicesCompleter(Git().get_branches())
    parser.add_argument("--resolve-images", "-r", action="store_true", help="Also resolve subcomponent AMI IDs and docker repo urls")
    subcomponent_group = parser.add_mutually_exclusive_group()
    subcomponent_group.add_argument("--stack", "-s", help="CloudFormation subcomponent to descent into").completer = \
        lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("stack", prefix, parsed_args, **kwargs)
    subcomponent_group.add_argument("--serverless", "-l", help="Serverless subcomponent to descent into").completer = \
        lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("serverless", prefix, parsed_args, **kwargs)
    subcomponent_group.add_argument("--docker", "-d", help="Docker image subcomponent to descent into").completer = \
        lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("docker", prefix, parsed_args, **kwargs)
    subcomponent_group.add_argument("--image", "-i", const="", nargs="?", help="AMI image subcomponent to descent into").completer = \
        lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("image", prefix, parsed_args, **kwargs)
    subcomponent_group.add_argument("--cdk", "-c", help="CDK subcomponent to descent into").completer = \
        lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("cdk", prefix, parsed_args, **kwargs)
    subcomponent_group.add_argument("--terraform", "-t", help="Terraform subcomponent to descent into").completer = \
        lambda prefix, parsed_args, **kwargs: component_typed_subcomponents("terraform", prefix, parsed_args, **kwargs)
    format_group = parser.add_mutually_exclusive_group()
    format_group.add_argument("--json", "-j", action="store_true", help="JSON format output (default)")
    format_group.add_argument("--yaml", "-y", action="store_true", help="YAML format output")
    format_group.add_argument("--properties", "-p", action="store_true", help="properties file format output")
    format_group.add_argument("--export-statements", "-e", action="store_true",
                              help="Output as eval-able export statements")

    argcomplete.autocomplete(parser)
    args = parser.parse_args()

    transform = json.dumps
    if args.export_statements:
        transform = map_to_exports
    if args.properties:
        transform = map_to_properties
    if args.yaml:
        transform = yaml.dump
    del args.export_statements
    del args.yaml
    del args.json
    del args.properties
    if (args.stack or args.serverless or args.docker or not isinstance(args.image, NoneType)) \
       and not args.component:
        parser.error("image, stack, doker or serverless do not make sense without component")
    print(transform(load_parameters(**vars(args))), end="")
def load_parameters(component=None, stack=None, serverless=None, docker=None, image=None, 
                    cdk=None, terraform=None, branch=None, resolve_images=False,
                    git=None):
    subc_name = ""
    if stack:
        subc_name = "stack=" + stack
    if serverless:
        subc_name = "serverless=" + serverless
    if docker:
        subc_name = "docker=" + docker
    if isinstance(image, six.string_types):
        subc_name = "image=" + image
    if cdk:
        subc_name = "cdk=" + cdk
    if terraform:
        subc_name = "terraform=" + terraform
    if not git:
        git = Git()
    with git:
        current_branch = git.get_current_branch()
        if not branch:
            branch = current_branch
        branch = branch.strip().split("origin/")[-1:][0]
        params_key = (component, subc_name, branch)
        if params_key in parameters:
            return parameters[params_key]
        ret = {
            "GIT_BRANCH": branch
        }
        account = resolve_account()
        if account:
            ret["ACCOUNT_ID"] = account
        if component:
            ret["COMPONENT"] = component
        prefix = ""
        if current_branch != branch:
            prefix = git.export_branch(branch) + os.sep
        files = [prefix + "infra.properties", prefix + "infra-" + branch + ".properties"]
        if component:
            files.append(prefix + component + os.sep + "infra.properties")
            files.append(prefix + component + os.sep + "infra-" + branch + ".properties")
            _add_subcomponent_file(prefix + component, branch, "stack", stack, files)
            _add_subcomponent_file(prefix + component, branch, "serverless", serverless, files)
            _add_subcomponent_file(prefix + component, branch, "cdk", cdk, files)
            _add_subcomponent_file(prefix + component, branch, "terraform", terraform, files)
            _add_subcomponent_file(prefix + component, branch, "docker", docker, files)
            _add_subcomponent_file(prefix + component, branch, "image", image, files)
            if isinstance(image, six.string_types):
                files.append(prefix + component + os.sep + "image" + os.sep + "infra.properties")
                files.append(prefix + component + os.sep + "image" + os.sep + "infra-" + branch + ".properties")
        initial_resolve = ret.copy()
        os.environ["DO_NOT_RESOLVE_EXTERNAL_REFS"] = "true"
        for file in files:
            if os.path.exists(file):
                import_parameter_file(file, initial_resolve)
        del os.environ["DO_NOT_RESOLVE_EXTERNAL_REFS"]
        if "REGION" not in initial_resolve:
            ret["REGION"] = region()
        else:
            ret["REGION"] = initial_resolve["REGION"]
        if not "AWS_DEFAULT_REGION" in os.environ:
            os.environ["AWS_DEFAULT_REGION"] = ret["REGION"]
        if "paramEnvId" not in initial_resolve:
            ret["paramEnvId"] = branch
        else:
            ret["paramEnvId"] = initial_resolve["paramEnvId"]
        for file in files:
            if os.path.exists(file):
                import_parameter_file(file, ret)
        if (serverless or stack or cdk or terraform) and resolve_images:
            image_branch = branch
            if 'BAKE_IMAGE_BRANCH' in ret:
                image_branch = ret['BAKE_IMAGE_BRANCH']
            for docker in [dockerdir.split("/docker-")[1] for dockerdir in glob(component + os.sep + "docker-*")]:
                try:
                    ret['paramDockerUri' + docker] = resolve_docker_uri(component, 'paramDockerUri' + docker, image_branch, git)
                except ClientError:
                    # Best effor to load docker uris, but ignore errors since the repo might not
                    # actually be in use. Missing and used uris will result in an error later.
                    pass
            for image_name in [imagedir.split("/image")[1].replace("-", "").lower() for imagedir in glob(component + os.sep + "image*")]:
                try:
                    image = resolve_ami(ret, component, image_name, image_branch, branch, git)
                    if image:
                        ret['paramAmi' + image_name] = image['ImageId']
                        ret['paramAmiName' + image_name] = image['Name']
                        env_param_name = "AMI_ID"
                        if image_name:
                            env_param_name +=  "_" + image_name.upper()
                        ret[env_param_name] = image['ImageId']
                except ClientError:
                    # Best effor to load ami info, but ignore errors since the image might not
                    # actually be in use. Missing and used images will result in an error later.
                    pass
        if "REGION" not in ret:
            ret["REGION"] = region()
        if "paramEnvId" not in ret:
            ret["paramEnvId"] = branch
        if "ORIG_STACK_NAME" in os.environ:
            ret["ORIG_STACK_NAME"] = os.environ["ORIG_STACK_NAME"]
            if "STACK_NAME" not in ret:
                ret["STACK_NAME"] = component + "-" + ret["ORIG_STACK_NAME"] + "-" + ret["paramEnvId"]
        for k, v in list(os.environ.items()):
            if k.startswith("ORIG_") and k.endswith("_NAME"):
                ret[k] = v
        if "ORIG_DOCKER_NAME" in os.environ:
            if "DOCKER_NAME" not in ret:
                ret["DOCKER_NAME"] = component + "/" + ret["paramEnvId"] + "-" + ret["ORIG_DOCKER_NAME"]
        if "JENKINS_JOB_PREFIX" not in ret:
            ret["JENKINS_JOB_PREFIX"] = "ndt" + ret["paramEnvId"]
        parameters[params_key] = ret
        return ret
def list_jobs(export_job_properties=False,
              branch=None,
              json=False,
              component=None):
    ret = {"branches": []}
    arr = []
    param_files = {}
    with Git() as git:
        current_project = Project(root=guess_project_root())
        if branch:
            branches = [branch]
        else:
            branches = git.get_branches()
        components = []
        for c_branch in branches:
            branch_obj = {"name": c_branch, "components": []}
            ret["branches"].append(branch_obj)
            if c_branch == git.get_current_branch():
                project = current_project
            else:
                root = git.export_branch(c_branch)
                project = Project(root=root, branch=c_branch)
            if component:
                c_component = project.get_component(component)
                if not c_component:
                    print("No matching components")
                    if json:
                        return {}
                    else:
                        return []
                branch_obj["components"].append({
                    "name": c_component.name,
                    "subcomponents": []
                })
                components.append(c_component)
            else:
                for c_component in project.get_components():
                    branch_obj["components"].append({
                        "name": c_component.name,
                        "subcomponents": []
                    })
                    components.append(c_component)
        if not json and export_job_properties:
            try:
                mkdir(current_project.root + sep + "job-properties")
            except OSError as err:
                # Directory already exists is ok
                if err.errno == 17:
                    pass
                else:
                    raise err
        if json:
            _collect_json(components, ret, export_job_properties, git)
        else:
            arr, param_files = _collect_prop_files(components,
                                                   export_job_properties,
                                                   current_project.root, git)
            if export_job_properties:
                _write_prop_files(param_files)
    if json:
        return ret
    else:
        return arr