def __init__(self, credentials_descriptor): """ Generates aws credentials for nordstrom users. Args: credentials_descriptor: A dictionary containing serialized credentialer, satisfying the yac/schema/aws/credentialer.json schema Raises: ValidationError: if a inputs fails schema validation """ validate(credentials_descriptor, "yac/schema/stacks/aws/credentialer.json") self.accounts = search("accounts", credentials_descriptor, []) self.region = search("region", credentials_descriptor, []) # if urls not provided, use defaults self.token_endpoint_url = search('"token-endpoint-url"', credentials_descriptor, TOKEN_ENDPOINT_URL) self.role_endpoint_url = search('"role-endpoint-url"', credentials_descriptor, ROLE_ENDPOINT_URL) # initialize the inputs (for driving user prompts) self.inputs = Inputs(search("Inputs", credentials_descriptor, {}))
def __init__(self, serialized_inputs_cacher): validate(serialized_inputs_cacher, "yac/schema/inputs_cacher.json") self.enabled = search("enabled", serialized_inputs_cacher, False) self.path = search("path", serialized_inputs_cacher, "") self.exclusions = search('"exclusions"', serialized_inputs_cacher, [])
def __init__(self, serialized_service_description, alias): validate(serialized_service_description, "yac/schema/description.json") self.name = search('"name"', serialized_service_description,"") self.summary = search('summary', serialized_service_description,"") self.version = search('version', serialized_service_description,"") self.repo = search('repo', serialized_service_description,"") self.default_alias = search('"default-alias"', serialized_service_description,"") if alias: self.alias = alias else: self.alias = self.default_alias
def __init__(self, serialized_boot_files): # first validate. this should throw an exception if # required fields aren't present # validate(serialized_boot_files, "yac/schema/boot_files.json") self.files = search("files", serialized_boot_files, []) self.directories = search("directories", serialized_boot_files, [])
def __init__(self, vault_configs): validate(vault_configs, "yac/schema/vaults/file.json") self.vault_path = search('"vault-path"', vault_configs) # default to json format self.format = search("format", vault_configs, "json") self.initialized = False self.ready = False
def __init__(self, serialized_stack): # args: # serialized_stack: dict containing the stack object from the Stack stanza # in the Servicefile # use the yac.lib.schema module to validate the serialized_stack, e.g.: validate(serialized_stack, "yac/schema/stacks/gcp/stack.json") # use the yac.lib.search module to grab fields from the serialized_stack, e.g.: self.name = search('name', serialized_stack) self.resources = search("Resources",serialized_stack, {}) self.conditions = search("Conditions",serialized_stack, {})
def _get_test_groups(self, group_names, test_groups): # return the list of test groups to run # arg: # group_names: array of group names, where each group name # can optionally include a test name within the # group (e.g. ['group-b',group-a:test-b',etc]) # test_groups: all test groups # # returns: # groups_to_run: list of test group objects to be run # err: string holding error message if one or more names in group_names # fails to match any existing test group groups_to_run = [] err = "" for group_name in group_names: # see if group name includes a test name if ":" in group_name: name_parts = group_name.split(":") group_name_key = name_parts[0] test_name = name_parts[1] else: group_name_key = group_name test_name = "" this_group = search("[?name=='%s'] | [0]" % group_name_key, test_groups, {}) if this_group: if test_name: # isolate the test from the group this_test = search("tests[?name=='%s'] | [0]" % test_name, this_group, {}) if this_test: # only include this test this_group['tests'] = [this_test] else: err = "test '%s' not found in test group '%s'" % ( test_name, group_name_key) groups_to_run = groups_to_run + [this_group] else: err = "test group '%s' not found" % group_name return groups_to_run, err
def __init__( self, test_name, target, artillery_descriptor, test_results, params ): self.test_name = test_name self.target = target self.config_path = search('config',artillery_descriptor,"") self.test_assertions = search('assertions',artillery_descriptor,{}) self.variables = search('variables',artillery_descriptor,{}) self.test_results = test_results self.params = params self.artillery_aggregates = {}
def __init__(self, vault_configs): validate(vault_configs, "yac/schema/vaults/keepass.json") self.vault_path = search('"vault-path"', vault_configs) vault_pwd_path = search('"vault-pwd-path"', vault_configs) self.vault_pwd = search('"vault-pwd"', vault_configs) if not self.vault_pwd and vault_pwd_path and os.path.exists( vault_pwd_path): self.vault_pwd = read_pwd_file(vault_pwd_path) self.ready = False self.initialized = False self.kp = None
def __init__(self, int_test_descriptor): # validate the test descriptor # note: this will raise an exception if validation fails validate(int_test_descriptor, "yac/schema/integration_tests.json") # raw because it my contain instrinsics self.raw_target_map = search('"target-map"', int_test_descriptor, {}) self.results_store = search('"results-store"', int_test_descriptor, {}) self.tests = search('tests', int_test_descriptor, []) self.test_groups = search('"test-groups"', int_test_descriptor, []) self.test_results = TestResults()
def __init__(self, serialized_kubernetes_stack, namespace=""): self.resource_array = search("resources",serialized_kubernetes_stack,[]) self.namespace = namespace
def __init__(self, credentialers_descriptor): self.credentialers = {} for credentialer_descriptor in credentialers_descriptor: name = search('name',credentialer_descriptor,"") credentialer_type = search('type',credentialer_descriptor,"") self.credentialers[name], err = get_credentials_provider(credentialer_type, credentialer_descriptor) if err: print("credentialer provider for type '%s' not available. err: %s ... exiting"%(credentialer_type,err)) exit(1)
def __init__(self, serialized_input): """ Inputs are typically used when a service provider wants to create an easy installation experience for their service. Args: serialized_input: A dictionary containing serialized input, satisfying the yac/schema/input.json schema Raises: ValidationError: if a inputs fails schema validation """ validate(serialized_input, "yac/schema/input.json") self.key = search("key",serialized_input,"") self.type = search("type",serialized_input,"") self.title = search("description",serialized_input,"") self.help = search("help",serialized_input,"") self.required = search("required",serialized_input,True) self.options = search("options",serialized_input,[]) self.conditions = search("conditions",serialized_input,{})
def __init__(self, serialized_stack): self.type = "aws-cloudformation" # validate. this should raise an error if required # fields aren't present validate(serialized_stack, "yac/schema/stacks/aws/stack.json") self.parameters = search("Parameters", serialized_stack, {}) self.resources = search("Resources", serialized_stack, {}) self.conditions = search("Conditions", serialized_stack, {}) # for mapping cloud formation parameters to yac params self.cf_param_map = ParameterMapping( search("ParameterMapping", serialized_stack, {})) self.boot_files = BootFiles(search('"BootFiles"', serialized_stack, {}))
def status_pod(pod): # pull the status of each container in the pod into a list of bools statuses = search("status.container_statuses[*].ready", pod.to_dict()) # make sure each container is status=True pod_ready = False not in statuses print("pod: %s, ready?: %s" % (pod.metadata.name, pod_ready)) return pod_ready
def __init__(self, notifier_descriptor, logger=None): # first validate. this should throw an exception if # required fields aren't present validate(notifier_descriptor, "yac/schema/notifier/slack.json") self.info_channel = search('"info-channel"', notifier_descriptor, "") self.warning_channel = search('"warning-channel"', notifier_descriptor, "") self.api_key = search('"api-key"', notifier_descriptor, "") self.client = SlackClient(self.api_key) if logger: self.logger = logger else: self.logger = get_yac_logger()
def __init__(self, vault_configs={}): validate(vault_configs, "yac/schema/vaults/s3.json") self.vault_bucket = search('"bucket"', vault_configs) self.vault_s3_path = search('"vault-path"', vault_configs) self.format = search("format", vault_configs, "json") vault_file_local_path = get_vault_local_path(self.vault_bucket, self.vault_s3_path) file_vault_config = { "vault-path": vault_file_local_path, "format": self.format } FileVault.__init__(self, file_vault_config) self.session = None
def __init__(self, credentials_descriptor): validate(credentials_descriptor, "yac/schema/stacks/k8s/credentialer.json") self.namespace = search("namespace", credentials_descriptor,"") self.clusters = search("clusters", credentials_descriptor, ["nonprod","prod"]) # if tokens are input there should be one per cluster self.tokens = search("tokens", credentials_descriptor, []) self.secrets = Secrets(search('"Secrets"', credentials_descriptor,{})) # initialize the inputs (for driving user prompts) self.inputs = Inputs(search("Inputs", credentials_descriptor,{})) # for integration testing it is useful to write files to a # root director other than user's home self.rootdir = search("rootdir", credentials_descriptor,"")
def __init__(self, serialized_artifact): validate(serialized_artifact, "yac/schema/makers/container_image.json") self.name = search("name",serialized_artifact,"") self.description = search("description",serialized_artifact,"") self.image = search("image",serialized_artifact) # the registry where the images should be pushed # defaults to artifactory self.registry = search('registry', serialized_artifact, ARTIFACTORY_URL) # initialize the inputs (for driving user prompts) self.inputs = Inputs(search("Inputs", serialized_artifact,{})) self.secrets = Secrets(search('"Secrets"', serialized_artifact,{})) # client for most operations self.client = docker.DockerClient('tcp://%s:%s'%(BUILDER_HOST, BUILDER_PORT)) # client for "low-level" build operations (e.g. builds that send # the details on each layer built to stdout ) # TODO: figure out why auth isn't working from inside a container # with this one self.api_client = docker.APIClient('tcp://%s:%s'%(BUILDER_HOST, BUILDER_PORT))
def get_engine(engine_type, engine_key, serialized_obj): engine_configs = get_engine_configs() instance = None err = "" if engine_type: module_path = search( "%s[?key=='%s'] | [0].module" % (engine_type, engine_key), engine_configs) class_name = search( "%s[?key=='%s'] | [0].class" % (engine_type, engine_key), engine_configs) provider_module, err = get_module(module_path) if not err and class_name: class_ = getattr(provider_module, class_name) instance = class_(serialized_obj) return instance, err
def __init__(self, serialized_pipeline): # first validate. this should throw an exception if # required fields aren't present validate(serialized_pipeline, "yac/schema/pipeline.json") self.deploy_branch = jmespath.search('"deploy-branch"', serialized_pipeline) self.rollback_branch = jmespath.search('"rollback-branch"', serialized_pipeline) self.setup = jmespath.search('setup', serialized_pipeline) self.notifications = search('notifications',serialized_pipeline) self.stages = search('stages',serialized_pipeline,[]) self.stage_names = search('stages[*].name',serialized_pipeline,[])
def cost(self, params, context=""): total_cost = 0 if self.resource_array: # render intrinsics in deployments rendered_deployments = apply_intrinsics(self.resource_array, params) mem_cost_map = {"129M": 3.96} per_cpu_cost = 15.72 resources = search("[*].spec.template.spec.resources.requests", rendered_deployments, []) pod_counts = search("[*].spec.replicas", rendered_deployments) mem_cost = 0 cpu_cost = 0 print( "deployment cost is based on number of pods and memory and cpu requests for each" ) for i, resource in enumerate(resources): pod_count = float(pod_counts[i]) if 'memory' in resource and resource['memory'] in mem_cost_map: mem_cost = mem_cost_map[resource['memory']] if 'cpu' in resource: cpu_cost = float(resource['cpu']) * per_cpu_cost total_cost = total_cost + 6.05 * (pod_count * (mem_cost + cpu_cost)) else: total_cost = total_cost + 6.05 * (pod_count) return total_cost
def __init__(self, serialized_stack): if serialized_stack: stack_type = search('type', serialized_stack, "") self.impl, err = get_stack_provider(stack_type, serialized_stack) if err: print( "stack provider for type '%s' not available. err: %s ... exiting" % (stack_type, err)) exit(1) else: self.impl = None
def get_token(self, k8s_resources, builder_account_name, context): # first get the name of the secret from the 'secrets' attribute # of the service account kubectl_command = "kubectl --context=%s get sa %s -o json" % ( context, builder_account_name) print("cmd: %s" % kubectl_command) sa_str, err = k8s_resources.run_kubectl(kubectl_command) token = "" if not err: sa_dict = json.loads(sa_str) secret_name = search("secrets | [0].name", sa_dict) if secret_name: token, err = self.secrets.get_secret_value( context, secret_name, "token") return token, err
def __init__(self, serialized_artifact): validate(serialized_artifact, "yac/schema/makers/ami.json") self.name = search('name', serialized_artifact) self.description = search('description', serialized_artifact) # the aws profile aliasing the account to build in self.profile = search('profile', serialized_artifact) # path to the packer file self.packer_file = search('"packer-file"', serialized_artifact) # directory containing files that should be included in the build self.packable_dir = search('"packer-dir"', serialized_artifact, "") self.secrets = Secrets(search('"Secrets"', serialized_artifact, {})) # initialize the inputs (for driving user prompts) self.inputs = Inputs(search("Inputs", serialized_artifact, {}))
def _get_tests(self, test_names, rendered_tests): # return the list of tests to run # arg: # test_names: string list of test names # rendered_tests: tests with rendered intrinsics # returns: # tests_to_run: list of tests objects to be run # err: string holding error message if or more names in test_names # fails to match any existing test tests_to_run = [] err = "" for test_name in test_names: this_test = search("[?name=='%s']" % test_name, rendered_tests, []) if this_test: tests_to_run = tests_to_run + this_test else: err = "test %s not found" % test_name return tests_to_run, err
def __init__(self, serialized_stack): """ Args: serialized_stack: A dictionary containing serialized k8s stack, satisfying the yac/schema/stacks/k8s/stack.json schema Returns: A K8sStack instance Raises: ValidationError: if a serialized_stack fails schema validation """ # first validate. this should raise an error if # required fields aren't present validate(serialized_stack, "yac/schema/stacks/k8s/stack.json") self.type = "kubernetes" self.namespace = search("namespace", serialized_stack, "") self.configmaps = ConfigMaps(serialized_stack) self.secrets = Secrets(serialized_stack) self.deployments = Deployments(serialized_stack) self.ingresses = Ingresses(serialized_stack) self.statefulsets = StatefulSets(serialized_stack) self.services = Services(serialized_stack) self.pvcs = PVCs(serialized_stack) # generic resources (orchestrated via kubectl) self.resources = Resources(serialized_stack)
def register_engine(engine_type, engine_key, module_path, class_name, configs_path=""): configs = get_engine_configs(configs_path) err = "" provider_module, err = get_module(module_path) if not err and class_name: class_ = getattr(provider_module, class_name) instance = class_({}) # make sure engine does not already exist existing_module_path = search( "%s[?key=='%s'] | [0].module" % (engine_type, engine_key), configs) if existing_module_path: err = "'%s' engine with key %s already exists" % (engine_type, engine_key) if not err: configs[engine_type].append({ "key": engine_key, "module": module_path, "class": class_name }) # write the engine configs to disk write_engine_configs(configs, configs_path) return err
def __init__(self, serialized_kubernetes_stack): self.resource_array = search("secrets",serialized_kubernetes_stack,[])
def __init__(self, serialized_task): self.name = search("name", serialized_task, "") self.description = search("description", serialized_task, "") self.module = search("module", serialized_task, "") self.inputs = Inputs(search("Inputs", serialized_task, {}))