def check_config(workflow: "Workflow", data: dict) -> dict: if not data: return data # check for a compose_flow section compose_flow = data.get("compose_flow", None) if not compose_flow: return data extends = compose_flow.get("config", {}).get("extends") if not extends: return data configs = [] for item in extends: with open(item) as fh: configs.append(yaml_load(fh)) # append the config that was read in to the configs list configs.append(data) # mash them together data = remerge(configs) return data
def get_config() -> dict: data = None if os.path.exists(DC_CONFIG_FILE): with open(DC_CONFIG_FILE, 'r') as fh: data = yaml_load(fh) return data
def data(self): if self._data: return self._data compose_content = self.load() self._data = yaml_load(compose_content) return self._data
def render_config(workflow: "Workflow", config: dict) -> dict: # render the config prior to returning it content = yaml_dump(config) rendered_content = render(content, env=workflow.environment.data) rendered_config = yaml_load(rendered_content) print(f"rendered_config={rendered_config}") return rendered_config
def data(self): try: content = self.load() except NoSuchConfig: content = {} # if the content is empty return immediately if not content: return content rendered = render(content) return yaml_load(rendered)
def test_set_resources_empty_resources_left_alone(self, *mocks): """ Ensures when services.x.deploy doesn't exist it is not added """ content = get_content("profiles/no_constraints.yml") data = yaml_load(content) service_name, data = list(data["services"].items())[0] profile = Profile(self.workflow) profile.set_resources(service_name, data) self.assertEqual("deploy" not in data, True)
def test_set_resources_memory_reservation_no_limit(self, *mocks): """ Ensures memory limit is matched to reservation when no limit is given """ merge_profile_mock = mocks[0] merge_profile_mock.return_value = get_content( 'profiles/reservation_no_limit.yml') profile = Profile(self.workflow) param = {} content = profile._compile( param) # the param is ignored because it's using the mock merge_profile_mock.assert_called_with(param) data = yaml_load(content) resources = data['services']['app']['deploy']['resources'] self.assertEqual(resources['limits']['memory'], resources['reservations']['memory'])
def test_set_resources_memory_limit_and_reservation(self, *mocks): """ Ensures memory reservation and limit are left alone when they are both defined """ merge_profile_mock = mocks[0] merge_profile_mock.return_value = get_content( 'profiles/limit_and_reservation.yml') profile = Profile(self.workflow) param = {} content = profile._compile( param) # the param is ignored because it's using the mock merge_profile_mock.assert_called_with(param) data = yaml_load(content) resources = data['services']['app']['deploy']['resources'] self.assertEqual('10M', resources['reservations']['memory']) self.assertEqual('100M', resources['limits']['memory'])
def read_project_config(workflow: "Workflow") -> dict: """Reads the project config from the filesystem """ data = {} logger = get_logger() compose_flow_filename = workflow.args.compose_flow_filename if compose_flow_filename: paths = [compose_flow_filename] else: config_name_config_file = f"compose-flow.{workflow.config_basename}.yml" project_name_config_file = f"compose-flow.{workflow.project_name}.yml" paths = [ config_name_config_file, project_name_config_file, DC_CONFIG_PATH ] for item in paths: filename = os.path.basename(item) logger.debug(f"looking for {filename}") if os.path.exists(filename): with open(filename, "r") as fh: data = yaml_load(fh) break else: logger.warning( f"compose-flow config not found; tried {paths} in {os.getcwd()}") outfile = f"compose-flow-{workflow.config_name}-config.yml" with open(outfile, "w") as fh: fh.write(yaml_dump(data)) return data
def test_set_resources_memory_limit_no_reservation(self, *mocks): """ Ensures memory reservation is matched to limit when no reservation is given """ merge_profile_mock = mocks[0] merge_profile_mock.return_value = get_content( "profiles/limit_no_reservation.yml" ) profile = Profile(self.workflow) param = {} content = profile._compile( param ) # the param is ignored because it's using the mock merge_profile_mock.assert_called_with(param) data = yaml_load(content) resources = data["services"]["app"]["deploy"]["resources"] self.assertEqual( resources["reservations"]["memory"], resources["limits"]["memory"] )
def _compile(self, profile: dict) -> str: """ Compiles the profile into a single docker compose file Args: profile: The profile name to compile Returns: compiled compose file as a string """ if self._compiled_profile: return self._compiled_profile content = merge_profile(profile) # perform transformations on the compiled profile if content: data = yaml_load(content) # check if the environment needs to be copied from another service data = self._copy_environment(data) # see if any services need to be expanded out data = self._check_cf_config(data) # drop the compose_flow section if it exists data.pop("compose_flow", None) # for each service inject DOCKER_STACK and DOCKER_SERVICE for service_name, service_data in data.get("services", {}).items(): service_environment = service_data.setdefault( "environment", []) # convert the service_environment into a dict service_environment_d = {} for item in service_environment: item_split = item.split("=", 1) k = item_split[0] if len(item_split) > 1: v = item_split[1] else: v = None service_environment_d[k] = v for k, v in ( ("DOCKER_SERVICE", service_name), ("DOCKER_STACK", self.workflow.config_name), ): if k not in service_environment_d: service_environment_d[k] = v # reconstruct the k=v list honoring empty values service_environment_l = [] for k, v in service_environment_d.items(): if v is None: val = k else: val = f"{k}={v}" service_environment_l.append(val) # dump back out as list service_data["environment"] = service_environment_l # enforce resources self.set_resources(service_name, service_data) content = yaml_dump(data) self._compiled_profile = content return content