def check_config(workflow: "Workflow", data: dict) -> dict:
if not data:
return data
# check for a compose_flow section
compose_flow = data.get("compose_flow", None)
if not compose_flow:
return data
extends = compose_flow.get("config", {}).get("extends")
if not extends:
return data
configs = []
for item in extends:
with open(item) as fh:
configs.append(yaml_load(fh))
# append the config that was read in to the configs list
configs.append(data)
# mash them together
data = remerge(configs)
return data
def get_config() -> dict:
data = None
if os.path.exists(DC_CONFIG_FILE):
with open(DC_CONFIG_FILE, 'r') as fh:
data = yaml_load(fh)
return data
def data(self):
if self._data:
return self._data
compose_content = self.load()
self._data = yaml_load(compose_content)
return self._data
def render_config(workflow: "Workflow", config: dict) -> dict:
# render the config prior to returning it
content = yaml_dump(config)
rendered_content = render(content, env=workflow.environment.data)
rendered_config = yaml_load(rendered_content)
print(f"rendered_config={rendered_config}")
return rendered_config
def data(self):
try:
content = self.load()
except NoSuchConfig:
content = {}
# if the content is empty return immediately
if not content:
return content
rendered = render(content)
return yaml_load(rendered)
def test_set_resources_empty_resources_left_alone(self, *mocks):
"""
Ensures when services.x.deploy doesn't exist it is not added
"""
content = get_content("profiles/no_constraints.yml")
data = yaml_load(content)
service_name, data = list(data["services"].items())[0]
profile = Profile(self.workflow)
profile.set_resources(service_name, data)
self.assertEqual("deploy" not in data, True)
def test_set_resources_memory_reservation_no_limit(self, *mocks):
"""
Ensures memory limit is matched to reservation when no limit is given
"""
merge_profile_mock = mocks[0]
merge_profile_mock.return_value = get_content(
'profiles/reservation_no_limit.yml')
profile = Profile(self.workflow)
param = {}
content = profile._compile(
param) # the param is ignored because it's using the mock
merge_profile_mock.assert_called_with(param)
data = yaml_load(content)
resources = data['services']['app']['deploy']['resources']
self.assertEqual(resources['limits']['memory'],
resources['reservations']['memory'])
def test_set_resources_memory_limit_and_reservation(self, *mocks):
"""
Ensures memory reservation and limit are left alone when they are both defined
"""
merge_profile_mock = mocks[0]
merge_profile_mock.return_value = get_content(
'profiles/limit_and_reservation.yml')
profile = Profile(self.workflow)
param = {}
content = profile._compile(
param) # the param is ignored because it's using the mock
merge_profile_mock.assert_called_with(param)
data = yaml_load(content)
resources = data['services']['app']['deploy']['resources']
self.assertEqual('10M', resources['reservations']['memory'])
self.assertEqual('100M', resources['limits']['memory'])
def read_project_config(workflow: "Workflow") -> dict:
"""Reads the project config from the filesystem
"""
data = {}
logger = get_logger()
compose_flow_filename = workflow.args.compose_flow_filename
if compose_flow_filename:
paths = [compose_flow_filename]
else:
config_name_config_file = f"compose-flow.{workflow.config_basename}.yml"
project_name_config_file = f"compose-flow.{workflow.project_name}.yml"
paths = [
config_name_config_file, project_name_config_file, DC_CONFIG_PATH
]
for item in paths:
filename = os.path.basename(item)
logger.debug(f"looking for {filename}")
if os.path.exists(filename):
with open(filename, "r") as fh:
data = yaml_load(fh)
break
else:
logger.warning(
f"compose-flow config not found; tried {paths} in {os.getcwd()}")
outfile = f"compose-flow-{workflow.config_name}-config.yml"
with open(outfile, "w") as fh:
fh.write(yaml_dump(data))
return data
def test_set_resources_memory_limit_no_reservation(self, *mocks):
"""
Ensures memory reservation is matched to limit when no reservation is given
"""
merge_profile_mock = mocks[0]
merge_profile_mock.return_value = get_content(
"profiles/limit_no_reservation.yml"
)
profile = Profile(self.workflow)
param = {}
content = profile._compile(
param
) # the param is ignored because it's using the mock
merge_profile_mock.assert_called_with(param)
data = yaml_load(content)
resources = data["services"]["app"]["deploy"]["resources"]
self.assertEqual(
resources["reservations"]["memory"], resources["limits"]["memory"]
)
def _compile(self, profile: dict) -> str:
"""
Compiles the profile into a single docker compose file
Args:
profile: The profile name to compile
Returns:
compiled compose file as a string
"""
if self._compiled_profile:
return self._compiled_profile
content = merge_profile(profile)
# perform transformations on the compiled profile
if content:
data = yaml_load(content)
# check if the environment needs to be copied from another service
data = self._copy_environment(data)
# see if any services need to be expanded out
data = self._check_cf_config(data)
# drop the compose_flow section if it exists
data.pop("compose_flow", None)
# for each service inject DOCKER_STACK and DOCKER_SERVICE
for service_name, service_data in data.get("services", {}).items():
service_environment = service_data.setdefault(
"environment", [])
# convert the service_environment into a dict
service_environment_d = {}
for item in service_environment:
item_split = item.split("=", 1)
k = item_split[0]
if len(item_split) > 1:
v = item_split[1]
else:
v = None
service_environment_d[k] = v
for k, v in (
("DOCKER_SERVICE", service_name),
("DOCKER_STACK", self.workflow.config_name),
):
if k not in service_environment_d:
service_environment_d[k] = v
# reconstruct the k=v list honoring empty values
service_environment_l = []
for k, v in service_environment_d.items():
if v is None:
val = k
else:
val = f"{k}={v}"
service_environment_l.append(val)
# dump back out as list
service_data["environment"] = service_environment_l
# enforce resources
self.set_resources(service_name, service_data)
content = yaml_dump(data)
self._compiled_profile = content
return content
