def run(self, remainder: List[str]):
"""Runs the current command on all compositions in their current order."""
help_only = len(remainder) == 1 and remainder[0] in ["--help", "-h"]
if not help_only:
cls = self.__class__.__name__
LOGGER.info(
f"Running {cls} command '{self.command}' with remainder '{remainder}'..."
)
results = []
order = getattr(Composition, self.command)._order # Class lookup
for composition in order(self):
action = getattr(composition, self.command) # Instance lookup
res = action(remainder)
if help_only:
# If only help on a subcommand is requested, print it out *once* and
# leave.
LOGGER.debug("Only printing help then exiting.")
try:
print(res.stdout)
except AttributeError:
print("No help available.")
return
results.append(res)
return results
def decorator(func):
nonlocal desc, name
# If the decorated function's actual name and docstring are already proper,
# we can use them here. Otherwise, they need to be set explicitly.
if desc is None:
desc = func.__doc__.splitlines()[0]
if name is None:
name = func.__name__
COMMANDS[name] = CommandMetadata(desc, order)
LOGGER.debug(f"Registered {name} as a command.")
@wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs)
wrapper._order = order
return wrapper
def outsiders(dump=True):
"""Gets all requests originating from outside (not in DynDNS history) IPs."""
entries = []
outside_ips = set()
inside_ips = get_home_ips()
for n, line in enumerate(docker_logs("proxy")):
entry = json.loads(line)
timestamp = aware_timestamp(entry["ts"])
if n == 0:
log_start = timestamp.isoformat()
start_delta = now() - timestamp
try:
ip, _ = entry["request"]["remote_addr"].split(":")
except KeyError:
# Not a server log line, perhaps a startup log line like
# '{"level":"warn","ts":1615631067.0608304,"logger":"admin","msg":"admin endpoint disabled"}'
continue
if ip in inside_ips:
continue
outside_ips.add(ip)
entries.append(entry)
ip_infos = ipinfos(outside_ips)
LOGGER.info(f"Logs start at {log_start}, aka {start_delta} ago.")
LOGGER.info(f"Found {len(entries)} request(s) from outside IPs.")
LOGGER.info(f"These came from {len(outside_ips)} unique outside IPs:")
LOGGER.info(ip_infos)
if dump:
dumps = {Path("outside_requests.json"): entries, Path("ipinfos.json"): ip_infos}
for file, data in dumps.items():
with open(file, "w") as f:
LOGGER.info(f"Dumping to {file}.")
json.dump(data, f, indent=4)
return entries
def _run(cmd, subcmd, flags=None, **kwargs) -> CompletedProcess:
if flags is None:
flags = []
else:
if isinstance(flags, str):
# Allow flags to be a string, so that we don't have to create a list for a
# single flag argument (`["--all"]` vs. `"--all"`).
flags = split(flags)
try:
return text_run(cmd + split(subcmd) + flags, **kwargs)
except CalledTextProcessError as e:
# Examine the standard error output, trying to find matches. If a match is found,
# convert raised exception to a more fitting one.
for regex, exception in _STDERR_CONVERSIONS.items():
match = regex.search(e.stderr)
if match:
LOGGER.warning(
f"Process' stderr matched against {regex}, raising {exception}"
)
raise exception(**vars(e)) from e
raise
def text_run(cmd, **kwargs):
try:
return run(cmd, check=True, capture_output=True, text=True, **kwargs)
except CalledProcessError as e:
LOGGER.error(e.stderr)
raise CalledTextProcessError(**vars(e))
def wrapper(*args, **kwargs):
nonlocal backoff
start = dt.now()
LOGGER.debug("Starting function call retries.")
n = 1
while True:
LOGGER.info(f"Running try number {n}.")
try:
res = func(*args, **kwargs)
LOGGER.info("Call succeeded.")
return res
except on_exception:
LOGGER.warning(f"Call failed (raised {on_exception}).")
n += 1
now = dt.now()
delta = now - start
if delta > timeout:
LOGGER.error("Timeout reached, exiting.")
raise
LOGGER.info(f"Sleeping for {backoff}.")
time.sleep(backoff.total_seconds())
backoff *= 2 # exponential
from importlib import import_module
from control.util.log import LOGGER
try:
module = "docker"
docker = import_module(module)
_DOCKER = docker.from_env()
except ModuleNotFoundError:
# `object()` instances don't support attribute assignment,
# we need something more capable:
from types import SimpleNamespace
# Allow the script to run without the above module installed. For this to work,
# we have fake to all currently used attributes correctly. That is pretty stupid
# and definitely doesn't scale well, but it seemed fun.
LOGGER.error(f"Module '{module}' not found, disabling silently.")
_DOCKER = SimpleNamespace()
_DOCKER.containers = SimpleNamespace()
_DOCKER.containers.list = lambda: []
def docker_logs(container_name):
"""Fetches all available log lines (entries) from a container."""
container = _DOCKER.containers.get(container_name)
return [entry for entry in container.logs().decode("utf8").split("\n") if entry]
def lexec(self, remainder):
"""Execute a command on all services with a certain, truthy label.
There's two types of labels:
1. Lists simply allow labels to be present, with no value/value of `None`.
2. Mappings additionally assign values to labels.
I decided to only allow mappings so that a change in behaviour can be
triggered in the YAML by flipping the value, not removing the entire label
(like you would have to do with lists).
"""
parser = argparse.ArgumentParser(description=dedent("""\
Like `exec`, but finds services to operate on automatically, by
filtering by the passed label.
The remainder here is forwarded to `exec`, *without* the `SERVICE`
part (so only the command to execute and flags to that command.).
"""))
parser.add_argument(
"-e",
"--exec_options",
help=
"Options to be passed to exec itself, not the command to be executed."
+
" WARNING: Must be quoted *in the shell* to avoid wrong processing.",
)
parser.add_argument(
"label", help="Execute on services where this label is truthy.")
parser.add_argument(
"remainder",
# For an overview of `nargs`, see also https://stackoverflow.com/a/31243133/11477374
nargs=argparse.REMAINDER,
help="Remainder is the command (with flags) to be executed for each"
+ " identified service.",
)
args = parser.parse_args(remainder)
label = args.label
LOGGER.info(f"Got label '{label}'")
results = []
for service_name, service_config in self["services"].items():
LOGGER.debug(f"Working on service '{service_name}'...")
try:
do_exec = bool(service_config["labels"][label])
LOGGER.debug(f"Found label, evaluated to '{do_exec}'")
except KeyError:
LOGGER.debug("No such label, or no labels at all.")
continue
except TypeError:
LOGGER.debug("Labels exist but they are not a mapping.")
continue
if do_exec:
LOGGER.info("Executing on this service.")
results.append(
self.exec(
# Examples in comments:
split(args.exec_options) # ['-T']
+ [service_name] # ['main']
+ args.remainder # ['ls', '-lah']
))
return results
def __new__(cls, name, bases, namespace):
new_cls = super().__new__(cls, name, bases, namespace)
new_cls._base_cmd = ["docker-compose", "--no-ansi"]
base_subcommands = {
name: CommandMetadata(desc, order)
for name, desc, order in [
("build", "Build or rebuild services.", sorted),
("config", "Validate and view the Compose file.", sorted),
("create", "Create services.", sorted),
(
"down",
"Stop and remove containers, networks, images, and volumes.",
sorted_reverse,
),
("events", "Receive real time events from containers.",
sorted),
("exec", "Execute a command in a running container.", sorted),
("help", "Get help on a command.", sorted),
("images", "List images.", sorted),
("kill", "Kill containers.", sorted),
("logs", "View output from containers.", sorted),
("pause", "Pause services.", sorted),
("port", "Print the public port for a port binding.", sorted),
("ps", "List containers.", sorted),
("pull", "Pull service images.", sorted),
("push", "Push service images.", sorted),
("restart", "Restart services.", sorted),
("rm", "Remove stopped containers.", sorted),
("run", "Run a one-off command.", sorted),
("scale", "Set number of containers for a service.", sorted),
("start", "Start services.", sorted),
("stop", "Stop services.", sorted_reverse),
("top", "Display the running processes.", sorted),
("unpause", "Unpause services.", sorted),
("up", "Create and start containers.", sorted),
("version", "Show the Docker-Compose version information.",
sorted),
]
}
# Dynamically provide attributes for each known `docker-compose` command, which
# runs the corresponding command and passes all flags on. We could also solve
# this via `__getattr__` in the class definition, but that's ugly and boring,
# plus we don't get autocompletion on available commands.
for subcmd, metadata in base_subcommands.items():
# Keep argument local to lambda using the signature, see also
# https://docs.python.org/3/faq/programming.html#why-do-lambdas-defined-in-a-loop-with-different-values-all-return-the-same-result
def subcmd_factory(subcmd=subcmd):
"""Returns a specialized function with fixed arguments.
By specifically introducing `command` in the function signature,
a new scope is created to allow the new returned function to access the
correct object.
"""
@_register_command(
name=subcmd,
desc=metadata.desc,
order=metadata.order,
)
def subcmd_run(self, flags=None):
# For docker-compose, it's important to run in the correct working
# directory to resolve all files, e.g. `.env` files.
return _run(
self._base_cmd,
flags=flags,
subcmd=subcmd,
cwd=self.cwd,
)
return subcmd_run
name = subcmd.lower()
LOGGER.debug(f"Providing method '{name}' for {new_cls}.")
setattr(new_cls, name, subcmd_factory())
return new_cls
from pathlib import Path
from shlex import quote
from control.util.log import LOGGER
from control.util.procs import text_run
PYTHON_PACKAGE_ROOT = Path(__file__).parent.parent
LOGGER.debug(f"Python package root: {PYTHON_PACKAGE_ROOT}")
PYTHON_PROJECT_ROOT = PYTHON_PACKAGE_ROOT.parent
LOGGER.debug(f"Python project root: {PYTHON_PROJECT_ROOT}")
PROJECT_ROOT = PYTHON_PROJECT_ROOT.parent
LOGGER.debug(f"Project root: {PROJECT_ROOT}")
def strip_last_suffix(path: Path) -> Path:
return path.with_suffix("")
def locate_executable(cmd) -> Path:
"""Returns the absolute path to the passed command, searching `$PATH`."""
# `Path` doesn't strip trailing newline and would carry it along, so `strip`.
return Path(text_run(["which", quote(cmd)]).stdout.strip())