def run(self, remainder: List[str]): """Runs the current command on all compositions in their current order.""" help_only = len(remainder) == 1 and remainder[0] in ["--help", "-h"] if not help_only: cls = self.__class__.__name__ LOGGER.info( f"Running {cls} command '{self.command}' with remainder '{remainder}'..." ) results = [] order = getattr(Composition, self.command)._order # Class lookup for composition in order(self): action = getattr(composition, self.command) # Instance lookup res = action(remainder) if help_only: # If only help on a subcommand is requested, print it out *once* and # leave. LOGGER.debug("Only printing help then exiting.") try: print(res.stdout) except AttributeError: print("No help available.") return results.append(res) return results
def decorator(func): nonlocal desc, name # If the decorated function's actual name and docstring are already proper, # we can use them here. Otherwise, they need to be set explicitly. if desc is None: desc = func.__doc__.splitlines()[0] if name is None: name = func.__name__ COMMANDS[name] = CommandMetadata(desc, order) LOGGER.debug(f"Registered {name} as a command.") @wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) wrapper._order = order return wrapper
def outsiders(dump=True): """Gets all requests originating from outside (not in DynDNS history) IPs.""" entries = [] outside_ips = set() inside_ips = get_home_ips() for n, line in enumerate(docker_logs("proxy")): entry = json.loads(line) timestamp = aware_timestamp(entry["ts"]) if n == 0: log_start = timestamp.isoformat() start_delta = now() - timestamp try: ip, _ = entry["request"]["remote_addr"].split(":") except KeyError: # Not a server log line, perhaps a startup log line like # '{"level":"warn","ts":1615631067.0608304,"logger":"admin","msg":"admin endpoint disabled"}' continue if ip in inside_ips: continue outside_ips.add(ip) entries.append(entry) ip_infos = ipinfos(outside_ips) LOGGER.info(f"Logs start at {log_start}, aka {start_delta} ago.") LOGGER.info(f"Found {len(entries)} request(s) from outside IPs.") LOGGER.info(f"These came from {len(outside_ips)} unique outside IPs:") LOGGER.info(ip_infos) if dump: dumps = {Path("outside_requests.json"): entries, Path("ipinfos.json"): ip_infos} for file, data in dumps.items(): with open(file, "w") as f: LOGGER.info(f"Dumping to {file}.") json.dump(data, f, indent=4) return entries
def _run(cmd, subcmd, flags=None, **kwargs) -> CompletedProcess: if flags is None: flags = [] else: if isinstance(flags, str): # Allow flags to be a string, so that we don't have to create a list for a # single flag argument (`["--all"]` vs. `"--all"`). flags = split(flags) try: return text_run(cmd + split(subcmd) + flags, **kwargs) except CalledTextProcessError as e: # Examine the standard error output, trying to find matches. If a match is found, # convert raised exception to a more fitting one. for regex, exception in _STDERR_CONVERSIONS.items(): match = regex.search(e.stderr) if match: LOGGER.warning( f"Process' stderr matched against {regex}, raising {exception}" ) raise exception(**vars(e)) from e raise
def text_run(cmd, **kwargs): try: return run(cmd, check=True, capture_output=True, text=True, **kwargs) except CalledProcessError as e: LOGGER.error(e.stderr) raise CalledTextProcessError(**vars(e))
def wrapper(*args, **kwargs): nonlocal backoff start = dt.now() LOGGER.debug("Starting function call retries.") n = 1 while True: LOGGER.info(f"Running try number {n}.") try: res = func(*args, **kwargs) LOGGER.info("Call succeeded.") return res except on_exception: LOGGER.warning(f"Call failed (raised {on_exception}).") n += 1 now = dt.now() delta = now - start if delta > timeout: LOGGER.error("Timeout reached, exiting.") raise LOGGER.info(f"Sleeping for {backoff}.") time.sleep(backoff.total_seconds()) backoff *= 2 # exponential
from importlib import import_module from control.util.log import LOGGER try: module = "docker" docker = import_module(module) _DOCKER = docker.from_env() except ModuleNotFoundError: # `object()` instances don't support attribute assignment, # we need something more capable: from types import SimpleNamespace # Allow the script to run without the above module installed. For this to work, # we have fake to all currently used attributes correctly. That is pretty stupid # and definitely doesn't scale well, but it seemed fun. LOGGER.error(f"Module '{module}' not found, disabling silently.") _DOCKER = SimpleNamespace() _DOCKER.containers = SimpleNamespace() _DOCKER.containers.list = lambda: [] def docker_logs(container_name): """Fetches all available log lines (entries) from a container.""" container = _DOCKER.containers.get(container_name) return [entry for entry in container.logs().decode("utf8").split("\n") if entry]
def lexec(self, remainder): """Execute a command on all services with a certain, truthy label. There's two types of labels: 1. Lists simply allow labels to be present, with no value/value of `None`. 2. Mappings additionally assign values to labels. I decided to only allow mappings so that a change in behaviour can be triggered in the YAML by flipping the value, not removing the entire label (like you would have to do with lists). """ parser = argparse.ArgumentParser(description=dedent("""\ Like `exec`, but finds services to operate on automatically, by filtering by the passed label. The remainder here is forwarded to `exec`, *without* the `SERVICE` part (so only the command to execute and flags to that command.). """)) parser.add_argument( "-e", "--exec_options", help= "Options to be passed to exec itself, not the command to be executed." + " WARNING: Must be quoted *in the shell* to avoid wrong processing.", ) parser.add_argument( "label", help="Execute on services where this label is truthy.") parser.add_argument( "remainder", # For an overview of `nargs`, see also https://stackoverflow.com/a/31243133/11477374 nargs=argparse.REMAINDER, help="Remainder is the command (with flags) to be executed for each" + " identified service.", ) args = parser.parse_args(remainder) label = args.label LOGGER.info(f"Got label '{label}'") results = [] for service_name, service_config in self["services"].items(): LOGGER.debug(f"Working on service '{service_name}'...") try: do_exec = bool(service_config["labels"][label]) LOGGER.debug(f"Found label, evaluated to '{do_exec}'") except KeyError: LOGGER.debug("No such label, or no labels at all.") continue except TypeError: LOGGER.debug("Labels exist but they are not a mapping.") continue if do_exec: LOGGER.info("Executing on this service.") results.append( self.exec( # Examples in comments: split(args.exec_options) # ['-T'] + [service_name] # ['main'] + args.remainder # ['ls', '-lah'] )) return results
def __new__(cls, name, bases, namespace): new_cls = super().__new__(cls, name, bases, namespace) new_cls._base_cmd = ["docker-compose", "--no-ansi"] base_subcommands = { name: CommandMetadata(desc, order) for name, desc, order in [ ("build", "Build or rebuild services.", sorted), ("config", "Validate and view the Compose file.", sorted), ("create", "Create services.", sorted), ( "down", "Stop and remove containers, networks, images, and volumes.", sorted_reverse, ), ("events", "Receive real time events from containers.", sorted), ("exec", "Execute a command in a running container.", sorted), ("help", "Get help on a command.", sorted), ("images", "List images.", sorted), ("kill", "Kill containers.", sorted), ("logs", "View output from containers.", sorted), ("pause", "Pause services.", sorted), ("port", "Print the public port for a port binding.", sorted), ("ps", "List containers.", sorted), ("pull", "Pull service images.", sorted), ("push", "Push service images.", sorted), ("restart", "Restart services.", sorted), ("rm", "Remove stopped containers.", sorted), ("run", "Run a one-off command.", sorted), ("scale", "Set number of containers for a service.", sorted), ("start", "Start services.", sorted), ("stop", "Stop services.", sorted_reverse), ("top", "Display the running processes.", sorted), ("unpause", "Unpause services.", sorted), ("up", "Create and start containers.", sorted), ("version", "Show the Docker-Compose version information.", sorted), ] } # Dynamically provide attributes for each known `docker-compose` command, which # runs the corresponding command and passes all flags on. We could also solve # this via `__getattr__` in the class definition, but that's ugly and boring, # plus we don't get autocompletion on available commands. for subcmd, metadata in base_subcommands.items(): # Keep argument local to lambda using the signature, see also # https://docs.python.org/3/faq/programming.html#why-do-lambdas-defined-in-a-loop-with-different-values-all-return-the-same-result def subcmd_factory(subcmd=subcmd): """Returns a specialized function with fixed arguments. By specifically introducing `command` in the function signature, a new scope is created to allow the new returned function to access the correct object. """ @_register_command( name=subcmd, desc=metadata.desc, order=metadata.order, ) def subcmd_run(self, flags=None): # For docker-compose, it's important to run in the correct working # directory to resolve all files, e.g. `.env` files. return _run( self._base_cmd, flags=flags, subcmd=subcmd, cwd=self.cwd, ) return subcmd_run name = subcmd.lower() LOGGER.debug(f"Providing method '{name}' for {new_cls}.") setattr(new_cls, name, subcmd_factory()) return new_cls
from pathlib import Path from shlex import quote from control.util.log import LOGGER from control.util.procs import text_run PYTHON_PACKAGE_ROOT = Path(__file__).parent.parent LOGGER.debug(f"Python package root: {PYTHON_PACKAGE_ROOT}") PYTHON_PROJECT_ROOT = PYTHON_PACKAGE_ROOT.parent LOGGER.debug(f"Python project root: {PYTHON_PROJECT_ROOT}") PROJECT_ROOT = PYTHON_PROJECT_ROOT.parent LOGGER.debug(f"Project root: {PROJECT_ROOT}") def strip_last_suffix(path: Path) -> Path: return path.with_suffix("") def locate_executable(cmd) -> Path: """Returns the absolute path to the passed command, searching `$PATH`.""" # `Path` doesn't strip trailing newline and would carry it along, so `strip`. return Path(text_run(["which", quote(cmd)]).stdout.strip())