def to_command(self, *args, **kwargs):
abspath = os.path.abspath(inspect.getfile(self.callFunc))
ctxpath = image_mod.Image.get_contextual_path(abspath)
if hostdet.is_wsl():
import conducto.internal.build as cib
ctxpath = cib._split_windocker(ctxpath)
elif hostdet.is_windows():
ctxpath = hostdet.windows_docker_path(ctxpath)
parts = [
"conducto",
f"__conducto_path:{ctxpath}:endpath__",
self.callFunc.__name__,
]
sig = inspect.signature(self.callFunc)
bound = sig.bind(*args, **kwargs)
for k, v in bound.arguments.items():
if v is True:
parts.append(f"--{k}")
continue
if v is False:
parts.append(f"--no-{k}")
continue
if client_utils.isiterable(v):
parts += ["--{}={}".format(k, t.List.join(map(t.serialize, v)))]
else:
parts += ["--{}={}".format(k, t.serialize(v))]
return " ".join(pipes.quote(part) for part in parts)
def docker_available_drives():
import string
if hostdet.is_wsl():
kwargs = dict(check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
drives = []
for drive in string.ascii_lowercase:
drivedir = f"{drive}:\\"
try:
subprocess.run(f"wslpath -u {drivedir}", shell=True, **kwargs)
drives.append(drive)
except subprocess.CalledProcessError:
pass
else:
from ctypes import windll # Windows only
# get all drives
drive_bitmask = windll.kernel32.GetLogicalDrives()
letters = string.ascii_lowercase
drives = [
letters[i] for i, v in enumerate(bin(drive_bitmask)) if v == "1"
]
# filter to fixed drives
is_fixed = lambda x: windll.kernel32.GetDriveTypeW(f"{x}:\\") == 3
drives = [d for d in drives if is_fixed(d.upper())]
return drives
def build(
node,
build_mode=constants.BuildMode.DEPLOY_TO_CLOUD,
use_shell=False,
use_app=True,
retention=7,
is_public=False,
):
assert node.parent is None
assert node.name == "/"
if hostdet.is_wsl():
required_drives = _wsl_translate_locations(node)
elif hostdet.is_windows():
required_drives = _windows_translate_locations(node)
if hostdet.is_wsl() or hostdet.is_windows():
available = docker_available_drives()
unavailable = set(required_drives).difference(available)
if len(unavailable) > 0:
msg = f"The drive {unavailable.pop()} is used in an image context, but is not available in Docker. Review your Docker Desktop file sharing settings."
raise hostdet.WindowsMapError(msg)
from .. import api
# refresh the token for every pipeline launch
# Force in case of cognito change
node.token = token = api.Auth().get_token_from_shell(force=True)
serialization = node.serialize()
command = " ".join(pipes.quote(a) for a in sys.argv)
# Register pipeline, get <pipeline_id>
cloud = build_mode == constants.BuildMode.DEPLOY_TO_CLOUD
pipeline_id = api.Pipeline().create(
token,
command,
cloud=cloud,
retention=retention,
tags=node.tags or [],
title=node.title,
is_public=is_public,
)
launch_from_serialization(serialization, pipeline_id, build_mode,
use_shell, use_app, token)
def write(self):
config_file = self.__get_config_file()
# Create config dir if doesn't exist.
config_dir = os.path.dirname(config_file)
if not os.path.isdir(config_dir):
# local import due to import loop
import conducto.internal.host_detection as hostdet
if hostdet.is_wsl():
# Create .conducto directory in the window's users homedir.
# Symlink that to the linux user's homedir. This is
# back-translated to a docker friendly path on docker mounting.
fallback_error = """\
There was an error creating the conducto configuration files at ~/.conducto.
The .conducto folder must be accessible to docker and so it must be on a
Windows drive. You can set that up manually by executing the following
commands:
mkdir /mnt/c/Users/<winuser>/.conducto
ln -sf /mnt/c/Users/<winuser>/.conducto ~/.conducto
"""
try:
cmdline = ["wslpath", "-u", r"C:\Windows\system32\cmd.exe"]
proc = subprocess.run(cmdline, stdout=subprocess.PIPE)
cmdpath = proc.stdout.decode("utf-8").strip()
cmdline = [cmdpath, "/C", "echo %USERPROFILE%"]
proc = subprocess.run(cmdline, stdout=subprocess.PIPE)
winprofile = proc.stdout.decode("utf-8").strip()
cmdline = ["wslpath", "-u", winprofile]
proc = subprocess.run(cmdline, stdout=subprocess.PIPE)
homedir = proc.stdout.decode("utf-8").strip()
win_config_dir = os.path.join(homedir, ".conducto")
if not os.path.isdir(win_config_dir):
os.mkdir(win_config_dir)
cmdline = ["ln", "-s", win_config_dir, config_dir]
subprocess.run(cmdline, stdout=subprocess.PIPE)
except subprocess.CalledProcessError:
raise RuntimeError(fallback_error)
else:
os.mkdir(config_dir)
with open(config_file, "w") as config_fh:
self.config.write(config_fh)
def start_container(payload, live):
import random
image = get_param(payload, "image", default={})
image_name = image["name_debug"]
if live and not image.get("path_map"):
raise ValueError(
f"Cannot do livedebug for image {image['name']} because it does not have a `copy_dir` or `path_map`"
)
container_name = "conducto_debug_" + str(random.randrange(1 << 64))
print("Launching docker container...")
if live:
print("Context will be mounted read-write")
print("Make modifications on your local machine, "
"and they will be reflected in the container.")
options = []
if get_param(payload, "requires_docker"):
options.append("-v /var/run/docker.sock:/var/run/docker.sock")
options.append(f'--cpus {get_param(payload, "cpu")}')
options.append(f'--memory {get_param(payload, "mem") * 1024**3}')
# TODO: Should actually pass these variables from manager, iff local
local_basedir = constants.ConductoPaths.get_local_base_dir()
if hostdet.is_wsl():
local_basedir = os.path.realpath(local_basedir)
local_basedir = hostdet.wsl_host_docker_path(local_basedir)
elif hostdet.is_windows():
local_basedir = hostdet.windows_docker_path(local_basedir)
remote_basedir = f"{get_home_dir_for_image(image_name)}/.conducto"
options.append(f"-v {local_basedir}:{remote_basedir}")
if live:
for external, internal in image["path_map"].items():
if not os.path.isabs(internal):
internal = get_work_dir_for_image(image_name) + "/" + internal
options.append(f"-v {external}:{internal}")
command = f"docker run {' '.join(options)} --name={container_name} {image_name} tail -f /dev/null "
subprocess.Popen(command, shell=True)
time.sleep(1)
return container_name
def relpath(path):
"""
Construct a path with decoration to enable translation inside a docker
image for a node. This may be used to construct path parameters to a
command line tool.
This is used internally by :py:class:`conducto.Exec` when used with a
Python callable to construct the command line which executes that callable
in the pipeline.
"""
ctxpath = Image.get_contextual_path(path)
if hostdet.is_wsl():
import conducto.internal.build as cib
ctxpath = cib._split_windocker(ctxpath)
elif hostdet.is_windows():
ctxpath = hostdet.windows_docker_path(ctxpath)
return f"__conducto_path:{ctxpath}:endpath__"
def run_in_local_container(token,
pipeline_id,
update_token=False,
inject_env=None,
is_migration=False):
# Remote base dir will be verified by container.
local_basedir = constants.ConductoPaths.get_local_base_dir()
if inject_env is None:
inject_env = {}
if hostdet.is_wsl():
local_basedir = os.path.realpath(local_basedir)
local_basedir = hostdet.wsl_host_docker_path(local_basedir)
elif hostdet.is_windows():
local_basedir = hostdet.windows_docker_path(local_basedir)
else:
subp = subprocess.Popen(
"head -1 /proc/self/cgroup|cut -d/ -f3",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
container_id, err = subp.communicate()
container_id = container_id.decode("utf-8").strip()
if container_id:
# Mount to the ~/.conducto of the host machine and not of the container
import json
subp = subprocess.Popen(
f"docker inspect -f '{{{{ json .Mounts }}}}' {container_id}",
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
)
mount_data, err = subp.communicate()
if subp.returncode == 0:
mounts = json.loads(mount_data)
for mount in mounts:
if mount["Destination"] == local_basedir:
local_basedir = mount["Source"]
break
# The homedir inside the manager is /root
remote_basedir = "/root/.conducto"
tag = api.Config().get_image_tag()
manager_image = constants.ImageUtil.get_manager_image(tag)
ccp = constants.ConductoPaths
pipelinebase = ccp.get_local_path(pipeline_id,
expand=False,
base=remote_basedir)
# Note: This path is in the docker which is always unix
pipelinebase = pipelinebase.replace(os.path.sep, "/")
serialization = f"{pipelinebase}/{ccp.SERIALIZATION}"
container_name = f"conducto_manager_{pipeline_id}"
network_name = os.getenv("CONDUCTO_NETWORK",
f"conducto_network_{pipeline_id}")
if not is_migration:
try:
client_utils.subprocess_run([
"docker", "network", "create", network_name, "--label=conducto"
])
except client_utils.CalledProcessError as e:
if f"network with name {network_name} already exists" in e.stderr.decode(
):
pass
else:
raise
flags = [
# Detached mode.
"-d",
# Remove container when done.
"--rm",
# --name is the name of the container, as in when you do `docker ps`
# --hostname is the name of the host inside the container.
# Set them equal so that the manager can use socket.gethostname() to
# spin up workers that connect to its network.
"--name",
container_name,
"--network",
network_name,
"--hostname",
container_name,
"--label",
"conducto",
# Mount local conducto basedir on container. Allow TaskServer
# to access config and serialization and write logs.
"-v",
f"{local_basedir}:{remote_basedir}",
# Mount docker sock so we can spin out task workers.
"-v",
"/var/run/docker.sock:/var/run/docker.sock",
# Specify expected base dir for container to verify.
"-e",
f"CONDUCTO_BASE_DIR_VERIFY={remote_basedir}",
"-e",
f"CONDUCTO_LOCAL_BASE_DIR={local_basedir}",
"-e",
f"CONDUCTO_LOCAL_HOSTNAME={socket.gethostname()}",
"-e",
f"CONDUCTO_NETWORK={network_name}",
]
for env_var in (
"CONDUCTO_URL",
"CONDUCTO_CONFIG",
"IMAGE_TAG",
"CONDUCTO_DEV_REGISTRY",
):
if os.environ.get(env_var):
flags.extend(["-e", f"{env_var}={os.environ[env_var]}"])
for k, v in inject_env.items():
flags.extend(["-e", f"{k}={v}"])
if hostdet.is_wsl() or hostdet.is_windows():
drives = docker_available_drives()
if docker_desktop_23():
flags.extend(["-e", "WINDOWS_HOST=host_mnt"])
else:
flags.extend(["-e", "WINDOWS_HOST=plain"])
for d in drives:
# Mount whole system read-only to enable rebuilding images as needed
mount = f"type=bind,source={d}:/,target={constants.ConductoPaths.MOUNT_LOCATION}/{d.lower()},readonly"
flags += ["--mount", mount]
else:
# Mount whole system read-only to enable rebuilding images as needed
mount = f"type=bind,source=/,target={constants.ConductoPaths.MOUNT_LOCATION},readonly"
flags += ["--mount", mount]
if _manager_debug():
flags[0] = "-it"
flags += ["-e", "CONDUCTO_LOG_LEVEL=0"]
capture_output = False
else:
capture_output = True
mcpu = _manager_cpu()
if mcpu > 0:
flags += ["--cpus", str(mcpu)]
# WSL doesn't persist this into containers natively
# Have to have this configured so that we can use host docker creds to pull containers
docker_basedir = constants.ConductoPaths.get_local_docker_config_dir()
if docker_basedir:
flags += ["-v", f"{docker_basedir}:/root/.docker"]
cmd_parts = [
"python",
"-m",
"manager.src",
"-p",
pipeline_id,
"-i",
serialization,
"--profile",
api.Config().default_profile,
"--local",
]
if update_token:
cmd_parts += ["--update_token", "--token", token]
if manager_image.startswith("conducto/"):
docker_parts = ["docker", "pull", manager_image]
log.debug(" ".join(pipes.quote(s) for s in docker_parts))
client_utils.subprocess_run(
docker_parts,
capture_output=capture_output,
msg="Error pulling manager container",
)
# Run manager container.
docker_parts = ["docker", "run"] + flags + [manager_image] + cmd_parts
log.debug(" ".join(pipes.quote(s) for s in docker_parts))
client_utils.subprocess_run(
docker_parts,
msg="Error starting manager container",
capture_output=capture_output,
)
# When in debug mode the manager is run attached and it makes no sense to
# follow that up with waiting for the manager to start.
if not _manager_debug():
log.debug(
f"Verifying manager docker startup pipeline_id={pipeline_id}")
def _get_docker_output():
p = subprocess.run(["docker", "ps"], stdout=subprocess.PIPE)
return p.stdout.decode("utf-8")
pl = constants.PipelineLifecycle
target = pl.active - pl.standby
# wait 45 seconds, but this should be quick
for _ in range(
int(constants.ManagerAppParams.WAIT_TIME_SECS /
constants.ManagerAppParams.POLL_INTERVAL_SECS)):
time.sleep(constants.ManagerAppParams.POLL_INTERVAL_SECS)
log.debug(f"awaiting program {pipeline_id} active")
data = api.Pipeline().get(token, pipeline_id)
if data["status"] in target and data["pgw"] not in ["", None]:
break
dps = _get_docker_output()
if container_name not in dps:
attached = [param for param in docker_parts if param != "-d"]
dockerrun = " ".join(pipes.quote(s) for s in attached)
msg = f"There was an error starting the docker container. Try running the command below for more diagnostics or contact us on Slack at ConductoHQ.\n{dockerrun}"
raise RuntimeError(msg)
else:
# timeout, return error
raise RuntimeError(
f"no manager connection to pgw for {pipeline_id} after {constants.ManagerAppParams.WAIT_TIME_SECS} seconds"
)
log.debug(f"Manager docker connected to pgw pipeline_id={pipeline_id}")