def create_new_deployment(
runner: Runner, deployment_arg: str, image_name: str, expose: PortMapping,
add_custom_nameserver: bool
) -> Tuple[str, str]:
"""
Create a new Deployment, return its name and Kubernetes label.
"""
span = runner.span()
run_id = runner.session_id
runner.show(
"Starting network proxy to cluster using "
"new Deployment {}".format(deployment_arg)
)
def remove_existing_deployment(quiet=False):
if not quiet:
runner.show("Cleaning up Deployment {}".format(deployment_arg))
runner.check_call(
runner.kubectl(
"delete",
"--ignore-not-found",
"svc,deploy",
"--selector=telepresence=" + run_id,
)
)
runner.add_cleanup("Delete new deployment", remove_existing_deployment)
remove_existing_deployment(quiet=True)
command = [
"run", # This will result in using Deployment:
"--restart=Always",
"--limits=cpu=100m,memory=256Mi",
"--requests=cpu=25m,memory=64Mi",
deployment_arg,
"--image=" + image_name,
"--labels=telepresence=" + run_id,
]
# Provide a stable argument ordering. Reverse it because that happens to
# make some current tests happy but in the long run that's totally
# arbitrary and doesn't need to be maintained. See issue 494.
for port in sorted(expose.remote(), reverse=True):
command.append("--port={}".format(port))
if expose.remote():
command.append("--expose")
# If we're on local VM we need to use different nameserver to prevent
# infinite loops caused by sshuttle:
if add_custom_nameserver:
command.append(
"--env=TELEPRESENCE_NAMESERVER=" + get_alternate_nameserver()
)
try:
runner.check_call(runner.kubectl(command))
except CalledProcessError as exc:
raise runner.fail(
"Failed to create deployment {}:\n{}".format(
deployment_arg, exc.stderr
)
)
span.end()
return deployment_arg, run_id
def main():
"""
Top-level function for Telepresence
"""
########################################
# Preliminaries: No changes to the machine or the cluster, no cleanup
# Capture environment info and the user's intent
# Check for a subcommand
with crash_reporting():
args = command_parse_args(None, only_for_commands=True)
if args is not None:
command_main(args)
with crash_reporting():
args = parse_args() # tab-completion stuff goes here
runner = Runner(Output(args.logfile), None, args.verbose)
span = runner.span()
runner.add_cleanup("Stop time tracking", span.end)
runner.kubectl = KubeInfo(runner, args)
start_proxy = proxy.setup(runner, args)
do_connect = connect.setup(runner, args)
get_remote_env, write_env_files = remote_env.setup(runner, args)
launch = outbound.setup(runner, args)
mount_remote = mount.setup(runner, args)
final_checks(runner, args)
# Usage tracking
call_scout(runner, args)
########################################
# Now it's okay to change things
with runner.cleanup_handling(), crash_reporting(runner):
# Set up the proxy pod (operation -> pod name)
remote_info = start_proxy(runner)
# Connect to the proxy (pod name -> ssh object)
socks_port, ssh = do_connect(runner, remote_info)
# Capture remote environment information (ssh object -> env info)
env = get_remote_env(runner, remote_info)
# Handle filesystem stuff
mount_dir = mount_remote(runner, env, ssh)
# Maybe write environment files
write_env_files(runner, env)
# Set up outbound networking (pod name, ssh object)
# Launch user command with the correct environment (...)
user_process = launch(
runner, remote_info, env, socks_port, ssh, mount_dir
)
wait_for_exit(runner, user_process)
def create_with_cleanup(runner: Runner, manifests: Iterable[Manifest]) -> None:
"""Create resources and set up their removal at cleanup.
Uses "kubectl create" with the supplied manifests to create resources.
Assumes that all the created resources include the telepresence label so it
can use a label selector to delete those resources.
"""
kinds = set(str(manifest["kind"]).capitalize() for manifest in manifests)
kinds_display = ", ".join(kinds)
manifest_list = make_k8s_list(manifests)
manifest_json = json.dumps(manifest_list)
try:
runner.check_call(runner.kubectl("create", "-f", "-"),
input=manifest_json.encode("utf-8"))
except CalledProcessError as exc:
raise runner.fail("Failed to create {}:\n{}".format(
kinds_display, exc.stderr))
def clean_up() -> None:
runner.show("Cleaning up {}".format(kinds_display))
runner.check_call(
runner.kubectl(
"delete",
"--ignore-not-found",
"--wait=false",
"--selector=telepresence=" + runner.session_id,
",".join(kinds),
))
runner.add_cleanup("Delete proxy {}".format(kinds_display), clean_up)
def sip_workaround(runner: Runner, existing_paths: str,
unsupported_tools_path: str) -> str:
"""
Workaround System Integrity Protection.
Newer OS X don't allow injecting libraries into binaries in /bin, /sbin and
/usr. We therefore make a copy of them and modify $PATH to point at their
new location. It's only ~100MB so this should be pretty fast!
:param existing_paths: Current $PATH.
:param unsupported_tools_path: Path where we have custom versions of ping
etc. Needs to be first in list so that we override system versions.
"""
protected = {"/bin", "/sbin", "/usr/sbin", "/usr/bin"}
# Remove protected paths from $PATH:
paths = [p for p in existing_paths.split(":") if p not in protected]
# Add temp dir
bin_dir = mkdtemp(dir="/tmp")
paths.insert(0, bin_dir)
runner.add_cleanup("Remove SIP workaround binaries", rmtree, bin_dir)
for directory in protected:
for file in os.listdir(directory):
try:
copy(os.path.join(directory, file), bin_dir)
except IOError:
continue
os.chmod(os.path.join(bin_dir, file), 0o775)
paths = [unsupported_tools_path] + paths
# Return new $PATH
return ":".join(paths)
def main():
"""
Top-level function for Telepresence
"""
with crash_reporting():
########################################
# Preliminaries: No changes to the machine or the cluster, no cleanup
# Capture environment info
args = parse_args() # tab-completion stuff goes here
runner = Runner(args.logfile, args.verbose)
span = runner.span()
runner.add_cleanup("Stop time tracking", span.end)
set_kube_command(runner, args)
with runner.cleanup_handling(), crash_reporting(runner):
########################################
# Intent: Fast, user prompts here, cleanup available
# Capture the user's intent
start_proxy = proxy.setup(runner, args)
do_connect = connect.setup(runner, args)
get_remote_env, write_env_files = remote_env.setup(runner, args)
launch = outbound.setup(runner, args)
mount_remote = mount.setup(runner, args)
final_checks(runner, args)
# Usage tracking
call_scout(runner, args)
########################################
# Action: Perform the user's intended operation(s)
# Now it's okay to change things
# Set up the proxy pod (operation -> pod name)
remote_info = start_proxy(runner)
# Connect to the proxy (pod name -> ssh object)
socks_port, ssh = do_connect(runner, remote_info)
# Capture remote environment information (ssh object -> env info)
env, pod_info = get_remote_env(runner, ssh, remote_info)
# Handle filesystem stuff
mount_dir = mount_remote(runner, env, ssh)
# Maybe write environment files
write_env_files(runner, env)
# Set up outbound networking (pod name, ssh object)
# Launch user command with the correct environment (...)
user_process = launch(
runner, remote_info, env, socks_port, ssh, mount_dir, pod_info
)
runner.wait_for_exit(user_process)
def swap_deployment_openshift(runner: Runner, deployment_arg: str,
expose: PortMapping,
add_custom_nameserver: bool) -> Tuple[str, str]:
"""
Swap out an existing DeploymentConfig and also clears any triggers
which were registered, otherwise replaced telepresence pod would
be immediately swapped back to the original one because of
image change trigger.
Returns (Deployment name, unique K8s label, JSON of original container that
was swapped out.)
"""
run_id = runner.session_id
deployment, container = _split_deployment_container(deployment_arg)
dc_json_with_triggers = json.loads(
runner.get_output(
runner.kubectl("get", "dc/{}".format(deployment), "-o", "json",
"--export")))
runner.check_call(
runner.kubectl("set", "triggers", "dc/{}".format(deployment),
"--remove-all"))
dc_json = json.loads(
runner.get_output(
runner.kubectl("get", "dc/{}".format(deployment), "-o", "json",
"--export")))
def apply_json(json_config):
runner.check_call(runner.kubectl("replace", "-f", "-"),
input=json.dumps(json_config).encode("utf-8"))
# Now that we've updated the deployment config,
# let's rollout latest version to apply the changes
runner.check_call(
runner.kubectl("rollout", "latest", "dc/{}".format(deployment)))
runner.check_call(
runner.kubectl("rollout", "status", "-w",
"dc/{}".format(deployment)))
runner.add_cleanup("Restore original deployment config", apply_json,
dc_json_with_triggers)
container = _get_container_name(container, dc_json)
new_dc_json = new_swapped_deployment(
dc_json,
container,
run_id,
expose,
add_custom_nameserver,
)
apply_json(new_dc_json)
return deployment, run_id
def swap_deployment_openshift(
runner: Runner, args: argparse.Namespace) -> Tuple[str, str, Dict]:
"""
Swap out an existing DeploymentConfig.
Returns (Deployment name, unique K8s label, JSON of original container that
was swapped out.)
In practice OpenShift doesn't seem to do the right thing when a
DeploymentConfig is updated. In particular, we need to disable the image
trigger so that we can use the new image, but the replicationcontroller
then continues to deploy the existing image.
So instead we use a different approach than for Kubernetes, replacing the
current ReplicationController with one that uses the Telepresence image,
then restores it. We delete the pods to force the RC to do its thing.
"""
run_id = runner.session_id
deployment_name, *container_name = args.swap_deployment.split(":", 1)
if container_name:
container_name = container_name[0]
rcs = runner.get_output(
runner.kubectl(
"get", "rc", "-o", "name", "--selector",
"openshift.io/deployment-config.name=" + deployment_name))
rc_name = sorted(rcs.split(),
key=lambda n: int(n.split("-")[-1]))[0].split("/", 1)[1]
rc_json = json.loads(
runner.get_output(runner.kubectl("get", "rc", "-o", "json", "--export",
rc_name),
stderr=STDOUT))
def apply_json(json_config):
runner.check_call(runner.kubectl("apply", "-f", "-"),
input=json.dumps(json_config).encode("utf-8"))
# Now that we've updated the replication controller, delete pods to
# make sure changes get applied:
runner.check_call(
runner.kubectl("delete", "pod", "--selector",
"deployment=" + rc_name))
runner.add_cleanup("Restore original replication controller", apply_json,
rc_json)
# If no container name was given, just use the first one:
if not container_name:
container_name = rc_json["spec"]["template"]["spec"]["containers"][0][
"name"]
new_rc_json, orig_container_json = new_swapped_deployment(
rc_json,
container_name,
run_id,
TELEPRESENCE_REMOTE_IMAGE,
args.method == "vpn-tcp" and args.in_local_vm,
)
apply_json(new_rc_json)
return deployment_name, run_id, orig_container_json
def swap_deployment_openshift(runner: Runner, deployment_arg: str,
image_name: str, expose: PortMapping,
add_custom_nameserver: bool) -> Tuple[str, str]:
"""
Swap out an existing DeploymentConfig.
Returns (Deployment name, unique K8s label, JSON of original container that
was swapped out.)
In practice OpenShift doesn't seem to do the right thing when a
DeploymentConfig is updated. In particular, we need to disable the image
trigger so that we can use the new image, but the replicationcontroller
then continues to deploy the existing image.
So instead we use a different approach than for Kubernetes, replacing the
current ReplicationController with one that uses the Telepresence image,
then restores it. We delete the pods to force the RC to do its thing.
"""
run_id = runner.session_id
deployment, container = _split_deployment_container(deployment_arg)
rcs = runner.get_output(
runner.kubectl("get", "rc", "-o", "name", "--selector",
"openshift.io/deployment-config.name=" + deployment))
rc_name = sorted(rcs.split(),
key=lambda n: int(n.split("-")[-1]))[0].split("/", 1)[1]
rc_json = json.loads(
runner.get_output(runner.kubectl("get", "rc", "-o", "json", "--export",
rc_name),
stderr=STDOUT))
def apply_json(json_config):
runner.check_call(runner.kubectl("apply", "-f", "-"),
input=json.dumps(json_config).encode("utf-8"))
# Now that we've updated the replication controller, delete pods to
# make sure changes get applied:
runner.check_call(
runner.kubectl("delete", "pod", "--selector",
"deployment=" + rc_name))
runner.add_cleanup("Restore original replication controller", apply_json,
rc_json)
container = _get_container_name(container, rc_json)
new_rc_json, orig_container_json = new_swapped_deployment(
rc_json,
container,
run_id,
image_name,
add_custom_nameserver,
)
apply_json(new_rc_json)
_merge_expose_ports(expose, orig_container_json)
return deployment, run_id
def connect_sshuttle(runner: Runner, remote_info: RemoteInfo,
hosts_or_ips: List[str], ssh: SSH) -> None:
"""Connect to Kubernetes using sshuttle."""
span = runner.span()
sshuttle_method = "auto"
if runner.platform == "linux":
# sshuttle tproxy mode seems to have issues:
sshuttle_method = "nat"
runner.launch(
"sshuttle",
get_sshuttle_command(ssh, sshuttle_method) + [
# DNS proxy running on remote pod:
"--to-ns",
"127.0.0.1:9053",
] + get_proxy_cidrs(runner, remote_info, hosts_or_ips),
keep_session=True, # Avoid trouble with interactive sudo
)
# sshuttle will take a while to startup. We can detect it being up when
# DNS resolution of services starts working. We use a specific single
# segment so any search/domain statements in resolv.conf are applied,
# which then allows the DNS proxy to detect the suffix domain and
# filter it out.
# On Macs, and perhaps elsewhere, there is OS-level caching of
# NXDOMAIN, so bypass caching by sending new domain each time. Another,
# less robust alternative, is to `killall -HUP mDNSResponder`.
subspan = runner.span("sshuttle-wait")
countdown = 3
for idx in runner.loop_until(35, 0.1):
# Construct a different name each time to avoid NXDOMAIN caching.
name = "hellotelepresence-{}".format(idx)
runner.write("Wait for vpn-tcp connection: {}".format(name))
if dns_lookup(runner, name, 5):
countdown -= 1
runner.write("Resolved {}. {} more...".format(name, countdown))
if countdown == 0:
break
# The loop uses a single segment to try to capture suffix or search
# path in the proxy. However, in some network setups, single-segment
# names don't get resolved the normal way. To see whether we're running
# into this, also try to resolve a name with many dots. This won't
# resolve successfully but will show up in the logs. See also:
# https://github.com/telepresenceio/telepresence/issues/242. We use a
# short timeout here because (1) this takes a long time for some users
# and (2) we're only looking for a log entry; we don't expect this to
# succeed and don't benefit from waiting for the NXDOMAIN.
many_dotted_name = "{}.a.sanity.check.telepresence.io".format(name)
dns_lookup(runner, many_dotted_name, 1)
if countdown != 0:
runner.add_cleanup("Diagnose vpn-tcp", log_info_vpn_crash, runner)
raise RuntimeError("vpn-tcp tunnel did not connect")
subspan.end()
span.end()
def launch_inject(
runner: Runner,
command: List[str],
socks_port: int,
env_overrides: Dict[str, str],
) -> Popen:
"""
Launch the user's command under torsocks
"""
torsocks_env = set_up_torsocks(runner, socks_port)
env_overrides.update(torsocks_env)
env = get_local_env(runner, env_overrides, True)
process = Popen(command, env=env)
runner.add_cleanup("Terminate local process", terminate_local_process,
runner, process)
return process
def launch_vpn(
runner: Runner,
remote_info: RemoteInfo,
command: List[str],
also_proxy: List[str],
env_overrides: Dict[str, str],
ssh: SSH,
) -> Popen:
"""
Launch sshuttle and the user's command
"""
connect_sshuttle(runner, remote_info, also_proxy, ssh)
env = get_local_env(runner, env_overrides, False)
process = Popen(command, env=env)
runner.add_cleanup("Terminate local process", terminate_local_process,
runner, process)
return process
def run_local_command(
runner: Runner,
remote_info: RemoteInfo,
args: argparse.Namespace,
env_overrides: Dict[str, str],
socks_port: int,
ssh: SSH,
) -> Popen:
"""--run-shell/--run support, run command locally."""
env = os.environ.copy()
env.update(env_overrides)
# Don't use runner.popen() since we want to give program access to current
# stdout and stderr if it wants it.
env["PROMPT_COMMAND"] = ('PS1="@{}|$PS1";unset PROMPT_COMMAND'.format(
args.context))
# Inject replacements for unsupported tools like ping:
unsupported_tools_path = get_unsupported_tools(runner,
args.method != "inject-tcp")
env["PATH"] = unsupported_tools_path + ":" + env["PATH"]
# Make sure we use "bash", no "/bin/bash", so we get the copied version on
# OS X:
if args.run is None:
# We skip .bashrc since it might e.g. have kubectl running to get bash
# autocomplete, and Go programs don't like DYLD on macOS at least (see
# https://github.com/datawire/telepresence/issues/125).
command = ["bash", "--norc"]
else:
command = args.run
if args.method == "inject-tcp":
setup_torsocks(runner, env, socks_port, unsupported_tools_path)
p = Popen(["torsocks"] + command, env=env)
elif args.method == "vpn-tcp":
connect_sshuttle(runner, remote_info, args, env, ssh)
p = Popen(command, env=env)
def terminate_if_alive():
runner.write("Shutting down local process...\n")
if p.poll() is None:
runner.write("Killing local process...\n")
kill_process(p)
runner.add_cleanup("Terminate local process", terminate_if_alive)
return p
def create_new_deployment(runner: Runner,
args: argparse.Namespace) -> Tuple[str, str]:
"""Create a new Deployment, return its name and Kubernetes label."""
span = runner.span()
run_id = runner.session_id
def remove_existing_deployment():
runner.get_output(
runner.kubectl(
"delete",
"--ignore-not-found",
"svc,deploy",
"--selector=telepresence=" + run_id,
))
runner.add_cleanup("Delete new deployment", remove_existing_deployment)
remove_existing_deployment()
if args.needs_root:
image_name = TELEPRESENCE_REMOTE_IMAGE_PRIV
else:
image_name = TELEPRESENCE_REMOTE_IMAGE
command = [
"run",
# This will result in using Deployment:
"--restart=Always",
"--limits=cpu=100m,memory=256Mi",
"--requests=cpu=25m,memory=64Mi",
args.new_deployment,
"--image=" + image_name,
"--labels=telepresence=" + run_id,
]
# Provide a stable argument ordering. Reverse it because that happens to
# make some current tests happy but in the long run that's totally
# arbitrary and doesn't need to be maintained. See issue 494.
for port in sorted(args.expose.remote(), reverse=True):
command.append("--port={}".format(port))
if args.expose.remote():
command.append("--expose")
# If we're on local VM we need to use different nameserver to prevent
# infinite loops caused by sshuttle:
if args.method == "vpn-tcp" and args.in_local_vm:
command.append("--env=TELEPRESENCE_NAMESERVER=" +
get_alternate_nameserver())
runner.get_output(runner.kubectl(command))
span.end()
return args.new_deployment, run_id
def command_main(args):
"""
Top-level function for Telepresence when executing subcommands
"""
with crash_reporting():
runner = Runner(Output(args.logfile), None, args.verbose)
span = runner.span()
runner.add_cleanup("Stop time tracking", span.end)
runner.kubectl = KubeInfo(runner, args)
args.operation = args.command
args.method = "teleproxy"
call_scout(runner, args)
if args.command == "outbound":
return outbound.command(runner)
raise runner.fail("Not implemented!")
def launch_vpn(
runner: Runner,
remote_info: RemoteInfo,
command: List[str],
also_proxy: List[str],
env_overrides: Dict[str, str],
ssh: SSH,
) -> Popen:
"""
Launch sshuttle and the user's command
"""
connect_sshuttle(runner, remote_info, also_proxy, ssh)
env = get_local_env(runner, env_overrides, False)
try:
process = Popen(command, env=env)
except OSError as exc:
raise runner.fail("Failed to launch your command: {}".format(exc))
runner.add_cleanup("Terminate local process", terminate_local_process,
runner, process)
return process
def act(self, runner: Runner) -> RemoteInfo:
runner.show("Starting network proxy to cluster by swapping out " +
"{} {} ".format(self.deployment_type, self.intent.name) +
"with a proxy Pod")
def resize_original(replicas: str) -> None:
"""Resize the original deployment (kubectl scale)."""
runner.check_call(
runner.kubectl("scale", self.deployment_type, self.intent.name,
"--replicas={}".format(replicas)))
create_with_cleanup(runner, self.manifests)
# Scale down the original deployment
runner.add_cleanup("Re-scale original deployment", resize_original,
self.original_replicas)
resize_original("0")
wait_for_pod(runner, self.remote_info)
return self.remote_info
def call_scout(runner: Runner, args):
config_root = Path(Path.home() / ".config" / "telepresence")
config_root.mkdir(parents=True, exist_ok=True)
id_file = Path(config_root / "id")
scout_kwargs = dict(kubectl_version=runner.kubectl.command_version,
kubernetes_version=runner.kubectl.cluster_version,
operation=args.operation,
method=args.method)
try:
with id_file.open('x') as f:
install_id = str(uuid4())
f.write(install_id)
scout_kwargs["new_install"] = True
except FileExistsError:
with id_file.open('r') as f:
install_id = f.read()
scout_kwargs["new_install"] = False
scout = Scout("telepresence", __version__, install_id)
scouted = scout.report(**scout_kwargs)
runner.write("Scout info: {}".format(scouted))
my_version = get_numeric_version(__version__)
try:
latest = get_numeric_version(scouted["latest_version"])
except (KeyError, ValueError):
latest = my_version
if latest > my_version:
message = ("\nTelepresence {} is available (you're running {}). "
"https://www.telepresence.io/reference/changelog").format(
scouted["latest_version"], __version__)
def ver_notice():
runner.show(message)
runner.add_cleanup("Show version notice", ver_notice)
def launch_local(
runner: Runner,
command: List[str],
env_overrides: Dict[str, str],
replace_dns_tools: bool,
) -> Popen:
# Compute user process environment
env = os.environ.copy()
env.update(env_overrides)
env["PROMPT_COMMAND"] = ('PS1="@{}|$PS1";unset PROMPT_COMMAND'.format(
runner.kubectl.context))
env["PATH"] = apply_workarounds(runner, env["PATH"], replace_dns_tools)
# Launch the user process
runner.show("Setup complete. Launching your command.")
try:
process = Popen(command, env=env)
except OSError as exc:
raise runner.fail("Failed to launch your command: {}".format(exc))
runner.add_cleanup("Terminate local process", terminate_local_process,
runner, process)
return process
def supplant_deployment(runner: Runner,
args: argparse.Namespace) -> Tuple[str, str, Dict]:
"""
Swap out an existing Deployment, supplant method.
Native Kubernetes version.
Returns (Deployment name, unique K8s label, JSON of original container that
was swapped out.)
"""
span = runner.span()
run_id = runner.session_id
deployment_name, *container_name = args.swap_deployment.split(":", 1)
if container_name:
container_name = container_name[0]
deployment_json = get_deployment_json(
runner,
deployment_name,
args.context,
args.namespace,
"deployment",
)
# If no container name was given, just use the first one:
if not container_name:
container_name = deployment_json["spec"]["template"]["spec"][
"containers"][0]["name"]
# If we're on local VM we need to use different nameserver to
# prevent infinite loops caused by sshuttle.
add_custom_nameserver = args.method == "vpn-tcp" and args.in_local_vm
if args.needs_root:
image_name = TELEPRESENCE_REMOTE_IMAGE_PRIV
else:
image_name = TELEPRESENCE_REMOTE_IMAGE
new_deployment_json, orig_container_json = new_swapped_deployment(
deployment_json,
container_name,
run_id,
image_name,
add_custom_nameserver,
)
# Compute a new name that isn't too long, i.e. up to 63 characters.
# Trim the original name until "tel-{run_id}-{pod_id}" fits.
# https://github.com/kubernetes/community/blob/master/contributors/design-proposals/architecture/identifiers.md
new_deployment_name = "{name:.{max_width}s}-{id}".format(
name=deployment_json["metadata"]["name"],
id=run_id,
max_width=(50 - (len(run_id) + 1)))
new_deployment_json["metadata"]["name"] = new_deployment_name
def resize_original(replicas):
"""Resize the original deployment (kubectl scale)"""
runner.check_call(
runner.kubectl("scale", "deployment", deployment_name,
"--replicas={}".format(replicas)))
def delete_new_deployment(check):
"""Delete the new (copied) deployment"""
ignore = []
if not check:
ignore = ["--ignore-not-found"]
runner.check_call(
runner.kubectl("delete", "deployment", new_deployment_name,
*ignore))
# Launch the new deployment
runner.add_cleanup("Delete new deployment", delete_new_deployment, True)
delete_new_deployment(False) # Just in case
runner.check_call(runner.kubectl("apply", "-f", "-"),
input=json.dumps(new_deployment_json).encode("utf-8"))
# Scale down the original deployment
runner.add_cleanup("Re-scale original deployment", resize_original,
deployment_json["spec"]["replicas"])
resize_original(0)
span.end()
return new_deployment_name, run_id, orig_container_json
def create_new_deployment(
runner: Runner,
deployment_arg: str,
expose: PortMapping,
custom_nameserver: Optional[str],
service_account: str,
) -> Tuple[str, str]:
"""
Create a new Deployment, return its name and Kubernetes label.
"""
span = runner.span()
run_id = runner.session_id
runner.show(
"Starting network proxy to cluster using "
"new Deployment {}".format(deployment_arg)
)
def remove_existing_deployment(quiet=False):
if not quiet:
runner.show("Cleaning up Deployment {}".format(deployment_arg))
runner.check_call(
runner.kubectl(
"delete",
"--ignore-not-found",
"svc,deploy",
"--selector=telepresence=" + run_id,
)
)
runner.add_cleanup("Delete new deployment", remove_existing_deployment)
remove_existing_deployment(quiet=True)
# Define the deployment as yaml
env = {}
if custom_nameserver:
# If we're on local VM we need to use different nameserver to prevent
# infinite loops caused by sshuttle:
env["TELEPRESENCE_NAMESERVER"] = custom_nameserver
# Create the deployment via yaml
deployment_yaml = _get_deployment_yaml(
deployment_arg,
run_id,
get_image_name(runner, expose),
service_account,
env,
)
try:
runner.check_call(
runner.kubectl("create", "-f", "-"),
input=deployment_yaml.encode("utf-8")
)
except CalledProcessError as exc:
raise runner.fail(
"Failed to create deployment {}:\n{}".format(
deployment_arg, exc.stderr
)
)
# Expose the deployment with a service
if expose.remote():
command = [
"expose",
"deployment",
deployment_arg,
]
# Provide a stable argument ordering. Reverse it because that
# happens to make some current tests happy but in the long run
# that's totally arbitrary and doesn't need to be maintained.
# See issue 494.
for port in sorted(expose.remote(), reverse=True):
command.append("--port={}".format(port))
try:
runner.check_call(runner.kubectl(*command))
except CalledProcessError as exc:
raise runner.fail(
"Failed to expose deployment {}:\n{}".format(
deployment_arg, exc.stderr
)
)
span.end()
return deployment_arg, run_id
def connect(runner: Runner, remote_info: RemoteInfo, is_container_mode: bool,
expose: PortMapping) -> Tuple[int, SSH]:
"""
Start all the processes that handle remote proxying.
Return (local port of SOCKS proxying tunnel, SSH instance).
"""
span = runner.span()
# Keep local copy of pod logs, for debugging purposes:
runner.launch(
"kubectl logs",
runner.kubectl("logs", "-f", remote_info.pod_name, "--container",
remote_info.container_name),
bufsize=0,
)
ssh = SSH(runner, find_free_port())
# forward remote port to here, by tunneling via remote SSH server:
runner.launch(
"kubectl port-forward",
runner.kubectl("port-forward", remote_info.pod_name,
"{}:8022".format(ssh.port)))
if is_container_mode:
# kubectl port-forward currently only listens on loopback. So we
# portforward from the docker0 interface on Linux, and the lo0 alias we
# added on OS X, to loopback (until we can use kubectl port-forward
# option to listen on docker0 -
# https://github.com/kubernetes/kubernetes/pull/46517, or all our users
# have latest version of Docker for Mac, which has nicer solution -
# https://github.com/datawire/telepresence/issues/224).
if runner.platform == "linux":
# If ip addr is available use it if not fall back to ifconfig.
missing = runner.depend(["ip", "ifconfig"])
if "ip" not in missing:
docker_interfaces = re.findall(
r"(\d+\.\d+\.\d+\.\d+)",
runner.get_output(["ip", "addr", "show", "dev",
"docker0"]))
elif "ifconfig" not in missing:
docker_interfaces = re.findall(
r"(\d+\.\d+\.\d+\.\d+)",
runner.get_output(["ifconfig", "docker0"]))
else:
raise runner.fail(
"""At least one of "ip addr" or "ifconfig" must be """ +
"available to retrieve Docker interface info.")
if len(docker_interfaces) == 0:
raise runner.fail("No interface for docker found")
docker_interface = docker_interfaces[0]
else:
# The way to get routing from container to host is via an alias on
# lo0 (https://docs.docker.com/docker-for-mac/networking/). We use
# an IP range that is assigned for testing network devices and
# therefore shouldn't conflict with real IPs or local private
# networks (https://tools.ietf.org/html/rfc6890).
runner.check_call(
["sudo", "ifconfig", "lo0", "alias", MAC_LOOPBACK_IP])
runner.add_cleanup(
"Mac Loopback", runner.check_call,
["sudo", "ifconfig", "lo0", "-alias", MAC_LOOPBACK_IP])
docker_interface = MAC_LOOPBACK_IP
runner.launch("socat for docker", [
"socat", "TCP4-LISTEN:{},bind={},reuseaddr,fork".format(
ssh.port,
docker_interface,
), "TCP4:127.0.0.1:{}".format(ssh.port)
])
ssh.wait()
# In Docker mode this happens inside the local Docker container:
if not is_container_mode:
expose_local_services(
runner,
ssh,
list(expose.local_to_remote()),
)
# Start tunnels for the SOCKS proxy (local -> remote)
# and the local server for the proxy to poll (remote -> local).
socks_port = find_free_port()
local_server_port = find_free_port()
runner.track_background(
launch_local_server(local_server_port, runner.output))
forward_args = [
"-L127.0.0.1:{}:127.0.0.1:9050".format(socks_port),
"-R9055:127.0.0.1:{}".format(local_server_port)
]
runner.launch("SSH port forward (socks and proxy poll)",
ssh.bg_command(forward_args))
span.end()
return socks_port, ssh
def run_docker_command(
runner: Runner,
remote_info: RemoteInfo,
args: argparse.Namespace,
remote_env: Dict[str, str],
ssh: SSH,
mount_dir: Optional[str],
) -> Popen:
"""
--docker-run support.
Connect using sshuttle running in a Docker container, and then run user
container.
:param args: Command-line args to telepresence binary.
:param remote_env: Dictionary with environment on remote pod.
:param mount_dir: Path to local directory where remote pod's filesystem is
mounted.
"""
if SUDO_FOR_DOCKER:
runner.require_sudo()
# Update environment:
remote_env["TELEPRESENCE_METHOD"] = "container" # mostly just for tests :(
# Extract --publish flags and add them to the sshuttle container, which is
# responsible for defining the network entirely.
docker_args, publish_args = parse_docker_args(args.docker_run)
# Start the sshuttle container:
name = random_name()
config = {
"port":
ssh.port,
"cidrs":
get_proxy_cidrs(
runner, args, remote_info, remote_env["KUBERNETES_SERVICE_HOST"]
),
"expose_ports":
list(args.expose.local_to_remote()),
}
if runner.platform == "darwin":
config["ip"] = MAC_LOOPBACK_IP
# Image already has tini init so doesn't need --init option:
span = runner.span()
runner.launch(
"Network container",
docker_runify(
publish_args + [
"--rm", "--privileged", "--name=" +
name, TELEPRESENCE_LOCAL_IMAGE, "proxy",
json.dumps(config)
]
),
killer=make_docker_kill(runner, name)
)
# Wait for sshuttle to be running:
while True:
try:
runner.check_call(
docker_runify([
"--network=container:" + name, "--rm",
TELEPRESENCE_LOCAL_IMAGE, "wait"
])
)
except CalledProcessError as e:
if e.returncode == 100:
# We're good!
break
elif e.returncode == 125:
# Docker failure, probably due to original container not
# starting yet... so sleep and try again:
sleep(1)
continue
else:
raise
else:
raise RuntimeError(
"Waiting container exited prematurely. File a bug, please!"
)
# Start the container specified by the user:
container_name = random_name()
docker_command = docker_runify([
"--name=" + container_name,
"--network=container:" + name,
],
env=True)
# Prepare container environment
for key in remote_env:
docker_command.append("-e={}".format(key))
docker_env = os.environ.copy()
docker_env.update(remote_env)
if mount_dir:
docker_command.append("--volume={}:{}".format(mount_dir, mount_dir))
# Don't add --init if the user is doing something with it
init_args = [
arg for arg in docker_args
if arg == "--init" or arg.startswith("--init=")
]
# Older versions of Docker don't have --init:
if not init_args and "--init" in runner.get_output([
"docker", "run", "--help"
]):
docker_command += ["--init"]
docker_command += docker_args
span.end()
p = Popen(docker_command, env=docker_env)
def terminate_if_alive():
runner.write("Shutting down containers...\n")
if p.poll() is None:
runner.write("Killing local container...\n")
make_docker_kill(runner, container_name)()
runner.add_cleanup("Terminate local container", terminate_if_alive)
return p
def supplant_deployment(runner: Runner, deployment_arg: str, image_name: str,
expose: PortMapping,
add_custom_nameserver: bool) -> Tuple[str, str]:
"""
Swap out an existing Deployment, supplant method.
Native Kubernetes version.
Returns (Deployment name, unique K8s label, JSON of original container that
was swapped out.)
"""
span = runner.span()
run_id = runner.session_id
deployment, container = _split_deployment_container(deployment_arg)
deployment_json = get_deployment_json(
runner,
deployment,
"deployment",
)
container = _get_container_name(container, deployment_json)
new_deployment_json, orig_container_json = new_swapped_deployment(
deployment_json,
container,
run_id,
image_name,
add_custom_nameserver,
)
# Compute a new name that isn't too long, i.e. up to 63 characters.
# Trim the original name until "tel-{run_id}-{pod_id}" fits.
# https://github.com/kubernetes/community/blob/master/contributors/design-proposals/architecture/identifiers.md
new_deployment_name = "{name:.{max_width}s}-{id}".format(
name=deployment_json["metadata"]["name"],
id=run_id,
max_width=(50 - (len(run_id) + 1)))
new_deployment_json["metadata"]["name"] = new_deployment_name
def resize_original(replicas):
"""Resize the original deployment (kubectl scale)"""
runner.check_call(
runner.kubectl("scale", "deployment", deployment,
"--replicas={}".format(replicas)))
def delete_new_deployment(check):
"""Delete the new (copied) deployment"""
ignore = []
if not check:
ignore = ["--ignore-not-found"]
runner.check_call(
runner.kubectl("delete", "deployment", new_deployment_name,
*ignore))
# Launch the new deployment
runner.add_cleanup("Delete new deployment", delete_new_deployment, True)
delete_new_deployment(False) # Just in case
runner.check_call(runner.kubectl("apply", "-f", "-"),
input=json.dumps(new_deployment_json).encode("utf-8"))
# Scale down the original deployment
runner.add_cleanup("Re-scale original deployment", resize_original,
deployment_json["spec"]["replicas"])
resize_original(0)
_merge_expose_ports(expose, orig_container_json)
span.end()
return new_deployment_name, run_id
def run_docker_command(
runner: Runner,
remote_info: RemoteInfo,
docker_run: List[str],
expose: PortMapping,
to_pod: List[int],
from_pod: List[int],
container_to_host: PortMapping,
remote_env: Dict[str, str],
ssh: SSH,
mount_dir: Optional[str],
use_docker_mount: Optional[bool],
pod_info: Dict[str, str],
) -> "subprocess.Popen[bytes]":
"""
--docker-run support.
Connect using sshuttle running in a Docker container, and then run user
container.
:param remote_env: Dictionary with environment on remote pod.
:param mount_dir: Path to local directory where remote pod's filesystem is
mounted.
"""
# Update environment:
remote_env["TELEPRESENCE_METHOD"] = "container" # mostly just for tests :(
# Extract --publish flags and add them to the sshuttle container, which is
# responsible for defining the network entirely.
docker_args, publish_args = parse_docker_args(docker_run)
# Point a host port to the network container's sshd
container_sshd_port = find_free_port()
publish_args.append(
"--publish=127.0.0.1:{}:38022/tcp".format(container_sshd_port)
)
local_ssh = SSH(runner, container_sshd_port, "[email protected]")
# Start the network (sshuttle) container:
name = random_name()
config = {
"cidrs": ["0/0"],
"expose_ports": list(expose.local_to_remote()),
"to_pod": to_pod,
"from_pod": from_pod,
}
dns_args = []
if "hostname" in pod_info:
dns_args.append("--hostname={}".format(pod_info["hostname"].strip()))
if "hosts" in pod_info:
dns_args.extend(parse_hosts_aliases(pod_info["hosts"]))
if "resolv" in pod_info:
dns_args.extend(parse_resolv_conf(pod_info["resolv"]))
# Image already has tini init so doesn't need --init option:
span = runner.span()
runner.launch(
"Network container",
runner.docker(
"run", *publish_args, *dns_args, "--rm", "--privileged",
"--name=" + name, TELEPRESENCE_LOCAL_IMAGE, "proxy",
json.dumps(config)
),
killer=make_docker_kill(runner, name),
keep_session=runner.sudo_for_docker,
)
# Set up ssh tunnel to allow the container to reach the cluster
if not local_ssh.wait():
raise RuntimeError("SSH to the network container failed to start.")
container_forward_args = ["-R", "38023:127.0.0.1:{}".format(ssh.port)]
for container_port, host_port in container_to_host.local_to_remote():
if runner.chatty:
runner.show(
"Forwarding container port {} to host port {}.".format(
container_port, host_port
)
)
container_forward_args.extend([
"-R", "{}:127.0.0.1:{}".format(container_port, host_port)
])
runner.launch(
"Local SSH port forward", local_ssh.bg_command(container_forward_args)
)
# Wait for sshuttle to be running:
sshuttle_ok = False
for _ in runner.loop_until(120, 1):
try:
runner.check_call(
runner.docker(
"run", "--network=container:" + name, "--rm",
TELEPRESENCE_LOCAL_IMAGE, "wait"
)
)
except subprocess.CalledProcessError as e:
if e.returncode == 100:
# We're good!
sshuttle_ok = True
break
elif e.returncode == 125:
# Docker failure, probably due to original container not
# starting yet... so try again:
continue
else:
raise
else:
raise RuntimeError(
"Waiting container exited prematurely. File a bug, please!"
)
if not sshuttle_ok:
# This used to loop forever. Now we time out after two minutes.
raise RuntimeError(
"Waiting for network container timed out. File a bug, please!"
)
# Start the container specified by the user:
container_name = random_name()
docker_command = runner.docker(
"run",
"--name=" + container_name,
"--network=container:" + name,
env=True,
)
# Prepare container environment
for key in remote_env:
docker_command.append("-e={}".format(key))
docker_env = os.environ.copy()
docker_env.update(remote_env)
if mount_dir:
if use_docker_mount:
mount_volume = "telepresence-" + runner.session_id
else:
mount_volume = mount_dir
docker_command.append("--volume={}:{}".format(mount_volume, mount_dir))
# Don't add --init if the user is doing something with it
init_args = [
arg for arg in docker_args
if arg == "--init" or arg.startswith("--init=")
]
# Older versions of Docker don't have --init:
docker_run_help = runner.get_output(["docker", "run", "--help"])
if not init_args and "--init" in docker_run_help:
docker_command += ["--init"]
docker_command += docker_args
span.end()
runner.show("Setup complete. Launching your container.")
process = subprocess.Popen(docker_command, env=docker_env)
def terminate_if_alive() -> None:
runner.write("Shutting down containers...\n")
if process.poll() is None:
runner.write("Killing local container...\n")
make_docker_kill(runner, container_name)()
runner.add_cleanup("Terminate local container", terminate_if_alive)
return process
