def _load_recent_sessions(must_exist=True):
"""
Load recent sessions from RECENTS_FILE.
:param must_exist: if True and session file does not exist, exit with error
:return: dict of latest sessions
"""
global _recent_sessions
if _recent_sessions is not None:
return _recent_sessions
if os.path.exists(RECENTS_FILE):
_recent_sessions = OrderedDict()
try:
for line in open(RECENTS_FILE, "rt"):
key, args = line.strip().split(":::", 1)
_recent_sessions[key] = args
except Exception as exc:
message(ff("Error reading {RECENTS_FILE}: {exc}"))
_recent_sessions = None
if _recent_sessions is None and must_exist:
bye("no recent radiopadre sessions and no arguments given. Run with -h for help."
)
return _recent_sessions
def update_installation(enable_pull=False):
global docker_image
enable_pull = enable_pull or config.AUTO_INIT or config.UPDATE
if config.CONTAINER_DEV:
update_server_from_repository()
docker_image = config.DOCKER_IMAGE
if check_output(ff("docker image inspect {docker_image}")) is None:
if not enable_pull:
bye(
ff(" Radiopadre docker image {docker_image} not found. Re-run with --update or --auto-init perhaps?"
))
message(
ff(" Radiopadre docker image {docker_image} not found locally"))
else:
message(ff(" Using radiopadre docker image {docker_image}"))
if enable_pull:
warning(ff("Calling docker pull {docker_image}"))
warning(
" (This may take a few minutes if the image is not up to date...)"
)
try:
subprocess.call([docker, "pull", docker_image])
except subprocess.CalledProcessError as exc:
if config.IGNORE_UPDATE_ERRORS:
warning(
"docker pull failed, but --ignore-update-errors is set, proceeding anyway"
)
return
raise exc
def kill_sessions(session_dict, session_ids, ignore_fail=False):
kill_cont = " ".join(session_ids)
message(" killing containers: {}".format(kill_cont))
for cont in session_ids:
if cont not in session_dict:
bye("no such radiopadre container: {}".format(cont))
name, path, _, _, _ = session_dict[cont]
session_id_file = "{}/{}".format(SESSION_INFO_DIR, name)
if os.path.exists(session_id_file):
subprocess.call(["rm", "-fr", session_id_file])
shell(ff("{docker} kill {kill_cont}"), ignore_fail=True)
def ssh_remote_interactive(command, fail_retcode=None):
"""Runs command on remote host. Returns the exit status if 0, or None if the exit status matches fail_retcode.
Any other non-zero exit status (or any other error) will result in an exception.
"""
try:
return subprocess.check_call(SSH_OPTS + [command])
except subprocess.CalledProcessError as exc:
if exc.returncode == fail_retcode:
return None
message(ff("ssh {command} failed with exit code {exc.returncode}"))
raise
def init(binary, docker_binary=None):
global singularity, has_docker
singularity = binary
_init_session_dir()
if docker_binary:
# check that we actually have docker permissions
if check_output(docker_binary + " ps") is None:
message(
"can't connect to docker daemon, will proceed without docker")
has_docker = None
else:
has_docker = docker_binary
docker.init(docker_binary)
def ssh_remote(command, fail_retcode=None, stderr=DEVNULL):
"""Runs command on remote host. Returns its output if the exit status is 0, or None if the exit status matches fail_retcode.
Any other non-zero exit status (or any other error) will result in an exception.
"""
try:
return subprocess.check_output(SSH_OPTS + [command],
stderr=stderr).decode('utf-8')
except subprocess.CalledProcessError as exc:
if exc.returncode == fail_retcode:
return None
message(ff("ssh {command} failed with exit code {exc.returncode}"))
raise
def _run_container(container_name,
docker_opts,
jupyter_port,
browser_urls,
singularity=False):
message("Running {}".format(" ".join(map(str, docker_opts))))
if singularity:
message(
" (When using singularity and the image is not yet available locally, this can take a few minutes the first time you run.)"
)
if config.CONTAINER_DEBUG:
docker_process = subprocess.Popen(docker_opts,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr)
else:
docker_process = subprocess.Popen(
docker_opts,
stdout=DEVNULL,
stderr=DEVNULL if config.NON_INTERACTIVE else sys.stderr)
#stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr,
#env=os.environ)
if config.NBCONVERT:
message("Waiting for conversion to finish")
docker_process.wait()
return None
else:
# pause to let the Jupyter server spin up
wait = await_server_startup(jupyter_port,
process=docker_process,
init_wait=1,
server_name="notebook container")
if wait is None:
if docker_process.returncode is not None:
bye(
ff("container unexpectedly exited with return code {docker_process.returncode}"
))
bye(
ff("unable to connect to jupyter notebook server on port {jupyter_port}"
))
message(
ff("Container started. The jupyter notebook server is running on port {jupyter_port} (after {wait:.2f} secs)"
))
if browser_urls:
iglesia.register_helpers(*run_browser(*browser_urls))
# give things a second (to let the browser command print its stuff, if it wants to)
time.sleep(1)
return docker_process
def update_server_from_repository():
"""
Updates the radiopadre git working directory, if necessary
:return:
"""
if config.UPDATE and config.SERVER_INSTALL_PATH and os.path.isdir(
config.SERVER_INSTALL_PATH + "/.git"):
if config.SERVER_INSTALL_BRANCH:
cmd = ff(
"cd {config.SERVER_INSTALL_PATH} && git fetch origin && git checkout {config.SERVER_INSTALL_BRANCH} && git pull"
)
else:
cmd = ff("cd {config.SERVER_INSTALL_PATH} && git pull")
message(
ff("--update specified, --server-install-path at {config.SERVER_INSTALL_PATH} will be updated via"
))
message(ff(" {cmd}"))
if shell(cmd):
bye("update failed")
def list_sessions():
"""Returns OrderedDict (ordered by uptime) of running containers with their session IDs. Clears up dead sessions.
Dict is id -> [name, path, uptime, session_id, ports]"""
container_dict = _ps_containers()
# match session files to containers
for session_dir in glob.glob(SESSION_INFO_DIR + "/radiopadre-*"):
name = os.path.basename(session_dir)
if name not in container_dict:
message(
" container {} is no longer running, clearing up session dir"
.format(name))
subprocess.call(["rm", "-fr", session_dir])
continue
try:
container_dict[name][3], container_dict[name][
4] = read_session_info(session_dir)
except ValueError:
message(ff(" invalid session dir {session_dir}"))
continue
output = OrderedDict()
# check for containers without session info and form up output dict
for name, (id_, path, time, session_id, ports) in container_dict.items():
if session_id is None:
message(" container {} has no session dir -- killing it".format(
name))
subprocess.call([docker, "kill", id_])
else:
output[id_] = [name, path, time, session_id, ports]
return output
def await_server_startup(port,
process=None,
server_name="jupyter notebook server",
init_wait=2,
wait=60):
"""
Waits for a server process to start up, tries to connect to the specified port,
returns when successful
:param port: port number
:param process: if not None, waits on the process and checks its return code
:param init_wait: number of second to wait before trying to connect
:param wait: total number of seconds to wait before giving up
:return: number of seconds elapsed before connection, or None if failed
"""
# pause to let the Jupyter server spin up
t0 = time.time()
time.sleep(init_wait)
# then try to connect to it
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
for retry in range(int(wait / .1)):
# try to connect
try:
sock.connect(("localhost", port))
del sock
return time.time() - t0
except socket.error:
pass
if not retry:
message(
ff("Waiting for up to {wait} secs for the {server_name} to come up"
))
# sleep, check process
if process is not None:
process.poll()
if process.returncode is not None:
return None
time.sleep(.1)
return None
def help_yourself(problem, suggestion=None):
"""
Prints a "help yourself" message and exits
"""
message("{}".format(problem))
message(
ff("Please ssh {config.REMOTE_HOST} and sort it out yourself, then rerun this script"
))
if suggestion:
message(ff("({suggestion})"))
sys.exit(1)
def init_specific_options(remote_host, notebook_path, options):
global _DEFAULT_KEYS
global _CMDLINE_DEFAULTS
parser = configparser.ConfigParser()
hostname = ff("{remote_host}") if remote_host else "local sesssion"
session = ff("{hostname}:{notebook_path}")
config_exists = os.path.exists(CONFIG_FILE)
use_config_files = not options.remote and not options.inside_container
# try to read config file for host and host:path (not in --remote mode though)
if use_config_files and config_exists:
parser.read(CONFIG_FILE)
for sect_key in "global defaults", hostname, session:
if parser.has_section(sect_key):
section = dict(parser.items(sect_key))
if section:
message(
ff(" loading settings from {CONFIG_FILE} [{sect_key}]"
))
for key in _DEFAULT_KEYS:
lkey = key.lower()
if lkey in section:
value = _get_config_value(section, lkey)
if value != globals()[key]:
message(ff(" {key} = {value}"))
globals()[key] = value
# update using command-line options
command_line_updated = []
for key in globals().keys():
if re.match("^[A-Z]", key):
optname = key.lower()
opt_switch = "--" + optname.replace("_", "-")
value = getattr(options, optname, None)
# skip DEFAULT_VALUE placeholders, trust in config
if value is DEFAULT_VALUE or value is None:
continue
if type(value) is list:
value = ",".join(value)
if value is not _CMDLINE_DEFAULTS.get(key, None):
if use_config_files:
# do not mark options such as --update for saving
if value is not _CMDLINE_DEFAULTS.get(key) and DefaultConfig.get(key) is not None \
and opt_switch not in NON_PERSISTING_OPTIONS:
command_line_updated.append(key)
message(ff(" command line specifies {key} = {value}"))
globals()[key] = value
# save new config
if use_config_files and command_line_updated:
if options.save_config_host:
message(
ff(" saving command-line settings to {CONFIG_FILE} [{hostname}]"
))
if not parser.has_section(hostname):
parser.add_section(hostname)
for key in command_line_updated:
parser.set(hostname, key, _set_config_value(key))
if options.save_config_session:
if not parser.has_section(session):
parser.add_section(session)
message(
ff(" saving command-line settings to {CONFIG_FILE} [{session}]"
))
for key in command_line_updated:
parser.set(session, key, _set_config_value(key))
if options.save_config_host or options.save_config_session:
radiopadre_dir = make_radiopadre_dir()
with open(CONFIG_FILE + ".new", "w") as configfile:
if not config_exists:
message(ff(" creating new config file {CONFIG_FILE}"))
if not parser.has_section('global defaults'):
configfile.write(
"[global defaults]\n# defaults that apply to all sessions go here\n\n"
)
parser.write(configfile)
configfile.write("\n\n## default settings follow\n")
for key, value in DefaultConfig.items():
configfile.write("# {} = {}\n".format(key.lower(), value))
# if successful, rename files
if config_exists:
if os.path.exists(CONFIG_FILE + ".old"):
os.unlink(CONFIG_FILE + ".old")
os.rename(CONFIG_FILE, CONFIG_FILE + ".old")
os.rename(CONFIG_FILE + ".new", CONFIG_FILE)
message(ff("saved updated config to {CONFIG_FILE}"))
def update_installation():
# are we already running inside a virtualenv? Proceed directly if so
# (see https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv)
# pip install command with -v repeated for each VERBOSE increment
pip_install = "pip install " + "-v " * min(max(config.VERBOSE - 1, 0), 3)
if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix')
and sys.base_prefix != sys.prefix):
if sys.prefix == config.RADIOPADRE_VENV:
message(
ff("Running inside radiopadre virtual environment {sys.prefix}"
))
else:
message(
ff("Running inside non-default virtual environment {sys.prefix}"
))
message(ff("Will assume radiopadre has been installed here."))
config.RADIOPADRE_VENV = sys.prefix
if config.VENV_REINSTALL:
bye("Can't --venv-reinstall from inside the virtualenv itself.")
# Otherwise check for virtualenv, nuke/remake one if needed, then activate it
else:
config.RADIOPADRE_VENV = os.path.expanduser(config.RADIOPADRE_VENV)
activation_script = os.path.join(config.RADIOPADRE_VENV,
"bin/activate_this.py")
# see if a reinstall is needed
if config.AUTO_INIT and config.VENV_REINSTALL and os.path.exists(
config.RADIOPADRE_VENV):
if not os.path.exists(activation_script):
error(ff("{activation_script} does not exist. Bat country!"))
bye(
ff("Refusing to touch this virtualenv. Please remove it by hand if you must."
))
cmd = ff("rm -fr {config.RADIOPADRE_VENV}")
warning(ff("Found a virtualenv in {config.RADIOPADRE_VENV}."))
warning("However, --venv-reinstall was specified. About to run:")
warning(" " + cmd)
if config.FULL_CONSENT:
warning(
"--full-consent given, so not asking for confirmation.")
else:
warning(ff("Your informed consent is required!"))
inp = INPUT(
ff("Please enter 'yes' to rm -fr {config.RADIOPADRE_VENV}: "
)).strip()
if inp != "yes":
bye(ff("'{inp}' is not a 'yes'. Phew!"))
message("OK, nuking it!")
shell(cmd)
new_venv = False
if not os.path.exists(config.RADIOPADRE_VENV):
if config.AUTO_INIT:
message(ff("Creating virtualenv {config.RADIOPADRE_VENV}"))
shell(ff("virtualenv -p python3 {config.RADIOPADRE_VENV}"))
new_venv = True
else:
error(
ff("Radiopadre virtualenv {config.RADIOPADRE_VENV} doesn't exist."
))
bye(ff("Try re-running with --auto-init to reinstall it."))
message(
ff(" Activating the radiopadre virtualenv via {activation_script}"
))
with open(activation_script) as f:
code = compile(f.read(), activation_script, 'exec')
exec(code, dict(__file__=activation_script), {})
if new_venv:
extras = config.VENV_EXTRAS.split(
",") if config.VENV_EXTRAS else []
# add numpy explicitly to quicken up pyregion install
extras.append("numpy")
if extras:
extras = " ".join(extras)
message(ff("Installing specified extras: {extras}"))
shell(ff("{pip_install} {extras}"))
# now check for a radiopadre install inside the venv
have_install = check_output("pip show radiopadre")
if have_install:
install_info = dict(
[x.split(": ", 1) for x in have_install.split("\n") if ': ' in x])
version = install_info.get("Version", "unknown")
if config.UPDATE:
warning(
ff("radiopadre (version {version}) is installed, but --update specified."
))
else:
message(ff("radiopadre (version {version}) is installed."))
if not have_install or config.UPDATE:
if config.SERVER_INSTALL_PATH and os.path.exists(
config.SERVER_INSTALL_PATH):
message(
ff("--server-install-path {config.SERVER_INSTALL_PATH} is configured and exists."
))
update_server_from_repository()
install = ff("-e {config.SERVER_INSTALL_PATH}")
elif config.SERVER_INSTALL_REPO:
if config.SERVER_INSTALL_REPO == "default":
config.SERVER_INSTALL_REPO = config.DEFAULT_SERVER_INSTALL_REPO
branch = config.SERVER_INSTALL_BRANCH or "master"
if config.SERVER_INSTALL_PATH:
message(
ff("--server-install-path and --server-install-repo configured, will clone and install"
))
cmd = ff(
"git clone -b {branch} {config.SERVER_INSTALL_REPO} {config.SERVER_INSTALL_PATH}"
)
message(ff("Running {cmd}"))
shell(cmd)
install = ff("-e {config.SERVER_INSTALL_PATH}")
else:
message(
ff("only --server-install-repo specified, will install directly from git"
))
install = ff("git+{config.SERVER_INSTALL_REPO}@{branch}")
elif config.SERVER_INSTALL_PIP:
message(
ff("--server-install-pip {config.SERVER_INSTALL_PIP} is configured."
))
install = config.SERVER_INSTALL_PIP
else:
bye("no radiopadre installation method specified (see --server-install options)"
)
cmd = ff("{pip_install} -U {install}")
message(ff("Running {cmd}"))
shell(cmd)
def start_session(container_name, selected_ports, userside_ports,
notebook_path, browser_urls):
from iglesia import ROOTDIR
from radiopadre_client.server import JUPYTER_OPTS
# get hostname
os.environ["HOSTNAME"] = subprocess.check_output("/bin/hostname").decode()
jupyter_port = selected_ports[0]
userside_http_port = userside_ports[2]
if config.NBCONVERT:
JUPYTER_OPTS.append(notebook_path)
else:
JUPYTER_OPTS += [
ff("--port={jupyter_port}"), "--no-browser", "--browser=/dev/null"
] # --no-browser alone seems to be ignored
if config.INSIDE_CONTAINER_PORTS or config.CONTAINER_TEST:
JUPYTER_OPTS += ["--allow-root", "--ip=0.0.0.0"]
# if LOAD_NOTEBOOK:
# JUPYTER_OPTS.append(LOAD_NOTEBOOK if type(LOAD_NOTEBOOK) is str else LOAD_NOTEBOOK[0])
# pass configured ports to radiopadre kernel
os.environ['RADIOPADRE_SELECTED_PORTS'] = ":".join(map(
str, selected_ports))
os.environ['RADIOPADRE_USERSIDE_PORTS'] = ":".join(map(
str, userside_ports))
# get base path of radiopadre install
radiopadre_base = subprocess.check_output(
ff(""". {config.RADIOPADRE_VENV}/bin/activate && \
python -c "import importlib; print(importlib.find_loader('radiopadre').get_filename())" """
),
shell=True).decode().strip()
radiopadre_base = os.path.dirname(os.path.dirname(radiopadre_base))
message(
ff("Detected radiopadre directory within virtualenv as {radiopadre_base}"
))
# default JS9 dir goes off the virtualenv
os.environ.setdefault("RADIOPADRE_JS9_DIR",
ff("{config.RADIOPADRE_VENV}/js9-www"))
iglesia.init_helpers(radiopadre_base,
verbose=config.VERBOSE > 0,
run_js9=not config.NBCONVERT,
run_carta=not config.NBCONVERT)
## start jupyter process
jupyter_path = config.RADIOPADRE_VENV + "/bin/jupyter"
message("Starting: {} {} in {}".format(jupyter_path,
" ".join(JUPYTER_OPTS),
os.getcwd()))
notebook_proc = subprocess.Popen([jupyter_path] + JUPYTER_OPTS,
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr,
bufsize=1,
universal_newlines=True,
env=os.environ)
## use this instead to debug the sessison
#notebook_proc = subprocess.Popen([config.RADIOPADRE_VENV+"/bin/ipython"],
# stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr,
# env=os.environ)
if config.NBCONVERT:
message("Waiting for conversion to finish")
notebook_proc.wait()
else:
iglesia.register_helpers(notebook_proc)
# launch browser
if browser_urls:
iglesia.register_helpers(*run_browser(*browser_urls))
# elif not config.REMOTE_MODE_PORTS and not config.INSIDE_CONTAINER_PORTS:
# message("Please point your browser to {}".format(" ".join(browser_urls)))
# pause to let the Jupyter server spin up
wait = await_server_startup(jupyter_port,
init_wait=0,
process=notebook_proc)
if wait is None:
if notebook_proc.returncode is not None:
bye(
ff("jupyter unexpectedly exited with return code {notebook_proc.returncode}"
))
bye(
ff("unable to connect to jupyter notebook server on port {jupyter_port}"
))
message(
ff("The jupyter notebook server is running on port {jupyter_port} (after {wait:.2f} secs)"
))
if config.CONTAINER_TEST:
message(ff("--container-test was specified, dry run is complete"))
sys.exit(0)
try:
while True:
if config.INSIDE_CONTAINER_PORTS:
debug("inside container -- sleeping indefinitely")
time.sleep(100000)
else:
a = INPUT("Type 'exit' to kill the session: ")
if notebook_proc.poll() is not None:
message("The notebook server has exited with code {}".
format(notebook_proc.poll()))
sys.exit(0)
if a.lower() == 'exit':
message("Exit request received")
sys.exit(0)
except BaseException as exc:
if type(exc) is KeyboardInterrupt:
message("Caught Ctrl+C")
status = 1
elif type(exc) is EOFError:
message("Input channel has closed")
status = 1
elif type(exc) is SystemExit:
status = getattr(exc, 'code', 0)
message("Exiting with status {}".format(status))
else:
message("Caught exception {} ({})".format(exc, type(exc)))
status = 1
sys.exit(status)
def kill_container(name):
message(ff("Killing container {name}"))
shell(ff("{docker} kill {name}"), ignore_fail=True)
def run_remote_session(command, copy_initial_notebook, notebook_path,
extra_arguments):
SSH_MUX_OPTS = "-o ControlPath=/tmp/ssh_mux_radiopadre_%C -o ControlMaster=auto -o ControlPersist=1h".split(
)
SCP_OPTS = ["scp"] + SSH_MUX_OPTS
# SSH_OPTS = ["ssh", "-t"] + SSH_MUX_OPTS + [config.REMOTE_HOST]
SSH_OPTS = ["ssh"] + SSH_MUX_OPTS + [config.REMOTE_HOST]
# See, possibly: https://stackoverflow.com/questions/44348083/how-to-send-sigint-ctrl-c-to-current-remote-process-over-ssh-without-t-optio
# master ssh connection, to be closed when we exit
message(
ff("Opening ssh connection to {config.REMOTE_HOST}. You may be prompted for your password."
))
debug(" {}".format(" ".join(SSH_OPTS)))
ssh_master = subprocess.check_call(SSH_OPTS + ["exit"], stderr=DEVNULL)
# raw_input("Continue?")
def help_yourself(problem, suggestion=None):
"""
Prints a "help yourself" message and exits
"""
message("{}".format(problem))
message(
ff("Please ssh {config.REMOTE_HOST} and sort it out yourself, then rerun this script"
))
if suggestion:
message(ff("({suggestion})"))
sys.exit(1)
def ssh_remote(command, fail_retcode=None, stderr=DEVNULL):
"""Runs command on remote host. Returns its output if the exit status is 0, or None if the exit status matches fail_retcode.
Any other non-zero exit status (or any other error) will result in an exception.
"""
try:
return subprocess.check_output(SSH_OPTS + [command],
stderr=stderr).decode('utf-8')
except subprocess.CalledProcessError as exc:
if exc.returncode == fail_retcode:
return None
message(ff("ssh {command} failed with exit code {exc.returncode}"))
raise
def ssh_remote_v(command, fail_retcode=None):
return ssh_remote(command, fail_retcode, stderr=sys.stderr)
def ssh_remote_interactive(command, fail_retcode=None):
"""Runs command on remote host. Returns the exit status if 0, or None if the exit status matches fail_retcode.
Any other non-zero exit status (or any other error) will result in an exception.
"""
try:
return subprocess.check_call(SSH_OPTS + [command])
except subprocess.CalledProcessError as exc:
if exc.returncode == fail_retcode:
return None
message(ff("ssh {command} failed with exit code {exc.returncode}"))
raise
def scp_to_remote(path, remote_path):
return subprocess.check_output(
SCP_OPTS + [path, "{}:{}".format(config.REMOTE_HOST, remote_path)])
def check_remote_file(remote_file, test="-x"):
"""
Checks that a remote file exists. 'test' is specified bash-style, e.g. "-x" for executable.
Can also use -f and -f, for example.
Returns True or False, or raises an exception on other errors.
"""
return ssh_remote(
"if [ {} {} ]; then exit 0; else exit 199; fi".format(
test, remote_file),
fail_retcode=199,
stderr=DEVNULL) is not None
def check_remote_command(command):
"""
Checks that remote host has a particular command available (by running 'which' on the remote).
Returns True or False, or raises an exception on other errors.
"""
if config.SKIP_CHECKS:
return command
return (ssh_remote("which " + command, fail_retcode=1, stderr=DEVNULL)
or "").strip()
# --update or --auto-init disables --skip-checks
if config.SKIP_CHECKS:
if config.UPDATE:
message("Note that --update implies --no-skip-checks")
config.SKIP_CHECKS = False
elif config.AUTO_INIT:
message("Note that --auto-init implies --no-skip-checks")
config.SKIP_CHECKS = False
# propagate our config to command-line arguments
remote_config = config.get_config_dict()
remote_config['BROWSER'] = 'None'
remote_config['SKIP_CHECKS'] = False
remote_config['VENV_REINSTALL'] = False
# Check for various remote bits
if config.VERBOSE and not config.SKIP_CHECKS:
message(ff("Checking installation on {config.REMOTE_HOST}."))
has_git = check_remote_command("git")
USE_VENV = has_singularity = has_docker = None
for backend in config.BACKEND:
remote_config["BACKEND"] = backend
if backend == "venv" and check_remote_command(
"virtualenv") and check_remote_command("pip"):
USE_VENV = True
break
elif backend == "docker":
has_docker = check_remote_command("docker")
if has_docker:
break
elif backend == "singularity":
has_singularity = check_remote_command("singularity")
if has_singularity:
break
message(
ff("The '{backend}' back-end is not available on {config.REMOTE_HOST}, skipping."
))
else:
bye(
ff("None of the specified back-ends are available on {config.REMOTE_HOST}."
))
if remote_config["BACKEND"] != "docker":
config.CONTAINER_PERSIST = config.CONTAINER_DEBUG = False
# which runscript to look for
runscript0 = "run-radiopadre"
# form up remote venv path, but do not expand ~ at this point (it may be a different username on the remote)
env = os.environ.copy()
env.setdefault("RADIOPADRE_DIR", config.REMOTE_RADIOPADRE_DIR
or "~/.radiopadre")
config.RADIOPADRE_VENV = (config.RADIOPADRE_VENV or "").format(**env)
# this variable used in error and info messages
remote_venv = ff("{config.REMOTE_HOST}:{config.RADIOPADRE_VENV}")
# pip install command with -v repeated for each VERBOSE increment
pip_install = "pip install " + "-v " * min(max(config.VERBOSE - 1, 0), 3)
# do we want to do an install/update -- will be forced to True (if we must install),
# or False if we can't update
do_update = config.UPDATE
if config.SKIP_CHECKS:
runscript = ff(
"if [ -f {config.RADIOPADRE_VENV}/bin/activate ]; then " +
"source {config.RADIOPADRE_VENV}/bin/activate; fi; run-radiopadre "
)
do_update = False
else:
runscript = None
# (a) if --auto-init and --venv-reinstall specified, zap remote virtualenv if present
if config.AUTO_INIT and config.VENV_REINSTALL:
if not config.RADIOPADRE_VENV:
bye(
ff("Can't do --auto-init --venv-reinstall because --radiopadre-venv is not set"
))
if "~" in config.RADIOPADRE_VENV:
config.RADIOPADRE_VENV = ssh_remote(
ff("echo {config.RADIOPADRE_VENV}")).strip(
) # expand "~" on remote
if check_remote_file(ff("{config.RADIOPADRE_VENV}"), "-d"):
if not check_remote_file(
ff("{config.RADIOPADRE_VENV}/bin/activate"), "-f"):
error(
ff("{remote_venv}/bin/activate} does not exist. Bat country!"
))
bye(
ff("Refusing to touch this virtualenv. Please remove it by hand if you must."
))
cmd = ff("rm -fr {config.RADIOPADRE_VENV}")
warning(ff("Found a virtualenv in {remote_venv}."))
warning(
"However, --venv-reinstall was specified. About to run:")
warning(ff(" ssh {config.REMOTE_HOST} " + cmd))
if config.FULL_CONSENT:
warning(
"--full-consent given, so not asking for confirmation."
)
else:
warning(ff("Your informed consent is required!"))
inp = INPUT(
ff("Please enter 'yes' to rm -fr {remote_venv}: ")
).strip()
if inp != "yes":
bye(ff("'{inp}' is not a 'yes'. Phew!"))
message("OK, nuking it!")
ssh_remote(cmd)
# force update
do_update = True
# (b) look inside venv
if runscript is None and config.RADIOPADRE_VENV:
if "~" in config.RADIOPADRE_VENV:
config.RADIOPADRE_VENV = ssh_remote(
ff("echo {config.RADIOPADRE_VENV}")).strip(
) # expand "~" on remote
if check_remote_file(ff("{config.RADIOPADRE_VENV}/bin/activate"),
"-f"):
if ssh_remote(ff(
"source {config.RADIOPADRE_VENV}/bin/activate && which {runscript0}"
),
fail_retcode=1):
runscript = ff(
"source {config.RADIOPADRE_VENV}/bin/activate && {runscript0}"
)
message(
ff("Using remote client script within {config.RADIOPADRE_VENV}"
))
else:
message(
ff("Remote virtualenv {config.RADIOPADRE_VENV} exists, but does not contain a radiopadre-client installation."
))
else:
message(ff("No remote virtualenv found at {remote_venv}"))
# (c) just try `which` directly
if runscript is None:
runscript = check_remote_command(runscript0)
if runscript:
message(ff("Using remote client script at {runscript}"))
runscript = ff()
do_update = False
if config.UPDATE:
warning(
ff("ignoring --update for client since it isn't in a virtualenv"
))
else:
message(ff("No remote client script {runscript0} found"))
runscript = None
## No runscript found on remote?
## First, figure out whether to reinstall a virtualenv for it
if not runscript:
message(ff("No {runscript0} script found on {config.REMOTE_HOST}"))
if not config.AUTO_INIT:
bye(
ff("Try re-running with --auto-init to install radiopadre-client on {config.REMOTE_HOST}."
))
if not config.RADIOPADRE_VENV:
bye(ff("Can't do --auto-init because --virtual-env is not set."))
message("Trying to --auto-init an installation for you...")
# try to auto-init a virtual environment
if not check_remote_file(ff("{config.RADIOPADRE_VENV}/bin/activate"),
"-f"):
message(ff("Creating virtualenv {remote_venv}"))
ssh_remote_v(ff("virtualenv -p python3 {config.RADIOPADRE_VENV}"))
extras = config.VENV_EXTRAS.split(
",") if config.VENV_EXTRAS else []
# add numpy explicitly to quicken up pyregion install
extras.append("numpy")
if extras:
extras = " ".join(extras)
message(ff("Installing specified extras: {extras}"))
ssh_remote_v(
ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} {extras}"
))
else:
message(ff("Installing into existing virtualenv {remote_venv}"))
# Now, figure out how to install or update the client package
if not runscript or do_update:
# installing from a specified existing path
if config.CLIENT_INSTALL_PATH and check_remote_file(
config.CLIENT_INSTALL_PATH, "-d"):
install_path = config.CLIENT_INSTALL_PATH
message(
ff("--client-install-path {install_path} is configured and exists on {config.REMOTE_HOST}."
))
# update if managed by git
if check_remote_file(ff("{install_path}/.git"),
"-d") and config.UPDATE:
if has_git:
if config.CLIENT_INSTALL_BRANCH:
cmd = ff(
"cd {install_path} && git fetch origin && git checkout {config.CLIENT_INSTALL_BRANCH} && git pull"
)
else:
cmd = ff("cd {install_path} && git pull")
warning(
ff("--update specified and git detected, will attempt to update via"
))
message(ff(" {cmd}"))
ssh_remote_v(cmd)
else:
warning(
ff("--update specified, but no git command found on {config.REMOTE_HOST}"
))
install_path = "-e " + install_path
# else, installing from git
elif config.CLIENT_INSTALL_REPO:
if config.CLIENT_INSTALL_REPO == "default":
config.CLIENT_INSTALL_REPO = config.DEFAULT_CLIENT_INSTALL_REPO
branch = config.CLIENT_INSTALL_BRANCH or "master"
if config.CLIENT_INSTALL_PATH:
message(
ff("--client-install-path and --client-install-repo configured, will attempt"
))
cmd = ff(
"git clone -b {branch} {config.CLIENT_INSTALL_REPO} {config.CLIENT_INSTALL_PATH}"
)
message(ff(" ssh {config.REMOTE_HOST} {cmd}"))
ssh_remote_v(cmd)
install_path = ff("-e {config.CLIENT_INSTALL_PATH}")
else:
message(
ff("--client-install-repo is configured, will try to install directly from git"
))
install_path = ff("git+{config.CLIENT_INSTALL_REPO}@{branch}")
# now pip install
message(
ff("Doing pip install -e {install_path} in {config.RADIOPADRE_VENV}"
))
ssh_remote_v(
ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} -e {install_path}"
))
# else, installing directly from pip
elif config.CLIENT_INSTALL_PIP:
message(
ff("--client-install-pip {config.CLIENT_INSTALL_PIP} is configured."
))
install_path = config.CLIENT_INSTALL_PIP
else:
bye("no radiopadre-client installation method specified (see --client-install options)"
)
# now install
message(
ff("Will attempt to pip install -U {install_path} in {remote_venv}"
))
ssh_remote_v(
ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} -U {install_path}"
))
# sanity check
if ssh_remote(ff(
"source {config.RADIOPADRE_VENV}/bin/activate && which {runscript0}"
),
fail_retcode=1):
runscript = ff(
"source {config.RADIOPADRE_VENV}/bin/activate && {runscript0}")
else:
bye(
ff("Something went wrong during installation on {config.REMOTE_HOST}, since I still don't see the {runscript0} script"
))
message("Success!")
runscript = ff(
"export RADIOPADRE_DIR={config.REMOTE_RADIOPADRE_DIR}; {runscript}")
# copy notebook to remote
if copy_initial_notebook:
if not os.path.exists(copy_initial_notebook):
bye("{} doesn't exist".format(copy_initial_notebook))
if check_remote_file(notebook_path or ".", "-d"):
nbpath = "{}/{}".format(notebook_path or ".",
copy_initial_notebook)
if check_remote_file(nbpath, "-ff("):
message(
ff("remote notebook {nbpath} exists, will not copy over"))
else:
message(
ff("remote notebook {nbpath} doesn't exist, will copy over"
))
scp_to_remote(copy_initial_notebook, notebook_path)
notebook_path = nbpath
# allocate 5 suggested ports (in resume mode, this will be overridden by the session settings)
starting_port = 10000 + os.getuid() * 3
ports = []
for _ in range(5):
starting_port = find_unused_port(starting_port + 1, 10000)
ports.append(starting_port)
remote_config["remote"] = ":".join(map(str, ports))
# turn the remote_config dict into a command line
runscript += " " + " ".join(
config.get_options_list(remote_config, quote=True))
runscript += " '{}' {}".format(
command if command is not "load" else notebook_path,
" ".join(extra_arguments))
# start ssh subprocess to launch notebook
args = list(SSH_OPTS) + ["shopt -s huponexit && " + runscript]
if config.VERBOSE:
message("running {}".format(" ".join(args)))
else:
message(ff("running radiopadre client on {config.REMOTE_HOST}"))
ssh = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1,
universal_newlines=True)
poller = Poller()
poller.register_process(ssh, config.REMOTE_HOST,
config.REMOTE_HOST + " stderr")
if not USE_VENV:
poller.register_file(sys.stdin, "stdin")
container_name = None
urls = []
remote_running = False
status = 0
try:
while remote_running is not None and poller.fdlabels:
fdlist = poller.poll(verbose=config.VERBOSE > 1)
for fname, fobj in fdlist:
try:
line = fobj.readline()
except EOFError:
line = b''
empty_line = not line
line = (line.decode('utf-8')
if type(line) is bytes else line).rstrip()
if fobj is sys.stdin and line == 'D' and config.CONTAINER_PERSIST:
sys.exit(0)
# break out if ssh closes
if empty_line:
poller.unregister_file(fobj)
if ssh.stdout not in poller and ssh.stderr not in poller:
message(
ff("The ssh process to {config.REMOTE_HOST} has exited"
))
remote_running = None
break
continue
# print remote output
print_output = False
if fobj is ssh.stderr:
print_output = not line.startswith("Shared connection to")
else:
print_output = not line.startswith(
"radiopadre:") or command != 'load'
if not empty_line and (config.VERBOSE or print_output):
for key, dispatch in _dispatch_message.items():
if key in line:
dispatch(u"{}: {}".format(fname, line))
break
else:
message(u"{}: {}".format(fname, line))
if not line:
continue
# if remote is not yet started, check output
if not remote_running:
# check for session ID
match = re.match(
".*Session ID/notebook token is '([0-9a-f]+)'", line)
if match:
config.SESSION_ID = match.group(1)
continue
# check for notebook port, and launch second ssh when we have it
re_ports = ":".join(["([\\d]+)"] *
10) # form up regex for ddd:ddd:...
match = re.match(ff(".*Selected ports: {re_ports}[\s]*$"),
line)
if match:
ports = list(map(int, match.groups()))
remote_ports = ports[:5]
local_ports = ports[5:]
if config.VERBOSE:
message(
"Detected ports {}:{}:{}:{}:{} -> {}:{}:{}:{}:{}"
.format(*ports))
ssh2_args = ["ssh"] + SSH_MUX_OPTS + [
"-O", "forward", config.REMOTE_HOST
]
for loc, rem in zip(local_ports, remote_ports):
ssh2_args += [
"-L",
"localhost:{}:localhost:{}".format(loc, rem)
]
# tell mux process to forward the ports
if config.VERBOSE:
message(
"sending forward request to ssh mux process".
format(ssh2_args))
subprocess.call(ssh2_args)
continue
# check for launch URL
match = re.match(".*Browse to URL: ([^\s\033]+)", line)
if match:
urls.append(match.group(1))
continue
# check for container name
match = re.match(".*Container name: ([^\s\033]+)", line)
if match:
container_name = match.group(1)
continue
if "jupyter notebook server is running" in line:
remote_running = True
time.sleep(1)
iglesia.register_helpers(*run_browser(*urls))
message(
"The remote radiopadre session is now fully up")
if USE_VENV or not config.CONTAINER_PERSIST:
message("Press Ctrl+C to kill the remote session")
else:
message(
"Press D<Enter> to detach from remote session, or Ctrl+C to kill it"
)
except SystemExit as exc:
message(ff("SystemExit: {exc.code}"))
status = exc.code
except KeyboardInterrupt:
message("Ctrl+C caught")
status = 1
except Exception as exc:
traceback.print_exc()
message("Exception caught: {}".format(str(exc)))
if remote_running and ssh.poll() is None:
message("Asking remote session to exit, nicely")
try:
try:
ssh.stdin.write("exit\n")
except TypeError:
ssh.stdin.write(b"exit\n") # because f**k you python
except IOError:
debug(" looks like it's already exited")
# if status and not USE_VENV and container_name:
# message(ff("killing remote container {container_name}"))
# try:
# if has_docker:
# ssh_remote(ff("{has_docker} kill {container_name}"))
# elif has_singularity:
# from .backends.singularity import get_singularity_image
# singularity_image = get_singularity_image(config.DOCKER_IMAGE)
# ssh_remote(ff("{has_singularity} instance.stop {singularity_image} {container_name}"))
# except subprocess.CalledProcessError as exc:
# message(exc.output.decode())
for i in range(10, 0, -1):
if ssh.poll() is not None:
debug("Remote session has exited")
ssh.wait()
break
message(ff("Waiting for remote session to exit ({i})"))
time.sleep(1)
else:
message(ff("Remote session hasn't exited, killing the ssh process"))
ssh.kill()
return status
def start_session(container_name, selected_ports, userside_ports,
notebook_path, browser_urls):
from iglesia import ABSROOTDIR, LOCAL_SESSION_DIR, SHADOW_SESSION_DIR
radiopadre_dir = make_radiopadre_dir()
docker_local = make_dir(radiopadre_dir + "/.docker-local")
js9_tmp = make_dir(radiopadre_dir + "/.js9-tmp")
session_info_dir = get_session_info_dir(container_name)
# message(ff("Container name: {container_name}")) # remote script will parse it
os.environ["RADIOPADRE_CONTAINER_NAME"] = container_name
os.environ["XDG_RUNTIME_DIR"] = ""
docker_opts = ["--workdir", ABSROOTDIR]
# setup mounts for work dir and home dir, if needed
homedir = os.path.expanduser("~")
docker_opts += [
"-B",
"{}:{}{}".format(ABSROOTDIR, ABSROOTDIR,
""), # ":ro" if orig_rootdir else ""),
"-B",
"{}:{}".format(radiopadre_dir, radiopadre_dir),
# hides /home/user/.local, which if exposed, can confuse jupyter and ipython
"-B",
"{}:{}".format(docker_local,
os.path.realpath(os.path.join(homedir, ".local"))),
# mount session info directory (needed to serve e.g. js9prefs.js)
"-B",
"{}:{}".format(session_info_dir, LOCAL_SESSION_DIR),
"-B",
"{}:{}".format(session_info_dir, SHADOW_SESSION_DIR),
# mount a writeable tmp dir for the js9 install -- needed by js9helper
"-B",
"{}:/.radiopadre/venv/js9-www/tmp".format(js9_tmp),
]
if config.CONTAINER_DEV:
if os.path.isdir(config.CLIENT_INSTALL_PATH):
docker_opts += [
"-B",
"{}:/radiopadre-client".format(config.CLIENT_INSTALL_PATH)
]
if os.path.isdir(config.SERVER_INSTALL_PATH):
docker_opts += [
"-B", "{}:/radiopadre".format(config.SERVER_INSTALL_PATH)
]
# if not config.CONTAINER_DEBUG:
# command = [singularity, "instance.start"] + docker_opts + \
# [singularity_image, container_name]
# message("running {}".format(" ".join(map(str, command))))
# subprocess.call(command)
# docker_opts = [singularity, "exec", "instance://{}".format(container_name)]
# else:
# docker_opts = [singularity, "exec" ] + docker_opts + [singularity_image]
docker_opts = [singularity, "run"] + docker_opts + [singularity_image]
container_ports = selected_ports
# build up command-line arguments
docker_opts += _collect_runscript_arguments(container_ports +
userside_ports)
if notebook_path:
docker_opts.append(notebook_path)
_run_container(container_name,
docker_opts,
jupyter_port=selected_ports[0],
browser_urls=browser_urls,
singularity=True)
if config.NBCONVERT:
return
try:
while True:
a = INPUT("Type 'exit' to kill the container session: ")
if a.lower() == 'exit':
sys.exit(0)
except BaseException as exc:
if type(exc) is KeyboardInterrupt:
message("Caught Ctrl+C")
status = 1
elif type(exc) is SystemExit:
status = getattr(exc, 'code', 0)
message("Exiting with status {}".format(status))
else:
message("Caught exception {} ({})".format(exc, type(exc)))
status = 1
if status:
message("Killing the container")
subprocess.call([
singularity, "instance.stop", singularity_image, container_name
],
stdout=DEVNULL)
sys.exit(status)
def run_radiopadre_server(command, arguments, notebook_path, workdir=None):
global backend
# message("Welcome to Radiopadre!")
USE_VENV = USE_DOCKER = USE_SINGULARITY = False
for backend in config.BACKEND:
if backend == "venv" and find_which("virtualenv"):
USE_VENV = True
import radiopadre_client.backends.venv
backend = radiopadre_client.backends.venv
backend.init()
break
elif backend == "docker":
has_docker = find_which("docker")
if has_docker:
USE_DOCKER = True
message(ff("Using {has_docker} for container mode"))
import radiopadre_client.backends.docker
backend = radiopadre_client.backends.docker
backend.init(binary=has_docker)
break
elif backend == "singularity":
has_docker = find_which("docker")
has_singularity = find_which("singularity")
if has_singularity:
USE_SINGULARITY = True
message(ff("Using {has_singularity} for container mode"))
import radiopadre_client.backends.singularity
backend = radiopadre_client.backends.singularity
backend.init(binary=has_singularity, docker_binary=has_docker)
break
message(ff("The '{backend}' back-end is not available."))
else:
bye(ff("None of the specified back-ends are available."))
# if not None, gives the six port assignments
attaching_to_ports = container_name = None
# ### ps/ls command
if command == 'ps' or command == 'ls':
session_dict = backend.list_sessions()
num = len(session_dict)
message("{} session{} running".format(num, "s" if num != 1 else ""))
for i, (id, (name, path, uptime, session_id,
ports)) in enumerate(session_dict.items()):
print("{i}: id {id}, name {name}, in {path}, up since {uptime}".
format(**locals()))
sys.exit(0)
# ### kill command
if command == 'kill':
session_dict = backend.list_sessions()
if not session_dict:
bye("no sessions running, nothing to kill")
if arguments[0] == "all":
kill_sessions = session_dict.keys()
else:
kill_sessions = [
backend.identify_session(session_dict, arg)
for arg in arguments
]
backend.kill_sessions(session_dict, kill_sessions)
sys.exit(0)
## attach command
if command == "resume":
session_dict = backend.list_sessions()
if arguments:
id_ = backend.identify_session(session_dict, arguments[0])
else:
if not session_dict:
bye("no sessions running, nothing to attach to")
config.SESSION_ID = session_dict.keys()[0]
container_name, path, _, _, attaching_to_ports = session_dict[id_]
message(
ff(" Attaching to existing session {config.SESSION_ID} running in {path}"
))
# load command
elif command == 'load':
attaching_to_ports = None
# else unknown command
else:
bye("unknown command {}".format(command))
running_session_dict = None
# ### SETUP LOCAL SESSION PROPERTIES: container_name, session_id, port assignments
# REATTACH MODE: everything is read from the session file
if attaching_to_ports:
# session_id and container_name already set above. Ports read from session file and printed to the console
# for the benefit of the remote end (if any)
jupyter_port, helper_port, http_port, carta_port, carta_ws_port = selected_ports = attaching_to_ports[:
5]
userside_ports = attaching_to_ports[5:]
# INSIDE CONTAINER: internal ports are fixed, userside ports are passed in, name is passed in, session ID is read from file
elif config.INSIDE_CONTAINER_PORTS:
message("started the radiopadre container")
container_name = os.environ['RADIOPADRE_CONTAINER_NAME']
config.SESSION_ID = os.environ['RADIOPADRE_SESSION_ID']
selected_ports = config.INSIDE_CONTAINER_PORTS[:5]
userside_ports = config.INSIDE_CONTAINER_PORTS[5:]
message(" Inside container, using ports {}".format(" ".join(
map(str, config.INSIDE_CONTAINER_PORTS))))
# NORMAL MODE: find unused internal ports. Userside ports are passed from remote if in remote mode, or same in local mode
else:
if not USE_VENV:
container_name = "radiopadre-{}-{}".format(config.USER,
uuid.uuid4().hex)
message(ff("Starting new session in container {container_name}"))
# get dict of running sessions (for GRIM_REAPER later)
running_session_dict = backend.list_sessions()
else:
container_name = None
message("Starting new session in virtual environment")
selected_ports = [find_unused_port(1024)]
for i in range(4):
selected_ports.append(find_unused_port(selected_ports[-1] + 1))
if config.REMOTE_MODE_PORTS:
userside_ports = config.REMOTE_MODE_PORTS
else:
userside_ports = selected_ports
os.environ['RADIOPADRE_SESSION_ID'] = config.SESSION_ID = uuid.uuid4(
).hex
# write out session file
if container_name:
backend.save_session_info(container_name, selected_ports,
userside_ports)
global userside_jupyter_port # needed for it to be visible to ff() from a list comprehension
jupyter_port, helper_port, http_port, carta_port, carta_ws_port = selected_ports
userside_jupyter_port, userside_helper_port, userside_http_port, userside_carta_port, userside_carta_ws_port = userside_ports
# print port assignments to console -- in remote mode, remote script will parse this out
if not config.INSIDE_CONTAINER_PORTS:
message(" Selected ports: {}".format(":".join(
map(str, selected_ports + userside_ports))))
message(ff(" Session ID/notebook token is '{config.SESSION_ID}'"))
if container_name is not None:
message(ff(" Container name: {container_name}"))
# ### will we be starting a browser?
browser = False
if config.INSIDE_CONTAINER_PORTS:
if config.VERBOSE:
message(
" Running inside container -- not opening a browser in here.")
elif config.REMOTE_MODE_PORTS:
if config.VERBOSE:
message(" Remote mode -- not opening a browser locally.")
elif os.environ.get("SSH_CLIENT"):
message("You appear to have logged in via ssh.")
message(
"You're logged in via ssh, so I'm not opening a web browser for you."
)
message(
"Please manually browse to the URL printed by Jupyter below. You will probably want to employ ssh"
)
message(
"port forwarding if you want to browse this notebook from your own machine."
)
browser = False
else:
message("You appear to have a local session.")
if not config.BROWSER:
message("--no-browser is set, we will not invoke a browser.")
message("Please manually browse to the URL printed below.")
browser = False
else:
message(
ff("We'll attempt to open a web browser (using '{config.BROWSER}') as needed. Use --no-browser to disable this."
))
browser = True
# ### ATTACHING TO EXISTING SESSION: complete the attachment and exit
if attaching_to_ports:
url = ff(
"http://localhost:{userside_jupyter_port}/tree#running?token={session_id}"
)
# in local mode, see if we need to open a browser. Else just print the URL -- remote script will pick it up
if not config.REMOTE_MODE_PORTS and browser:
message(ff("driving browser: {config.BROWSER} {url}"))
subprocess.call([config.BROWSER, url], stdout=DEVNULL)
time.sleep(1)
else:
message(ff("Browse to URL: {url}"), color="GREEN")
# emit message so remote initiates browsing
if config.REMOTE_MODE_PORTS:
message(
"The Jupyter Notebook is running inside the reattached session, presumably"
)
if config.VERBOSE:
message("sleeping")
while True:
time.sleep(1000000)
sys.exit(0)
# ### NEW SESSION: from this point on, we're opening a new session
# ### setup working directory and notebook paths
global LOAD_DIR
global LOAD_NOTEBOOK
# if explicit notebook directory is given, change into it before doing anything else
if notebook_path:
if os.path.isdir(notebook_path):
os.chdir(notebook_path)
notebook_path = '.'
LOAD_DIR = True
LOAD_NOTEBOOK = None
else:
nbdir = os.path.dirname(notebook_path)
if nbdir:
if not os.path.isdir(nbdir):
bye("{} doesn't exist".format(nbdir))
os.chdir(nbdir)
notebook_path = os.path.basename(notebook_path)
LOAD_DIR = False
LOAD_NOTEBOOK = notebook_path
else:
LOAD_DIR = '.'
LOAD_NOTEBOOK = None
# message(ff("{LOAD_DIR} {LOAD_NOTEBOOK} {notebook_path}"))
#
if config.NBCONVERT and not LOAD_NOTEBOOK:
bye("a notebook must be specified in order to use --nbconvert")
# if using containers (and not inside a container), see if older sessions need to be reaped
if running_session_dict and config.GRIM_REAPER:
kill_sessions = []
for cont, (_, path, _, sid, _) in running_session_dict.items():
if sid != config.SESSION_ID and os.path.samefile(
path, os.getcwd()):
message(ff("reaping older session {sid}"))
kill_sessions.append(cont)
if kill_sessions:
backend.kill_sessions(running_session_dict,
kill_sessions,
ignore_fail=True)
# virtual environment
os.environ["RADIOPADRE_VENV"] = config.RADIOPADRE_VENV
# init paths & environment
iglesia.init()
iglesia.set_userside_ports(userside_ports)
global JUPYTER_OPTS
if config.NBCONVERT:
JUPYTER_OPTS = [
"nbconvert", "--ExecutePreprocessor.timeout=600", "--no-input",
"--to", "html_embed", "--execute"
]
os.environ["RADIOPADRE_NBCONVERT"] = "True"
else:
JUPYTER_OPTS = [
"notebook",
"--ContentsManager.pre_save_hook=radiopadre_utils.notebook_utils._notebook_save_hook",
"--ContentsManager.allow_hidden=True"
]
os.environ.pop("RADIOPADRE_NBCONVERT", None)
# update installation etc.
backend.update_installation()
# (when running natively (i.e. in a virtual environment), the notebook app doesn't pass the token to the browser
# command properly... so let it pick its own token then)
# if options.remote or options.config.INSIDE_CONTAINER_PORTS or not options.virtual_env:
JUPYTER_OPTS += [
ff("--NotebookApp.token='{config.SESSION_ID}'"),
ff("--NotebookApp.custom_display_url='http://localhost:{userside_jupyter_port}'"
)
]
#=== figure out whether we initialize or load a notebook
os.chdir(iglesia.SERVER_BASEDIR)
if iglesia.SNOOP_MODE:
warning(
ff("{iglesia.ABSROOTDIR} is not writable for you, so radiopadre is operating in snoop mode."
))
ALL_NOTEBOOKS = glob.glob("*.ipynb")
if iglesia.SNOOP_MODE and not ALL_NOTEBOOKS:
orig_notebooks = glob.glob(os.path.join(iglesia.ABSROOTDIR, "*.ipynb"))
if orig_notebooks:
message(
" No notebooks in shadow directory: will copy notebooks from target."
)
message(" Copying {} notebooks from {}".format(
len(orig_notebooks), iglesia.ABSROOTDIR))
for nb in orig_notebooks:
shutil.copyfile(nb, './' + os.path.basename(nb))
ALL_NOTEBOOKS = glob.glob("*.ipynb")
message(" Available notebooks: " + " ".join(ALL_NOTEBOOKS))
if not config.INSIDE_CONTAINER_PORTS:
# if no notebooks in place, see if we need to create a default
if not ALL_NOTEBOOKS:
if config.DEFAULT_NOTEBOOK:
message(
ff(" No notebooks yet: will create {config.DEFAULT_NOTEBOOK}"
))
LOAD_DIR = True
open(config.DEFAULT_NOTEBOOK,
'wt').write(default_notebook_code)
ALL_NOTEBOOKS = [config.DEFAULT_NOTEBOOK]
else:
message(
" No notebooks and no default. Displaying directory only."
)
LOAD_DIR = True
LOAD_NOTEBOOK = None
# expand globs and apply auto-load as needed
if LOAD_NOTEBOOK:
LOAD_NOTEBOOK = [
nb for nb in ALL_NOTEBOOKS
if fnmatch.fnmatch(os.path.basename(nb), LOAD_NOTEBOOK)
]
elif config.AUTO_LOAD == "1":
LOAD_NOTEBOOK = ALL_NOTEBOOKS[0] if ALL_NOTEBOOKS else None
message(ff(" Auto-loading {LOAD_NOTEBOOK[0]}."))
elif config.AUTO_LOAD:
LOAD_NOTEBOOK = [
nb for nb in ALL_NOTEBOOKS
if fnmatch.fnmatch(os.path.basename(nb), config.AUTO_LOAD)
]
if LOAD_NOTEBOOK:
message(" Auto-loading {}".format(" ".join(LOAD_NOTEBOOK)))
else:
message(
ff(" No notebooks matching --auto-load {config.AUTO_LOAD}"
))
urls = []
if LOAD_DIR:
urls.append(
ff("http://localhost:{userside_jupyter_port}/?token={config.SESSION_ID}"
))
if LOAD_NOTEBOOK:
urls += [
ff("http://localhost:{userside_jupyter_port}/notebooks/{nb}?token={config.SESSION_ID}"
) for nb in LOAD_NOTEBOOK
]
if not config.NBCONVERT:
for url in urls[::-1]:
message(ff("Browse to URL: {url}"), color="GREEN")
if config.CARTA_BROWSER:
url = ff(
"http://localhost:{iglesia.CARTA_PORT}/?socketUrl=ws://localhost:{iglesia.CARTA_WS_PORT}"
)
message(ff("Browse to URL: {url} (CARTA file browser)"),
color="GREEN")
urls.append(url)
# now we're ready to start the session
backend.start_session(container_name, selected_ports, userside_ports,
notebook_path, browser and urls)
def update_installation(rebuild=False, docker_pull=True):
global docker_image
global singularity_image
docker_image = config.DOCKER_IMAGE
singularity_image = os.path.expanduser(get_singularity_image(docker_image))
# this will be True if we need to build the image
build_image = False
# clearly true if no image
if not os.path.exists(singularity_image):
if config.SINGULARITY_AUTO_BUILD:
message(ff("Singularity image {singularity_image} does not exist"))
build_image = True
else:
error(
ff("Singularity image {singularity_image} does not exist, and auto-build is disabled"
))
bye(ff("Re-run with --singularity-auto-build to proceed"))
# also true if rebuild forced by flags or config or command line
elif rebuild or config.SINGULARITY_REBUILD:
config.SINGULARITY_AUTO_BUILD = build_image = True
message(
ff("--singularity-rebuild specified, removing singularity image {singularity_image}"
))
# pull down docker image first
if has_docker and docker_pull:
message(
"Checking docker image (from which our singularity image is built)"
)
docker.update_installation(enable_pull=True)
# if we're not forced to build yet, check for an update
if config.UPDATE and not build_image:
if has_docker:
# check timestamp of docker image
docker_image_time = None
output = check_output(
ff("{has_docker} image inspect {docker_image} -f '{{{{ .Created }}}}'"
)).strip()
message(ff(" docker image timestamp is {output}"))
# in Python 3.7 we have datetime.fromisoformat(date_string), but for now we muddle:
match = output and re.match("(^.*)[.](\d+)Z", output)
if match:
try:
docker_image_time = calendar.timegm(
time.strptime(match.group(1), "%Y-%m-%dT%H:%M:%S"))
docker_image_time = datetime.datetime.utcfromtimestamp(
docker_image_time)
except ValueError:
pass
sing_image_time = datetime.datetime.utcfromtimestamp(
os.path.getmtime(singularity_image))
message(" singularity image timestamp is {}".format(
sing_image_time.isoformat()))
if docker_image_time is None:
warning(
ff("can't parse docker image timestamp '{output}', rebuilding {singularity_image} just in case"
))
build_image = True
elif docker_image_time > sing_image_time:
warning(
ff("rebuilding outdated singularity image {singularity_image}"
))
build_image = True
else:
message(
ff("singularity image {singularity_image} is up-to-date"))
else:
message(
ff("--update specified but no docker access, assuming {singularity_image} is up-to-date"
))
# now build if needed
if build_image:
warning(
ff("Rebuilding singularity image from docker://{docker_image}"))
warning(ff(" (This may take a few minutes....)"))
singularity_image_new = singularity_image + ".new.img"
if os.path.exists(singularity_image_new):
os.unlink(singularity_image_new)
cmd = [
singularity, "build", singularity_image_new,
ff("docker://{docker_image}")
]
message("running " + " ".join(cmd))
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as exc:
if config.IGNORE_UPDATE_ERRORS:
if os.path.exists(singularity_image):
warning(
"singularity build failed but --ignore-update-errors is set, proceeding with old image"
)
else:
error(
"singularity build failed, --ignore-update-errors is set, but we have no older image"
)
raise
else:
raise
# move old image
message(ff("Build successful, renaming to {singularity_image}"))
os.rename(singularity_image_new, singularity_image)
else:
message(
ff("Using existing radiopadre singularity image {singularity_image}"
))
# not supported with Singularity
config.CONTAINER_PERSIST = False
def preinit_js9():
"""Pre-initialization, when Javascript is not available yet. Determines paths and starts helper processs"""
global radiopadre_kernel
import radiopadre_kernel
import iglesia
global JS9_HELPER_PORT, JS9_DIR
JS9_DIR = iglesia.JS9_DIR
JS9_HELPER_PORT = iglesia.JS9HELPER_PORT
try:
global JS9_ERROR
if not os.path.exists(JS9_DIR):
raise JS9Error(f"{JS9_DIR} does not exist")
message(f"Using JS9 install in {JS9_DIR}")
global RADIOPADRE_INSTALL_PREFIX
global RADIOPADRE_LOCAL_PREFIX
global JS9_INSTALL_PREFIX
global JS9_INIT_HTML_STATIC
global JS9_INIT_HTML_DYNAMIC
global JS9_SCRIPT_PREFIX
global JS9_LOCAL_SETTINGS
RADIOPADRE_INSTALL_PREFIX = f"{radiopadre_kernel.SHADOW_URL_PREFIX}/radiopadre-www" # URL used to access radiopadre code
RADIOPADRE_LOCAL_PREFIX = f"{radiopadre_kernel.SHADOW_URL_PREFIX}/{radiopadre_kernel.ABSROOTDIR}/.radiopadre" # URL used to access radiopadre aux dir
JS9_INSTALL_PREFIX = f"{radiopadre_kernel.SHADOW_URL_PREFIX}/js9-www" # URL used to access JS9 code
JS9_SCRIPT_PREFIX = radiopadre_kernel.SHADOW_URL_PREFIX
JS9_LOCAL_SETTINGS = f"{radiopadre_kernel.SESSION_URL}/js9prefs.js"
# load templated init HTML
try:
with open(os.path.join(DIRNAME, "js9-init-static-template.html"), "rt") as inp:
JS9_INIT_HTML_STATIC = inp.read().format(**globals())
with open(os.path.join(DIRNAME, "js9-init-dynamic-template.html"), "rt") as inp:
JS9_INIT_HTML_DYNAMIC = inp.read().format(**globals())
except Exception as exc:
traceback.print_exc()
JS9_ERROR = "Error reading init templates: {}".format(str(exc))
except JS9Error as exc:
if exc.message:
JS9_ERROR = exc.message
# on error, init code replaced by error message
if JS9_ERROR:
error(f"JS9 init error: {JS9_ERROR}")
# def init_js9():
# """Final initialization, when Javascript can be injected"""
# from IPython.display import Javascript, display
# display(Javascript("""
# <link type='image/x-icon' rel='shortcut icon' href='/static/js9-www/favicon.ico'>
# <link type='text/css' rel='stylesheet' href='/static/js9-www/js9support.css'>
# <link type='text/css' rel='stylesheet' href='/static/js9-www/js9.css'>
# <link rel='apple-touch-icon' href='/static/js9-www/images/js9-apple-touch-icon.png'>
# <script type='text/javascript' src='/static/js9-www/js9prefs.js'></script>
# <script type='text/javascript'> console.log('loaded JS9 prefs 1') </script>
# <script type='text/javascript' src='/files/.radiopadre-session/js9prefs.js'></script>
# <script type='text/javascript'> console.log('loaded JS9 prefs 2')</script>
# <script type='text/javascript' src='/static/js9-www/js9support.min.js'></script>
# <script type='text/javascript' src='/static/js9-www/js9.min.js'></script>
# <script type='text/javascript' src='/static/js9-www/js9plugins.js'></script>
# <script type='text/javascript'> console.log('loaded JS9 components') </script>
# <script type='text/javascript' src='/static/radiopadre-www/js9partners.js'></script>
# <script type='text/javascript'> console.log('loaded JS9 partner plugin') </script>
# <script type='text/javascript' src='/static/js9colormaps.js'></script>\
# """),)
def run_browser(*urls):
"""
Runs a browser pointed to URL(s), in background if config.BROWSER_BG is True.
If config.BROWSER_MULTI is set, runs a browser per URL, else feeds all URLs to one browser invocation
Returns list of processes (in BROWSER_BG mode).
"""
from . import config
procs = []
# open browser if needed
if config.BROWSER:
message("Running {} {}\r".format(config.BROWSER, " ".join(urls)))
message(
" if this fails, specify a correct browser invocation command with --browser and rerun,"
)
message(
" or else browse to the URL given above (\"Browse to URL:\") yourself."
)
# sometimes the notebook does not respond immediately, so take a second
time.sleep(1)
if config.BROWSER_MULTI:
commands = [[config.BROWSER] + list(urls)]
else:
commands = [[config.BROWSER, url] for url in urls]
for command in commands:
try:
if config.BROWSER_BG:
procs.append(
subprocess.Popen(command,
stdin=DEVZERO,
stdout=sys.stdout,
stderr=sys.stderr))
else:
subprocess.call(command, stdout=DEVNULL)
except OSError as exc:
if exc.errno == 2:
message(ff("{config.BROWSER} not found"))
else:
raise
else:
message(
"--no-browser given, or browser not set, not opening a browser for you\r"
)
message("Please browse to: {}\n".format(" ".join(urls)))
return procs
def check_recent_sessions(options, argv, parser=None):
"""
Loads a recent session if requested
:param options: Options object from ArgumentParser
:param argv: Argument list (from sys.argv[1:] initially)
:param parser: ArgumentParser object used to (re)parse the options
:return: options, argv
Where options and argv may have been loaded from the recent
options file
"""
make_radiopadre_dir()
# load history
try:
readline.read_history_file(HISTORY_FILE)
except IOError:
pass
resume_session = None
# a single-digit argument resumes session #N
if len(options.arguments) == 1 and re.match("^\d$", options.arguments[0]):
resume_session = int(options.arguments[0])
# no arguments is resume session #0
elif not options.arguments:
resume_session = 0
if resume_session is not None:
last = _load_recent_sessions()
num_recent = len(last)
if resume_session >= num_recent:
bye(ff("no recent session #{resume_session}"))
message("Your most recent radiopadre sessions are:")
message("")
for i, (_, opts) in enumerate(list(last.items())[::-1]):
message(" [#{0}] {1}".format(i, opts), color="GREEN")
message("")
print(
"\nInteractive startup mode. Edit arguments and press Enter to run, or Ctrl+C to bail out. "
)
print(
" (Ctrl+U + <NUM> + Enter will paste other recent session arguments from the list above)\n"
)
inp = None
cmdline = ''
readline.set_startup_hook(lambda: readline.insert_text(cmdline))
while inp is None:
# form up list of fake args to be re-parsed for the last session
cmdline = list(last.items())[-(resume_session + 1)][1]
# non-persisting options raised in command line shall be appended to the fake args
for opt in config.NON_PERSISTING_OPTIONS:
if opt.startswith("--") and getattr(options, opt[2:].replace(
"-", "_"), None):
cmdline += " " + opt
cmdline += " "
## colors confuse Ctrl+U and such
# prompt = ff("{logger.Colors.GREEN}[#{resume_session}]:{logger.Colors.ENDC} ")
prompt = ff("[#{resume_session}] ")
inp = INPUT(prompt)
inp = inp.strip()
if not inp:
resume_session = 0
inp = None
elif re.match("^\d+$", inp):
res = int(inp)
if res >= num_recent:
warning(ff("no recent session #{res}"))
else:
resume_session = res
readline.remove_history_item(1)
inp = None
readline.set_startup_hook(None)
global _last_input
_last_input = inp
argv = shlex.split(inp, posix=False)
options = parser.parse_args(argv)
return options, argv
CACHE_URL_BASE = CACHE_URL_ROOT[:-len(subdir
)] if subdir else CACHE_URL_ROOT
# when running nbconvert, it doesn't know about the magic "/files" URL, and just needs a local filename
global NBCONVERT
NBCONVERT = bool(os.environ.get("RADIOPADRE_NBCONVERT"))
files_prefix = "." if NBCONVERT else "/files"
if SNOOP_MODE:
FILE_URL_ROOT = f"{files_prefix}{subdir}/.radiopadre.content/"
NOTEBOOK_URL_ROOT = f"/notebooks{subdir}/.radiopadre.content/"
else:
FILE_URL_ROOT = f"{files_prefix}{subdir}/"
NOTEBOOK_URL_ROOT = f"/notebooks{subdir}/"
# init JS9 sources
from . import js9
js9.preinit_js9()
if ROOTDIR is None:
from iglesia import logger
# enable logging
log = logger.init("radiopadre.kernel") #, use_formatter=False)
log.setLevel(logging.DEBUG)
log.addHandler(log_handler)
LOGFILE = logger.enable_logfile("kernel")
logger.disable_printing()
message("initializing radiopadre_kernel")
init()
def start_session(container_name, selected_ports, userside_ports,
notebook_path, browser_urls):
from iglesia import ABSROOTDIR, LOCAL_SESSION_DIR, SHADOW_SESSION_DIR, SNOOP_MODE
radiopadre_dir = make_radiopadre_dir()
docker_local = make_dir(radiopadre_dir + "/.docker-local")
js9_tmp = make_dir(radiopadre_dir + "/.js9-tmp")
session_info_dir = get_session_info_dir(container_name)
message(
ff("Container name: {container_name}")) # remote script will parse it
docker_opts = [
docker,
"run",
"--rm",
"--name",
container_name,
"--cap-add=SYS_ADMIN",
"-w",
ABSROOTDIR,
"--user",
"{}:{}".format(os.getuid(), os.getgid()),
"-e",
"USER={}".format(os.environ["USER"]),
"-e",
"HOME={}".format(os.environ["HOME"]),
"-e",
"RADIOPADRE_DIR={}".format(radiopadre_dir),
"-e",
ff("RADIOPADRE_CONTAINER_NAME={container_name}"),
"-e",
ff("RADIOPADRE_SESSION_ID={config.SESSION_ID}"),
]
# enable detached mode if not debugging, and also if not doing conversion non-interactively
if not config.CONTAINER_DEBUG and not config.NBCONVERT:
docker_opts.append("-d")
for port1, port2 in zip(selected_ports, CONTAINER_PORTS):
docker_opts += ["-p", "{}:{}/tcp".format(port1, port2)]
container_ports = list(CONTAINER_PORTS)
# setup mounts for work dir and home dir, if needed
homedir = os.path.expanduser("~")
docker_opts += [
"-v",
"{}:{}{}".format(ABSROOTDIR, ABSROOTDIR, ":ro" if SNOOP_MODE else ""),
"-v",
"{}:{}".format(homedir, homedir),
"-v",
"{}:{}".format(radiopadre_dir, radiopadre_dir),
## hides /home/user/.local, which can confuse jupyter and ipython
## into seeing e.g. kernelspecs that they should not see
"-v",
"{}:{}/.local".format(docker_local, homedir),
# mount session info directory (needed to serve e.g. js9prefs.js)
"-v",
"{}:{}".format(session_info_dir, LOCAL_SESSION_DIR),
"-v",
"{}:{}".format(session_info_dir, SHADOW_SESSION_DIR),
# mount a writeable tmp dir for the js9 install -- needed by js9helper
"-v",
"{}:/.radiopadre/venv/js9-www/tmp".format(js9_tmp),
"--label",
"radiopadre.user={}".format(USER),
"--label",
"radiopadre.dir={}".format(os.getcwd()),
]
if config.CONTAINER_DEV:
if os.path.isdir(SERVER_INSTALL_PATH):
docker_opts += ["-v", "{}:/radiopadre".format(SERVER_INSTALL_PATH)]
if os.path.isdir(CLIENT_INSTALL_PATH):
docker_opts += [
"-v", "{}:/radiopadre-client".format(CLIENT_INSTALL_PATH)
]
# add image
docker_opts.append(docker_image)
# build up command-line arguments
docker_opts += _collect_runscript_arguments(container_ports +
userside_ports)
if notebook_path:
docker_opts.append(notebook_path)
_run_container(container_name,
docker_opts,
jupyter_port=selected_ports[0],
browser_urls=browser_urls)
if config.NBCONVERT:
return
global running_container
running_container = container_name
atexit.register(reap_running_container)
if config.CONTAINER_PERSIST and config.CONTAINER_DETACH:
message("exiting: container session will remain running.")
running_container = None # to avoid reaping
sys.exit(0)
else:
if config.CONTAINER_PERSIST:
prompt = "Type 'exit' to kill the container session, or 'D' to detach: "
else:
prompt = "Type 'exit' to kill the container session: "
try:
while True:
a = INPUT(prompt)
if a.lower() == 'exit':
sys.exit(0)
if a.upper(
) == 'D' and config.CONTAINER_PERSIST and container_name:
running_container = None # to avoid reaping
sys.exit(0)
except BaseException as exc:
if type(exc) is KeyboardInterrupt:
message("Caught Ctrl+C")
status = 1
elif type(exc) is SystemExit:
status = getattr(exc, 'code', 0)
message("Exiting with status {}".format(status))
else:
message("Caught exception {} ({})".format(exc, type(exc)))
status = 1
# if not status:
# running_container = None # to avoid reaping
sys.exit(status)
