def save_recent_session(session_key, argv):
"""
Saves session arguments into recents file.
:param session_key: key to save under (only one session per key is saved)
:param argv: argument list to save
:return: None
"""
# add current line to history, if not already there
cmdline = " ".join(
[x if x and not ' ' in x else "'{}'".format(x) for x in argv])
if not _last_input:
if cmdline != readline.get_history_item(
readline.get_current_history_length()):
readline.add_history(cmdline)
make_radiopadre_dir()
try:
readline.write_history_file(HISTORY_FILE)
except IOError:
traceback.print_exc()
warning("Error writing history file (see above). Proceeding anyway.")
readline.clear_history()
# reform command-line without persisting options
cmdline = " ".join([
x if x and not ' ' in x else "'{}'".format(x) for x in argv
if x not in config.NON_PERSISTING_OPTIONS
])
recents = _load_recent_sessions(False) or OrderedDict()
session_key = ":".join(map(str, session_key))
if session_key in recents:
del recents[session_key]
if len(recents) >= 5:
del recents[list(recents.keys())[0]]
recents[session_key] = cmdline
make_radiopadre_dir()
with open(RECENTS_FILE, 'wt') as rf:
for key, cmdline in recents.items():
rf.write("{}:::{}\n".format(key, cmdline))
global _recent_sessions
_recent_sessions = recents
def _get_config_value(section, key):
globalval = globals().get(key.upper())
value = section[key]
if globalval is None or isinstance(globalval, six.string_types):
return value
elif type(globalval) is bool:
if value.lower() in {'no', '0', 'false'}:
return False
elif value.lower() in {'yes', '1', 'true'}:
return True
else:
warning("unrecognized setting {} = {} in config, assuming False".
format(key, value))
return False
elif type(globalval) is int:
return int(value)
else:
raise TypeError("unsupported type {} for option {}".format(
type(globalval), key))
def update_installation(enable_pull=False):
global docker_image
enable_pull = enable_pull or config.AUTO_INIT or config.UPDATE
if config.CONTAINER_DEV:
update_server_from_repository()
docker_image = config.DOCKER_IMAGE
if check_output(ff("docker image inspect {docker_image}")) is None:
if not enable_pull:
bye(
ff(" Radiopadre docker image {docker_image} not found. Re-run with --update or --auto-init perhaps?"
))
message(
ff(" Radiopadre docker image {docker_image} not found locally"))
else:
message(ff(" Using radiopadre docker image {docker_image}"))
if enable_pull:
warning(ff("Calling docker pull {docker_image}"))
warning(
" (This may take a few minutes if the image is not up to date...)"
)
try:
subprocess.call([docker, "pull", docker_image])
except subprocess.CalledProcessError as exc:
if config.IGNORE_UPDATE_ERRORS:
warning(
"docker pull failed, but --ignore-update-errors is set, proceeding anyway"
)
return
raise exc
def init():
"""Initializes radiopadre kernel"""
iglesia.init()
global FILE_URL_ROOT, NOTEBOOK_URL_ROOT, CACHE_URL_BASE, CACHE_URL_ROOT, \
SHADOW_URL_PREFIX
global \
ABSROOTDIR, ROOTDIR, DISPLAY_ROOTDIR, SHADOW_HOME, SERVER_BASEDIR, SHADOW_BASEDIR, \
SHADOW_ROOTDIR, SESSION_DIR, SESSION_URL, SESSION_ID, \
VERBOSE, HOSTNAME, SNOOP_MODE
from iglesia import \
ABSROOTDIR, ROOTDIR, DISPLAY_ROOTDIR, SHADOW_HOME, SERVER_BASEDIR, SHADOW_BASEDIR, \
SHADOW_ROOTDIR, SESSION_DIR, SESSION_URL, SESSION_ID, \
VERBOSE, HOSTNAME, SNOOP_MODE
# setup for snoop mode. Browsing /home/other/path/to,
if SNOOP_MODE:
# for a Jupyter basedir of ~/.radiopadre/home/other/path, this becomes /home/other/path
unshadowed_server_base = SERVER_BASEDIR[len(SHADOW_HOME):]
# Otherwise it'd better have been /home/other/path/to to begin with!
if not _is_subdir(ABSROOTDIR, unshadowed_server_base):
error(
f"""The requested directory {ABSROOTDIR} is not under {unshadowed_server_base}.
This is probably a bug! """)
# Since Jupyter is running under ~/.radiopadre/home/other/path, we can serve other's files from
# /home/other/path/to as /files/to/.content
subdir = SHADOW_ROOTDIR[
len(SERVER_BASEDIR
):] # this becomes "/to" (or "" if paths are the same)
# but do make sure that the .content symlink is in place!
_make_symlink(ABSROOTDIR, SHADOW_ROOTDIR + "/.radiopadre.content")
# else running in native mode
else:
if not _is_subdir(ABSROOTDIR, SERVER_BASEDIR):
warning(
f"""The requested directory {ABSROOTDIR} is not under {SERVER_BASEDIR}.
This is probably a bug! """)
# for a server dir of /home/user/path, and an ABSROOTDIR of /home/oms/path/to, get the subdir
subdir = ABSROOTDIR[
len(SERVER_BASEDIR
):] # this becomes "/to" (or "" if paths are the same)
os.chdir(ABSROOTDIR)
ROOTDIR = '.'
## check casacore availability
global casacore_tables
try:
import casacore.tables as casacore_tables
except Exception as exc:
casacore_tables = None
warning(
"casacore.tables failed to import. Table browsing functionality will not be available."
)
radiopadre_base = os.path.dirname(os.path.dirname(__file__))
# # pre-init JS9 stuff and run JS9 helper
# js9.preinit_js9(in_container, helper_port, userside_helper_port, http_rewrites)
iglesia.init_helpers(radiopadre_base)
# now a port is available (set up in init_helpers()), form up URLs
SHADOW_URL_PREFIX = f"http://localhost:{iglesia.HTTPSERVER_PORT}/{SESSION_ID}"
CACHE_URL_ROOT = SHADOW_URL_PREFIX + ABSROOTDIR
CACHE_URL_BASE = CACHE_URL_ROOT[:-len(subdir
)] if subdir else CACHE_URL_ROOT
# when running nbconvert, it doesn't know about the magic "/files" URL, and just needs a local filename
global NBCONVERT
NBCONVERT = bool(os.environ.get("RADIOPADRE_NBCONVERT"))
files_prefix = "." if NBCONVERT else "/files"
if SNOOP_MODE:
FILE_URL_ROOT = f"{files_prefix}{subdir}/.radiopadre.content/"
NOTEBOOK_URL_ROOT = f"/notebooks{subdir}/.radiopadre.content/"
else:
FILE_URL_ROOT = f"{files_prefix}{subdir}/"
NOTEBOOK_URL_ROOT = f"/notebooks{subdir}/"
# init JS9 sources
from . import js9
js9.preinit_js9()
def run_radiopadre_server(command, arguments, notebook_path, workdir=None):
global backend
# message("Welcome to Radiopadre!")
USE_VENV = USE_DOCKER = USE_SINGULARITY = False
for backend in config.BACKEND:
if backend == "venv" and find_which("virtualenv"):
USE_VENV = True
import radiopadre_client.backends.venv
backend = radiopadre_client.backends.venv
backend.init()
break
elif backend == "docker":
has_docker = find_which("docker")
if has_docker:
USE_DOCKER = True
message(ff("Using {has_docker} for container mode"))
import radiopadre_client.backends.docker
backend = radiopadre_client.backends.docker
backend.init(binary=has_docker)
break
elif backend == "singularity":
has_docker = find_which("docker")
has_singularity = find_which("singularity")
if has_singularity:
USE_SINGULARITY = True
message(ff("Using {has_singularity} for container mode"))
import radiopadre_client.backends.singularity
backend = radiopadre_client.backends.singularity
backend.init(binary=has_singularity, docker_binary=has_docker)
break
message(ff("The '{backend}' back-end is not available."))
else:
bye(ff("None of the specified back-ends are available."))
# if not None, gives the six port assignments
attaching_to_ports = container_name = None
# ### ps/ls command
if command == 'ps' or command == 'ls':
session_dict = backend.list_sessions()
num = len(session_dict)
message("{} session{} running".format(num, "s" if num != 1 else ""))
for i, (id, (name, path, uptime, session_id,
ports)) in enumerate(session_dict.items()):
print("{i}: id {id}, name {name}, in {path}, up since {uptime}".
format(**locals()))
sys.exit(0)
# ### kill command
if command == 'kill':
session_dict = backend.list_sessions()
if not session_dict:
bye("no sessions running, nothing to kill")
if arguments[0] == "all":
kill_sessions = session_dict.keys()
else:
kill_sessions = [
backend.identify_session(session_dict, arg)
for arg in arguments
]
backend.kill_sessions(session_dict, kill_sessions)
sys.exit(0)
## attach command
if command == "resume":
session_dict = backend.list_sessions()
if arguments:
id_ = backend.identify_session(session_dict, arguments[0])
else:
if not session_dict:
bye("no sessions running, nothing to attach to")
config.SESSION_ID = session_dict.keys()[0]
container_name, path, _, _, attaching_to_ports = session_dict[id_]
message(
ff(" Attaching to existing session {config.SESSION_ID} running in {path}"
))
# load command
elif command == 'load':
attaching_to_ports = None
# else unknown command
else:
bye("unknown command {}".format(command))
running_session_dict = None
# ### SETUP LOCAL SESSION PROPERTIES: container_name, session_id, port assignments
# REATTACH MODE: everything is read from the session file
if attaching_to_ports:
# session_id and container_name already set above. Ports read from session file and printed to the console
# for the benefit of the remote end (if any)
jupyter_port, helper_port, http_port, carta_port, carta_ws_port = selected_ports = attaching_to_ports[:
5]
userside_ports = attaching_to_ports[5:]
# INSIDE CONTAINER: internal ports are fixed, userside ports are passed in, name is passed in, session ID is read from file
elif config.INSIDE_CONTAINER_PORTS:
message("started the radiopadre container")
container_name = os.environ['RADIOPADRE_CONTAINER_NAME']
config.SESSION_ID = os.environ['RADIOPADRE_SESSION_ID']
selected_ports = config.INSIDE_CONTAINER_PORTS[:5]
userside_ports = config.INSIDE_CONTAINER_PORTS[5:]
message(" Inside container, using ports {}".format(" ".join(
map(str, config.INSIDE_CONTAINER_PORTS))))
# NORMAL MODE: find unused internal ports. Userside ports are passed from remote if in remote mode, or same in local mode
else:
if not USE_VENV:
container_name = "radiopadre-{}-{}".format(config.USER,
uuid.uuid4().hex)
message(ff("Starting new session in container {container_name}"))
# get dict of running sessions (for GRIM_REAPER later)
running_session_dict = backend.list_sessions()
else:
container_name = None
message("Starting new session in virtual environment")
selected_ports = [find_unused_port(1024)]
for i in range(4):
selected_ports.append(find_unused_port(selected_ports[-1] + 1))
if config.REMOTE_MODE_PORTS:
userside_ports = config.REMOTE_MODE_PORTS
else:
userside_ports = selected_ports
os.environ['RADIOPADRE_SESSION_ID'] = config.SESSION_ID = uuid.uuid4(
).hex
# write out session file
if container_name:
backend.save_session_info(container_name, selected_ports,
userside_ports)
global userside_jupyter_port # needed for it to be visible to ff() from a list comprehension
jupyter_port, helper_port, http_port, carta_port, carta_ws_port = selected_ports
userside_jupyter_port, userside_helper_port, userside_http_port, userside_carta_port, userside_carta_ws_port = userside_ports
# print port assignments to console -- in remote mode, remote script will parse this out
if not config.INSIDE_CONTAINER_PORTS:
message(" Selected ports: {}".format(":".join(
map(str, selected_ports + userside_ports))))
message(ff(" Session ID/notebook token is '{config.SESSION_ID}'"))
if container_name is not None:
message(ff(" Container name: {container_name}"))
# ### will we be starting a browser?
browser = False
if config.INSIDE_CONTAINER_PORTS:
if config.VERBOSE:
message(
" Running inside container -- not opening a browser in here.")
elif config.REMOTE_MODE_PORTS:
if config.VERBOSE:
message(" Remote mode -- not opening a browser locally.")
elif os.environ.get("SSH_CLIENT"):
message("You appear to have logged in via ssh.")
message(
"You're logged in via ssh, so I'm not opening a web browser for you."
)
message(
"Please manually browse to the URL printed by Jupyter below. You will probably want to employ ssh"
)
message(
"port forwarding if you want to browse this notebook from your own machine."
)
browser = False
else:
message("You appear to have a local session.")
if not config.BROWSER:
message("--no-browser is set, we will not invoke a browser.")
message("Please manually browse to the URL printed below.")
browser = False
else:
message(
ff("We'll attempt to open a web browser (using '{config.BROWSER}') as needed. Use --no-browser to disable this."
))
browser = True
# ### ATTACHING TO EXISTING SESSION: complete the attachment and exit
if attaching_to_ports:
url = ff(
"http://localhost:{userside_jupyter_port}/tree#running?token={session_id}"
)
# in local mode, see if we need to open a browser. Else just print the URL -- remote script will pick it up
if not config.REMOTE_MODE_PORTS and browser:
message(ff("driving browser: {config.BROWSER} {url}"))
subprocess.call([config.BROWSER, url], stdout=DEVNULL)
time.sleep(1)
else:
message(ff("Browse to URL: {url}"), color="GREEN")
# emit message so remote initiates browsing
if config.REMOTE_MODE_PORTS:
message(
"The Jupyter Notebook is running inside the reattached session, presumably"
)
if config.VERBOSE:
message("sleeping")
while True:
time.sleep(1000000)
sys.exit(0)
# ### NEW SESSION: from this point on, we're opening a new session
# ### setup working directory and notebook paths
global LOAD_DIR
global LOAD_NOTEBOOK
# if explicit notebook directory is given, change into it before doing anything else
if notebook_path:
if os.path.isdir(notebook_path):
os.chdir(notebook_path)
notebook_path = '.'
LOAD_DIR = True
LOAD_NOTEBOOK = None
else:
nbdir = os.path.dirname(notebook_path)
if nbdir:
if not os.path.isdir(nbdir):
bye("{} doesn't exist".format(nbdir))
os.chdir(nbdir)
notebook_path = os.path.basename(notebook_path)
LOAD_DIR = False
LOAD_NOTEBOOK = notebook_path
else:
LOAD_DIR = '.'
LOAD_NOTEBOOK = None
# message(ff("{LOAD_DIR} {LOAD_NOTEBOOK} {notebook_path}"))
#
if config.NBCONVERT and not LOAD_NOTEBOOK:
bye("a notebook must be specified in order to use --nbconvert")
# if using containers (and not inside a container), see if older sessions need to be reaped
if running_session_dict and config.GRIM_REAPER:
kill_sessions = []
for cont, (_, path, _, sid, _) in running_session_dict.items():
if sid != config.SESSION_ID and os.path.samefile(
path, os.getcwd()):
message(ff("reaping older session {sid}"))
kill_sessions.append(cont)
if kill_sessions:
backend.kill_sessions(running_session_dict,
kill_sessions,
ignore_fail=True)
# virtual environment
os.environ["RADIOPADRE_VENV"] = config.RADIOPADRE_VENV
# init paths & environment
iglesia.init()
iglesia.set_userside_ports(userside_ports)
global JUPYTER_OPTS
if config.NBCONVERT:
JUPYTER_OPTS = [
"nbconvert", "--ExecutePreprocessor.timeout=600", "--no-input",
"--to", "html_embed", "--execute"
]
os.environ["RADIOPADRE_NBCONVERT"] = "True"
else:
JUPYTER_OPTS = [
"notebook",
"--ContentsManager.pre_save_hook=radiopadre_utils.notebook_utils._notebook_save_hook",
"--ContentsManager.allow_hidden=True"
]
os.environ.pop("RADIOPADRE_NBCONVERT", None)
# update installation etc.
backend.update_installation()
# (when running natively (i.e. in a virtual environment), the notebook app doesn't pass the token to the browser
# command properly... so let it pick its own token then)
# if options.remote or options.config.INSIDE_CONTAINER_PORTS or not options.virtual_env:
JUPYTER_OPTS += [
ff("--NotebookApp.token='{config.SESSION_ID}'"),
ff("--NotebookApp.custom_display_url='http://localhost:{userside_jupyter_port}'"
)
]
#=== figure out whether we initialize or load a notebook
os.chdir(iglesia.SERVER_BASEDIR)
if iglesia.SNOOP_MODE:
warning(
ff("{iglesia.ABSROOTDIR} is not writable for you, so radiopadre is operating in snoop mode."
))
ALL_NOTEBOOKS = glob.glob("*.ipynb")
if iglesia.SNOOP_MODE and not ALL_NOTEBOOKS:
orig_notebooks = glob.glob(os.path.join(iglesia.ABSROOTDIR, "*.ipynb"))
if orig_notebooks:
message(
" No notebooks in shadow directory: will copy notebooks from target."
)
message(" Copying {} notebooks from {}".format(
len(orig_notebooks), iglesia.ABSROOTDIR))
for nb in orig_notebooks:
shutil.copyfile(nb, './' + os.path.basename(nb))
ALL_NOTEBOOKS = glob.glob("*.ipynb")
message(" Available notebooks: " + " ".join(ALL_NOTEBOOKS))
if not config.INSIDE_CONTAINER_PORTS:
# if no notebooks in place, see if we need to create a default
if not ALL_NOTEBOOKS:
if config.DEFAULT_NOTEBOOK:
message(
ff(" No notebooks yet: will create {config.DEFAULT_NOTEBOOK}"
))
LOAD_DIR = True
open(config.DEFAULT_NOTEBOOK,
'wt').write(default_notebook_code)
ALL_NOTEBOOKS = [config.DEFAULT_NOTEBOOK]
else:
message(
" No notebooks and no default. Displaying directory only."
)
LOAD_DIR = True
LOAD_NOTEBOOK = None
# expand globs and apply auto-load as needed
if LOAD_NOTEBOOK:
LOAD_NOTEBOOK = [
nb for nb in ALL_NOTEBOOKS
if fnmatch.fnmatch(os.path.basename(nb), LOAD_NOTEBOOK)
]
elif config.AUTO_LOAD == "1":
LOAD_NOTEBOOK = ALL_NOTEBOOKS[0] if ALL_NOTEBOOKS else None
message(ff(" Auto-loading {LOAD_NOTEBOOK[0]}."))
elif config.AUTO_LOAD:
LOAD_NOTEBOOK = [
nb for nb in ALL_NOTEBOOKS
if fnmatch.fnmatch(os.path.basename(nb), config.AUTO_LOAD)
]
if LOAD_NOTEBOOK:
message(" Auto-loading {}".format(" ".join(LOAD_NOTEBOOK)))
else:
message(
ff(" No notebooks matching --auto-load {config.AUTO_LOAD}"
))
urls = []
if LOAD_DIR:
urls.append(
ff("http://localhost:{userside_jupyter_port}/?token={config.SESSION_ID}"
))
if LOAD_NOTEBOOK:
urls += [
ff("http://localhost:{userside_jupyter_port}/notebooks/{nb}?token={config.SESSION_ID}"
) for nb in LOAD_NOTEBOOK
]
if not config.NBCONVERT:
for url in urls[::-1]:
message(ff("Browse to URL: {url}"), color="GREEN")
if config.CARTA_BROWSER:
url = ff(
"http://localhost:{iglesia.CARTA_PORT}/?socketUrl=ws://localhost:{iglesia.CARTA_WS_PORT}"
)
message(ff("Browse to URL: {url} (CARTA file browser)"),
color="GREEN")
urls.append(url)
# now we're ready to start the session
backend.start_session(container_name, selected_ports, userside_ports,
notebook_path, browser and urls)
def update_installation():
# are we already running inside a virtualenv? Proceed directly if so
# (see https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv)
# pip install command with -v repeated for each VERBOSE increment
pip_install = "pip install " + "-v " * min(max(config.VERBOSE - 1, 0), 3)
if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix')
and sys.base_prefix != sys.prefix):
if sys.prefix == config.RADIOPADRE_VENV:
message(
ff("Running inside radiopadre virtual environment {sys.prefix}"
))
else:
message(
ff("Running inside non-default virtual environment {sys.prefix}"
))
message(ff("Will assume radiopadre has been installed here."))
config.RADIOPADRE_VENV = sys.prefix
if config.VENV_REINSTALL:
bye("Can't --venv-reinstall from inside the virtualenv itself.")
# Otherwise check for virtualenv, nuke/remake one if needed, then activate it
else:
config.RADIOPADRE_VENV = os.path.expanduser(config.RADIOPADRE_VENV)
activation_script = os.path.join(config.RADIOPADRE_VENV,
"bin/activate_this.py")
# see if a reinstall is needed
if config.AUTO_INIT and config.VENV_REINSTALL and os.path.exists(
config.RADIOPADRE_VENV):
if not os.path.exists(activation_script):
error(ff("{activation_script} does not exist. Bat country!"))
bye(
ff("Refusing to touch this virtualenv. Please remove it by hand if you must."
))
cmd = ff("rm -fr {config.RADIOPADRE_VENV}")
warning(ff("Found a virtualenv in {config.RADIOPADRE_VENV}."))
warning("However, --venv-reinstall was specified. About to run:")
warning(" " + cmd)
if config.FULL_CONSENT:
warning(
"--full-consent given, so not asking for confirmation.")
else:
warning(ff("Your informed consent is required!"))
inp = INPUT(
ff("Please enter 'yes' to rm -fr {config.RADIOPADRE_VENV}: "
)).strip()
if inp != "yes":
bye(ff("'{inp}' is not a 'yes'. Phew!"))
message("OK, nuking it!")
shell(cmd)
new_venv = False
if not os.path.exists(config.RADIOPADRE_VENV):
if config.AUTO_INIT:
message(ff("Creating virtualenv {config.RADIOPADRE_VENV}"))
shell(ff("virtualenv -p python3 {config.RADIOPADRE_VENV}"))
new_venv = True
else:
error(
ff("Radiopadre virtualenv {config.RADIOPADRE_VENV} doesn't exist."
))
bye(ff("Try re-running with --auto-init to reinstall it."))
message(
ff(" Activating the radiopadre virtualenv via {activation_script}"
))
with open(activation_script) as f:
code = compile(f.read(), activation_script, 'exec')
exec(code, dict(__file__=activation_script), {})
if new_venv:
extras = config.VENV_EXTRAS.split(
",") if config.VENV_EXTRAS else []
# add numpy explicitly to quicken up pyregion install
extras.append("numpy")
if extras:
extras = " ".join(extras)
message(ff("Installing specified extras: {extras}"))
shell(ff("{pip_install} {extras}"))
# now check for a radiopadre install inside the venv
have_install = check_output("pip show radiopadre")
if have_install:
install_info = dict(
[x.split(": ", 1) for x in have_install.split("\n") if ': ' in x])
version = install_info.get("Version", "unknown")
if config.UPDATE:
warning(
ff("radiopadre (version {version}) is installed, but --update specified."
))
else:
message(ff("radiopadre (version {version}) is installed."))
if not have_install or config.UPDATE:
if config.SERVER_INSTALL_PATH and os.path.exists(
config.SERVER_INSTALL_PATH):
message(
ff("--server-install-path {config.SERVER_INSTALL_PATH} is configured and exists."
))
update_server_from_repository()
install = ff("-e {config.SERVER_INSTALL_PATH}")
elif config.SERVER_INSTALL_REPO:
if config.SERVER_INSTALL_REPO == "default":
config.SERVER_INSTALL_REPO = config.DEFAULT_SERVER_INSTALL_REPO
branch = config.SERVER_INSTALL_BRANCH or "master"
if config.SERVER_INSTALL_PATH:
message(
ff("--server-install-path and --server-install-repo configured, will clone and install"
))
cmd = ff(
"git clone -b {branch} {config.SERVER_INSTALL_REPO} {config.SERVER_INSTALL_PATH}"
)
message(ff("Running {cmd}"))
shell(cmd)
install = ff("-e {config.SERVER_INSTALL_PATH}")
else:
message(
ff("only --server-install-repo specified, will install directly from git"
))
install = ff("git+{config.SERVER_INSTALL_REPO}@{branch}")
elif config.SERVER_INSTALL_PIP:
message(
ff("--server-install-pip {config.SERVER_INSTALL_PIP} is configured."
))
install = config.SERVER_INSTALL_PIP
else:
bye("no radiopadre installation method specified (see --server-install options)"
)
cmd = ff("{pip_install} -U {install}")
message(ff("Running {cmd}"))
shell(cmd)
def run_remote_session(command, copy_initial_notebook, notebook_path,
extra_arguments):
SSH_MUX_OPTS = "-o ControlPath=/tmp/ssh_mux_radiopadre_%C -o ControlMaster=auto -o ControlPersist=1h".split(
)
SCP_OPTS = ["scp"] + SSH_MUX_OPTS
# SSH_OPTS = ["ssh", "-t"] + SSH_MUX_OPTS + [config.REMOTE_HOST]
SSH_OPTS = ["ssh"] + SSH_MUX_OPTS + [config.REMOTE_HOST]
# See, possibly: https://stackoverflow.com/questions/44348083/how-to-send-sigint-ctrl-c-to-current-remote-process-over-ssh-without-t-optio
# master ssh connection, to be closed when we exit
message(
ff("Opening ssh connection to {config.REMOTE_HOST}. You may be prompted for your password."
))
debug(" {}".format(" ".join(SSH_OPTS)))
ssh_master = subprocess.check_call(SSH_OPTS + ["exit"], stderr=DEVNULL)
# raw_input("Continue?")
def help_yourself(problem, suggestion=None):
"""
Prints a "help yourself" message and exits
"""
message("{}".format(problem))
message(
ff("Please ssh {config.REMOTE_HOST} and sort it out yourself, then rerun this script"
))
if suggestion:
message(ff("({suggestion})"))
sys.exit(1)
def ssh_remote(command, fail_retcode=None, stderr=DEVNULL):
"""Runs command on remote host. Returns its output if the exit status is 0, or None if the exit status matches fail_retcode.
Any other non-zero exit status (or any other error) will result in an exception.
"""
try:
return subprocess.check_output(SSH_OPTS + [command],
stderr=stderr).decode('utf-8')
except subprocess.CalledProcessError as exc:
if exc.returncode == fail_retcode:
return None
message(ff("ssh {command} failed with exit code {exc.returncode}"))
raise
def ssh_remote_v(command, fail_retcode=None):
return ssh_remote(command, fail_retcode, stderr=sys.stderr)
def ssh_remote_interactive(command, fail_retcode=None):
"""Runs command on remote host. Returns the exit status if 0, or None if the exit status matches fail_retcode.
Any other non-zero exit status (or any other error) will result in an exception.
"""
try:
return subprocess.check_call(SSH_OPTS + [command])
except subprocess.CalledProcessError as exc:
if exc.returncode == fail_retcode:
return None
message(ff("ssh {command} failed with exit code {exc.returncode}"))
raise
def scp_to_remote(path, remote_path):
return subprocess.check_output(
SCP_OPTS + [path, "{}:{}".format(config.REMOTE_HOST, remote_path)])
def check_remote_file(remote_file, test="-x"):
"""
Checks that a remote file exists. 'test' is specified bash-style, e.g. "-x" for executable.
Can also use -f and -f, for example.
Returns True or False, or raises an exception on other errors.
"""
return ssh_remote(
"if [ {} {} ]; then exit 0; else exit 199; fi".format(
test, remote_file),
fail_retcode=199,
stderr=DEVNULL) is not None
def check_remote_command(command):
"""
Checks that remote host has a particular command available (by running 'which' on the remote).
Returns True or False, or raises an exception on other errors.
"""
if config.SKIP_CHECKS:
return command
return (ssh_remote("which " + command, fail_retcode=1, stderr=DEVNULL)
or "").strip()
# --update or --auto-init disables --skip-checks
if config.SKIP_CHECKS:
if config.UPDATE:
message("Note that --update implies --no-skip-checks")
config.SKIP_CHECKS = False
elif config.AUTO_INIT:
message("Note that --auto-init implies --no-skip-checks")
config.SKIP_CHECKS = False
# propagate our config to command-line arguments
remote_config = config.get_config_dict()
remote_config['BROWSER'] = 'None'
remote_config['SKIP_CHECKS'] = False
remote_config['VENV_REINSTALL'] = False
# Check for various remote bits
if config.VERBOSE and not config.SKIP_CHECKS:
message(ff("Checking installation on {config.REMOTE_HOST}."))
has_git = check_remote_command("git")
USE_VENV = has_singularity = has_docker = None
for backend in config.BACKEND:
remote_config["BACKEND"] = backend
if backend == "venv" and check_remote_command(
"virtualenv") and check_remote_command("pip"):
USE_VENV = True
break
elif backend == "docker":
has_docker = check_remote_command("docker")
if has_docker:
break
elif backend == "singularity":
has_singularity = check_remote_command("singularity")
if has_singularity:
break
message(
ff("The '{backend}' back-end is not available on {config.REMOTE_HOST}, skipping."
))
else:
bye(
ff("None of the specified back-ends are available on {config.REMOTE_HOST}."
))
if remote_config["BACKEND"] != "docker":
config.CONTAINER_PERSIST = config.CONTAINER_DEBUG = False
# which runscript to look for
runscript0 = "run-radiopadre"
# form up remote venv path, but do not expand ~ at this point (it may be a different username on the remote)
env = os.environ.copy()
env.setdefault("RADIOPADRE_DIR", config.REMOTE_RADIOPADRE_DIR
or "~/.radiopadre")
config.RADIOPADRE_VENV = (config.RADIOPADRE_VENV or "").format(**env)
# this variable used in error and info messages
remote_venv = ff("{config.REMOTE_HOST}:{config.RADIOPADRE_VENV}")
# pip install command with -v repeated for each VERBOSE increment
pip_install = "pip install " + "-v " * min(max(config.VERBOSE - 1, 0), 3)
# do we want to do an install/update -- will be forced to True (if we must install),
# or False if we can't update
do_update = config.UPDATE
if config.SKIP_CHECKS:
runscript = ff(
"if [ -f {config.RADIOPADRE_VENV}/bin/activate ]; then " +
"source {config.RADIOPADRE_VENV}/bin/activate; fi; run-radiopadre "
)
do_update = False
else:
runscript = None
# (a) if --auto-init and --venv-reinstall specified, zap remote virtualenv if present
if config.AUTO_INIT and config.VENV_REINSTALL:
if not config.RADIOPADRE_VENV:
bye(
ff("Can't do --auto-init --venv-reinstall because --radiopadre-venv is not set"
))
if "~" in config.RADIOPADRE_VENV:
config.RADIOPADRE_VENV = ssh_remote(
ff("echo {config.RADIOPADRE_VENV}")).strip(
) # expand "~" on remote
if check_remote_file(ff("{config.RADIOPADRE_VENV}"), "-d"):
if not check_remote_file(
ff("{config.RADIOPADRE_VENV}/bin/activate"), "-f"):
error(
ff("{remote_venv}/bin/activate} does not exist. Bat country!"
))
bye(
ff("Refusing to touch this virtualenv. Please remove it by hand if you must."
))
cmd = ff("rm -fr {config.RADIOPADRE_VENV}")
warning(ff("Found a virtualenv in {remote_venv}."))
warning(
"However, --venv-reinstall was specified. About to run:")
warning(ff(" ssh {config.REMOTE_HOST} " + cmd))
if config.FULL_CONSENT:
warning(
"--full-consent given, so not asking for confirmation."
)
else:
warning(ff("Your informed consent is required!"))
inp = INPUT(
ff("Please enter 'yes' to rm -fr {remote_venv}: ")
).strip()
if inp != "yes":
bye(ff("'{inp}' is not a 'yes'. Phew!"))
message("OK, nuking it!")
ssh_remote(cmd)
# force update
do_update = True
# (b) look inside venv
if runscript is None and config.RADIOPADRE_VENV:
if "~" in config.RADIOPADRE_VENV:
config.RADIOPADRE_VENV = ssh_remote(
ff("echo {config.RADIOPADRE_VENV}")).strip(
) # expand "~" on remote
if check_remote_file(ff("{config.RADIOPADRE_VENV}/bin/activate"),
"-f"):
if ssh_remote(ff(
"source {config.RADIOPADRE_VENV}/bin/activate && which {runscript0}"
),
fail_retcode=1):
runscript = ff(
"source {config.RADIOPADRE_VENV}/bin/activate && {runscript0}"
)
message(
ff("Using remote client script within {config.RADIOPADRE_VENV}"
))
else:
message(
ff("Remote virtualenv {config.RADIOPADRE_VENV} exists, but does not contain a radiopadre-client installation."
))
else:
message(ff("No remote virtualenv found at {remote_venv}"))
# (c) just try `which` directly
if runscript is None:
runscript = check_remote_command(runscript0)
if runscript:
message(ff("Using remote client script at {runscript}"))
runscript = ff()
do_update = False
if config.UPDATE:
warning(
ff("ignoring --update for client since it isn't in a virtualenv"
))
else:
message(ff("No remote client script {runscript0} found"))
runscript = None
## No runscript found on remote?
## First, figure out whether to reinstall a virtualenv for it
if not runscript:
message(ff("No {runscript0} script found on {config.REMOTE_HOST}"))
if not config.AUTO_INIT:
bye(
ff("Try re-running with --auto-init to install radiopadre-client on {config.REMOTE_HOST}."
))
if not config.RADIOPADRE_VENV:
bye(ff("Can't do --auto-init because --virtual-env is not set."))
message("Trying to --auto-init an installation for you...")
# try to auto-init a virtual environment
if not check_remote_file(ff("{config.RADIOPADRE_VENV}/bin/activate"),
"-f"):
message(ff("Creating virtualenv {remote_venv}"))
ssh_remote_v(ff("virtualenv -p python3 {config.RADIOPADRE_VENV}"))
extras = config.VENV_EXTRAS.split(
",") if config.VENV_EXTRAS else []
# add numpy explicitly to quicken up pyregion install
extras.append("numpy")
if extras:
extras = " ".join(extras)
message(ff("Installing specified extras: {extras}"))
ssh_remote_v(
ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} {extras}"
))
else:
message(ff("Installing into existing virtualenv {remote_venv}"))
# Now, figure out how to install or update the client package
if not runscript or do_update:
# installing from a specified existing path
if config.CLIENT_INSTALL_PATH and check_remote_file(
config.CLIENT_INSTALL_PATH, "-d"):
install_path = config.CLIENT_INSTALL_PATH
message(
ff("--client-install-path {install_path} is configured and exists on {config.REMOTE_HOST}."
))
# update if managed by git
if check_remote_file(ff("{install_path}/.git"),
"-d") and config.UPDATE:
if has_git:
if config.CLIENT_INSTALL_BRANCH:
cmd = ff(
"cd {install_path} && git fetch origin && git checkout {config.CLIENT_INSTALL_BRANCH} && git pull"
)
else:
cmd = ff("cd {install_path} && git pull")
warning(
ff("--update specified and git detected, will attempt to update via"
))
message(ff(" {cmd}"))
ssh_remote_v(cmd)
else:
warning(
ff("--update specified, but no git command found on {config.REMOTE_HOST}"
))
install_path = "-e " + install_path
# else, installing from git
elif config.CLIENT_INSTALL_REPO:
if config.CLIENT_INSTALL_REPO == "default":
config.CLIENT_INSTALL_REPO = config.DEFAULT_CLIENT_INSTALL_REPO
branch = config.CLIENT_INSTALL_BRANCH or "master"
if config.CLIENT_INSTALL_PATH:
message(
ff("--client-install-path and --client-install-repo configured, will attempt"
))
cmd = ff(
"git clone -b {branch} {config.CLIENT_INSTALL_REPO} {config.CLIENT_INSTALL_PATH}"
)
message(ff(" ssh {config.REMOTE_HOST} {cmd}"))
ssh_remote_v(cmd)
install_path = ff("-e {config.CLIENT_INSTALL_PATH}")
else:
message(
ff("--client-install-repo is configured, will try to install directly from git"
))
install_path = ff("git+{config.CLIENT_INSTALL_REPO}@{branch}")
# now pip install
message(
ff("Doing pip install -e {install_path} in {config.RADIOPADRE_VENV}"
))
ssh_remote_v(
ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} -e {install_path}"
))
# else, installing directly from pip
elif config.CLIENT_INSTALL_PIP:
message(
ff("--client-install-pip {config.CLIENT_INSTALL_PIP} is configured."
))
install_path = config.CLIENT_INSTALL_PIP
else:
bye("no radiopadre-client installation method specified (see --client-install options)"
)
# now install
message(
ff("Will attempt to pip install -U {install_path} in {remote_venv}"
))
ssh_remote_v(
ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} -U {install_path}"
))
# sanity check
if ssh_remote(ff(
"source {config.RADIOPADRE_VENV}/bin/activate && which {runscript0}"
),
fail_retcode=1):
runscript = ff(
"source {config.RADIOPADRE_VENV}/bin/activate && {runscript0}")
else:
bye(
ff("Something went wrong during installation on {config.REMOTE_HOST}, since I still don't see the {runscript0} script"
))
message("Success!")
runscript = ff(
"export RADIOPADRE_DIR={config.REMOTE_RADIOPADRE_DIR}; {runscript}")
# copy notebook to remote
if copy_initial_notebook:
if not os.path.exists(copy_initial_notebook):
bye("{} doesn't exist".format(copy_initial_notebook))
if check_remote_file(notebook_path or ".", "-d"):
nbpath = "{}/{}".format(notebook_path or ".",
copy_initial_notebook)
if check_remote_file(nbpath, "-ff("):
message(
ff("remote notebook {nbpath} exists, will not copy over"))
else:
message(
ff("remote notebook {nbpath} doesn't exist, will copy over"
))
scp_to_remote(copy_initial_notebook, notebook_path)
notebook_path = nbpath
# allocate 5 suggested ports (in resume mode, this will be overridden by the session settings)
starting_port = 10000 + os.getuid() * 3
ports = []
for _ in range(5):
starting_port = find_unused_port(starting_port + 1, 10000)
ports.append(starting_port)
remote_config["remote"] = ":".join(map(str, ports))
# turn the remote_config dict into a command line
runscript += " " + " ".join(
config.get_options_list(remote_config, quote=True))
runscript += " '{}' {}".format(
command if command is not "load" else notebook_path,
" ".join(extra_arguments))
# start ssh subprocess to launch notebook
args = list(SSH_OPTS) + ["shopt -s huponexit && " + runscript]
if config.VERBOSE:
message("running {}".format(" ".join(args)))
else:
message(ff("running radiopadre client on {config.REMOTE_HOST}"))
ssh = subprocess.Popen(args,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
bufsize=1,
universal_newlines=True)
poller = Poller()
poller.register_process(ssh, config.REMOTE_HOST,
config.REMOTE_HOST + " stderr")
if not USE_VENV:
poller.register_file(sys.stdin, "stdin")
container_name = None
urls = []
remote_running = False
status = 0
try:
while remote_running is not None and poller.fdlabels:
fdlist = poller.poll(verbose=config.VERBOSE > 1)
for fname, fobj in fdlist:
try:
line = fobj.readline()
except EOFError:
line = b''
empty_line = not line
line = (line.decode('utf-8')
if type(line) is bytes else line).rstrip()
if fobj is sys.stdin and line == 'D' and config.CONTAINER_PERSIST:
sys.exit(0)
# break out if ssh closes
if empty_line:
poller.unregister_file(fobj)
if ssh.stdout not in poller and ssh.stderr not in poller:
message(
ff("The ssh process to {config.REMOTE_HOST} has exited"
))
remote_running = None
break
continue
# print remote output
print_output = False
if fobj is ssh.stderr:
print_output = not line.startswith("Shared connection to")
else:
print_output = not line.startswith(
"radiopadre:") or command != 'load'
if not empty_line and (config.VERBOSE or print_output):
for key, dispatch in _dispatch_message.items():
if key in line:
dispatch(u"{}: {}".format(fname, line))
break
else:
message(u"{}: {}".format(fname, line))
if not line:
continue
# if remote is not yet started, check output
if not remote_running:
# check for session ID
match = re.match(
".*Session ID/notebook token is '([0-9a-f]+)'", line)
if match:
config.SESSION_ID = match.group(1)
continue
# check for notebook port, and launch second ssh when we have it
re_ports = ":".join(["([\\d]+)"] *
10) # form up regex for ddd:ddd:...
match = re.match(ff(".*Selected ports: {re_ports}[\s]*$"),
line)
if match:
ports = list(map(int, match.groups()))
remote_ports = ports[:5]
local_ports = ports[5:]
if config.VERBOSE:
message(
"Detected ports {}:{}:{}:{}:{} -> {}:{}:{}:{}:{}"
.format(*ports))
ssh2_args = ["ssh"] + SSH_MUX_OPTS + [
"-O", "forward", config.REMOTE_HOST
]
for loc, rem in zip(local_ports, remote_ports):
ssh2_args += [
"-L",
"localhost:{}:localhost:{}".format(loc, rem)
]
# tell mux process to forward the ports
if config.VERBOSE:
message(
"sending forward request to ssh mux process".
format(ssh2_args))
subprocess.call(ssh2_args)
continue
# check for launch URL
match = re.match(".*Browse to URL: ([^\s\033]+)", line)
if match:
urls.append(match.group(1))
continue
# check for container name
match = re.match(".*Container name: ([^\s\033]+)", line)
if match:
container_name = match.group(1)
continue
if "jupyter notebook server is running" in line:
remote_running = True
time.sleep(1)
iglesia.register_helpers(*run_browser(*urls))
message(
"The remote radiopadre session is now fully up")
if USE_VENV or not config.CONTAINER_PERSIST:
message("Press Ctrl+C to kill the remote session")
else:
message(
"Press D<Enter> to detach from remote session, or Ctrl+C to kill it"
)
except SystemExit as exc:
message(ff("SystemExit: {exc.code}"))
status = exc.code
except KeyboardInterrupt:
message("Ctrl+C caught")
status = 1
except Exception as exc:
traceback.print_exc()
message("Exception caught: {}".format(str(exc)))
if remote_running and ssh.poll() is None:
message("Asking remote session to exit, nicely")
try:
try:
ssh.stdin.write("exit\n")
except TypeError:
ssh.stdin.write(b"exit\n") # because f**k you python
except IOError:
debug(" looks like it's already exited")
# if status and not USE_VENV and container_name:
# message(ff("killing remote container {container_name}"))
# try:
# if has_docker:
# ssh_remote(ff("{has_docker} kill {container_name}"))
# elif has_singularity:
# from .backends.singularity import get_singularity_image
# singularity_image = get_singularity_image(config.DOCKER_IMAGE)
# ssh_remote(ff("{has_singularity} instance.stop {singularity_image} {container_name}"))
# except subprocess.CalledProcessError as exc:
# message(exc.output.decode())
for i in range(10, 0, -1):
if ssh.poll() is not None:
debug("Remote session has exited")
ssh.wait()
break
message(ff("Waiting for remote session to exit ({i})"))
time.sleep(1)
else:
message(ff("Remote session hasn't exited, killing the ssh process"))
ssh.kill()
return status
def check_recent_sessions(options, argv, parser=None):
"""
Loads a recent session if requested
:param options: Options object from ArgumentParser
:param argv: Argument list (from sys.argv[1:] initially)
:param parser: ArgumentParser object used to (re)parse the options
:return: options, argv
Where options and argv may have been loaded from the recent
options file
"""
make_radiopadre_dir()
# load history
try:
readline.read_history_file(HISTORY_FILE)
except IOError:
pass
resume_session = None
# a single-digit argument resumes session #N
if len(options.arguments) == 1 and re.match("^\d$", options.arguments[0]):
resume_session = int(options.arguments[0])
# no arguments is resume session #0
elif not options.arguments:
resume_session = 0
if resume_session is not None:
last = _load_recent_sessions()
num_recent = len(last)
if resume_session >= num_recent:
bye(ff("no recent session #{resume_session}"))
message("Your most recent radiopadre sessions are:")
message("")
for i, (_, opts) in enumerate(list(last.items())[::-1]):
message(" [#{0}] {1}".format(i, opts), color="GREEN")
message("")
print(
"\nInteractive startup mode. Edit arguments and press Enter to run, or Ctrl+C to bail out. "
)
print(
" (Ctrl+U + <NUM> + Enter will paste other recent session arguments from the list above)\n"
)
inp = None
cmdline = ''
readline.set_startup_hook(lambda: readline.insert_text(cmdline))
while inp is None:
# form up list of fake args to be re-parsed for the last session
cmdline = list(last.items())[-(resume_session + 1)][1]
# non-persisting options raised in command line shall be appended to the fake args
for opt in config.NON_PERSISTING_OPTIONS:
if opt.startswith("--") and getattr(options, opt[2:].replace(
"-", "_"), None):
cmdline += " " + opt
cmdline += " "
## colors confuse Ctrl+U and such
# prompt = ff("{logger.Colors.GREEN}[#{resume_session}]:{logger.Colors.ENDC} ")
prompt = ff("[#{resume_session}] ")
inp = INPUT(prompt)
inp = inp.strip()
if not inp:
resume_session = 0
inp = None
elif re.match("^\d+$", inp):
res = int(inp)
if res >= num_recent:
warning(ff("no recent session #{res}"))
else:
resume_session = res
readline.remove_history_item(1)
inp = None
readline.set_startup_hook(None)
global _last_input
_last_input = inp
argv = shlex.split(inp, posix=False)
options = parser.parse_args(argv)
return options, argv
def update_installation(rebuild=False, docker_pull=True):
global docker_image
global singularity_image
docker_image = config.DOCKER_IMAGE
singularity_image = os.path.expanduser(get_singularity_image(docker_image))
# this will be True if we need to build the image
build_image = False
# clearly true if no image
if not os.path.exists(singularity_image):
if config.SINGULARITY_AUTO_BUILD:
message(ff("Singularity image {singularity_image} does not exist"))
build_image = True
else:
error(
ff("Singularity image {singularity_image} does not exist, and auto-build is disabled"
))
bye(ff("Re-run with --singularity-auto-build to proceed"))
# also true if rebuild forced by flags or config or command line
elif rebuild or config.SINGULARITY_REBUILD:
config.SINGULARITY_AUTO_BUILD = build_image = True
message(
ff("--singularity-rebuild specified, removing singularity image {singularity_image}"
))
# pull down docker image first
if has_docker and docker_pull:
message(
"Checking docker image (from which our singularity image is built)"
)
docker.update_installation(enable_pull=True)
# if we're not forced to build yet, check for an update
if config.UPDATE and not build_image:
if has_docker:
# check timestamp of docker image
docker_image_time = None
output = check_output(
ff("{has_docker} image inspect {docker_image} -f '{{{{ .Created }}}}'"
)).strip()
message(ff(" docker image timestamp is {output}"))
# in Python 3.7 we have datetime.fromisoformat(date_string), but for now we muddle:
match = output and re.match("(^.*)[.](\d+)Z", output)
if match:
try:
docker_image_time = calendar.timegm(
time.strptime(match.group(1), "%Y-%m-%dT%H:%M:%S"))
docker_image_time = datetime.datetime.utcfromtimestamp(
docker_image_time)
except ValueError:
pass
sing_image_time = datetime.datetime.utcfromtimestamp(
os.path.getmtime(singularity_image))
message(" singularity image timestamp is {}".format(
sing_image_time.isoformat()))
if docker_image_time is None:
warning(
ff("can't parse docker image timestamp '{output}', rebuilding {singularity_image} just in case"
))
build_image = True
elif docker_image_time > sing_image_time:
warning(
ff("rebuilding outdated singularity image {singularity_image}"
))
build_image = True
else:
message(
ff("singularity image {singularity_image} is up-to-date"))
else:
message(
ff("--update specified but no docker access, assuming {singularity_image} is up-to-date"
))
# now build if needed
if build_image:
warning(
ff("Rebuilding singularity image from docker://{docker_image}"))
warning(ff(" (This may take a few minutes....)"))
singularity_image_new = singularity_image + ".new.img"
if os.path.exists(singularity_image_new):
os.unlink(singularity_image_new)
cmd = [
singularity, "build", singularity_image_new,
ff("docker://{docker_image}")
]
message("running " + " ".join(cmd))
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError as exc:
if config.IGNORE_UPDATE_ERRORS:
if os.path.exists(singularity_image):
warning(
"singularity build failed but --ignore-update-errors is set, proceeding with old image"
)
else:
error(
"singularity build failed, --ignore-update-errors is set, but we have no older image"
)
raise
else:
raise
# move old image
message(ff("Build successful, renaming to {singularity_image}"))
os.rename(singularity_image_new, singularity_image)
else:
message(
ff("Using existing radiopadre singularity image {singularity_image}"
))
# not supported with Singularity
config.CONTAINER_PERSIST = False