def add_to_parser(parser): """Adds parser options corresponding to global defaults (that have not been added to the parser already)""" global _DEFAULT_KEYS, _CMDLINE_DEFAULTS for key in _DEFAULT_KEYS: default_conf = DefaultConfig[key] lkey = key.lower() optname = lkey.replace("_", "-") default_cmdline = parser.get_default(lkey) # no command-line switch for this option? Add it if default_cmdline is None: parser.add_argument("--" + optname, type=type(default_conf), metavar=key, help=ff("overrides the {key} config setting.")) _CMDLINE_DEFAULTS[key] = default_conf # else check for opposite-value switch else: if type(DefaultConfig[key] ) is bool and "--" + optname not in NON_PERSISTING_OPTIONS: if default_cmdline is 0: parser.add_argument("--no-" + optname, action="store_false", dest=lkey, help=ff("opposite of --{optname}.")) elif default_cmdline is 1: parser.add_argument("--" + optname, action="store_true", help=ff("opposite of --no-{optname}.")) _CMDLINE_DEFAULTS[key] = default_cmdline
def update_installation(enable_pull=False): global docker_image enable_pull = enable_pull or config.AUTO_INIT or config.UPDATE if config.CONTAINER_DEV: update_server_from_repository() docker_image = config.DOCKER_IMAGE if check_output(ff("docker image inspect {docker_image}")) is None: if not enable_pull: bye( ff(" Radiopadre docker image {docker_image} not found. Re-run with --update or --auto-init perhaps?" )) message( ff(" Radiopadre docker image {docker_image} not found locally")) else: message(ff(" Using radiopadre docker image {docker_image}")) if enable_pull: warning(ff("Calling docker pull {docker_image}")) warning( " (This may take a few minutes if the image is not up to date...)" ) try: subprocess.call([docker, "pull", docker_image]) except subprocess.CalledProcessError as exc: if config.IGNORE_UPDATE_ERRORS: warning( "docker pull failed, but --ignore-update-errors is set, proceeding anyway" ) return raise exc
def _run_container(container_name, docker_opts, jupyter_port, browser_urls, singularity=False): message("Running {}".format(" ".join(map(str, docker_opts)))) if singularity: message( " (When using singularity and the image is not yet available locally, this can take a few minutes the first time you run.)" ) if config.CONTAINER_DEBUG: docker_process = subprocess.Popen(docker_opts, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr) else: docker_process = subprocess.Popen( docker_opts, stdout=DEVNULL, stderr=DEVNULL if config.NON_INTERACTIVE else sys.stderr) #stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, #env=os.environ) if config.NBCONVERT: message("Waiting for conversion to finish") docker_process.wait() return None else: # pause to let the Jupyter server spin up wait = await_server_startup(jupyter_port, process=docker_process, init_wait=1, server_name="notebook container") if wait is None: if docker_process.returncode is not None: bye( ff("container unexpectedly exited with return code {docker_process.returncode}" )) bye( ff("unable to connect to jupyter notebook server on port {jupyter_port}" )) message( ff("Container started. The jupyter notebook server is running on port {jupyter_port} (after {wait:.2f} secs)" )) if browser_urls: iglesia.register_helpers(*run_browser(*browser_urls)) # give things a second (to let the browser command print its stuff, if it wants to) time.sleep(1) return docker_process
def help_yourself(problem, suggestion=None): """ Prints a "help yourself" message and exits """ message("{}".format(problem)) message( ff("Please ssh {config.REMOTE_HOST} and sort it out yourself, then rerun this script" )) if suggestion: message(ff("({suggestion})")) sys.exit(1)
def list_sessions(): """Returns OrderedDict (ordered by uptime) of running containers with their session IDs. Clears up dead sessions. Dict is id -> [name, path, uptime, session_id, ports]""" container_dict = _ps_containers() # match session files to containers for session_dir in glob.glob(SESSION_INFO_DIR + "/radiopadre-*"): name = os.path.basename(session_dir) if name not in container_dict: message( " container {} is no longer running, clearing up session dir" .format(name)) subprocess.call(["rm", "-fr", session_dir]) continue try: container_dict[name][3], container_dict[name][ 4] = read_session_info(session_dir) except ValueError: message(ff(" invalid session dir {session_dir}")) continue output = OrderedDict() # check for containers without session info and form up output dict for name, (id_, path, time, session_id, ports) in container_dict.items(): if session_id is None: message(" container {} has no session dir -- killing it".format( name)) subprocess.call([docker, "kill", id_]) else: output[id_] = [name, path, time, session_id, ports] return output
def _load_recent_sessions(must_exist=True): """ Load recent sessions from RECENTS_FILE. :param must_exist: if True and session file does not exist, exit with error :return: dict of latest sessions """ global _recent_sessions if _recent_sessions is not None: return _recent_sessions if os.path.exists(RECENTS_FILE): _recent_sessions = OrderedDict() try: for line in open(RECENTS_FILE, "rt"): key, args = line.strip().split(":::", 1) _recent_sessions[key] = args except Exception as exc: message(ff("Error reading {RECENTS_FILE}: {exc}")) _recent_sessions = None if _recent_sessions is None and must_exist: bye("no recent radiopadre sessions and no arguments given. Run with -h for help." ) return _recent_sessions
def read_session_info(container_name): """Reads the given session ID file. Returns session_id, ports, or else throws a ValueError""" dirname = get_session_info_dir(container_name) session_file = ff("{dirname}/info") if not os.path.exists(session_file): raise ValueError(ff("invalid session dir {dirname}")) comps = open(session_file, "rt").read().strip().split(" ") if len(comps) != 11: raise ValueError(ff("invalid session dir {dirname}")) session_id = comps[0] try: ports = map(int, comps[1:]) except: raise ValueError(ff("invalid session dir {dirname}")) return session_id, ports
def update_server_from_repository(): """ Updates the radiopadre git working directory, if necessary :return: """ if config.UPDATE and config.SERVER_INSTALL_PATH and os.path.isdir( config.SERVER_INSTALL_PATH + "/.git"): if config.SERVER_INSTALL_BRANCH: cmd = ff( "cd {config.SERVER_INSTALL_PATH} && git fetch origin && git checkout {config.SERVER_INSTALL_BRANCH} && git pull" ) else: cmd = ff("cd {config.SERVER_INSTALL_PATH} && git pull") message( ff("--update specified, --server-install-path at {config.SERVER_INSTALL_PATH} will be updated via" )) message(ff(" {cmd}")) if shell(cmd): bye("update failed")
def get_options_list(config_dict, quote=True): """ :param config_dict: dictionary of config settings quote: if True, values will placed in single quotes. Use this if forming up a shell string command, ultimately. :return: list of command-line arguments """ args = [] # turn the remote_config dict into a command line for key, value in config_dict.items(): opt = key.lower().replace("_", "-") #if value != DefaultConfig.get(key): if value is True: args.append(ff("--{opt}")) elif value is not False and value is not None: if type(value) is list: value = ",".join(map(str, value)) args += [ff("--{opt}"), ff("'{value}'") if quote else str(value)] return args
def kill_sessions(session_dict, session_ids, ignore_fail=False): kill_cont = " ".join(session_ids) message(" killing containers: {}".format(kill_cont)) for cont in session_ids: if cont not in session_dict: bye("no such radiopadre container: {}".format(cont)) name, path, _, _, _ = session_dict[cont] session_id_file = "{}/{}".format(SESSION_INFO_DIR, name) if os.path.exists(session_id_file): subprocess.call(["rm", "-fr", session_id_file]) shell(ff("{docker} kill {kill_cont}"), ignore_fail=True)
def ssh_remote_interactive(command, fail_retcode=None): """Runs command on remote host. Returns the exit status if 0, or None if the exit status matches fail_retcode. Any other non-zero exit status (or any other error) will result in an exception. """ try: return subprocess.check_call(SSH_OPTS + [command]) except subprocess.CalledProcessError as exc: if exc.returncode == fail_retcode: return None message(ff("ssh {command} failed with exit code {exc.returncode}")) raise
def ssh_remote(command, fail_retcode=None, stderr=DEVNULL): """Runs command on remote host. Returns its output if the exit status is 0, or None if the exit status matches fail_retcode. Any other non-zero exit status (or any other error) will result in an exception. """ try: return subprocess.check_output(SSH_OPTS + [command], stderr=stderr).decode('utf-8') except subprocess.CalledProcessError as exc: if exc.returncode == fail_retcode: return None message(ff("ssh {command} failed with exit code {exc.returncode}")) raise
def run_browser(*urls): """ Runs a browser pointed to URL(s), in background if config.BROWSER_BG is True. If config.BROWSER_MULTI is set, runs a browser per URL, else feeds all URLs to one browser invocation Returns list of processes (in BROWSER_BG mode). """ from . import config procs = [] # open browser if needed if config.BROWSER: message("Running {} {}\r".format(config.BROWSER, " ".join(urls))) message( " if this fails, specify a correct browser invocation command with --browser and rerun," ) message( " or else browse to the URL given above (\"Browse to URL:\") yourself." ) # sometimes the notebook does not respond immediately, so take a second time.sleep(1) if config.BROWSER_MULTI: commands = [[config.BROWSER] + list(urls)] else: commands = [[config.BROWSER, url] for url in urls] for command in commands: try: if config.BROWSER_BG: procs.append( subprocess.Popen(command, stdin=DEVZERO, stdout=sys.stdout, stderr=sys.stderr)) else: subprocess.call(command, stdout=DEVNULL) except OSError as exc: if exc.errno == 2: message(ff("{config.BROWSER} not found")) else: raise else: message( "--no-browser given, or browser not set, not opening a browser for you\r" ) message("Please browse to: {}\n".format(" ".join(urls))) return procs
def await_server_startup(port, process=None, server_name="jupyter notebook server", init_wait=2, wait=60): """ Waits for a server process to start up, tries to connect to the specified port, returns when successful :param port: port number :param process: if not None, waits on the process and checks its return code :param init_wait: number of second to wait before trying to connect :param wait: total number of seconds to wait before giving up :return: number of seconds elapsed before connection, or None if failed """ # pause to let the Jupyter server spin up t0 = time.time() time.sleep(init_wait) # then try to connect to it sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) for retry in range(int(wait / .1)): # try to connect try: sock.connect(("localhost", port)) del sock return time.time() - t0 except socket.error: pass if not retry: message( ff("Waiting for up to {wait} secs for the {server_name} to come up" )) # sleep, check process if process is not None: process.poll() if process.returncode is not None: return None time.sleep(.1) return None
def _init_session_dir(): radiopadre_dir = make_radiopadre_dir() global SESSION_INFO_DIR SESSION_INFO_DIR = ff("{radiopadre_dir}/sessions") make_dir(SESSION_INFO_DIR)
def init_specific_options(remote_host, notebook_path, options): global _DEFAULT_KEYS global _CMDLINE_DEFAULTS parser = configparser.ConfigParser() hostname = ff("{remote_host}") if remote_host else "local sesssion" session = ff("{hostname}:{notebook_path}") config_exists = os.path.exists(CONFIG_FILE) use_config_files = not options.remote and not options.inside_container # try to read config file for host and host:path (not in --remote mode though) if use_config_files and config_exists: parser.read(CONFIG_FILE) for sect_key in "global defaults", hostname, session: if parser.has_section(sect_key): section = dict(parser.items(sect_key)) if section: message( ff(" loading settings from {CONFIG_FILE} [{sect_key}]" )) for key in _DEFAULT_KEYS: lkey = key.lower() if lkey in section: value = _get_config_value(section, lkey) if value != globals()[key]: message(ff(" {key} = {value}")) globals()[key] = value # update using command-line options command_line_updated = [] for key in globals().keys(): if re.match("^[A-Z]", key): optname = key.lower() opt_switch = "--" + optname.replace("_", "-") value = getattr(options, optname, None) # skip DEFAULT_VALUE placeholders, trust in config if value is DEFAULT_VALUE or value is None: continue if type(value) is list: value = ",".join(value) if value is not _CMDLINE_DEFAULTS.get(key, None): if use_config_files: # do not mark options such as --update for saving if value is not _CMDLINE_DEFAULTS.get(key) and DefaultConfig.get(key) is not None \ and opt_switch not in NON_PERSISTING_OPTIONS: command_line_updated.append(key) message(ff(" command line specifies {key} = {value}")) globals()[key] = value # save new config if use_config_files and command_line_updated: if options.save_config_host: message( ff(" saving command-line settings to {CONFIG_FILE} [{hostname}]" )) if not parser.has_section(hostname): parser.add_section(hostname) for key in command_line_updated: parser.set(hostname, key, _set_config_value(key)) if options.save_config_session: if not parser.has_section(session): parser.add_section(session) message( ff(" saving command-line settings to {CONFIG_FILE} [{session}]" )) for key in command_line_updated: parser.set(session, key, _set_config_value(key)) if options.save_config_host or options.save_config_session: radiopadre_dir = make_radiopadre_dir() with open(CONFIG_FILE + ".new", "w") as configfile: if not config_exists: message(ff(" creating new config file {CONFIG_FILE}")) if not parser.has_section('global defaults'): configfile.write( "[global defaults]\n# defaults that apply to all sessions go here\n\n" ) parser.write(configfile) configfile.write("\n\n## default settings follow\n") for key, value in DefaultConfig.items(): configfile.write("# {} = {}\n".format(key.lower(), value)) # if successful, rename files if config_exists: if os.path.exists(CONFIG_FILE + ".old"): os.unlink(CONFIG_FILE + ".old") os.rename(CONFIG_FILE, CONFIG_FILE + ".old") os.rename(CONFIG_FILE + ".new", CONFIG_FILE) message(ff("saved updated config to {CONFIG_FILE}"))
def check_recent_sessions(options, argv, parser=None): """ Loads a recent session if requested :param options: Options object from ArgumentParser :param argv: Argument list (from sys.argv[1:] initially) :param parser: ArgumentParser object used to (re)parse the options :return: options, argv Where options and argv may have been loaded from the recent options file """ make_radiopadre_dir() # load history try: readline.read_history_file(HISTORY_FILE) except IOError: pass resume_session = None # a single-digit argument resumes session #N if len(options.arguments) == 1 and re.match("^\d$", options.arguments[0]): resume_session = int(options.arguments[0]) # no arguments is resume session #0 elif not options.arguments: resume_session = 0 if resume_session is not None: last = _load_recent_sessions() num_recent = len(last) if resume_session >= num_recent: bye(ff("no recent session #{resume_session}")) message("Your most recent radiopadre sessions are:") message("") for i, (_, opts) in enumerate(list(last.items())[::-1]): message(" [#{0}] {1}".format(i, opts), color="GREEN") message("") print( "\nInteractive startup mode. Edit arguments and press Enter to run, or Ctrl+C to bail out. " ) print( " (Ctrl+U + <NUM> + Enter will paste other recent session arguments from the list above)\n" ) inp = None cmdline = '' readline.set_startup_hook(lambda: readline.insert_text(cmdline)) while inp is None: # form up list of fake args to be re-parsed for the last session cmdline = list(last.items())[-(resume_session + 1)][1] # non-persisting options raised in command line shall be appended to the fake args for opt in config.NON_PERSISTING_OPTIONS: if opt.startswith("--") and getattr(options, opt[2:].replace( "-", "_"), None): cmdline += " " + opt cmdline += " " ## colors confuse Ctrl+U and such # prompt = ff("{logger.Colors.GREEN}[#{resume_session}]:{logger.Colors.ENDC} ") prompt = ff("[#{resume_session}] ") inp = INPUT(prompt) inp = inp.strip() if not inp: resume_session = 0 inp = None elif re.match("^\d+$", inp): res = int(inp) if res >= num_recent: warning(ff("no recent session #{res}")) else: resume_session = res readline.remove_history_item(1) inp = None readline.set_startup_hook(None) global _last_input _last_input = inp argv = shlex.split(inp, posix=False) options = parser.parse_args(argv) return options, argv
def update_installation(): # are we already running inside a virtualenv? Proceed directly if so # (see https://stackoverflow.com/questions/1871549/determine-if-python-is-running-inside-virtualenv) # pip install command with -v repeated for each VERBOSE increment pip_install = "pip install " + "-v " * min(max(config.VERBOSE - 1, 0), 3) if hasattr(sys, 'real_prefix') or (hasattr(sys, 'base_prefix') and sys.base_prefix != sys.prefix): if sys.prefix == config.RADIOPADRE_VENV: message( ff("Running inside radiopadre virtual environment {sys.prefix}" )) else: message( ff("Running inside non-default virtual environment {sys.prefix}" )) message(ff("Will assume radiopadre has been installed here.")) config.RADIOPADRE_VENV = sys.prefix if config.VENV_REINSTALL: bye("Can't --venv-reinstall from inside the virtualenv itself.") # Otherwise check for virtualenv, nuke/remake one if needed, then activate it else: config.RADIOPADRE_VENV = os.path.expanduser(config.RADIOPADRE_VENV) activation_script = os.path.join(config.RADIOPADRE_VENV, "bin/activate_this.py") # see if a reinstall is needed if config.AUTO_INIT and config.VENV_REINSTALL and os.path.exists( config.RADIOPADRE_VENV): if not os.path.exists(activation_script): error(ff("{activation_script} does not exist. Bat country!")) bye( ff("Refusing to touch this virtualenv. Please remove it by hand if you must." )) cmd = ff("rm -fr {config.RADIOPADRE_VENV}") warning(ff("Found a virtualenv in {config.RADIOPADRE_VENV}.")) warning("However, --venv-reinstall was specified. About to run:") warning(" " + cmd) if config.FULL_CONSENT: warning( "--full-consent given, so not asking for confirmation.") else: warning(ff("Your informed consent is required!")) inp = INPUT( ff("Please enter 'yes' to rm -fr {config.RADIOPADRE_VENV}: " )).strip() if inp != "yes": bye(ff("'{inp}' is not a 'yes'. Phew!")) message("OK, nuking it!") shell(cmd) new_venv = False if not os.path.exists(config.RADIOPADRE_VENV): if config.AUTO_INIT: message(ff("Creating virtualenv {config.RADIOPADRE_VENV}")) shell(ff("virtualenv -p python3 {config.RADIOPADRE_VENV}")) new_venv = True else: error( ff("Radiopadre virtualenv {config.RADIOPADRE_VENV} doesn't exist." )) bye(ff("Try re-running with --auto-init to reinstall it.")) message( ff(" Activating the radiopadre virtualenv via {activation_script}" )) with open(activation_script) as f: code = compile(f.read(), activation_script, 'exec') exec(code, dict(__file__=activation_script), {}) if new_venv: extras = config.VENV_EXTRAS.split( ",") if config.VENV_EXTRAS else [] # add numpy explicitly to quicken up pyregion install extras.append("numpy") if extras: extras = " ".join(extras) message(ff("Installing specified extras: {extras}")) shell(ff("{pip_install} {extras}")) # now check for a radiopadre install inside the venv have_install = check_output("pip show radiopadre") if have_install: install_info = dict( [x.split(": ", 1) for x in have_install.split("\n") if ': ' in x]) version = install_info.get("Version", "unknown") if config.UPDATE: warning( ff("radiopadre (version {version}) is installed, but --update specified." )) else: message(ff("radiopadre (version {version}) is installed.")) if not have_install or config.UPDATE: if config.SERVER_INSTALL_PATH and os.path.exists( config.SERVER_INSTALL_PATH): message( ff("--server-install-path {config.SERVER_INSTALL_PATH} is configured and exists." )) update_server_from_repository() install = ff("-e {config.SERVER_INSTALL_PATH}") elif config.SERVER_INSTALL_REPO: if config.SERVER_INSTALL_REPO == "default": config.SERVER_INSTALL_REPO = config.DEFAULT_SERVER_INSTALL_REPO branch = config.SERVER_INSTALL_BRANCH or "master" if config.SERVER_INSTALL_PATH: message( ff("--server-install-path and --server-install-repo configured, will clone and install" )) cmd = ff( "git clone -b {branch} {config.SERVER_INSTALL_REPO} {config.SERVER_INSTALL_PATH}" ) message(ff("Running {cmd}")) shell(cmd) install = ff("-e {config.SERVER_INSTALL_PATH}") else: message( ff("only --server-install-repo specified, will install directly from git" )) install = ff("git+{config.SERVER_INSTALL_REPO}@{branch}") elif config.SERVER_INSTALL_PIP: message( ff("--server-install-pip {config.SERVER_INSTALL_PIP} is configured." )) install = config.SERVER_INSTALL_PIP else: bye("no radiopadre installation method specified (see --server-install options)" ) cmd = ff("{pip_install} -U {install}") message(ff("Running {cmd}")) shell(cmd)
def start_session(container_name, selected_ports, userside_ports, notebook_path, browser_urls): from iglesia import ROOTDIR from radiopadre_client.server import JUPYTER_OPTS # get hostname os.environ["HOSTNAME"] = subprocess.check_output("/bin/hostname").decode() jupyter_port = selected_ports[0] userside_http_port = userside_ports[2] if config.NBCONVERT: JUPYTER_OPTS.append(notebook_path) else: JUPYTER_OPTS += [ ff("--port={jupyter_port}"), "--no-browser", "--browser=/dev/null" ] # --no-browser alone seems to be ignored if config.INSIDE_CONTAINER_PORTS or config.CONTAINER_TEST: JUPYTER_OPTS += ["--allow-root", "--ip=0.0.0.0"] # if LOAD_NOTEBOOK: # JUPYTER_OPTS.append(LOAD_NOTEBOOK if type(LOAD_NOTEBOOK) is str else LOAD_NOTEBOOK[0]) # pass configured ports to radiopadre kernel os.environ['RADIOPADRE_SELECTED_PORTS'] = ":".join(map( str, selected_ports)) os.environ['RADIOPADRE_USERSIDE_PORTS'] = ":".join(map( str, userside_ports)) # get base path of radiopadre install radiopadre_base = subprocess.check_output( ff(""". {config.RADIOPADRE_VENV}/bin/activate && \ python -c "import importlib; print(importlib.find_loader('radiopadre').get_filename())" """ ), shell=True).decode().strip() radiopadre_base = os.path.dirname(os.path.dirname(radiopadre_base)) message( ff("Detected radiopadre directory within virtualenv as {radiopadre_base}" )) # default JS9 dir goes off the virtualenv os.environ.setdefault("RADIOPADRE_JS9_DIR", ff("{config.RADIOPADRE_VENV}/js9-www")) iglesia.init_helpers(radiopadre_base, verbose=config.VERBOSE > 0, run_js9=not config.NBCONVERT, run_carta=not config.NBCONVERT) ## start jupyter process jupyter_path = config.RADIOPADRE_VENV + "/bin/jupyter" message("Starting: {} {} in {}".format(jupyter_path, " ".join(JUPYTER_OPTS), os.getcwd())) notebook_proc = subprocess.Popen([jupyter_path] + JUPYTER_OPTS, stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, bufsize=1, universal_newlines=True, env=os.environ) ## use this instead to debug the sessison #notebook_proc = subprocess.Popen([config.RADIOPADRE_VENV+"/bin/ipython"], # stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr, # env=os.environ) if config.NBCONVERT: message("Waiting for conversion to finish") notebook_proc.wait() else: iglesia.register_helpers(notebook_proc) # launch browser if browser_urls: iglesia.register_helpers(*run_browser(*browser_urls)) # elif not config.REMOTE_MODE_PORTS and not config.INSIDE_CONTAINER_PORTS: # message("Please point your browser to {}".format(" ".join(browser_urls))) # pause to let the Jupyter server spin up wait = await_server_startup(jupyter_port, init_wait=0, process=notebook_proc) if wait is None: if notebook_proc.returncode is not None: bye( ff("jupyter unexpectedly exited with return code {notebook_proc.returncode}" )) bye( ff("unable to connect to jupyter notebook server on port {jupyter_port}" )) message( ff("The jupyter notebook server is running on port {jupyter_port} (after {wait:.2f} secs)" )) if config.CONTAINER_TEST: message(ff("--container-test was specified, dry run is complete")) sys.exit(0) try: while True: if config.INSIDE_CONTAINER_PORTS: debug("inside container -- sleeping indefinitely") time.sleep(100000) else: a = INPUT("Type 'exit' to kill the session: ") if notebook_proc.poll() is not None: message("The notebook server has exited with code {}". format(notebook_proc.poll())) sys.exit(0) if a.lower() == 'exit': message("Exit request received") sys.exit(0) except BaseException as exc: if type(exc) is KeyboardInterrupt: message("Caught Ctrl+C") status = 1 elif type(exc) is EOFError: message("Input channel has closed") status = 1 elif type(exc) is SystemExit: status = getattr(exc, 'code', 0) message("Exiting with status {}".format(status)) else: message("Caught exception {} ({})".format(exc, type(exc))) status = 1 sys.exit(status)
def run_remote_session(command, copy_initial_notebook, notebook_path, extra_arguments): SSH_MUX_OPTS = "-o ControlPath=/tmp/ssh_mux_radiopadre_%C -o ControlMaster=auto -o ControlPersist=1h".split( ) SCP_OPTS = ["scp"] + SSH_MUX_OPTS # SSH_OPTS = ["ssh", "-t"] + SSH_MUX_OPTS + [config.REMOTE_HOST] SSH_OPTS = ["ssh"] + SSH_MUX_OPTS + [config.REMOTE_HOST] # See, possibly: https://stackoverflow.com/questions/44348083/how-to-send-sigint-ctrl-c-to-current-remote-process-over-ssh-without-t-optio # master ssh connection, to be closed when we exit message( ff("Opening ssh connection to {config.REMOTE_HOST}. You may be prompted for your password." )) debug(" {}".format(" ".join(SSH_OPTS))) ssh_master = subprocess.check_call(SSH_OPTS + ["exit"], stderr=DEVNULL) # raw_input("Continue?") def help_yourself(problem, suggestion=None): """ Prints a "help yourself" message and exits """ message("{}".format(problem)) message( ff("Please ssh {config.REMOTE_HOST} and sort it out yourself, then rerun this script" )) if suggestion: message(ff("({suggestion})")) sys.exit(1) def ssh_remote(command, fail_retcode=None, stderr=DEVNULL): """Runs command on remote host. Returns its output if the exit status is 0, or None if the exit status matches fail_retcode. Any other non-zero exit status (or any other error) will result in an exception. """ try: return subprocess.check_output(SSH_OPTS + [command], stderr=stderr).decode('utf-8') except subprocess.CalledProcessError as exc: if exc.returncode == fail_retcode: return None message(ff("ssh {command} failed with exit code {exc.returncode}")) raise def ssh_remote_v(command, fail_retcode=None): return ssh_remote(command, fail_retcode, stderr=sys.stderr) def ssh_remote_interactive(command, fail_retcode=None): """Runs command on remote host. Returns the exit status if 0, or None if the exit status matches fail_retcode. Any other non-zero exit status (or any other error) will result in an exception. """ try: return subprocess.check_call(SSH_OPTS + [command]) except subprocess.CalledProcessError as exc: if exc.returncode == fail_retcode: return None message(ff("ssh {command} failed with exit code {exc.returncode}")) raise def scp_to_remote(path, remote_path): return subprocess.check_output( SCP_OPTS + [path, "{}:{}".format(config.REMOTE_HOST, remote_path)]) def check_remote_file(remote_file, test="-x"): """ Checks that a remote file exists. 'test' is specified bash-style, e.g. "-x" for executable. Can also use -f and -f, for example. Returns True or False, or raises an exception on other errors. """ return ssh_remote( "if [ {} {} ]; then exit 0; else exit 199; fi".format( test, remote_file), fail_retcode=199, stderr=DEVNULL) is not None def check_remote_command(command): """ Checks that remote host has a particular command available (by running 'which' on the remote). Returns True or False, or raises an exception on other errors. """ if config.SKIP_CHECKS: return command return (ssh_remote("which " + command, fail_retcode=1, stderr=DEVNULL) or "").strip() # --update or --auto-init disables --skip-checks if config.SKIP_CHECKS: if config.UPDATE: message("Note that --update implies --no-skip-checks") config.SKIP_CHECKS = False elif config.AUTO_INIT: message("Note that --auto-init implies --no-skip-checks") config.SKIP_CHECKS = False # propagate our config to command-line arguments remote_config = config.get_config_dict() remote_config['BROWSER'] = 'None' remote_config['SKIP_CHECKS'] = False remote_config['VENV_REINSTALL'] = False # Check for various remote bits if config.VERBOSE and not config.SKIP_CHECKS: message(ff("Checking installation on {config.REMOTE_HOST}.")) has_git = check_remote_command("git") USE_VENV = has_singularity = has_docker = None for backend in config.BACKEND: remote_config["BACKEND"] = backend if backend == "venv" and check_remote_command( "virtualenv") and check_remote_command("pip"): USE_VENV = True break elif backend == "docker": has_docker = check_remote_command("docker") if has_docker: break elif backend == "singularity": has_singularity = check_remote_command("singularity") if has_singularity: break message( ff("The '{backend}' back-end is not available on {config.REMOTE_HOST}, skipping." )) else: bye( ff("None of the specified back-ends are available on {config.REMOTE_HOST}." )) if remote_config["BACKEND"] != "docker": config.CONTAINER_PERSIST = config.CONTAINER_DEBUG = False # which runscript to look for runscript0 = "run-radiopadre" # form up remote venv path, but do not expand ~ at this point (it may be a different username on the remote) env = os.environ.copy() env.setdefault("RADIOPADRE_DIR", config.REMOTE_RADIOPADRE_DIR or "~/.radiopadre") config.RADIOPADRE_VENV = (config.RADIOPADRE_VENV or "").format(**env) # this variable used in error and info messages remote_venv = ff("{config.REMOTE_HOST}:{config.RADIOPADRE_VENV}") # pip install command with -v repeated for each VERBOSE increment pip_install = "pip install " + "-v " * min(max(config.VERBOSE - 1, 0), 3) # do we want to do an install/update -- will be forced to True (if we must install), # or False if we can't update do_update = config.UPDATE if config.SKIP_CHECKS: runscript = ff( "if [ -f {config.RADIOPADRE_VENV}/bin/activate ]; then " + "source {config.RADIOPADRE_VENV}/bin/activate; fi; run-radiopadre " ) do_update = False else: runscript = None # (a) if --auto-init and --venv-reinstall specified, zap remote virtualenv if present if config.AUTO_INIT and config.VENV_REINSTALL: if not config.RADIOPADRE_VENV: bye( ff("Can't do --auto-init --venv-reinstall because --radiopadre-venv is not set" )) if "~" in config.RADIOPADRE_VENV: config.RADIOPADRE_VENV = ssh_remote( ff("echo {config.RADIOPADRE_VENV}")).strip( ) # expand "~" on remote if check_remote_file(ff("{config.RADIOPADRE_VENV}"), "-d"): if not check_remote_file( ff("{config.RADIOPADRE_VENV}/bin/activate"), "-f"): error( ff("{remote_venv}/bin/activate} does not exist. Bat country!" )) bye( ff("Refusing to touch this virtualenv. Please remove it by hand if you must." )) cmd = ff("rm -fr {config.RADIOPADRE_VENV}") warning(ff("Found a virtualenv in {remote_venv}.")) warning( "However, --venv-reinstall was specified. About to run:") warning(ff(" ssh {config.REMOTE_HOST} " + cmd)) if config.FULL_CONSENT: warning( "--full-consent given, so not asking for confirmation." ) else: warning(ff("Your informed consent is required!")) inp = INPUT( ff("Please enter 'yes' to rm -fr {remote_venv}: ") ).strip() if inp != "yes": bye(ff("'{inp}' is not a 'yes'. Phew!")) message("OK, nuking it!") ssh_remote(cmd) # force update do_update = True # (b) look inside venv if runscript is None and config.RADIOPADRE_VENV: if "~" in config.RADIOPADRE_VENV: config.RADIOPADRE_VENV = ssh_remote( ff("echo {config.RADIOPADRE_VENV}")).strip( ) # expand "~" on remote if check_remote_file(ff("{config.RADIOPADRE_VENV}/bin/activate"), "-f"): if ssh_remote(ff( "source {config.RADIOPADRE_VENV}/bin/activate && which {runscript0}" ), fail_retcode=1): runscript = ff( "source {config.RADIOPADRE_VENV}/bin/activate && {runscript0}" ) message( ff("Using remote client script within {config.RADIOPADRE_VENV}" )) else: message( ff("Remote virtualenv {config.RADIOPADRE_VENV} exists, but does not contain a radiopadre-client installation." )) else: message(ff("No remote virtualenv found at {remote_venv}")) # (c) just try `which` directly if runscript is None: runscript = check_remote_command(runscript0) if runscript: message(ff("Using remote client script at {runscript}")) runscript = ff() do_update = False if config.UPDATE: warning( ff("ignoring --update for client since it isn't in a virtualenv" )) else: message(ff("No remote client script {runscript0} found")) runscript = None ## No runscript found on remote? ## First, figure out whether to reinstall a virtualenv for it if not runscript: message(ff("No {runscript0} script found on {config.REMOTE_HOST}")) if not config.AUTO_INIT: bye( ff("Try re-running with --auto-init to install radiopadre-client on {config.REMOTE_HOST}." )) if not config.RADIOPADRE_VENV: bye(ff("Can't do --auto-init because --virtual-env is not set.")) message("Trying to --auto-init an installation for you...") # try to auto-init a virtual environment if not check_remote_file(ff("{config.RADIOPADRE_VENV}/bin/activate"), "-f"): message(ff("Creating virtualenv {remote_venv}")) ssh_remote_v(ff("virtualenv -p python3 {config.RADIOPADRE_VENV}")) extras = config.VENV_EXTRAS.split( ",") if config.VENV_EXTRAS else [] # add numpy explicitly to quicken up pyregion install extras.append("numpy") if extras: extras = " ".join(extras) message(ff("Installing specified extras: {extras}")) ssh_remote_v( ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} {extras}" )) else: message(ff("Installing into existing virtualenv {remote_venv}")) # Now, figure out how to install or update the client package if not runscript or do_update: # installing from a specified existing path if config.CLIENT_INSTALL_PATH and check_remote_file( config.CLIENT_INSTALL_PATH, "-d"): install_path = config.CLIENT_INSTALL_PATH message( ff("--client-install-path {install_path} is configured and exists on {config.REMOTE_HOST}." )) # update if managed by git if check_remote_file(ff("{install_path}/.git"), "-d") and config.UPDATE: if has_git: if config.CLIENT_INSTALL_BRANCH: cmd = ff( "cd {install_path} && git fetch origin && git checkout {config.CLIENT_INSTALL_BRANCH} && git pull" ) else: cmd = ff("cd {install_path} && git pull") warning( ff("--update specified and git detected, will attempt to update via" )) message(ff(" {cmd}")) ssh_remote_v(cmd) else: warning( ff("--update specified, but no git command found on {config.REMOTE_HOST}" )) install_path = "-e " + install_path # else, installing from git elif config.CLIENT_INSTALL_REPO: if config.CLIENT_INSTALL_REPO == "default": config.CLIENT_INSTALL_REPO = config.DEFAULT_CLIENT_INSTALL_REPO branch = config.CLIENT_INSTALL_BRANCH or "master" if config.CLIENT_INSTALL_PATH: message( ff("--client-install-path and --client-install-repo configured, will attempt" )) cmd = ff( "git clone -b {branch} {config.CLIENT_INSTALL_REPO} {config.CLIENT_INSTALL_PATH}" ) message(ff(" ssh {config.REMOTE_HOST} {cmd}")) ssh_remote_v(cmd) install_path = ff("-e {config.CLIENT_INSTALL_PATH}") else: message( ff("--client-install-repo is configured, will try to install directly from git" )) install_path = ff("git+{config.CLIENT_INSTALL_REPO}@{branch}") # now pip install message( ff("Doing pip install -e {install_path} in {config.RADIOPADRE_VENV}" )) ssh_remote_v( ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} -e {install_path}" )) # else, installing directly from pip elif config.CLIENT_INSTALL_PIP: message( ff("--client-install-pip {config.CLIENT_INSTALL_PIP} is configured." )) install_path = config.CLIENT_INSTALL_PIP else: bye("no radiopadre-client installation method specified (see --client-install options)" ) # now install message( ff("Will attempt to pip install -U {install_path} in {remote_venv}" )) ssh_remote_v( ff("source {config.RADIOPADRE_VENV}/bin/activate && {pip_install} -U {install_path}" )) # sanity check if ssh_remote(ff( "source {config.RADIOPADRE_VENV}/bin/activate && which {runscript0}" ), fail_retcode=1): runscript = ff( "source {config.RADIOPADRE_VENV}/bin/activate && {runscript0}") else: bye( ff("Something went wrong during installation on {config.REMOTE_HOST}, since I still don't see the {runscript0} script" )) message("Success!") runscript = ff( "export RADIOPADRE_DIR={config.REMOTE_RADIOPADRE_DIR}; {runscript}") # copy notebook to remote if copy_initial_notebook: if not os.path.exists(copy_initial_notebook): bye("{} doesn't exist".format(copy_initial_notebook)) if check_remote_file(notebook_path or ".", "-d"): nbpath = "{}/{}".format(notebook_path or ".", copy_initial_notebook) if check_remote_file(nbpath, "-ff("): message( ff("remote notebook {nbpath} exists, will not copy over")) else: message( ff("remote notebook {nbpath} doesn't exist, will copy over" )) scp_to_remote(copy_initial_notebook, notebook_path) notebook_path = nbpath # allocate 5 suggested ports (in resume mode, this will be overridden by the session settings) starting_port = 10000 + os.getuid() * 3 ports = [] for _ in range(5): starting_port = find_unused_port(starting_port + 1, 10000) ports.append(starting_port) remote_config["remote"] = ":".join(map(str, ports)) # turn the remote_config dict into a command line runscript += " " + " ".join( config.get_options_list(remote_config, quote=True)) runscript += " '{}' {}".format( command if command is not "load" else notebook_path, " ".join(extra_arguments)) # start ssh subprocess to launch notebook args = list(SSH_OPTS) + ["shopt -s huponexit && " + runscript] if config.VERBOSE: message("running {}".format(" ".join(args))) else: message(ff("running radiopadre client on {config.REMOTE_HOST}")) ssh = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=1, universal_newlines=True) poller = Poller() poller.register_process(ssh, config.REMOTE_HOST, config.REMOTE_HOST + " stderr") if not USE_VENV: poller.register_file(sys.stdin, "stdin") container_name = None urls = [] remote_running = False status = 0 try: while remote_running is not None and poller.fdlabels: fdlist = poller.poll(verbose=config.VERBOSE > 1) for fname, fobj in fdlist: try: line = fobj.readline() except EOFError: line = b'' empty_line = not line line = (line.decode('utf-8') if type(line) is bytes else line).rstrip() if fobj is sys.stdin and line == 'D' and config.CONTAINER_PERSIST: sys.exit(0) # break out if ssh closes if empty_line: poller.unregister_file(fobj) if ssh.stdout not in poller and ssh.stderr not in poller: message( ff("The ssh process to {config.REMOTE_HOST} has exited" )) remote_running = None break continue # print remote output print_output = False if fobj is ssh.stderr: print_output = not line.startswith("Shared connection to") else: print_output = not line.startswith( "radiopadre:") or command != 'load' if not empty_line and (config.VERBOSE or print_output): for key, dispatch in _dispatch_message.items(): if key in line: dispatch(u"{}: {}".format(fname, line)) break else: message(u"{}: {}".format(fname, line)) if not line: continue # if remote is not yet started, check output if not remote_running: # check for session ID match = re.match( ".*Session ID/notebook token is '([0-9a-f]+)'", line) if match: config.SESSION_ID = match.group(1) continue # check for notebook port, and launch second ssh when we have it re_ports = ":".join(["([\\d]+)"] * 10) # form up regex for ddd:ddd:... match = re.match(ff(".*Selected ports: {re_ports}[\s]*$"), line) if match: ports = list(map(int, match.groups())) remote_ports = ports[:5] local_ports = ports[5:] if config.VERBOSE: message( "Detected ports {}:{}:{}:{}:{} -> {}:{}:{}:{}:{}" .format(*ports)) ssh2_args = ["ssh"] + SSH_MUX_OPTS + [ "-O", "forward", config.REMOTE_HOST ] for loc, rem in zip(local_ports, remote_ports): ssh2_args += [ "-L", "localhost:{}:localhost:{}".format(loc, rem) ] # tell mux process to forward the ports if config.VERBOSE: message( "sending forward request to ssh mux process". format(ssh2_args)) subprocess.call(ssh2_args) continue # check for launch URL match = re.match(".*Browse to URL: ([^\s\033]+)", line) if match: urls.append(match.group(1)) continue # check for container name match = re.match(".*Container name: ([^\s\033]+)", line) if match: container_name = match.group(1) continue if "jupyter notebook server is running" in line: remote_running = True time.sleep(1) iglesia.register_helpers(*run_browser(*urls)) message( "The remote radiopadre session is now fully up") if USE_VENV or not config.CONTAINER_PERSIST: message("Press Ctrl+C to kill the remote session") else: message( "Press D<Enter> to detach from remote session, or Ctrl+C to kill it" ) except SystemExit as exc: message(ff("SystemExit: {exc.code}")) status = exc.code except KeyboardInterrupt: message("Ctrl+C caught") status = 1 except Exception as exc: traceback.print_exc() message("Exception caught: {}".format(str(exc))) if remote_running and ssh.poll() is None: message("Asking remote session to exit, nicely") try: try: ssh.stdin.write("exit\n") except TypeError: ssh.stdin.write(b"exit\n") # because f**k you python except IOError: debug(" looks like it's already exited") # if status and not USE_VENV and container_name: # message(ff("killing remote container {container_name}")) # try: # if has_docker: # ssh_remote(ff("{has_docker} kill {container_name}")) # elif has_singularity: # from .backends.singularity import get_singularity_image # singularity_image = get_singularity_image(config.DOCKER_IMAGE) # ssh_remote(ff("{has_singularity} instance.stop {singularity_image} {container_name}")) # except subprocess.CalledProcessError as exc: # message(exc.output.decode()) for i in range(10, 0, -1): if ssh.poll() is not None: debug("Remote session has exited") ssh.wait() break message(ff("Waiting for remote session to exit ({i})")) time.sleep(1) else: message(ff("Remote session hasn't exited, killing the ssh process")) ssh.kill() return status
def kill_container(name): message(ff("Killing container {name}")) shell(ff("{docker} kill {name}"), ignore_fail=True)
def update_installation(rebuild=False, docker_pull=True): global docker_image global singularity_image docker_image = config.DOCKER_IMAGE singularity_image = os.path.expanduser(get_singularity_image(docker_image)) # this will be True if we need to build the image build_image = False # clearly true if no image if not os.path.exists(singularity_image): if config.SINGULARITY_AUTO_BUILD: message(ff("Singularity image {singularity_image} does not exist")) build_image = True else: error( ff("Singularity image {singularity_image} does not exist, and auto-build is disabled" )) bye(ff("Re-run with --singularity-auto-build to proceed")) # also true if rebuild forced by flags or config or command line elif rebuild or config.SINGULARITY_REBUILD: config.SINGULARITY_AUTO_BUILD = build_image = True message( ff("--singularity-rebuild specified, removing singularity image {singularity_image}" )) # pull down docker image first if has_docker and docker_pull: message( "Checking docker image (from which our singularity image is built)" ) docker.update_installation(enable_pull=True) # if we're not forced to build yet, check for an update if config.UPDATE and not build_image: if has_docker: # check timestamp of docker image docker_image_time = None output = check_output( ff("{has_docker} image inspect {docker_image} -f '{{{{ .Created }}}}'" )).strip() message(ff(" docker image timestamp is {output}")) # in Python 3.7 we have datetime.fromisoformat(date_string), but for now we muddle: match = output and re.match("(^.*)[.](\d+)Z", output) if match: try: docker_image_time = calendar.timegm( time.strptime(match.group(1), "%Y-%m-%dT%H:%M:%S")) docker_image_time = datetime.datetime.utcfromtimestamp( docker_image_time) except ValueError: pass sing_image_time = datetime.datetime.utcfromtimestamp( os.path.getmtime(singularity_image)) message(" singularity image timestamp is {}".format( sing_image_time.isoformat())) if docker_image_time is None: warning( ff("can't parse docker image timestamp '{output}', rebuilding {singularity_image} just in case" )) build_image = True elif docker_image_time > sing_image_time: warning( ff("rebuilding outdated singularity image {singularity_image}" )) build_image = True else: message( ff("singularity image {singularity_image} is up-to-date")) else: message( ff("--update specified but no docker access, assuming {singularity_image} is up-to-date" )) # now build if needed if build_image: warning( ff("Rebuilding singularity image from docker://{docker_image}")) warning(ff(" (This may take a few minutes....)")) singularity_image_new = singularity_image + ".new.img" if os.path.exists(singularity_image_new): os.unlink(singularity_image_new) cmd = [ singularity, "build", singularity_image_new, ff("docker://{docker_image}") ] message("running " + " ".join(cmd)) try: subprocess.check_call(cmd) except subprocess.CalledProcessError as exc: if config.IGNORE_UPDATE_ERRORS: if os.path.exists(singularity_image): warning( "singularity build failed but --ignore-update-errors is set, proceeding with old image" ) else: error( "singularity build failed, --ignore-update-errors is set, but we have no older image" ) raise else: raise # move old image message(ff("Build successful, renaming to {singularity_image}")) os.rename(singularity_image_new, singularity_image) else: message( ff("Using existing radiopadre singularity image {singularity_image}" )) # not supported with Singularity config.CONTAINER_PERSIST = False
def start_session(container_name, selected_ports, userside_ports, notebook_path, browser_urls): from iglesia import ABSROOTDIR, LOCAL_SESSION_DIR, SHADOW_SESSION_DIR, SNOOP_MODE radiopadre_dir = make_radiopadre_dir() docker_local = make_dir(radiopadre_dir + "/.docker-local") js9_tmp = make_dir(radiopadre_dir + "/.js9-tmp") session_info_dir = get_session_info_dir(container_name) message( ff("Container name: {container_name}")) # remote script will parse it docker_opts = [ docker, "run", "--rm", "--name", container_name, "--cap-add=SYS_ADMIN", "-w", ABSROOTDIR, "--user", "{}:{}".format(os.getuid(), os.getgid()), "-e", "USER={}".format(os.environ["USER"]), "-e", "HOME={}".format(os.environ["HOME"]), "-e", "RADIOPADRE_DIR={}".format(radiopadre_dir), "-e", ff("RADIOPADRE_CONTAINER_NAME={container_name}"), "-e", ff("RADIOPADRE_SESSION_ID={config.SESSION_ID}"), ] # enable detached mode if not debugging, and also if not doing conversion non-interactively if not config.CONTAINER_DEBUG and not config.NBCONVERT: docker_opts.append("-d") for port1, port2 in zip(selected_ports, CONTAINER_PORTS): docker_opts += ["-p", "{}:{}/tcp".format(port1, port2)] container_ports = list(CONTAINER_PORTS) # setup mounts for work dir and home dir, if needed homedir = os.path.expanduser("~") docker_opts += [ "-v", "{}:{}{}".format(ABSROOTDIR, ABSROOTDIR, ":ro" if SNOOP_MODE else ""), "-v", "{}:{}".format(homedir, homedir), "-v", "{}:{}".format(radiopadre_dir, radiopadre_dir), ## hides /home/user/.local, which can confuse jupyter and ipython ## into seeing e.g. kernelspecs that they should not see "-v", "{}:{}/.local".format(docker_local, homedir), # mount session info directory (needed to serve e.g. js9prefs.js) "-v", "{}:{}".format(session_info_dir, LOCAL_SESSION_DIR), "-v", "{}:{}".format(session_info_dir, SHADOW_SESSION_DIR), # mount a writeable tmp dir for the js9 install -- needed by js9helper "-v", "{}:/.radiopadre/venv/js9-www/tmp".format(js9_tmp), "--label", "radiopadre.user={}".format(USER), "--label", "radiopadre.dir={}".format(os.getcwd()), ] if config.CONTAINER_DEV: if os.path.isdir(SERVER_INSTALL_PATH): docker_opts += ["-v", "{}:/radiopadre".format(SERVER_INSTALL_PATH)] if os.path.isdir(CLIENT_INSTALL_PATH): docker_opts += [ "-v", "{}:/radiopadre-client".format(CLIENT_INSTALL_PATH) ] # add image docker_opts.append(docker_image) # build up command-line arguments docker_opts += _collect_runscript_arguments(container_ports + userside_ports) if notebook_path: docker_opts.append(notebook_path) _run_container(container_name, docker_opts, jupyter_port=selected_ports[0], browser_urls=browser_urls) if config.NBCONVERT: return global running_container running_container = container_name atexit.register(reap_running_container) if config.CONTAINER_PERSIST and config.CONTAINER_DETACH: message("exiting: container session will remain running.") running_container = None # to avoid reaping sys.exit(0) else: if config.CONTAINER_PERSIST: prompt = "Type 'exit' to kill the container session, or 'D' to detach: " else: prompt = "Type 'exit' to kill the container session: " try: while True: a = INPUT(prompt) if a.lower() == 'exit': sys.exit(0) if a.upper( ) == 'D' and config.CONTAINER_PERSIST and container_name: running_container = None # to avoid reaping sys.exit(0) except BaseException as exc: if type(exc) is KeyboardInterrupt: message("Caught Ctrl+C") status = 1 elif type(exc) is SystemExit: status = getattr(exc, 'code', 0) message("Exiting with status {}".format(status)) else: message("Caught exception {} ({})".format(exc, type(exc))) status = 1 # if not status: # running_container = None # to avoid reaping sys.exit(status)
def run_radiopadre_server(command, arguments, notebook_path, workdir=None): global backend # message("Welcome to Radiopadre!") USE_VENV = USE_DOCKER = USE_SINGULARITY = False for backend in config.BACKEND: if backend == "venv" and find_which("virtualenv"): USE_VENV = True import radiopadre_client.backends.venv backend = radiopadre_client.backends.venv backend.init() break elif backend == "docker": has_docker = find_which("docker") if has_docker: USE_DOCKER = True message(ff("Using {has_docker} for container mode")) import radiopadre_client.backends.docker backend = radiopadre_client.backends.docker backend.init(binary=has_docker) break elif backend == "singularity": has_docker = find_which("docker") has_singularity = find_which("singularity") if has_singularity: USE_SINGULARITY = True message(ff("Using {has_singularity} for container mode")) import radiopadre_client.backends.singularity backend = radiopadre_client.backends.singularity backend.init(binary=has_singularity, docker_binary=has_docker) break message(ff("The '{backend}' back-end is not available.")) else: bye(ff("None of the specified back-ends are available.")) # if not None, gives the six port assignments attaching_to_ports = container_name = None # ### ps/ls command if command == 'ps' or command == 'ls': session_dict = backend.list_sessions() num = len(session_dict) message("{} session{} running".format(num, "s" if num != 1 else "")) for i, (id, (name, path, uptime, session_id, ports)) in enumerate(session_dict.items()): print("{i}: id {id}, name {name}, in {path}, up since {uptime}". format(**locals())) sys.exit(0) # ### kill command if command == 'kill': session_dict = backend.list_sessions() if not session_dict: bye("no sessions running, nothing to kill") if arguments[0] == "all": kill_sessions = session_dict.keys() else: kill_sessions = [ backend.identify_session(session_dict, arg) for arg in arguments ] backend.kill_sessions(session_dict, kill_sessions) sys.exit(0) ## attach command if command == "resume": session_dict = backend.list_sessions() if arguments: id_ = backend.identify_session(session_dict, arguments[0]) else: if not session_dict: bye("no sessions running, nothing to attach to") config.SESSION_ID = session_dict.keys()[0] container_name, path, _, _, attaching_to_ports = session_dict[id_] message( ff(" Attaching to existing session {config.SESSION_ID} running in {path}" )) # load command elif command == 'load': attaching_to_ports = None # else unknown command else: bye("unknown command {}".format(command)) running_session_dict = None # ### SETUP LOCAL SESSION PROPERTIES: container_name, session_id, port assignments # REATTACH MODE: everything is read from the session file if attaching_to_ports: # session_id and container_name already set above. Ports read from session file and printed to the console # for the benefit of the remote end (if any) jupyter_port, helper_port, http_port, carta_port, carta_ws_port = selected_ports = attaching_to_ports[: 5] userside_ports = attaching_to_ports[5:] # INSIDE CONTAINER: internal ports are fixed, userside ports are passed in, name is passed in, session ID is read from file elif config.INSIDE_CONTAINER_PORTS: message("started the radiopadre container") container_name = os.environ['RADIOPADRE_CONTAINER_NAME'] config.SESSION_ID = os.environ['RADIOPADRE_SESSION_ID'] selected_ports = config.INSIDE_CONTAINER_PORTS[:5] userside_ports = config.INSIDE_CONTAINER_PORTS[5:] message(" Inside container, using ports {}".format(" ".join( map(str, config.INSIDE_CONTAINER_PORTS)))) # NORMAL MODE: find unused internal ports. Userside ports are passed from remote if in remote mode, or same in local mode else: if not USE_VENV: container_name = "radiopadre-{}-{}".format(config.USER, uuid.uuid4().hex) message(ff("Starting new session in container {container_name}")) # get dict of running sessions (for GRIM_REAPER later) running_session_dict = backend.list_sessions() else: container_name = None message("Starting new session in virtual environment") selected_ports = [find_unused_port(1024)] for i in range(4): selected_ports.append(find_unused_port(selected_ports[-1] + 1)) if config.REMOTE_MODE_PORTS: userside_ports = config.REMOTE_MODE_PORTS else: userside_ports = selected_ports os.environ['RADIOPADRE_SESSION_ID'] = config.SESSION_ID = uuid.uuid4( ).hex # write out session file if container_name: backend.save_session_info(container_name, selected_ports, userside_ports) global userside_jupyter_port # needed for it to be visible to ff() from a list comprehension jupyter_port, helper_port, http_port, carta_port, carta_ws_port = selected_ports userside_jupyter_port, userside_helper_port, userside_http_port, userside_carta_port, userside_carta_ws_port = userside_ports # print port assignments to console -- in remote mode, remote script will parse this out if not config.INSIDE_CONTAINER_PORTS: message(" Selected ports: {}".format(":".join( map(str, selected_ports + userside_ports)))) message(ff(" Session ID/notebook token is '{config.SESSION_ID}'")) if container_name is not None: message(ff(" Container name: {container_name}")) # ### will we be starting a browser? browser = False if config.INSIDE_CONTAINER_PORTS: if config.VERBOSE: message( " Running inside container -- not opening a browser in here.") elif config.REMOTE_MODE_PORTS: if config.VERBOSE: message(" Remote mode -- not opening a browser locally.") elif os.environ.get("SSH_CLIENT"): message("You appear to have logged in via ssh.") message( "You're logged in via ssh, so I'm not opening a web browser for you." ) message( "Please manually browse to the URL printed by Jupyter below. You will probably want to employ ssh" ) message( "port forwarding if you want to browse this notebook from your own machine." ) browser = False else: message("You appear to have a local session.") if not config.BROWSER: message("--no-browser is set, we will not invoke a browser.") message("Please manually browse to the URL printed below.") browser = False else: message( ff("We'll attempt to open a web browser (using '{config.BROWSER}') as needed. Use --no-browser to disable this." )) browser = True # ### ATTACHING TO EXISTING SESSION: complete the attachment and exit if attaching_to_ports: url = ff( "http://localhost:{userside_jupyter_port}/tree#running?token={session_id}" ) # in local mode, see if we need to open a browser. Else just print the URL -- remote script will pick it up if not config.REMOTE_MODE_PORTS and browser: message(ff("driving browser: {config.BROWSER} {url}")) subprocess.call([config.BROWSER, url], stdout=DEVNULL) time.sleep(1) else: message(ff("Browse to URL: {url}"), color="GREEN") # emit message so remote initiates browsing if config.REMOTE_MODE_PORTS: message( "The Jupyter Notebook is running inside the reattached session, presumably" ) if config.VERBOSE: message("sleeping") while True: time.sleep(1000000) sys.exit(0) # ### NEW SESSION: from this point on, we're opening a new session # ### setup working directory and notebook paths global LOAD_DIR global LOAD_NOTEBOOK # if explicit notebook directory is given, change into it before doing anything else if notebook_path: if os.path.isdir(notebook_path): os.chdir(notebook_path) notebook_path = '.' LOAD_DIR = True LOAD_NOTEBOOK = None else: nbdir = os.path.dirname(notebook_path) if nbdir: if not os.path.isdir(nbdir): bye("{} doesn't exist".format(nbdir)) os.chdir(nbdir) notebook_path = os.path.basename(notebook_path) LOAD_DIR = False LOAD_NOTEBOOK = notebook_path else: LOAD_DIR = '.' LOAD_NOTEBOOK = None # message(ff("{LOAD_DIR} {LOAD_NOTEBOOK} {notebook_path}")) # if config.NBCONVERT and not LOAD_NOTEBOOK: bye("a notebook must be specified in order to use --nbconvert") # if using containers (and not inside a container), see if older sessions need to be reaped if running_session_dict and config.GRIM_REAPER: kill_sessions = [] for cont, (_, path, _, sid, _) in running_session_dict.items(): if sid != config.SESSION_ID and os.path.samefile( path, os.getcwd()): message(ff("reaping older session {sid}")) kill_sessions.append(cont) if kill_sessions: backend.kill_sessions(running_session_dict, kill_sessions, ignore_fail=True) # virtual environment os.environ["RADIOPADRE_VENV"] = config.RADIOPADRE_VENV # init paths & environment iglesia.init() iglesia.set_userside_ports(userside_ports) global JUPYTER_OPTS if config.NBCONVERT: JUPYTER_OPTS = [ "nbconvert", "--ExecutePreprocessor.timeout=600", "--no-input", "--to", "html_embed", "--execute" ] os.environ["RADIOPADRE_NBCONVERT"] = "True" else: JUPYTER_OPTS = [ "notebook", "--ContentsManager.pre_save_hook=radiopadre_utils.notebook_utils._notebook_save_hook", "--ContentsManager.allow_hidden=True" ] os.environ.pop("RADIOPADRE_NBCONVERT", None) # update installation etc. backend.update_installation() # (when running natively (i.e. in a virtual environment), the notebook app doesn't pass the token to the browser # command properly... so let it pick its own token then) # if options.remote or options.config.INSIDE_CONTAINER_PORTS or not options.virtual_env: JUPYTER_OPTS += [ ff("--NotebookApp.token='{config.SESSION_ID}'"), ff("--NotebookApp.custom_display_url='http://localhost:{userside_jupyter_port}'" ) ] #=== figure out whether we initialize or load a notebook os.chdir(iglesia.SERVER_BASEDIR) if iglesia.SNOOP_MODE: warning( ff("{iglesia.ABSROOTDIR} is not writable for you, so radiopadre is operating in snoop mode." )) ALL_NOTEBOOKS = glob.glob("*.ipynb") if iglesia.SNOOP_MODE and not ALL_NOTEBOOKS: orig_notebooks = glob.glob(os.path.join(iglesia.ABSROOTDIR, "*.ipynb")) if orig_notebooks: message( " No notebooks in shadow directory: will copy notebooks from target." ) message(" Copying {} notebooks from {}".format( len(orig_notebooks), iglesia.ABSROOTDIR)) for nb in orig_notebooks: shutil.copyfile(nb, './' + os.path.basename(nb)) ALL_NOTEBOOKS = glob.glob("*.ipynb") message(" Available notebooks: " + " ".join(ALL_NOTEBOOKS)) if not config.INSIDE_CONTAINER_PORTS: # if no notebooks in place, see if we need to create a default if not ALL_NOTEBOOKS: if config.DEFAULT_NOTEBOOK: message( ff(" No notebooks yet: will create {config.DEFAULT_NOTEBOOK}" )) LOAD_DIR = True open(config.DEFAULT_NOTEBOOK, 'wt').write(default_notebook_code) ALL_NOTEBOOKS = [config.DEFAULT_NOTEBOOK] else: message( " No notebooks and no default. Displaying directory only." ) LOAD_DIR = True LOAD_NOTEBOOK = None # expand globs and apply auto-load as needed if LOAD_NOTEBOOK: LOAD_NOTEBOOK = [ nb for nb in ALL_NOTEBOOKS if fnmatch.fnmatch(os.path.basename(nb), LOAD_NOTEBOOK) ] elif config.AUTO_LOAD == "1": LOAD_NOTEBOOK = ALL_NOTEBOOKS[0] if ALL_NOTEBOOKS else None message(ff(" Auto-loading {LOAD_NOTEBOOK[0]}.")) elif config.AUTO_LOAD: LOAD_NOTEBOOK = [ nb for nb in ALL_NOTEBOOKS if fnmatch.fnmatch(os.path.basename(nb), config.AUTO_LOAD) ] if LOAD_NOTEBOOK: message(" Auto-loading {}".format(" ".join(LOAD_NOTEBOOK))) else: message( ff(" No notebooks matching --auto-load {config.AUTO_LOAD}" )) urls = [] if LOAD_DIR: urls.append( ff("http://localhost:{userside_jupyter_port}/?token={config.SESSION_ID}" )) if LOAD_NOTEBOOK: urls += [ ff("http://localhost:{userside_jupyter_port}/notebooks/{nb}?token={config.SESSION_ID}" ) for nb in LOAD_NOTEBOOK ] if not config.NBCONVERT: for url in urls[::-1]: message(ff("Browse to URL: {url}"), color="GREEN") if config.CARTA_BROWSER: url = ff( "http://localhost:{iglesia.CARTA_PORT}/?socketUrl=ws://localhost:{iglesia.CARTA_WS_PORT}" ) message(ff("Browse to URL: {url} (CARTA file browser)"), color="GREEN") urls.append(url) # now we're ready to start the session backend.start_session(container_name, selected_ports, userside_ports, notebook_path, browser and urls)
def kill_container(name): singularity_image = get_singularity_image(config.DOCKER_IMAGE) shell(ff("{singularity} instance.stop {singularity_image} {name}"))