def _list(stdscr, args): import pandas as pd from xmen.utils import load_params pd.set_option('display.width', 1000) pd.set_option('display.max_columns', 1000) pd.set_option('display.max_colwidth', args.max_width) pd.set_option('display.max_rows', args.max_rows) if args.pattern is None: pattern = os.getcwd() else: pattern = os.path.abspath(os.path.expanduser(args.pattern[0])) params = os.path.join(pattern, 'params.yml') if os.path.exists(params): # print the params.yml file to screen print(f'Content of {params}') import ruamel.yaml from xmen.utils import recursive_print_lines with open(os.path.join(params), 'r') as params_yml: params = ruamel.yaml.load(params_yml, ruamel.yaml.RoundTripLoader) lines = recursive_print_lines(params) for l in lines: print(l) else: # print(args.pattern) # global_exp_manager = GlobalExperimentManager() config = Config() paths = config.filter(pattern) params = load_params(paths) if args.list: if args.pattern is None: pattern += '.*' args.pattern = pattern from xmen.list import notebook_display, args_to_filters args.filters += [ '_notes', '_created|_timestamps', '_purpose', '_version' ] notebook_display(params, *args_to_filters(args)) elif args.interval is None: if args.pattern is None: args.pattern = '.*' from xmen.utils import load_params from xmen.list import args_to_filters, visualise_params data_frame, root = visualise_params(params, *args_to_filters(args)) print(f'Roots relative to {root}') print(data_frame) else: if args.pattern is None: args.pattern = f'.*' from xmen.utils import load_params from xmen.list import interactive_display interactive_display(stdscr, args)
def send_request_task(q_requests, q_response): import socket from xmen.config import Config import time config = Config() context = ssl.create_default_context() try: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as ss: with context.wrap_socket(ss, server_hostname=config.server_host) as s: s.connect((config.server_host, config.server_port)) while True: try: try: request = q_requests.get(False) except queue.Empty: pass if not request: break send(request, s) response = receive(s) q_response.put(response) time.sleep(0.1) except queue.Empty: pass except (socket.error, IOError): pass
def send_request_task(q_request, q_response=None, hook=None): from xmen.config import Config import time config = Config() context = ssl.create_default_context() with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as ss: with context.wrap_socket(ss, server_hostname=config.server_host) as s: while True: try: s.connect((config.server_host, config.server_port)) except (socket.error, OSError): time.sleep(1.) else: while True: try: request = q_request.get() if not request: return if hook: request = hook(request) send(request, s) response = receive(s) if q_response is not None: q_response.put(response) except queue.Empty: pass except KeyboardInterrupt: pass
def _python(args): config = Config() if args.list is not None: print(f'The following python experiments are currently linked') for k, v in config.python_experiments.items(): print(f'{k}: {v}') if args.add is not None: try: with config as c: c.add_python(*args.add) print( f'Added experiment {args.add[-1]} from module {args.add[-2]}') except Exception as m: print(m) print( f'ERROR: failed to add experiment {args.add[-1]} from module {args.add[-2]}' ) if args.remove is not None: with config as c: if args.remove in c.python_experiments: path = c.python_experiments.pop(args.remove) if '.xmen' in path: os.remove(path) print(f'Successfully removed {args.remove}!') if args.name: import subprocess if args.name[0] not in config.python_experiments: print(f'No experiments found matching {args.name[0]}') exit() args = [config.python_experiments[args.name[0]]] + args.flags subprocess.call(args)
def _save(self, defaults_dir=None): """Save experiment to either a defaults.yml file or a params.yml file depending on its status""" if self._status == DEFAULT: path = os.path.join(os.path.join(defaults_dir, 'defaults.yml')) else: path = os.path.join(self.directory, 'params.yml') if self.status == RUNNING: self._timestamps['last'] = get_time() # save parameters (always) string = self.as_yaml() with open(path, 'w') as file: file.write(string) if self.status not in [DEFAULT, REGISTERED] and not self._is_debug: request = self.to_update_request() for q in self._queues: # get one item and put one item try: q.get(block=False) except queue.Empty: pass q.put(request, block=False) elif self.status == REGISTERED and not self._is_debug: # the global configuration will link with the server... Config().link(self.root)
def send_request(request): from xmen.config import Config config = Config() context = ssl.create_default_context() with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as ss: with context.wrap_socket(ss, server_hostname=config.server_host) as s: s.connect((config.server_host, config.server_port)) send(request, s) response = receive(s) return decode_response(response)
def to_update_request(self): from xmen.config import Config config = Config() data = self.as_yaml() root = f'{config.local_user}@{config.local_host}:{self.root}' return UpdateExperiment(user=config.user, password=config.password, root=root, data=data, status=self.status)
def _relink(args): from xmen.utils import load_param, save_param config = Config() if args.root == '': args.root = os.getcwd() args.root = os.path.abspath(args.root) if args.experiments is None: # relink experiment managers if args.recursive: managers = [ os.path.dirname(p) for p in glob.glob(args.root + '/**/experiment.yml', recursive=True) ] if len(managers) == 0: print( f"No roots found for pattern {args.root + '/**/experiment.yml'}" ) else: managers = [args.root] for manager in managers: experiment_manager = ExperimentManager(manager) experiment_manager.replant(manager) # relink experiments if args.recursive: roots = [ os.path.dirname(p) for p in glob.glob(args.root + '/**/params.yml', recursive=True) ] if len(managers) == 0: print( f"No roots found for pattern {args.root + '/**/params.yml'}" ) else: roots = [args.root] roots = [r for r in roots if r not in config.linked] config.link(roots) config.clean() for r in roots: params = load_param(r) if params['_root'] != r: params['_root'] = r save_param(params, r) else: experiment_manager = ExperimentManager(args.root) experiment_manager.check_initialised() experiment_manager.relink(args.experiments)
def interactive_display(stdscr, args): """interactively display results with various search queries""" import curses import multiprocessing as mp import queue from xmen.utils import dic_from_yml from collections import OrderedDict global root global results global default_pattern global expand_helps global short_root global last_request_time default_pattern = None expand_helps = False short_root = True stdscr.refresh() rows, cols = stdscr.getmaxyx() if rows < 9: raise NotEnoughRows pos_x, pos_y = 0, 0 # load cached results from xmen.config import Config config = Config() results, last_request_time = config.cache(load=True) meta, message = [], [] # initialise colours if curses.has_colors(): curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK) curses.init_pair(2, curses.COLOR_RED, curses.COLOR_BLACK) curses.init_pair(3, curses.COLOR_CYAN, curses.COLOR_BLACK) curses.init_pair(4, curses.COLOR_YELLOW, curses.COLOR_BLACK) curses.init_pair(5, curses.COLOR_GREEN, curses.COLOR_BLACK) curses.init_pair(6, curses.COLOR_MAGENTA, curses.COLOR_BLACK) WHITE = curses.color_pair(1) RED = curses.color_pair(2) CYAN = curses.color_pair(3) YELLOW = curses.color_pair(4) GREEN = curses.color_pair(5) MAGNETA = curses.color_pair(6) global window window = curses.newwin(rows, cols, 0, 0) def generate_table(results, args): dics, roots = results.values(), results.keys() data_frame, root = visualise_params(dics, *args_to_filters(args), roots=roots, short_root=short_root) return data_frame, root def toggle(args, name, update=None): global default_pattern val = getattr(args, name) if name == 'display_meta': if update in meta: meta.remove(update) else: meta.append(update) update = '|'.join(meta) setattr(args, name, update if update else '^$') elif name == 'display_messages': if update in message: message.remove(update) else: message.append(update) update = '|'.join(message) setattr(args, name, update if update else '^$') elif name == 'filters': if update == '': setattr(args, name, [args.filters[0]]) else: setattr(args, name, getattr(args, name) + [update]) elif name == 'filter_params': if update == '': setattr(args, name, [args.filter_params[0]]) else: setattr(args, name, getattr(args, name) + [update]) elif name == 'status_filter': if update == '': setattr(args, name, '[^(deleted)]') else: setattr(args, name, update) elif name == 'pattern': if default_pattern is None: default_patten = args.pattern if update == '': setattr(args, name, default_patten) else: setattr(args, name, update) else: if update is None: update = DEFAULTS[name] if val == '^$': setattr(args, name, update) else: setattr(args, name, '^$') return args def display_row(i, pad, table=None, x=None): from tabulate import tabulate if x is None: x = tabulate(table, headers='keys', tablefmt='github').split('\n') x0 = x[0] xi = x[i] # stdscr.addstr(i + 2, 1, xx) root = [ ii for ii, hh in enumerate(x0.split('|')) if hh.strip() == 'root' ] status = [ ii for ii, hh in enumerate(x0.split('|')) if hh.strip() == 'status' ] if i == 0: xi = xi.replace('|', '') pad.addstr(i, 1, xi, curses.A_BOLD) else: componenents = xi.split('|') if status: for j, c in enumerate(componenents): if j == status[0]: col = { 'deleted': curses.A_DIM, 'registered': WHITE, 'timeout': CYAN, 'stopped': MAGNETA, 'running': YELLOW, 'error': RED, 'finished': GREEN } cc = col.get(c.strip(), None) if cc is None: offset = sum(len(_) for _ in componenents[:j]) + 1 pad.addstr(i, offset, c) else: pad.addstr( i, sum(len(_) for _ in componenents[:j]) + 1, c, cc) else: pad.addstr(i, sum(len(_) for _ in componenents[:j]) + 1, c) else: pad.addstr(i, 1, ''.join(componenents)) def display_table(table): import curses global pad, window, expand_helps rows, cols = stdscr.getmaxyx() window.erase() if last_time is not None: window.addstr(0, 0, f'Last update recieved @ {last_request_time}', curses.A_DIM) else: window.addstr(0, 0, f'No running experiments', curses.A_DIM) legend_pad = curses.newpad(5, 500) if expand_helps: legend_pad.addstr(0, 0, 'Help: (press o to minimise)', curses.A_BOLD) legend_pad.addstr( 1, 0, 'R=expand-roots d=date s=status v=version p=purpose t=monitor-message M=meta ' ) legend_pad.addstr( 2, 0, 'G=gpu S=slurm c=cpu n=network V=virtual w=swap O=os D=disks') legend_pad.addstr( 3, 0, 'r=filter-roots z = filter-status f=filter-parameters') legend_pad.addstr(4, 0, '(press r, z, f for more info)') else: legend_pad.addstr(0, 0, 'Toggles:', curses.A_BOLD) legend_pad.addstr( 1, 0, f'date = {args.display_date} meta = {args.display_meta}, messages={args.display_messages}, version={args.display_version}' ) legend_pad.addstr( 2, 0, f'pattern = {args.pattern}, status = {args.status_filter}') legend_pad.addstr(3, 0, f'filters={args.filters}') legend_pad.addstr(4, 0, f'(for more help press o)') if table is not None: from tabulate import tabulate x = tabulate(table, headers='keys', tablefmt='github').split('\n') x.pop(1) if len(x) > rows - 6: filler = '|'.join([ ' ...'.ljust(len(xx), ' ') if xx else '' for ii, xx in enumerate(x[1].split('|')) ]) filler = filler.replace('...', ' ', 1) x = [x[0], filler] + x[-(rows - 9):] pad = curses.newpad(len(x), len(x[0])) for i, xx in enumerate(x): display_row(i, pad, x=x) else: pad = curses.newpad(3, 1000) pad.addstr(0, 0, '') pad.addstr(1, 4, f'No experiments found!', RED | curses.A_BOLD) pad.addstr(2, 4, f'Try resetting the filters...') window.noutrefresh() pad.noutrefresh(pos_x, 0, 1, 0, rows - 5, cols - 1) legend_pad.noutrefresh(0, 0, rows - 5, 0, rows - 1, cols - 1) curses.doupdate() def visualise_results(results, args): dics = OrderedDict([ (r, d) for k, (r, s, u, a, up, d) in results.items() if re.match(args.pattern, r) and re.match(args.status_filter, s) ]) data_frame = None if dics: data_frame, root = generate_table(dics, args) display_table(data_frame) try: from xmen.server import send_request_task from xmen.utils import dic_from_yml import time import multiprocessing manager = mp.Manager() q_request = manager.Queue(maxsize=1) q_response = manager.Queue(maxsize=1) q_processed = manager.Queue(maxsize=1) update_requests(q_request, last_request_time, config) p_request = multiprocessing.Process(target=send_request_task, args=(q_request, q_response)) p_request.start() p_process = multiprocessing.Process(target=process_results_task, args=(q_response, q_processed)) p_process.start() # get initial results view if not results: stdscr.addstr( 0, 1, 'Downloading user content from server. This could take a minute but only needs to happen once...', curses.A_BOLD) stdscr.refresh() try: updates, last_request_time = q_processed.get(timeout=180.) except queue.Empty: stdscr.addstr( 0, 1, 'Timeout occurred after 180s. Is the server connected?', curses.A_BOLD) return results.update(updates) config.cache(save=(results, last_request_time)) update_requests(q_request, last_request_time, config) last_time = time.time() visualise_results(results, args) while True: if time.time() - last_time > args.interval: # request experiment updates try: updates, last_request_time = q_processed.get(False) except queue.Empty: pass else: results.update(updates) config.cache(save=(results, last_request_time)) visualise_results(results, args) update_requests(q_request, last_request_time, config) last_time = time.time() if stdscr is not None: stdscr.timeout(1000) c = stdscr.getch() if c == ord('d'): toggle(args, 'display_date') # args.display_date = not args.display_date visualise_results(results, args) if c == ord('v'): toggle(args, 'display_version') visualise_results(results, args) if c == ord('s'): toggle(args, 'display_status') visualise_results(results, args) if c == ord('p'): toggle(args, 'display_purpose') visualise_results(results, args) if c == ord('M'): toggle( args, 'display_meta', '_meta_(slurm_job|root$|name$|mac$|host$|user$|home$)') visualise_results(results, args) if c == ord('V'): toggle(args, 'display_meta', '_meta_virtual.*') visualise_results(results, args) if c == ord('w'): toggle(args, 'display_meta', '_meta_swap.*') visualise_results(results, args) if c == ord('O'): toggle(args, 'display_meta', '_meta_system.*') visualise_results(results, args) if c == ord('D'): toggle(args, 'display_meta', '_meta_disks.*') visualise_results(results, args) if c == ord('c'): toggle(args, 'display_meta', '_meta_cpu.*') visualise_results(results, args) if c == ord('G'): toggle(args, 'display_meta', '_meta_gpu.*') visualise_results(results, args) if c == ord('n'): toggle(args, 'display_meta', '_meta_network.*') visualise_results(results, args) if c == ord('S'): args = toggle(args, 'display_meta', '_meta_slurm.*') visualise_results(results, args) if c == ord('t'): args = toggle( args, 'display_messages', '_messages_(last|e$|s$|wall$|end$|next$|m_step$|m_load$)' ) visualise_results(results, args) if c == ord('f'): from curses.textpad import Textbox, rectangle rows, cols = stdscr.getmaxyx() window.move(rows - 5, 0) window.clrtoeol() window.move(rows - 4, 0) window.clrtoeol() window.move(rows - 3, 0) window.clrtoeol() window.move(rows - 2, 0) window.clrtoeol() window.move(rows - 1, 0) window.clrtoeol() window.addstr(rows - 3, 0, 'Search: (enter "" to reset)', curses.A_BOLD) window.addstr( rows - 2, 0, 'eg. none to reset, ".*", "_messages_.*", "_meta_.*", _version_git_branch=="xmen", a==10, loss<2.0' ) window.refresh() win = curses.newwin(1, cols, rows - 1, 0) text_box = Textbox(win) text = text_box.edit(validate=manage_backspace).strip() if text and not text.startswith('_'): text = '(?!_)' + text args = toggle(args, 'filters', text) visualise_results(results, args) if c == ord('R'): short_root = not short_root visualise_results(results, args) if c == ord('r'): from curses.textpad import Textbox, rectangle rows, cols = stdscr.getmaxyx() window.move(rows - 5, 0) window.clrtoeol() window.move(rows - 4, 0) window.clrtoeol() window.move(rows - 3, 0) window.clrtoeol() window.move(rows - 2, 0) window.clrtoeol() window.move(rows - 1, 0) window.clrtoeol() window.addstr(rows - 3, 0, 'Root: (enter "" to reset)', curses.A_BOLD) window.addstr( rows - 2, 0, 'eg. ".*", "{user}@{host}:{pattern}" where user, host and pattern are regex' ) window.refresh() win = curses.newwin(1, cols, rows - 1, 0) text_box = Textbox(win) text = text_box.edit(validate=manage_backspace).strip() toggle(args, 'pattern', text.strip()) visualise_results(results, args) if c == ord('z'): from curses.textpad import Textbox, rectangle rows, cols = stdscr.getmaxyx() window.move(rows - 5, 0) window.clrtoeol() window.move(rows - 4, 0) window.clrtoeol() window.move(rows - 3, 0) window.clrtoeol() window.move(rows - 2, 0) window.clrtoeol() window.move(rows - 1, 0) window.clrtoeol() window.addstr(rows - 3, 0, 'Status: (enter "" to reset)', curses.A_BOLD) window.addstr(rows - 2, 0, 'eg. "running", ".*", "deleted') window.refresh() win = curses.newwin(1, cols, rows - 1, 0) text_box = Textbox(win) text = text_box.edit(validate=manage_backspace).strip() toggle(args, 'status_filter', text.strip()) visualise_results(results, args) if c == ord("o"): expand_helps = not expand_helps visualise_results(results, args) except KeyboardInterrupt: p_request.terminate() p_process.terminate() raise KeyboardInterrupt
def setup(): from xmen.config import Config print(DESCRIPTION) Config().setup()
def _config(args): with Config() as config: if args.interactive is not None: config.setup() if args.disable_prompt is not None: config.prompt = False elif args.enable_prompt is not None: config.prompt = True if args.disable_save_conda is not None: config.save_conda = False elif args.enable_save_conda is not None: config.save_conda = True if args.register_user is not None: from getpass import getpass user = input('username: '******'password: '******'confirm password: '******'ERROR: Passwords do not match') else: config.register_user(user, password) if args.change_password is not None: from getpass import getpass password = getpass(prompt='Current Password: '******'New Password: '******'Confirm new Password: '******'ERROR: Passwords do not match') else: config.change_password(password, new_password) if args.disable_stdout_to_txt is not None: config.redirect_stdout = False elif args.enable_stdout_to_txt is not None: config.redirect_stdout = True if args.disable_requeue is not None: config.requeue = False elif args.enable_requeue is not None: config.requeue = True if args.header is not None: if os.path.exists(args.header): config.header = open(args.header, 'r').read() else: config.header = args.header if args.server_host: print(f'Updating server host to {args.server_host}') config.server_host = args.server_host if args.server_port: print(f'Updating server port to {args.server_port}') config.server_port = args.server_port if args.list is not None: print(config) if args.clean is not None: config.clean() if args.sync is not None: roots = config.filter(os.getcwd() + '.*') print('Synchronising...') for r in roots: print(r) config.sync(roots) if args.sync_all is not None: print('Synchronising...') for r in config.linked: print(r) config.sync()
params = load_param(r) if params['_root'] != r: params['_root'] = r save_param(params, r) else: experiment_manager = ExperimentManager(args.root) experiment_manager.check_initialised() experiment_manager.relink(args.experiments) relink_parser.set_defaults(func=_relink) ####################################################################################################################### # list ####################################################################################################################### config = Config() list_parser = subparsers.add_parser('list', help='list experiments to screen') # ---- Modes list_parser.add_argument( '-i', '--interval', type=float, default=None, const=1., nargs='?', help= 'Run in interactive mode. If interval is passed then the table will be updated ' 'every n seconds (defaults to 1.).') list_parser.add_argument('-N', '--list', action='store_true',
'other flags and can be used in combination with --to_root, --to_defaults, and --link', 'root': 'Generate a run script and defaults.yml file for interfacing with xgent', 'debug': 'Run experiment in debug mode. The experiments debug will be called before registering.', 'txt': 'Also log stdout and stderr to an out.txt file. Enabled by default (default taken from xmen config)', 'restart': 'Restart the experiment', 'purpose': 'A string giving the purpose of the current experiment' } from xmen.config import Config CONFIG = Config() import textwrap for k in helps: helps[k] = '\n'.join(textwrap.wrap(helps[k], 50)) class NullRoot(str): def __new__(cls): obj = str.__new__(cls, os.devnull) return obj def __add__(self, other): return self