def _list(stdscr, args):
import pandas as pd
from xmen.utils import load_params
pd.set_option('display.width', 1000)
pd.set_option('display.max_columns', 1000)
pd.set_option('display.max_colwidth', args.max_width)
pd.set_option('display.max_rows', args.max_rows)
if args.pattern is None:
pattern = os.getcwd()
else:
pattern = os.path.abspath(os.path.expanduser(args.pattern[0]))
params = os.path.join(pattern, 'params.yml')
if os.path.exists(params):
# print the params.yml file to screen
print(f'Content of {params}')
import ruamel.yaml
from xmen.utils import recursive_print_lines
with open(os.path.join(params), 'r') as params_yml:
params = ruamel.yaml.load(params_yml, ruamel.yaml.RoundTripLoader)
lines = recursive_print_lines(params)
for l in lines:
print(l)
else:
# print(args.pattern)
# global_exp_manager = GlobalExperimentManager()
config = Config()
paths = config.filter(pattern)
params = load_params(paths)
if args.list:
if args.pattern is None:
pattern += '.*'
args.pattern = pattern
from xmen.list import notebook_display, args_to_filters
args.filters += [
'_notes', '_created|_timestamps', '_purpose', '_version'
]
notebook_display(params, *args_to_filters(args))
elif args.interval is None:
if args.pattern is None:
args.pattern = '.*'
from xmen.utils import load_params
from xmen.list import args_to_filters, visualise_params
data_frame, root = visualise_params(params, *args_to_filters(args))
print(f'Roots relative to {root}')
print(data_frame)
else:
if args.pattern is None:
args.pattern = f'.*'
from xmen.utils import load_params
from xmen.list import interactive_display
interactive_display(stdscr, args)
def send_request_task(q_requests, q_response):
import socket
from xmen.config import Config
import time
config = Config()
context = ssl.create_default_context()
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as ss:
with context.wrap_socket(ss,
server_hostname=config.server_host) as s:
s.connect((config.server_host, config.server_port))
while True:
try:
try:
request = q_requests.get(False)
except queue.Empty:
pass
if not request:
break
send(request, s)
response = receive(s)
q_response.put(response)
time.sleep(0.1)
except queue.Empty:
pass
except (socket.error, IOError):
pass
def send_request_task(q_request, q_response=None, hook=None):
from xmen.config import Config
import time
config = Config()
context = ssl.create_default_context()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as ss:
with context.wrap_socket(ss, server_hostname=config.server_host) as s:
while True:
try:
s.connect((config.server_host, config.server_port))
except (socket.error, OSError):
time.sleep(1.)
else:
while True:
try:
request = q_request.get()
if not request:
return
if hook:
request = hook(request)
send(request, s)
response = receive(s)
if q_response is not None:
q_response.put(response)
except queue.Empty:
pass
except KeyboardInterrupt:
pass
def _python(args):
config = Config()
if args.list is not None:
print(f'The following python experiments are currently linked')
for k, v in config.python_experiments.items():
print(f'{k}: {v}')
if args.add is not None:
try:
with config as c:
c.add_python(*args.add)
print(
f'Added experiment {args.add[-1]} from module {args.add[-2]}')
except Exception as m:
print(m)
print(
f'ERROR: failed to add experiment {args.add[-1]} from module {args.add[-2]}'
)
if args.remove is not None:
with config as c:
if args.remove in c.python_experiments:
path = c.python_experiments.pop(args.remove)
if '.xmen' in path:
os.remove(path)
print(f'Successfully removed {args.remove}!')
if args.name:
import subprocess
if args.name[0] not in config.python_experiments:
print(f'No experiments found matching {args.name[0]}')
exit()
args = [config.python_experiments[args.name[0]]] + args.flags
subprocess.call(args)
def _save(self, defaults_dir=None):
"""Save experiment to either a defaults.yml file or a params.yml file depending on its status"""
if self._status == DEFAULT:
path = os.path.join(os.path.join(defaults_dir, 'defaults.yml'))
else:
path = os.path.join(self.directory, 'params.yml')
if self.status == RUNNING:
self._timestamps['last'] = get_time()
# save parameters (always)
string = self.as_yaml()
with open(path, 'w') as file:
file.write(string)
if self.status not in [DEFAULT, REGISTERED] and not self._is_debug:
request = self.to_update_request()
for q in self._queues:
# get one item and put one item
try:
q.get(block=False)
except queue.Empty:
pass
q.put(request, block=False)
elif self.status == REGISTERED and not self._is_debug:
# the global configuration will link with the server...
Config().link(self.root)
def send_request(request):
from xmen.config import Config
config = Config()
context = ssl.create_default_context()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as ss:
with context.wrap_socket(ss, server_hostname=config.server_host) as s:
s.connect((config.server_host, config.server_port))
send(request, s)
response = receive(s)
return decode_response(response)
def to_update_request(self):
from xmen.config import Config
config = Config()
data = self.as_yaml()
root = f'{config.local_user}@{config.local_host}:{self.root}'
return UpdateExperiment(user=config.user,
password=config.password,
root=root,
data=data,
status=self.status)
def _relink(args):
from xmen.utils import load_param, save_param
config = Config()
if args.root == '':
args.root = os.getcwd()
args.root = os.path.abspath(args.root)
if args.experiments is None:
# relink experiment managers
if args.recursive:
managers = [
os.path.dirname(p)
for p in glob.glob(args.root + '/**/experiment.yml',
recursive=True)
]
if len(managers) == 0:
print(
f"No roots found for pattern {args.root + '/**/experiment.yml'}"
)
else:
managers = [args.root]
for manager in managers:
experiment_manager = ExperimentManager(manager)
experiment_manager.replant(manager)
# relink experiments
if args.recursive:
roots = [
os.path.dirname(p)
for p in glob.glob(args.root + '/**/params.yml',
recursive=True)
]
if len(managers) == 0:
print(
f"No roots found for pattern {args.root + '/**/params.yml'}"
)
else:
roots = [args.root]
roots = [r for r in roots if r not in config.linked]
config.link(roots)
config.clean()
for r in roots:
params = load_param(r)
if params['_root'] != r:
params['_root'] = r
save_param(params, r)
else:
experiment_manager = ExperimentManager(args.root)
experiment_manager.check_initialised()
experiment_manager.relink(args.experiments)
def interactive_display(stdscr, args):
"""interactively display results with various search queries"""
import curses
import multiprocessing as mp
import queue
from xmen.utils import dic_from_yml
from collections import OrderedDict
global root
global results
global default_pattern
global expand_helps
global short_root
global last_request_time
default_pattern = None
expand_helps = False
short_root = True
stdscr.refresh()
rows, cols = stdscr.getmaxyx()
if rows < 9:
raise NotEnoughRows
pos_x, pos_y = 0, 0
# load cached results
from xmen.config import Config
config = Config()
results, last_request_time = config.cache(load=True)
meta, message = [], []
# initialise colours
if curses.has_colors():
curses.init_pair(1, curses.COLOR_WHITE, curses.COLOR_BLACK)
curses.init_pair(2, curses.COLOR_RED, curses.COLOR_BLACK)
curses.init_pair(3, curses.COLOR_CYAN, curses.COLOR_BLACK)
curses.init_pair(4, curses.COLOR_YELLOW, curses.COLOR_BLACK)
curses.init_pair(5, curses.COLOR_GREEN, curses.COLOR_BLACK)
curses.init_pair(6, curses.COLOR_MAGENTA, curses.COLOR_BLACK)
WHITE = curses.color_pair(1)
RED = curses.color_pair(2)
CYAN = curses.color_pair(3)
YELLOW = curses.color_pair(4)
GREEN = curses.color_pair(5)
MAGNETA = curses.color_pair(6)
global window
window = curses.newwin(rows, cols, 0, 0)
def generate_table(results, args):
dics, roots = results.values(), results.keys()
data_frame, root = visualise_params(dics,
*args_to_filters(args),
roots=roots,
short_root=short_root)
return data_frame, root
def toggle(args, name, update=None):
global default_pattern
val = getattr(args, name)
if name == 'display_meta':
if update in meta:
meta.remove(update)
else:
meta.append(update)
update = '|'.join(meta)
setattr(args, name, update if update else '^$')
elif name == 'display_messages':
if update in message:
message.remove(update)
else:
message.append(update)
update = '|'.join(message)
setattr(args, name, update if update else '^$')
elif name == 'filters':
if update == '':
setattr(args, name, [args.filters[0]])
else:
setattr(args, name, getattr(args, name) + [update])
elif name == 'filter_params':
if update == '':
setattr(args, name, [args.filter_params[0]])
else:
setattr(args, name, getattr(args, name) + [update])
elif name == 'status_filter':
if update == '':
setattr(args, name, '[^(deleted)]')
else:
setattr(args, name, update)
elif name == 'pattern':
if default_pattern is None:
default_patten = args.pattern
if update == '':
setattr(args, name, default_patten)
else:
setattr(args, name, update)
else:
if update is None:
update = DEFAULTS[name]
if val == '^$':
setattr(args, name, update)
else:
setattr(args, name, '^$')
return args
def display_row(i, pad, table=None, x=None):
from tabulate import tabulate
if x is None:
x = tabulate(table, headers='keys', tablefmt='github').split('\n')
x0 = x[0]
xi = x[i]
# stdscr.addstr(i + 2, 1, xx)
root = [
ii for ii, hh in enumerate(x0.split('|')) if hh.strip() == 'root'
]
status = [
ii for ii, hh in enumerate(x0.split('|')) if hh.strip() == 'status'
]
if i == 0:
xi = xi.replace('|', '')
pad.addstr(i, 1, xi, curses.A_BOLD)
else:
componenents = xi.split('|')
if status:
for j, c in enumerate(componenents):
if j == status[0]:
col = {
'deleted': curses.A_DIM,
'registered': WHITE,
'timeout': CYAN,
'stopped': MAGNETA,
'running': YELLOW,
'error': RED,
'finished': GREEN
}
cc = col.get(c.strip(), None)
if cc is None:
offset = sum(len(_) for _ in componenents[:j]) + 1
pad.addstr(i, offset, c)
else:
pad.addstr(
i,
sum(len(_) for _ in componenents[:j]) + 1, c,
cc)
else:
pad.addstr(i,
sum(len(_) for _ in componenents[:j]) + 1,
c)
else:
pad.addstr(i, 1, ''.join(componenents))
def display_table(table):
import curses
global pad, window, expand_helps
rows, cols = stdscr.getmaxyx()
window.erase()
if last_time is not None:
window.addstr(0, 0, f'Last update recieved @ {last_request_time}',
curses.A_DIM)
else:
window.addstr(0, 0, f'No running experiments', curses.A_DIM)
legend_pad = curses.newpad(5, 500)
if expand_helps:
legend_pad.addstr(0, 0, 'Help: (press o to minimise)',
curses.A_BOLD)
legend_pad.addstr(
1, 0,
'R=expand-roots d=date s=status v=version p=purpose t=monitor-message M=meta '
)
legend_pad.addstr(
2, 0,
'G=gpu S=slurm c=cpu n=network V=virtual w=swap O=os D=disks')
legend_pad.addstr(
3, 0,
'r=filter-roots z = filter-status f=filter-parameters')
legend_pad.addstr(4, 0, '(press r, z, f for more info)')
else:
legend_pad.addstr(0, 0, 'Toggles:', curses.A_BOLD)
legend_pad.addstr(
1, 0,
f'date = {args.display_date} meta = {args.display_meta}, messages={args.display_messages}, version={args.display_version}'
)
legend_pad.addstr(
2, 0,
f'pattern = {args.pattern}, status = {args.status_filter}')
legend_pad.addstr(3, 0, f'filters={args.filters}')
legend_pad.addstr(4, 0, f'(for more help press o)')
if table is not None:
from tabulate import tabulate
x = tabulate(table, headers='keys', tablefmt='github').split('\n')
x.pop(1)
if len(x) > rows - 6:
filler = '|'.join([
' ...'.ljust(len(xx), ' ') if xx else ''
for ii, xx in enumerate(x[1].split('|'))
])
filler = filler.replace('...', ' ', 1)
x = [x[0], filler] + x[-(rows - 9):]
pad = curses.newpad(len(x), len(x[0]))
for i, xx in enumerate(x):
display_row(i, pad, x=x)
else:
pad = curses.newpad(3, 1000)
pad.addstr(0, 0, '')
pad.addstr(1, 4, f'No experiments found!', RED | curses.A_BOLD)
pad.addstr(2, 4, f'Try resetting the filters...')
window.noutrefresh()
pad.noutrefresh(pos_x, 0, 1, 0, rows - 5, cols - 1)
legend_pad.noutrefresh(0, 0, rows - 5, 0, rows - 1, cols - 1)
curses.doupdate()
def visualise_results(results, args):
dics = OrderedDict([
(r, d) for k, (r, s, u, a, up, d) in results.items()
if re.match(args.pattern, r) and re.match(args.status_filter, s)
])
data_frame = None
if dics:
data_frame, root = generate_table(dics, args)
display_table(data_frame)
try:
from xmen.server import send_request_task
from xmen.utils import dic_from_yml
import time
import multiprocessing
manager = mp.Manager()
q_request = manager.Queue(maxsize=1)
q_response = manager.Queue(maxsize=1)
q_processed = manager.Queue(maxsize=1)
update_requests(q_request, last_request_time, config)
p_request = multiprocessing.Process(target=send_request_task,
args=(q_request, q_response))
p_request.start()
p_process = multiprocessing.Process(target=process_results_task,
args=(q_response, q_processed))
p_process.start()
# get initial results view
if not results:
stdscr.addstr(
0, 1,
'Downloading user content from server. This could take a minute but only needs to happen once...',
curses.A_BOLD)
stdscr.refresh()
try:
updates, last_request_time = q_processed.get(timeout=180.)
except queue.Empty:
stdscr.addstr(
0, 1,
'Timeout occurred after 180s. Is the server connected?',
curses.A_BOLD)
return
results.update(updates)
config.cache(save=(results, last_request_time))
update_requests(q_request, last_request_time, config)
last_time = time.time()
visualise_results(results, args)
while True:
if time.time() - last_time > args.interval:
# request experiment updates
try:
updates, last_request_time = q_processed.get(False)
except queue.Empty:
pass
else:
results.update(updates)
config.cache(save=(results, last_request_time))
visualise_results(results, args)
update_requests(q_request, last_request_time, config)
last_time = time.time()
if stdscr is not None:
stdscr.timeout(1000)
c = stdscr.getch()
if c == ord('d'):
toggle(args, 'display_date')
# args.display_date = not args.display_date
visualise_results(results, args)
if c == ord('v'):
toggle(args, 'display_version')
visualise_results(results, args)
if c == ord('s'):
toggle(args, 'display_status')
visualise_results(results, args)
if c == ord('p'):
toggle(args, 'display_purpose')
visualise_results(results, args)
if c == ord('M'):
toggle(
args, 'display_meta',
'_meta_(slurm_job|root$|name$|mac$|host$|user$|home$)')
visualise_results(results, args)
if c == ord('V'):
toggle(args, 'display_meta', '_meta_virtual.*')
visualise_results(results, args)
if c == ord('w'):
toggle(args, 'display_meta', '_meta_swap.*')
visualise_results(results, args)
if c == ord('O'):
toggle(args, 'display_meta', '_meta_system.*')
visualise_results(results, args)
if c == ord('D'):
toggle(args, 'display_meta', '_meta_disks.*')
visualise_results(results, args)
if c == ord('c'):
toggle(args, 'display_meta', '_meta_cpu.*')
visualise_results(results, args)
if c == ord('G'):
toggle(args, 'display_meta', '_meta_gpu.*')
visualise_results(results, args)
if c == ord('n'):
toggle(args, 'display_meta', '_meta_network.*')
visualise_results(results, args)
if c == ord('S'):
args = toggle(args, 'display_meta', '_meta_slurm.*')
visualise_results(results, args)
if c == ord('t'):
args = toggle(
args, 'display_messages',
'_messages_(last|e$|s$|wall$|end$|next$|m_step$|m_load$)'
)
visualise_results(results, args)
if c == ord('f'):
from curses.textpad import Textbox, rectangle
rows, cols = stdscr.getmaxyx()
window.move(rows - 5, 0)
window.clrtoeol()
window.move(rows - 4, 0)
window.clrtoeol()
window.move(rows - 3, 0)
window.clrtoeol()
window.move(rows - 2, 0)
window.clrtoeol()
window.move(rows - 1, 0)
window.clrtoeol()
window.addstr(rows - 3, 0, 'Search: (enter "" to reset)',
curses.A_BOLD)
window.addstr(
rows - 2, 0,
'eg. none to reset, ".*", "_messages_.*", "_meta_.*", _version_git_branch=="xmen", a==10, loss<2.0'
)
window.refresh()
win = curses.newwin(1, cols, rows - 1, 0)
text_box = Textbox(win)
text = text_box.edit(validate=manage_backspace).strip()
if text and not text.startswith('_'):
text = '(?!_)' + text
args = toggle(args, 'filters', text)
visualise_results(results, args)
if c == ord('R'):
short_root = not short_root
visualise_results(results, args)
if c == ord('r'):
from curses.textpad import Textbox, rectangle
rows, cols = stdscr.getmaxyx()
window.move(rows - 5, 0)
window.clrtoeol()
window.move(rows - 4, 0)
window.clrtoeol()
window.move(rows - 3, 0)
window.clrtoeol()
window.move(rows - 2, 0)
window.clrtoeol()
window.move(rows - 1, 0)
window.clrtoeol()
window.addstr(rows - 3, 0, 'Root: (enter "" to reset)',
curses.A_BOLD)
window.addstr(
rows - 2, 0,
'eg. ".*", "{user}@{host}:{pattern}" where user, host and pattern are regex'
)
window.refresh()
win = curses.newwin(1, cols, rows - 1, 0)
text_box = Textbox(win)
text = text_box.edit(validate=manage_backspace).strip()
toggle(args, 'pattern', text.strip())
visualise_results(results, args)
if c == ord('z'):
from curses.textpad import Textbox, rectangle
rows, cols = stdscr.getmaxyx()
window.move(rows - 5, 0)
window.clrtoeol()
window.move(rows - 4, 0)
window.clrtoeol()
window.move(rows - 3, 0)
window.clrtoeol()
window.move(rows - 2, 0)
window.clrtoeol()
window.move(rows - 1, 0)
window.clrtoeol()
window.addstr(rows - 3, 0, 'Status: (enter "" to reset)',
curses.A_BOLD)
window.addstr(rows - 2, 0, 'eg. "running", ".*", "deleted')
window.refresh()
win = curses.newwin(1, cols, rows - 1, 0)
text_box = Textbox(win)
text = text_box.edit(validate=manage_backspace).strip()
toggle(args, 'status_filter', text.strip())
visualise_results(results, args)
if c == ord("o"):
expand_helps = not expand_helps
visualise_results(results, args)
except KeyboardInterrupt:
p_request.terminate()
p_process.terminate()
raise KeyboardInterrupt
def setup():
from xmen.config import Config
print(DESCRIPTION)
Config().setup()
def _config(args):
with Config() as config:
if args.interactive is not None:
config.setup()
if args.disable_prompt is not None:
config.prompt = False
elif args.enable_prompt is not None:
config.prompt = True
if args.disable_save_conda is not None:
config.save_conda = False
elif args.enable_save_conda is not None:
config.save_conda = True
if args.register_user is not None:
from getpass import getpass
user = input('username: '******'password: '******'confirm password: '******'ERROR: Passwords do not match')
else:
config.register_user(user, password)
if args.change_password is not None:
from getpass import getpass
password = getpass(prompt='Current Password: '******'New Password: '******'Confirm new Password: '******'ERROR: Passwords do not match')
else:
config.change_password(password, new_password)
if args.disable_stdout_to_txt is not None:
config.redirect_stdout = False
elif args.enable_stdout_to_txt is not None:
config.redirect_stdout = True
if args.disable_requeue is not None:
config.requeue = False
elif args.enable_requeue is not None:
config.requeue = True
if args.header is not None:
if os.path.exists(args.header):
config.header = open(args.header, 'r').read()
else:
config.header = args.header
if args.server_host:
print(f'Updating server host to {args.server_host}')
config.server_host = args.server_host
if args.server_port:
print(f'Updating server port to {args.server_port}')
config.server_port = args.server_port
if args.list is not None:
print(config)
if args.clean is not None:
config.clean()
if args.sync is not None:
roots = config.filter(os.getcwd() + '.*')
print('Synchronising...')
for r in roots:
print(r)
config.sync(roots)
if args.sync_all is not None:
print('Synchronising...')
for r in config.linked:
print(r)
config.sync()
params = load_param(r)
if params['_root'] != r:
params['_root'] = r
save_param(params, r)
else:
experiment_manager = ExperimentManager(args.root)
experiment_manager.check_initialised()
experiment_manager.relink(args.experiments)
relink_parser.set_defaults(func=_relink)
#######################################################################################################################
# list
#######################################################################################################################
config = Config()
list_parser = subparsers.add_parser('list', help='list experiments to screen')
# ---- Modes
list_parser.add_argument(
'-i',
'--interval',
type=float,
default=None,
const=1.,
nargs='?',
help=
'Run in interactive mode. If interval is passed then the table will be updated '
'every n seconds (defaults to 1.).')
list_parser.add_argument('-N',
'--list',
action='store_true',
'other flags and can be used in combination with --to_root, --to_defaults, and --link',
'root':
'Generate a run script and defaults.yml file for interfacing with xgent',
'debug':
'Run experiment in debug mode. The experiments debug will be called before registering.',
'txt':
'Also log stdout and stderr to an out.txt file. Enabled by default (default taken from xmen config)',
'restart':
'Restart the experiment',
'purpose':
'A string giving the purpose of the current experiment'
}
from xmen.config import Config
CONFIG = Config()
import textwrap
for k in helps:
helps[k] = '\n'.join(textwrap.wrap(helps[k], 50))
class NullRoot(str):
def __new__(cls):
obj = str.__new__(cls, os.devnull)
return obj
def __add__(self, other):
return self