def commented_to_py(x, seq=tuple): from ruamel.yaml.comments import CommentedSeq, CommentedMap if type(x) is CommentedMap: return {k: commented_to_py(v) for k, v in x.items()} if type(x) is CommentedSeq: return seq(commented_to_py(v) for v in x) else: return x
def multi_client_server_2(): from xmen.utils import commented_to_py # get the hostname import struct import ruamel.yaml from xmen.experiment import IncompatibleYmlException host = socket.gethostname() port = 6011 # initiate port no above 1024 server_socket = socket.socket() # get instance # look closely. The bind() function takes tuple as argument server_socket.bind((host, port)) # bind host address and port together # configure how many client the server can listen simultaneously server_socket.listen(10) while True: conn, address = server_socket.accept() print("Connection from: " + str(address)) length = conn.recv(struct.calcsize('Q')) if length: length, = struct.unpack('Q', length) print('length = ', length) params = conn.recv(length).decode() yaml = ruamel.yaml.YAML() try: params = yaml.load(params) except: raise IncompatibleYmlException params = {k: commented_to_py(v) for k, v in params.items()} print(params) conn.close() # close the connection
def from_yml(self, path, copy=False): """Load state from either a ``params.yml`` or ``defaults.yml`` file (inferred from the filename). The status of the experiment will be updated to ``'default'`` if ``'defaults.yml'`` file else ``'registered'`` if ``params.yml`` file. If copy is ``True`` then only user defined parameters themselves will be copied from the params.yml file. """ from xmen.utils import dic_from_yml params = dic_from_yml(path=path) # backward compatibility if '_name' in params: params['_root'] = os.path.join(params['_root'], params.pop('_name')) if '_created' in params: params['_timestamps'] = get_timestamps() params['_timestamps']['created'] = params.pop('_created') params = { k: commented_to_py(v) for k, v in params.items() if k in self.__dict__ } if copy: # Copy only parameter values themselves (and not specials) params = {k: v for k, v in params.items() if not k.startswith('_')} # update created date self.__dict__.update(params)
def load_params(roots): """Load params.yml files into a list of dictionaries from a list of paths""" import ruamel.yaml from xmen.utils import commented_to_py out = [] for path in roots: with open(os.path.join(path, 'params.yml'), 'r') as params_yml: params = ruamel.yaml.load(params_yml, ruamel.yaml.RoundTripLoader) params = {k: commented_to_py(v) for k, v in params.items()} out.append(params) return out
def send(dic, conn, typ='safe'): """Send dictionary to connected socket""" from xmen.utils import commented_to_py import gzip if hasattr(dic, '_asdict'): dic = dic._asdict() # data is always sent without comments dic = {k: commented_to_py(v) for k, v in dic.items()} string = dic_to_yaml(dic, typ, True) buffer = string.encode() # compress buffer = gzip.compress(buffer) conn.sendall(struct.pack('Q', len(buffer)) + buffer)
def updates_client(q, interval): from xmen.utils import commented_to_py import os import socket import time # get the hostname import struct import ruamel.yaml from xmen.experiment import IncompatibleYmlException host = '193.62.124.26' port = 6011 # initiate port no above 1024 server_socket = socket.socket() # get instance # look closely. The bind() function takes tuple as argument server_socket.bind((host, port)) # bind host address and port together # configure how many client the server can listen simultaneously server_socket.listen(10) while True: updates = {} last = time.time() while time.time() - last < interval: conn, address = server_socket.accept() # print("Connection from: " + str(address)) length = conn.recv(struct.calcsize('Q')) if length: length, = struct.unpack('Q', length) # print('length = ', length) params = conn.recv(length).decode() yaml = ruamel.yaml.YAML() try: params = yaml.load(params) except: raise IncompatibleYmlException params = {k: commented_to_py(v) for k, v in params.items()} print(params['_name']) # print(params) updates[os.path.join(params['_root'], params['_name'])] = params conn.close() # close the connection if updates: q.put((updates, last))
def receive(conn, typ='safe', default_flow_style=False): """Receive dictionary over connected socket""" import ruamel.yaml from xmen.utils import IncompatibleYmlException from xmen.utils import commented_to_py import gzip length = receive_message(conn, struct.calcsize('Q')) dic = None if length: length, = struct.unpack('Q', length) buffer = receive_message(conn, length) buffer = gzip.decompress(buffer) yaml = ruamel.yaml.YAML(typ=typ) yaml.default_flow_style = default_flow_style try: dic = yaml.load(buffer) except Exception as m: raise IncompatibleYmlException dic = {k: commented_to_py(v) for k, v in dic.items()} return dic
def recursive_print_lines(dic, helps=None, start=''): import collections from ruamel.yaml.comments import CommentedMap lines = [] dic = commented_to_py(dic) for k, v in dic.items(): if type(v) is dict or type(v) is collections.OrderedDict or type(v) is CommentedMap: lines += [f'{k}:'] lines += [' ' + start + l for l in recursive_print_lines(v)] elif v is not None: h = '' if helps is not None: h = helps.get(k, None) if h is not None: h = helps[k].split(":")[1].strip() else: h = '' if h != '': lines += [f'{start}{k}: {v} # {h}'] else: lines += [f'{start}{k}: {v}'] return lines