def __init__(self, run_function: Callable[..., Dict], runs: Sequence = (), database_connector: str = './db') -> None: self.runs = list(runs) self.run_function = run_function self.db = Database(database_connector)
def run_client(self, client_name: str = "{hostname}_{pid}", server: str = 'localhost', port: int = 8342, parameter_update: Optional[ParameterDict] = None, max_time=None) -> None: """Start runner client that retrieves configuration from server, runs it and reports results back to server. Args: client_name: name of the client. Has to be unique for each client. Placeholders {hostname} and {pid} can be used to generate unique name. server: url to server port: port as specified in run_server parameter_update: Used to override/extend parameters received from the server. Typical use cases is to set optimization or GPU parameters for some clients to make some clients simulate on CPU, others on GPU max_time: maximum runtime in seconds: the client runs scenario after scenario, but starts only a new scenario if not more than max_time seconds have passed since this function was called. So the time given here should be the total maximum runtime minus a typical runtime for one setup """ from urllib.request import urlopen from urllib.error import URLError import time parameter_update = {} if parameter_update is None else parameter_update url = f"http://{server}:{port}" client_name = client_name.format(hostname=socket.gethostname(), pid=os.getpid()) start_time = time.time() while True: try: if max_time is not None and (time.time() - start_time) > max_time: print("Stopping client - maximum time reached") break http_response = urlopen(url + "/next_scenario", data=json.dumps({ 'client_name': client_name }).encode()) scenario = json.loads(http_response.read().decode()) if scenario['status'] != 'ok': break original_params = scenario['params'].copy() scenario['params'].update(parameter_update) result = self.run_function(**scenario['params']) answer = { 'params': original_params, 'changed_params': parameter_update, 'result': result, 'env': Database.get_environment(), 'client_name': client_name } urlopen(url + '/result', data=json.dumps(answer).encode()) except URLError: print( f"Cannot connect to server {url} retrying in 5 seconds..." ) sleep(5)
def local(a): if a.database: self.db = Database(a.database) self.run_scenarios_not_in_database(json.loads( a.parameter_override))
def server(a): if a.database: self.db = Database(a.database) self.run_server(a.host, a.port)
class ParameterStudy: """Manages and runs multiple configurations locally or distributed and stores results in NoSQL database. To run a parameter study, define a run function that takes all parameters as keyword arguments and returns the results as a (possibly nested) dictionary. Then, define the parameter sets that this function should be run with. Examples: >>> import tempfile >>> >>> def dummy_run_function(p1, p2, p3, p4): ... print("Run called with", p1, p2, p3, p4) ... return { 'result1': p1 * p2, 'result2': p3 + p4 } >>> >>> with tempfile.TemporaryDirectory() as tmp_dir: ... ps = ParameterStudy(dummy_run_function, database_connector=tmp_dir) ... ps.add_run({'p1': 5, 'p2': 42, 'p3': 'abc', 'p4': 'def'}) ... ps.add_combinations( [('p1', [1, 2]), ... ('p3', ['x', 'y'])], constant_parameters={'p2': 5, 'p4': 'z' }) ... ps.run() ... ps.run_scenarios_not_in_database() ... ps.run_from_command_line(argv=['local']) # alternative to run - exposes a command line interface if ... # no argv is passed. Does not run anything here, because ... # configuration already in database are skipped Run called with 2 5 y z Run called with 2 5 x z Run called with 1 5 y z Run called with 1 5 x z Run called with 5 42 abc def Above example runs all parameter combinations locally and stores the returned result in the NoSQL database. It is also possible to distribute the runs to multiple processes, by starting a server on one machine and multiple executing runners on other machines. The server distributes configurations to the runners, collects their results to stores the results in the database. """ Run = namedtuple("Run", ['parameter_dict', 'weight']) def __init__(self, run_function: Callable[..., Dict], runs: Sequence = (), database_connector: str = './db') -> None: self.runs = list(runs) self.run_function = run_function self.db = Database(database_connector) def add_run(self, parameter_dict: ParameterDict, weight: int = 1) -> None: """Schedule a dictionary of parameters to run in this parameter study. Args: parameter_dict: used as keyword arguments to the run function. weight: weight of the run configuration which should be proportional to runtime of this case, used for progress display and distribution to processes. """ self.runs.append(self.Run(parameter_dict, weight)) def add_combinations( self, degrees_of_freedom: Sequence[Tuple[str, Sequence[Any]]], constant_parameters: Optional[ParameterDict] = None, filter_function: Optional[FilterFunction] = None, runtime_weight_function: Optional[WeightFunction] = None) -> None: """Add all possible combinations of given parameters as runs. This is a convenience function to simulate all possible parameter combinations of a scenario. Configurations can be filtered and weighted by passing filter- and weighting functions. Args: degrees_of_freedom: defines for each parameter the possible values it can take on constant_parameters: parameter dict, for parameters that should not be changed filter_function: optional function that receives a parameter dict and returns the potentially modified dict or None if this combination should not be added. runtime_weight_function: function mapping a parameter dict to the runtime weight (see weight at add_runs) Examples: degrees_of_freedom = [('p1', [1,2]), ('p2', ['a', 'b'])] is equivalent to calling add_run four times, with all possible parameter combinations. """ parameter_names = [e[0] for e in degrees_of_freedom] parameter_values = [e[1] for e in degrees_of_freedom] default_params_dict = {} if constant_parameters is None else constant_parameters for value_tuple in itertools.product(*parameter_values): params_dict = deepcopy(default_params_dict) params_dict.update({ name: value for name, value in zip(parameter_names, value_tuple) }) params = DotDict(params_dict) if filter_function: params = filter_function(params) if params is None: continue weight = 1 if not runtime_weight_function else runtime_weight_function( params) self.add_run(params, weight) def run(self, process: int = 0, num_processes: int = 1, parameter_update: Optional[ParameterDict] = None) -> None: """Runs all added configurations. Args: process: configurations are split into num_processes chunks according to weights and only the process'th chunk is run. To run all, use process=0 and num_processes=1 num_processes: see above parameter_update: Extend/override all configurations with this dictionary. """ parameter_update = {} if parameter_update is None else parameter_update own_runs = self._distribute_runs(self.runs, process, num_processes) for run in own_runs: parameter_dict = run.parameter_dict.copy() parameter_dict.update(parameter_update) result = self.run_function(**parameter_dict) self.db.save(run.parameter_dict, result, None, changed_params=parameter_update) def run_scenarios_not_in_database( self, parameter_update: Optional[ParameterDict] = None) -> None: """Same as run method, but runs only configuration for which no result is in the database yet.""" parameter_update = {} if parameter_update is None else parameter_update filtered_runs = self._filter_already_simulated(self.runs) for run in filtered_runs: parameter_dict = run.parameter_dict.copy() parameter_dict.update(parameter_update) result = self.run_function(**parameter_dict) self.db.save(run.parameter_dict, result, changed_params=parameter_update) def run_server(self, ip: str = "0.0.0.0", port: int = 8342): """Runs server to supply runner clients with scenarios to simulate and collect results from them. Skips scenarios that are already in the database.""" from http.server import BaseHTTPRequestHandler, HTTPServer filtered_runs = self._filter_already_simulated(self.runs) if not filtered_runs: print("No Scenarios to simulate") return class ParameterStudyServer(BaseHTTPRequestHandler): parameterStudy = self all_runs = filtered_runs runs = filtered_runs.copy() currently_running = {} finished_runs = [] def next_scenario(self, received_json_data): client_name = received_json_data['client_name'] if len(self.runs) > 0: run_status = "%d/%d" % (len( self.finished_runs), len(self.all_runs)) work_status = "%d/%d" % (sum(r.weight for r in self.finished_runs), sum(r.weight for r in self.all_runs)) format_args = { 'remaining': len(self.runs), 'time': datetime.datetime.now().strftime("%H:%M:%S"), 'client_name': client_name, 'run_status': run_status, 'work_status': work_status, } scenario = self.runs.pop(0) print( " {time} {client_name} fetched scenario. Scenarios: {run_status}, Work: {work_status}" .format(**format_args)) self.currently_running[client_name] = scenario return {'status': 'ok', 'params': scenario.parameter_dict} else: return {'status': 'finished'} def result(self, received_json_data): client_name = received_json_data['client_name'] run = self.currently_running[client_name] self.finished_runs.append(run) del self.currently_running[client_name] d = received_json_data def hash_dict(dictionary): import hashlib return hashlib.sha1( json.dumps(dictionary, sort_keys=True).encode()).hexdigest() assert hash_dict(d['params']) == hash_dict(run.parameter_dict), \ str(d['params']) + "is not equal to " + str(run.parameter_dict) self.parameterStudy.db.save(run.parameter_dict, result=d['result'], env=d['env'], changed_params=d['changed_params']) return {} # noinspection PyPep8Naming def do_POST(self) -> None: mapping = { '/next_scenario': self.next_scenario, '/result': self.result } if self.path in mapping.keys(): data = self._read_contents() self.send_response(200) self.send_header("Content-type", "application/json") self.end_headers() json_data = json.loads(data) response = mapping[self.path](json_data) self.wfile.write(json.dumps(response).encode()) else: self.send_response(400) # noinspection PyPep8Naming def do_GET(self): return self.do_POST() def _read_contents(self): return self.rfile.read(int( self.headers['Content-Length'])).decode() def log_message(self, fmt, *args): return print( f"Listening to connections on {ip}:{port}. Scenarios to simulate: {len(filtered_runs)}" ) server = HTTPServer((ip, port), ParameterStudyServer) while len(ParameterStudyServer.currently_running) > 0 or len( ParameterStudyServer.runs) > 0: server.handle_request() server.handle_request() def run_client(self, client_name: str = "{hostname}_{pid}", server: str = 'localhost', port: int = 8342, parameter_update: Optional[ParameterDict] = None, max_time=None) -> None: """Start runner client that retrieves configuration from server, runs it and reports results back to server. Args: client_name: name of the client. Has to be unique for each client. Placeholders {hostname} and {pid} can be used to generate unique name. server: url to server port: port as specified in run_server parameter_update: Used to override/extend parameters received from the server. Typical use cases is to set optimization or GPU parameters for some clients to make some clients simulate on CPU, others on GPU max_time: maximum runtime in seconds: the client runs scenario after scenario, but starts only a new scenario if not more than max_time seconds have passed since this function was called. So the time given here should be the total maximum runtime minus a typical runtime for one setup """ from urllib.request import urlopen from urllib.error import URLError import time parameter_update = {} if parameter_update is None else parameter_update url = f"http://{server}:{port}" client_name = client_name.format(hostname=socket.gethostname(), pid=os.getpid()) start_time = time.time() while True: try: if max_time is not None and (time.time() - start_time) > max_time: print("Stopping client - maximum time reached") break http_response = urlopen(url + "/next_scenario", data=json.dumps({ 'client_name': client_name }).encode()) scenario = json.loads(http_response.read().decode()) if scenario['status'] != 'ok': break original_params = scenario['params'].copy() scenario['params'].update(parameter_update) result = self.run_function(**scenario['params']) answer = { 'params': original_params, 'changed_params': parameter_update, 'result': result, 'env': Database.get_environment(), 'client_name': client_name } urlopen(url + '/result', data=json.dumps(answer).encode()) except URLError: print( f"Cannot connect to server {url} retrying in 5 seconds..." ) sleep(5) def run_from_command_line(self, argv: Optional[Sequence[str]] = None) -> None: """Exposes interface to command line with possibility to run directly or distributed via server/client.""" from argparse import ArgumentParser def server(a): if a.database: self.db = Database(a.database) self.run_server(a.host, a.port) def client(a): self.run_client(a.client_name, a.host, a.port, json.loads(a.parameter_override), a.max_time) def local(a): if a.database: self.db = Database(a.database) self.run_scenarios_not_in_database(json.loads( a.parameter_override)) parser = ArgumentParser() subparsers = parser.add_subparsers() local_parser = subparsers.add_parser( 'local', aliases=['l'], help="Run scenarios locally which are not yet in database", ) local_parser.add_argument("-d", "--database", type=str, default="") local_parser.add_argument( "-P", "--parameter_override", type=str, default="{}", help= "JSON: the parameter dictionary is updated with these parameters. Use this to " "set host specific options like GPU call parameters. Enclose in \" " ) local_parser.set_defaults(func=local) server_parser = subparsers.add_parser( 'server', aliases=['s'], help="Runs server to distribute different scenarios to workers", ) server_parser.add_argument("-p", "--port", type=int, default=8342, help="Port to listen on") server_parser.add_argument("-H", "--host", type=str, default="0.0.0.0", help="IP/Hostname to listen on") server_parser.add_argument("-d", "--database", type=str, default="") server_parser.set_defaults(func=server) client_parser = subparsers.add_parser( 'client', aliases=['c'], help= "Runs a worker client connection to scenario distribution server") client_parser.add_argument("-p", "--port", type=int, default=8342, help="Port to connect to") client_parser.add_argument("-H", "--host", type=str, default="localhost", help="Host or IP to connect to") client_parser.add_argument( "-n", "--client_name", type=str, default="{hostname}_{pid}", help= "Unique client name, you can use {hostname} and {pid} as placeholder" ) client_parser.add_argument( "-P", "--parameter_override", type=str, default="{}", help= "JSON: the parameter dictionary is updated with these parameters. Use this to " "set host specific options like GPU call parameters. Enclose in \" " ) client_parser.add_argument( "-t", "--max_time", type=int, default=None, help="If more than this time in seconds has passed, " "the client stops running scenarios.") client_parser.set_defaults(func=client) args = parser.parse_args(argv) if not len(vars(args)): parser.print_help() else: args.func(args) def _filter_already_simulated(self, all_runs): """Removes all runs from the given list, that are already in the database""" already_simulated = {json.dumps(e.params) for e in self.db.filter({})} return [ r for r in all_runs if json.dumps(r.parameter_dict) not in already_simulated ] @staticmethod def _distribute_runs(all_runs, process, num_processes): """Partitions runs by their weights into num_processes chunks and returns the process's chunk.""" sorted_runs = sorted(all_runs, key=lambda e: e.weight, reverse=True) result = sorted_runs[process::num_processes] result.reverse() # start with faster scenarios return result