def _do_run(self, start_year, end_year, *args, **kwargs): running_ids = [] for iter in range(len(self.run_ids_dict.keys())): run_id = self.run_ids_dict.keys()[iter] finished_year, server = self.run_ids_dict[run_id] while True: # wait for available server selected_server, restart = self.select_server(server) if selected_server is None: # no server available time.sleep(60) runs_by_status = self.get_run_manager().get_runs_by_status(running_ids) finished_runs = runs_by_status.get('done', []) failed_runs = runs_by_status.get('failed', []) finished_and_failed_runs = finished_runs + failed_runs self.enable_servers(finished_runs, failed_runs) if len(failed_runs) > 0: raise for run in finished_and_failed_runs: running_ids.remove(run) else: break self.set_environment_for_this_run(selected_server, run_id) config = self.get_run_manager().get_resources_for_run_id_from_history(run_id=self.run_id) self.prepare_for_this_run(config, self.configuration_update.get(selected_server, {}), restart) if start_year is None: this_start_year = config['years'][0] else: this_start_year = start_year running_ids.append(run_id) RemoteRun._do_run(self, max(this_start_year, finished_year+1), end_year, config, background=True)
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs): self.run_ids_dict = {} # dict of run_id and finished year if run_id_file is None: config = get_config_from_opus_path(configuration_path) self.number_of_runs = config.get("number_of_runs", 1) root_seed = config.get("seed", None) seed(root_seed) # generate different seed for each run (each seed contains 1 number) seed_array = randint(1, 2**30, self.number_of_runs) for irun in range(self.number_of_runs): config['seed'] = (seed_array[irun], ) RemoteRun.prepare_for_run(self, config=config, prepare_cache=False) self.run_ids_dict[self.run_id] = (0, 'NA') self.run_id_file = self.default_run_id_file else: self.read_run_id_file(run_id_file) self.run_id_file = run_id_file self.get_run_manager() self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime()) logger.log_status("run_id_file: %s" % self.run_id_file) self.write_into_run_id_file() return None
def __init__(self, server_file, hostname, username, password, *args, **kwargs): self.servers_info = {} if server_file is not None: # read hostname-file host_names_and_paths_array = load_table_from_text_file( server_file, comment='#')[0] passw = None for i in range(host_names_and_paths_array.shape[0]): self.servers_info[host_names_and_paths_array[i, 0]] = { 'username': host_names_and_paths_array[i, 1], 'opus_path': host_names_and_paths_array[i, 2], 'communication_path_root': host_names_and_paths_array[i, 3], 'number_of_processes': int(host_names_and_paths_array[i, 4]) } previous_passw = None if (host_names_and_paths_array[i, 0] <> 'localhost'): default = '' if passw is not None: default = ' [previous password]' previous_passw = passw passw = getpass.getpass( 'Password for %s@%s%s:' % (self.servers_info[host_names_and_paths_array[i, 0]] ['username'], host_names_and_paths_array[i, 0], default)) if len(passw) == 0: passw = previous_passw self.servers_info[host_names_and_paths_array[ i, 0]]['password'] = passw else: self.servers_info[host_names_and_paths_array[ i, 0]]['password'] = None else: self.servers_info[hostname] = { 'username': username, 'opus_path': self.remote_opus_path, 'communication_path_root': self.remote_communication_path_root, 'number_of_processes': 1, 'password': password } for server in self.servers_info.keys(): self.servers_info[server][ 'python_command'] = self.python_commands.get( server, self.python_command) self.servers_info[server]['running'] = 0 # 0 processes are running RemoteRun.__init__(self, hostname, username, password, *args, **kwargs)
def _do_run(self, start_year, end_year, *args, **kwargs): running_ids = [] for iter in range(len(self.run_ids_dict.keys())): run_id = self.run_ids_dict.keys()[iter] finished_year, server = self.run_ids_dict[run_id] while True: # wait for available server selected_server, restart = self.select_server(server) if selected_server is None: # no server available time.sleep(60) runs_by_status = self.get_run_manager().get_runs_by_status( running_ids) finished_runs = runs_by_status.get('done', []) failed_runs = runs_by_status.get('failed', []) finished_and_failed_runs = finished_runs + failed_runs self.enable_servers(finished_runs, failed_runs) if len(failed_runs) > 0: raise for run in finished_and_failed_runs: running_ids.remove(run) else: break self.set_environment_for_this_run(selected_server, run_id) config = self.get_run_manager( ).get_resources_for_run_id_from_history(run_id=self.run_id) self.prepare_for_this_run( config, self.configuration_update.get(selected_server, {}), restart) if start_year is None: this_start_year = config['years'][0] else: this_start_year = start_year running_ids.append(run_id) RemoteRun._do_run(self, max(this_start_year, finished_year + 1), end_year, config, background=True)
def __init__(self, server_file, hostname, username, password, *args, **kwargs): self.servers_info = {} if server_file is not None: # read hostname-file host_names_and_paths_array = load_table_from_text_file(server_file, comment='#')[0] passw = None for i in range(host_names_and_paths_array.shape[0]): self.servers_info[host_names_and_paths_array[i,0]] = { 'username': host_names_and_paths_array[i,1], 'opus_path': host_names_and_paths_array[i,2], 'communication_path_root': host_names_and_paths_array[i,3], 'number_of_processes': int(host_names_and_paths_array[i,4])} previous_passw = None if (host_names_and_paths_array[i,0] <> 'localhost'): default = '' if passw is not None: default = ' [previous password]' previous_passw = passw passw = getpass.getpass('Password for %s@%s%s:' % ( self.servers_info[host_names_and_paths_array[i,0]]['username'], host_names_and_paths_array[i,0], default)) if len(passw) == 0: passw = previous_passw self.servers_info[host_names_and_paths_array[i,0]]['password'] = passw else: self.servers_info[host_names_and_paths_array[i,0]]['password'] = None else: self.servers_info[hostname] = { 'username': username, 'opus_path': self.remote_opus_path, 'communication_path_root': self.remote_communication_path_root, 'number_of_processes': 1, 'password': password } for server in self.servers_info.keys(): self.servers_info[server]['python_command'] = self.python_commands.get(server, self.python_command) self.servers_info[server]['running'] = 0 # 0 processes are running RemoteRun.__init__(self, hostname, username, password, *args, **kwargs)
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs): self.run_ids_dict = {} # dict of run_id and finished year if run_id_file is None: config = get_config_from_opus_path(configuration_path) self.number_of_runs = config.get("number_of_runs", 1) root_seed = config.get("seed", None) seed(root_seed) # generate different seed for each run (each seed contains 1 number) seed_array = randint(1,2**30, self.number_of_runs) for irun in range(self.number_of_runs): config['seed']= (seed_array[irun],) RemoteRun.prepare_for_run(self, config=config, prepare_cache=False) self.run_ids_dict[self.run_id] = (0, 'NA') self.run_id_file = self.default_run_id_file else: self.read_run_id_file(run_id_file) self.run_id_file = run_id_file self.get_run_manager() self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime()) logger.log_status("run_id_file: %s" % self.run_id_file) self.write_into_run_id_file() return None