def _do_run(self, start_year, end_year, *args, **kwargs):
running_ids = []
for iter in range(len(self.run_ids_dict.keys())):
run_id = self.run_ids_dict.keys()[iter]
finished_year, server = self.run_ids_dict[run_id]
while True: # wait for available server
selected_server, restart = self.select_server(server)
if selected_server is None: # no server available
time.sleep(60)
runs_by_status = self.get_run_manager().get_runs_by_status(running_ids)
finished_runs = runs_by_status.get('done', [])
failed_runs = runs_by_status.get('failed', [])
finished_and_failed_runs = finished_runs + failed_runs
self.enable_servers(finished_runs, failed_runs)
if len(failed_runs) > 0:
raise
for run in finished_and_failed_runs:
running_ids.remove(run)
else:
break
self.set_environment_for_this_run(selected_server, run_id)
config = self.get_run_manager().get_resources_for_run_id_from_history(run_id=self.run_id)
self.prepare_for_this_run(config, self.configuration_update.get(selected_server, {}), restart)
if start_year is None:
this_start_year = config['years'][0]
else:
this_start_year = start_year
running_ids.append(run_id)
RemoteRun._do_run(self, max(this_start_year, finished_year+1), end_year, config, background=True)
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs):
self.run_ids_dict = {} # dict of run_id and finished year
if run_id_file is None:
config = get_config_from_opus_path(configuration_path)
self.number_of_runs = config.get("number_of_runs", 1)
root_seed = config.get("seed", None)
seed(root_seed)
# generate different seed for each run (each seed contains 1 number)
seed_array = randint(1, 2**30, self.number_of_runs)
for irun in range(self.number_of_runs):
config['seed'] = (seed_array[irun], )
RemoteRun.prepare_for_run(self,
config=config,
prepare_cache=False)
self.run_ids_dict[self.run_id] = (0, 'NA')
self.run_id_file = self.default_run_id_file
else:
self.read_run_id_file(run_id_file)
self.run_id_file = run_id_file
self.get_run_manager()
self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime())
logger.log_status("run_id_file: %s" % self.run_id_file)
self.write_into_run_id_file()
return None
def __init__(self, server_file, hostname, username, password, *args,
**kwargs):
self.servers_info = {}
if server_file is not None: # read hostname-file
host_names_and_paths_array = load_table_from_text_file(
server_file, comment='#')[0]
passw = None
for i in range(host_names_and_paths_array.shape[0]):
self.servers_info[host_names_and_paths_array[i, 0]] = {
'username': host_names_and_paths_array[i, 1],
'opus_path': host_names_and_paths_array[i, 2],
'communication_path_root': host_names_and_paths_array[i,
3],
'number_of_processes': int(host_names_and_paths_array[i,
4])
}
previous_passw = None
if (host_names_and_paths_array[i, 0] <> 'localhost'):
default = ''
if passw is not None:
default = ' [previous password]'
previous_passw = passw
passw = getpass.getpass(
'Password for %s@%s%s:' %
(self.servers_info[host_names_and_paths_array[i, 0]]
['username'], host_names_and_paths_array[i,
0], default))
if len(passw) == 0:
passw = previous_passw
self.servers_info[host_names_and_paths_array[
i, 0]]['password'] = passw
else:
self.servers_info[host_names_and_paths_array[
i, 0]]['password'] = None
else:
self.servers_info[hostname] = {
'username': username,
'opus_path': self.remote_opus_path,
'communication_path_root': self.remote_communication_path_root,
'number_of_processes': 1,
'password': password
}
for server in self.servers_info.keys():
self.servers_info[server][
'python_command'] = self.python_commands.get(
server, self.python_command)
self.servers_info[server]['running'] = 0 # 0 processes are running
RemoteRun.__init__(self, hostname, username, password, *args, **kwargs)
def _do_run(self, start_year, end_year, *args, **kwargs):
running_ids = []
for iter in range(len(self.run_ids_dict.keys())):
run_id = self.run_ids_dict.keys()[iter]
finished_year, server = self.run_ids_dict[run_id]
while True: # wait for available server
selected_server, restart = self.select_server(server)
if selected_server is None: # no server available
time.sleep(60)
runs_by_status = self.get_run_manager().get_runs_by_status(
running_ids)
finished_runs = runs_by_status.get('done', [])
failed_runs = runs_by_status.get('failed', [])
finished_and_failed_runs = finished_runs + failed_runs
self.enable_servers(finished_runs, failed_runs)
if len(failed_runs) > 0:
raise
for run in finished_and_failed_runs:
running_ids.remove(run)
else:
break
self.set_environment_for_this_run(selected_server, run_id)
config = self.get_run_manager(
).get_resources_for_run_id_from_history(run_id=self.run_id)
self.prepare_for_this_run(
config, self.configuration_update.get(selected_server, {}),
restart)
if start_year is None:
this_start_year = config['years'][0]
else:
this_start_year = start_year
running_ids.append(run_id)
RemoteRun._do_run(self,
max(this_start_year, finished_year + 1),
end_year,
config,
background=True)
def __init__(self, server_file, hostname, username, password, *args, **kwargs):
self.servers_info = {}
if server_file is not None: # read hostname-file
host_names_and_paths_array = load_table_from_text_file(server_file, comment='#')[0]
passw = None
for i in range(host_names_and_paths_array.shape[0]):
self.servers_info[host_names_and_paths_array[i,0]] = {
'username': host_names_and_paths_array[i,1],
'opus_path': host_names_and_paths_array[i,2],
'communication_path_root': host_names_and_paths_array[i,3],
'number_of_processes': int(host_names_and_paths_array[i,4])}
previous_passw = None
if (host_names_and_paths_array[i,0] <> 'localhost'):
default = ''
if passw is not None:
default = ' [previous password]'
previous_passw = passw
passw = getpass.getpass('Password for %s@%s%s:' % (
self.servers_info[host_names_and_paths_array[i,0]]['username'],
host_names_and_paths_array[i,0], default))
if len(passw) == 0:
passw = previous_passw
self.servers_info[host_names_and_paths_array[i,0]]['password'] = passw
else:
self.servers_info[host_names_and_paths_array[i,0]]['password'] = None
else:
self.servers_info[hostname] = {
'username': username,
'opus_path': self.remote_opus_path,
'communication_path_root': self.remote_communication_path_root,
'number_of_processes': 1,
'password': password
}
for server in self.servers_info.keys():
self.servers_info[server]['python_command'] = self.python_commands.get(server, self.python_command)
self.servers_info[server]['running'] = 0 # 0 processes are running
RemoteRun.__init__(self, hostname, username, password, *args, **kwargs)
def prepare_for_run(self, configuration_path, run_id_file=None, **kwargs):
self.run_ids_dict = {} # dict of run_id and finished year
if run_id_file is None:
config = get_config_from_opus_path(configuration_path)
self.number_of_runs = config.get("number_of_runs", 1)
root_seed = config.get("seed", None)
seed(root_seed)
# generate different seed for each run (each seed contains 1 number)
seed_array = randint(1,2**30, self.number_of_runs)
for irun in range(self.number_of_runs):
config['seed']= (seed_array[irun],)
RemoteRun.prepare_for_run(self, config=config, prepare_cache=False)
self.run_ids_dict[self.run_id] = (0, 'NA')
self.run_id_file = self.default_run_id_file
else:
self.read_run_id_file(run_id_file)
self.run_id_file = run_id_file
self.get_run_manager()
self.date_time_str = time.strftime('%Y_%m_%d_%H_%M', time.localtime())
logger.log_status("run_id_file: %s" % self.run_id_file)
self.write_into_run_id_file()
return None
