def run_per_ip(cmd, env_vars_for_mpi=None, use_devnull=False): if os.environ.get('OMPI_COMM_WORLD_SIZE') is not None: raise RuntimeError( "Function run_per_ip is not meant to be run from within an OpenMPI context. It is intended to invoke mpirun by itelf." ) if not is_valid_multi_node_config(): print( "************************* Single-HLS mode *************************" ) run_cmd_as_subprocess(cmd, use_devnull) else: if os.environ.get('DOCKER_SSHD_PORT'): portnum = os.environ.get('DOCKER_SSHD_PORT') else: portnum = 3022 scmd = f"mpirun --allow-run-as-root --mca plm_rsh_args -p{portnum} --tag-output --merge-stderr-to-stdout --prefix /usr/lib/habanalabs/openmpi/ -H {os.environ.get('MULTI_HLS_IPS')} " if env_vars_for_mpi is not None: for env_var in env_vars_for_mpi: scmd += f"-x {env_var} " scmd += cmd print( f"{socket.gethostname()}: In MULTI NODE run_per_ip(): scmd = {scmd}" ) run_cmd_as_subprocess(scmd, use_devnull)
def create_multi_worker_setup(self): assert self.use_horovod and self.num_workers_per_hls > 1, "Horovod run requires at least 2 workers" self.run_config_env_variables[ 'NUM_WORKERS_PER_HLS'] = f"{self.num_workers_per_hls}" tmp_dir = get_canonical_path("$HOME/tmp/") run_per_ip(f"mkdir -p {str(tmp_dir)}", ['MULTI_HLS_IPS', 'PYTHONPATH'], False) print(f"MULTI_HLS_IPS={os.environ.get('MULTI_HLS_IPS')}") # OpenMPI process bind resource type. mpi_map_by = "socket" # Get lscpu cmd = 'lscpu | grep \"CPU(s):\"' lscpu_output = [] with subprocess.Popen(cmd, shell=True, executable='/bin/bash', stdout=subprocess.PIPE, stderr=subprocess.STDOUT) as proc: lscpu_output = proc.stdout.read() # Determine the optimal value of resources per process of OpenMPI binding based on local lscpu. if mpi_map_by == "socket": mpi_map_by_pe = int( lscpu_output.split()[1]) // self.num_workers_per_hls // 2 elif mpi_map_by == "slot": mpi_map_by_pe = int( lscpu_output.split()[1]) // self.num_workers_per_hls else: raise Exception("mpi_map_by must be either 'socket' or 'slot'.") print(f"mpi_map_by_pe = {mpi_map_by_pe}") output_file_name = str(tmp_dir.joinpath("demo_bert_log/")) self.mpirun_cmd = "mpirun" self.mpirun_cmd += " --allow-run-as-root" self.mpirun_cmd += f" --tag-output --merge-stderr-to-stdout --output-filename {output_file_name}" if mpi_map_by_pe > 0: self.mpirun_cmd += f" --bind-to core --map-by {mpi_map_by}:PE={mpi_map_by_pe}" hcl_config_path = '' if is_valid_multi_node_config(): hcl_config_path = self.create_multi_hls_setup(tmp_dir) else: hcl_config_path = self.create_single_hls_setup(tmp_dir) print(f"HCL_CONFIG_PATH = {str(os.environ.get('HCL_CONFIG_PATH'))}") print(f"hcl_config_path = {hcl_config_path} ->") print_file_contents(hcl_config_path) os.environ['MPIRUN_CMD'] = self.mpirun_cmd print( f"{self.__class__.__name__} create_multi_worker_setup(): self.mpirun_cmd = {self.mpirun_cmd}" ) print( f"{self.__class__.__name__} create_multi_worker_setup(): MPIRUN_CMD = {os.environ.get('MPIRUN_CMD')}" )
def prepare_results_path(self, results_dir): try: if self.use_horovod and is_valid_multi_node_config(): prepare_output_dir_path = Path(__file__).parent.parent.parent.joinpath('common').joinpath('prepare_output_dir.py') run_per_ip(f"python3 {str(prepare_output_dir_path)} {results_dir}", ['MULTI_HLS_IPS', 'PYTHONPATH'], False) else: prepare_output_dir.prepare_output_dir_r(results_dir) except Exception as exc: raise RuntimeError(f"Error in {self.__class__.__name__} prepare_results_path({results_dir})") from exc
def create_pretraining_data(self, seq_length, max_pred_per_seq): try: if self.use_horovod and is_valid_multi_node_config(): create_pt_data_path = Path(__file__).parent.joinpath('create_pretraining_data_overfit.py') run_per_ip(f"python3 {str(create_pt_data_path)} {self.dataset_path} {self.pretrained_model} {seq_length} {max_pred_per_seq}", ['MULTI_HLS_IPS', 'PYTHONPATH'], False) else: create_pretraining_data_overfit.create_pretraining_data_overfit_r(self.dataset_path, self.pretrained_model, seq_length, max_pred_per_seq) except Exception as exc: raise RuntimeError(f"Error in {self.__class__.__name__} create_pretraining_data({self.dataset_path} {self.pretrained_model} {seq_length} {max_pred_per_seq})") from exc
def check_dirs(self, largs): try: if self.use_horovod and is_valid_multi_node_config(): check_dirs_path = Path(__file__).parent.parent.parent.joinpath( 'common').joinpath('check_dirs.py') run_per_ip(f"python3 {str(check_dirs_path)} {largs}", ['MULTI_HLS_IPS', 'PYTHONPATH'], False) else: check_dirs.check_dirs_r(largs.split()) except Exception as exc: raise RuntimeError( f"Error in {self.__class__.__name__} check_dirs(largs)" ) from exc
def download_dataset(self): try: if self.use_horovod and is_valid_multi_node_config(): download_dataset_path = Path(__file__).parent.joinpath( 'download').joinpath('download_dataset.py') run_per_ip( f"python3 {str(download_dataset_path)} {self.args.dataset_path}", ['MULTI_HLS_IPS', 'PYTHONPATH'], False) else: download_dataset.download_dataset_r(self.args.dataset_path) except Exception as exc: raise RuntimeError( f"Error in {self.__class__.__name__} download_dataset()" ) from exc
def prepare_output_dir(self): try: if self.use_horovod and is_valid_multi_node_config(): prepare_output_dir_squad_path = Path(__file__).parent.joinpath( 'prepare_output_dir_squad.py') run_per_ip( f"python3 {str(prepare_output_dir_squad_path)} {self.args.output_dir} {self.batch_size} {self.max_seq_len}", ['MULTI_HLS_IPS', 'PYTHONPATH'], False) else: prepare_output_dir_squad.prepare_output_dir_squad_r( self.args.output_dir, self.batch_size, self.max_seq_len) except Exception as exc: raise RuntimeError( f"Error in {self.__class__.__name__} prepare_output_dir()" ) from exc
def download_pretrained_model(self, horovod_run): try: if horovod_run and is_valid_multi_node_config(): download_pretrained_model_path = Path( __file__).parent.joinpath('download').joinpath( 'download_pretrained_model.py') run_per_ip( f"python3 {str(download_pretrained_model_path)} {self.pretrained_url} {self.pretrained_model} False", ['MULTI_HLS_IPS', 'PYTHONPATH'], False) else: download_pretrained_model.download_pretrained_model_r( self.pretrained_url, self.pretrained_model, False) except Exception as exc: raise RuntimeError( f"Error in {self.__class__.__name__} download_pretrained_model()" ) from exc
def generate_mpi_hostfile(file_path): mpi_hostfile_path = '' if is_valid_multi_node_config(): multi_hls_nodes = get_multi_node_config_nodes() print("Generating MPI hostfile...") file_name = "hostfile" os.makedirs(get_canonical_path(file_path), mode=0o777, exist_ok=True) mpi_hostfile_path = get_canonical_path(file_path).joinpath(file_name) if os.path.exists(mpi_hostfile_path): #os.remove(mpi_hostfile_path) cmd = f"rm -f {str(mpi_hostfile_path)}" run_cmd_as_subprocess(cmd) print(f"Path: {mpi_hostfile_path}") out_fid = open(mpi_hostfile_path, 'a') config_str = '' for node in multi_hls_nodes: config_str += f"{node} slots=8\n" print(f"MPI hostfile: \n{config_str}") out_fid.write(config_str) out_fid.close() return mpi_hostfile_path