def _reinitialize_logging_and_call_train(args): # NOTE: it looks like logging setting is cleared # by launching processes with "spawn" method. # Within each worker process, # logging configuraiton must be set again. from espnet.asr.pytorch_backend.asr import train setup_logging(args.verbose) train(args)
def main(cmd_args): """Run the main training function.""" parser = get_parser() args, _ = parser.parse_known_args(cmd_args) if args.backend == "chainer" and args.train_dtype != "float32": raise NotImplementedError( f"chainer backend does not support --train-dtype {args.train_dtype}." "Use --dtype float32." ) if args.ngpu == 0 and args.train_dtype in ("O0", "O1", "O2", "O3", "float16"): raise ValueError( f"--train-dtype {args.train_dtype} does not support the CPU backend." ) from espnet.utils.dynamic_import import dynamic_import if args.model_module is None: model_module = "espnet.nets." + args.backend + "_backend.e2e_asr:E2E" else: model_module = args.model_module model_class = dynamic_import(model_module) model_class.add_arguments(parser) args = parser.parse_args(cmd_args) args.model_module = model_module if "chainer_backend" in args.model_module: args.backend = "chainer" if "pytorch_backend" in args.model_module: args.backend = "pytorch" # add version info in args args.version = __version__ # logging info if args.verbose > 0: logging.basicConfig( level=logging.INFO, format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", ) else: logging.basicConfig( level=logging.WARN, format="%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s", ) logging.warning("Skip DEBUG/INFO messages") # If --ngpu is not given, # 1. if CUDA_VISIBLE_DEVICES is set, all visible devices # 2. if nvidia-smi exists, use all devices # 3. else ngpu=0 if args.ngpu is None: cvd = os.environ.get("CUDA_VISIBLE_DEVICES") if cvd is not None: ngpu = len(cvd.split(",")) else: logging.warning("CUDA_VISIBLE_DEVICES is not set.") try: p = subprocess.run( ["nvidia-smi", "-L"], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) except (subprocess.CalledProcessError, FileNotFoundError): ngpu = 0 else: ngpu = len(p.stderr.decode().split("\n")) - 1 else: if is_torch_1_2_plus and args.ngpu != 1: logging.debug( "There are some bugs with multi-GPU processing in PyTorch 1.2+" + " (see https://github.com/pytorch/pytorch/issues/21108)" ) ngpu = args.ngpu logging.info(f"ngpu: {ngpu}") # display PYTHONPATH logging.info("python path = " + os.environ.get("PYTHONPATH", "(None)")) # set random seed logging.info("random seed = %d" % args.seed) random.seed(args.seed) np.random.seed(args.seed) # load dictionary for debug log if args.dict is not None: with open(args.dict, "rb") as f: dictionary = f.readlines() char_list = [entry.decode("utf-8").split(" ")[0] for entry in dictionary] char_list.insert(0, "<blank>") char_list.append("<eos>") # for non-autoregressive maskctc model if "maskctc" in args.model_module: char_list.append("<mask>") args.char_list = char_list else: args.char_list = None # train logging.info("backend = " + args.backend) if args.num_spkrs == 1: if args.backend == "chainer": from espnet.asr.chainer_backend.asr import train train(args) elif args.backend == "pytorch": from espnet.asr.pytorch_backend.asr import train train(args) else: raise ValueError("Only chainer and pytorch are supported.") else: # FIXME(kamo): Support --model-module if args.backend == "pytorch": from espnet.asr.pytorch_backend.asr_mix import train train(args) else: raise ValueError("Only pytorch is supported.")
def main(cmd_args): parser = get_parser() args, _ = parser.parse_known_args(cmd_args) if args.backend == "chainer" and args.train_dtype != "float32": raise NotImplementedError( f"chainer backend does not support --train-dtype {args.train_dtype}." "Use --dtype float32.") if args.ngpu == 0 and args.train_dtype in ("O0", "O1", "O2", "O3", "float16"): raise ValueError(f"--train-dtype {args.train_dtype} does not support the CPU backend.") from espnet.utils.dynamic_import import dynamic_import if args.model_module is None: model_module = "espnet.nets." + args.backend + "_backend.e2e_asr:E2E" else: model_module = args.model_module model_class = dynamic_import(model_module) model_class.add_arguments(parser) args = parser.parse_args(cmd_args) args.model_module = model_module if 'chainer_backend' in args.model_module: args.backend = 'chainer' if 'pytorch_backend' in args.model_module: args.backend = 'pytorch' # logging info if args.verbose > 0: logging.basicConfig( level=logging.INFO, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') else: logging.basicConfig( level=logging.WARN, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') logging.warning('Skip DEBUG/INFO messages') # If --ngpu is not given, # 1. if CUDA_VISIBLE_DEVICES is set, all visible devices # 2. if nvidia-smi exists, use all devices # 3. else ngpu=0 if args.ngpu is None: cvd = os.environ.get("CUDA_VISIBLE_DEVICES") if cvd is not None: ngpu = len(cvd.split(',')) else: logging.warning("CUDA_VISIBLE_DEVICES is not set.") try: p = subprocess.run(['nvidia-smi', '-L'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) except (subprocess.CalledProcessError, FileNotFoundError): ngpu = 0 else: ngpu = len(p.stderr.decode().split('\n')) - 1 else: if is_torch_1_2_plus: assert args.ngpu == 1, "There are some bugs with multi-GPU processing in PyTorch 1.2+" \ " (see https://github.com/pytorch/pytorch/issues/21108)" ngpu = args.ngpu logging.info(f"ngpu: {ngpu}") # display PYTHONPATH logging.info('python path = ' + os.environ.get('PYTHONPATH', '(None)')) # set random seed logging.info('random seed = %d' % args.seed) random.seed(args.seed) np.random.seed(args.seed) # load dictionary for debug log if args.dict is not None: with open(args.dict, 'rb') as f: dictionary = f.readlines() char_list = [entry.decode('utf-8').split(' ')[0] for entry in dictionary] char_list.insert(0, '<blank>') char_list.append('<eos>') args.char_list = char_list else: args.char_list = None # train logging.info('backend = ' + args.backend) if args.num_spkrs == 1: if args.backend == "chainer": from espnet.asr.chainer_backend.asr import train train(args) elif args.backend == "pytorch": from espnet.asr.pytorch_backend.asr import train train(args) else: raise ValueError("Only chainer and pytorch are supported.") else: # FIXME(kamo): Support --model-module if args.backend == "pytorch": from espnet.asr.pytorch_backend.asr_mix import train train(args) else: raise ValueError("Only pytorch is supported.")
def main(cmd_args): parser = argparse.ArgumentParser() # general configuration parser.add_argument('--ngpu', default=0, type=int, help='Number of GPUs') parser.add_argument('--backend', default='chainer', type=str, choices=['chainer', 'pytorch'], help='Backend library') parser.add_argument('--outdir', type=str, required=True, help='Output directory') parser.add_argument('--debugmode', default=1, type=int, help='Debugmode') parser.add_argument('--dict', required=True, help='Dictionary') parser.add_argument('--seed', default=1, type=int, help='Random seed') parser.add_argument('--debugdir', type=str, help='Output directory for debugging') parser.add_argument('--resume', '-r', default='', nargs='?', help='Resume the training from snapshot') parser.add_argument('--minibatches', '-N', type=int, default='-1', help='Process only N minibatches (for debug)') parser.add_argument('--verbose', '-V', default=0, type=int, help='Verbose option') parser.add_argument('--tensorboard-dir', default=None, type=str, nargs='?', help="Tensorboard log dir path") # task related parser.add_argument('--train-json', type=str, default=None, help='Filename of train label data (json)') parser.add_argument('--valid-json', type=str, default=None, help='Filename of validation label data (json)') # network architecture parser.add_argument('--model-module', type=str, default=None, help='model defined module (default: espnet.nets.xxx_backend.e2e_asr)') # encoder parser.add_argument('--num-spkrs', default=1, type=int, choices=[1, 2], help='Number of speakers in the speech.') parser.add_argument('--etype', default='blstmp', type=str, choices=['lstm', 'blstm', 'lstmp', 'blstmp', 'vgglstmp', 'vggblstmp', 'vgglstm', 'vggblstm', 'gru', 'bgru', 'grup', 'bgrup', 'vgggrup', 'vggbgrup', 'vgggru', 'vggbgru'], help='Type of encoder network architecture') parser.add_argument('--elayers-sd', default=4, type=int, help='Number of encoder layers for speaker differentiate part. (multi-speaker asr mode only)') parser.add_argument('--elayers', default=4, type=int, help='Number of encoder layers (for shared recognition part in multi-speaker asr mode)') parser.add_argument('--eunits', '-u', default=300, type=int, help='Number of encoder hidden units') parser.add_argument('--eprojs', default=320, type=int, help='Number of encoder projection units') parser.add_argument('--subsample', default=1, type=str, help='Subsample input frames x_y_z means subsample every x frame at 1st layer, ' 'every y frame at 2nd layer etc.') # loss parser.add_argument('--ctc_type', default='warpctc', type=str, choices=['builtin', 'warpctc'], help='Type of CTC implementation to calculate loss.') # attention parser.add_argument('--atype', default='dot', type=str, choices=['noatt', 'dot', 'add', 'location', 'coverage', 'coverage_location', 'location2d', 'location_recurrent', 'multi_head_dot', 'multi_head_add', 'multi_head_loc', 'multi_head_multi_res_loc'], help='Type of attention architecture') parser.add_argument('--adim', default=320, type=int, help='Number of attention transformation dimensions') parser.add_argument('--awin', default=5, type=int, help='Window size for location2d attention') parser.add_argument('--aheads', default=4, type=int, help='Number of heads for multi head attention') parser.add_argument('--aconv-chans', default=-1, type=int, help='Number of attention convolution channels \ (negative value indicates no location-aware attention)') parser.add_argument('--aconv-filts', default=100, type=int, help='Number of attention convolution filters \ (negative value indicates no location-aware attention)') parser.add_argument('--spa', action='store_true', help='Enable speaker parallel attention.') # decoder parser.add_argument('--dtype', default='lstm', type=str, choices=['lstm', 'gru'], help='Type of decoder network architecture') parser.add_argument('--dlayers', default=1, type=int, help='Number of decoder layers') parser.add_argument('--dunits', default=320, type=int, help='Number of decoder hidden units') parser.add_argument('--mtlalpha', default=0.5, type=float, help='Multitask learning coefficient, alpha: alpha*ctc_loss + (1-alpha)*att_loss ') parser.add_argument('--lsm-type', const='', default='', type=str, nargs='?', choices=['', 'unigram'], help='Apply label smoothing with a specified distribution type') parser.add_argument('--lsm-weight', default=0.0, type=float, help='Label smoothing weight') parser.add_argument('--sampling-probability', default=0.0, type=float, help='Ratio of predicted labels fed back to decoder') # recognition options to compute CER/WER parser.add_argument('--report-cer', default=False, action='store_true', help='Compute CER on development set') parser.add_argument('--report-wer', default=False, action='store_true', help='Compute WER on development set') parser.add_argument('--nbest', type=int, default=1, help='Output N-best hypotheses') parser.add_argument('--beam-size', type=int, default=4, help='Beam size') parser.add_argument('--penalty', default=0.0, type=float, help='Incertion penalty') parser.add_argument('--maxlenratio', default=0.0, type=float, help="""Input length ratio to obtain max output length. If maxlenratio=0.0 (default), it uses a end-detect function to automatically find maximum hypothesis lengths""") parser.add_argument('--minlenratio', default=0.0, type=float, help='Input length ratio to obtain min output length') parser.add_argument('--ctc-weight', default=0.3, type=float, help='CTC weight in joint decoding') parser.add_argument('--rnnlm', type=str, default=None, help='RNNLM model file to read') parser.add_argument('--rnnlm-conf', type=str, default=None, help='RNNLM model config file to read') parser.add_argument('--lm-weight', default=0.1, type=float, help='RNNLM weight.') parser.add_argument('--sym-space', default='<space>', type=str, help='Space symbol') parser.add_argument('--sym-blank', default='<blank>', type=str, help='Blank symbol') # model (parameter) related parser.add_argument('--dropout-rate', default=0.0, type=float, help='Dropout rate for the encoder') parser.add_argument('--dropout-rate-decoder', default=0.0, type=float, help='Dropout rate for the decoder') # minibatch related parser.add_argument('--sortagrad', default=0, type=int, nargs='?', help="How many epochs to use sortagrad for. 0 = deactivated, -1 = all epochs") parser.add_argument('--batch-size', '-b', default=50, type=int, help='Batch size') parser.add_argument('--maxlen-in', default=800, type=int, metavar='ML', help='Batch size is reduced if the input sequence length > ML') parser.add_argument('--maxlen-out', default=150, type=int, metavar='ML', help='Batch size is reduced if the output sequence length > ML') parser.add_argument('--n_iter_processes', default=0, type=int, help='Number of processes of iterator') parser.add_argument('--preprocess-conf', type=str, default=None, help='The configuration file for the pre-processing') # optimization related parser.add_argument('--opt', default='adadelta', type=str, choices=['adadelta', 'adam', 'noam'], help='Optimizer') parser.add_argument('--accum-grad', default=1, type=int, help='Number of gradient accumuration') parser.add_argument('--eps', default=1e-8, type=float, help='Epsilon constant for optimizer') parser.add_argument('--eps-decay', default=0.01, type=float, help='Decaying ratio of epsilon') parser.add_argument('--weight-decay', default=0.0, type=float, help='Weight decay ratio') parser.add_argument('--criterion', default='acc', type=str, choices=['loss', 'acc'], help='Criterion to perform epsilon decay') parser.add_argument('--threshold', default=1e-4, type=float, help='Threshold to stop iteration') parser.add_argument('--epochs', '-e', default=30, type=int, help='Maximum number of epochs') parser.add_argument('--early-stop-criterion', default='validation/main/acc', type=str, nargs='?', help="Value to monitor to trigger an early stopping of the training") parser.add_argument('--patience', default=3, type=int, nargs='?', help="Number of epochs to wait without improvement before stopping the training") parser.add_argument('--grad-clip', default=5, type=float, help='Gradient norm threshold to clip') parser.add_argument('--num-save-attention', default=3, type=int, help='Number of samples of attention to be saved') # transfer learning related parser.add_argument('--asr-model', default=False, nargs='?', help='Pre-trained ASR model') parser.add_argument('--mt-model', default=False, nargs='?', help='Pre-trained MT model') args, _ = parser.parse_known_args(cmd_args) from importlib import import_module if args.model_module is not None: model_module = import_module(args.model_module) assert hasattr(model_module, "E2E") if hasattr(model_module, "add_arguments"): model_module.add_arguments(parser) args = parser.parse_args(cmd_args) if args.model_module is None: args.model_module = "espnet.nets." + args.backend + "_backend.e2e_asr" else: args.backend = "chainer" if "chainer" in args.model_module else "pytorch" # logging info if args.verbose > 0: logging.basicConfig( level=logging.INFO, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') else: logging.basicConfig( level=logging.WARN, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') logging.warning('Skip DEBUG/INFO messages') # check CUDA_VISIBLE_DEVICES if args.ngpu > 0: # python 2 case if platform.python_version_tuple()[0] == '2': if "clsp.jhu.edu" in subprocess.check_output(["hostname", "-f"]): cvd = subprocess.check_output(["/usr/local/bin/free-gpu", "-n", str(args.ngpu)]).strip() logging.info('CLSP: use gpu' + cvd) os.environ['CUDA_VISIBLE_DEVICES'] = cvd # python 3 case else: if "clsp.jhu.edu" in subprocess.check_output(["hostname", "-f"]).decode(): cvd = subprocess.check_output(["/usr/local/bin/free-gpu", "-n", str(args.ngpu)]).decode().strip() logging.info('CLSP: use gpu' + cvd) os.environ['CUDA_VISIBLE_DEVICES'] = cvd cvd = os.environ.get("CUDA_VISIBLE_DEVICES") if cvd is None: logging.warning("CUDA_VISIBLE_DEVICES is not set.") elif args.ngpu != len(cvd.split(",")): logging.error("#gpus is not matched with CUDA_VISIBLE_DEVICES.") sys.exit(1) # display PYTHONPATH logging.info('python path = ' + os.environ.get('PYTHONPATH', '(None)')) # set random seed logging.info('random seed = %d' % args.seed) random.seed(args.seed) np.random.seed(args.seed) # load dictionary for debug log if args.dict is not None: with open(args.dict, 'rb') as f: dictionary = f.readlines() char_list = [entry.decode('utf-8').split(' ')[0] for entry in dictionary] char_list.insert(0, '<blank>') char_list.append('<eos>') args.char_list = char_list else: args.char_list = None # train logging.info('backend = ' + args.backend) if args.num_spkrs == 1: if args.backend == "chainer": from espnet.asr.chainer_backend.asr import train train(args) elif args.backend == "pytorch": from espnet.asr.pytorch_backend.asr import train train(args) else: raise ValueError("Only chainer and pytorch are supported.") elif args.num_spkrs > 1: if args.backend == "pytorch": from espnet.asr.pytorch_backend.asr_mix import train train(args) else: raise ValueError("Only pytorch is supported.")
def main(cmd_args): parser = get_parser() args, _ = parser.parse_known_args(cmd_args) from espnet.utils.dynamic_import import dynamic_import if args.model_module is not None: model_class = dynamic_import(args.model_module) model_class.add_arguments(parser) args = parser.parse_args(cmd_args) if args.model_module is None: args.model_module = "espnet.nets." + args.backend + "_backend.e2e_asr:E2E" if 'chainer_backend' in args.model_module: args.backend = 'chainer' if 'pytorch_backend' in args.model_module: args.backend = 'pytorch' # logging info if args.verbose > 0: logging.basicConfig( level=logging.INFO, format= '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') else: logging.basicConfig( level=logging.WARN, format= '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') logging.warning('Skip DEBUG/INFO messages') # check CUDA_VISIBLE_DEVICES if args.ngpu > 0: # python 2 case if platform.python_version_tuple()[0] == '2': if "clsp.jhu.edu" in subprocess.check_output(["hostname", "-f"]): cvd = subprocess.check_output( ["/usr/local/bin/free-gpu", "-n", str(args.ngpu)]).strip() logging.info('CLSP: use gpu' + cvd) os.environ['CUDA_VISIBLE_DEVICES'] = cvd # python 3 case else: if "clsp.jhu.edu" in subprocess.check_output(["hostname", "-f"]).decode(): cvd = subprocess.check_output( ["/usr/local/bin/free-gpu", "-n", str(args.ngpu)]).decode().strip() logging.info('CLSP: use gpu' + cvd) os.environ['CUDA_VISIBLE_DEVICES'] = cvd cvd = os.environ.get("CUDA_VISIBLE_DEVICES") if cvd is None: logging.warning("CUDA_VISIBLE_DEVICES is not set.") elif args.ngpu != len(cvd.split(",")): logging.error("#gpus is not matched with CUDA_VISIBLE_DEVICES.") sys.exit(1) # display PYTHONPATH logging.info('python path = ' + os.environ.get('PYTHONPATH', '(None)')) # set random seed logging.info('random seed = %d' % args.seed) random.seed(args.seed) np.random.seed(args.seed) # load dictionary for debug log if args.dict is not None: with open(args.dict, 'rb') as f: dictionary = f.readlines() char_list = [ entry.decode('utf-8').split(' ')[0] for entry in dictionary ] char_list.insert(0, '<blank>') char_list.append('<eos>') args.char_list = char_list else: args.char_list = None # train logging.info('backend = ' + args.backend) if args.num_spkrs == 1: if args.backend == "chainer": from espnet.asr.chainer_backend.asr import train train(args) elif args.backend == "pytorch": if args.parallel_mode == 'ddp': import torch import torch.distributed as dist torch.cuda.set_device(args.local_rank) dist.init_process_group(backend='nccl', init_method='env://') from espnet.asr.pytorch_backend.asrtts import train train(args) else: raise ValueError("Only chainer and pytorch are supported.") elif args.num_spkrs > 1: if args.backend == "pytorch": from espnet.asr.pytorch_backend.asr_mix import train train(args) else: raise ValueError("Only pytorch is supported.")
def main(cmd_args): """Run the main training function.""" parser = get_parser() args, _ = parser.parse_known_args(cmd_args) if args.backend == "chainer" and args.train_dtype != "float32": raise NotImplementedError( f"chainer backend does not support --train-dtype {args.train_dtype}." "Use --dtype float32." ) if args.ngpu == 0 and args.train_dtype in ("O0", "O1", "O2", "O3", "float16"): raise ValueError( f"--train-dtype {args.train_dtype} does not support the CPU backend." ) from espnet.utils.dynamic_import import dynamic_import if args.model_module is None: if args.num_spkrs == 1: model_module = "espnet.nets." + args.backend + "_backend.e2e_asr:E2E" else: model_module = "espnet.nets." + args.backend + "_backend.e2e_asr_mix:E2E" else: model_module = args.model_module model_class = dynamic_import(model_module) model_class.add_arguments(parser) args = parser.parse_args(cmd_args) args.model_module = model_module if "chainer_backend" in args.model_module: args.backend = "chainer" if "pytorch_backend" in args.model_module: args.backend = "pytorch" # add version info in args args.version = __version__ # logging info setup_logging(args.verbose) # If --ngpu is not given, # 1. if CUDA_VISIBLE_DEVICES is set, all visible devices # 2. if nvidia-smi exists, use all devices # 3. else ngpu=0 if args.ngpu is None: cvd = os.environ.get("CUDA_VISIBLE_DEVICES") if cvd is not None: ngpu = len(cvd.split(",")) else: logging.warning("CUDA_VISIBLE_DEVICES is not set.") try: p = subprocess.run( ["nvidia-smi", "-L"], stdout=subprocess.PIPE, stderr=subprocess.PIPE ) except (subprocess.CalledProcessError, FileNotFoundError): ngpu = 0 else: ngpu = len(p.stderr.decode().split("\n")) - 1 else: if args.ngpu != 1: logging.debug( "There are some bugs with multi-GPU processing in PyTorch 1.2+" + " (see https://github.com/pytorch/pytorch/issues/21108)" ) ngpu = args.ngpu if args.use_ddp and ngpu <= 0: raise ValueError("DDP requires at least 1 GPU.") logging.info(f"ngpu: {ngpu}") # display PYTHONPATH logging.info("python path = " + os.environ.get("PYTHONPATH", "(None)")) # set random seed logging.info("random seed = %d" % args.seed) random.seed(args.seed) np.random.seed(args.seed) # load dictionary for debug log if args.dict is not None: with open(args.dict, "rb") as f: dictionary = f.readlines() char_list = [entry.decode("utf-8").split(" ")[0] for entry in dictionary] char_list.insert(0, "<blank>") char_list.append("<eos>") # for non-autoregressive maskctc model if "maskctc" in args.model_module: char_list.append("<mask>") args.char_list = char_list else: args.char_list = None # train logging.info("backend = " + args.backend) if args.use_ddp: # When using DDP, only PyTorch is supported. # Chainer is out-of-scope. if args.num_spkrs == 1: if args.backend == "chainer": raise ValueError("Chainer with DDP is not supported.") from espnet.distributed.pytorch_backend.launch import ( launch, set_start_method, ) # NOTE: it's necessary to set "spawn" as a multiprocessing # start method. Because, in this use case, CUDA initialization # procedure has been already done, but CUDA context can't be # shared with processes. # By default, multiprocessing tries to launch a process with # "fork" method. But, it will make processes which share # memory address spaces with a parent process. # To ensure a separate memory space, "spawn" method is required. set_start_method("spawn") launch(_reinitialize_logging_and_call_train, args, args.ngpu) else: raise ValueError("Single speaker is only supported when using DDP.") else: if args.num_spkrs == 1: if args.backend == "chainer": from espnet.asr.chainer_backend.asr import train train(args) elif args.backend == "pytorch": from espnet.asr.pytorch_backend.asr import train train(args) else: raise ValueError("Only chainer and pytorch are supported.") else: # FIXME(kamo): Support --model-module if args.backend == "pytorch": from espnet.asr.pytorch_backend.asr_mix import train train(args) else: raise ValueError("Only pytorch is supported.")
def main(cmd_args): parser = get_parser() args, _ = parser.parse_known_args(cmd_args) from espnet.utils.dynamic_import import dynamic_import if args.model_module is not None: model_class = dynamic_import(args.model_module) model_class.add_arguments(parser) args = parser.parse_args(cmd_args) if args.model_module is None: args.model_module = "espnet.nets." + args.backend + "_backend.e2e_asr:E2E" if 'chainer_backend' in args.model_module: args.backend = 'chainer' if 'pytorch_backend' in args.model_module: args.backend = 'pytorch' # logging info if args.verbose > 0: logging.basicConfig( level=logging.INFO, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') else: logging.basicConfig( level=logging.WARN, format='%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') logging.warning('Skip DEBUG/INFO messages') # If --ngpu is not given, # 1. if CUDA_VISIBLE_DEVICES is set, all visible devices # 2. if nvidia-smi exists, use all devices # 3. else ngpu=0 if args.ngpu is None: cvd = os.environ.get("CUDA_VISIBLE_DEVICES") if cvd is not None: ngpu = len(cvd.split(',')) else: logging.warning("CUDA_VISIBLE_DEVICES is not set.") try: p = subprocess.run(['nvidia-smi', '-L'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) except (subprocess.CalledProcessError, FileNotFoundError): ngpu = 0 else: ngpu = len(p.stderr.decode().split('\n')) - 1 else: ngpu = args.ngpu logging.info(f"ngpu: {ngpu}") # display PYTHONPATH logging.info('python path = ' + os.environ.get('PYTHONPATH', '(None)')) # set random seed logging.info('random seed = %d' % args.seed) random.seed(args.seed) np.random.seed(args.seed) # load dictionary for debug log if args.dict is not None: with open(args.dict, 'rb') as f: dictionary = f.readlines() char_list = [entry.decode('utf-8').split(' ')[0] for entry in dictionary] char_list.insert(0, '<blank>') char_list.append('<eos>') args.char_list = char_list else: args.char_list = None # train logging.info('backend = ' + args.backend) if args.num_spkrs == 1: if args.backend == "chainer": from espnet.asr.chainer_backend.asr import train train(args) elif args.backend == "pytorch": from espnet.asr.pytorch_backend.asr import train train(args) else: raise ValueError("Only chainer and pytorch are supported.") elif args.num_spkrs > 1: if args.backend == "pytorch": from espnet.asr.pytorch_backend.asr_mix import train train(args) else: raise ValueError("Only pytorch is supported.")
def main(cmd_args): parser = get_parser() args, _ = parser.parse_known_args(cmd_args) if args.backend == "chainer" and args.train_dtype != "float32": raise NotImplementedError( f"chainer backend does not support --train-dtype {args.train_dtype}." "Use --dtype float32.") if args.ngpu == 0 and args.train_dtype in ("O0", "O1", "O2", "O3", "float16"): raise ValueError( f"--train-dtype {args.train_dtype} does not support the CPU backend." ) from espnet.utils.dynamic_import import dynamic_import if args.model_module is None: model_module = "espnet.nets." + args.backend + "_backend.e2e_asr:E2E" else: model_module = args.model_module model_class = dynamic_import(model_module) model_class.add_arguments(parser) args = parser.parse_args(cmd_args) args.model_module = model_module if 'chainer_backend' in args.model_module: args.backend = 'chainer' if 'pytorch_backend' in args.model_module: args.backend = 'pytorch' # logging info if args.verbose > 0: logging.basicConfig( level=logging.INFO, format= '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') else: logging.basicConfig( level=logging.WARN, format= '%(asctime)s (%(module)s:%(lineno)d) %(levelname)s: %(message)s') logging.warning('Skip DEBUG/INFO messages') # If --ngpu is not given, # 1. if CUDA_VISIBLE_DEVICES is set, all visible devices # 2. if nvidia-smi exists, use all devices # 3. else ngpu=0 if args.ngpu is None: cvd = os.environ.get("CUDA_VISIBLE_DEVICES") if cvd is not None: ngpu = len(cvd.split(',')) else: logging.warning("CUDA_VISIBLE_DEVICES is not set.") try: p = subprocess.run(['nvidia-smi', '-L'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) except (subprocess.CalledProcessError, FileNotFoundError): ngpu = 0 else: ngpu = len(p.stderr.decode().split('\n')) - 1 else: if "fit.vutbr.cz" in subprocess.check_output(["hostname", "-f"]).decode(): command = 'nvidia-smi --query-gpu=memory.free,memory.total \ --format=csv |tail -n+2| awk \'BEGIN{FS=" "}{if ($1 / $3 > 0.99) print NR-1}\'' try: cvd = str( subprocess.check_output( command, shell=True).decode().rsplit('\n')[0:args.ngpu]) cvd = cvd.replace("]", "") cvd = cvd.replace("[", "") cvd = cvd.replace("'", "") logging.info("Selected GPU is: " + str(cvd)) os.environ['CUDA_VISIBLE_DEVICES'] = cvd except subprocess.CalledProcessError: logging.info("No GPU seems to be available") #ngpu = args.ngpu #logging.info(f"ngpu: {ngpu}") # display PYTHONPATH logging.info('python path = ' + os.environ.get('PYTHONPATH', '(None)')) # set random seed logging.info('random seed = %d' % args.seed) random.seed(args.seed) np.random.seed(args.seed) # load dictionary for debug log if args.dict is not None: with open(args.dict, 'rb') as f: dictionary = f.readlines() char_list = [ entry.decode('utf-8').split(' ')[0] for entry in dictionary ] char_list.insert(0, '<blank>') char_list.append('<eos>') args.char_list = char_list else: args.char_list = None # train logging.info('backend = ' + args.backend) if args.num_spkrs == 1: if args.backend == "chainer": from espnet.asr.chainer_backend.asr import train train(args) elif args.backend == "pytorch": from espnet.asr.pytorch_backend.asr import train train(args) else: raise ValueError("Only chainer and pytorch are supported.") else: # FIXME(kamo): Support --model-module if args.backend == "pytorch": from espnet.asr.pytorch_backend.asr_mix import train train(args) else: raise ValueError("Only pytorch is supported.")