import argparse
from utilbox.regex_util import regex_key_val
from euterpe.common.loader import DataLoader
def parse():
parser = argparse.ArgumentParser()
parser.add_argument('--kv', type=str)
parser.add_argument('--set', type=str)
return parser.parse_args()
pass
if __name__ == '__main__':
args = parse()
list_kv = DataLoader._read_key_val(args.kv)
list_set = DataLoader._read_key(args.set)
list_subset = DataLoader._subset_data(list_kv, list_set)
for k, v in list_subset:
print('{} {}'.format(k, v))
pass
if opts['config'] is None:
opts['config'] = os.path.splitext(opts['model'])[0] + '.cfg'
model = ModelSerializer.load_config(opts['config'])
model.train(False)
model.load_state_dict(torch.load(opts['model']))
# tacotron util helper #
helper = TacotronHelper(feat_cfg)
if opts['gpu'] >= 0:
torch.cuda.set_device(opts['gpu'])
model.cuda()
data_cfg_target = yaml.load(open(opts['data_cfg_tgt']))
data_iterator_target = DataLoader.load_feat_single(
feat_path=data_cfg_target['feat']['all'])
data_cfg_source = yaml.load(open(opts['data_cfg_src']))
data_iterator_source = DataLoader.load_feat_single(
feat_path=data_cfg_source['feat']['all'])
feat_stat = pickle.load(open(data_cfg_source['feat']['stat'], 'rb'))
feat_sil = feat_sil_from_stat(feat_stat)
# encoding latin1 for python 3 #
scaler = pickle.load(open(data_cfg_source['scaler'], 'rb'),
encoding='latin1') if data_cfg_source.get(
'scaler', None) is not None else None
### generation process ###
# create folder for generated result #
if opts['path'] is None:
opts = vars(parse())
print(opts)
# set default device #
np.random.seed(opts['seed'])
torch.manual_seed(opts['seed'])
if opts['gpu'] >= 0:
torch.cuda.set_device(opts['gpu'])
torch.cuda.manual_seed(opts['seed'])
group = opts['group']
# dataset #
data_cfg = yaml.load(open(opts['data_cfg']))
feat_iterator = DataLoader.load_feat(data_cfg,
data_type=LoaderDataType.TYPE_NP,
in_memory=opts['mem'])
text_iterator = DataLoader.load_text(data_cfg)
feat_stat = pickle.load(open(data_cfg['feat']['stat'], 'rb'))
feat_sil = feat_sil_from_stat(feat_stat)
print("Finish loading dataset ...")
NDIM = feat_iterator['train'].get_feat_dim() * group
NVOCAB = len(text_iterator['train'].get_map_text2idx())
if opts['model_pt'] is not None:
model = ModelSerializer.load_config(
os.path.join(opts['model_pt'], 'model.cfg'))
model.load_state_dict(
os.makedirs(os.path.join(tmpdir, 'meta'), exist_ok=False)
print('\nCreate temporary dir: {}'.format(tmpdir), file=sys.stderr)
else:
tmpdir = opts['path']
# if not exist, create new dir #
if not os.path.isdir(tmpdir):
os.makedirs(tmpdir, exist_ok=False)
os.makedirs(os.path.join(tmpdir, 'meta'), exist_ok=False)
else:
assert os.listdir(
tmpdir) == [], "target directory path must be empty"
os.makedirs(os.path.join(tmpdir, 'meta'), exist_ok=False)
if list_kv is None:
if opts['set'] is not None:
data_iterator = DataLoader.load_feat_single(
feat_path=data_cfg['feat']['all'], key=opts['set'])
text_iterator = DataLoader.load_text_single(
text_path=data_cfg['text']['all'],
key=opts['set'],
vocab=map_text2idx)
else:
data_iterator = DataLoader.load_feat_single(
feat_path=data_cfg['feat']['all'])
text_iterator = DataLoader.load_text_single(
text_path=data_cfg['text']['all'], vocab=map_text2idx)
else:
assert opts['mode'] == 'pred'
text_iterator = TextIterator(text_kv=list_kv,
map_text2idx=map_text2idx)
# sort text from longest to shortest #
print(opts)
# set default device #
np.random.seed(opts['seed'])
torch.manual_seed(opts['seed'])
if opts['gpu'] >= 0:
torch.cuda.set_device(opts['gpu'])
torch.cuda.manual_seed(opts['seed'])
group = opts['group']
# dataset #
data_in_cfg = yaml.load(open(opts['data_in_cfg']))
data_out_cfg = yaml.load(open(opts['data_out_cfg']))
feat_in_iterator = DataLoader.load_feat(data_in_cfg,
data_type=LoaderDataType.TYPE_NP,
in_memory=opts['mem'])
feat_out_iterator = DataLoader.load_feat(data_out_cfg,
data_type=LoaderDataType.TYPE_NP,
in_memory=opts['mem'])
feat_in_stat = pickle.load(open(data_in_cfg['feat']['stat'], 'rb'))
feat_in_sil = feat_sil_from_stat(feat_in_stat)
feat_out_stat = pickle.load(open(data_out_cfg['feat']['stat'], 'rb'))
feat_out_sil = feat_sil_from_stat(feat_out_stat)
print("Finish loading dataset ...")
NDIM_IN = feat_in_iterator['train'].get_feat_dim() * group
if args.wav_scp is None and args.feat_scp is None:
logger.info('fallback ... read feat_scp from data_cfg')
args.feat_scp = data_cfg['feat']['all']
if args.mode == 'tf':
assert args.data_cfg is not None
text_iterator = TextIterator(path=data_cfg['text']['all'],
map_text2idx=map_text2idx)
if args.wav_scp is not None:
# list all wav files #
list_key_wav = regex_key_val.findall(open(args.wav_scp).read())
if args.set is not None:
if os.path.exists(args.set):
list_key_wav = DataLoader._subset_data(
list_key_wav, DataLoader._read_key(args.set))
else:
args.set = args.set.split(' ')
list_key_wav = DataLoader._subset_data(list_key_wav, args.set)
list_key_wav = sorted(list_key_wav, key=lambda x: x[0])
# lazy load -- saving memory #
def lazy_generate_feat(path, cfg):
_feat = generate_feat_opts(path=path, cfg=cfg)
if scaler is not None:
_feat = scaler.transform(_feat)
return _feat
list_feat = []
list_feat_len = []
list_key = [x[0] for x in list_key_wav]
opts = vars(parse())
assert_parse(opts)
print(opts)
# set default device #
np.random.seed(opts['seed'])
torch.manual_seed(opts['seed'])
if opts['gpu'] >= 0:
torch.cuda.set_device(opts['gpu'])
torch.cuda.manual_seed(opts['seed'])
random.seed(opts['seed'])
# dataset #
data_pair_cfg = yaml.load(open(opts['data_pair_cfg']))
feat_iterator = DataLoader.load_feat(data_pair_cfg,
data_type=LoaderDataType.TYPE_NP,
in_memory=opts['mem'])
text_iterator = DataLoader.load_text(data_pair_cfg)
map_text2idx = text_iterator['train'].get_map_text2idx()
data_unpair_text_cfg = yaml.load(open(opts['data_unpair_text_cfg']))
data_unpair_speech_cfg = yaml.load(open(opts['data_unpair_speech_cfg']))
# feat_iterator['unpair'] = DataIteratorNP.load_feat(data_unpair_speech_cfg)['train']
# text_iterator['unpair'] = DataIteratorNP.load_text(data_unpair_text_cfg)['train']
feat_iterator['unpair'] = DataLoader.load_feat_single(
data_unpair_speech_cfg['feat']['all'],
data_unpair_speech_cfg['feat']['set']['train'])
text_iterator['unpair'] = DataLoader.load_text_single(
data_unpair_speech_cfg['text']['all'],
data_unpair_speech_cfg['text']['set']['train'],
vocab=map_text2idx)
type=str,
default=None,
help='path to save speaker vector')
parser.add_argument('--dump', action='store_true', default=False)
return parser.parse_args()
if __name__ == '__main__':
args = parse()
args.config = os.path.splitext(args.model)[0] + '.cfg'
data_cfg = yaml.load(open(args.data_cfg))
# construct all subset feat & feat_len
if os.path.exists(args.key):
_key = DataLoader._read_key(args.key)
else:
_key = args.key.split()
_feat_all = DataLoader._read_key_val(data_cfg['feat']['all'])
_feat_len_all_path = '{}_len{}'.format(
*os.path.splitext(data_cfg['feat']['all']))
_feat_len_all = DataLoader._read_key_val(_feat_len_all_path)
_feat_kv = DataLoader._subset_data(_feat_all, _key)
_feat_len_kv = DataLoader._subset_data(_feat_len_all, _key)
model = ModelSerializer.load_config(args.config)
model.load_state_dict(torch.load(args.model))
model.eval() # set as eval mode
if args.gpu >= 0:
torch.cuda.set_device(args.gpu)