def get_upstream_model(args):
print('[run_downstream] - getting upstream model:', args.upstream)
if args.upstream == 'transformer':
options = {
'ckpt_file': args.ckpt,
'load_pretrain': 'True',
'no_grad': 'True' if not args.fine_tune else 'False',
'dropout': 'default',
'spec_aug': 'False',
'spec_aug_prev': 'True',
'weighted_sum': 'True' if args.weighted_sum else 'False',
'select_layer': -1,
}
upstream_model = TRANSFORMER(options, args.input_dim)
upstream_model.permute_input = False
elif args.upstream == 'apc':
raise NotImplementedError
elif args.upstream == 'baseline':
upstream_model = dummy_upstream(args.input_dim)
else:
raise NotImplementedError ######### plug in your upstream pre-trained model here #########
assert (hasattr(upstream_model, 'forward'))
assert (hasattr(upstream_model, 'out_dim'))
return upstream_model
def test_transformer(args, input_dim):
from transformer.nn_transformer import TRANSFORMER
options = {'ckpt_file' : args.test,
'load_pretrain' : 'True',
'no_grad' : 'True',
'dropout' : 'default',
'spec_aug' : 'False',
'spec_aug_prev' : 'True',
'weighted_sum' : 'False',
'select_layer' : -1,
}
upstream_model = TRANSFORMER(options, input_dim)
print('[upstream runner] - successfully loaded, valid checkpoint: ', args.test)
return upstream_model
def get_upstream_model(upstream_opts, upconfig, input_dim, ckpt):
start_new = strtobool(str(upstream_opts['start_new']))
fine_tune = strtobool(str(upstream_opts['fine_tune']))
specaug = strtobool(str(upstream_opts['specaug']))
encoder_feat = strtobool(str(upstream_opts['encoder_feat']))
options = {
'ckpt_file': ckpt,
'load_pretrain': 'True' if not start_new else 'False',
'no_grad': 'True' if not fine_tune else 'False',
'dropout': 'default',
'spec_aug': 'False',
'spec_aug_prev': 'True' if specaug else 'False',
'weighted_sum': 'False',
'select_layer': -1,
'encoder_feat': 'True' if encoder_feat else 'False'
}
if upconfig == 'default':
upconfig = None
upstream_model = TRANSFORMER(options, input_dim, config=upconfig)
upstream_model.permute_input = False
assert (hasattr(upstream_model, 'forward'))
assert (hasattr(upstream_model, 'out_dim'))
return upstream_model
def run_over_layer(layer, real_todo, adve_todo):
if layer != 'feature':
options = {
'ckpt_file':
'result/result_transformer/mockingjay/LinearLarge-libri/model-500000.ckpt',
'load_pretrain': 'True',
'no_grad': 'True',
'dropout': 'default',
'spec_aug': 'False',
'spec_aug_prev': 'True',
'weighted_sum': 'False',
'select_layer': layer,
'permute_input': 'True',
}
mockingjay = TRANSFORMER(options=options, inp_dim=160)
mockingjay.eval()
episode = []
for i in range(len(real_todo)):
r = torch.FloatTensor(np.load(real_todo[i])).unsqueeze(1)
a = torch.FloatTensor(np.load(adve_todo[i])).unsqueeze(1)
min_len = min(r.shape[0], a.shape[0])
if layer == 'feature':
l2 = compute_lnsr(r[:min_len].data.cpu().numpy(),
a[:min_len].data.cpu().numpy(),
norm_L2=True)
else:
r_hidd = mockingjay(r[:min_len])
a_hidd = mockingjay(a[:min_len])
l2 = compute_lnsr(r_hidd.data.cpu().numpy(),
a_hidd.data.cpu().numpy(),
norm_L2=True)
episode.append(l2)
return np.mean(episode)
permute_input: str, ['True', 'False'], this attribute is for the forward method. If Ture then input ouput is in the shape of (T, B, D), if False then in (B, T, D)
"""
options = {
'ckpt_file':
'./result/result_transformer/on-the-fly-melBase960-b12-T-libri/states-1000000.ckpt',
'load_pretrain': 'True',
'no_grad': 'True',
'dropout': 'default',
'spec_aug': 'False',
'spec_aug_prev': 'True',
'weighted_sum': 'False',
'select_layer': -1,
'permute_input': 'False',
}
# setup the transformer model
model = TRANSFORMER(options=options,
inp_dim=0) # set inp_dim to 0 for auto setup
# load raw wav
example_wav = '../LibriSpeech/test-clean/61/70970/61-70970-0000.flac'
input_wav = TRANSFORMER.load_data(
example_wav, **model.config['online']) # size: (seq_len, dim) = (97200, 1)
# forward
input_wav = input_wav.unsqueeze(
0) # add batch dim, size: (batch, seq_len, dim) = (1, 97200, 1)
output_repr = model(
input_wav
) # preprocessing of "wav -> acoustic feature" is done during forward
# show size
print('input_wav size:', input_wav.size())
spec_aug_prev: str, ['True', 'False'], apply spec augment on input acoustic features if True, else apply on output representations (used for ASR training)
weighted_sum: str, ['True', 'False'], whether to use a learnable weighted sum to integrate hidden representations from all layers, if False then use the last
select_layer: int, select from all hidden representations, set to -1 to select the last (will only be used when weighted_sum is False)
"""
options = {
'ckpt_file':
'./result/result_transformer/tera/fmllrBase960-F-N-K-libri/states-1000000.ckpt',
'load_pretrain': 'True',
'no_grad': 'True',
'dropout': 'default',
'spec_aug': 'False',
'spec_aug_prev': 'True',
'weighted_sum': 'False',
'select_layer': -1,
}
transformer = TRANSFORMER(options=options, inp_dim=40)
transformer.permute_input = False # Set to False to take input as (B, T, D), otherwise take (T, B, D)
# setup your downstream class model
classifier = example_classifier(input_dim=768, hidden_dim=128,
class_num=2).cuda()
# construct the optimizer
params = list(transformer.named_parameters()) + list(
classifier.named_parameters())
optimizer = get_optimizer(params=params,
lr=4e-3,
warmup_proportion=0.7,
training_steps=50000)
# forward
select_layer: int, select from all hidden representations, set to -1 to select the last (will only be used when weighted_sum is False)
permute_input: str, ['True', 'False'], this attribute is for the forward method. If Ture then input ouput is in the shape of (T, B, D), if False then in (B, T, D)
"""
options = {
'ckpt_file':
'./result/result_transformer/tera/fmllrBase960-F-N-K-libri/states-1000000.ckpt',
'load_pretrain': 'True',
'no_grad': 'True',
'dropout': 'default',
'spec_aug': 'False',
'spec_aug_prev': 'True',
'weighted_sum': 'False',
'select_layer': -1,
'permute_input': 'False',
}
transformer = TRANSFORMER(options=options, inp_dim=40)
# setup your downstream class model
classifier = example_classifier(input_dim=768, hidden_dim=128,
class_num=2).cuda()
# construct the optimizer
params = list(transformer.named_parameters()) + list(
classifier.named_parameters())
optimizer = get_optimizer(params=params,
lr=4e-3,
warmup_proportion=0.7,
training_steps=50000)
# forward
example_inputs = torch.zeros(